Fixed disconnect due to max characters

.gitmodules

@@ -4,7 +4,7 @@
 	branch = master
 [submodule "shared"]
 	path = shared
-	url = https://git.did.science/WolverinDEV/TeaSpeak-SharedLib.git
+	url = https://git.did.science/TeaSpeak/TeaSpeakLibrary.git
 [submodule "music"]
 	path = music
 	url = https://github.com/TeaSpeak/TeaMusic-Providers.git

CMakeLists.txt

@@ -61,6 +61,8 @@ find_package(Opus REQUIRED)
 find_package(spdlog REQUIRED)
 find_package(Jemalloc REQUIRED)
 find_package(Protobuf REQUIRED)
+message("${zstd_DIR}")
+find_package(zstd REQUIRED)
 
 include_directories(${StringVariable_INCLUDE_DIR})
 add_subdirectory(music/)
@@ -103,4 +105,5 @@ add_definitions(-DINET -DINET6)
 add_subdirectory(shared/)
 add_subdirectory(server/)
 add_subdirectory(license/)
-add_subdirectory(MusicBot/)
+add_subdirectory(MusicBot/)
+add_subdirectory(file/)

MusicBot/src/MusicPlayer.cpp

@@ -30,9 +30,12 @@ void AbstractMusicPlayer::unregisterEventHandler(const std::string& string) {
 }
 
 void AbstractMusicPlayer::fireEvent(MusicEvent event) {
-    std::lock_guard lock(this->eventLock);
-    auto listCopy = this->eventHandlers; //Copy for remove while fire
-    for(const auto& entry : listCopy)
+    decltype(this->eventHandlers) handlers{};
+    {
+        std::lock_guard lock(this->eventLock);
+        handlers = this->eventHandlers; //Copy for remove while fire
+    }
+    for(const auto& entry : handlers)
         entry.second(event);
 }
 
@@ -75,11 +78,16 @@ void manager::loadProviders(const std::string& path) {
     }
 
     deque<fs::path> paths;
-    for(const auto& entry : fs::directory_iterator(dir)){
+    error_code error_code{};
+    for(const auto& entry : fs::directory_iterator(dir, error_code)){
         if(!entry.path().has_extension()) continue;
         if(entry.path().extension().string() == ".so")
             paths.push_back(entry.path());
     }
+    if(error_code) {
+        log::log(log::err, "Failed to scan the target directory (" + dir.string() + "): " + error_code.message());
+        return;
+    }
     std::sort(paths.begin(), paths.end(), [](const fs::path& a, const fs::path& b){ return a.filename().string() < b.filename().string(); });
 
     int index = 0;

file/CMakeLists.txt

@@ -0,0 +1,41 @@
+cmake_minimum_required(VERSION 3.6)
+project(TeaSpeak-Files)
+
+#set(CMAKE_CXX_STANDARD 17)
+#set(CMAKE_CXX_STANDARD_REQUIRED ON)
+
+add_library(TeaSpeak-FileServer STATIC
+        local_server/LocalFileProvider.cpp
+        local_server/LocalFileSystem.cpp
+        local_server/LocalFileTransfer.cpp
+        local_server/LocalFileTransferClientWorker.cpp
+        local_server/LocalFileTransferDisk.cpp
+        local_server/LocalFileTransferNetwork.cpp
+        local_server/clnpath.cpp
+        local_server/NetTools.cpp
+        local_server/Config.cpp
+        local_server/HTTPUtils.cpp
+)
+
+target_link_libraries(TeaSpeak-FileServer PUBLIC TeaSpeak ${StringVariable_LIBRARIES_STATIC} stdc++fs
+        libevent::core libevent::pthreads
+        # We're not linking this here, since we may later use DataPipes::shared linking
+        # DataPipes::core::static
+        openssl::ssl::shared
+        openssl::crypto::shared
+)
+
+target_include_directories(TeaSpeak-FileServer PUBLIC include/)
+target_compile_options(TeaSpeak-FileServer PUBLIC "-Wswitch-enum")
+
+add_executable(TeaSpeak-FileServerTest test/main.cpp)
+target_link_libraries(TeaSpeak-FileServerTest PUBLIC TeaSpeak-FileServer
+        TeaMusic            #Static (Must be in here, so we link against TeaMusic which uses C++11. That forbids GCC to use the newer glibc version)
+        CXXTerminal::static
+        DataPipes::core::static
+        stdc++fs
+)
+target_compile_options(TeaSpeak-FileServerTest PUBLIC -static-libgcc -static-libstdc++)
+
+add_executable(FileServer-CLNText local_server/clnpath.cpp)
+target_compile_definitions(FileServer-CLNText PUBLIC -DCLN_EXEC)

file/include/files/Config.h

@@ -0,0 +1,13 @@
+#pragma once
+
+#include <memory>
+#include <pipes/ssl.h>
+
+namespace ts::server::file::config {
+    enum struct Key {
+        SSL_OPTION_SUPPLIER
+    };
+
+    extern void value_updated(Key /* value */);
+    extern std::function<std::shared_ptr<pipes::SSL::Options>()> ssl_option_supplier;
+}

file/include/files/ExecuteResponse.h

@@ -0,0 +1,77 @@
+#pragma once
+
+namespace ts::server::file {
+    enum struct ExecuteStatus {
+        UNKNOWN,
+        WAITING,
+        SUCCESS,
+        ERROR
+    };
+
+    template<typename VariantType, typename T, std::size_t index = 0>
+    constexpr std::size_t variant_index() {
+        if constexpr (index == std::variant_size_v<VariantType>) {
+            return index;
+        } else if constexpr (std::is_same_v<std::variant_alternative_t<index, VariantType>, T>) {
+            return index;
+        } else {
+            return variant_index<VariantType, T, index + 1>();
+        }
+    }
+
+    struct EmptyExecuteResponse { };
+    template <class error_t, class response_t = EmptyExecuteResponse>
+    class ExecuteResponse {
+            typedef std::variant<EmptyExecuteResponse, error_t, response_t> variant_t;
+        public:
+            ExecuteStatus status{ExecuteStatus::WAITING};
+
+            [[nodiscard]] inline auto response() const -> const response_t& { return std::get<response_t>(this->response_); }
+
+            template <typename = std::enable_if_t<!std::is_void<error_t>::value>>
+            [[nodiscard]] inline const error_t& error() const { return std::get<error_t>(this->response_); }
+
+            inline void wait() const {
+                std::unique_lock nlock{this->notify_mutex};
+                this->notify_cv.wait(nlock, [&]{ return this->status != ExecuteStatus::WAITING; });
+            }
+
+            template<typename _Rep, typename _Period>
+            [[nodiscard]] inline bool wait_for(const std::chrono::duration<_Rep, _Period>& time) const {
+                std::unique_lock nlock{this->notify_mutex};
+                return this->notify_cv.wait_for(nlock, time, [&]{ return this->status != ExecuteStatus::WAITING; });
+            }
+
+            template <typename... Args>
+            inline void emplace_success(Args&&... args) {
+                constexpr auto success_index = variant_index<variant_t, response_t>();
+
+                std::lock_guard rlock{this->notify_mutex};
+                this->response_.template emplace<success_index, Args...>(std::forward<Args>(args)...);
+                this->status = ExecuteStatus::SUCCESS;
+                this->notify_cv.notify_all();
+            }
+
+            template <typename... Args>
+            inline void emplace_fail(Args&&... args) {
+                constexpr auto error_index = variant_index<variant_t, error_t>();
+
+                std::lock_guard rlock{this->notify_mutex};
+                this->response_.template emplace<error_index, Args...>(std::forward<Args>(args)...);
+                this->status = ExecuteStatus::ERROR;
+                this->notify_cv.notify_all();
+            }
+
+            [[nodiscard]] inline bool succeeded() const {
+                return this->status == ExecuteStatus::SUCCESS;
+            }
+
+            ExecuteResponse(std::mutex& notify_mutex, std::condition_variable& notify_cv)
+                    : notify_mutex{notify_mutex}, notify_cv{notify_cv} {}
+        private:
+            variant_t response_{}; /* void* as default value so we don't initialize error_t or response_t */
+
+            std::mutex& notify_mutex;
+            std::condition_variable& notify_cv;
+    };
+}

file/include/files/FileServer.h

@@ -0,0 +1,420 @@
+#pragma once
+
+#include <string>
+#include <chrono>
+#include <Definitions.h>
+#include <condition_variable>
+#include <utility>
+#include <variant>
+#include <deque>
+#include <functional>
+#include <atomic>
+
+#include "./ExecuteResponse.h"
+
+#define TRANSFER_KEY_LENGTH (32)
+#define TRANSFER_MEDIA_BYTES_LENGTH (32)
+
+namespace ts::server::file {
+    class VirtualFileServer;
+
+    namespace filesystem {
+        template <typename ErrorCodes>
+        struct DetailedError {
+            ErrorCodes error_type{ErrorCodes::UNKNOWN};
+            std::string error_message{};
+
+            DetailedError(ErrorCodes type, std::string extraMessage) : error_type{type}, error_message{std::move(extraMessage)} {}
+        };
+
+        enum struct DirectoryQueryErrorType {
+            UNKNOWN,
+            PATH_EXCEEDS_ROOT_PATH,
+            PATH_IS_A_FILE,
+            PATH_DOES_NOT_EXISTS,
+            FAILED_TO_LIST_FILES
+        };
+        constexpr std::array<std::string_view, 5> directory_query_error_messages = {
+            "unknown error",
+            "path exceeds base path",
+            "path is a file",
+            "path does not exists",
+            "failed to list files"
+        };
+
+        typedef DetailedError<DirectoryQueryErrorType> DirectoryQueryError;
+
+        struct DirectoryEntry {
+            enum Type {
+                UNKNOWN,
+                DIRECTORY,
+                FILE
+            };
+
+            Type type{Type::UNKNOWN};
+            std::string name{};
+            std::chrono::system_clock::time_point modified_at{};
+
+            size_t size{0}; /* file only */
+            bool empty{false}; /* directory only */
+        };
+
+        enum struct DirectoryModifyErrorType {
+            UNKNOWN,
+            PATH_EXCEEDS_ROOT_PATH,
+            PATH_ALREADY_EXISTS,
+            FAILED_TO_CREATE_DIRECTORIES
+        };
+        typedef DetailedError<DirectoryModifyErrorType> DirectoryModifyError;
+
+        enum struct FileModifyErrorType {
+            UNKNOWN,
+            PATH_EXCEEDS_ROOT_PATH,
+            TARGET_PATH_EXCEEDS_ROOT_PATH,
+            PATH_DOES_NOT_EXISTS,
+            TARGET_PATH_ALREADY_EXISTS,
+            FAILED_TO_DELETE_FILES,
+            FAILED_TO_RENAME_FILE,
+            FAILED_TO_CREATE_DIRECTORIES,
+
+            SOME_FILES_ARE_LOCKED
+        };
+        typedef DetailedError<FileModifyErrorType> FileModifyError;
+
+        enum struct FileDeleteErrorType {
+            UNKNOWN,
+        };
+        typedef DetailedError<FileDeleteErrorType> FileDeleteError;
+
+        struct FileDeleteResponse {
+            enum struct StatusType {
+                SUCCESS,
+
+                PATH_EXCEEDS_ROOT_PATH,
+                PATH_DOES_NOT_EXISTS,
+                FAILED_TO_DELETE_FILES,
+                SOME_FILES_ARE_LOCKED
+            };
+
+            struct DeleteResult {
+                StatusType status{StatusType::SUCCESS};
+                std::string error_detail{};
+
+                DeleteResult(StatusType status, std::string errorDetail) : status{status},
+                                                                                  error_detail{std::move(errorDetail)} {}
+            };
+
+            std::vector<DeleteResult> delete_results{};
+        };
+
+        enum struct ServerCommandErrorType {
+            UNKNOWN,
+            FAILED_TO_CREATE_DIRECTORIES,
+            FAILED_TO_DELETE_DIRECTORIES
+        };
+        typedef DetailedError<ServerCommandErrorType> ServerCommandError;
+
+        struct FileInfoResponse {
+            enum struct StatusType {
+                SUCCESS,
+
+                PATH_EXCEEDS_ROOT_PATH,
+                PATH_DOES_NOT_EXISTS,
+
+                FAILED_TO_QUERY_INFO,
+                UNKNOWN_FILE_TYPE
+            };
+
+            struct FileInfo {
+                StatusType status{StatusType::SUCCESS};
+                std::string error_detail{};
+
+                DirectoryEntry info{};
+
+                FileInfo(StatusType status, std::string errorDetail, DirectoryEntry info) : status{status},
+                                                                                  error_detail{std::move(errorDetail)}, info{std::move(info)} {}
+            };
+
+            std::vector<FileInfo> file_info{};
+        };
+
+        enum struct FileInfoErrorType {
+            UNKNOWN,
+        };
+        typedef DetailedError<FileInfoErrorType> FileInfoError;
+
+        class AbstractProvider {
+            public:
+                typedef ExecuteResponse<DirectoryQueryError, std::deque<DirectoryEntry>> directory_query_response_t;
+
+                /* server */
+                [[nodiscard]] virtual std::shared_ptr<ExecuteResponse<ServerCommandError>> initialize_server(const std::shared_ptr<VirtualFileServer> &/* server */) = 0;
+                [[nodiscard]] virtual std::shared_ptr<ExecuteResponse<ServerCommandError>> delete_server(const std::shared_ptr<VirtualFileServer> &/* server */) = 0;
+
+                /* channels */
+                [[nodiscard]] virtual std::shared_ptr<ExecuteResponse<FileInfoError, FileInfoResponse>> query_channel_info(const std::shared_ptr<VirtualFileServer> &/* server */, const std::vector<std::tuple<ChannelId, std::string>>& /* files */) = 0;
+                [[nodiscard]] virtual std::shared_ptr<directory_query_response_t> query_channel_directory(const std::shared_ptr<VirtualFileServer> &/* server */, ChannelId /* channel */, const std::string& /* path */) = 0;
+                [[nodiscard]] virtual std::shared_ptr<ExecuteResponse<DirectoryModifyError>> create_channel_directory(const std::shared_ptr<VirtualFileServer> &/* server */, ChannelId /* channel */, const std::string& /* path */) = 0;
+                [[nodiscard]] virtual std::shared_ptr<ExecuteResponse<FileDeleteError, FileDeleteResponse>> delete_channel_files(const std::shared_ptr<VirtualFileServer> &/* server */, ChannelId /* channel */, const std::vector<std::string>& /* paths */) = 0;
+                [[nodiscard]] virtual std::shared_ptr<ExecuteResponse<FileModifyError>> rename_channel_file(const std::shared_ptr<VirtualFileServer> &/* server */, ChannelId /* channel */, const std::string& /* path */, ChannelId /* target channel */, const std::string& /* target */) = 0;
+
+                /* icons */
+                [[nodiscard]] virtual std::shared_ptr<ExecuteResponse<FileInfoError, FileInfoResponse>> query_icon_info(const std::shared_ptr<VirtualFileServer> &/* server */, const std::vector<std::string>& /* names */) = 0;
+                [[nodiscard]] virtual std::shared_ptr<directory_query_response_t> query_icon_directory(const std::shared_ptr<VirtualFileServer> &/* server */) = 0;
+                [[nodiscard]] virtual std::shared_ptr<ExecuteResponse<FileDeleteError, FileDeleteResponse>> delete_icons(const std::shared_ptr<VirtualFileServer> &/* server */, const std::vector<std::string>& /* names */) = 0;
+
+                /* avatars */
+                [[nodiscard]] virtual std::shared_ptr<ExecuteResponse<FileInfoError, FileInfoResponse>> query_avatar_info(const std::shared_ptr<VirtualFileServer> &/* server */, const std::vector<std::string>& /* names */) = 0;
+                [[nodiscard]] virtual std::shared_ptr<directory_query_response_t> query_avatar_directory(const std::shared_ptr<VirtualFileServer> &/* server */) = 0;
+                [[nodiscard]] virtual std::shared_ptr<ExecuteResponse<FileDeleteError, FileDeleteResponse>> delete_avatars(const std::shared_ptr<VirtualFileServer> &/* server */, const std::vector<std::string>& /* names */) = 0;
+            private:
+        };
+    }
+
+    namespace transfer {
+        typedef uint16_t transfer_id;
+
+        struct Transfer {
+            transfer_id server_transfer_id{0};
+            transfer_id client_transfer_id{0};
+
+            std::shared_ptr<VirtualFileServer> server{nullptr};
+            ChannelId channel_id{0};
+
+            ClientId client_id{0};
+            std::string client_unique_id{};
+
+            char transfer_key[TRANSFER_KEY_LENGTH]{};
+            std::chrono::system_clock::time_point initialized_timestamp{};
+            enum Direction {
+                DIRECTION_UNKNOWN,
+                DIRECTION_UPLOAD,
+                DIRECTION_DOWNLOAD
+            } direction{DIRECTION_UNKNOWN};
+
+            struct Address {
+                std::string hostname{};
+                uint16_t port{0};
+            };
+            std::vector<Address> server_addresses{};
+
+            enum TargetType {
+                TARGET_TYPE_UNKNOWN,
+                TARGET_TYPE_CHANNEL_FILE,
+                TARGET_TYPE_ICON,
+                TARGET_TYPE_AVATAR
+            } target_type{TARGET_TYPE_UNKNOWN};
+            std::string target_file_path{};
+            std::string absolute_file_path{};
+
+            std::string relative_file_path{};
+            std::string file_name{};
+
+            int64_t max_bandwidth{-1};
+            size_t expected_file_size{0}; /* incl. the offset! */
+            size_t file_offset{0};
+            bool override_exiting{false};
+        };
+
+        struct TransferStatistics {
+            uint64_t network_bytes_send{0};
+            uint64_t network_bytes_received{0};
+
+            uint64_t delta_network_bytes_send{0};
+            uint64_t delta_network_bytes_received{0};
+
+            uint64_t file_bytes_transferred{0};
+            uint64_t delta_file_bytes_transferred{0};
+
+            size_t file_start_offset{0};
+            size_t file_current_offset{0};
+            size_t file_total_size{0};
+
+            double average_speed{0};
+            double current_speed{0};
+        };
+
+        struct TransferInitError {
+            enum Type {
+                UNKNOWN,
+
+                INVALID_FILE_TYPE,
+                FILE_DOES_NOT_EXISTS,
+                FILE_IS_NOT_A_FILE,
+
+                CLIENT_TOO_MANY_TRANSFERS,
+                SERVER_TOO_MANY_TRANSFERS,
+
+                SERVER_QUOTA_EXCEEDED,
+                CLIENT_QUOTA_EXCEEDED,
+
+                IO_ERROR
+            } error_type{UNKNOWN};
+            std::string error_message{};
+
+            TransferInitError(Type errorType, std::string errorMessage) : error_type{errorType},
+                                                                                 error_message{std::move(errorMessage)} {}
+        };
+
+        struct TransferActionError {
+            enum Type {
+                UNKNOWN,
+
+                UNKNOWN_TRANSFER
+            } error_type{UNKNOWN};
+            std::string error_message{};
+        };
+
+        struct TransferError {
+            enum Type {
+                UNKNOWN,
+
+                TRANSFER_TIMEOUT,
+
+                DISK_IO_ERROR,
+                DISK_TIMEOUT,
+                DISK_INITIALIZE_ERROR,
+
+                NETWORK_IO_ERROR,
+
+                UNEXPECTED_CLIENT_DISCONNECT,
+                UNEXPECTED_DISK_EOF,
+
+                USER_REQUEST
+            } error_type{UNKNOWN};
+            std::string error_message{};
+        };
+
+        struct ActiveFileTransfer {
+            transfer_id client_transfer_id{0};
+            transfer_id server_transfer_id{0};
+
+            Transfer::Direction direction{Transfer::DIRECTION_UNKNOWN};
+
+            ClientId client_id{};
+            std::string client_unique_id{};
+
+            std::string file_path{};
+            std::string file_name{};
+
+            size_t expected_size{};
+            size_t size_done{};
+
+            enum Status {
+                NOT_STARTED,
+                RUNNING,
+                INACTIVE /* (not used yet) */
+            } status{Status::NOT_STARTED};
+
+            std::chrono::milliseconds runtime{};
+
+            double average_speed{0};
+            double current_speed{0};
+        };
+
+        enum struct TransferListError {
+            UNKNOWN
+        };
+
+        class AbstractProvider {
+            public:
+                struct TransferInfo {
+                    std::string file_path{};
+                    std::string client_unique_id{};
+                    ClientId client_id{};
+
+                    bool override_exiting{false}; /* only for upload valid */
+
+                    size_t file_offset{0};
+                    size_t expected_file_size{0};
+
+                    int64_t max_bandwidth{-1};
+                    int64_t max_concurrent_transfers{-1};
+
+                    /* only used for upload, for download the quotas could be checked before */
+                    size_t download_server_quota_limit{(size_t) -1};
+                    size_t download_client_quota_limit{(size_t) -1};
+                };
+
+                virtual std::shared_ptr<ExecuteResponse<TransferInitError, std::shared_ptr<Transfer>>>
+                initialize_channel_transfer(Transfer::Direction /* direction */, const std::shared_ptr<VirtualFileServer>& /* server */, ChannelId /* channel */, const TransferInfo& /* info */) = 0;
+
+                virtual std::shared_ptr<ExecuteResponse<TransferInitError, std::shared_ptr<Transfer>>>
+                initialize_icon_transfer(Transfer::Direction /* direction */, const std::shared_ptr<VirtualFileServer>& /* server */, const TransferInfo& /* info */) = 0;
+
+                virtual std::shared_ptr<ExecuteResponse<TransferInitError, std::shared_ptr<Transfer>>>
+                initialize_avatar_transfer(Transfer::Direction /* direction */, const std::shared_ptr<VirtualFileServer>& /* server */, const TransferInfo& /* info */) = 0;
+
+                virtual std::shared_ptr<ExecuteResponse<TransferListError, std::vector<ActiveFileTransfer>>> list_transfer() = 0;
+
+                virtual std::shared_ptr<ExecuteResponse<TransferActionError>> stop_transfer(const std::shared_ptr<VirtualFileServer>& /* server */, transfer_id /* id */, bool /* flush */) = 0;
+
+                std::function<void(const std::shared_ptr<Transfer>&)> callback_transfer_registered{}; /* transfer has been registered */
+                std::function<void(const std::shared_ptr<Transfer>&)> callback_transfer_started{}; /* transfer has been started */
+                std::function<void(const std::shared_ptr<Transfer>&)> callback_transfer_finished{}; /* transfer has been finished */
+                std::function<void(const std::shared_ptr<Transfer>&, const TransferStatistics&)> callback_transfer_statistics{};
+                std::function<void(const std::shared_ptr<Transfer>&, const transfer::TransferStatistics&, const TransferError&)> callback_transfer_aborted{}; /* an error happened while transferring the data  */
+        };
+    }
+
+    class VirtualFileServer {
+        public:
+            explicit VirtualFileServer(ServerId server_id, std::string unique_id) : server_id_{server_id}, unique_id_{std::move(unique_id)} {}
+
+            [[nodiscard]] inline auto unique_id() const -> const std::string& { return this->unique_id_; }
+            [[nodiscard]] inline auto server_id() const -> ServerId { return this->server_id_; }
+
+            [[nodiscard]] inline auto max_networking_upload_bandwidth() const -> int64_t { return this->max_networking_upload_bandwidth_; }
+            virtual void max_networking_upload_bandwidth(int64_t value) {
+                this->max_networking_upload_bandwidth_ = value;
+            }
+
+            [[nodiscard]] inline auto max_networking_download_bandwidth() const -> int64_t { return this->max_networking_download_bandwidth_; }
+            virtual void max_networking_download_bandwidth(int64_t value) {
+                this->max_networking_download_bandwidth_ = value;
+            }
+
+            [[nodiscard]] inline auto generate_transfer_id() {
+                return ++this->current_transfer_id;
+            }
+        private:
+            ServerId server_id_;
+            std::string unique_id_;
+
+            int64_t max_networking_upload_bandwidth_{-1};
+            int64_t max_networking_download_bandwidth_{-1};
+
+            std::atomic<transfer::transfer_id> current_transfer_id{0};
+    };
+
+    class AbstractFileServer {
+        public:
+            [[nodiscard]] virtual std::string file_base_path() const = 0;
+            [[nodiscard]] virtual filesystem::AbstractProvider& file_system() = 0;
+            [[nodiscard]] virtual transfer::AbstractProvider& file_transfer() = 0;
+
+            [[nodiscard]] inline auto virtual_servers() const -> std::deque<std::shared_ptr<VirtualFileServer>> {
+                std::lock_guard slock{this->servers_mutex};
+                return this->servers_;
+            }
+
+            [[nodiscard]] inline auto find_virtual_server(ServerId server_id) const -> std::shared_ptr<VirtualFileServer> {
+                std::lock_guard slock{this->servers_mutex};
+                auto it = std::find_if(this->servers_.begin(), this->servers_.end(), [&](const std::shared_ptr<VirtualFileServer>& server) {
+                    return server->server_id() == server_id;
+                });
+                return it == this->servers_.end() ? nullptr : *it;
+            }
+
+            virtual std::shared_ptr<VirtualFileServer> register_server(ServerId /* server id */) = 0;
+            virtual void unregister_server(ServerId /* server id */) = 0;
+        protected:
+            mutable std::mutex servers_mutex{};
+            std::deque<std::shared_ptr<VirtualFileServer>> servers_{};
+    };
+
+    extern bool initialize(std::string& /* error */, const std::string& /* host names */, uint16_t /* port */);
+    extern void finalize();
+
+    extern std::shared_ptr<AbstractFileServer> server();
+}

file/local_server/Config.cpp

@@ -0,0 +1,13 @@
+//
+// Created by WolverinDEV on 21/05/2020.
+//
+
+#include "files/Config.h"
+
+using namespace ts::server::file;
+
+std::function<std::shared_ptr<pipes::SSL::Options>()> config::ssl_option_supplier{nullptr};
+
+void config::value_updated(ts::server::file::config::Key) {
+    /* we currently do nothing */
+}

file/local_server/HTTPUtils.cpp

@@ -0,0 +1,35 @@
+//
+// Created by WolverinDEV on 22/05/2020.
+//
+
+#include <pipes/misc/http.h>
+#include "HTTPUtils.h"
+
+bool http::parse_url_parameters(const std::string_view &query, std::map<std::string, std::string>& result) {
+    const auto query_offset = query.find('?');
+    if(query_offset == std::string::npos) return false;
+
+    const auto query_end_offset = query.find('#', query_offset); /* fragment (if there is any) */
+
+    auto offset = query_offset + 1;
+    size_t next_param;
+    while(offset > 0) {
+        next_param = query.find('&', offset);
+        if(next_param >= query_end_offset)
+            next_param = query_end_offset;
+
+        if(offset >= next_param)
+            break;
+
+        /* parameter: [offset;next_param) */
+        const auto param_view = query.substr(offset, next_param - offset);
+        const auto assignment_index = param_view.find('=');
+        if(assignment_index == std::string::npos)
+            result[std::string{param_view}] = "";
+        else
+            result[std::string{param_view.substr(0, assignment_index)}] = http::decode_url(std::string{param_view.substr(assignment_index + 1)});
+
+        offset = next_param + 1;
+    }
+    return true;
+}

file/local_server/HTTPUtils.h

@@ -0,0 +1,7 @@
+#pragma once
+
+#include <map>
+
+namespace http {
+    bool parse_url_parameters(const std::string_view& /* url */, std::map<std::string, std::string>& /* result */);
+}

file/local_server/LocalFileProvider.cpp

@@ -0,0 +1,148 @@
+//
+// Created by WolverinDEV on 28/04/2020.
+//
+
+#include <netinet/in.h>
+#include <log/LogUtils.h>
+#include "LocalFileProvider.h"
+#include "LocalFileSystem.h"
+#include "LocalFileTransfer.h"
+
+using namespace ts::server;
+using LocalFileServer = file::LocalFileProvider;
+using LocalVirtualFileServer = file::LocalVirtualFileServer;
+
+std::shared_ptr<LocalFileServer> server_instance{};
+bool file::initialize(std::string &error, const std::string& hostnames, uint16_t port) {
+    server_instance = std::make_shared<LocalFileProvider>();
+
+    if(!server_instance->initialize(error)) {
+        server_instance = nullptr;
+        return false;
+    }
+
+    bool any_bind{false};
+    for(const auto& binding : net::resolve_bindings(hostnames, port)) {
+        if(!get<2>(binding).empty()) {
+            logError(LOG_FT, "Failed to resolve binding for {}: {}", get<0>(binding), get<2>(binding));
+            continue;
+        }
+
+        auto result = dynamic_cast<transfer::LocalFileTransfer&>(server_instance->file_transfer()).add_network_binding({ get<0>(binding), get<1>(binding) });
+        switch (result) {
+            case transfer::NetworkingBindResult::SUCCESS:
+                any_bind = true;
+                break;
+
+            case transfer::NetworkingBindResult::OUT_OF_MEMORY:
+                logWarning(LOG_FT, "Failed to listen to address {}: Out of memory", get<0>(binding));
+                continue;
+
+            case transfer::NetworkingBindResult::FAILED_TO_LISTEN:
+                logWarning(LOG_FT, "Failed to listen on {}: {}/{}", get<0>(binding), errno, strerror(errno));
+                continue;
+
+            case transfer::NetworkingBindResult::FAILED_TO_BIND:
+                logWarning(LOG_FT, "Failed to bind on {}: {}/{}", get<0>(binding), errno, strerror(errno));
+                continue;
+
+            case transfer::NetworkingBindResult::BINDING_ALREADY_EXISTS:
+                logWarning(LOG_FT, "Failed to bind on {}: binding already exists", get<0>(binding));
+                continue;
+
+            case transfer::NetworkingBindResult::NETWORKING_NOT_INITIALIZED:
+                logWarning(LOG_FT, "Failed to bind on {}: networking not initialized", get<0>(binding));
+                continue;
+
+            case transfer::NetworkingBindResult::FAILED_TO_ALLOCATE_SOCKET:
+                logWarning(LOG_FT, "Failed to allocate a socket for {}: {}/{}", get<0>(binding), errno, strerror(errno));
+                continue;
+        }
+    }
+
+    return true;
+}
+
+void file::finalize() {
+    auto server = std::exchange(server_instance, nullptr);
+    if(!server) return;
+
+    server->finalize();
+}
+
+std::shared_ptr<file::AbstractFileServer> file::server() {
+    return server_instance;
+}
+
+LocalFileServer::LocalFileProvider() {
+    this->file_system_ = new filesystem::LocalFileSystem();
+    this->file_transfer_ = new transfer::LocalFileTransfer(this->file_system_);
+}
+LocalFileServer::~LocalFileProvider() {
+    delete this->file_transfer_;
+    delete this->file_system_;
+};
+
+bool LocalFileServer::initialize(std::string &error) {
+    if(!this->file_system_->initialize(error, "files/"))
+        return false;
+
+    if(!this->file_transfer_->start()) {
+        error = "transfer server startup failed";
+        this->file_system_->finalize();
+        return false;
+    }
+    return true;
+}
+
+void LocalFileServer::finalize() {
+    this->file_transfer_->stop();
+    this->file_system_->finalize();
+}
+
+file::filesystem::AbstractProvider &LocalFileServer::file_system() {
+    return *this->file_system_;
+}
+
+file::transfer::AbstractProvider &LocalFileServer::file_transfer() {
+    return *this->file_transfer_;
+}
+
+std::string file::LocalFileProvider::file_base_path() const {
+    return this->file_system_->root_path();
+}
+
+std::shared_ptr<file::VirtualFileServer> LocalFileServer::register_server(ServerId server_id) {
+    auto server = this->find_virtual_server(server_id);
+    if(server) return server;
+
+    server = std::make_shared<file::LocalVirtualFileServer>(server_id, std::to_string(server_id));
+    {
+        std::lock_guard slock{this->servers_mutex};
+        this->servers_.push_back(server);
+    }
+
+    return server;
+}
+
+void LocalFileServer::unregister_server(ServerId server_id) {
+    auto server_unique_id = std::to_string(server_id);
+
+    std::lock_guard slock{this->servers_mutex};
+    auto it = std::find_if(this->servers_.begin(), this->servers_.end(), [&](const std::shared_ptr<VirtualFileServer>& server) {
+        return server->unique_id() == server_unique_id;
+    });
+
+    if(it == this->servers_.end()) return;
+    this->servers_.erase(it);
+}
+
+void LocalVirtualFileServer::max_networking_upload_bandwidth(int64_t value) {
+    VirtualFileServer::max_networking_upload_bandwidth(value);
+    this->upload_throttle.set_max_bandwidth(value);
+}
+
+void LocalVirtualFileServer::max_networking_download_bandwidth(int64_t value) {
+    VirtualFileServer::max_networking_download_bandwidth(value);
+    this->download_throttle.set_max_bandwidth(value);
+}

file/local_server/LocalFileProvider.h

@@ -0,0 +1,52 @@
+#pragma once
+
+#include <files/FileServer.h>
+#include <deque>
+#include <utility>
+#include <thread>
+#include <shared_mutex>
+#include <sys/socket.h>
+#include <pipes/ws.h>
+#include <pipes/ssl.h>
+#include <misc/net.h>
+#include <misc/spin_mutex.h>
+#include <random>
+#include <misc/memtracker.h>
+#include "./NetTools.h"
+
+namespace ts::server::file {
+    namespace filesystem { class LocalFileSystem; }
+    namespace transfer { class LocalFileTransfer; }
+
+    class LocalVirtualFileServer : public VirtualFileServer {
+        public:
+            explicit LocalVirtualFileServer(ServerId server_id, std::string unique_id) : VirtualFileServer{server_id, std::move(unique_id)} {}
+
+            void max_networking_upload_bandwidth(int64_t value) override;
+            void max_networking_download_bandwidth(int64_t value) override;
+
+            networking::NetworkThrottle upload_throttle{};
+            networking::NetworkThrottle download_throttle{};
+    };
+
+    class LocalFileProvider : public AbstractFileServer {
+        public:
+            LocalFileProvider();
+            virtual ~LocalFileProvider();
+
+            [[nodiscard]] bool initialize(std::string& /* error */);
+            void finalize();
+
+            [[nodiscard]] std::string file_base_path() const override;
+
+            filesystem::AbstractProvider &file_system() override;
+            transfer::AbstractProvider &file_transfer() override;
+
+
+            std::shared_ptr<VirtualFileServer> register_server(ServerId /* server id */) override;
+            void unregister_server(ServerId /* server id */) override;
+        private:
+            filesystem::LocalFileSystem* file_system_;
+            transfer::LocalFileTransfer* file_transfer_;
+    };
+}

file/local_server/LocalFileSystem.cpp

@@ -0,0 +1,457 @@
+//
+// Created by WolverinDEV on 29/04/2020.
+//
+#include <experimental/filesystem>
+#define FS_INCLUDED
+
+#include <log/LogUtils.h>
+#include "./LocalFileSystem.h"
+#include "./clnpath.h"
+
+using namespace ts::server::file;
+using namespace ts::server::file::filesystem;
+namespace fs = std::experimental::filesystem;
+using directory_query_response_t = AbstractProvider::directory_query_response_t;
+
+LocalFileSystem::~LocalFileSystem() = default;
+
+bool LocalFileSystem::initialize(std::string &error_message, const std::string &root_path_string) {
+    auto root_path = fs::u8path(root_path_string);
+    std::error_code error{};
+
+    if(!fs::exists(root_path, error)) {
+        if(error)
+            logWarning(LOG_FT, "Failed to check root path existence. Assuming it does not exist. ({}/{})", error.value(), error.message());
+        if(!fs::create_directories(root_path, error) || error) {
+            error_message = "Failed to create root file system at " + root_path_string + ": " + std::to_string(error.value()) + "/" + error.message();
+            return false;
+        }
+    }
+
+    auto croot = clnpath(fs::absolute(root_path).string());
+    logMessage(LOG_FT, "Started file system root at {}", croot);
+    this->root_path_ = croot;
+    return true;
+}
+
+void LocalFileSystem::finalize() {}
+
+fs::path LocalFileSystem::server_path(const std::shared_ptr<VirtualFileServer> &server) {
+    return fs::u8path(this->root_path_) / fs::u8path("server_" + std::to_string(server->server_id()));
+}
+
+fs::path LocalFileSystem::server_channel_path(const std::shared_ptr<VirtualFileServer> &server, ts::ChannelId cid) {
+    return this->server_path(server) / fs::u8path("channel_" + std::to_string(cid));
+}
+
+bool LocalFileSystem::exceeds_base_path(const fs::path &base, const fs::path &target) {
+    auto rel_target = clnpath(target.string());
+    if(rel_target.starts_with("..")) return true;
+
+    auto base_string = clnpath(fs::absolute(base).string());
+    auto target_string = clnpath(fs::absolute(target).string());
+    return !target_string.starts_with(base_string);
+}
+
+bool LocalFileSystem::is_any_file_locked(const fs::path &base, const std::string &path, std::string &locked_file) {
+    auto c_path = clnpath(fs::absolute(base / fs::u8path(path)).string());
+
+    std::lock_guard lock{this->locked_files_mutex};
+    for(const auto& lfile : this->locked_files_) {
+        if(lfile.starts_with(c_path)) {
+            locked_file = lfile.substr(base.string().length());
+            return true;
+        }
+    }
+
+    return false;
+}
+
+std::string LocalFileSystem::target_file_path(FileCategory type, const std::shared_ptr<VirtualFileServer> &server, ts::ChannelId cid, const std::string &path) {
+    fs::path target_path{};
+    switch (type) {
+        case FileCategory::CHANNEL:
+            target_path = this->server_channel_path(server, cid) / path;
+            break;
+        case FileCategory::ICON:
+            target_path = this->server_path(server) / "icons" / path;
+            break;
+        case FileCategory::AVATAR:
+            target_path = this->server_path(server) / "avatars" / path;
+            break;
+    }
+
+    return clnpath(fs::absolute(target_path).string());
+}
+
+std::string LocalFileSystem::absolute_avatar_path(const std::shared_ptr<VirtualFileServer> &sid, const std::string &path) {
+    return this->target_file_path(FileCategory::AVATAR, sid, 0, path);
+}
+
+std::string LocalFileSystem::absolute_icon_path(const std::shared_ptr<VirtualFileServer> &sid, const std::string &path) {
+    return this->target_file_path(FileCategory::ICON, sid, 0, path);
+}
+
+std::string LocalFileSystem::absolute_channel_path(const std::shared_ptr<VirtualFileServer> &sid, ChannelId cid, const std::string &path) {
+    return this->target_file_path(FileCategory::CHANNEL, sid, cid, path);
+}
+
+void LocalFileSystem::lock_file(const std::string &c_path) {
+    std::lock_guard lock{this->locked_files_mutex};
+    this->locked_files_.push_back(c_path);
+}
+
+void LocalFileSystem::unlock_file(const std::string &c_path) {
+    std::lock_guard lock{this->locked_files_mutex};
+
+    this->locked_files_.erase(std::remove_if(this->locked_files_.begin(), this->locked_files_.end(), [&](const auto& p) { return p == c_path; }), this->locked_files_.end());
+}
+
+std::shared_ptr<ExecuteResponse<ServerCommandError>> LocalFileSystem::initialize_server(const std::shared_ptr<VirtualFileServer> &id) {
+    auto path = this->server_path(id);
+    std::error_code error{};
+
+    auto response = this->create_execute_response<ServerCommandError>();
+
+    if(!fs::exists(path, error)) {
+        if(!fs::create_directories(path, error) || error) {
+            response->emplace_fail(ServerCommandErrorType::FAILED_TO_CREATE_DIRECTORIES, std::to_string(error.value()) + "/" + error.message());
+            return response;
+        }
+    }
+
+    //TODO: Copy the default icon
+
+    response->emplace_success();
+    return response;
+}
+
+std::shared_ptr<ExecuteResponse<ServerCommandError>> LocalFileSystem::delete_server(const std::shared_ptr<VirtualFileServer> &id) {
+    auto path = this->server_path(id);
+    std::error_code error{};
+
+    auto response = this->create_execute_response<ServerCommandError>();
+
+    //TODO: Stop all running file transfers!
+
+    if(fs::exists(path, error)) {
+        if(!fs::remove_all(path, error) || error) {
+            response->emplace_fail(ServerCommandErrorType::FAILED_TO_DELETE_DIRECTORIES, std::to_string(error.value()) + "/" + error.message());
+            return response;
+        }
+    }
+
+    response->emplace_success();
+    return response;
+}
+
+std::shared_ptr<directory_query_response_t> LocalFileSystem::query_directory(const fs::path &base,
+                                                                             const std::string &path,
+                                                                             bool allow_non_existance) {
+    std::error_code error{};
+    auto response = this->create_execute_response<DirectoryQueryError, std::deque<DirectoryEntry>>();
+    auto target_path = base / fs::u8path(path);
+    if(this->exceeds_base_path(base, target_path)) {
+        response->emplace_fail(DirectoryQueryErrorType::PATH_EXCEEDS_ROOT_PATH, "");
+        return response;
+    }
+
+    if(!fs::exists(target_path, error)) {
+        if(allow_non_existance)
+            response->emplace_success();
+        else
+            response->emplace_fail(DirectoryQueryErrorType::PATH_DOES_NOT_EXISTS, "");
+        return response;
+    } else if(error) {
+        logWarning(LOG_FT, "Failed to check for file at {}: {}. Assuming it does not exists.", target_path.string(), error.value(), error.message());
+        response->emplace_fail(DirectoryQueryErrorType::PATH_DOES_NOT_EXISTS, "");
+        return response;
+    }
+
+    if(!fs::is_directory(target_path, error)) {
+        response->emplace_fail(DirectoryQueryErrorType::PATH_IS_A_FILE, "");
+        return response;
+    } else if(error) {
+        logWarning(LOG_FT, "Failed to check for directory at {}: {}. Assuming its not a directory.", target_path.string(), error.value(), error.message());
+        response->emplace_fail(DirectoryQueryErrorType::PATH_IS_A_FILE, "");
+        return response;
+    }
+
+    std::deque<DirectoryEntry> entries{};
+    for(auto& entry : fs::directory_iterator(target_path, error)) {
+        auto status = entry.status(error);
+        if(error) {
+            logWarning(LOG_FT, "Failed to query file status for {} ({}/{}). Skipping entry for directory query.", entry.path().string(), error.value(), error.message());
+            continue;
+        }
+
+        if(status.type() == fs::file_type::directory) {
+            auto& dentry = entries.emplace_back();
+            dentry.type = DirectoryEntry::DIRECTORY;
+            dentry.name = entry.path().filename();
+
+            dentry.empty = fs::is_empty(entry.path(), error);
+            if(error)
+                logWarning(LOG_FT, "Failed to query directory empty state for directory {} ({}/{})", target_path.string(), error.value(), error.message());
+
+            dentry.modified_at = fs::last_write_time(entry.path(), error);
+            if(error)
+                logWarning(LOG_FT, "Failed to query last write time for directory {} ({}/{})", entry.path().string(), error.value(), error.message());
+            dentry.size = 0;
+        } else if(status.type() == fs::file_type::regular) {
+            auto& dentry = entries.emplace_back();
+            dentry.type = DirectoryEntry::FILE;
+            dentry.name = entry.path().filename();
+
+            dentry.modified_at = fs::last_write_time(entry.path(), error);
+            if(error)
+                logWarning(LOG_FT, "Failed to query last write time for file {} ({}/{}).", entry.path().string(), error.value(), error.message());
+            dentry.size = fs::file_size(entry.path(), error);
+            if(error)
+                logWarning(LOG_FT, "Failed to query size for file {} ({}/{}).", entry.path().string(), error.value(), error.message());
+        } else {
+            logWarning(LOG_FT, "Directory query listed an unknown file type for file {} ({}).", entry.path().string(), (int) status.type());
+        }
+    }
+    if(error && entries.empty()) {
+        response->emplace_fail(DirectoryQueryErrorType::FAILED_TO_LIST_FILES, std::to_string(error.value()) + "/" + error.message());
+        return response;
+    }
+    response->emplace_success(std::forward<decltype(entries)>(entries));
+    return response;
+}
+
+std::shared_ptr<directory_query_response_t> LocalFileSystem::query_icon_directory(const std::shared_ptr<VirtualFileServer> &id) {
+    return this->query_directory(this->server_path(id) / fs::u8path("icons"), "/", true);
+}
+
+std::shared_ptr<directory_query_response_t> LocalFileSystem::query_avatar_directory(const std::shared_ptr<VirtualFileServer> &id) {
+    return this->query_directory(this->server_path(id) / fs::u8path("avatars"), "/", true);
+}
+
+std::shared_ptr<directory_query_response_t> LocalFileSystem::query_channel_directory(const std::shared_ptr<VirtualFileServer> &id, ChannelId channelId, const std::string &path) {
+    return this->query_directory(this->server_channel_path(id, channelId), path, false);
+}
+
+std::shared_ptr<ExecuteResponse<DirectoryModifyError>> LocalFileSystem::create_channel_directory(const std::shared_ptr<VirtualFileServer> &id, ChannelId channelId, const std::string &path) {
+    auto channel_path_root = this->server_channel_path(id, channelId);
+    std::error_code error{};
+
+    auto response = this->create_execute_response<DirectoryModifyError>();
+    auto target_path = channel_path_root / fs::u8path(path);
+    if(this->exceeds_base_path(channel_path_root, target_path)) {
+        response->emplace_fail(DirectoryModifyErrorType::PATH_EXCEEDS_ROOT_PATH, "");
+        return response;
+    }
+
+    if(fs::exists(target_path, error)) {
+        response->emplace_fail(DirectoryModifyErrorType::PATH_ALREADY_EXISTS, "");
+        return response;
+    } else if(error) {
+        logWarning(LOG_FT, "Failed to check for file at {}: {}. Assuming it does not exists.", target_path.string(), error.value(), error.message());
+    }
+
+    if(!fs::create_directories(target_path, error) || error) {
+        response->emplace_fail(DirectoryModifyErrorType::FAILED_TO_CREATE_DIRECTORIES, std::to_string(error.value()) + "/" + error.message());
+        return response;
+    }
+
+    response->emplace_success();
+    return response;
+}
+
+std::shared_ptr<ExecuteResponse<FileModifyError>> LocalFileSystem::rename_channel_file(const std::shared_ptr<VirtualFileServer> &id, ChannelId channelId, const std::string &current_path_string, ChannelId targetChannelId, const std::string &new_path_string) {
+    auto channel_path_root = this->server_channel_path(id, channelId);
+    auto target_path_root = this->server_channel_path(id, targetChannelId);
+    std::error_code error{};
+    std::string locked_file{};
+
+    auto response = this->create_execute_response<FileModifyError>();
+    auto current_path = channel_path_root / fs::u8path(current_path_string);
+    auto target_path = target_path_root / fs::u8path(new_path_string);
+
+    if(this->exceeds_base_path(channel_path_root, current_path)) {
+        response->emplace_fail(FileModifyErrorType::PATH_EXCEEDS_ROOT_PATH, "");
+        return response;
+    }
+    if(this->exceeds_base_path(target_path_root, target_path)) {
+        response->emplace_fail(FileModifyErrorType::TARGET_PATH_EXCEEDS_ROOT_PATH, "");
+        return response;
+    }
+
+    if(!fs::exists(current_path, error)) {
+        response->emplace_fail(FileModifyErrorType::PATH_DOES_NOT_EXISTS, "");
+        return response;
+    } else if(error) {
+        logWarning(LOG_FT, "Failed to check for file at {}: {}. Assuming it does not exists.", current_path.string(), error.value(), error.message());
+        response->emplace_fail(FileModifyErrorType::PATH_DOES_NOT_EXISTS, "");
+        return response;
+    }
+
+    if(!fs::exists(target_path.parent_path(), error)) {
+        if(!fs::create_directories(target_path.parent_path(), error)) {
+            response->emplace_fail(FileModifyErrorType::FAILED_TO_CREATE_DIRECTORIES, std::to_string(error.value()) + "/" + error.message());
+            return response;
+        }
+    } else if(error) {
+        logWarning(LOG_FT, "Failed to test for target directory existence for {}: {}/{}. Assuming it does exists", target_path.parent_path().string(), error.value(), error.message());
+    }
+
+    if(fs::exists(target_path, error)) {
+        response->emplace_fail(FileModifyErrorType::TARGET_PATH_ALREADY_EXISTS, "");
+        return response;
+    } else if(error) {
+        logWarning(LOG_FT, "Failed to check for file at {}: {}. Assuming it does exists.", current_path.string(), error.value(), error.message());
+        response->emplace_fail(FileModifyErrorType::TARGET_PATH_ALREADY_EXISTS, "");
+        return response;
+    }
+
+    if(this->is_any_file_locked(channel_path_root, current_path, locked_file)) {
+        response->emplace_fail(FileModifyErrorType::SOME_FILES_ARE_LOCKED, locked_file);
+        return response;
+    }
+
+    fs::rename(current_path, target_path, error);
+    if(error) {
+        response->emplace_fail(FileModifyErrorType::FAILED_TO_RENAME_FILE, std::to_string(error.value()) + "/" + error.message());
+        return response;
+    }
+
+    response->emplace_success();
+    return response;
+}
+
+std::shared_ptr<ExecuteResponse<FileDeleteError, FileDeleteResponse>> LocalFileSystem::delete_files(const fs::path &base,
+                                                                               const std::vector<std::string> &paths) {
+    std::error_code error{};
+    std::string locked_file{};
+    auto response = this->create_execute_response<FileDeleteError, FileDeleteResponse>();
+
+    std::vector<FileDeleteResponse::DeleteResult> delete_results{};
+    for(const auto& path : paths) {
+        auto target_path = base / fs::u8path(path);
+
+        if(!fs::exists(target_path, error)) {
+            delete_results.emplace_back(FileDeleteResponse::StatusType::PATH_DOES_NOT_EXISTS, "");
+            continue;
+        } else if(error) {
+            logWarning(LOG_FT, "Failed to check for file at {}: {}. Assuming it does exists.", target_path.string(), error.value(), error.message());
+            delete_results.emplace_back(FileDeleteResponse::StatusType::PATH_DOES_NOT_EXISTS, "");
+            continue;
+        }
+
+        if(this->is_any_file_locked(base, path, locked_file)) {
+            delete_results.emplace_back(FileDeleteResponse::StatusType::SOME_FILES_ARE_LOCKED, locked_file);
+            continue;
+        }
+
+        if(!fs::remove_all(target_path, error) || error) {
+            delete_results.emplace_back(FileDeleteResponse::StatusType::FAILED_TO_DELETE_FILES, std::to_string(error.value()) + "/" + error.message());
+            continue;
+        }
+
+        delete_results.emplace_back(FileDeleteResponse::StatusType::SUCCESS, "");
+    }
+
+    response->emplace_success(FileDeleteResponse{delete_results});
+    return response;
+}
+
+std::shared_ptr<ExecuteResponse<FileDeleteError, FileDeleteResponse>> LocalFileSystem::delete_channel_files(const std::shared_ptr<VirtualFileServer> &id, ChannelId channelId, const std::vector<std::string> &path) {
+    return this->delete_files(this->server_channel_path(id, channelId), path);
+}
+
+std::shared_ptr<ExecuteResponse<FileDeleteError, FileDeleteResponse>> LocalFileSystem::delete_icons(const std::shared_ptr<VirtualFileServer> &id, const std::vector<std::string> &icon) {
+    return this->delete_files(this->server_path(id) / fs::u8path("icons"), icon);
+}
+
+std::shared_ptr<ExecuteResponse<FileDeleteError, FileDeleteResponse>> LocalFileSystem::delete_avatars(const std::shared_ptr<VirtualFileServer> &id, const std::vector<std::string> &avatar) {
+    return this->delete_files(this->server_path(id) / fs::u8path("avatars"), avatar);
+}
+
+std::shared_ptr<ExecuteResponse<FileInfoError, FileInfoResponse>> LocalFileSystem::query_file_info(const std::vector<std::tuple<fs::path, std::string>> &paths) {
+    std::error_code error{};
+    auto response = this->create_execute_response<FileInfoError, FileInfoResponse>();
+    std::vector<FileInfoResponse::FileInfo> file_infos{};
+    file_infos.reserve(paths.size());
+
+    for(const auto& [base, path] : paths) {
+        auto target_path = base / fs::u8path(path);
+        if(this->exceeds_base_path(base, target_path)) {
+            file_infos.emplace_back(FileInfoResponse::StatusType::PATH_EXCEEDS_ROOT_PATH, "", DirectoryEntry{});
+            continue;
+        }
+
+        if(!fs::exists(target_path, error)) {
+            file_infos.emplace_back(FileInfoResponse::StatusType::PATH_DOES_NOT_EXISTS, "", DirectoryEntry{});
+            continue;
+        } else if(error) {
+            logWarning(LOG_FT, "Failed to check for file at {}: {}. Assuming it does not exists.", target_path.string(), error.value(), error.message());
+
+            file_infos.emplace_back(FileInfoResponse::StatusType::PATH_DOES_NOT_EXISTS, "", DirectoryEntry{});
+            continue;
+        }
+
+        auto status = fs::status(target_path, error);
+        if(error) {
+            logWarning(LOG_FT, "Failed to query file status for {} ({}/{}). Skipping entry for file info query.", target_path.string(), error.value(), error.message());
+
+            file_infos.emplace_back(FileInfoResponse::StatusType::FAILED_TO_QUERY_INFO, "", DirectoryEntry{});
+            continue;
+        }
+
+        if(status.type() == fs::file_type::directory) {
+            DirectoryEntry dentry{};
+            dentry.type = DirectoryEntry::DIRECTORY;
+            dentry.name = target_path.filename();
+            dentry.empty = fs::is_empty(target_path, error);
+            if(error)
+                logWarning(LOG_FT, "Failed to query directory empty state for directory {} ({}/{})", target_path.string(), error.value(), error.message());
+
+            dentry.modified_at = fs::last_write_time(target_path, error);
+            if(error)
+                logWarning(LOG_FT, "Failed to query last write time for directory {} ({}/{})", target_path.string(), error.value(), error.message());
+            dentry.size = 0;
+            file_infos.emplace_back(FileInfoResponse::StatusType::SUCCESS, "", dentry);
+        } else if(status.type() == fs::file_type::regular) {
+            DirectoryEntry dentry{};
+            dentry.type = DirectoryEntry::FILE;
+            dentry.name = target_path.filename();
+
+            dentry.modified_at = fs::last_write_time(target_path, error);
+            if(error)
+                logWarning(LOG_FT, "Failed to query last write time for file {} ({}/{}).", target_path.string(), error.value(), error.message());
+            dentry.size = fs::file_size(target_path, error);
+            if(error)
+                logWarning(LOG_FT, "Failed to query size for file {} ({}/{}).", target_path.string(), error.value(), error.message());
+            file_infos.emplace_back(FileInfoResponse::StatusType::SUCCESS, "", dentry);
+        } else {
+            logWarning(LOG_FT, "File info query contains an unknown file type for file {} ({}).", target_path.string(), (int) status.type());
+            file_infos.emplace_back(FileInfoResponse::StatusType::UNKNOWN_FILE_TYPE, "", DirectoryEntry{});
+        }
+    }
+
+    response->emplace_success(FileInfoResponse{file_infos});
+    return response;
+}
+
+std::shared_ptr<ExecuteResponse<FileInfoError, FileInfoResponse>> LocalFileSystem::query_channel_info(const std::shared_ptr<VirtualFileServer> &sid, const std::vector<std::tuple<ChannelId, std::string>>& files) {
+    std::vector<std::tuple<fs::path, std::string>> file_paths{};
+    for(const auto& [channelId, path] : files)
+        file_paths.emplace_back(this->server_channel_path(sid, channelId), path);
+    return this->query_file_info(file_paths);
+}
+
+std::shared_ptr<ExecuteResponse<FileInfoError, FileInfoResponse>> LocalFileSystem::query_icon_info(const std::shared_ptr<VirtualFileServer> &sid, const std::vector<std::string> &paths) {
+    std::vector<std::tuple<fs::path, std::string>> file_paths{};
+    for(const auto& path : paths)
+        file_paths.emplace_back(this->server_path(sid) / fs::u8path("icons"), path);
+    return this->query_file_info(file_paths);
+}
+
+std::shared_ptr<ExecuteResponse<FileInfoError, FileInfoResponse> > LocalFileSystem::query_avatar_info(const std::shared_ptr<VirtualFileServer> &sid, const std::vector<std::string> &paths) {
+    std::vector<std::tuple<fs::path, std::string>> file_paths{};
+    for(const auto& path : paths)
+        file_paths.emplace_back(this->server_path(sid) / fs::u8path("avatars"), path);
+    return this->query_file_info(file_paths);
+}

file/local_server/LocalFileSystem.h

@@ -0,0 +1,117 @@
+//
+// Created by WolverinDEV on 16/09/2020.
+//
+
+#pragma once
+
+#pragma once
+
+#include <files/FileServer.h>
+#include <deque>
+#include <utility>
+#include <thread>
+#include <shared_mutex>
+#include <sys/socket.h>
+#include <pipes/ws.h>
+#include <pipes/ssl.h>
+#include <misc/net.h>
+#include <misc/spin_mutex.h>
+#include <random>
+#include <misc/memtracker.h>
+#include "./NetTools.h"
+
+namespace ts::server::file::filesystem {
+#ifdef FS_INCLUDED
+    namespace fs = std::experimental::filesystem;
+#endif
+
+    class LocalFileSystem : public filesystem::AbstractProvider {
+            using FileModifyError = filesystem::FileModifyError;
+            using DirectoryModifyError = filesystem::DirectoryModifyError;
+        public:
+            enum struct FileCategory {
+                ICON,
+                AVATAR,
+                CHANNEL
+            };
+
+            virtual ~LocalFileSystem();
+
+            bool initialize(std::string & /* error */, const std::string & /* root path */);
+            void finalize();
+
+            void lock_file(const std::string& /* absolute path */);
+            void unlock_file(const std::string& /* absolute path */);
+
+            [[nodiscard]] inline const auto &root_path() const { return this->root_path_; }
+
+            [[nodiscard]] std::string absolute_avatar_path(const std::shared_ptr<VirtualFileServer> &, const std::string&);
+            [[nodiscard]] std::string absolute_icon_path(const std::shared_ptr<VirtualFileServer> &, const std::string&);
+            [[nodiscard]] std::string absolute_channel_path(const std::shared_ptr<VirtualFileServer> &, ChannelId, const std::string&);
+
+            std::shared_ptr<ExecuteResponse<ServerCommandError>> initialize_server(const std::shared_ptr<VirtualFileServer> & /* server */) override;
+            std::shared_ptr<ExecuteResponse<ServerCommandError>> delete_server(const std::shared_ptr<VirtualFileServer> & /* server */) override;
+
+            std::shared_ptr<ExecuteResponse<FileInfoError, FileInfoResponse>>
+            query_channel_info(const std::shared_ptr<VirtualFileServer> & /* server */,  const std::vector<std::tuple<ChannelId, std::string>>& /* names */) override;
+
+            std::shared_ptr<directory_query_response_t>
+            query_channel_directory(const std::shared_ptr<VirtualFileServer> & id, ChannelId channelId, const std::string &string) override;
+
+            std::shared_ptr<ExecuteResponse<DirectoryModifyError>>
+            create_channel_directory(const std::shared_ptr<VirtualFileServer> & id, ChannelId channelId, const std::string &string) override;
+
+            std::shared_ptr<ExecuteResponse<FileDeleteError, FileDeleteResponse>>
+            delete_channel_files(const std::shared_ptr<VirtualFileServer> & id, ChannelId channelId, const std::vector<std::string> &string) override;
+
+            std::shared_ptr<ExecuteResponse<FileModifyError>>
+            rename_channel_file(const std::shared_ptr<VirtualFileServer> & id, ChannelId channelId, const std::string &, ChannelId, const std::string &) override;
+
+            std::shared_ptr<ExecuteResponse<FileInfoError, FileInfoResponse>>
+            query_icon_info(const std::shared_ptr<VirtualFileServer> & /* server */, const std::vector<std::string>& /* names */) override;
+
+            std::shared_ptr<directory_query_response_t> query_icon_directory(const std::shared_ptr<VirtualFileServer> & id) override;
+
+            std::shared_ptr<ExecuteResponse<FileDeleteError, FileDeleteResponse>>
+            delete_icons(const std::shared_ptr<VirtualFileServer> & id, const std::vector<std::string> &string) override;
+
+            std::shared_ptr<ExecuteResponse<FileInfoError, FileInfoResponse>>
+            query_avatar_info(const std::shared_ptr<VirtualFileServer> & /* server */, const std::vector<std::string>& /* names */) override;
+
+            std::shared_ptr<directory_query_response_t> query_avatar_directory(const std::shared_ptr<VirtualFileServer> & id) override;
+
+            std::shared_ptr<ExecuteResponse<FileDeleteError, FileDeleteResponse>>
+            delete_avatars(const std::shared_ptr<VirtualFileServer> & id, const std::vector<std::string> &string) override;
+
+        private:
+#ifdef FS_INCLUDED
+            [[nodiscard]] fs::path server_path(const std::shared_ptr<VirtualFileServer> &);
+            [[nodiscard]] fs::path server_channel_path(const std::shared_ptr<VirtualFileServer> &, ChannelId);
+            [[nodiscard]] static bool exceeds_base_path(const fs::path& /* base */, const fs::path& /* target */);
+            [[nodiscard]] bool is_any_file_locked(const fs::path& /* base */, const std::string& /* path */, std::string& /* file (relative to the base) */);
+
+            [[nodiscard]] std::shared_ptr<ExecuteResponse<FileDeleteError, FileDeleteResponse>>
+            delete_files(const fs::path& /* base */, const std::vector<std::string> &string);
+
+            [[nodiscard]] std::shared_ptr<directory_query_response_t>
+            query_directory(const fs::path& /* base */, const std::string &string, bool);
+
+            [[nodiscard]] std::shared_ptr<ExecuteResponse<FileInfoError, FileInfoResponse>>
+            query_file_info(const std::vector<std::tuple<fs::path, std::string>> &string);
+#endif
+
+            template <typename error_t, typename result_t = EmptyExecuteResponse>
+            std::shared_ptr<ExecuteResponse<error_t, result_t>> create_execute_response() {
+                return std::make_shared<ExecuteResponse<error_t, result_t>>(this->result_notify_mutex, this->result_notify_cv);
+            }
+            std::string target_file_path(FileCategory type, const std::shared_ptr<VirtualFileServer> &sid, ts::ChannelId cid, const std::string &path);
+
+            std::mutex result_notify_mutex{};
+            std::condition_variable result_notify_cv{};
+
+            std::string root_path_{};
+
+            std::mutex locked_files_mutex{};
+            std::deque<std::string> locked_files_{};
+    };
+}

file/local_server/LocalFileTransfer.cpp

@@ -0,0 +1,385 @@
+//
+// Created by WolverinDEV on 04/05/2020.
+//
+
+#include <cassert>
+#include <event2/event.h>
+#include <log/LogUtils.h>
+#include <random>
+#include "./LocalFileProvider.h"
+#include "./LocalFileTransfer.h"
+#include <experimental/filesystem>
+
+namespace fs = std::experimental::filesystem;
+using namespace ts::server::file;
+using namespace ts::server::file::transfer;
+
+Buffer* transfer::allocate_buffer(size_t size) {
+    auto total_size = sizeof(Buffer) + size;
+    auto buffer = (Buffer*) malloc(total_size);
+    new (buffer) Buffer{};
+    buffer->capacity = size;
+    return buffer;
+}
+
+void transfer::free_buffer(Buffer* buffer) {
+    buffer->~Buffer();
+    free(buffer);
+}
+
+FileClient::~FileClient() {
+    this->flush_network_buffer();
+    this->flush_disk_buffer();
+
+    assert(!this->disk_buffer.buffer_head);
+    assert(!this->network_buffer.buffer_head);
+
+    assert(!this->file.file_descriptor);
+    assert(!this->file.currently_processing);
+    assert(!this->file.next_client);
+
+    assert(!this->networking.event_read);
+    assert(!this->networking.event_write);
+
+    assert(this->state == STATE_DISCONNECTED);
+    memtrack::freed<FileClient>(this);
+}
+
+LocalFileTransfer::LocalFileTransfer(filesystem::LocalFileSystem *fs) : file_system_{fs} {}
+LocalFileTransfer::~LocalFileTransfer() = default;
+
+bool LocalFileTransfer::start() {
+    (void) this->start_client_worker();
+
+    {
+        auto start_result = this->start_disk_io();
+        switch (start_result) {
+            case DiskIOStartResult::SUCCESS:
+                break;
+            case DiskIOStartResult::OUT_OF_MEMORY:
+                logError(LOG_FT, "Failed to start disk worker (Out of memory)");
+                goto error_exit_disk;
+            default:
+                logError(LOG_FT, "Failed to start disk worker ({})", (int) start_result);
+                goto error_exit_disk;
+        }
+    }
+
+
+    {
+        auto start_result = this->start_networking();
+        switch (start_result) {
+            case NetworkingStartResult::SUCCESS:
+                break;
+
+            case NetworkingStartResult::OUT_OF_MEMORY:
+                logError(LOG_FT, "Failed to start networking (Out of memory)");
+                goto error_exit_network;
+
+            default:
+                logError(LOG_FT, "Failed to start networking ({})", (int) start_result);
+                goto error_exit_network;
+        }
+    }
+
+    return true;
+    error_exit_network:
+    this->shutdown_networking();
+
+    error_exit_disk:
+    this->shutdown_disk_io();
+    this->shutdown_client_worker();
+    return false;
+}
+
+void LocalFileTransfer::stop() {
+    this->shutdown_networking();
+    this->shutdown_disk_io();
+    this->shutdown_client_worker();
+}
+
+std::shared_ptr<ExecuteResponse<TransferInitError, std::shared_ptr<Transfer>>>
+        LocalFileTransfer::initialize_icon_transfer(Transfer::Direction direction, const std::shared_ptr<VirtualFileServer> &server, const TransferInfo &info) {
+    return this->initialize_transfer(direction, server, 0, Transfer::TARGET_TYPE_ICON, info);
+}
+
+std::shared_ptr<ExecuteResponse<TransferInitError, std::shared_ptr<Transfer>>>
+        LocalFileTransfer::initialize_avatar_transfer(Transfer::Direction direction, const std::shared_ptr<VirtualFileServer> &server, const TransferInfo &info) {
+    return this->initialize_transfer(direction, server, 0, Transfer::TARGET_TYPE_AVATAR, info);
+}
+
+std::shared_ptr<ExecuteResponse<TransferInitError, std::shared_ptr<Transfer>>>
+        LocalFileTransfer::initialize_channel_transfer(Transfer::Direction direction, const std::shared_ptr<VirtualFileServer> &server, ChannelId cid, const TransferInfo &info) {
+    return this->initialize_transfer(direction, server, cid, Transfer::TARGET_TYPE_CHANNEL_FILE, info);
+}
+
+std::shared_ptr<ExecuteResponse<TransferInitError, std::shared_ptr<Transfer>>> LocalFileTransfer::initialize_transfer(
+        Transfer::Direction direction, const std::shared_ptr<VirtualFileServer> &server, ChannelId cid,
+        Transfer::TargetType ttype,
+        const TransferInfo &info) {
+    auto response = this->create_execute_response<TransferInitError, std::shared_ptr<Transfer>>();
+
+    std::lock_guard clock{this->transfer_create_mutex};
+    if(info.max_concurrent_transfers > 0) {
+        std::unique_lock tlock{this->transfers_mutex};
+        {
+            auto transfers = std::count_if(this->transfers_.begin(), this->transfers_.end(), [&](const std::shared_ptr<FileClient>& client) {
+                return client->transfer && client->transfer->client_unique_id == info.client_unique_id && client->state < FileClient::STATE_FLUSHING;
+            });
+            transfers += std::count_if(this->pending_transfers.begin(), this->pending_transfers.end(), [&](const std::shared_ptr<Transfer>& transfer) {
+                return transfer->client_unique_id == info.client_unique_id;
+            });
+
+            if(transfers >= info.max_concurrent_transfers) {
+                response->emplace_fail(TransferInitError::CLIENT_TOO_MANY_TRANSFERS, std::to_string(transfers));
+                return response;
+            }
+        }
+
+        {
+            auto server_transfers = this->pending_transfers.size();
+            server_transfers += std::count_if(this->transfers_.begin(), this->transfers_.end(), [&](const std::shared_ptr<FileClient>& client) {
+                return client->transfer;
+            });
+            if(server_transfers >= this->max_concurrent_transfers) {
+                response->emplace_fail(TransferInitError::SERVER_TOO_MANY_TRANSFERS, std::to_string(server_transfers));
+                return response;
+            }
+        }
+    }
+
+    auto transfer = std::make_shared<Transfer>();
+    transfer->server_transfer_id = server->generate_transfer_id();
+    transfer->server = server;
+    transfer->channel_id = cid;
+    transfer->target_type = ttype;
+    transfer->direction = direction;
+
+    transfer->client_id = 0; /* must be provided externally */
+    transfer->client_transfer_id = 0; /* must be provided externally */
+
+    transfer->server_addresses.reserve(this->network.bindings.size());
+    for(auto& binding : this->network.bindings) {
+        if(!binding->file_descriptor) continue;
+
+        transfer->server_addresses.emplace_back(Transfer::Address{binding->hostname, net::port(binding->address)});
+    }
+
+    transfer->target_file_path = info.file_path;
+    transfer->file_offset = info.file_offset;
+    transfer->expected_file_size = info.expected_file_size;
+    transfer->max_bandwidth = info.max_bandwidth;
+    transfer->client_unique_id = info.client_unique_id;
+    transfer->client_id = info.client_id;
+
+    constexpr static std::string_view kTokenCharacters{"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"};
+    for(auto& c : transfer->transfer_key)
+        c = kTokenCharacters[transfer_random_token_generator() % kTokenCharacters.length()];
+    transfer->transfer_key[0] = (char) 'r'; /* (114) */ /* a non valid SSL header type to indicate that we're using a file transfer key and not doing a SSL handshake */
+    transfer->transfer_key[1] = (char) 'a'; /* ( 97) */
+    transfer->transfer_key[2] = (char) 'w'; /* (119) */
+
+    transfer->initialized_timestamp = std::chrono::system_clock::now();
+
+    {
+        std::string absolute_path{};
+        switch (transfer->target_type) {
+            case Transfer::TARGET_TYPE_AVATAR:
+                absolute_path = this->file_system_->absolute_avatar_path(transfer->server, transfer->target_file_path);
+                break;
+            case Transfer::TARGET_TYPE_ICON:
+                absolute_path = this->file_system_->absolute_icon_path(transfer->server, transfer->target_file_path);
+                break;
+            case Transfer::TARGET_TYPE_CHANNEL_FILE:
+                absolute_path = this->file_system_->absolute_channel_path(transfer->server, transfer->channel_id, transfer->target_file_path);
+                break;
+            case Transfer::TARGET_TYPE_UNKNOWN:
+            default:
+                response->emplace_fail(TransferInitError::INVALID_FILE_TYPE, "");
+                return response;
+        }
+        transfer->absolute_file_path = absolute_path;
+
+        const auto root_path_length = this->file_system_->root_path().size();
+        if(root_path_length < absolute_path.size())
+            transfer->relative_file_path = absolute_path.substr(root_path_length);
+        else
+            transfer->relative_file_path = "error";
+        transfer->file_name = fs::u8path(absolute_path).filename();
+    }
+
+    if(direction == Transfer::DIRECTION_DOWNLOAD) {
+        auto path = fs::u8path(transfer->absolute_file_path);
+        std::error_code error{};
+        if(!fs::exists(path, error)) {
+            response->emplace_fail(TransferInitError::FILE_DOES_NOT_EXISTS, "");
+            return response;
+        } else if(error) {
+            logWarning(LOG_FT, "Failed to check for file at {}: {}. Assuming it does not exists.", transfer->absolute_file_path, error.value(), error.message());
+            response->emplace_fail(TransferInitError::FILE_DOES_NOT_EXISTS, "");
+            return response;
+        }
+
+        auto status = fs::status(path, error);
+        if(error) {
+            logWarning(LOG_FT, "Failed to status for file at {}: {}. Ignoring file transfer.", transfer->absolute_file_path, error.value(), error.message());
+            response->emplace_fail(TransferInitError::IO_ERROR, "stat");
+            return response;
+        }
+
+        if(status.type() != fs::file_type::regular) {
+            response->emplace_fail(TransferInitError::FILE_IS_NOT_A_FILE, "");
+            return response;
+        }
+
+        transfer->expected_file_size = fs::file_size(path, error);
+        if(error) {
+            logWarning(LOG_FT, "Failed to get file size for file at {}: {}. Ignoring file transfer.", transfer->absolute_file_path, error.value(), error.message());
+            response->emplace_fail(TransferInitError::IO_ERROR, "file_size");
+            return response;
+        }
+
+        if(info.download_client_quota_limit > 0 && info.download_client_quota_limit <= transfer->expected_file_size) {
+            response->emplace_fail(TransferInitError::CLIENT_QUOTA_EXCEEDED, "");
+            return response;
+        }
+        if(info.download_server_quota_limit > 0 && info.download_server_quota_limit <= transfer->expected_file_size) {
+            response->emplace_fail(TransferInitError::SERVER_QUOTA_EXCEEDED, "");
+            return response;
+        }
+    }
+
+    {
+        std::lock_guard tlock{this->transfers_mutex};
+        this->pending_transfers.push_back(transfer);
+    }
+
+    switch (transfer->target_type) {
+        case Transfer::TARGET_TYPE_AVATAR:
+            logMessage(LOG_FT, "Initialized avatar transfer for avatar \"{}\" ({} bytes, transferring {} bytes).", transfer->target_file_path, transfer->expected_file_size, transfer->expected_file_size - transfer->file_offset);
+            break;
+        case Transfer::TARGET_TYPE_ICON:
+            logMessage(LOG_FT, "Initialized icon transfer for icon \"{}\" ({} bytes, transferring {} bytes).",
+                       transfer->target_file_path, transfer->expected_file_size, transfer->expected_file_size - transfer->file_offset);
+            break;
+        case Transfer::TARGET_TYPE_CHANNEL_FILE:
+            logMessage(LOG_FT, "Initialized channel transfer for file \"{}/{}\" ({} bytes, transferring {} bytes).",
+                       transfer->channel_id, transfer->target_file_path, transfer->expected_file_size, transfer->expected_file_size - transfer->file_offset);
+            break;
+        case Transfer::TARGET_TYPE_UNKNOWN:
+        default:
+            response->emplace_fail(TransferInitError::INVALID_FILE_TYPE, "");
+            return response;
+    }
+
+    if(auto callback{this->callback_transfer_registered}; callback)
+        callback(transfer);
+
+    response->emplace_success(std::move(transfer));
+    return response;
+}
+
+std::shared_ptr<ExecuteResponse<TransferActionError>> LocalFileTransfer::stop_transfer(const std::shared_ptr<VirtualFileServer>& server, transfer_id id, bool flush) {
+    auto response = this->create_execute_response<TransferActionError>();
+
+    std::shared_ptr<Transfer> transfer{};
+    std::shared_ptr<FileClient> connected_transfer{};
+
+    {
+        std::lock_guard tlock{this->transfers_mutex};
+
+        auto ct_it = std::find_if(this->transfers_.begin(), this->transfers_.end(), [&](const std::shared_ptr<FileClient>& t) {
+            return t->transfer && t->transfer->server_transfer_id == id && t->transfer->server == server;
+        });
+        if(ct_it != this->transfers_.end())
+            connected_transfer = *ct_it;
+        else {
+            auto t_it = std::find_if(this->pending_transfers.begin(), this->pending_transfers.end(), [&](const std::shared_ptr<Transfer>& t) {
+                return t->server_transfer_id == id && t->server == server;
+            });
+            if(t_it != this->pending_transfers.end()) {
+                transfer = *t_it;
+                this->pending_transfers.erase(t_it);
+            }
+        }
+    }
+
+    if(!transfer) {
+        if(connected_transfer)
+            transfer = connected_transfer->transfer;
+        else {
+            response->emplace_fail(TransferActionError{TransferActionError::UNKNOWN_TRANSFER, ""});
+            return response;
+        }
+    }
+
+    if(connected_transfer) {
+        this->invoke_aborted_callback(connected_transfer, { TransferError::USER_REQUEST, "" });
+        logMessage(LOG_FT, "{} Stopping transfer due to an user request.", connected_transfer->log_prefix());
+
+        std::unique_lock slock{connected_transfer->state_mutex};
+        this->disconnect_client(connected_transfer, slock, flush);
+    } else {
+        this->invoke_aborted_callback(transfer, { TransferError::USER_REQUEST, "" });
+        logMessage(LOG_FT, "Removing pending file transfer for id {}", id);
+    }
+
+    response->emplace_success();
+    return response;
+}
+
+inline void apply_transfer_info(const std::shared_ptr<Transfer>& transfer, ActiveFileTransfer& info) {
+    info.server_transfer_id = transfer->server_transfer_id;
+    info.client_transfer_id = transfer->client_transfer_id;
+    info.direction = transfer->direction;
+    info.client_id = transfer->client_id;
+    info.client_unique_id = transfer->client_unique_id;
+
+    info.file_path = transfer->relative_file_path;
+    info.file_name = transfer->file_name;
+
+    info.expected_size = transfer->expected_file_size;
+}
+
+std::shared_ptr<ExecuteResponse<TransferListError, std::vector<ActiveFileTransfer>>> LocalFileTransfer::list_transfer() {
+    std::vector<ActiveFileTransfer> transfer_infos{};
+    auto response = this->create_execute_response<TransferListError, std::vector<ActiveFileTransfer>>();
+
+    std::unique_lock tlock{this->transfers_mutex};
+    auto awaiting_transfers = this->pending_transfers;
+    auto running_transfers = this->transfers_;
+    tlock.unlock();
+
+    transfer_infos.reserve(awaiting_transfers.size() + running_transfers.size());
+    for(const auto& transfer : awaiting_transfers) {
+        ActiveFileTransfer info{};
+        apply_transfer_info(transfer, info);
+        info.size_done = transfer->file_offset;
+
+        info.status = ActiveFileTransfer::NOT_STARTED;
+        info.runtime = std::chrono::milliseconds{0};
+
+        info.average_speed = 0;
+        info.current_speed = 0;
+        transfer_infos.push_back(info);
+    }
+
+    for(const auto& client : running_transfers) {
+        auto transfer = client->transfer;
+        if(!transfer) continue;
+
+        ActiveFileTransfer info{};
+        apply_transfer_info(transfer, info);
+        info.size_done = transfer->file_offset + client->statistics.file_transferred.total_bytes;
+
+        info.status = ActiveFileTransfer::RUNNING;
+        info.runtime = std::chrono::floor<std::chrono::milliseconds>(std::chrono::system_clock::now() - client->timings.key_received);
+
+        info.average_speed = client->statistics.file_transferred.average_bandwidth();
+        info.current_speed = client->statistics.file_transferred.current_bandwidth();
+        transfer_infos.push_back(info);
+    }
+
+    response->emplace_success(std::move(transfer_infos));
+    return response;
+}

file/local_server/LocalFileTransfer.h

@@ -0,0 +1,446 @@
+//
+// Created by WolverinDEV on 16/09/2020.
+//
+
+#pragma once
+
+#pragma once
+
+#include <files/FileServer.h>
+#include <deque>
+#include <utility>
+#include <thread>
+#include <shared_mutex>
+#include <sys/socket.h>
+#include <pipes/ws.h>
+#include <pipes/ssl.h>
+#include <misc/net.h>
+#include <misc/spin_mutex.h>
+#include <random>
+#include <misc/memtracker.h>
+#include "./NetTools.h"
+#include "LocalFileSystem.h"
+
+#define TRANSFER_MAX_CACHED_BYTES (1024 * 1024 * 1) // Buffer up to 1mb
+
+namespace ts::server::file::transfer {
+    class LocalFileTransfer;
+
+    struct Buffer {
+        Buffer* next{nullptr};
+
+        size_t capacity{0};
+        size_t length{0};
+        size_t offset{0};
+
+        char data[1]{};
+    };
+    [[nodiscard]] extern Buffer* allocate_buffer(size_t);
+    extern void free_buffer(Buffer*);
+
+    /* all variables are locked via the state_mutex */
+    struct FileClient : std::enable_shared_from_this<FileClient> {
+        LocalFileTransfer* handle;
+        std::shared_ptr<Transfer> transfer{nullptr};
+
+        std::shared_mutex state_mutex{};
+        enum {
+            STATE_AWAITING_KEY, /* includes SSL/HTTP init */
+            STATE_TRANSFERRING,
+            STATE_FLUSHING,
+            STATE_DISCONNECTED
+        } state{STATE_AWAITING_KEY};
+
+        bool finished_signal_send{false};
+
+        enum NetworkingProtocol {
+            PROTOCOL_UNKNOWN,
+            PROTOCOL_HTTPS,
+            PROTOCOL_TS_V1
+        };
+
+        enum HTTPUploadState {
+            HTTP_AWAITING_HEADER,
+            HTTP_STATE_AWAITING_BOUNDARY,
+            HTTP_STATE_AWAITING_BOUNDARY_END,
+            HTTP_STATE_UPLOADING,
+            HTTP_STATE_DOWNLOADING
+        };
+
+        struct {
+            bool file_locked{false};
+            int file_descriptor{0};
+
+            bool currently_processing{false};
+            FileClient* next_client{nullptr};
+
+            bool query_media_bytes{false};
+            uint8_t media_bytes[TRANSFER_MEDIA_BYTES_LENGTH]{0};
+            uint8_t media_bytes_length{0};
+        } file{};
+
+        struct {
+            size_t provided_bytes{0};
+            char key[TRANSFER_KEY_LENGTH]{0};
+        } transfer_key{};
+
+        struct {
+            std::mutex mutex{};
+            size_t bytes{0};
+
+            bool buffering_stopped{false};
+            bool write_disconnected{false};
+
+            Buffer* buffer_head{nullptr};
+            Buffer** buffer_tail{&buffer_head};
+        } network_buffer{};
+
+        struct {
+            std::mutex mutex{};
+            size_t bytes{0};
+
+            bool buffering_stopped{false};
+            bool write_disconnected{false};
+
+            Buffer* buffer_head{nullptr};
+            Buffer** buffer_tail{&buffer_head};
+        } disk_buffer{};
+
+        struct {
+            sockaddr_storage address{};
+            int file_descriptor{0};
+
+            NetworkingProtocol protocol{PROTOCOL_UNKNOWN};
+
+            struct event* event_read{nullptr};
+            struct event* event_write{nullptr};
+            struct event* event_throttle{nullptr};
+
+            bool add_event_write{false}, add_event_read{false};
+
+            std::chrono::system_clock::time_point disconnect_timeout{};
+
+            networking::NetworkThrottle client_throttle{};
+            /* the right side is the server throttle */
+            networking::DualNetworkThrottle throttle{&client_throttle, &networking::NetworkThrottle::kNoThrottle};
+
+            pipes::SSL pipe_ssl{};
+            bool pipe_ssl_init{false};
+            std::unique_ptr<Buffer, decltype(free_buffer)*> http_header_buffer{nullptr, free_buffer};
+            HTTPUploadState http_state{HTTPUploadState::HTTP_AWAITING_HEADER};
+
+            std::string http_boundary{};
+
+            /* Only read the transfer key length at the beginning. We than have the actual limit which will be set via throttle */
+            size_t max_read_buffer_size{TRANSFER_KEY_LENGTH};
+        } networking{};
+
+        struct {
+            networking::TransferStatistics network_send{};
+            networking::TransferStatistics network_received{};
+
+            networking::TransferStatistics file_transferred{};
+
+            networking::TransferStatistics disk_bytes_read{};
+            networking::TransferStatistics disk_bytes_write{};
+        } statistics{};
+
+        struct {
+            std::chrono::system_clock::time_point last_write{};
+            std::chrono::system_clock::time_point last_read{};
+
+            std::chrono::system_clock::time_point connected{};
+            std::chrono::system_clock::time_point key_received{};
+            std::chrono::system_clock::time_point disconnecting{};
+        } timings;
+
+        explicit FileClient(LocalFileTransfer* handle) : handle{handle} { memtrack::allocated<FileClient>(this); }
+        ~FileClient();
+
+        void add_network_write_event(bool /* ignore bandwidth limits */);
+        void add_network_write_event_nolock(bool /* ignore bandwidth limits */);
+
+        /* will check if we've enough space in out read buffer again */
+        void add_network_read_event(bool /* ignore bandwidth limits */);
+
+        bool send_file_bytes(const void* /* buffer */, size_t /* length */);
+        bool enqueue_network_buffer_bytes(const void* /* buffer */, size_t /* length */);
+        bool enqueue_disk_buffer_bytes(const void* /* buffer */, size_t /* length */);
+
+        /* these function clear the buffers and set the write disconnected flags to true so no new buffers will be enqueued */
+        size_t flush_network_buffer();
+        void flush_disk_buffer();
+
+        [[nodiscard]] bool buffers_flushed();
+        [[nodiscard]] inline std::string log_prefix() const { return "[" + net::to_string(this->networking.address) + "]"; }
+    };
+
+    enum struct DiskIOStartResult {
+        SUCCESS,
+        OUT_OF_MEMORY
+    };
+
+    enum struct NetworkingStartResult {
+        SUCCESS,
+        OUT_OF_MEMORY
+    };
+
+    enum struct NetworkingBindResult {
+        SUCCESS,
+
+        BINDING_ALREADY_EXISTS,
+        NETWORKING_NOT_INITIALIZED,
+        FAILED_TO_ALLOCATE_SOCKET, /* errno is set */
+        FAILED_TO_BIND,
+        FAILED_TO_LISTEN,
+
+        OUT_OF_MEMORY,
+    };
+
+    enum struct NetworkingUnbindResult {
+        SUCCESS,
+        UNKNOWN_BINDING
+    };
+
+    enum struct ClientWorkerStartResult {
+        SUCCESS
+    };
+
+    enum struct NetworkInitializeResult {
+        SUCCESS,
+        OUT_OF_MEMORY
+    };
+
+    enum struct FileInitializeResult {
+        SUCCESS,
+
+        INVALID_TRANSFER_DIRECTION,
+        OUT_OF_MEMORY,
+
+        PROCESS_FILE_LIMIT_REACHED,
+        SYSTEM_FILE_LIMIT_REACHED,
+
+        FILE_IS_BUSY,
+        FILE_DOES_NOT_EXISTS,
+        FILE_SYSTEM_ERROR,
+        FILE_IS_A_DIRECTORY,
+
+        FILE_TOO_LARGE,
+        DISK_IS_READ_ONLY,
+
+        FILE_SEEK_FAILED,
+        FILE_SIZE_MISMATCH,
+
+        FILE_IS_NOT_ACCESSIBLE,
+
+        FAILED_TO_READ_MEDIA_BYTES,
+        COUNT_NOT_CREATE_DIRECTORIES,
+
+        MAX
+    };
+
+    constexpr static std::array<std::string_view, (size_t) FileInitializeResult::MAX> kFileInitializeResultMessages{
+            /* SUCCESS */                       "success",
+
+            /* INVALID_TRANSFER_DIRECTION */    "invalid file transfer direction",
+            /* OUT_OF_MEMORY */                 "out of memory",
+
+            /* PROCESS_FILE_LIMIT_REACHED */    "process file limit reached",
+            /* SYSTEM_FILE_LIMIT_REACHED */     "system file limit reached",
+
+            /* FILE_IS_BUSY */                  "target file is busy",
+            /* FILE_DOES_NOT_EXISTS */          "target file does not exists",
+            /* FILE_SYSTEM_ERROR */             "internal file system error",
+            /* FILE_IS_A_DIRECTORY */           "target file is a directory",
+
+            /* FILE_TOO_LARGE */                "file is too large",
+            /* DISK_IS_READ_ONLY */             "disk is in read only mode",
+
+            /* FILE_SEEK_FAILED */              "failed to seek to target file offset",
+            /* FILE_SIZE_MISMATCH */            "file size miss match",
+            /* FILE_IS_NOT_ACCESSIBLE */        "file is not accessible",
+            /* FAILED_TO_READ_MEDIA_BYTES */    "failed to read file media bytes",
+            /* COUNT_NOT_CREATE_DIRECTORIES */  "could not create required directories"
+    };
+
+    enum struct TransferKeyApplyResult {
+        SUCCESS,
+        FILE_ERROR,
+        UNKNOWN_KEY,
+
+        INTERNAL_ERROR
+    };
+
+    enum struct TransferUploadRawResult {
+        MORE_DATA_TO_RECEIVE,
+        FINISH,
+        FINISH_OVERFLOW,
+
+        /* UNKNOWN ERROR */
+    };
+
+    enum struct TransferUploadHTTPResult {
+        MORE_DATA_TO_RECEIVE,
+        FINISH,
+
+        BOUNDARY_MISSING,
+        BOUNDARY_TOKEN_INVALID,
+        BOUNDARY_INVALID,
+
+        MISSING_CONTENT_TYPE,
+        INVALID_CONTENT_TYPE
+        /* UNKNOWN ERROR */
+    };
+
+    struct NetworkBinding {
+        std::string hostname{};
+        sockaddr_storage address{};
+    };
+
+    struct ActiveNetworkBinding : std::enable_shared_from_this<ActiveNetworkBinding> {
+        std::string hostname{};
+        sockaddr_storage address{};
+
+        int file_descriptor{-1};
+        struct event* accept_event{nullptr};
+
+        LocalFileTransfer* handle{nullptr};
+    };
+
+    class LocalFileTransfer : public AbstractProvider {
+        public:
+            explicit LocalFileTransfer(filesystem::LocalFileSystem*);
+            ~LocalFileTransfer();
+
+            [[nodiscard]] bool start();
+            void stop();
+
+            [[nodiscard]] NetworkingBindResult add_network_binding(const NetworkBinding& /* binding */);
+            [[nodiscard]] std::vector<NetworkBinding> active_network_bindings();
+            [[nodiscard]] NetworkingUnbindResult remove_network_binding(const NetworkBinding& /* binding */);
+
+            std::shared_ptr<ExecuteResponse<TransferInitError, std::shared_ptr<Transfer>>>
+            initialize_channel_transfer(Transfer::Direction direction, const std::shared_ptr<VirtualFileServer>& server, ChannelId channelId,
+                                        const TransferInfo &info) override;
+
+            std::shared_ptr<ExecuteResponse<TransferInitError, std::shared_ptr<Transfer>>>
+            initialize_icon_transfer(Transfer::Direction direction, const std::shared_ptr<VirtualFileServer>& server, const TransferInfo &info) override;
+
+            std::shared_ptr<ExecuteResponse<TransferInitError, std::shared_ptr<Transfer>>>
+            initialize_avatar_transfer(Transfer::Direction direction, const std::shared_ptr<VirtualFileServer>& server, const TransferInfo &info) override;
+
+            std::shared_ptr<ExecuteResponse<TransferListError, std::vector<ActiveFileTransfer>>> list_transfer() override;
+
+            std::shared_ptr<ExecuteResponse<TransferActionError>> stop_transfer(const std::shared_ptr<VirtualFileServer>& /* server */, transfer_id id, bool) override;
+        private:
+            enum struct DiskIOLoopState {
+                STOPPED,
+                RUNNING,
+
+                STOPPING,
+                FORCE_STOPPING
+            };
+            filesystem::LocalFileSystem* file_system_;
+
+            size_t max_concurrent_transfers{1024};
+            std::mt19937 transfer_random_token_generator{std::random_device{}()};
+
+            std::mutex result_notify_mutex{};
+            std::condition_variable result_notify_cv{};
+
+            std::mutex transfers_mutex{};
+            std::mutex transfer_create_mutex{};
+            std::deque<std::shared_ptr<FileClient>> transfers_{};
+            std::deque<std::shared_ptr<Transfer>> pending_transfers{};
+
+            enum ServerState {
+                STOPPED,
+                RUNNING
+            } state{ServerState::STOPPED};
+
+            struct {
+                bool active{false};
+
+                std::thread dispatch_thread{};
+                std::mutex mutex{};
+                std::condition_variable notify_cv{};
+            } disconnect;
+
+            struct {
+                std::mutex mutex;
+
+                bool active{false};
+                std::thread dispatch_thread{};
+                struct event_base* event_base{nullptr};
+
+                std::deque<std::shared_ptr<ActiveNetworkBinding>> bindings{};
+            } network{};
+
+            struct {
+                DiskIOLoopState state{DiskIOLoopState::STOPPED};
+                std::thread dispatch_thread{};
+                std::mutex queue_lock{};
+                std::condition_variable notify_work_awaiting{};
+                std::condition_variable notify_client_processed{};
+
+                FileClient* queue_head{nullptr};
+                FileClient** queue_tail{&queue_head};
+            } disk_io{};
+
+            template <typename error_t, typename result_t = EmptyExecuteResponse>
+            std::shared_ptr<ExecuteResponse<error_t, result_t>> create_execute_response() {
+                return std::make_shared<ExecuteResponse<error_t, result_t>>(this->result_notify_mutex, this->result_notify_cv);
+            }
+
+            std::shared_ptr<ExecuteResponse<TransferInitError, std::shared_ptr<Transfer>>>
+            initialize_transfer(Transfer::Direction, const std::shared_ptr<VirtualFileServer> &, ChannelId, Transfer::TargetType, const TransferInfo &info);
+
+            [[nodiscard]] NetworkingStartResult start_networking();
+            [[nodiscard]] DiskIOStartResult start_disk_io();
+            [[nodiscard]] ClientWorkerStartResult start_client_worker();
+
+            void shutdown_networking();
+            void shutdown_disk_io();
+            void shutdown_client_worker();
+
+            void disconnect_client(const std::shared_ptr<FileClient>& /* client */, std::unique_lock<std::shared_mutex>& /* state lock */, bool /* flush network */);
+
+            [[nodiscard]] NetworkInitializeResult initialize_networking(const std::shared_ptr<FileClient>& /* client */, int /* file descriptor */);
+            /* might block 'till all IO operations have been succeeded */
+            void finalize_networking(const std::shared_ptr<FileClient>& /* client */, std::unique_lock<std::shared_mutex>& /* state lock */);
+
+            [[nodiscard]] FileInitializeResult initialize_file_io(const std::shared_ptr<FileClient>& /* client */);
+            void finalize_file_io(const std::shared_ptr<FileClient>& /* client */, std::unique_lock<std::shared_mutex>& /* state lock */);
+
+            [[nodiscard]] bool initialize_client_ssl(const std::shared_ptr<FileClient>& /* client */);
+            void finalize_client_ssl(const std::shared_ptr<FileClient>& /* client */);
+
+            void enqueue_disk_io(const std::shared_ptr<FileClient>& /* client */);
+            void execute_disk_io(const std::shared_ptr<FileClient>& /* client */);
+
+            void test_disconnecting_state(const std::shared_ptr<FileClient>& /* client */);
+
+            [[nodiscard]] TransferUploadRawResult handle_transfer_upload_raw(const std::shared_ptr<FileClient>& /* client */, const char * /* buffer */, size_t /* length */, size_t* /* bytes written */);
+            [[nodiscard]] TransferUploadHTTPResult handle_transfer_upload_http(const std::shared_ptr<FileClient>& /* client */, const char * /* buffer */, size_t /* length */);
+
+            void send_http_response(const std::shared_ptr<FileClient>& /* client */, http::HttpResponse& /* response */);
+
+            static void callback_transfer_network_write(int, short, void*);
+            static void callback_transfer_network_read(int, short, void*);
+            static void callback_transfer_network_throttle(int, short, void*);
+            static void callback_transfer_network_accept(int, short, void*);
+
+            static void dispatch_loop_client_worker(void*);
+            static void dispatch_loop_network(void*);
+            static void dispatch_loop_disk_io(void*);
+
+            void report_transfer_statistics(const std::shared_ptr<FileClient>& /* client */);
+            [[nodiscard]] TransferStatistics generate_transfer_statistics_report(const std::shared_ptr<FileClient>& /* client */);
+            void invoke_aborted_callback(const std::shared_ptr<FileClient>& /* client */, const TransferError& /* error */);
+            void invoke_aborted_callback(const std::shared_ptr<Transfer>& /* pending transfer */, const TransferError& /* error */);
+
+            size_t handle_transfer_read(const std::shared_ptr<FileClient>& /* client */, const char* /* buffer */, size_t /* bytes */);
+            size_t handle_transfer_read_raw(const std::shared_ptr<FileClient>& /* client */, const char* /* buffer */, size_t /* bytes */);
+            [[nodiscard]] TransferKeyApplyResult handle_transfer_key_provided(const std::shared_ptr<FileClient>& /* client */, std::string& /* error */);
+    };
+}

file/local_server/LocalFileTransferClientWorker.cpp

@@ -0,0 +1,271 @@
+//
+// Created by WolverinDEV on 04/05/2020.
+//
+
+#include <cassert>
+#include <event2/event.h>
+#include <log/LogUtils.h>
+#include "./LocalFileProvider.h"
+#include "./LocalFileTransfer.h"
+
+using namespace ts::server::file;
+using namespace ts::server::file::transfer;
+
+ClientWorkerStartResult LocalFileTransfer::start_client_worker() {
+    assert(!this->disconnect.active);
+    this->disconnect.active = true;
+
+    this->disconnect.dispatch_thread = std::thread(&LocalFileTransfer::dispatch_loop_client_worker, this);
+    return ClientWorkerStartResult::SUCCESS;
+}
+
+void LocalFileTransfer::shutdown_client_worker() {
+    if(!this->disconnect.active) return;
+    this->disconnect.active = false;
+
+    this->disconnect.notify_cv.notify_all();
+    if(this->disconnect.dispatch_thread.joinable())
+        this->disconnect.dispatch_thread.join();
+
+    {
+        std::unique_lock tlock{this->transfers_mutex};
+        if(!this->transfers_.empty())
+            logWarning(LOG_FT, "Shutting down disconnect worker even thou we still have some active clients. This could cause memory leaks.");
+    }
+}
+
+void LocalFileTransfer::disconnect_client(const std::shared_ptr<FileClient> &client, std::unique_lock<std::shared_mutex>& state_lock, bool flush) {
+    assert(state_lock.owns_lock());
+
+    if(client->state == FileClient::STATE_DISCONNECTED || (client->state == FileClient::STATE_FLUSHING && flush)) {
+        return; /* shall NOT happen */
+    }
+
+#define del_ev_noblock(event) if(event) event_del_noblock(event)
+
+    client->state = flush ? FileClient::STATE_FLUSHING : FileClient::STATE_DISCONNECTED;
+    client->timings.disconnecting = std::chrono::system_clock::now();
+    if(flush) {
+        const auto network_flush_time = client->networking.throttle.expected_writing_time(client->network_buffer.bytes) + std::chrono::seconds{10};
+
+        del_ev_noblock(client->networking.event_read);
+
+        client->networking.disconnect_timeout = std::chrono::system_clock::now() + network_flush_time;
+        debugMessage(LOG_FT, "{} Disconnecting client. Flushing pending bytes (max {} seconds)", client->log_prefix(), std::chrono::floor<std::chrono::seconds>(network_flush_time).count());
+
+        client->add_network_write_event_nolock(false);
+        this->enqueue_disk_io(client);
+    } else {
+        del_ev_noblock(client->networking.event_read);
+        del_ev_noblock(client->networking.event_write);
+        del_ev_noblock(client->networking.event_throttle);
+
+        this->disconnect.notify_cv.notify_one();
+    }
+
+#undef del_ev_noblock
+}
+
+void LocalFileTransfer::test_disconnecting_state(const std::shared_ptr<FileClient> &client) {
+    if(client->state != FileClient::STATE_FLUSHING)
+        return;
+
+    if(!client->buffers_flushed())
+        return;
+
+    debugMessage(LOG_FT, "{} Disk and network buffers are flushed. Closing connection.", client->log_prefix());
+    std::unique_lock s_lock{client->state_mutex};
+    this->disconnect_client(client, s_lock, false);
+}
+
+void LocalFileTransfer::dispatch_loop_client_worker(void *ptr_transfer) {
+    auto provider = reinterpret_cast<LocalFileTransfer*>(ptr_transfer);
+
+    while(provider->disconnect.active) {
+        {
+            std::unique_lock dlock{provider->disconnect.mutex};
+            provider->disconnect.notify_cv.wait_for(dlock, std::chrono::milliseconds {500}); /* report all 500ms the statistics */
+        }
+        /* run the disconnect worker at least once before exiting */
+
+        /* transfer statistics */
+        {
+            std::unique_lock tlock{provider->transfers_mutex};
+            auto transfers = provider->transfers_;
+            tlock.unlock();
+            for(const auto& transfer : transfers) {
+                switch(transfer->state) {
+                    case FileClient::STATE_TRANSFERRING:
+                        break;
+                    case FileClient::STATE_FLUSHING:
+                        if(!transfer->transfer)
+                            continue;
+
+                        if(transfer->transfer->direction != Transfer::DIRECTION_DOWNLOAD)
+                            continue;
+
+                        if(transfer->buffers_flushed())
+                            continue;
+
+                        break; /* we're still transferring (sending data) */
+                    case FileClient::STATE_AWAITING_KEY:
+                    case FileClient::STATE_DISCONNECTED:
+                    default:
+                        continue;
+                }
+
+                provider->report_transfer_statistics(transfer);
+            }
+        }
+
+        {
+            std::deque<std::shared_ptr<Transfer>> timeouted_transfers{};
+
+            {
+                std::unique_lock tlock{provider->transfers_mutex};
+
+                auto now = std::chrono::system_clock::now();
+                std::copy_if(provider->pending_transfers.begin(), provider->pending_transfers.end(), std::back_inserter(timeouted_transfers), [&](const std::shared_ptr<Transfer>& t) {
+                    return t->initialized_timestamp + std::chrono::seconds{10} < now;
+                });
+                provider->pending_transfers.erase(std::remove_if(provider->pending_transfers.begin(), provider->pending_transfers.end(), [&](const auto& t) {
+                    return std::find(timeouted_transfers.begin(), timeouted_transfers.end(), t) != timeouted_transfers.end();
+                }), provider->pending_transfers.end());
+            }
+
+            for(const auto& pt : timeouted_transfers)
+                provider->invoke_aborted_callback(pt, { TransferError::TRANSFER_TIMEOUT, "" });
+
+            if(!timeouted_transfers.empty())
+                logMessage(LOG_FT, "Removed {} pending transfers because no request has been made for them.", timeouted_transfers.size());
+        }
+
+
+        {
+            std::deque<std::shared_ptr<FileClient>> disconnected_clients{};
+            {
+                std::unique_lock tlock{provider->transfers_mutex};
+
+                auto now = std::chrono::system_clock::now();
+                std::copy_if(provider->transfers_.begin(), provider->transfers_.end(), std::back_inserter(disconnected_clients), [&](const std::shared_ptr<FileClient>& t) {
+                    std::shared_lock slock{t->state_mutex};
+                    if(t->state == FileClient::STATE_DISCONNECTED) {
+                        return true;
+                    } else if(t->state == FileClient::STATE_AWAITING_KEY) {
+                        return t->timings.connected + std::chrono::seconds{10} < now;
+                    } else if(t->state == FileClient::STATE_TRANSFERRING) {
+                        assert(t->transfer);
+                        if(t->transfer->direction == Transfer::DIRECTION_UPLOAD) {
+                            return t->timings.last_read + std::chrono::seconds{5} < now;
+                        } else if(t->transfer->direction == Transfer::DIRECTION_DOWNLOAD) {
+                            return t->timings.last_write + std::chrono::seconds{5} < now;
+                        }
+                    } else if(t->state == FileClient::STATE_FLUSHING) {
+                        if(t->networking.disconnect_timeout.time_since_epoch().count() > 0)
+                            return t->networking.disconnect_timeout + std::chrono::seconds{5} < now;
+                        return t->timings.disconnecting + std::chrono::seconds{30} < now;
+                    }
+                    return false;
+                });
+                provider->transfers_.erase(std::remove_if(provider->transfers_.begin(), provider->transfers_.end(), [&](const auto& t) {
+                    return std::find(disconnected_clients.begin(), disconnected_clients.end(), t) != disconnected_clients.end();
+                }), provider->transfers_.end());
+            }
+
+            for(auto& client : disconnected_clients) {
+                switch(client->state) {
+                    case FileClient::STATE_AWAITING_KEY:
+                        logMessage(LOG_FT, "{} Received no key. Dropping client.", client->log_prefix());
+                        break;
+                    case FileClient::STATE_TRANSFERRING:
+                        logMessage(LOG_FT, "{} Networking timeout. Dropping client", client->log_prefix());
+                        provider->invoke_aborted_callback(client, { TransferError::TRANSFER_TIMEOUT, "" });
+                        break;
+                    case FileClient::STATE_FLUSHING:
+                        if(!client->buffers_flushed())
+                            logMessage(LOG_FT, "{} Failed to flush connection. Dropping client", client->log_prefix());
+                        else
+                            ; /* we just awaited a client disconnect */
+                        break;
+                    case FileClient::STATE_DISCONNECTED:
+                    default:
+                        break;
+                }
+                {
+                    std::unique_lock slock{client->state_mutex};
+                    client->state = FileClient::STATE_DISCONNECTED;
+                    /*
+                     * First of all disconnect the client from the network so no actions could be triggered by that way.
+                     * Secondly finalize all network components, so no data is pending anywhere
+                     * Thirdly drop the client's disk worker (if it's an upload the data should be written already, else we don't care anyways)
+                     */
+                    provider->finalize_networking(client, slock);
+                    provider->finalize_client_ssl(client);
+                    provider->finalize_file_io(client, slock);
+                }
+
+                debugMessage(LOG_FT, "{} Destroying transfer.", client->log_prefix());
+            }
+        }
+    }
+}
+
+void LocalFileTransfer::report_transfer_statistics(const std::shared_ptr<FileClient> &client) {
+    auto callback{this->callback_transfer_statistics};
+    if(!callback) return;
+
+    callback(client->transfer, this->generate_transfer_statistics_report(client));
+}
+
+TransferStatistics LocalFileTransfer::generate_transfer_statistics_report(const std::shared_ptr<FileClient> &client) {
+    TransferStatistics stats{};
+
+    stats.network_bytes_send = client->statistics.network_send.total_bytes;
+    stats.network_bytes_received = client->statistics.network_received.total_bytes;
+    stats.file_bytes_transferred = client->statistics.file_transferred.total_bytes;
+
+    stats.delta_network_bytes_received = client->statistics.network_received.take_delta();
+    stats.delta_network_bytes_send = client->statistics.network_received.take_delta();
+
+    stats.delta_file_bytes_transferred = client->statistics.file_transferred.take_delta();
+
+    stats.file_start_offset = client->transfer->file_offset;
+    stats.file_current_offset = client->statistics.file_transferred.total_bytes + client->transfer->file_offset;
+    stats.file_total_size = client->transfer->expected_file_size;
+
+    stats.average_speed = client->statistics.file_transferred.average_bandwidth();
+    stats.current_speed = client->statistics.file_transferred.current_bandwidth();
+
+    return stats;
+}
+
+void LocalFileTransfer::invoke_aborted_callback(const std::shared_ptr<FileClient> &client,
+                                                const ts::server::file::transfer::TransferError &error) {
+    auto callback{this->callback_transfer_aborted};
+    if(!callback || !client->transfer) return;
+
+    callback(client->transfer, this->generate_transfer_statistics_report(client), error);
+}
+
+void LocalFileTransfer::invoke_aborted_callback(const std::shared_ptr<Transfer> &transfer,
+                                                const ts::server::file::transfer::TransferError &error) {
+    auto callback{this->callback_transfer_aborted};
+    if(!callback) return;
+
+    TransferStatistics stats{};
+
+    stats.network_bytes_send = 0;
+    stats.network_bytes_received = 0;
+    stats.file_bytes_transferred = 0;
+
+    stats.delta_network_bytes_received = 0;
+    stats.delta_network_bytes_send = 0;
+
+    stats.delta_file_bytes_transferred = 0;
+
+    stats.file_start_offset = transfer->file_offset;
+    stats.file_current_offset = transfer->file_offset;
+    stats.file_total_size = transfer->expected_file_size;
+
+    callback(transfer, stats, error);
+}

file/local_server/LocalFileTransferDisk.cpp

@@ -0,0 +1,566 @@
+//
+// Created by WolverinDEV on 04/05/2020.
+//
+
+#include <cassert>
+#include <event.h>
+#include <experimental/filesystem>
+#include <log/LogUtils.h>
+#include "./LocalFileProvider.h"
+#include "./LocalFileTransfer.h"
+#include "duration_utils.h"
+
+using namespace ts::server::file;
+using namespace ts::server::file::transfer;
+namespace fs = std::experimental::filesystem;
+
+DiskIOStartResult LocalFileTransfer::start_disk_io() {
+    assert(this->disk_io.state == DiskIOLoopState::STOPPED);
+
+    this->disk_io.state = DiskIOLoopState::RUNNING;
+    this->disk_io.dispatch_thread = std::thread(&LocalFileTransfer::dispatch_loop_disk_io, this);
+    return DiskIOStartResult::SUCCESS;
+}
+
+void LocalFileTransfer::shutdown_disk_io() {
+    if(this->disk_io.state == DiskIOLoopState::STOPPED) return;
+
+    this->disk_io.state = DiskIOLoopState::STOPPING;
+    {
+        std::unique_lock qlock{this->disk_io.queue_lock};
+        this->disk_io.notify_work_awaiting.notify_all();
+        while(this->disk_io.queue_head)
+            this->disk_io.notify_client_processed.wait_for(qlock, std::chrono::seconds{10});
+
+        if(this->disk_io.queue_head) {
+            logWarning(0, "Failed to flush disk IO. Force aborting.");
+            this->disk_io.state = DiskIOLoopState::FORCE_STOPPING;
+            this->disk_io.notify_work_awaiting.notify_all();
+            this->disk_io.notify_client_processed.wait(qlock);
+        }
+    }
+
+    if(this->disk_io.dispatch_thread.joinable())
+        this->disk_io.dispatch_thread.join();
+
+    this->disk_io.state = DiskIOLoopState::STOPPED;
+}
+
+void LocalFileTransfer::dispatch_loop_disk_io(void *provider_ptr) {
+    auto provider = reinterpret_cast<LocalFileTransfer*>(provider_ptr);
+
+    std::shared_ptr<FileClient> client{};
+    while(true) {
+        {
+            std::unique_lock qlock{provider->disk_io.queue_lock};
+            if(client) {
+                client->file.currently_processing = false;
+                provider->disk_io.notify_client_processed.notify_all();
+                client = nullptr;
+            }
+
+            provider->disk_io.notify_work_awaiting.wait(qlock, [&]{ return provider->disk_io.state != DiskIOLoopState::RUNNING || provider->disk_io.queue_head != nullptr; });
+
+            if(provider->disk_io.queue_head) {
+                try {
+                    client = provider->disk_io.queue_head->shared_from_this();
+                } catch (std::bad_weak_ptr& ex) {
+                    logCritical(LOG_FT, "Disk worker encountered a bad weak ptr. This indicated something went horribly wrong! Please submit this on https://forum.teaspeak.de !!!");
+                    client.reset();
+                    continue;
+                }
+
+                provider->disk_io.queue_head = provider->disk_io.queue_head->file.next_client;
+                if(!provider->disk_io.queue_head)
+                    provider->disk_io.queue_tail = &provider->disk_io.queue_head;
+            }
+
+            if(provider->disk_io.state != DiskIOLoopState::RUNNING) {
+                if(provider->disk_io.state == DiskIOLoopState::STOPPING) {
+                    if(!client)
+                        break;
+                    /* break only if all clients have been flushed */
+                } else {
+                    /* force stopping without any flushing */
+                    auto fclient = &*client;
+                    while(fclient)
+                        fclient = std::exchange(fclient->file.next_client, nullptr);
+
+                    provider->disk_io.queue_head = nullptr;
+                    provider->disk_io.queue_tail = &provider->disk_io.queue_head;
+                    break;
+                }
+            }
+
+            if(!client)
+                continue;
+
+            client->file.currently_processing = true;
+            client->file.next_client = nullptr;
+        }
+
+        provider->execute_disk_io(client);
+    }
+    provider->disk_io.notify_client_processed.notify_all();
+}
+
+bool FileClient::enqueue_disk_buffer_bytes(const void *snd_buffer, size_t size) {
+    auto tbuffer = allocate_buffer(size);
+    tbuffer->length = size;
+    tbuffer->offset = 0;
+    memcpy(tbuffer->data, snd_buffer, size);
+
+    size_t buffer_size;
+    {
+        std::lock_guard block{this->disk_buffer.mutex};
+        if(this->disk_buffer.write_disconnected)
+            goto write_disconnected;
+
+        *this->disk_buffer.buffer_tail = tbuffer;
+        this->disk_buffer.buffer_tail = &tbuffer->next;
+        buffer_size = (this->disk_buffer.bytes += size);
+    }
+
+    return buffer_size > TRANSFER_MAX_CACHED_BYTES;
+
+    write_disconnected:
+    free_buffer(tbuffer);
+    return false;
+}
+
+void FileClient::flush_disk_buffer() {
+    Buffer* current_head;
+    {
+        std::lock_guard block{this->disk_buffer.mutex};
+
+        this->disk_buffer.write_disconnected = true;
+        this->disk_buffer.bytes = 0;
+        current_head = std::exchange(this->disk_buffer.buffer_head, nullptr);
+        this->disk_buffer.buffer_tail = &this->disk_buffer.buffer_head;
+    }
+
+    while(current_head) {
+        auto next = current_head->next;
+        free_buffer(current_head);
+        current_head = next;
+    }
+}
+
+bool FileClient::buffers_flushed() {
+    {
+        std::lock_guard db_lock{this->disk_buffer.mutex};
+        if(this->disk_buffer.bytes > 0)
+            return false;
+    }
+
+    {
+        std::lock_guard nb_lock{this->network_buffer.mutex};
+        if(this->network_buffer.bytes > 0)
+            return false;
+    }
+
+    return true;
+}
+
+FileInitializeResult LocalFileTransfer::initialize_file_io(const std::shared_ptr<FileClient> &transfer) {
+    FileInitializeResult result{FileInitializeResult::SUCCESS};
+    assert(transfer->transfer);
+
+    std::shared_lock slock{transfer->state_mutex};
+    auto& file_data = transfer->file;
+    assert(!file_data.file_descriptor);
+    assert(!file_data.next_client);
+
+    auto absolute_path = transfer->transfer->absolute_file_path;
+    {
+        unsigned int open_flags{0};
+        if(transfer->transfer->direction == Transfer::DIRECTION_DOWNLOAD) {
+            open_flags = O_RDONLY;
+
+            std::error_code fs_error{};
+            if(absolute_path.empty() || !fs::exists(absolute_path, fs_error)) {
+                result = FileInitializeResult::FILE_DOES_NOT_EXISTS;
+                goto error_exit;
+            } else if(fs_error) {
+                logWarning(LOG_FT, "{} Failed to check for file existence of {}: {}/{}", transfer->log_prefix(), absolute_path, fs_error.value(), fs_error.message());
+                result = FileInitializeResult::FILE_SYSTEM_ERROR;
+                goto error_exit;
+            }
+        } else if(transfer->transfer->direction == Transfer::DIRECTION_UPLOAD) {
+            std::error_code fs_error{};
+            auto parent_path = fs::u8path(absolute_path).parent_path();
+            if(!fs::exists(parent_path)) {
+                if(!fs::create_directories(parent_path, fs_error) || fs_error) {
+                    logError(LOG_FT, "{} Failed to create required directories for file upload for {}: {}/{}", transfer->log_prefix(), absolute_path, fs_error.value(), fs_error.message());
+                    result = FileInitializeResult::COUNT_NOT_CREATE_DIRECTORIES;
+                    goto error_exit;
+                }
+            } else if(fs_error) {
+                logWarning(LOG_FT, "{} Failed to check for directory existence of {}: {}/{}. Assuming it exists", transfer->log_prefix(), parent_path.string(), fs_error.value(), fs_error.message());
+            }
+
+            open_flags = (unsigned) O_WRONLY | (unsigned) O_CREAT;
+            if(transfer->transfer->override_exiting)
+                open_flags |= (unsigned) O_TRUNC;
+        } else {
+            return FileInitializeResult::INVALID_TRANSFER_DIRECTION;
+        }
+
+        file_data.file_descriptor = open(absolute_path.c_str(), (int) open_flags, 0644);
+        if(file_data.file_descriptor <= 0) {
+            const auto errno_ = errno;
+            switch (errno_) {
+                case EACCES:
+                    result = FileInitializeResult::FILE_IS_NOT_ACCESSIBLE;
+                    break;
+                case EDQUOT:
+                    logWarning(LOG_FT, "{} Disk inode limit has been reached. Failed to start file transfer for file {}", transfer->log_prefix(), absolute_path);
+                    result = FileInitializeResult::FILE_SYSTEM_ERROR;
+                    break;
+                case EISDIR:
+                    result = FileInitializeResult::FILE_IS_A_DIRECTORY;
+                    break;
+                case EMFILE:
+                    result = FileInitializeResult::PROCESS_FILE_LIMIT_REACHED;
+                    break;
+                case ENFILE:
+                    result = FileInitializeResult::SYSTEM_FILE_LIMIT_REACHED;
+                    break;
+                case ETXTBSY:
+                    result = FileInitializeResult::FILE_IS_BUSY;
+                    break;
+                case EROFS:
+                    result = FileInitializeResult::DISK_IS_READ_ONLY;
+                    break;
+                default:
+                    logWarning(LOG_FT, "{} Failed to start file {} for file {}: {}/{}", transfer->log_prefix(),
+                               transfer->transfer->direction == Transfer::DIRECTION_DOWNLOAD ? "download" : "upload", absolute_path, errno_, strerror(errno_));
+                    result = FileInitializeResult::FILE_SYSTEM_ERROR;
+                    break;
+            }
+            goto error_exit;
+        }
+    }
+
+    this->file_system_->lock_file(absolute_path);
+    transfer->file.file_locked = true;
+
+    if(transfer->transfer->direction == Transfer::DIRECTION_UPLOAD) {
+        if(ftruncate(file_data.file_descriptor, transfer->transfer->expected_file_size) != 0) {
+            const auto errno_ = errno;
+            switch (errno_) {
+                case EACCES:
+                    logWarning(LOG_FT, "{} File {} got inaccessible on truncating, but not on opening.", transfer->log_prefix(), absolute_path);
+                    result = FileInitializeResult::FILE_IS_NOT_ACCESSIBLE;
+                    goto error_exit;
+
+                case EFBIG:
+                    result = FileInitializeResult::FILE_TOO_LARGE;
+                    goto error_exit;
+
+                case EIO:
+                    logWarning(LOG_FT, "{} A disk IO error occurred while resizing file {}.", transfer->log_prefix(), absolute_path);
+                    result = FileInitializeResult::FILE_IS_NOT_ACCESSIBLE;
+                    goto error_exit;
+
+                case EROFS:
+                    logWarning(LOG_FT, "{} Failed to resize file {} because disk is in read only mode.", transfer->log_prefix(), absolute_path);
+                    result = FileInitializeResult::FILE_IS_NOT_ACCESSIBLE;
+                    goto error_exit;
+                default:
+                    debugMessage(LOG_FT, "{} Failed to truncate file {}: {}/{}. Trying to upload file anyways.", transfer->log_prefix(), absolute_path, errno_, strerror(errno_));
+            }
+        }
+    } else if(transfer->transfer->direction == Transfer::DIRECTION_DOWNLOAD) {
+        auto file_size = lseek(file_data.file_descriptor, 0, SEEK_END);
+        if(file_size != transfer->transfer->expected_file_size) {
+            logWarning(LOG_FT, "{} Expected target file to be of size {}, but file is actually of size {}", transfer->log_prefix(), transfer->transfer->expected_file_size, file_size);
+            result = FileInitializeResult::FILE_SIZE_MISMATCH;
+            goto error_exit;
+        }
+        if(transfer->file.query_media_bytes && file_size > 0) {
+            auto new_pos = lseek(file_data.file_descriptor, 0, SEEK_SET);
+            if(new_pos < 0) {
+                logWarning(LOG_FT, "{} Failed to seek to file start: {}/{}", transfer->log_prefix(), errno, strerror(errno));
+                result = FileInitializeResult::FILE_SEEK_FAILED;
+                goto error_exit;
+            }
+
+            auto read = ::read(file_data.file_descriptor, transfer->file.media_bytes, TRANSFER_MEDIA_BYTES_LENGTH);
+            if(read <= 0) {
+                logWarning(LOG_FT, "{} Failed to read file media bytes: {}/{}", transfer->log_prefix(), errno, strerror(errno));
+                result = FileInitializeResult::FAILED_TO_READ_MEDIA_BYTES;
+                goto error_exit;
+            }
+            transfer->file.media_bytes_length = read;
+        }
+    }
+
+    {
+        auto new_pos = lseek(file_data.file_descriptor, transfer->transfer->file_offset, SEEK_SET);
+        if(new_pos < 0) {
+            logWarning(LOG_FT, "{} Failed to seek to target file offset ({}): {}/{}", transfer->log_prefix(), transfer->transfer->file_offset, errno, strerror(errno));
+            result = FileInitializeResult::FILE_SEEK_FAILED;
+            goto error_exit;
+        } else if(new_pos != transfer->transfer->file_offset) {
+            logWarning(LOG_FT, "{} File rw offset mismatch after seek. Expected {} but received {}", transfer->log_prefix(), transfer->transfer->file_offset, new_pos);
+            result = FileInitializeResult::FILE_SEEK_FAILED;
+            goto error_exit;
+        }
+        logTrace(LOG_FT, "{} Seek to file offset {}. New actual offset is {}", transfer->log_prefix(), transfer->transfer->file_offset, new_pos);
+    }
+
+    return FileInitializeResult::SUCCESS;
+    error_exit:
+    if(std::exchange(transfer->file.file_locked, false))
+        this->file_system_->unlock_file(absolute_path);
+
+    if(file_data.file_descriptor > 0)
+        ::close(file_data.file_descriptor);
+    file_data.file_descriptor = 0;
+
+    return result;
+}
+
+void LocalFileTransfer::finalize_file_io(const std::shared_ptr<FileClient> &transfer,
+                                         std::unique_lock<std::shared_mutex> &state_lock) {
+    assert(state_lock.owns_lock());
+    if(!transfer->transfer) {
+        return;
+    }
+
+    auto absolute_path = transfer->transfer->absolute_file_path;
+
+    auto& file_data = transfer->file;
+
+    state_lock.unlock();
+    {
+        std::unique_lock dlock{this->disk_io.queue_lock};
+        while(true) {
+            if(file_data.currently_processing) {
+                this->disk_io.notify_client_processed.wait(dlock);
+                continue;
+            }
+
+            if(this->disk_io.queue_head == &*transfer) {
+                this->disk_io.queue_head = file_data.next_client;
+                if (!this->disk_io.queue_head) {
+                    this->disk_io.queue_tail = &this->disk_io.queue_head;
+                }
+            } else if(file_data.next_client || this->disk_io.queue_tail == &file_data.next_client) {
+                FileClient* head{this->disk_io.queue_head};
+                while(head->file.next_client != &*transfer) {
+                    assert(head->file.next_client);
+                    head = head->file.next_client;
+                }
+
+                head->file.next_client = file_data.next_client;
+                if(!file_data.next_client)
+                    this->disk_io.queue_tail = &head->file.next_client;
+            }
+            file_data.next_client = nullptr;
+            break;
+        }
+    }
+    state_lock.lock();
+
+    if(std::exchange(file_data.file_locked, false)) {
+        this->file_system_->unlock_file(absolute_path);
+    }
+
+    if(file_data.file_descriptor > 0)
+        ::close(file_data.file_descriptor);
+    file_data.file_descriptor = 0;
+}
+
+void LocalFileTransfer::enqueue_disk_io(const std::shared_ptr<FileClient> &client) {
+    if(!client->file.file_descriptor) {
+        return;
+    }
+
+    if(!client->transfer) {
+        return;
+    }
+
+    if(client->transfer->direction == Transfer::DIRECTION_DOWNLOAD) {
+        if(client->state != FileClient::STATE_TRANSFERRING) {
+            return;
+        }
+
+        if(client->disk_buffer.bytes > TRANSFER_MAX_CACHED_BYTES) {
+            return;
+        }
+    } else if(client->transfer->direction == Transfer::DIRECTION_UPLOAD) {
+        /* we don't do this check because this might be a flush instruction, where the buffer is actually zero bytes filled */
+        /*
+        if(client->buffer.bytes == 0)
+            return;
+        */
+    }
+
+    std::lock_guard dlock{this->disk_io.queue_lock};
+    if(client->file.next_client || this->disk_io.queue_tail == &client->file.next_client) {
+        return;
+    }
+
+    *this->disk_io.queue_tail = &*client;
+    this->disk_io.queue_tail = &client->file.next_client;
+
+    this->disk_io.notify_work_awaiting.notify_all();
+}
+
+void LocalFileTransfer::execute_disk_io(const std::shared_ptr<FileClient> &client) {
+    if(!client->transfer) return;
+
+    if(client->transfer->direction == Transfer::DIRECTION_UPLOAD) {
+        Buffer* buffer;
+        size_t buffer_left_size;
+
+        while(true) {
+            {
+                std::lock_guard block{client->disk_buffer.mutex};
+                buffer = client->disk_buffer.buffer_head;
+                buffer_left_size = client->disk_buffer.bytes;
+            }
+            if(!buffer) {
+                assert(buffer_left_size == 0);
+                break;
+            }
+
+            assert(buffer->offset < buffer->length);
+            auto written = ::write(client->file.file_descriptor, buffer->data + buffer->offset, buffer->length - buffer->offset);
+            if(written <= 0) {
+                if(written == 0) {
+                    /* EOF, how the hell is this event possible?! */
+                    auto offset_written = client->statistics.disk_bytes_write.total_bytes + client->transfer->file_offset;
+                    auto aoffset = lseek(client->file.file_descriptor, 0, SEEK_CUR);
+                    logError(LOG_FT, "{} Received unexpected file write EOF. EOF received at {} but expected {}. Actual file offset: {}. Closing transfer.",
+                            client->log_prefix(), offset_written, client->transfer->expected_file_size, aoffset);
+
+
+                    this->invoke_aborted_callback(client, { TransferError::UNEXPECTED_DISK_EOF, strerror(errno) });
+
+                    client->flush_disk_buffer();
+                    {
+                        std::unique_lock slock{client->state_mutex};
+                        client->handle->disconnect_client(client, slock, true);
+                    }
+                } else {
+                    if(errno == EAGAIN) {
+                        //TODO: Timeout?
+                        this->enqueue_disk_io(client);
+                        break;
+                    }
+
+                    auto offset_written = client->statistics.disk_bytes_write.total_bytes + client->transfer->file_offset;
+                    auto aoffset = lseek(client->file.file_descriptor, 0, SEEK_CUR);
+                    logError(LOG_FT, "{} Received write to disk IO error. Write pointer is at {} of {}. Actual file offset: {}. Closing transfer.",
+                            client->log_prefix(), offset_written, client->transfer->expected_file_size, aoffset);
+
+                    this->invoke_aborted_callback(client, { TransferError::DISK_IO_ERROR, strerror(errno) });
+
+                    client->flush_disk_buffer();
+                    {
+                        std::unique_lock slock{client->state_mutex};
+                        client->handle->disconnect_client(client, slock, true);
+                    }
+                }
+                return;
+            } else {
+                buffer->offset += written;
+                assert(buffer->offset <= buffer->length);
+                if(buffer->length == buffer->offset) {
+                    {
+                        std::lock_guard block{client->disk_buffer.mutex};
+                        client->disk_buffer.buffer_head = buffer->next;
+                        if(!buffer->next)
+                            client->disk_buffer.buffer_tail = &client->disk_buffer.buffer_head;
+
+                        assert(client->disk_buffer.bytes >= written);
+                        client->disk_buffer.bytes -= written;
+                        buffer_left_size = client->disk_buffer.bytes;
+                        (void) buffer_left_size; /* trick my IDE here a bit */
+                    }
+
+                    free_buffer(buffer);
+                } else {
+                    std::lock_guard block{client->disk_buffer.mutex};
+                    assert(client->disk_buffer.bytes >= written);
+                    client->disk_buffer.bytes -= written;
+                    buffer_left_size = client->disk_buffer.bytes;
+                    (void) buffer_left_size; /* trick my IDE here a bit */
+                }
+
+                client->statistics.disk_bytes_write.increase_bytes(written);
+            }
+        }
+
+        if(buffer_left_size > 0) {
+            this->enqueue_disk_io(client);
+        } else if(client->state == FileClient::STATE_FLUSHING) {
+            this->test_disconnecting_state(client);
+        }
+
+        if(client->state == FileClient::STATE_TRANSFERRING && buffer_left_size < TRANSFER_MAX_CACHED_BYTES / 2) {
+            if(client->disk_buffer.buffering_stopped)
+                logMessage(LOG_FT, "{} Starting network read, buffer is capable for reading again.", client->log_prefix());
+            client->add_network_read_event(false);
+        }
+    } else if(client->transfer->direction == Transfer::DIRECTION_DOWNLOAD) {
+        if(client->state == FileClient::STATE_FLUSHING) {
+            client->flush_disk_buffer(); /* just in case, file download usually don't write to the disk */
+            return;
+        }
+
+        while(true) {
+            constexpr auto buffer_capacity{4096};
+            char buffer[buffer_capacity];
+
+            auto read = ::read(client->file.file_descriptor, buffer, buffer_capacity);
+            if(read <= 0) {
+                if(read == 0) {
+                    /* EOF */
+                    auto offset_send = client->statistics.disk_bytes_read.total_bytes + client->transfer->file_offset;
+                    if(client->transfer->expected_file_size == offset_send) {
+                        debugMessage(LOG_FT, "{} Finished file reading. Flushing and disconnecting transfer. Reading took {} seconds.",
+                                client->log_prefix(), duration_to_string(std::chrono::system_clock::now() - client->timings.key_received));
+                    } else {
+                        auto aoffset = lseek(client->file.file_descriptor, 0, SEEK_CUR);
+                        logError(LOG_FT, "{} Received unexpected read EOF. EOF received at {} but expected {}. Actual file offset: {}. Disconnecting client.",
+                                client->log_prefix(), offset_send, client->transfer->expected_file_size, aoffset);
+
+                        this->invoke_aborted_callback(client, { TransferError::UNEXPECTED_DISK_EOF, "" });
+                    }
+                } else {
+                    if(errno == EAGAIN) {
+                        this->enqueue_disk_io(client);
+                        return;
+                    }
+
+                    logWarning(LOG_FT, "{} Failed to read from file {} ({}/{}). Aborting transfer.", client->log_prefix(), client->transfer->absolute_file_path, errno, strerror(errno));
+
+                    this->invoke_aborted_callback(client, { TransferError::DISK_IO_ERROR, strerror(errno) });
+                }
+
+                std::unique_lock slock{client->state_mutex};
+                client->handle->disconnect_client(client, slock, true);
+                return;
+            } else {
+                auto buffer_full = client->send_file_bytes(buffer, read);
+                client->statistics.disk_bytes_read.increase_bytes(read);
+                client->statistics.file_transferred.increase_bytes(read);
+
+                std::shared_lock slock{client->state_mutex};
+                if(buffer_full) {
+                    //logTrace(LOG_FT, "{} Stopping buffering from disk. Buffer full ({}bytes)", client->log_prefix(), client->buffer.bytes);
+                    break;
+                }
+
+                /* we've stuff to write again, yeahr */
+                client->add_network_write_event(false);
+            }
+        }
+    } else {
+        logError(LOG_FT, "{} Disk IO scheduled, but transfer direction is unknown.", client->log_prefix());
+    }
+}

file/local_server/NetTools.cpp

@@ -0,0 +1,9 @@
+//
+// Created by WolverinDEV on 12/05/2020.
+//
+
+#include "./NetTools.h"
+
+using namespace ts::server::file::networking;
+
+NetworkThrottle NetworkThrottle::kNoThrottle{-1};

file/local_server/NetTools.h

@@ -0,0 +1,166 @@
+#pragma once
+
+#include <cstddef>
+#include <chrono>
+#include <mutex>
+#include <cassert>
+#include <array>
+
+#include <misc/spin_mutex.h>
+#include <numeric>
+
+namespace ts::server::file::networking {
+    struct NetworkThrottle {
+        constexpr static auto kThrottleTimespanMs{250};
+
+        typedef uint8_t span_t;
+
+        static NetworkThrottle kNoThrottle;
+
+        ssize_t max_bytes{0};
+
+        span_t current_index{0};
+        size_t bytes_send{0};
+
+        mutable spin_mutex mutex{};
+
+        inline bool increase_bytes(size_t bytes) {
+            auto current_ms = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
+            auto current_span = (span_t) (current_ms / kThrottleTimespanMs);
+
+            std::lock_guard slock{this->mutex};
+            if(this->current_index != current_span) {
+                this->current_index = current_span;
+                this->bytes_send = bytes;
+            } else {
+                this->bytes_send += bytes;
+            }
+            return this->max_bytes > 0 && this->bytes_send >= this->max_bytes;
+        }
+
+        inline void set_max_bandwidth(ssize_t bytes_per_second) {
+            std::lock_guard slock{this->mutex};
+            if(bytes_per_second <= 0)
+                this->max_bytes = -1;
+            else
+                this->max_bytes = bytes_per_second * kThrottleTimespanMs / 1000;
+        }
+
+        [[nodiscard]] inline bool should_throttle(timeval& next_timestamp) {
+            if(this->max_bytes <= 0) return false;
+
+            auto current_ms = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
+            auto current_span = (span_t) (current_ms / kThrottleTimespanMs);
+
+            std::lock_guard slock{this->mutex};
+            if(this->max_bytes <= 0) return false; /* we've to test here again, else out arithmetic will fail */
+            if(this->current_index != current_span) return false;
+            if(this->bytes_send < this->max_bytes) return false;
+
+            next_timestamp.tv_usec = (kThrottleTimespanMs - current_ms % kThrottleTimespanMs) * 1000;
+            next_timestamp.tv_sec =  next_timestamp.tv_usec / 1000000;
+            next_timestamp.tv_usec -= next_timestamp.tv_sec * 1000000;
+            return true;
+        }
+
+        [[nodiscard]] inline size_t bytes_left() const {
+            if(this->max_bytes <= 0) return (size_t) -1;
+
+            auto current_ms = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
+            auto current_span = (span_t) (current_ms / kThrottleTimespanMs);
+
+            std::lock_guard slock{this->mutex};
+            if(this->max_bytes <= 0) return false; /* we've to test here again, else out arithmetic will fail */
+            if(this->current_index != current_span) return this->max_bytes;
+            if(this->bytes_send < this->max_bytes) return this->max_bytes - this->bytes_send;
+            return 0;
+        }
+
+        [[nodiscard]] inline std::chrono::milliseconds expected_writing_time(size_t bytes) const {
+            std::lock_guard slock{this->mutex};
+
+            if(this->max_bytes <= 0) return std::chrono::milliseconds{0};
+            return std::chrono::seconds{bytes / (this->max_bytes * (1000 / kThrottleTimespanMs))};
+        }
+    };
+
+    struct DualNetworkThrottle {
+        NetworkThrottle *left, *right;
+
+        explicit DualNetworkThrottle(NetworkThrottle* left, NetworkThrottle* right) : left{left}, right{right} {
+            assert(left);
+            assert(right);
+        }
+
+        [[nodiscard]] inline size_t bytes_left() const {
+            return std::min(this->left->bytes_left(), this->right->bytes_left());
+        }
+
+        [[nodiscard]] inline std::chrono::milliseconds expected_writing_time(size_t bytes) const {
+            return std::max(this->left->expected_writing_time(bytes), this->right->expected_writing_time(bytes));
+        }
+
+        [[nodiscard]] inline bool should_throttle(timeval& next_timestamp) const {
+            bool throttle = false;
+            timeval right_timestamp{};
+            throttle |= this->left->should_throttle(next_timestamp);
+            throttle |= this->right->should_throttle(right_timestamp);
+            if(!throttle) return false;
+
+            if(right_timestamp.tv_sec > next_timestamp.tv_sec || (right_timestamp.tv_sec == next_timestamp.tv_sec && right_timestamp.tv_usec > next_timestamp.tv_usec))
+                next_timestamp = right_timestamp;
+            return true;
+        }
+
+        inline bool increase_bytes(size_t bytes) {
+            bool result = false;
+            result |= this->left->increase_bytes(bytes);
+            result |= this->right->increase_bytes(bytes);
+            return result;
+        }
+    };
+
+    struct TransferStatistics {
+        constexpr static auto kMeasureTimespanMs{1000};
+        constexpr static auto kAverageTimeCount{60};
+        typedef uint8_t span_t;
+
+        size_t total_bytes{0};
+        size_t delta_bytes{0}; /* used for statistics propagation */
+
+        span_t span_index{0};
+        size_t span_bytes{0};
+        std::array<size_t, kAverageTimeCount> history{};
+
+        spin_mutex mutex{};
+
+        inline void increase_bytes(size_t bytes) {
+            auto current_ms = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
+            auto current_span = (span_t) (current_ms / kMeasureTimespanMs);
+
+            std::lock_guard slock{this->mutex};
+            this->total_bytes += bytes;
+
+            if(this->span_index != current_span)
+                this->history[this->span_index % kAverageTimeCount] = std::exchange(this->span_bytes, 0);
+            this->span_index = current_span;
+            this->span_bytes += bytes;
+        }
+
+        [[nodiscard]] inline size_t take_delta() {
+            std::lock_guard slock{this->mutex};
+            assert(this->delta_bytes <= this->total_bytes);
+            auto delta = this->total_bytes - this->delta_bytes;
+            this->delta_bytes = this->total_bytes;
+            return delta;
+        }
+
+        [[nodiscard]] inline double current_bandwidth() const {
+            return (this->history[(this->span_index - 1) % kAverageTimeCount] * (double) 1000) / (double) kMeasureTimespanMs;
+        }
+
+        [[nodiscard]] inline double average_bandwidth() const {
+            return (std::accumulate(this->history.begin(), this->history.end(), 0UL) * (double) 1000) / (double) (kMeasureTimespanMs * kAverageTimeCount);
+        }
+    };
+}

file/local_server/clnpath.cpp

@@ -0,0 +1,283 @@
+//
+// Created by WolverinDEV on 29/04/2020.
+//
+
+#include "clnpath.h"
+#include <cstring>
+
+#define MAX_PATH_ELEMENTS   128  /* Number of levels of directory */
+void ts_clnpath(char *path)
+{
+    char           *src;
+    char           *dst;
+    char            c;
+    int             slash = 0;
+
+    /* Convert multiple adjacent slashes to single slash */
+    src = dst = path;
+    while ((c = *dst++ = *src++) != '\0')
+    {
+        if (c == '/')
+        {
+            slash = 1;
+            while (*src == '/')
+                src++;
+        }
+    }
+
+    if (slash == 0)
+        return;
+
+    /* Remove "./" from "./xxx" but leave "./" alone. */
+    /* Remove "/." from "xxx/." but reduce "/." to "/". */
+    /* Reduce "xxx/./yyy" to "xxx/yyy" */
+    src = dst = (*path == '/') ? path + 1 : path;
+    while (src[0] == '.' && src[1] == '/' && src[2] != '\0')
+        src += 2;
+    while ((c = *dst++ = *src++) != '\0')
+    {
+        if (c == '/' && src[0] == '.' && (src[1] == '\0' || src[1] == '/'))
+        {
+            src++;
+            dst--;
+        }
+    }
+    if (path[0] == '/' && path[1] == '.' &&
+        (path[2] == '\0' || (path[2] == '/' && path[3] == '\0')))
+        path[1] = '\0';
+
+    /* Remove trailing slash, if any.  There is at most one! */
+    /* dst is pointing one beyond terminating null */
+    if ((dst -= 2) > path && *dst == '/')
+        *dst++ = '\0';
+}
+
+bool ts_strequal(const char* a, const char* b) {
+    return strcmp(a, b) == 0;
+}
+
+int ts_tokenise(char* ostring, const char* del, char** result, int max_tokens) {
+    int num_tokens{0};
+
+    char* token, *string, *tofree;
+    tofree = string = strdup(ostring);
+    while ((token = strsep(&string, del)) != nullptr) {
+        result[num_tokens++] = strdup(token);
+        if(num_tokens > max_tokens)
+            break;
+    }
+
+    free(tofree);
+    return num_tokens;
+}
+
+/*
+** clnpath2() is not part of the basic clnpath() function because it can
+** change the meaning of a path name if there are symbolic links on the
+** system.  For example, suppose /usr/tmp is a symbolic link to /var/tmp.
+** If the user supplies /usr/tmp/../abcdef as the directory name, clnpath
+** would transform that to /usr/abcdef, not to /var/abcdef which is what
+** the kernel would interpret it as.
+*/
+void ts_clnpath2(char *path)
+{
+    char *token[MAX_PATH_ELEMENTS], *otoken[MAX_PATH_ELEMENTS];
+    int   ntok, ontok;
+
+    ts_clnpath(path);
+
+    /* Reduce "<name>/.." to "/" */
+    ntok = ontok = ts_tokenise(path, "/", otoken, MAX_PATH_ELEMENTS);
+    memcpy(token, otoken, sizeof(char*) * ntok);
+
+    if (ntok > 1) {
+        for (int i = 0; i < ntok - 1; i++)
+        {
+            if (!ts_strequal(token[i], "..") && ts_strequal(token[i + 1], ".."))
+            {
+                if (*token[i] == '\0')
+                    continue;
+                while (i < ntok - 1)
+                {
+                    token[i] = token[i + 2];
+                    i++;
+                }
+                ntok -= 2;
+                i = -1;     /* Restart enclosing for loop */
+            }
+        }
+    }
+
+    /* Reassemble string */
+    char *dst = path;
+    if (ntok == 0)
+    {
+        *dst++ = '.';
+        *dst = '\0';
+    }
+    else
+    {
+        if (token[0][0] == '\0')
+        {
+            int   i;
+            for (i = 1; i < ntok && ts_strequal(token[i], ".."); i++)
+                ;
+            if (i > 1)
+            {
+                int j;
+                for (j = 1; i < ntok; i++)
+                    token[j++] = token[i];
+                ntok = j;
+            }
+        }
+        if (ntok == 1 && token[0][0] == '\0')
+        {
+            *dst++ = '/';
+            *dst = '\0';
+        }
+        else
+        {
+            for (int i = 0; i < ntok; i++)
+            {
+                char *src = token[i];
+                while ((*dst++ = *src++) != '\0')
+                    ;
+                *(dst - 1) = '/';
+            }
+            *(dst - 1) = '\0';
+        }
+    }
+
+    for(int i{0}; i < ontok; i++)
+        ::free(otoken[i]);
+}
+
+std::string clnpath(const std::string_view& data) {
+    std::string result{data};
+    ts_clnpath2(result.data());
+    auto index = result.find((char) 0);
+    if(index != std::string::npos)
+        result.resize(index);
+    return result;
+}
+
+#ifdef CLN_EXEC
+typedef struct p1_test_case
+{
+    const char *input;
+    const char *output;
+} p1_test_case;
+
+/* This stress tests the cleaning, concentrating on the boundaries. */
+static const p1_test_case p1_tests[] =
+        {
+                { "/",                                  "/",            },
+                { "//",                                 "/",            },
+                { "///",                                "/",            },
+                { "/.",                                 "/",            },
+                { "/./",                                "/",            },
+                { "/./.",                               "/",            },
+                { "/././.profile",                      "/.profile",    },
+                { "./",                                 ".",            },
+                { "./.",                                ".",            },
+                { "././",                               ".",            },
+                { "./././.profile",                     ".profile",     },
+                { "abc/.",                              "abc",          },
+                { "abc/./def",                          "abc/def",      },
+                { "./abc",                              "abc",          },
+
+                { "//abcd///./abcd////",                "/abcd/abcd",                   },
+                { "//abcd///././../defg///ddd//.",      "/abcd/../defg/ddd",            },
+                { "/abcd/./../././defg/./././ddd",      "/abcd/../defg/ddd",            },
+                { "//abcd//././../defg///ddd//.///",    "/abcd/../defg/ddd",            },
+
+                /* Most of these are minimal interest in phase 1 */
+                { "/usr/tmp/clnpath.c",                 "/usr/tmp/clnpath.c",           },
+                { "/usr/tmp/",                          "/usr/tmp",                     },
+                { "/bin/..",                            "/bin/..",                      },
+                { "bin/..",                             "bin/..",                       },
+                { "/bin/.",                             "/bin",                         },
+                { "sub/directory",                      "sub/directory",                },
+                { "sub/directory/file",                 "sub/directory/file",           },
+                { "/part1/part2/../.././../",           "/part1/part2/../../..",        },
+                { "/.././../usr//.//bin/./cc",          "/../../usr/bin/cc",            },
+        };
+
+static void p1_tester(const void *data)
+{
+    const p1_test_case *test = (const p1_test_case *)data;
+    char  buffer[256];
+
+    strcpy(buffer, test->input);
+    ts_clnpath(buffer);
+    if (strcmp(buffer, test->output) == 0)
+        printf("<<%s>> cleans to <<%s>>\n", test->input, buffer);
+    else
+    {
+        fprintf(stderr, "<<%s>> - unexpected output from clnpath()\n", test->input);
+        fprintf(stderr, "Wanted <<%s>>\n", test->output);
+        fprintf(stderr, "Actual <<%s>>\n", buffer);
+    }
+}
+
+typedef struct p2_test_case
+{
+    const char *input;
+    const char *output;
+} p2_test_case;
+
+static const p2_test_case p2_tests[] =
+{
+    { "/abcd/../defg/ddd",              "/defg/ddd"         },
+    { "/bin/..",                        "/"                 },
+    { "bin/..",                         "."                 },
+    { "/usr/bin/..",                    "/usr"              },
+    { "/usr/bin/../..",                 "/"                 },
+    { "usr/bin/../..",                  "."                 },
+    { "../part/of/../the/way",          "../part/the/way"   },
+    { "/../part/of/../the/way",         "/part/the/way"     },
+    { "part1/part2/../../part3",        "part3"             },
+    { "part1/part2/../../../part3",     "../part3"          },
+    { "/part1/part2/../../../part3",    "/part3"            },
+    { "/part1/part2/../../../",         "/"                 },
+    { "/../../usr/bin/cc",              "/usr/bin/cc"       },
+    { "../../usr/bin/cc",               "../../usr/bin/cc"  },
+    { "part1/./part2/../../part3",      "part3"             },
+    { "./part1/part2/../../../part3",   "../part3"          },
+    { "/part1/part2/.././../../part3",  "/part3"            },
+    { "/part1/part2/../.././../",       "/"                 },
+    { "/.././..//./usr///bin/cc/",      "/usr/bin/cc"       },
+    {nullptr, nullptr}
+};
+
+static void p2_tester(const void *data)
+{
+    auto test = (const p2_test_case *)data;
+    char  buffer[256];
+
+    strcpy(buffer, test->input);
+    ts_clnpath2(buffer);
+    if (strcmp(buffer, test->output) == 0)
+        printf("<<%s>> cleans to <<%s>>\n", test->input, buffer);
+    else
+    {
+        fprintf(stderr, "<<%s>> - unexpected output from clnpath2()\n", test->input);
+        fprintf(stderr, "Wanted <<%s>>\n", test->output);
+        fprintf(stderr, "Actual <<%s>>\n", buffer);
+    }
+}
+
+int main() {
+    for(const auto& test : p1_tests) {
+        if(!test.input) break;
+        p1_tester(&test);
+    }
+
+    printf("------------------------------\n");
+    for(const auto& test : p2_tests) {
+        if(!test.input) break;
+        p2_tester(&test);
+    }
+}
+
+#endif

file/local_server/clnpath.h

@@ -0,0 +1,6 @@
+#pragma once
+
+#include <string>
+#include <string_view>
+
+extern std::string clnpath(const std::string_view&);

file/local_server/duration_utils.h

@@ -0,0 +1,35 @@
+#pragma once
+
+template <typename Rep, typename Per>
+inline std::string duration_to_string(std::chrono::duration<Rep, Per> ms) {
+    std::string result{};
+
+    {
+        auto hours = std::chrono::duration_cast<std::chrono::hours>(ms);
+        if(hours.count())
+            result += std::to_string(hours.count()) + " hours ";
+        ms -= hours;
+    }
+    {
+        auto minutes = std::chrono::duration_cast<std::chrono::minutes>(ms);
+        if(minutes.count())
+            result += std::to_string(minutes.count()) + " minutes ";
+        ms -= minutes;
+    }
+    {
+        auto seconds = std::chrono::duration_cast<std::chrono::seconds>(ms);
+        if(seconds.count())
+            result += std::to_string(seconds.count()) + " seconds ";
+        ms -= seconds;
+    }
+    if(result.empty()) {
+        auto milliseconds = std::chrono::duration_cast<std::chrono::milliseconds>(ms);
+        if(milliseconds.count())
+            result = std::to_string(milliseconds.count()) + " milliseconds ";
+    }
+    if(result.empty()) {
+        auto microseconds = std::chrono::duration_cast<std::chrono::microseconds>(ms);
+        result = std::to_string(microseconds.count()) + " microseconds ";
+    }
+    return result.substr(0, result.length() - 1);
+}

file/test/main.cpp

@@ -0,0 +1,242 @@
+//
+// Created by WolverinDEV on 29/04/2020.
+//
+
+#include <files/FileServer.h>
+#include <log/LogUtils.h>
+
+#include <experimental/filesystem>
+#include <local_server/clnpath.h>
+#include <event2/thread.h>
+#include <include/files/Config.h>
+#include <local_server/HTTPUtils.h>
+
+namespace fs = std::experimental::filesystem;
+
+using namespace ts::server;
+
+struct Nothing {};
+
+template <typename ErrorType, typename ResponseType>
+inline void print_fs_response(const std::string& message, const std::shared_ptr<file::ExecuteResponse<file::filesystem::DetailedError<ErrorType>, ResponseType>>& response) {
+    if(response->status == file::ExecuteStatus::ERROR)
+        logError(LOG_FT, "{}: {} => {}", message, (int) response->error().error_type, response->error().error_message);
+    else if(response->status == file::ExecuteStatus::SUCCESS)
+        logMessage(LOG_FT, "{}: success", message);
+    else
+        logWarning(LOG_FT, "Unknown response state ({})!", (int) response->status);
+}
+
+template <typename ErrorType, typename ResponseType>
+inline void print_ft_response(const std::string& message, const std::shared_ptr<file::ExecuteResponse<ErrorType, ResponseType>>& response) {
+    if(response->status == file::ExecuteStatus::ERROR)
+        logError(LOG_FT, "{}: {} => {}", message, (int) response->error().error_type, response->error().error_message);
+    else if(response->status == file::ExecuteStatus::SUCCESS)
+        logMessage(LOG_FT, "{}: success", message);
+    else
+        logWarning(LOG_FT, "Unknown response state ({})!", (int) response->status);
+}
+
+inline void print_query(const std::string& message, const file::filesystem::AbstractProvider::directory_query_response_t& response) {
+    if(response.status == file::ExecuteStatus::ERROR)
+        logError(LOG_FT, "{}: {} => {}", message, (int) response.error().error_type, response.error().error_message);
+    else if(response.status == file::ExecuteStatus::SUCCESS) {
+        const auto& entries = response.response();
+        logMessage(LOG_FT, "{}: Found {} entries", message, entries.size());
+        for(auto& entry : entries) {
+            if(entry.type == file::filesystem::DirectoryEntry::FILE)
+                logMessage(LOG_FT, "  - File {}", entry.name);
+            else if(entry.type == file::filesystem::DirectoryEntry::DIRECTORY)
+                logMessage(LOG_FT, "  - Directory {}", entry.name);
+            else
+                logMessage(LOG_FT, "  - Unknown {}", entry.name);
+            logMessage(LOG_FT, "    Write timestamp: {}", std::chrono::floor<std::chrono::seconds>(entry.modified_at.time_since_epoch()).count());
+            logMessage(LOG_FT, "    Size: {}", entry.size);
+        }
+    } else
+        logWarning(LOG_FT, "{}: Unknown response state ({})!", message, (int) response.status);
+}
+
+EVP_PKEY* ssl_generate_key() {
+    auto key = std::unique_ptr<EVP_PKEY, decltype(&EVP_PKEY_free)>(EVP_PKEY_new(), ::EVP_PKEY_free);
+
+    auto rsa = RSA_new();
+    auto e = std::unique_ptr<BIGNUM, decltype(&BN_free)>(BN_new(), ::BN_free);
+    BN_set_word(e.get(), RSA_F4);
+    if(!RSA_generate_key_ex(rsa, 2048, e.get(), nullptr)) return nullptr;
+    EVP_PKEY_assign_RSA(key.get(), rsa);
+    return key.release();
+}
+
+X509* ssl_generate_certificate(EVP_PKEY* key) {
+    auto cert = X509_new();
+    X509_set_pubkey(cert, key);
+
+    ASN1_INTEGER_set(X509_get_serialNumber(cert), 3);
+    X509_gmtime_adj(X509_get_notBefore(cert), 0);
+    X509_gmtime_adj(X509_get_notAfter(cert), 31536000L);
+
+    X509_NAME* name = nullptr;
+    name = X509_get_subject_name(cert);
+    //for(const auto& subject : this->subjects)
+    //    X509_NAME_add_entry_by_txt(name, subject.first.c_str(),  MBSTRING_ASC, (unsigned char *) subject.second.c_str(), subject.second.length(), -1, 0);
+    X509_set_subject_name(cert, name);
+
+    name = X509_get_issuer_name(cert);
+    //for(const auto& subject : this->issues)
+    //    X509_NAME_add_entry_by_txt(name, subject.first.c_str(),  MBSTRING_ASC, (unsigned char *) subject.second.c_str(), subject.second.length(), -1, 0);
+
+    X509_set_issuer_name(cert, name);
+
+    X509_sign(cert, key, EVP_sha512());
+    return cert;
+}
+
+int main() {
+    evthread_use_pthreads();
+    {
+        std::map<std::string, std::string> query{};
+        http::parse_url_parameters("http://www.example.org/suche?stichwort=wiki&no-arg&1arg=&ausgabe=liste&test=test#bla=d&blub=c", query);
+        for(const auto& [key, value] : query)
+            std::cout << key << " => " << value << std::endl;
+        return 0;
+    }
+
+    auto log_config = std::make_shared<logger::LoggerConfig>();
+    log_config->terminalLevel = spdlog::level::trace;
+    logger::setup(log_config);
+
+    std::string error{};
+    if(!file::initialize(error)) {
+        logError(LOG_FT, "Failed to initialize file server: {}", error);
+        return 0;
+    }
+    logMessage(LOG_FT, "File server started");
+    auto instance = file::server();
+
+
+    {
+        auto options = std::make_shared<pipes::SSL::Options>();
+        options->verbose_io = true;
+        options->context_method = SSLv23_method();
+        options->free_unused_keypairs = false;
+
+        {
+            std::shared_ptr<EVP_PKEY> pkey{ssl_generate_key(), ::EVP_PKEY_free};
+            std::shared_ptr<X509> cert{ssl_generate_certificate(&*pkey), ::X509_free};
+
+            options->default_keypair({pkey, cert});
+        }
+        file::config::ssl_option_supplier = [options]{
+            return options;
+        };
+    }
+
+#if 0
+    auto& fs = instance->file_system();
+    {
+        auto response = fs.initialize_server(0);
+        response->wait();
+        print_fs_response("Server init result", response);
+        if(response->status != file::ExecuteStatus::SUCCESS)
+            return 0;
+    }
+
+
+    {
+        auto response = fs.create_channel_directory(0, 2, "/");
+        response->wait();
+        print_fs_response("Channel dir create A", response);
+    }
+
+
+    {
+        auto response = fs.create_channel_directory(0, 2, "/test-folder/");
+        response->wait();
+        print_fs_response("Channel dir create B", response);
+    }
+
+
+    {
+        auto response = fs.create_channel_directory(0, 2, "../test-folder/");
+        response->wait();
+        print_fs_response("Channel dir create C", response);
+    }
+
+
+    {
+        auto response = fs.create_channel_directory(0, 2, "./test-folder/../test-folder-2");
+        response->wait();
+        print_fs_response("Channel dir create D", response);
+    }
+
+    {
+        auto response = fs.query_channel_directory(0, 2, "/");
+        response->wait();
+        print_query("Channel query", *response);
+    }
+
+    {
+        auto response = fs.query_icon_directory(0);
+        response->wait();
+        print_query("Icons", *response);
+    }
+
+    {
+        auto response = fs.query_avatar_directory(0);
+        response->wait();
+        print_query("Avatars", *response);
+    }
+
+    {
+        auto response = fs.rename_channel_file(0, 2, "./test-folder/../test-folder-2", "./test-folder/../test-folder-3");
+        response->wait();
+        print_fs_response("Folder rename A", response);
+    }
+
+    {
+        auto response = fs.rename_channel_file(0, 2, "./test-folder/../test-folder-3", "./test-folder/../test-folder-2");
+        response->wait();
+        print_fs_response("Folder rename B", response);
+    }
+#endif
+
+#if 0
+    auto& ft = instance->file_transfer();
+
+    ft.callback_transfer_finished = [](const std::shared_ptr<file::transfer::Transfer>& transfer) {
+        logMessage(0, "Transfer finished");
+    };
+
+    ft.callback_transfer_started = [](const std::shared_ptr<file::transfer::Transfer>& transfer) {
+        logMessage(0, "Transfer started");
+    };
+
+    ft.callback_transfer_aborted = [](const std::shared_ptr<file::transfer::Transfer>& transfer, const transfer::TransferStatistics&, const file::transfer::TransferError& error) {
+        logMessage(0, "Transfer aborted ({}/{})", (int) error.error_type, error.error_message);
+    };
+
+    ft.callback_transfer_statistics = [](const std::shared_ptr<file::transfer::Transfer>& transfer, const file::transfer::TransferStatistics& stats) {
+        logMessage(0, "Transfer stats. New file bytes: {}, delta bytes send {}", stats.delta_file_bytes_transferred, stats.delta_network_bytes_send);
+    };
+
+    {
+        auto response = ft.initialize_channel_transfer(file::transfer::Transfer::DIRECTION_UPLOAD, 0, 2, {
+                "test2.txt",
+                false,
+                4,
+                120,
+                32
+        });
+        response->wait();
+        print_ft_response("Download test.txt", response);
+        if(response->succeeded())
+            logMessage(LOG_FT, "Download key: {}", std::string{response->response()->transfer_key, TRANSFER_KEY_LENGTH});
+    }
+#endif
+
+    std::this_thread::sleep_for(std::chrono::seconds{120});
+    //TODO: Test file locking
+    file::finalize();
+    return 0;
+}

file/test/upload-content.bin

@@ -0,0 +1,2 @@
+Hello World
+xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx

file/todo.txt

@@ -0,0 +1 @@
+Test HTTPS file upload!

license/LicenseServerMain.cpp

@@ -18,7 +18,7 @@ using namespace license;
 
 /*
  * Requests/license: SELECT `tmp`.`keyId`, `tmp`.`key`, `tmp`.`ip`, `tmp`.`count`, `license_info`.`username`, `tmp`.`type` FROM (SELECT DISTINCT `license_request`.`keyId`, `key`, `license_request`.`ip`, COUNT(`license_request`.`keyId`) AS `count`, `license`.`type` FROM `license_request` INNER JOIN `license` ON `license`.`keyId` = `license_request`.`keyId` GROUP BY (`license_request`.`ip`)) AS `tmp` INNER JOIN `license_info` ON `license_info`.`keyId` = `tmp`.`keyId`
- * Different IP's: SELECT `tmp`.`keyId`, `license_info`.`username`, COUNT(`ip`) FROM (SELECT DISTINCT `ip`, `keyId` FROM `license_request`) AS `tmp` LEFT JOIN `license_info` ON `license_info`.`keyId` = `tmp`.`keyId` GROUP BY (`tmp`.`keyId`)
+ * Different IP's: SELECT `tmp`.`keyId`, `license_info`.`username`, COUNT(`ip`) FROM (SELECT DISTINCT `ip`, `keyId` FROM `license_request`) AS `tmp` LEFT JOIN `license_info` ON `license_info`.`keyId` = `tmp`.`keyId` GROUP BY (`tmp`.`keyId`) ORDER BY COUNT(`ip`) DESC
  *
  * Requests (license) / ip: SELECT DISTINCT `ip`, COUNT(`ip`) FROM `license_request` WHERE `keyId` = ? GROUP BY `ip`
  *
@@ -26,6 +26,8 @@ using namespace license;
  * //462
  *
  * SELECT DISTINCT(`ip`), `license_request`.`keyId`, `license_info`.`username` FROM `license_request` LEFT JOIN `license_info` ON `license_info`.`keyId` = `license_request`.`keyId` WHERE `timestamp` > (UNIX_TIMESTAMP() - 2 * 60 * 60) * 1000
+ * License too many request looking: SELECT `keyId`, `username`, COUNT(`ip`) FROM `unique_license_requests` GROUP BY `keyId` ORDER BY COUNT(`ip`) DESC
+ *
  *
  *
  * Online clients:   SELECT SUM(`clients`) FROM (SELECT DISTINCT(`ip`), `clients` FROM `history_online` WHERE `timestamp` > (UNIX_TIMESTAMP() - 60 * 60 * 2) * 1000) AS `a`
@@ -34,6 +36,18 @@ using namespace license;
  *
  * Empty instances: curl -ik "https://stats.teaspeak.de:27788?type=request&request_type=general" -X GET 2>&1 | grep "data:"
  */
+//SELECT SUM(`clients`) FROM `history_online` WHERE `keyId` = 701 OR `keyId` = 795 OR `keyId` = 792 OR `keyId` = 582 OR `keyId` = 753 OR `keyId` = 764 OR `keyId` = 761 OR `keyId` = 796 WHERE `timestamp` > (UNIX_TIMESTAMP() - 2.1 * 60 * 60) * 1000
+
+/*
+ * Extra users:
+ * - ServerSponsoring
+ * - Davide550
+ * - xDeyego?
+ * - latters
+ * - Pamonha
+ * - vova3639 (5 licenses)
+ */
+
 bool handle_command(string& line);
 
 shared_ptr<server::database::DatabaseHandler> license_manager;

license/server/DatabaseHandler.cpp

@@ -533,7 +533,7 @@ std::deque<std::unique_ptr<DatabaseHandler::GlobalVersionsStatistic>> DatabaseHa
 
 bool DatabaseHandler::register_license_upgrade(license_key_id_t old_key_id, license_key_id_t new_key_id,
         const std::chrono::system_clock::time_point &begin_timestamp, const std::chrono::system_clock::time_point &end_timestamp, const std::string &license_key) {
-    auto upgrade_id = std::chrono::system_clock::now().time_since_epoch().count();
+    auto upgrade_id = std::chrono::ceil<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
     auto sql_result = sql::command(this->sql(), "INSERT INTO `license_upgrades` (`upgrade_id`, `old_key_id`, `new_key_id`, `timestamp_begin`, `timestamp_end`, `valid`, `use_count`, `license`) VALUES"
                                                 "(:upgrade_id, :old_key_id, :new_key_id, :timestamp_begin, :timestamp_end, 1, 0, :license)",
                                                 variable{":upgrade_id", upgrade_id},

license/server/LicenseServer.cpp

@@ -134,16 +134,19 @@ void LicenseServer::handleEventWrite(int fd, short, void* ptrServer) {
     if(!client) return;
 
     buffer::RawBuffer* write_buffer{nullptr};
+    std::unique_lock write_lock(client->network.write_queue_lock);
     while(true) { //TODO: May add some kind of timeout?
-        std::lock_guard lock(client->network.write_queue_lock);
         write_buffer = TAILQ_FIRST(&client->network.write_queue);
         if(!write_buffer) return;
 
         auto writtenBytes = send(fd, &write_buffer->buffer[write_buffer->index], write_buffer->length - write_buffer->index, 0);
         if(writtenBytes <= 0) {
+            write_lock.unlock();
         	if(writtenBytes == -1 && errno == EAGAIN)
         		return;
 	        logError(LOG_LICENSE_CONTROLL, "Invalid write. Disconnecting remote client. Message: {}/{}", errno, strerror(errno));
+	        server->unregisterClient(client);
+	        return;
         } else {
             write_buffer->index += writtenBytes;
         }
@@ -187,7 +190,7 @@ void ConnectedClient::init() {
 	TAILQ_INIT(&network.write_queue);
 }
 
-void ConnectedClient::uninit() {
+void ConnectedClient::uninit(bool blocking) {
     {
         lock_guard queue_lock{this->network.write_queue_lock};
         ts::buffer::RawBuffer* buffer;
@@ -201,11 +204,21 @@ void ConnectedClient::uninit() {
         close(this->network.fileDescriptor);
 	    network.fileDescriptor = 0;
     }
-    if(this->network.readEvent) event_del(this->network.readEvent);
-    this->network.readEvent = nullptr;
 
-    if(this->network.writeEvent) event_del(this->network.writeEvent);
-    this->network.writeEvent = nullptr;
+    std::unique_lock elock{this->network.event_mutex};
+    auto read_event = std::exchange(this->network.readEvent, nullptr);
+    auto write_event = std::exchange(this->network.writeEvent, nullptr);
+    elock.unlock();
+
+    if(blocking) {
+        if(read_event) event_del_block(read_event);
+        if(write_event) event_del_block(write_event);
+    } else {
+        if(read_event) event_del_noblock(read_event);
+        if(write_event) event_del_noblock(write_event);
+    }
+    if(read_event) event_free(read_event);
+    if(write_event) event_free(write_event);
 }
 
 void LicenseServer::handleEventRead(int fd, short, void* ptrServer) {
@@ -221,12 +234,20 @@ void LicenseServer::handleEventRead(int fd, short, void* ptrServer) {
     if(read < 0){
         if(errno == EWOULDBLOCK) return;
         logError(LOG_LICENSE_CONTROLL, "Invalid read. Disconnecting remote client. Message: {}/{}", errno, strerror(errno));
-        event_del_noblock(client->network.readEvent);
+        {
+            std::lock_guard elock{client->network.event_mutex};
+            if(client->network.readEvent)
+                event_del_noblock(client->network.readEvent);
+        }
 	    server->closeConnection(client);
         return;
     } else if(read == 0) {
         logMessage(LOG_LICENSE_CONTROLL, "[CLIENT][" + client->address() + "] Received EOF for client. Removing client.");
-        event_del_noblock(client->network.readEvent);
+        {
+            std::lock_guard elock{client->network.event_mutex};
+            if(client->network.readEvent)
+                event_del_noblock(client->network.readEvent);
+        }
 	    server->closeConnection(client);
         return;
     }
@@ -312,7 +333,7 @@ void LicenseServer::unregisterClient(const std::shared_ptr<ConnectedClient> &cli
         std::lock_guard state_lock{client->protocol.state_lock};
         client->protocol.state = protocol::UNCONNECTED;
     }
-    client->uninit();
+    client->uninit(this_thread::get_id() != this->event_base_dispatch.get_id());
 }
 
 void LicenseServer::cleanup_clients() {

license/server/LicenseServer.h

@@ -31,6 +31,8 @@ namespace license {
 		    struct {
 			    sockaddr_in remoteAddr;
 			    int fileDescriptor = 0;
+
+			    std::mutex event_mutex{};
 			    event* readEvent = nullptr; /* protected via state_lock (check state and the use these variables) */
 			    event* writeEvent = nullptr;
 
@@ -57,7 +59,7 @@ namespace license {
 		    bool invalid_license = false;
 
             void init();
-            void uninit();
+            void uninit(bool /* blocking */);
             void sendPacket(const protocol::packet&);
 
 		    inline std::string address() { return inet_ntoa(network.remoteAddr.sin_addr); }

license/server/LicenseServerHandler.cpp

@@ -188,7 +188,7 @@ bool LicenseServer::handleServerValidation(shared_ptr<ConnectedClient> &client,
 		        logMessage(LOG_GENERAL, "[CLIENT][{}] Remote license has been deleted! Shutting down server!", client->address());
 	        } else {
 		        fill_info(response.mutable_license_info(), info, remote_license->data.licenceKey);
-		        auto is_invalid = !info->isValid();
+		        auto is_invalid = !info->isNotExpired();
 		        if(is_invalid) {
                     response.set_invalid_reason("license is invalid");
                     response.set_valid(false);
@@ -198,9 +198,15 @@ bool LicenseServer::handleServerValidation(shared_ptr<ConnectedClient> &client,
 	        }
         }
 		this->manager->logRequest(remote_license->key(), client->unique_identifier, client->address(), pkt.info().version(), response.valid());
-	} else {
+	}  else {
+	    /* shall never happen, by default each server has the default license */
 	    response.set_valid(true);
 	}
+    if(pkt.has_memory_valid() && !pkt.memory_valid()) {
+        response.set_invalid_reason("server memory seems to be invalid");
+        response.set_valid(false);
+        logError(LOG_GENERAL, "Server {} has patched license memory!", client->address());
+    }
 
 	if(client->protocol.version == 2) {
         if(response.valid())

license/server/WebAPI.cpp

@@ -297,17 +297,21 @@ void WebStatistics::handleEventWrite(int file_descriptor, short, void* ptr_serve
 	}
 
 
-	std::lock_guard<std::recursive_mutex> lock(client->execute_lock);
+	std::unique_lock elock(client->execute_lock);
 	if(client->buffer_write.empty()) return;
 	auto& buffer = client->buffer_write.front();
 
 	auto written = send(file_descriptor, buffer.data(), buffer.length(), MSG_DONTWAIT | MSG_NOSIGNAL);
 	if(written < 0){
+		elock.unlock();
+
 		if(errno == EWOULDBLOCK) return;
 		logError(LOG_LICENSE_WEB, "[{}] Invalid write: {}/{}. Closing connection.", client->client_prefix(), errno, strerror(errno));
 		server->close_connection(client);
 		return;
 	} else if(written == 0) {
+		elock.unlock();
+
 		logError(LOG_LICENSE_WEB, "[{}] Invalid write (eof). Closing connection", client->client_prefix());
 		server->close_connection(client);
 		return;
@@ -334,17 +338,20 @@ void WebStatistics::close_connection(const std::shared_ptr<license::web::WebStat
 		else; //TODO Error handling?
 	}
 
-	std::lock_guard<std::recursive_mutex> lock(client->execute_lock);
-	if(client->event_read) {
-		event_del(client->event_read);
-		event_free(client->event_read);
-		client->event_read = nullptr;
+	std::unique_lock elock(client->execute_lock);
+	auto event_read = std::exchange(client->event_read, nullptr);
+	auto event_write = std::exchange(client->event_write, nullptr);
+	elock.unlock();
+	if(event_read) {
+		event_del(event_read);
+		event_free(event_read);
 	}
-	if(client->event_write) {
-		event_del(client->event_write);
-		event_free(client->event_write);
-		client->event_write = nullptr;
+	if(event_write) {
+		event_del(event_write);
+		event_free(event_write);
 	}
+
+	elock.lock();
 	if(client->file_descriptor > 0) {
 		if(shutdown(client->file_descriptor, SHUT_RDWR) < 0); //TODO error handling
 		if(close(client->file_descriptor) < 0); //TODO error handling
@@ -353,6 +360,7 @@ void WebStatistics::close_connection(const std::shared_ptr<license::web::WebStat
 
 	if(client->pipe_websocket)
 		client->pipe_websocket = nullptr;
+
 	if(client->pipe_ssl) {
 		client->pipe_ssl->finalize();
 		client->pipe_ssl = nullptr;

license/server/WebAPI.h

@@ -92,7 +92,6 @@ namespace license {
 				std::deque<std::shared_ptr<Client>> clients;
 
 				threads::ThreadPool scheduler{1, "WebStatistics #"};
-
 			protected:
 				static void handleEventAccept(int, short, void*);
 				static void handleEventRead(int, short, void*);

license/shared/include/license/client.h

@@ -8,7 +8,7 @@
 #include "license.h"
 
 namespace license::client {
-    class LicenseServerClient {
+    class LicenseServerClient : public std::enable_shared_from_this<LicenseServerClient> {
         public:
             enum ConnectionState {
                 CONNECTING,

license/shared/include/license/license.h

@@ -348,7 +348,7 @@ namespace license {
 	    bool deleted{false};
 	    uint32_t upgrade_id{0};
 
-        inline bool isValid() { return (end.time_since_epoch().count() == 0 || std::chrono::system_clock::now() < this->end); }
+        inline bool isNotExpired() { return (end.time_since_epoch().count() == 0 || std::chrono::system_clock::now() < this->end); }
     };
 
     extern std::shared_ptr<License> readLocalLicence(const std::string &, std::string &);

license/shared/packets/LicenseRequest.proto

@@ -41,6 +41,7 @@ message ServerValidation {
     required bool license_info      = 2;
     optional bytes license          = 3;
     optional ServerInfo info        = 4; //Change somewhere to required but its currently for legacy support
+    optional bool memory_valid      = 5;
 }
 
 message LicenseResponse {

license/shared/src/client.cpp

@@ -7,6 +7,7 @@
 #include <event.h>
 #include <ThreadPool/ThreadHelper.h>
 #include <misc/endianness.h>
+#include <shared/include/license/client.h>
 #include "shared/include/license/client.h"
 #include "crypt.h"
 
@@ -42,11 +43,16 @@ LicenseServerClient::LicenseServerClient(const sockaddr_in &address, int pversio
 }
 
 LicenseServerClient::~LicenseServerClient() {
-    this->close_connection();
+    const auto is_event_loop = this->network.event_dispatch.get_id() == std::this_thread::get_id();
+    {
+        std::unique_lock slock{this->connection_lock};
+        this->connection_state = ConnectionState::UNCONNECTED;
+    }
+    this->cleanup_network_resources(); /* force cleanup ignoring the previous state */
+    if(is_event_loop) this->network.event_dispatch.detach();
 
     if(this->buffers.read)
         Buffer::free(this->buffers.read);
-    threads::save_join(this->network.event_dispatch, false);
 }
 
 bool LicenseServerClient::start_connection(std::string &error) {
@@ -88,14 +94,25 @@ bool LicenseServerClient::start_connection(std::string &error) {
     this->network.event_base = event_base_new();
     this->network.event_read = event_new(this->network.event_base, this->network.file_descriptor, EV_READ | EV_PERSIST, [](int, short e, void* _this) {
         auto client = reinterpret_cast<LicenseServerClient*>(_this);
+        auto client_ref = client->weak_from_this().lock();
+        if(!client_ref) {
+            logCritical(0, "Network callback for expired client (E011)!");
+            return;
+        }
         client->callback_read(e);
+        client_ref.reset();
     }, this);
     this->network.event_write = event_new(this->network.event_base, this->network.file_descriptor, EV_WRITE, [](int, short e, void* _this) {
         auto client = reinterpret_cast<LicenseServerClient*>(_this);
+        auto client_ref = client->weak_from_this().lock();
+        if(!client_ref) {
+            logCritical(0, "Network callback for expired client (E012)!");
+            return;
+        }
         client->callback_write(e);
+        client_ref.reset();
     }, this);
 
-    event_dispatch_spawned = true;
     this->network.event_dispatch = std::thread([&] {
         signal(SIGPIPE, SIG_IGN);
 
@@ -114,10 +131,6 @@ bool LicenseServerClient::start_connection(std::string &error) {
     return true;
     error_cleanup:
     this->cleanup_network_resources();
-    if(!event_dispatch_spawned) {
-        event_base_free(this->network.event_base);
-        this->network.event_base = nullptr;
-    }
     this->connection_state = ConnectionState::UNCONNECTED;
     return false;
 }
@@ -164,7 +177,7 @@ void LicenseServerClient::cleanup_network_resources() {
         auto buffer = TAILQ_FIRST(&this->buffers.write);
         while(buffer) {
             auto next = TAILQ_NEXT(buffer, tail);
-            Buffer::free(next);
+            Buffer::free(buffer);
             buffer = next;
         }
         TAILQ_INIT(&this->buffers.write);
@@ -243,8 +256,11 @@ void LicenseServerClient::callback_write(short events) {
     }
 
     if(events & EV_WRITE) {
-        if(this->connection_state == ConnectionState::CONNECTING)
+        if(this->connection_state == ConnectionState::CONNECTING) {
             this->callback_socket_connected();
+            if(this->connection_state == ConnectionState::UNCONNECTED) /* state may change in the callback */
+                return;
+        }
 
         ssize_t written_bytes{0};
 
@@ -365,6 +381,8 @@ void LicenseServerClient::handle_data(void *recv_buffer, size_t length) {
         if(buffer_length < header->length + sizeof(protocol::packet_header)) return;
 
         this->handle_raw_packet(header->packetId, buffer_ptr + buffer_offset + sizeof(protocol::packet_header), header->length);
+        if(this->connection_state == ConnectionState::UNCONNECTED) return; /* state may change while we're handing the packet */
+
         buffer_offset += header->length + sizeof(protocol::packet_header);
         buffer_length -= header->length + sizeof(protocol::packet_header);
     }

package.json

@@ -0,0 +1,8 @@
+{
+  "name": "TeaSpeak",
+  "version": "1.0.0",
+  "dependencies": {
+    "@types/node": "^14.11.2",
+    "yaml": "^1.10.0"
+  }
+}

server/CMakeLists.txt

@@ -43,14 +43,15 @@ set(SERVER_SOURCE_FILES
         # MySQLLibSSLFix.c
 
         src/client/ConnectedClient.cpp
-        src/client/voice/PrecomputedPuzzles.cpp
+        src/server/PrecomputedPuzzles.cpp
         src/client/voice/VoiceClient.cpp
         src/client/voice/VoiceClientHandschake.cpp
         src/client/voice/VoiceClientCommandHandler.cpp
-        src/client/voice/VoiceClientPacketHandler.cpp
-        src/client/voice/VoiceClientView.cpp
+        src/client/voice/VoiceClientConnectionPacketHandler.cpp
+        src/client/voice/PacketStatistics.cpp
         src/TS3ServerClientManager.cpp
         src/VirtualServer.cpp
+        src/FileServerHandler.cpp
         src/TS3ServerHeartbeat.cpp
         src/SignalHandler.cpp
         src/server/VoiceServer.cpp
@@ -61,14 +62,12 @@ set(SERVER_SOURCE_FILES
         src/client/command_handler/client.cpp
         src/client/command_handler/server.cpp
         src/client/command_handler/misc.cpp
+        src/client/command_handler/bulk_parsers.cpp
 
         src/client/ConnectedClientNotifyHandler.cpp
         src/VirtualServerManager.cpp
-        src/server/file/FileServer.cpp
         src/channel/ServerChannel.cpp
         src/channel/ClientChannelView.cpp
-        src/client/file/FileClient.cpp
-        src/client/file/FileClientIO.cpp
         src/Group.cpp
         src/manager/BanManager.cpp
         src/client/InternalClient.cpp
@@ -81,7 +80,6 @@ set(SERVER_SOURCE_FILES
         src/client/query/QueryClientCommands.cpp
         src/client/query/QueryClientNotify.cpp
 
-
         src/manager/IpListManager.cpp
 
         src/ConnectionStatistics.cpp
@@ -96,7 +94,6 @@ set(SERVER_SOURCE_FILES
         src/manager/LetterManager.cpp
         src/manager/PermissionNameManager.cpp
 
-        src/pinteraction/ApplicationInteraction.cpp
         src/ServerManagerSnapshot.cpp
         src/ServerManagerSnapshotDeploy.cpp
         src/client/music/Song.cpp
@@ -136,9 +133,34 @@ set(SERVER_SOURCE_FILES
         src/weblist/WebListManager.cpp
         src/weblist/TeamSpeakWebClient.cpp
 
+        src/snapshots/permission.cpp
+        src/snapshots/client.cpp
+        src/snapshots/channel.cpp
+        src/snapshots/server.cpp
+        src/snapshots/groups.cpp
+        src/snapshots/deploy.cpp
+        src/snapshots/music.cpp
+        src/snapshots/parser.cpp
+
+        src/manager/ActionLogger.cpp
+        src/manager/ActionLoggerImpl.cpp
+
         src/manager/ConversationManager.cpp
         src/client/SpeakingClientHandshake.cpp
-        src/client/command_handler/music.cpp src/client/command_handler/file.cpp)
+        src/client/command_handler/music.cpp src/client/command_handler/file.cpp
+
+        src/client/voice/PacketDecoder.cpp
+        src/client/voice/PacketEncoder.cpp
+        src/client/voice/ServerCommandExecutor.cpp
+        src/client/voice/PingHandler.cpp
+        src/client/voice/CryptSetupHandler.cpp
+
+        src/terminal/PipedTerminal.cpp
+
+        src/server/voice/UDPVoiceServer.cpp
+        src/server/voice/DatagramPacket.cpp
+)
+
 if (COMPILE_WEB_CLIENT)
     add_definitions(-DCOMPILE_WEB_CLIENT)
 
@@ -235,7 +257,7 @@ target_link_libraries(PermMapHelper
 
 SET(CPACK_PACKAGE_VERSION_MAJOR "1")
 SET(CPACK_PACKAGE_VERSION_MINOR "4")
-SET(CPACK_PACKAGE_VERSION_PATCH "10")
+SET(CPACK_PACKAGE_VERSION_PATCH "22")
 if (BUILD_TYPE_NAME EQUAL OFF)
     SET(CPACK_PACKAGE_VERSION_DATA "beta")
 elseif (BUILD_TYPE_NAME STREQUAL "")
@@ -257,6 +279,7 @@ target_link_libraries(TeaSpeakServer
         TeaLicenseHelper    #Static
         TeaMusic            #Static
         CXXTerminal::static  #Static
+        TeaSpeak-FileServer
         ${StringVariable_LIBRARIES_STATIC}
         ${YAML_CPP_LIBRARIES}
         pthread
@@ -278,10 +301,17 @@ target_link_libraries(TeaSpeakServer
 
         jsoncpp_lib
         ${ed25519_LIBRARIES_STATIC}
+        zstd::libzstd_static
 )
 
 if (COMPILE_WEB_CLIENT)
-    target_link_libraries(TeaSpeakServer ${glib20_DIR}/lib/x86_64-linux-gnu/libffi.so.7 ${nice_DIR}/lib/libnice.so.10)
+    file(GLOB GLIB20_ARCHS ${glib20_DIR}/lib/*)
+    list(LENGTH GLIB20_ARCHS GLIB20_ARCHS_LENGTH)
+    if (NOT ${GLIB20_ARCHS_LENGTH} EQUAL 1)
+        message(FATAL_ERROR "Missing arch specific folder for glib2.0 in ${glib20_DIR}. Found ${GLIB20_ARCHS_LENGTH} directories, expected 1.")
+    endif ()
+    list(GET GLIB20_ARCHS 0 GLIB20_ARCH_DIR)
+    target_link_libraries(TeaSpeakServer ${GLIB20_ARCH_DIR}/libffi.so.7 ${nice_DIR}/lib/libnice.so.10)
 endif ()
 
 # include_directories(${LIBRARY_PATH}/boringssl/include/)
@@ -307,4 +337,25 @@ if (NOT DISABLE_JEMALLOC)
             jemalloc
     )
     add_definitions(-DHAVE_JEMALLOC)
-endif ()
+endif ()
+
+
+add_executable(Snapshots-Permissions-Test src/snapshots/permission.cpp tests/snapshots/permission.cpp)
+target_link_libraries(Snapshots-Permissions-Test PUBLIC
+    TeaSpeak
+    CXXTerminal::static  #Static
+    ${StringVariable_LIBRARIES_STATIC}
+    ${YAML_CPP_LIBRARIES}
+    pthread
+    stdc++fs
+    libevent::core libevent::pthreads
+
+    #Require a so
+    sqlite3
+    DataPipes::rtc::shared
+
+    tomcrypt::static
+    tommath::static
+    ${glib20_DIR}/lib/x86_64-linux-gnu/libffi.so.7 ${nice_DIR}/lib/libnice.so.10
+)
+target_include_directories(Snapshots-Permissions-Test PUBLIC ${CMAKE_SOURCE_DIR}/server/src/)

server/environment/geoloc

@@ -1 +0,0 @@
-../repro/env/geoloc/

server/environment/providers

@@ -1 +0,0 @@
-../../music/bin/providers/

server/environment/resources

@@ -1 +0,0 @@
-../repro/env/resources/

server/main.cpp

@@ -1,3 +1,4 @@
+#include <dlfcn.h>
 #include <client/linux/handler/exception_handler.h>
 #include <iostream>
 #include <misc/strobf.h>
@@ -10,7 +11,6 @@
 #include "src/VirtualServer.h"
 #include "src/InstanceHandler.h"
 #include "src/server/QueryServer.h"
-#include "src/server/file/FileServer.h"
 #include "src/terminal/CommandHandler.h"
 #include "src/client/InternalClient.h"
 #include "src/SignalHandler.h"
@@ -44,11 +44,11 @@ extern void testTomMath();
     #define DB_NAME "TeaData.sqlite"
 #endif
 
-#include <regex>
 #include <codecvt>
+#include <src/terminal/PipedTerminal.h>
 #include "src/client/music/internal_provider/channel_replay/ChannelProvider.h"
 
-class CLIParser{
+class CLIParser {
     public:
         CLIParser (int &argc, char **argv){
             for (int i = 1; i < argc; i++)
@@ -114,6 +114,82 @@ int main(int argc, char** argv) {
     (void*) malloc_conf;
 #endif
 
+#if 0
+    {
+        //ts::property::list<ts::property::InstanceProperties>()
+        std::cout << "| Name | Type | Flags | Default Value | Description |  \n";
+        std::cout << "|:-- | -- | -- | -- |:-- |  \n";
+        for(const auto& property : ts::property::list<ts::property::VirtualServerProperties>()) {
+            std::cout << "| `" << property->name << "` | ";
+
+            switch(property->type_value) {
+                case ts::property::TYPE_STRING:
+                    std::cout << "String | ";
+                    break;
+
+                case ts::property::TYPE_BOOL:
+                    std::cout << "Boolean | ";
+                    break;
+
+                case ts::property::TYPE_SIGNED_NUMBER:
+                    std::cout << "Signed number | ";
+                    break;
+
+                case ts::property::TYPE_UNSIGNED_NUMBER:
+                    std::cout << "Unsigned number | ";
+                    break;
+
+                case ts::property::TYPE_FLOAT:
+                    std::cout << "Float | ";
+                    break;
+
+                default:
+                    std::cout << "Unknown | ";
+                    break;
+            }
+
+            std::string flags{};
+            if(property->flags & ts::property::FLAG_INTERNAL) { flags += ", internal"; }
+            if(property->flags & ts::property::FLAG_GLOBAL) { flags += ", global"; }
+
+            if(property->flags & ts::property::FLAG_SNAPSHOT) { flags += ", snapshot"; }
+            if(property->flags & ts::property::FLAG_SAVE) { flags += ", saved"; }
+            if(property->flags & ts::property::FLAG_SAVE_MUSIC) { flags += ", saved (music)"; }
+
+            if(property->flags & ts::property::FLAG_NEW) { flags += ", new"; }
+            if(property->flags & ts::property::FLAG_SERVER_VARIABLE) { flags += ", server variable"; }
+            if(property->flags & ts::property::FLAG_SERVER_VIEW) { flags += ", server view variable"; }
+
+            if(property->flags & ts::property::FLAG_CLIENT_VARIABLE) { flags += ", client variable"; }
+            if(property->flags & ts::property::FLAG_CLIENT_VIEW) { flags += ", client view variable"; }
+            if(property->flags & ts::property::FLAG_CLIENT_INFO) { flags += ", client info variable"; }
+
+            if(property->flags & ts::property::FLAG_CHANNEL_VARIABLE) { flags += ", channel variable"; }
+            if(property->flags & ts::property::FLAG_CHANNEL_VIEW) { flags += ", channel view variable"; }
+
+            // FLAG_GROUP_VIEW = FLAG_CHANNEL_VIEW << 1UL,
+            if(property->flags & ts::property::FLAG_INSTANCE_VARIABLE) { flags += ", instance variable"; }
+            if(property->flags & ts::property::FLAG_PLAYLIST_VARIABLE) { flags += ", playlist variable"; }
+            if(property->flags & ts::property::FLAG_USER_EDITABLE) { flags += ", editable"; }
+
+            if(!flags.empty()) {
+                std::cout << flags.substr(2);
+            }
+            std::cout << "| ";
+            if(property->default_value.empty()) {
+                std::cout << "empty";
+            } else {
+                std::cout << "`" << property->default_value << "` ";
+            }
+            std::cout << "| ";
+            std::cout << "No description ";
+            std::cout << "| ";
+            std::cout << "  \n";
+        }
+        return 0;
+    }
+#endif
+    
     CLIParser arguments(argc, argv);
     SSL_load_error_strings();
     OpenSSL_add_ssl_algorithms();
@@ -127,9 +203,8 @@ int main(int argc, char** argv) {
 
     if(!arguments.cmdOptionExists("--no-terminal")) {
         terminal::install();
-        if(!terminal::active()){ cerr << "could not setup terminal!" << endl; return -1; }
+        if(!terminal::active()) { cerr << "could not setup terminal!" << endl; return -1; }
     }
-    assert(ts::property::impl::validateUnique());
 
     if(arguments.cmdOptionExists("--help") || arguments.cmdOptionExists("-h")) {
         #define HELP_FMT "  {} {} | {}"
@@ -232,7 +307,6 @@ int main(int argc, char** argv) {
      if(true) return 0;
      */
 
-    //debugMessage(LOG_GENERAL, "Sizeof ViewEntry {} Sizeof LinkedTreeEntry {} Sizeof shared_ptr<ViewEntry> {} Sizeof ClientChannelView {}", sizeof(ts::ViewEntry), sizeof(ts::TreeView::LinkedTreeEntry), sizeof(shared_ptr<ts::ViewEntry>), sizeof(ts::ClientChannelView));
     {
         //http://git.mcgalaxy.de/WolverinDEV/tomcrypt/blob/develop/src/misc/crypt/crypt_inits.c#L40-86
         std::string descriptors = "LTGE";
@@ -265,7 +339,7 @@ int main(int argc, char** argv) {
     if(!cfgErrors.empty()){
         logErrorFmt(true, LOG_GENERAL, "Could not load configuration. Errors: (" + to_string(cfgErrors.size()) + ")");
         for(const auto& entry : cfgErrors)
-            logError(true, LOG_GENERAL, " - {}", entry);
+            logErrorFmt(true, LOG_GENERAL, " - {}", entry);
         logErrorFmt(true, LOG_GENERAL, "Stopping server...");
         goto stopApp;
     }
@@ -309,6 +383,32 @@ int main(int argc, char** argv) {
         logMessageFmt(true, LOG_GENERAL, strobf("[]---------------------------------------------------------[]").string());
     }
 
+    {
+        rlimit rlimit{0, 0};
+        //forum.teaspeak.de/index.php?threads/2570/
+        constexpr auto seek_help_message = "Fore more help visit the forum and read this thread (https://forum.teaspeak.de/index.php?threads/2570/).";
+        if(getrlimit(RLIMIT_NOFILE, &rlimit) != 0) {
+            //prlimit -n4096 -p pid_of_process
+            logWarningFmt(true, LOG_INSTANCE, "Failed to get open file rlimit ({}). Please ensure its over 16384.", strerror(errno));
+            logWarningFmt(true, LOG_INSTANCE, seek_help_message);
+        } else {
+            const auto original = rlimit.rlim_cur;
+            rlimit.rlim_cur = std::max(rlimit.rlim_cur, std::min(rlimit.rlim_max, (rlim_t) 16384));
+            if(original != rlimit.rlim_cur) {
+                if(setrlimit(RLIMIT_NOFILE, &rlimit) != 0) {
+                    logErrorFmt(true, LOG_INSTANCE, "Failed to set open file rlimit to {} ({}). Please ensure its over 16384.", rlimit.rlim_cur, strerror(errno));
+                    logWarningFmt(true, LOG_INSTANCE, seek_help_message);
+                    goto rlimit_updates;
+                }
+            }
+            if(rlimit.rlim_cur < 16384) {
+                logWarningFmt(true, LOG_INSTANCE, "Open file rlimit is bellow 16384 ({}). Please increase the system file descriptor limits.", rlimit.rlim_cur);
+                logWarningFmt(true, LOG_INSTANCE, seek_help_message);
+            }
+        }
+        rlimit_updates:;
+    }
+
     logMessage(LOG_GENERAL, "Starting TeaSpeak-Server v{}", build::version()->string(true));
     logMessage(LOG_GENERAL, "Starting music providers");
 
@@ -371,22 +471,18 @@ int main(int argc, char** argv) {
         auto password = arguments.cmdOptionExists("-q") ? arguments.get_option("-q") : arguments.get_option("--set_query_password");
         if(!password.empty()) {
             logMessageFmt(true, LOG_GENERAL, "Updating server admin query password to \"{}\"", password);
-            auto accounts = serverInstance->getQueryServer()->find_query_accounts_by_unique_id(serverInstance->getInitialServerAdmin()->getUid());
-            bool found = false;
-            for(const auto& account : accounts) {
-                if(account->bound_server != 0) continue;
+            auto account = serverInstance->getQueryServer()->find_query_account_by_name("serveradmin");
+            if(!account) {
+                logErrorFmt(true, LOG_GENERAL, "Failed to update server admin query password! Login does not exists!");
+            } else {
                 if(!serverInstance->getQueryServer()->change_query_password(account, password)) {
                     logErrorFmt(true, LOG_GENERAL, "Failed to update server admin query password! (Internal error)");
                 }
-                found = true;
-                break;
-            }
-            if(!found) {
-                logErrorFmt(true, LOG_GENERAL, "Failed to update server admin query password! Login does not exists!");
             }
         }
     }
 
+    terminal::initialize_pipe(arguments.get_option("--pipe-path"));
     if(terminal::instance()) terminal::instance()->setPrompt("§7> §f");
     while(mainThreadActive) {
         usleep(5 * 1000);
@@ -394,11 +490,23 @@ int main(int argc, char** argv) {
         if(terminal::instance()) {
             if(terminal::instance()->linesAvailable() > 0){
                 while(!(line = terminal::instance()->readLine("§7> §f")).empty())
-                    threads::Thread(THREAD_DETACHED, [line](){ terminal::chandler::handleCommand(line); });
+                    threads::Thread(THREAD_DETACHED, [line]{
+                        terminal::chandler::CommandHandle handle{};
+                        handle.command = line;
+
+                        if(!terminal::chandler::handleCommand(handle)) {
+                            for(const auto& response : handle.response)
+                                logErrorFmt(true, LOG_GENERAL, "{}", response);
+                        } else {
+                            for(const auto& response : handle.response)
+                                logMessageFmt(true, LOG_GENERAL, "{}", response);
+                        }
+                    });
             }
         }
     }
 
+    terminal::finalize_pipe();
 
     stopApp:
     logMessageFmt(true, LOG_GENERAL, "Stopping application");
@@ -420,4 +528,43 @@ int main(int argc, char** argv) {
         terminal::uninstall();
     mainThreadDone = true;
     return 0;
+}
+
+/* Fix for Virtuzzo 7 where sometimes the pthread create fails! */
+
+typedef int (*pthread_create_t)(pthread_t *thread, const pthread_attr_t *attr, void *(*start_routine) (void *), void *arg);
+pthread_create_t original_pthread_create{nullptr};
+int	pthread_create(pthread_t *thread, const pthread_attr_t *attr, void *(*start_routine) (void *), void *arg) {
+    if(!original_pthread_create) {
+        original_pthread_create = (pthread_create_t) dlsym(RTLD_NEXT, "pthread_create");
+        if(!original_pthread_create) {
+            std::cerr << "[CRITICAL] Missing original pthread_create function. Aborting execution!" << std::endl;
+            std::abort();
+        }
+    }
+
+    int result, attempt{0}, sleep{5};
+    while((result = original_pthread_create(thread, attr, start_routine, arg)) != 0 && errno == EAGAIN) {
+        if(attempt > 55) {
+            std::cerr << "[CRITICAL] pthread_create(...) cause EAGAIN for the last 50 attempts (~4.7seconds)! Aborting application execution!" << std::endl;
+            std::abort();
+        } else if(attempt > 5) {
+            /* let some other threads do work */
+            pthread_yield();
+        } else if(attempt == 0) {
+            std::string message{"[CRITICAL] Failed to spawn thread (Resource temporarily unavailable). Trying to recover."};
+            std::cerr << message << std::endl;
+        }
+
+        //std::string message{"[CRITICAL] pthread_create(...) cause EAGAIN! Trying again in " + std::to_string(sleep) + "usec (Attempt: " + std::to_string(attempt) + ")"};
+        //std::cerr << message << std::endl;
+        usleep(sleep);
+        attempt++;
+        sleep = (int) (sleep * 1.25);
+    }
+    if(attempt > 0) {
+        std::string message{"[CRITICAL] Successfully recovered from pthread_create() EAGAIN error. Took " + std::to_string(attempt) + " attempts."};
+        std::cerr << message << std::endl;
+    }
+    return result;
 }

server/repro/build.sh

@@ -6,6 +6,39 @@ if [[ -z "${BUILD_PATH}" ]]; then
     exit 1
 fi
 
+rm buildVersion.txt
+cp env/buildVersion.txt .
+rm -r env
+mkdir env
+cd env
+[[ $? -ne 0 ]] && {
+  echo "Failed to create the env"
+  exit 1
+}
+cp -rf ../../../git-teaspeak/default_files/{certs,commanddocs,geoloc,resources,*.sh} .
+[[ $? -ne 0 ]] && {
+  echo "Failed to copy env"
+  exit 1
+}
+
+cp -rf ../../../music/bin/providers .
+[[ $? -ne 0 ]] && {
+  echo "Failed to copy providers"
+  exit 1
+}
+#
+cp ../../environment/TeaSpeakServer .
+[[ $? -ne 0 ]] && {
+  echo "Failed to copy server"
+  exit 1
+}
+cd ..
+mv buildVersion.txt env/buildVersion.txt
+[[ $? -ne 0 ]] && {
+  echo "Failed to move the build version back"
+  exit 1
+}
+
 ./generate_version.sh "${BUILD_PATH}" || {
     echo "Failed to generate version! ($?)"
     exit 1
@@ -24,4 +57,4 @@ fi
 ./deploy_build.sh "${BUILD_PATH}" || {
     echo "Failed to deploy package! ($?)"
     exit 1
-}
+}

server/repro/env/TeaSpeakServer

@@ -1 +0,0 @@
-../../environment/TeaSpeakServer

server/repro/env/certs

@@ -1 +0,0 @@
-../../../git-teaspeak/default_files/certs/

server/repro/env/commanddocs

@@ -1 +0,0 @@
-../../../git-teaspeak/default_files/commanddocs/

server/repro/env/geoloc

@@ -1 +0,0 @@
-../../../git-teaspeak/default_files/geoloc/

server/repro/env/install_music.sh

@@ -1 +0,0 @@
-../../../git-teaspeak/default_files/install_music.sh

server/repro/env/providers

@@ -1 +0,0 @@
-../../../music/bin/providers/

server/repro/env/resources

@@ -1 +0,0 @@
-../../../git-teaspeak/default_files/resources/

server/repro/env/tealoop.sh

@@ -1 +0,0 @@
-../../../git-teaspeak/default_files/tealoop.sh

server/repro/env/teastart.sh

@@ -1 +0,0 @@
-../../../git-teaspeak/default_files/teastart.sh

server/repro/env/teastart_autorestart.sh

@@ -1 +0,0 @@
-../../../git-teaspeak/default_files/teastart_autorestart.sh

server/repro/env/teastart_minimal.sh

@@ -1 +0,0 @@
-../../../git-teaspeak/default_files/teastart_minimal.sh

server/repro/generate_libraries.sh

@@ -3,7 +3,7 @@
 #Required libraries:
 # "libssl.so"
 # "libcrypto.so"
-# "libDataPipes.so"
+# "libDataPipes-Rtc-Shared.so"
 # "libjemalloc.so.2"
 # "libsqlite3.so.0"
 # "libTeaMusic.so"
@@ -50,8 +50,8 @@ query_system_link "libcrypto.so.1.1"
 cp "${library_path}" . || { echo "failed to copy libcrypto.so.1.1"; exit 1; }
 
 # Setting up DataPipes
-library_path=$(realpath "${library_base}/DataPipes/${build_path}/lib/libDataPipes-RTC.so")
-cp "$library_path" . || { echo "failed to copy libDataPipes-RTC.so"; exit 1; }
+library_path=$(realpath "${library_base}/DataPipes/${build_path}/lib/libDataPipes-Rtc-Shared.so")
+cp "$library_path" . || { echo "failed to copy libDataPipes-Rtc-Shared.so"; exit 1; }
 _dp_path="$library_path"
 
 # Setting up Sqlite3

server/src/Configuration.cpp

@@ -1,5 +1,6 @@
 #include <utility>
 
+#include <log/LogUtils.h>
 #include "Configuration.h"
 #include "build.h"
 #include "../../license/shared/include/license/license.h"
@@ -46,14 +47,34 @@ bool config::server::strict_ut8_mode;
 bool config::server::show_invisible_clients_as_online;
 bool config::server::disable_ip_saving;
 bool config::server::default_music_bot;
+/*
+ * namespace limits {
+            extern size_t poke_message_length;
+            extern size_t talk_power_request_message_length;
+        }
+ */
+size_t config::server::limits::poke_message_length;
+size_t config::server::limits::talk_power_request_message_length;
+size_t config::server::limits::afk_message_length;
+
 ssize_t config::server::max_virtual_server;
 bool config::server::badges::allow_badges;
 bool config::server::badges::allow_overwolf;
 bool config::server::authentication::name;
 
 bool config::server::clients::teamspeak;
+std::string config::server::clients::extra_welcome_message_teamspeak;
+config::server::clients::WelcomeMessageType config::server::clients::extra_welcome_message_type_teamspeak;
+
 bool config::server::clients::teaweb;
+std::string config::server::clients::extra_welcome_message_teaweb;
+config::server::clients::WelcomeMessageType config::server::clients::extra_welcome_message_type_teaweb;
+
 bool config::server::clients::teaspeak;
+std::string config::server::clients::extra_welcome_message_teaspeak;
+config::server::clients::WelcomeMessageType config::server::clients::extra_welcome_message_type_teaspeak;
+
+bool config::server::clients::ignore_max_clone_permissions;
 
 uint16_t config::voice::default_voice_port;
 size_t config::voice::DefaultPuzzlePrecomputeSize;
@@ -115,8 +136,12 @@ bool config::web::activated;
 std::deque<std::tuple<std::string, std::string, std::string>> config::web::ssl::certificates;
 uint16_t config::web::webrtc_port_max;
 uint16_t config::web::webrtc_port_min;
-deque<string> config::web::ice_servers;
+bool config::web::stun_enabled;
+std::string config::web::stun_host;
+uint16_t config::web::stun_port;
 bool config::web::enable_upnp;
+bool config::web::udp_enabled;
+bool config::web::tcp_enabled;
 
 size_t config::log::vs_size;
 std::string config::log::path;
@@ -468,7 +493,6 @@ vector<string> config::parseConfig(const std::string& path) {
                 goto license_parsing;
             }
 
-            /*
             if(!config::license->isValid()) {
                 if(config::license->data.type == license::LicenseType::INVALID) {
                     errors.emplace_back(strobf("Give license isn't valid!").string());
@@ -478,38 +502,18 @@ vector<string> config::parseConfig(const std::string& path) {
                 logErrorFmt(true, LOG_GENERAL, strobf("The given license isn't valid!").string());
                 logErrorFmt(true, LOG_GENERAL, strobf("Falling back to the default license.").string());
                 teaspeak_license = "none";
+
                 goto license_parsing;
             }
-             */
         }
 
         {
             auto bindings = create_bindings();
             read_bindings(config, bindings);
             build_comments(comments, bindings);
-
-            for(const auto& entry : config::web::ice_servers) {
-                auto dp = entry.find(':');
-                if(dp == string::npos) {
-                    errors.emplace_back("Invalid ice server entry! Missing port");
-                    continue;
-                }
-                auto host = entry.substr(0, dp);
-                auto port = entry.substr(dp + 1);
-                if(port.find_last_not_of("0123456789") != string::npos)  {
-                    errors.push_back("Invalid ice server entry! Invalid port (" + port + ")");
-                    continue;
-                }
-
-                try {
-                    stoi(port);
-                } catch(std::exception& ex) {
-                    errors.push_back("Invalid ice server entry! Invalid port (" + port + ")");
-                }
-            }
         }
 
-        auto currentVersion = strobf("TeaSpeak ").string() + build::version()->string(true);
+        auto currentVersion = config::server::default_version();
         if(currentVersion != config::server::DefaultServerVersion) {
             auto ref = config::server::DefaultServerVersion;
             try {
@@ -573,6 +577,14 @@ std::vector<std::string> config::reload() {
 
         auto bindings = create_bindings();
         read_bindings(config, bindings, FLAG_RELOADABLE);
+
+
+        const auto& logConfig = logger::currentConfig();
+        if(logConfig) {
+            logConfig->logfileLevel = (spdlog::level::level_enum) ts::config::log::logfileLevel;
+            logConfig->terminalLevel = (spdlog::level::level_enum) ts::config::log::terminalLevel;
+            logger::updateLogLevels();
+        }
     } catch(const YAML::Exception& ex) {
         errors.emplace_back("Could not read config: " + ex.msg + " @" + to_string(ex.mark.line) + ":" + to_string(ex.mark.column));
         return errors;
@@ -1014,8 +1026,9 @@ std::deque<std::shared_ptr<EntryBinding>> config::create_bindings() {
     {
         BIND_GROUP(log)
         {
-            CREATE_BINDING("level", 0);
+            CREATE_BINDING("level", FLAG_RELOADABLE);
             BIND_INTEGRAL(config::log::logfileLevel, spdlog::level::debug, spdlog::level::trace, spdlog::level::off);
+            ADD_NOTE_RELOADABLE();
             ADD_DESCRIPTION("The log level within the log files");
             ADD_DESCRIPTION("Available types:");
             ADD_DESCRIPTION("  0: Trace");
@@ -1027,8 +1040,9 @@ std::deque<std::shared_ptr<EntryBinding>> config::create_bindings() {
             ADD_DESCRIPTION("  6: Off");
         }
         {
-            CREATE_BINDING("terminal_level", 0);
+            CREATE_BINDING("terminal_level", FLAG_RELOADABLE);
             BIND_INTEGRAL(config::log::terminalLevel, spdlog::level::info, spdlog::level::trace, spdlog::level::off);
+            ADD_NOTE_RELOADABLE();
             ADD_DESCRIPTION("The log level within the TeaSpeak server terminal");
             ADD_DESCRIPTION("Available types:");
             ADD_DESCRIPTION("  0: Trace");
@@ -1107,17 +1121,18 @@ std::deque<std::shared_ptr<EntryBinding>> config::create_bindings() {
         BIND_GROUP(query);
         {
             CREATE_BINDING("nl_char", 0);
-            BIND_STRING(config::query::newlineCharacter, "\r\n");
+            BIND_STRING(config::query::newlineCharacter, "\n");
             ADD_DESCRIPTION("Change the query newline character");
+            ADD_NOTE("NOTE: TS3 - Compatible bots may require \"\n\r\"");
         }
         {
             CREATE_BINDING("motd", 0);
-            BIND_STRING(config::query::motd, "TeaSpeak\r\nWelcome on the TeaSpeak ServerQuery interface.\r\n");
+            BIND_STRING(config::query::motd, "TeaSpeak\nWelcome on the TeaSpeak ServerQuery interface.\n");
             ADD_DESCRIPTION("The query welcome message");
 
             ADD_NOTE("If not like TeamSpeak then some applications may not recognize the Query");
             ADD_NOTE("Default TeamSpeak 3 MOTD:");
-            ADD_NOTE("TS3\r\nWelcome to the TeamSpeak 3 ServerQuery interface, type \"help\" for a list of commands and \"help <command>\" for information on a specific command.\r\n");
+            ADD_NOTE("TS3\n\rWelcome to the TeamSpeak 3 ServerQuery interface, type \"help\" for a list of commands and \"help <command>\" for information on a specific command.\n\r");
             ADD_NOTE("NOTE: Sometimes you have to append one \r\n more!");
         }
         {
@@ -1154,17 +1169,17 @@ std::deque<std::shared_ptr<EntryBinding>> config::create_bindings() {
                     "The start point for the server creation still apply.");
         }
         {
-            CREATE_BINDING("notifymute", 0);
+            CREATE_BINDING("notifymute", FLAG_RELOADABLE);
             BIND_BOOL(config::voice::notifyMuted, false);
             ADD_DESCRIPTION("Enable/disable the mute notify");
         }
         {
-            CREATE_BINDING("suppress_myts_warnings", 0);
+            CREATE_BINDING("suppress_myts_warnings", FLAG_RELOADABLE);
             BIND_BOOL(config::voice::suppress_myts_warnings, true);
             ADD_DESCRIPTION("Suppress the MyTS integration warnings");
         }
         {
-            CREATE_BINDING("allow_session_reinitialize", 0);
+            CREATE_BINDING("allow_session_reinitialize", FLAG_RELOADABLE);
             BIND_BOOL(config::voice::allow_session_reinitialize, true);
             ADD_DESCRIPTION("Enable/disable fast session reinitialisation.");
             ADD_SENSITIVE();
@@ -1198,19 +1213,19 @@ std::deque<std::shared_ptr<EntryBinding>> config::create_bindings() {
             }
         }
         {
-            CREATE_BINDING("connect_limit", 0);
+            CREATE_BINDING("connect_limit", FLAG_RELOADABLE);
             BIND_INTEGRAL(config::voice::connectLimit, 10, 0, 1024);
             ADD_DESCRIPTION("Maximum amount of join attempts per second.");
             ADD_NOTE("A value of zero means unlimited");
         }
         {
-            CREATE_BINDING("client_connect_limit", 0);
+            CREATE_BINDING("client_connect_limit", FLAG_RELOADABLE);
             BIND_INTEGRAL(config::voice::clientConnectLimit, 3, 0, 1024);
             ADD_DESCRIPTION("Maximum amount of join attempts per second per ip.");
             ADD_NOTE("A value of zero means unlimited");
         }
         {
-            CREATE_BINDING("protocol.experimental_31", 0);
+            CREATE_BINDING("protocol.experimental_31", FLAG_RELOADABLE);
             BIND_BOOL(config::experimental_31, false);
             ADD_DESCRIPTION("Enables the newer and safer protocol based on TeamSpeak's documented standard");
             ADD_NOTE("An invalid protocol chain could lead clients to calculate a wrong shared secret result");
@@ -1255,24 +1270,26 @@ std::deque<std::shared_ptr<EntryBinding>> config::create_bindings() {
             BIND_BOOL(config::server::delete_old_bans, true);
             ADD_DESCRIPTION("Enable/disable the deletion of old bans within the database");
         }
+#if 0
         {
             CREATE_BINDING("delete_missing_icon_permissions", 0);
             BIND_BOOL(config::server::delete_missing_icon_permissions, true);
             ADD_DESCRIPTION("Enable/disable the deletion of invalid icon id permissions");
         }
+#endif
         {
             CREATE_BINDING("allow_weblist", 0);
             BIND_BOOL(config::server::enable_teamspeak_weblist, true);
             ADD_DESCRIPTION("Enable/disable weblist reports globally! (Server setting wount be disabled, they will be just not send)");
         }
         {
-            CREATE_BINDING("strict_ut8_mode", 0);
+            CREATE_BINDING("strict_ut8_mode", FLAG_RELOADABLE);
             BIND_BOOL(config::server::strict_ut8_mode, false);
             ADD_DESCRIPTION("If enabled an error will be throws on invalid UTF-8 characters within the protocol (Query & Client).");
             ADD_DESCRIPTION("Else the property pair will be dropped silently!");
         }
         {
-            CREATE_BINDING("show_invisible_clients", 0);
+            CREATE_BINDING("show_invisible_clients", FLAG_RELOADABLE);
             BIND_BOOL(config::server::show_invisible_clients_as_online, true);
             ADD_DESCRIPTION("Allow anybody to send text messages to clients which are in invisible channels");
         }
@@ -1282,7 +1299,7 @@ std::deque<std::shared_ptr<EntryBinding>> config::create_bindings() {
             ADD_DESCRIPTION("Disable the saving of IP addresses within the database.");
         }
         {
-            CREATE_BINDING("default_music_bot", 0);
+            CREATE_BINDING("default_music_bot", FLAG_RELOADABLE);
             BIND_BOOL(config::server::default_music_bot, true);
             ADD_DESCRIPTION("Add by default a new music bot to each created virtual server.");
         }
@@ -1292,6 +1309,27 @@ std::deque<std::shared_ptr<EntryBinding>> config::create_bindings() {
             ADD_DESCRIPTION("Set the limit for maximal virtual servers. -1 means unlimited.");
             ADD_NOTE_RELOADABLE();
         }
+        {
+            BIND_GROUP(limits);
+
+            {
+                CREATE_BINDING("poke_message_length",  FLAG_RELOADABLE);
+                BIND_INTEGRAL(config::server::limits::poke_message_length, 1024, 1, 262144);
+                ADD_NOTE_RELOADABLE();
+            }
+
+            {
+                CREATE_BINDING("talk_power_request_message_length",  FLAG_RELOADABLE);
+                BIND_INTEGRAL(config::server::limits::talk_power_request_message_length, 50, 1, 262144);
+                ADD_NOTE_RELOADABLE();
+            }
+
+            {
+                CREATE_BINDING("afk_message_length",  FLAG_RELOADABLE);
+                BIND_INTEGRAL(config::server::limits::afk_message_length, 50, 1, 262144);
+                ADD_NOTE_RELOADABLE();
+            }
+        }
         {
             /*
             BIND_GROUP(badges);
@@ -1317,13 +1355,60 @@ std::deque<std::shared_ptr<EntryBinding>> config::create_bindings() {
             }
         }
         {
+            using WelcomeMessageType = config::server::clients::WelcomeMessageType;
             BIND_GROUP(clients);
+
+            /* TeamSpeak */
             {
                 CREATE_BINDING("teamspeak", FLAG_RELOADABLE);
                 BIND_BOOL(config::server::clients::teamspeak, true);
                 ADD_DESCRIPTION("Allow/disallow the TeamSpeak 3 client to join the server.");
                 ADD_NOTE_RELOADABLE();
             }
+            {
+                CREATE_BINDING("teamspeak_message", FLAG_RELOADABLE);
+                BIND_STRING(config::server::clients::extra_welcome_message_teamspeak, "");
+                ADD_DESCRIPTION("Add an extra welcome message for TeamSpeak client users");
+                ADD_NOTE_RELOADABLE();
+            }
+            {
+                CREATE_BINDING("teamspeak_message_type", FLAG_RELOADABLE);
+                BIND_INTEGRAL(config::server::clients::extra_welcome_message_type_teamspeak, WelcomeMessageType::WELCOME_MESSAGE_TYPE_NONE, WelcomeMessageType::WELCOME_MESSAGE_TYPE_MIN, WelcomeMessageType::WELCOME_MESSAGE_TYPE_MAX);
+                ADD_DESCRIPTION("The welcome message type modes");
+                ADD_DESCRIPTION(std::to_string(WelcomeMessageType::WELCOME_MESSAGE_TYPE_NONE) + " - None, do nothing");
+                ADD_DESCRIPTION(std::to_string(WelcomeMessageType::WELCOME_MESSAGE_TYPE_CHAT) + " - Message, sends this message before the server welcome message");
+                ADD_DESCRIPTION(std::to_string(WelcomeMessageType::WELCOME_MESSAGE_TYPE_POKE) + " - Message, pokes the client with the message when he enters the server");
+                ADD_NOTE_RELOADABLE();
+            }
+
+            /* TeaSpeak */
+            /*
+            {
+                CREATE_BINDING("teaspeak", FLAG_RELOADABLE);
+                BIND_BOOL(config::server::clients::teaspeak, true);
+                ADD_DESCRIPTION("Allow/disallow the TeaSpeak - Client to join the server.");
+                ADD_NOTE_RELOADABLE();
+            }
+            */
+            config::server::clients::teaspeak = true;
+            {
+                CREATE_BINDING("teaspeak_message", FLAG_RELOADABLE);
+                BIND_STRING(config::server::clients::extra_welcome_message_teaspeak, "");
+                ADD_DESCRIPTION("Add an extra welcome message for the TeaSpeak - Client users");
+                ADD_NOTE_RELOADABLE();
+            }
+            {
+                CREATE_BINDING("teaspeak_message_type", FLAG_RELOADABLE);
+                BIND_INTEGRAL(config::server::clients::extra_welcome_message_type_teaspeak, WelcomeMessageType::WELCOME_MESSAGE_TYPE_NONE, WelcomeMessageType::WELCOME_MESSAGE_TYPE_MIN, WelcomeMessageType::WELCOME_MESSAGE_TYPE_MAX);
+                ADD_DESCRIPTION("The welcome message type modes");
+                ADD_DESCRIPTION(std::to_string(WelcomeMessageType::WELCOME_MESSAGE_TYPE_NONE) + " - None, do nothing");
+                ADD_DESCRIPTION(std::to_string(WelcomeMessageType::WELCOME_MESSAGE_TYPE_CHAT) + " - Message, sends this message before the server welcome message");
+                ADD_DESCRIPTION(std::to_string(WelcomeMessageType::WELCOME_MESSAGE_TYPE_POKE) + " - Message, pokes the client with the message when he enters the server");
+                ADD_NOTE_RELOADABLE();
+            }
+
+
+            /* TeaWeb */
             {
                 CREATE_BINDING("teaweb", FLAG_RELOADABLE);
                 BIND_BOOL(config::server::clients::teaweb, true);
@@ -1331,9 +1416,26 @@ std::deque<std::shared_ptr<EntryBinding>> config::create_bindings() {
                 ADD_NOTE_RELOADABLE();
             }
             {
-                CREATE_BINDING("teaspeak", FLAG_RELOADABLE);
-                BIND_BOOL(config::server::clients::teaspeak, true);
-                ADD_DESCRIPTION("Allow/disallow the TeaSpeak - Client to join the server.");
+                CREATE_BINDING("teaweb_message", FLAG_RELOADABLE);
+                BIND_STRING(config::server::clients::extra_welcome_message_teaweb, "");
+                ADD_DESCRIPTION("Add an extra welcome message for the TeaSpeak - Web client users");
+                ADD_NOTE_RELOADABLE();
+            }
+            {
+                CREATE_BINDING("teaweb_message_type", FLAG_RELOADABLE);
+                BIND_INTEGRAL(config::server::clients::extra_welcome_message_type_teaweb, WelcomeMessageType::WELCOME_MESSAGE_TYPE_NONE, WelcomeMessageType::WELCOME_MESSAGE_TYPE_MIN, WelcomeMessageType::WELCOME_MESSAGE_TYPE_MAX);
+                ADD_DESCRIPTION("The welcome message type modes");
+                ADD_DESCRIPTION(std::to_string(WelcomeMessageType::WELCOME_MESSAGE_TYPE_NONE) + " - None, do nothing");
+                ADD_DESCRIPTION(std::to_string(WelcomeMessageType::WELCOME_MESSAGE_TYPE_CHAT) + " - Message, sends this message before the server welcome message");
+                ADD_DESCRIPTION(std::to_string(WelcomeMessageType::WELCOME_MESSAGE_TYPE_POKE) + " - Message, pokes the client with the message when he enters the server");
+                ADD_NOTE_RELOADABLE();
+            }
+
+            {
+                CREATE_BINDING("ignore_max_clone_permissions", FLAG_RELOADABLE);
+                BIND_BOOL(config::server::clients::ignore_max_clone_permissions, false);
+                ADD_DESCRIPTION("Allows you to disable the permission checks for i_client_max_clones_uid, i_client_max_clones_ip and i_client_max_clones_hwid");
+                ADD_SENSITIVE();
                 ADD_NOTE_RELOADABLE();
             }
         }
@@ -1426,9 +1528,34 @@ std::deque<std::shared_ptr<EntryBinding>> config::create_bindings() {
             ADD_NOTE("These ports must opened to use the voice bridge (Protocol: UDP)");
         }
         {
-            CREATE_BINDING("webrtc.ice", 0);
-            BIND_VECTOR(config::web::ice_servers, {"stun.l.google.com:19302"});
-            ADD_DESCRIPTION("A list of possible offered ice servers");
+            CREATE_BINDING("webrtc.stun.enabled", 0);
+            BIND_INTEGRAL(config::web::stun_enabled, false, false, true);
+            ADD_DESCRIPTION("Whatever to use a STUN server");
+            ADD_NOTE_RELOADABLE();
+        }
+        {
+            CREATE_BINDING("webrtc.stun.host", 0);
+            BIND_STRING(config::web::stun_host, "stun.l.google.com");
+            ADD_DESCRIPTION("Stun hostname");
+            ADD_NOTE_RELOADABLE();
+        }
+        {
+            CREATE_BINDING("webrtc.stun.port", 0);
+            BIND_INTEGRAL(config::web::stun_port, 19302, 1, 0xFFFF);
+            ADD_DESCRIPTION("Port of the stun server");
+            ADD_NOTE_RELOADABLE();
+        }
+        {
+            CREATE_BINDING("webrtc.udp", 0);
+            BIND_INTEGRAL(config::web::udp_enabled, true, false, true);
+            ADD_DESCRIPTION("Enable UDP for theweb client");
+            ADD_NOTE_RELOADABLE();
+        }
+        {
+            CREATE_BINDING("webrtc.tcp", 0);
+            BIND_INTEGRAL(config::web::tcp_enabled, true, false, true);
+            ADD_DESCRIPTION("Enable TCP for theweb client");
+            ADD_NOTE_RELOADABLE();
         }
     }
     {

server/src/Configuration.h

@@ -7,8 +7,10 @@
     #undef byte
 #endif
 #include <spdlog/common.h>
+#include <misc/strobf.h>
 #include "geo/GeoLocation.h"
 #include "../../license/shared/include/license/license.h"
+#include "build.h"
 
 namespace YAML {
     class Node;
@@ -76,6 +78,12 @@ namespace ts::config {
         extern bool disable_ip_saving;
         extern bool default_music_bot;
 
+        namespace limits {
+            extern size_t poke_message_length;
+            extern size_t talk_power_request_message_length;
+            extern size_t afk_message_length;
+        }
+
         namespace badges {
             extern bool allow_overwolf;
             extern bool allow_badges;
@@ -86,12 +94,35 @@ namespace ts::config {
         }
 
         namespace clients {
+            enum WelcomeMessageType {
+                WELCOME_MESSAGE_TYPE_MIN,
+                WELCOME_MESSAGE_TYPE_NONE = WELCOME_MESSAGE_TYPE_MIN,
+                WELCOME_MESSAGE_TYPE_CHAT,
+                WELCOME_MESSAGE_TYPE_POKE,
+                WELCOME_MESSAGE_TYPE_MAX
+            };
+
             extern bool teamspeak;
+            extern std::string extra_welcome_message_teamspeak;
+            extern WelcomeMessageType extra_welcome_message_type_teamspeak;
+
             extern bool teaspeak;
+            extern std::string extra_welcome_message_teaspeak;
+            extern WelcomeMessageType extra_welcome_message_type_teaspeak;
+
             extern bool teaweb;
+            extern std::string extra_welcome_message_teaweb;
+            extern WelcomeMessageType extra_welcome_message_type_teaweb;
+
+            extern bool ignore_max_clone_permissions;
         }
 
         extern ssize_t max_virtual_server;
+
+        __attribute__((always_inline)) inline std::string default_version() { return strobf("TeaSpeak ").string() + build::version()->string(true); }
+        __attribute__((always_inline)) inline bool check_server_version_with_license() {
+            return default_version() == DefaultServerVersion || (license->isPremium() && license->isValid());
+        }
     }
 
     namespace voice {
@@ -185,8 +216,15 @@ namespace ts::config {
 
         extern uint16_t webrtc_port_min;
         extern uint16_t webrtc_port_max;
-        extern std::deque<std::string> ice_servers;
+
         extern bool enable_upnp;
+
+        extern bool stun_enabled;
+        extern std::string stun_host;
+        extern uint16_t stun_port;
+
+        extern bool tcp_enabled;
+        extern bool udp_enabled;
     }
 
     namespace threads {

server/src/ConnectionStatistics.cpp

@@ -3,8 +3,9 @@
 //
 
 #include <misc/memtracker.h>
+
+#include <utility>
 #include "ConnectionStatistics.h"
-#include "VirtualServer.h"
 
 using namespace std;
 using namespace std::chrono;
@@ -13,321 +14,98 @@ using namespace ts::server;
 using namespace ts::stats;
 using namespace ts::protocol;
 
-ConnectionStatistics::ConnectionStatistics(const shared_ptr<ConnectionStatistics>& handle, bool properties) : handle(handle) {
+ConnectionStatistics::ConnectionStatistics(shared_ptr<ConnectionStatistics> handle) : handle(std::move(handle)) {
     memtrack::allocated<ConnectionStatistics>(this);
-
-    if(properties) {
-        this->properties = make_shared<Properties>(); //TODO load etc?
-        this->properties->register_property_type<property::ConnectionProperties>();
-    }
-
-    /*
-    this->properties->registerProperty("connection_packets_sent_speech", 0, PROP_STATISTIC);
-    this->properties->registerProperty("connection_bytes_sent_speech", 0, PROP_STATISTIC);
-    this->properties->registerProperty("connection_packets_received_speech", 0, PROP_STATISTIC);
-    this->properties->registerProperty("connection_bytes_received_speech", 0, PROP_STATISTIC);
-
-    this->properties->registerProperty("connection_packets_sent_keepalive", 0, PROP_STATISTIC);
-    this->properties->registerProperty("connection_bytes_sent_keepalive", 0, PROP_STATISTIC);
-    this->properties->registerProperty("connection_packets_received_keepalive", 0, PROP_STATISTIC);
-    this->properties->registerProperty("connection_bytes_received_keepalive", 0, PROP_STATISTIC);
-
-    this->properties->registerProperty("connection_packets_sent_control", 0, PROP_STATISTIC);
-    this->properties->registerProperty("connection_bytes_sent_control", 0, PROP_STATISTIC);
-    this->properties->registerProperty("connection_packets_received_control", 0, PROP_STATISTIC);
-    this->properties->registerProperty("connection_bytes_received_control", 0, PROP_STATISTIC);
-
-    this->properties->registerProperty("connection_packets_sent_total", 0, PROP_STATISTIC);
-    this->properties->registerProperty("connection_bytes_sent_total", 0, PROP_STATISTIC);
-    this->properties->registerProperty("connection_packets_received_total", 0, PROP_STATISTIC);
-    this->properties->registerProperty("connection_bytes_received_total", 0, PROP_STATISTIC);
-
-    this->properties->registerProperty("connection_bandwidth_sent_last_second_total", 0, PROP_STATISTIC);
-    this->properties->registerProperty("connection_bandwidth_sent_last_minute_total", 0, PROP_STATISTIC);
-    this->properties->registerProperty("connection_bandwidth_received_last_second_total", 0, PROP_STATISTIC);
-    this->properties->registerProperty("connection_bandwidth_received_last_minute_total", 0, PROP_STATISTIC);
-
-    this->properties->registerProperty("connection_filetransfer_bandwidth_sent", 0, PROP_STATISTIC);
-    this->properties->registerProperty("connection_filetransfer_bandwidth_received", 0, PROP_STATISTIC);
-    this->properties->registerProperty("connection_filetransfer_bytes_sent_total", 0, PROP_STATISTIC);
-    this->properties->registerProperty("connection_filetransfer_bytes_received_total", 0, PROP_STATISTIC);
-    */
 }
 
 ConnectionStatistics::~ConnectionStatistics() {
     memtrack::freed<ConnectionStatistics>(this);
-
-    {
-        lock_guard lock(this->history_lock_incoming);
-
-        for(auto entry : this->history_incoming)
-            if(entry->use_count.fetch_sub(1) == 1)
-                delete entry;
-        for(auto entry : this->history_file_incoming)
-            if(entry->use_count.fetch_sub(1) == 1)
-                delete entry;
-
-        this->history_incoming.clear();
-        this->history_file_incoming.clear();
-    }
-    {
-        lock_guard lock(this->history_lock_outgoing);
-
-        for(auto entry : this->history_outgoing)
-            if(entry->use_count.fetch_sub(1) == 1)
-                delete entry;
-        for(auto entry : this->history_file_outgoing)
-            if(entry->use_count.fetch_sub(1) == 1)
-                delete entry;
-
-        this->history_outgoing.clear();
-        this->history_file_outgoing.clear();
-    }
-}
-
-std::shared_ptr<Properties> ConnectionStatistics::statistics() {
-    return this->properties;
 }
 
 void ConnectionStatistics::logIncomingPacket(const category::value &category, size_t size) {
-    auto info_entry = new StatisticEntry{};
-    info_entry->timestamp = system_clock::now();
-    info_entry->size = uint16_t(size);
+    assert(category >= 0 && category <= 2);
+    this->statistics_second_current.connection_bytes_received[category] += size;
+    this->statistics_second_current.connection_packets_received[category] += 1;
 
-    this->_log_incoming_packet(info_entry, category);
-}
-
-void ConnectionStatistics::_log_incoming_packet(ts::stats::StatisticEntry *info_entry, int8_t index) {
-    if(index >= 0 && index <= 3) {
-        this->connection_packets_received[index] ++;
-        this->connection_bytes_received[index] += info_entry->size;
-    }
-    this->connection_packets_received[0] ++;
-    this->connection_bytes_received[0] += info_entry->size;
-
-    if(this->_measure_bandwidths) {
-        auto lock_count = info_entry->use_count++;
-        assert(lock_count >= 0);
-        (void) lock_count;
-
-        lock_guard lock(this->history_lock_incoming);
-        this->history_incoming.push_back(info_entry);
-    }
     if(this->handle)
-        this->handle->_log_incoming_packet(info_entry, index);
+        this->handle->logIncomingPacket(category, size);
 }
 
 void ConnectionStatistics::logOutgoingPacket(const category::value &category, size_t size) {
-    auto info_entry = new StatisticEntry{};
-    info_entry->timestamp = system_clock::now();
-    info_entry->size = uint16_t(size);
+    assert(category >= 0 && category <= 2);
+    this->statistics_second_current.connection_bytes_sent[category] += size;
+    this->statistics_second_current.connection_packets_sent[category] += 1;
 
-    this->_log_outgoing_packet(info_entry, category);
-}
-
-
-void ConnectionStatistics::_log_outgoing_packet(ts::stats::StatisticEntry *info_entry, int8_t index) {
-    if(index >= 0 && index <= 3) {
-        this->connection_packets_sent[index] ++;
-        this->connection_bytes_sent[index] += info_entry->size;
-    }
-    this->connection_packets_sent[0] ++;
-    this->connection_bytes_sent[0] += info_entry->size;
-
-    if(this->_measure_bandwidths) {
-        auto lock_count = info_entry->use_count++;
-        assert(lock_count >= 0);
-        (void) lock_count;
-
-        lock_guard lock(this->history_lock_outgoing);
-        this->history_outgoing.push_back(info_entry);
-    }
     if(this->handle)
-        this->handle->_log_outgoing_packet(info_entry, index);
+        this->handle->logOutgoingPacket(category, size);
 }
 
 /* file transfer */
-void ConnectionStatistics::logFileTransferIn(uint64_t bytes) {
-    auto info_entry = new StatisticEntry{};
-    info_entry->timestamp = system_clock::now();
-    info_entry->size = bytes;
-
-    this->_log_incoming_file_packet(info_entry);
-}
-
-void ConnectionStatistics::_log_incoming_file_packet(ts::stats::StatisticEntry *info_entry) {
-    this->file_bytes_received += info_entry->size;
-
-    if(this->_measure_bandwidths) {
-        auto lock_count = info_entry->use_count++;
-        assert(lock_count >= 0);
-        (void) lock_count;
-
-        lock_guard lock(this->history_lock_incoming);
-        this->history_file_incoming.push_back(info_entry);
-    }
+void ConnectionStatistics::logFileTransferIn(uint32_t bytes) {
+    this->statistics_second_current.file_bytes_received += bytes;
+    this->file_bytes_received += bytes;
 
     if(this->handle)
-        this->handle->_log_incoming_file_packet(info_entry);
+        this->handle->logFileTransferIn(bytes);
 }
 
-void ConnectionStatistics::logFileTransferOut(uint64_t bytes) {
-    auto info_entry = new StatisticEntry{};
-    info_entry->timestamp = system_clock::now();
-    info_entry->size = bytes;
-
-    this->_log_outgoing_file_packet(info_entry);
-}
-
-void ConnectionStatistics::_log_outgoing_file_packet(ts::stats::StatisticEntry *info_entry) {
-    this->file_bytes_sent += info_entry->size;
-
-    if(this->_measure_bandwidths) {
-        auto lock_count = info_entry->use_count++;
-        assert(lock_count >= 0);
-        (void) lock_count;
-
-        lock_guard lock(this->history_lock_outgoing);
-        this->history_file_outgoing.push_back(info_entry);
-    }
+void ConnectionStatistics::logFileTransferOut(uint32_t bytes) {
+    this->statistics_second_current.file_bytes_sent += bytes;
+    this->file_bytes_sent += bytes;
 
     if(this->handle)
-        this->handle->_log_outgoing_file_packet(info_entry);
+        this->handle->logFileTransferOut(bytes);
 }
 
 void ConnectionStatistics::tick() {
-    StatisticEntry* entry;
-    {
-        auto timeout_min = system_clock::now() - minutes(1);
+    auto now = std::chrono::system_clock::now();
+    auto time_difference = this->last_second_tick.time_since_epoch().count() > 0 ? now - this->last_second_tick : std::chrono::seconds{1};
+    if(time_difference >= std::chrono::seconds{1}) {
+        BandwidthEntry<uint32_t> current{};
+        current.atomic_exchange(this->statistics_second_current);
 
-        lock_guard lock(this->history_lock_incoming);
+        auto period_ms = std::chrono::floor<std::chrono::milliseconds>(time_difference).count();
+        auto current_normalized = current.mul<long double>(1000.0 / period_ms);
 
-        while(!this->history_incoming.empty() && (entry = this->history_incoming[0])->timestamp < timeout_min) {
-            if(entry->use_count.fetch_sub(1) == 1)
-                delete entry;
+        this->statistics_second = this->statistics_second.mul<long double>(.2) + current_normalized.mul<long double>(.8);
+        this->total_statistics += current;
 
-            this->history_incoming.pop_front();
-        }
+        auto current_second = std::chrono::floor<std::chrono::seconds>(now.time_since_epoch()).count();
+        if(statistics_minute_offset == 0)
+            statistics_minute_offset = current_second;
 
-        while(!this->history_file_incoming.empty() && (entry = this->history_file_incoming[0])->timestamp < timeout_min) {
-            if(entry->use_count.fetch_sub(1) == 1)
-                delete entry;
-
-            this->history_file_incoming.pop_front();
-        }
-    }
-    {
-        auto timeout_min = system_clock::now() - minutes(1);
-        lock_guard lock(this->history_lock_outgoing);
-
-        while(!this->history_outgoing.empty() && (entry = this->history_outgoing[0])->timestamp < timeout_min) {
-            if(entry->use_count.fetch_sub(1) == 1)
-                delete entry;
-
-            this->history_outgoing.pop_front();
-        }
-
-        while(!this->history_file_outgoing.empty() && (entry = this->history_file_outgoing[0])->timestamp < timeout_min) {
-            if(entry->use_count.fetch_sub(1) == 1)
-                delete entry;
-
-            this->history_file_outgoing.pop_front();
-        }
-    }
-
-    if(this->properties) {
-        auto& _properties = *this->properties;
-#define M(type, index) \
-    _properties[property::CONNECTION_BYTES_SENT_ ##type] = (uint64_t) this->connection_bytes_sent[index]; \
-    _properties[property::CONNECTION_PACKETS_SENT_ ##type] = (uint64_t) this->connection_packets_sent[index]; \
-    _properties[property::CONNECTION_BYTES_RECEIVED_ ##type] = (uint64_t) this->connection_bytes_received[index]; \
-    _properties[property::CONNECTION_PACKETS_RECEIVED_ ##type] = (uint64_t) this->connection_packets_received[index]; \
-
-        M(TOTAL, 0);
-        M(CONTROL, 1);
-        M(KEEPALIVE, 2);
-        M(SPEECH, 3);
-
-        _properties[property::CONNECTION_FILETRANSFER_BYTES_RECEIVED_TOTAL] = (uint64_t) this->file_bytes_received;
-        _properties[property::CONNECTION_FILETRANSFER_BYTES_SENT_TOTAL] = (uint64_t) this->file_bytes_sent;
-
-        _properties[property::CONNECTION_FILETRANSFER_BYTES_RECEIVED_TOTAL] = (uint64_t) this->file_bytes_received;
-        _properties[property::CONNECTION_FILETRANSFER_BYTES_SENT_TOTAL] = (uint64_t) this->file_bytes_sent;
+        /* fill all "lost" with the current bandwidth as well */
+        while(statistics_minute_offset <= current_second)
+            this->statistics_minute[statistics_minute_offset++ % this->statistics_minute.size()] = current_normalized;
+        this->last_second_tick = now;
     }
 }
-
-DataSummery ConnectionStatistics::dataReport() {
-    DataSummery report{};
-    auto minTimeout = system_clock::now() - seconds(1);
-
-
-    {
-        lock_guard lock(this->history_lock_incoming);
-
-        for(const auto& elm : this->history_incoming){
-            if(elm->timestamp >= minTimeout) {
-                report.recv_second += elm->size;
-            }
-
-            report.recv_minute += elm->size;
-        }
-
-        for(const auto& elm : this->history_file_incoming) {
-            report.file_recv += elm->size;
-        }
-    }
-
-    {
-        lock_guard lock(this->history_lock_outgoing);
-
-        for(const auto& elm : this->history_outgoing){
-            if(elm->timestamp >= minTimeout) {
-                report.send_second += elm->size;
-            }
-
-            report.send_minute += elm->size;
-        }
-
-        for(const auto& elm : this->history_file_outgoing) {
-            report.file_send += elm->size;
-        }
-    }
-
-    report.recv_minute /= 60;
-    report.send_minute /= 60;
-    return report;
+BandwidthEntry<uint32_t> ConnectionStatistics::minute_stats() const {
+    BandwidthEntry<uint32_t> result{};
+    for(const auto& second : this->statistics_minute)
+        result += second;
+    return result.mul<uint32_t>(1. / (double) this->statistics_minute.size());
 }
 
-FullReport ConnectionStatistics::full_report() {
-    FullReport report{};
+FileTransferStatistics ConnectionStatistics::file_stats() {
+    FileTransferStatistics result{};
 
-    for(size_t index = 0 ; index < 4; index++) {
-        report.connection_bytes_sent[index] = (uint64_t) this->connection_bytes_sent[index];
-        report.connection_packets_sent[index] = (uint64_t) this->connection_packets_sent[index];
-        report.connection_bytes_received[index] = (uint64_t) this->connection_bytes_received[index];
-        report.connection_packets_received[index] = (uint64_t) this->connection_packets_received[index];
-    }
+    result.bytes_received = this->file_bytes_received;
+    result.bytes_sent = this->file_bytes_sent;
 
-    report.file_bytes_sent = this->file_bytes_sent;
-    report.file_bytes_received = this->file_bytes_received;
-
-    return report;
+    return result;
 }
 
 std::pair<uint64_t, uint64_t> ConnectionStatistics::mark_file_bytes() {
     std::pair<uint64_t, uint64_t> result;
 
     {
-        lock_guard lock(this->history_lock_incoming);
         if(this->mark_file_bytes_received < this->file_bytes_received)
             result.second = this->file_bytes_received - this->mark_file_bytes_received;
         this->mark_file_bytes_received = (uint64_t) this->file_bytes_received;
     }
 
     {
-
-        lock_guard lock(this->history_lock_outgoing);
-
         if(this->mark_file_bytes_sent < this->file_bytes_sent)
             result.first = this->file_bytes_sent - this->mark_file_bytes_sent;
         this->mark_file_bytes_sent = (uint64_t) this->file_bytes_sent;

server/src/ConnectionStatistics.h

@@ -11,41 +11,97 @@ namespace ts {
     }
 
     namespace stats {
-        struct StatisticEntry {
-            std::atomic<int8_t> use_count{0};
-            std::chrono::time_point<std::chrono::system_clock> timestamp;
-            uint16_t size = 0;
+        template <typename value_t>
+        struct BandwidthEntry {
+            std::array<value_t, 3> connection_packets_sent{};
+            std::array<value_t, 3> connection_bytes_sent{};
+            std::array<value_t, 3> connection_packets_received{};
+            std::array<value_t, 3> connection_bytes_received{};
+
+            value_t file_bytes_sent{0};
+            value_t file_bytes_received{0};
+
+            template <typename other_type>
+            inline BandwidthEntry& operator=(const BandwidthEntry<other_type>& other) {
+                for(size_t index{0}; index < this->connection_packets_sent.size(); index++)
+                    this->connection_packets_sent[index] = other.connection_packets_sent[index];
+                for(size_t index{0}; index < this->connection_bytes_sent.size(); index++)
+                    this->connection_bytes_sent[index] = other.connection_bytes_sent[index];
+                for(size_t index{0}; index < this->connection_packets_received.size(); index++)
+                    this->connection_packets_received[index] = other.connection_packets_received[index];
+                for(size_t index{0}; index < this->connection_bytes_received.size(); index++)
+                    this->connection_bytes_received[index] = other.connection_bytes_received[index];
+
+                this->file_bytes_sent = other.file_bytes_sent;
+                this->file_bytes_received = other.file_bytes_received;
+                return *this;
+            }
+
+            template <typename target_t>
+            inline BandwidthEntry<target_t> mul(double factor) const {
+                BandwidthEntry<target_t> result{};
+                result = *this;
+                for(auto& val : result.connection_packets_sent) val *= factor;
+                for(auto& val : result.connection_bytes_sent) val *= factor;
+                for(auto& val : result.connection_packets_received) val *= factor;
+                for(auto& val : result.connection_bytes_received) val *= factor;
+
+                result.file_bytes_sent *= factor;
+                result.file_bytes_received *= factor;
+                return result;
+            }
+
+            template <typename other_type>
+            inline BandwidthEntry& operator+=(const BandwidthEntry<other_type>& other) {
+                for(size_t index{0}; index < this->connection_packets_sent.size(); index++)
+                    this->connection_packets_sent[index] += other.connection_packets_sent[index];
+                for(size_t index{0}; index < this->connection_bytes_sent.size(); index++)
+                    this->connection_bytes_sent[index] += other.connection_bytes_sent[index];
+                for(size_t index{0}; index < this->connection_packets_received.size(); index++)
+                    this->connection_packets_received[index] += other.connection_packets_received[index];
+                for(size_t index{0}; index < this->connection_bytes_received.size(); index++)
+                    this->connection_bytes_received[index] += other.connection_bytes_received[index];
+
+                this->file_bytes_sent += other.file_bytes_sent;
+                this->file_bytes_received += other.file_bytes_received;
+                return *this;
+            }
+
+            template <typename other_type>
+            inline BandwidthEntry operator+(const BandwidthEntry<other_type>& other) {
+                return BandwidthEntry{*this} += other;
+            }
+
+            template <typename atomic_t>
+            inline void atomic_exchange(BandwidthEntry<std::atomic<atomic_t>>& source) {
+                for(size_t index{0}; index < this->connection_packets_sent.size(); index++)
+                    this->connection_packets_sent[index] = source.connection_packets_sent[index].exchange(0);
+                for(size_t index{0}; index < this->connection_bytes_sent.size(); index++)
+                    this->connection_bytes_sent[index] = source.connection_bytes_sent[index].exchange(0);
+                for(size_t index{0}; index < this->connection_packets_received.size(); index++)
+                    this->connection_packets_received[index] = source.connection_packets_received[index].exchange(0);
+                for(size_t index{0}; index < this->connection_bytes_received.size(); index++)
+                    this->connection_bytes_received[index] = source.connection_bytes_received[index].exchange(0);
+
+                this->file_bytes_sent = source.file_bytes_sent.exchange(0);
+                this->file_bytes_received = source.file_bytes_received.exchange(0);
+            }
         };
 
-        struct DataSummery {
-            uint32_t send_minute;
-            uint32_t send_second;
-
-            uint32_t recv_minute;
-            uint32_t recv_second;
-
-            uint32_t file_recv;
-            uint32_t file_send;
-        };
-
-        struct FullReport {
-            uint64_t connection_packets_sent[4]{0, 0, 0, 0};
-            uint64_t connection_bytes_sent[4]{0, 0, 0, 0};
-            uint64_t connection_packets_received[4]{0, 0, 0, 0};
-            uint64_t connection_bytes_received[4]{0, 0, 0, 0};
-
-            uint64_t file_bytes_sent = 0;
-            uint64_t file_bytes_received = 0;
+        struct FileTransferStatistics {
+            uint64_t bytes_received{0};
+            uint64_t bytes_sent{0};
         };
 
         class ConnectionStatistics {
             public:
                 struct category {
+                    /* Only three categories. Map unknown to category 0 */
                     enum value {
                         COMMAND,
-                        ACK,
+                        KEEP_ALIVE,
                         VOICE,
-                        UNKNOWN
+                        UNKNOWN = COMMAND
                     };
 
                     constexpr static std::array<category::value, 16> lookup_table{
@@ -53,10 +109,10 @@ namespace ts {
                             VOICE, /* VoiceWhisper */
                             COMMAND, /* Command */
                             COMMAND, /* CommandLow */
-                            ACK, /* Ping */
-                            ACK, /* Pong */
-                            ACK, /* Ack */
-                            ACK, /* AckLow */
+                            KEEP_ALIVE, /* Ping */
+                            KEEP_ALIVE, /* Pong */
+                            COMMAND, /* Ack */
+                            COMMAND, /* AckLow */
                             COMMAND, /* */
 
                             UNKNOWN,
@@ -76,58 +132,38 @@ namespace ts {
                         return from_type(type.type());
                     }
                 };
-                explicit ConnectionStatistics(const std::shared_ptr<ConnectionStatistics>& /* root */, bool /* spawn properties */);
+                explicit ConnectionStatistics(std::shared_ptr<ConnectionStatistics>  /* root */);
                 ~ConnectionStatistics();
 
-                std::shared_ptr<Properties> statistics();
-
-                inline void logIncomingPacket(const protocol::ClientPacket& packet) { this->logIncomingPacket(category::from_type(packet.type()), packet.length()); }
                 void logIncomingPacket(const category::value& /* category */, size_t /* length */);
-                inline void logOutgoingPacket(const protocol::ServerPacket& packet) { this->logOutgoingPacket(category::from_type(packet.type()), packet.length()); }
                 void logOutgoingPacket(const category::value& /* category */, size_t /* length */);
-                void logFileTransferIn(uint64_t);
-                void logFileTransferOut(uint64_t);
+                void logFileTransferIn(uint32_t);
+                void logFileTransferOut(uint32_t);
 
                 void tick();
 
-                DataSummery dataReport();
-                FullReport full_report();
+                [[nodiscard]] inline const BandwidthEntry<uint64_t>& total_stats() const { return this->total_statistics; }
+                [[nodiscard]] inline BandwidthEntry<uint32_t> second_stats() const { return this->statistics_second; }
+                [[nodiscard]] BandwidthEntry<uint32_t> minute_stats() const;
 
+                FileTransferStatistics file_stats();
                 std::pair<uint64_t, uint64_t> mark_file_bytes();
-
-                inline bool measure_bandwidths() { return this->_measure_bandwidths; }
-                void measure_bandwidths(bool flag) { this->_measure_bandwidths = flag; }
-
-                inline bool has_properties() { return !!this->properties; }
             private:
-                bool _measure_bandwidths = true;
                 std::shared_ptr<ConnectionStatistics> handle;
-                std::shared_ptr<Properties> properties;
 
+                BandwidthEntry<uint64_t> total_statistics{};
 
-                std::atomic<uint64_t> connection_packets_sent[4]{0, 0, 0, 0};
-                std::atomic<uint64_t> connection_bytes_sent[4]{0, 0, 0, 0};
-                std::atomic<uint64_t> connection_packets_received[4]{0, 0, 0, 0};
-                std::atomic<uint64_t> connection_bytes_received[4]{0, 0, 0, 0};
+                BandwidthEntry<std::atomic<uint64_t>> statistics_second_current{};
+                BandwidthEntry<uint32_t> statistics_second{}; /* will be updated every second by the stats from the "current_second" */
+                std::array<BandwidthEntry<uint32_t>, 60> statistics_minute{};
+                uint32_t statistics_minute_offset{0}; /* pointing to the upcoming minute */
+                std::chrono::system_clock::time_point last_second_tick{};
 
-                std::atomic<uint64_t> file_bytes_sent = 0;
-                std::atomic<uint64_t> file_bytes_received = 0;
+                std::atomic<uint64_t> file_bytes_sent{0};
+                std::atomic<uint64_t> file_bytes_received{0};
 
-                std::atomic<uint64_t> mark_file_bytes_sent = 0;
-                std::atomic<uint64_t> mark_file_bytes_received = 0;
-
-                spin_lock history_lock_outgoing;
-                spin_lock history_lock_incoming;
-                std::deque<StatisticEntry*> history_file_incoming{};
-                std::deque<StatisticEntry*> history_file_outgoing{};
-                std::deque<StatisticEntry*> history_incoming{};
-                std::deque<StatisticEntry*> history_outgoing{};
-
-                void _log_incoming_packet(StatisticEntry */* statistics */, int8_t /* type index */);
-                void _log_outgoing_packet(StatisticEntry* /* statistics */, int8_t /* type index */);
-
-                void _log_incoming_file_packet(StatisticEntry */* statistics */);
-                void _log_outgoing_file_packet(StatisticEntry* /* statistics */);
+                uint64_t mark_file_bytes_sent{0};
+                uint64_t mark_file_bytes_received{0};
         };
     }
 }

server/src/DatabaseHelper.cpp

@@ -13,153 +13,165 @@ using namespace ts::permission;
 
 //#define DISABLE_CACHING
 
+struct ts::server::CachedPermissionManager {
+    ServerId server_id{0};
+    ClientDbId client_database_id{0};
+    std::weak_ptr<permission::v2::PermissionManager> instance{};
+
+    std::shared_ptr<permission::v2::PermissionManager> instance_ref{}; /* reference to the current instance, will be refreshed every time the instance gets accessed */
+    std::chrono::time_point<std::chrono::system_clock> last_access{};
+};
+
+
+struct ts::server::StartupCacheEntry {
+    ServerId sid{0};
+
+    std::deque<std::unique_ptr<StartupPermissionEntry>> permissions{};
+    std::deque<std::unique_ptr<StartupPropertyEntry>> properties{};
+};
+
 DatabaseHelper::DatabaseHelper(sql::SqlManager* srv) : sql(srv) {}
 DatabaseHelper::~DatabaseHelper() {
-    for(const auto& elm : cachedPermissionManagers)
-        delete elm;
-    cachedPermissionManagers.clear();
+    this->cached_permission_managers.clear();
 }
 
 void DatabaseHelper::tick() {
+    auto cache_timeout = std::chrono::system_clock::now() - std::chrono::minutes{10};
     {
-        threads::MutexLock l(this->permManagerLock);
-        auto cpy = this->cachedPermissionManagers;
-        for(const auto& mgr : cpy){
-            //if(mgr->ownLock && system_clock::now() - mgr->lastAccess > minutes(5)) //TODO remove instand delete!
-                mgr->ownLock.reset();
-            if(mgr->manager.expired()){
-                this->cachedPermissionManagers.erase(std::find(this->cachedPermissionManagers.begin(), this->cachedPermissionManagers.end(), mgr));
-                delete mgr;
-            }
-        }
-    }
+        std::lock_guard cp_lock{this->cached_permission_manager_lock};
 
-    {
-        threads::MutexLock l(this->propsLock);
-        auto pcpy = this->cachedProperties;
-        for(const auto& mgr : pcpy){
-            if(mgr->ownLock && system_clock::now() - mgr->lastAccess > minutes(5))
-                mgr->ownLock.reset();
-            if(mgr->properties.expired()) {
-                this->cachedProperties.erase(std::find(this->cachedProperties.begin(), this->cachedProperties.end(), mgr));
-                delete mgr;
-            }
-        }
+        this->cached_permission_managers.erase(std::remove_if(this->cached_permission_managers.begin(), this->cached_permission_managers.end(), [&](const std::unique_ptr<CachedPermissionManager>& manager) {
+            if(manager->last_access < cache_timeout)
+                manager->instance_ref = nullptr;
+
+            if(manager->instance.expired())
+                return true;
+
+            return false;
+        }), this->cached_permission_managers.end());
     }
 }
 
-int collectData(deque<shared_ptr<ClientDatabaseInfo>>* list, int length, char** values, char** columns){
-    shared_ptr<ClientDatabaseInfo> entry = std::make_shared<ClientDatabaseInfo>();
+constexpr static std::string_view kSqlBase{"SELECT `client_unique_id`, `client_database_id`, `client_nickname`, `client_created`, `client_last_connected`, `client_total_connections` FROM `clients_server`"};
+inline std::deque<std::shared_ptr<ClientDatabaseInfo>> query_database_client_info(sql::SqlManager* sql_manager, ServerId server_id, const std::string& query, const std::vector<variable>& variables) {
+    std::deque<std::shared_ptr<ClientDatabaseInfo>> result{};
 
-    for(int index = 0; index < length; index++)
-        if(strcmp(columns[index], "cldbid") == 0)
-            entry->cldbid = static_cast<ClientDbId>(stol(values[index]));
-        else if(strcmp(columns[index], "clientUid") == 0)
-            entry->uniqueId = values[index];
-        else if(strcmp(columns[index], "firstConnect") == 0)
-            entry->created = time_point<system_clock>() + seconds(stoll(values[index]));
-        else if(strcmp(columns[index], "lastConnect") == 0)
-            entry->lastjoin = time_point<system_clock>() + seconds(stoll(values[index]));
-        else if(strcmp(columns[index], "connections") == 0)
-            entry->connections = static_cast<uint32_t>(stoi(values[index]));
-        else if(strcmp(columns[index], "lastName") == 0)
-            entry->lastName = values[index];
-        else if(strcmp(columns[index], "serverId") == 0);
-        else logError(LOG_GENERAL, "Invalid db key for manager data. Key: {}", columns[index]);
+    sql::command command{sql_manager, query};
+    for(const auto& variable : variables)
+        command.value(variable);
+    auto sql_result = command.query([&](int length, std::string* values, std::string* names) {
+        auto entry = std::make_shared<ClientDatabaseInfo>();
 
-    list->push_back(entry);
-    return 0;
-}
+        auto index{0};
+        try {
+            assert(names[index] == "client_unique_id");
+            entry->client_unique_id = values[index++];
 
-#define MAX_QUERY 32
-std::deque<std::shared_ptr<ClientDatabaseInfo>> DatabaseHelper::queryDatabaseInfo(const std::shared_ptr<VirtualServer>& server, const std::deque<ClientDbId>& list) {
-    if(list.empty()) return {};
+            assert(names[index] == "client_database_id");
+            entry->client_database_id = std::stoull(values[index++]);
 
-    deque<shared_ptr<ClientDatabaseInfo>> result;
+            assert(names[index] == "client_nickname");
+            entry->client_nickname = values[index++];
 
-    if(list.size() <= MAX_QUERY){
-        std::string query = "SELECT * FROM `clients` WHERE `serverId` = :serverId AND (";
-        for(auto elm : list)
-            query += " `cldbid` = " + to_string(elm) + " OR";
-        query = query.substr(0, query.length() - 3) + ")";
-        logTrace(server ? server->getServerId() : 0, "[SQL] queryDatabseInfo(...) -> {}", query);
-        auto state = sql::command(this->sql, query, variable{":serverId", server ? server->getServerId() : 0}).query(std::function<decltype(collectData)>(collectData), &result);
-        auto pf = LOG_SQL_CMD;
-        pf(state);
-        if(!state) return {};
-    } else {
-        std::deque<ClientDbId> sub;
-        do {
-            sub.insert(sub.begin(), list.begin(), list.begin() + min(list.size(), (size_t) MAX_QUERY));
-            list.erase(list.begin(), list.begin() + min(list.size(), (size_t) MAX_QUERY));
+            assert(names[index] == "client_created");
+            entry->client_created = std::chrono::system_clock::time_point{} + std::chrono::seconds{std::stoull(values[index++])};
 
-            auto res = this->queryDatabaseInfo(server, sub);
-            result.insert(result.end(), res.begin(), res.end());
-            sub.clear();
-        } while(!list.empty());
+            assert(names[index] == "client_last_connected");
+            entry->client_last_connected = std::chrono::system_clock::time_point{} + std::chrono::seconds{std::stoull(values[index++])};
+
+            assert(names[index] == "client_total_connections");
+            entry->client_total_connections = std::stoull(values[index++]);
+
+            assert(index == length);
+        } catch (std::exception& ex) {
+            logError(server_id, "Failed to parse client base properties at index {}: {}. Query: {}",
+                     index - 1,
+                     ex.what(),
+                     query
+            );
+        }
+
+        result.push_back(std::move(entry));
+    });
+    if(!sql_result) {
+        logError(server_id, "Failed to query client database infos: {}; Query: {}", sql_result.fmtStr(), query);
+        return result;
     }
 
     return result;
 }
 
+std::deque<std::shared_ptr<ClientDatabaseInfo>> DatabaseHelper::queryDatabaseInfo(const std::shared_ptr<VirtualServer>& server, const std::deque<ClientDbId>& list) {
+    if(list.empty())
+        return {};
+
+    std::string valueList{};
+    for(const auto& element : list)
+        valueList += ", " + std::to_string(element);
+    valueList = valueList.substr(2);
+
+    auto serverId = server ? server->getServerId() : 0;
+    return query_database_client_info(this->sql, serverId, std::string{kSqlBase} + "WHERE `server_id` = :sid AND `client_database_id` IN (" + valueList + ")", {variable{":sid", serverId}});
+}
+
 std::deque<std::shared_ptr<ClientDatabaseInfo>> DatabaseHelper::queryDatabaseInfoByUid(const std::shared_ptr<VirtualServer> &server, std::deque<std::string> list) {
-    if(list.empty()) return {};
+    if(list.empty())
+        return {};
 
-    deque<shared_ptr<ClientDatabaseInfo>> result;
+    std::string valueList{};
+    for(size_t value_index{0}; value_index < list.size(); value_index++)
+        valueList += ", :v" + std::to_string(value_index);
+    valueList = valueList.substr(2);
 
-    if(list.size() <= MAX_QUERY){
-        std::string query = "SELECT * FROM `clients` WHERE `serverId` = :serverId AND (";
-        for(const auto &elm : list)
-            query += " `clientUid` = '" + elm + "' OR";
-        query = query.substr(0, query.length() - 3) + ")";
-        logTrace(server ? server->getServerId() : 0, "[SQL] queryDatabseInfoByUid(...) -> {}", query);
-        auto state = sql::command(this->sql, query, variable{":serverId", server ? server->getServerId() : 0}).query(function<decltype(collectData)>(collectData), &result);
-        auto pf = LOG_SQL_CMD;
-        pf(state);
-        if(!state) return {};
-    } else {
-        std::deque<std::string> sub;
-        do {
-            sub.insert(sub.begin(), list.begin(), list.begin() + min(list.size(), (size_t) MAX_QUERY));
-            list.erase(list.begin(), list.begin() + min(list.size(), (size_t) MAX_QUERY));
+    auto serverId = server ? server->getServerId() : 0;
 
-            auto res = this->queryDatabaseInfoByUid(server, sub);
-            result.insert(result.end(), res.begin(), res.end());
-            sub.clear();
-        } while(!list.empty());
-    }
+    std::vector<variable> values{};
+    values.reserve(list.size() + 1);
 
-    return result;
+    values.emplace_back(":sid", serverId);
+    for(size_t value_index{0}; value_index < list.size(); value_index++)
+        values.emplace_back(":v" + std::to_string(value_index), list[value_index]);
+
+    return query_database_client_info(this->sql, serverId, std::string{kSqlBase} + "WHERE `server_id` = :sid AND `client_unique_id` IN (" + valueList + ")", values);
 }
 
 bool DatabaseHelper::validClientDatabaseId(const std::shared_ptr<VirtualServer>& server, ClientDbId cldbid) { return cldbid > 0; } //TODO here check
 
 void DatabaseHelper::deleteClient(const std::shared_ptr<VirtualServer>& server, ClientDbId cldbid) {
-    ServerId sid = static_cast<ServerId>(server ? server->getServerId() : 0);
+    auto serverId = (ServerId) (server ? server->getServerId() : 0);
     {
-        lock_guard<threads::Mutex> lock(permManagerLock);
-        for(auto permMgr : this->cachedPermissionManagers)
-            if(permMgr->cldbid == cldbid && permMgr->sid == sid) {
-                this->cachedPermissionManagers.erase(std::find(this->cachedPermissionManagers.begin(), this->cachedPermissionManagers.end(), permMgr));
-                delete permMgr;
-                break;
-            }
+        lock_guard lock{cached_permission_manager_lock};
+
+        this->cached_permission_managers.erase(std::remove_if(this->cached_permission_managers.begin(), this->cached_permission_managers.end(), [&](const auto& entry) {
+            return entry->server_id == serverId && entry->client_database_id == cldbid;
+        }), this->cached_permission_managers.end());
     }
-    //TODO remove from props cache?
 
+    sql::result state{};
 
-    auto state = sql::command(this->sql, "DELETE FROM `properties` WHERE `serverId` = :sid AND (`type` = :type1 OR `type` = :type2) AND `id` = :id", variable{":sid", sid}, variable{":type1", property::PROP_TYPE_CONNECTION}, variable{":type2", property::PROP_TYPE_CLIENT}, variable{":id", cldbid}).execute();
-    state = sql::command(this->sql, "DELETE FROM `permissions` WHERE `serverId` = :sid AND `type` = :type AND `id` = :id", variable{":sid", sid}, variable{":type", permission::SQL_PERM_USER}, variable{":id", cldbid}).execute();
-    state = sql::command(this->sql, "DELETE FROM `clients` WHERE `serverId` = :sid AND `cldbid` = :id", variable{":sid", sid}, variable{":id", cldbid}).execute();
-    state = sql::command(this->sql, "DELETE FROM `bannedClients` WHERE `serverId` = :sid AND `invokerDbid` = :id", variable{":sid", sid}, variable{":id", cldbid}).execute();
-    state = sql::command(this->sql, "DELETE FROM `assignedGroups` WHERE `serverId` = :sid AND `cldbid` = :id", variable{":sid", sid}, variable{":id", cldbid}).execute();
+    state = sql::command(this->sql, "DELETE FROM `properties` WHERE `serverId` = :sid AND (`type` = :type1 OR `type` = :type2) AND `id` = :id", variable{":sid", serverId}, variable{":type1", property::PROP_TYPE_CONNECTION}, variable{":type2", property::PROP_TYPE_CLIENT}, variable{":id", cldbid}).execute();
+    state = sql::command(this->sql, "DELETE FROM `permissions` WHERE `serverId` = :sid AND `type` = :type AND `id` = :id", variable{":sid", serverId}, variable{":type", permission::SQL_PERM_USER}, variable{":id", cldbid}).execute();
+    state = sql::command(this->sql, "DELETE FROM `bannedClients` WHERE `serverId` = :sid AND `invokerDbid` = :id", variable{":sid", serverId}, variable{":id", cldbid}).execute();
+    state = sql::command(this->sql, "DELETE FROM `assignedGroups` WHERE `serverId` = :sid AND `cldbid` = :id", variable{":sid", serverId}, variable{":id", cldbid}).execute();
+
+    if(serverId == 0) {
+        state = sql::command(this->sql, "DELETE FROM `clients_server` WHERE `client_database_id` = :id", variable{":id", cldbid}).execute();
+        state = sql::command(this->sql, "DELETE FROM `clients` WHERE `client_database_id` = :id", variable{":id", cldbid}).execute();
+    } else {
+        state = sql::command(this->sql, "DELETE FROM `clients_server` WHERE `server_id` = :sid AND `client_database_id` = :id", variable{":sid", serverId}, variable{":id", cldbid}).execute();
+    }
 
     //TODO delete letters
     //TODO delete query
     //TODO delete complains
 }
 
-inline sql::result load_permissions_v2(const std::shared_ptr<VirtualServer>& server, v2::PermissionManager* manager, sql::command& command, bool test_channel /* only used for client permissions (client channel permissions) */) {
+inline sql::result load_permissions_v2(
+        const std::shared_ptr<VirtualServer>& server,
+        v2::PermissionManager* manager,
+        sql::command& command,
+        bool test_channel, /* only used for client permissions (client channel permissions) */
+        bool is_channel) {
     auto start = system_clock::now();
 
     auto server_id = server ? server->getServerId() : 0;
@@ -213,7 +225,7 @@ inline sql::result load_permissions_v2(const std::shared_ptr<VirtualServer>& ser
             return 0;
         }
 
-        if(channel_id == 0)
+        if(channel_id == 0 || is_channel)
             manager->load_permission(key, {value, granted}, skipped, negated, value != permNotGranted, granted != permNotGranted);
         else
             manager->load_permission(key, {value, granted}, channel_id, skipped, negated, value != permNotGranted, granted != permNotGranted);
@@ -225,26 +237,38 @@ inline sql::result load_permissions_v2(const std::shared_ptr<VirtualServer>& ser
     logTrace(server_id, "[SQL] load_permissions(\"{}\") took {}ms", command.sqlCommand(), duration_cast<milliseconds>(time).count());
 }
 
-#define UPDATE_COMMAND "UPDATE `permissions` SET `value` = :value, `grant` = :grant, `flag_skip` = :flag_skip, `flag_negate` = :flag_negate WHERE `serverId` = :serverId AND `type` = :type AND `id` = :id AND `permId` = :permId AND `channelId` = :chId"
-#define INSERT_COMMAND "INSERT INTO `permissions` (`serverId`, `type`, `id`, `channelId`, `permId`, `value`, `grant`, `flag_skip`, `flag_negate`) VALUES (:serverId, :type, :id, :chId, :permId, :value, :grant, :flag_skip, :flag_negate)"
-#define DELETE_COMMAND "DELETE FROM `permissions` WHERE `serverId` = :serverId AND `type` = :type AND `id` = :id AND `permId` = :permId AND `channelId` = :chId"
+constexpr static std::string_view kPermissionUpdateCommand{"UPDATE `permissions` SET `value` = :value, `grant` = :grant, `flag_skip` = :flag_skip, `flag_negate` = :flag_negate WHERE `serverId` = :serverId AND `type` = :type AND `id` = :id AND `permId` = :permId AND `channelId` = :chId"};
+constexpr static std::string_view kPermissionInsertCommand{"INSERT INTO `permissions` (`serverId`, `type`, `id`, `channelId`, `permId`, `value`, `grant`, `flag_skip`, `flag_negate`) VALUES (:serverId, :type, :id, :chId, :permId, :value, :grant, :flag_skip, :flag_negate)"};
+constexpr static std::string_view kPermissionDeleteCommand{"DELETE FROM `permissions` WHERE `serverId` = :serverId AND `type` = :type AND `id` = :id AND `permId` = :permId AND `channelId` = :chId"};
+
+std::shared_ptr<permission::v2::PermissionManager> DatabaseHelper::find_cached_permission_manager(ServerId server_id,
+                                                                                                  ClientDbId client_database_id) {
+    for(auto it = this->cached_permission_managers.begin(); it != this->cached_permission_managers.end(); it++) {
+        auto& cached_manager = *it;
+
+        if(cached_manager->client_database_id == client_database_id && cached_manager->server_id == server_id) {
+            auto manager = cached_manager->instance.lock();
+            if(!manager){
+                this->cached_permission_managers.erase(it);
+                break;
+            }
+
+            cached_manager->last_access = system_clock::now();
+            cached_manager->instance_ref = manager;
+            return manager;
+        }
+    }
+
+    return nullptr;
+}
 
 std::shared_ptr<v2::PermissionManager> DatabaseHelper::loadClientPermissionManager(const std::shared_ptr<VirtualServer>& server, ClientDbId cldbid) {
     auto server_id = server ? server->getServerId() : 0;
 #ifndef DISABLE_CACHING
     {
-        lock_guard<threads::Mutex> lock(permManagerLock);
-        for(auto permMgr : this->cachedPermissionManagers)
-            if(permMgr->cldbid == cldbid && permMgr->sid == (server ? server->getServerId() : 0)) {
-                auto ptr = permMgr->manager.lock();
-                if(!ptr){
-                    this->cachedPermissionManagers.erase(std::find(this->cachedPermissionManagers.begin(), this->cachedPermissionManagers.end(), permMgr));
-                    delete permMgr;
-                    break;
-                }
-                permMgr->lastAccess = system_clock::now();
-                return ptr;
-            }
+        std::lock_guard lock{cached_permission_manager_lock};
+        auto manager = this->find_cached_permission_manager(server_id, cldbid);
+        if(manager) return manager;
     }
 #endif
 
@@ -282,20 +306,28 @@ std::shared_ptr<v2::PermissionManager> DatabaseHelper::loadClientPermissionManag
                                     variable{":serverId", server ? server->getServerId() : 0},
                                     variable{":type", permission::SQL_PERM_USER},
                                     variable{":id", cldbid});
-        LOG_SQL_CMD(load_permissions_v2(server, permission_manager.get(), command, true));
+        LOG_SQL_CMD(load_permissions_v2(server, permission_manager.get(), command, true, false));
     }
 
 
 #ifndef DISABLE_CACHING
-    this->permManagerLock.lock();
-    auto entry = new CachedPermissionManager();
-    entry->sid = server_id;
-    entry->manager = permission_manager;
-    entry->ownLock = permission_manager;
-    entry->cldbid = cldbid;
-    entry->lastAccess = system_clock::now();
-    this->cachedPermissionManagers.push_back(entry);
-    this->permManagerLock.unlock();
+    auto cache_entry = std::make_unique<CachedPermissionManager>();
+    cache_entry->server_id = server_id;
+    cache_entry->instance = permission_manager;
+    cache_entry->instance_ref = permission_manager;
+    cache_entry->client_database_id = cldbid;
+    cache_entry->last_access = system_clock::now();
+
+    {
+        std::lock_guard cache_lock{this->cached_permission_manager_lock};
+
+        /* test if we might not got a second instance */
+        auto manager = this->find_cached_permission_manager(server_id, cldbid);
+        if(manager) return manager;
+
+        this->cached_permission_managers.push_back(std::move(cache_entry));
+    }
+
 #endif
     return permission_manager;
 }
@@ -308,7 +340,7 @@ void DatabaseHelper::saveClientPermissions(const std::shared_ptr<ts::server::Vir
 
     auto server_id = server ? server->getServerId() : 0;
     for(auto& update : updates) {
-        std::string query = update.flag_delete ? DELETE_COMMAND : (update.flag_db ? UPDATE_COMMAND : INSERT_COMMAND);
+        std::string query{update.flag_delete ? kPermissionDeleteCommand : (update.flag_db ? kPermissionUpdateCommand : kPermissionInsertCommand)};
 
         auto permission_data = permission::resolvePermissionData(update.permission);
         auto value = update.update_value == v2::delete_value ? permNotGranted : update.values.value;
@@ -364,7 +396,7 @@ std::shared_ptr<permission::v2::PermissionManager> DatabaseHelper::loadGroupPerm
                                 variable{":serverId", server ? server->getServerId() : 0},
                                 variable{":type", permission::SQL_PERM_GROUP},
                                 variable{":id", group_id});
-    LOG_SQL_CMD(load_permissions_v2(server, result.get(), command, false));
+    LOG_SQL_CMD(load_permissions_v2(server, result.get(), command, false, false));
     return result;
 }
 
@@ -375,7 +407,7 @@ void DatabaseHelper::saveGroupPermissions(const std::shared_ptr<ts::server::Virt
 
     auto server_id = server ? server->getServerId() : 0;
     for(auto& update : updates) {
-        std::string query = update.flag_delete ? DELETE_COMMAND : (update.flag_db ? UPDATE_COMMAND : INSERT_COMMAND);
+        std::string query{update.flag_delete ? kPermissionDeleteCommand : (update.flag_db ? kPermissionUpdateCommand : kPermissionInsertCommand)};
 
         auto permission_data = permission::resolvePermissionData(update.permission);
         auto value = update.update_value == v2::delete_value ? permNotGranted : update.values.value;
@@ -436,7 +468,7 @@ std::shared_ptr<permission::v2::PermissionManager> DatabaseHelper::loadPlaylistP
                                 variable{":serverId", server ? server->getServerId() : 0},
                                 variable{":type", permission::SQL_PERM_PLAYLIST},
                                 variable{":id", playlist_id});
-    LOG_SQL_CMD(load_permissions_v2(server, result.get(), command, false));
+    LOG_SQL_CMD(load_permissions_v2(server, result.get(), command, false, false));
     return result;
 }
 
@@ -447,7 +479,7 @@ void DatabaseHelper::savePlaylistPermissions(const std::shared_ptr<VirtualServer
 
     auto server_id = server ? server->getServerId() : 0;
     for(auto& update : updates) {
-        std::string query = update.flag_delete ? DELETE_COMMAND : (update.flag_db ? UPDATE_COMMAND : INSERT_COMMAND);
+        std::string query{update.flag_delete ? kPermissionDeleteCommand : (update.flag_db ? kPermissionUpdateCommand : kPermissionInsertCommand)};
 
         auto permission_data = permission::resolvePermissionData(update.permission);
         auto value = update.update_value == v2::delete_value ? permNotGranted : update.values.value;
@@ -502,7 +534,7 @@ std::shared_ptr<permission::v2::PermissionManager> DatabaseHelper::loadChannelPe
                                 variable{":chid", channel},
                                 variable{":id", 0},
                                 variable{":type", permission::SQL_PERM_CHANNEL});
-    LOG_SQL_CMD(load_permissions_v2(server, result.get(), command, false));
+    LOG_SQL_CMD(load_permissions_v2(server, result.get(), command, false, true));
     return result;
 }
 
@@ -513,7 +545,7 @@ void DatabaseHelper::saveChannelPermissions(const std::shared_ptr<ts::server::Vi
 
     auto server_id = server ? server->getServerId() : 0;
     for(auto& update : updates) {
-        std::string query = update.flag_delete ? DELETE_COMMAND : (update.flag_db ? UPDATE_COMMAND : INSERT_COMMAND);
+        std::string query{update.flag_delete ? kPermissionDeleteCommand : (update.flag_db ? kPermissionUpdateCommand : kPermissionInsertCommand)};
 
         auto value = update.update_value == v2::delete_value ? permNotGranted : update.values.value;
         auto grant = update.update_grant == v2::delete_value ? permNotGranted : update.values.grant;
@@ -555,47 +587,44 @@ std::shared_ptr<Properties> DatabaseHelper::default_properties_client(std::share
     return properties;
 }
 
-bool DatabaseHelper::assignDatabaseId(sql::SqlManager *sql, ServerId id, std::shared_ptr<DataClient> cl) {
+bool DatabaseHelper::assignDatabaseId(sql::SqlManager *sql, ServerId serverId, std::shared_ptr<DataClient> cl) {
     cl->loadDataForCurrentServer();
-    if(cl->getClientDatabaseId() == 0){ //Client does not exist
-        ClientDbId cldbid = 0;
-        auto res = sql::command(sql, "SELECT `cldbid` FROM `clients` WHERE `serverId` = 0 AND `clientUid` = :cluid", variable{":cluid", cl->getUid()}).query([](ClientDbId* ptr, int length, char** values, char** names){
-            *ptr = static_cast<ClientDbId>(stoll(values[0]));
-            return 0;
-        }, &cldbid);
-        auto pf = LOG_SQL_CMD;
-        pf(res);
-        if(!res) return false;
+    if(cl->getClientDatabaseId() == 0) {
+        /* client does not exists, create a new one */
 
-        auto insertTemplate = sql::model(sql, "INSERT INTO `clients` (`serverId`, `cldbId`,  `clientUid`, `lastName`,`firstConnect`,`lastConnect`, `connections`) VALUES (:serverId, :cldbid, :cluid, :name, :fconnect, :lconnect, :connections)",
-                                         variable{":cluid", cl->getUid()}, variable{":name", cl->getDisplayName()}, variable{":fconnect", duration_cast<seconds>(system_clock::now().time_since_epoch()).count()}, variable{":lconnect", 0}, variable{":connections", 0});
-        if(cldbid == 0){ //Completly new user
-            res = sql::command(sql, "SELECT `cldbid` FROM `clients` WHERE `serverId` = 0 ORDER BY `cldbid` DESC LIMIT 1").query([](ClientDbId* ptr, int length, char** values, char** names){
-                *ptr = static_cast<ClientDbId>(stoll(values[0]));
-                return 0;
-            }, &cldbid);
-            pf(res);
-            if(!res) return false;
+        sql::result sql_result{};
 
-            cldbid += 1;
-            res = insertTemplate.command().values(variable{":serverId", 0}, variable{":cldbid", cldbid}).execute(); //Insert global
-            pf(res);
-            if(!res) return false;
-            debugMessage(id, "Having new instance user on server {}. (Database ID: {} Name: {})", id, cldbid, cl->getDisplayName());
-        } else {
-            debugMessage(id, "Having new server user on server {}. (Database ID: {} Name: {})", id, cldbid, cl->getDisplayName());
+        std::string insert_or_ignore{sql->getType() == sql::TYPE_SQLITE ? "INSERT OR IGNORE" : "INSERT IGNORE"};
+
+        auto currentTimeSeconds = duration_cast<seconds>(system_clock::now().time_since_epoch()).count();
+        sql_result = sql::command{sql, insert_or_ignore + " INTO `clients` (`client_unique_id`, `client_created`) VALUES (:uniqueId, :now)",
+            variable{":uniqueId", cl->getUid()},
+            variable{":now", currentTimeSeconds}
+        }.execute();
+
+        if(!sql_result) {
+            logCritical(LOG_INSTANCE, "Failed to execute client insert command for {}: {}", cl->getUid(), sql_result.fmtStr());
+            return false;
         }
 
-        if(id != 0){ //Else already inserted
-            res = insertTemplate.command().values(variable{":serverId", id}, variable{":cldbid", cldbid}).execute();
-            pf(res);
-            if(!res) return false;
+        sql_result = sql::command{sql, "INSERT INTO `clients_server` (`server_id`, `client_unique_id`, `client_database_id`, `client_created`) SELECT :serverId, :uniqueId, `client_database_id`, :now FROM `clients` WHERE `client_unique_id` = :uniqueId;",
+            variable{":serverId", serverId},
+            variable{":uniqueId", cl->getUid()},
+            variable{":now", currentTimeSeconds}
+        }.execute();
+
+        if(!sql_result) {
+            logCritical(LOG_INSTANCE, "Failed to execute client server insert command for {}: {}", cl->getUid(), sql_result.fmtStr());
+            return false;
         }
 
-        return assignDatabaseId(sql, id, cl);
+        if(!cl->loadDataForCurrentServer())
+            return false;
+
+        debugMessage(serverId, "Successfully registered client {} for server {} with database id {}.", cl->getUid(), serverId, cl->getClientDatabaseId());
+        return true;
     }
 
-    logTrace(id, "Loaded client from database. Database id: {} Unique id: {}", cl->getClientDatabaseId(), cl->getUid());
     return true;
 }
 
@@ -617,8 +646,8 @@ inline sql::result load_properties(ServerId sid, deque<unique_ptr<FastPropertyEn
             }
         }
 
-        const auto &info = property::impl::info_key(type, key);
-        if(info->name == "undefined") {
+        const auto &info = property::find(type, key);
+        if(info.name == "undefined") {
             logError(sid, "Found unknown property in database! ({})", key);
             return 0;
         }
@@ -630,9 +659,9 @@ inline sql::result load_properties(ServerId sid, deque<unique_ptr<FastPropertyEn
         prop.setDbReference(true);
         */
 
-        auto data = make_unique<FastPropertyEntry>();
+        auto data = std::make_unique<FastPropertyEntry>();
+        data->type = &info;
         data->value = value;
-        data->type = info;
         properties.push_back(move(data));
         return 0;
     });
@@ -705,7 +734,7 @@ std::shared_ptr<Properties> DatabaseHelper::loadServerProperties(const std::shar
             sql = "INSERT INTO `properties` (`serverId`, `type`, `id`, `key`, `value`) VALUES (:sid, :type, :id, :key, :value)";
         }
 
-        logTrace(serverId, "Updating server property: " + prop.type().name + ". New value: " + prop.value() + ". Query: " + sql);
+        logTrace(serverId, "Updating server property: " + std::string{prop.type().name} + ". New value: " + prop.value() + ". Query: " + sql);
         sql::command(this->sql, sql,
                      variable{":sid", serverId},
                      variable{":type", property::PropertyType::PROP_TYPE_SERVER},
@@ -876,6 +905,7 @@ std::shared_ptr<Properties> DatabaseHelper::loadClientProperties(const std::shar
     if(server) {
         props->operator[](property::CLIENT_DESCRIPTION) = server->properties()[property::VIRTUALSERVER_DEFAULT_CLIENT_DESCRIPTION].value();
     }
+
     bool loaded = false;
     if(use_startup_cache && server) {
         shared_ptr<StartupCacheEntry> entry;
@@ -900,6 +930,7 @@ std::shared_ptr<Properties> DatabaseHelper::loadClientProperties(const std::shar
             loaded = true;
         }
     }
+
     if(!loaded) {
         auto command = sql::command(this->sql, "SELECT `key`, `value`, `type` FROM properties WHERE `serverId` = :serverId AND (`type` = :type1 OR `type` = :type2) AND `id` = :id", variable{":serverId", server ? server->getServerId() : 0}, variable{":type1", property::PropertyType::PROP_TYPE_CONNECTION}, variable{":type2", property::PropertyType::PROP_TYPE_CLIENT}, variable{":id", cldbid});
 
@@ -925,7 +956,7 @@ std::shared_ptr<Properties> DatabaseHelper::loadClientProperties(const std::shar
 
         if(!prop.isModified()) return;
         if((prop.type().flags & property::FLAG_SAVE) == 0 && (type != ClientType::CLIENT_MUSIC || (prop.type().flags & property::FLAG_SAVE_MUSIC) == 0)) {
-            logTrace(server ? server->getServerId() : 0, "[Property] Not saving property '" + prop.type().name + "', changed for " + to_string(cldbid) + " (New value: " + prop.value() + ")");
+            logTrace(server ? server->getServerId() : 0, "[Property] Not saving property '" + std::string{prop.type().name} + "', changed for " + to_string(cldbid) + " (New value: " + prop.value() + ")");
             return;
         }
         if(!prop.get_handle()) return;
@@ -933,15 +964,15 @@ std::shared_ptr<Properties> DatabaseHelper::loadClientProperties(const std::shar
         if(!prop.hasDbReference() && (prop.default_value() == prop.value())) return; //No changes to default value
         prop.setModified(false);
 
-        std::string sql;
+        std::string sqlCommand;
         if(prop.hasDbReference())
-            sql = "UPDATE `properties` SET `value` = :value WHERE `serverId` = :serverId AND `type` = :type AND `id` = :id AND `key` = :key";
+            sqlCommand = "UPDATE `properties` SET `value` = :value WHERE `serverId` = :serverId AND `type` = :type AND `id` = :id AND `key` = :key";
         else {
             prop.setDbReference(true);
-            sql = "INSERT INTO `properties` (`serverId`, `type`, `id`, `key`, `value`) VALUES (:serverId, :type, :id, :key, :value)";
+            sqlCommand = "INSERT INTO `properties` (`serverId`, `type`, `id`, `key`, `value`) VALUES (:serverId, :type, :id, :key, :value)";
         }
-        logTrace(server ? server->getServerId() : 0, "[Property] Changed property in db key: " + prop.type().name + " value: " + prop.value());
-        sql::command(this->sql, sql,
+        logTrace(server ? server->getServerId() : 0, "[Property] Changed property in db key: " + std::string{prop.type().name} + " value: " + prop.value());
+        sql::command(this->sql, sqlCommand,
                 variable{":serverId", server ? server->getServerId() : 0},
                 variable{":type", prop.type().type_property},
                 variable{":id", cldbid},
@@ -957,16 +988,62 @@ std::shared_ptr<Properties> DatabaseHelper::loadClientProperties(const std::shar
             return;
         }
 
-        std::string query;
-        if(prop.type() == property::CLIENT_TOTALCONNECTIONS)
-            query = "UPDATE `clients` SET `connections` = :value WHERE `serverId` = :serverId AND `cldbid` = :cldbid";
-        else if(prop.type() == property::CLIENT_NICKNAME)
-            query = "UPDATE `clients` SET `lastName` = :value WHERE `serverId` = :serverId AND `cldbid` = :cldbid";
-        else if(prop.type() == property::CLIENT_LASTCONNECTED)
-            query = "UPDATE `clients` SET `lastConnect` = :value WHERE `serverId` = :serverId AND `cldbid` = :cldbid";
-        if(query.empty()) return;
-        debugMessage(server ? server->getServerId() : 0, "[Property] Changing client property '" + prop.type().name + "' for " + to_string(cldbid) + " (New value: " + prop.value() + ", SQL: " + query + ")");
-        sql::command(this->sql, query, variable{":serverId", server ? server->getServerId() : 0}, variable{":cldbid", cldbid}, variable{":value", prop.value()}).executeLater().waitAndGetLater(LOG_SQL_CMD, {1, "future failed"});
+        std::string column;
+        if(prop.type().type_property == property::PROP_TYPE_CLIENT) {
+            switch (prop.type().property_index) {
+                case property::CLIENT_NICKNAME:
+                    column = "client_nickname";
+                    break;
+
+                case property::CLIENT_LASTCONNECTED:
+                    column = "client_last_connected";
+                    break;
+
+                case property::CLIENT_TOTALCONNECTIONS:
+                    column = "client_total_connections";
+                    break;
+
+                case property::CLIENT_MONTH_BYTES_UPLOADED:
+                    column = "client_month_upload";
+                    break;
+
+                case property::CLIENT_TOTAL_BYTES_UPLOADED:
+                    column = "client_total_upload";
+                    break;
+
+                case property::CLIENT_MONTH_BYTES_DOWNLOADED:
+                    column = "client_month_download";
+                    break;
+
+                case property::CLIENT_TOTAL_BYTES_DOWNLOADED:
+                    column = "client_total_download";
+                    break;
+
+                default:
+                    return;
+            }
+        } else if(prop.type().type_property == property::PROP_TYPE_CONNECTION) {
+            switch (prop.type().property_index) {
+                case property::CONNECTION_CLIENT_IP:
+                    column = "client_ip";
+                    break;
+
+                default:
+                    return;
+            }
+        }
+
+        debugMessage(server ? server->getServerId() : 0, "[Property] Changing client property '{}' for {} (New value: {}, Column: {})",
+            prop.type().name,
+            cldbid,
+            prop.value(),
+            column
+        );
+        sql::command(this->sql, "UPDATE `clients_server` SET `" + column + "` = :value WHERE `server_id` = :serverId AND `client_database_id` = :cldbid",
+                variable{":serverId", server ? server->getServerId() : 0},
+                variable{":cldbid", cldbid},
+                variable{":value", prop.value()}
+        ).executeLater().waitAndGetLater(LOG_SQL_CMD, {1, "future failed"});
     });
 
     return props;
@@ -1013,6 +1090,15 @@ void DatabaseHelper::clearStartupCache(ts::ServerId sid) {
     }
 }
 
+void DatabaseHelper::handleServerDelete(ServerId server_id) {
+    {
+        std::lock_guard pm_lock{this->cached_permission_manager_lock};
+        this->cached_permission_managers.erase(std::remove_if(this->cached_permission_managers.begin(), this->cached_permission_managers.end(), [&](const auto& entry) {
+            return entry->server_id == server_id;
+        }), this->cached_permission_managers.end());
+    }
+}
+
 //SELECT `serverId`, `type`, `id`, `key`, `value` FROM properties ORDER BY `serverId`
 //SELECT `serverId`, `type`, `id`, `channelId`, `permId`, `value`, `grant`, `flag_skip`, `flag_negate` FROM permissions ORDER BY `serverId`
 struct StartupPermissionArgument {
@@ -1119,8 +1205,8 @@ void DatabaseHelper::loadStartupPropertyCache() {
             }
         }
 
-        auto info = property::impl::info_key(type, key);
-        if(info == property::PropertyDescription::unknown) {
+        const auto& info = property::find(type, key);
+        if(info.is_undefined()) {
             logError(serverId, "Invalid property ({} | {})", key, type);
             return 0;
         }
@@ -1145,7 +1231,7 @@ void DatabaseHelper::loadStartupPropertyCache() {
         }
 
         auto entry = make_unique<StartupPropertyEntry>();
-        entry->info = info;
+        entry->info = &info;
         entry->value = value;
         entry->id = id;
         entry->type = type;
@@ -1154,13 +1240,19 @@ void DatabaseHelper::loadStartupPropertyCache() {
     }, &arg);
 }
 
-bool DatabaseHelper::deleteGroupPermissions(const std::shared_ptr<ts::server::VirtualServer> &server, ts::GroupId group_id) {
-    auto command = sql::command(this->sql, "DELETE FROM `permissions` WHERE `serverId` = :serverId AND `type` = :type AND `id` = :id",
-                                variable{":serverId", server ? server->getServerId() : 0},
-                                variable{":type", permission::SQL_PERM_GROUP},
-                                variable{":id", group_id}).execute();
-    LOG_SQL_CMD(command);
-    return !!command;
+void DatabaseHelper::deleteGroupArtifacts(ServerId server_id, GroupId group_id) {
+    sql::result result{};
+
+    result = sql::command(this->sql, "DELETE FROM `permissions` WHERE `serverId` = :serverId AND `type` = :type AND `id` = :id",
+                          variable{":serverId", server_id},
+                          variable{":type", permission::SQL_PERM_GROUP},
+                          variable{":id", group_id}).execute();
+    LOG_SQL_CMD(result);
+
+    result = sql::command(this->sql, "DELETE FROM `tokens` WHERE `serverId` = :serverId AND `targetGroup` = :id",
+                          variable{":serverId", server_id},
+                          variable{":id", group_id}).execute();
+    LOG_SQL_CMD(result);
 }
 
 bool DatabaseHelper::deleteChannelPermissions(const std::shared_ptr<ts::server::VirtualServer> &server, ts::ChannelId channel_id) {
@@ -1201,4 +1293,82 @@ bool DatabaseHelper::deletePlaylist(const std::shared_ptr<ts::server::VirtualSer
     ).executeLater().waitAndGetLater(LOG_SQL_CMD, {-1, "failed to delete playlist properties for playlist " + to_string(playlist_id)});
 
     return true;
+}
+
+constexpr static auto kDBListQuery{R"(
+SELECT `clients`.*, `properties`.`value` as `client_description` FROM (
+    SELECT
+       `clients_server`.`client_database_id`,
+       `clients_server`.`client_unique_id`,
+       `clients_server`.`client_nickname`,
+       `clients_server`.`client_ip`,
+       `clients_server`.`client_created`,
+       `clients_server`.`client_last_connected`,
+       `clients_server`.`client_total_connections`,
+       `clients`.`client_login_name` FROM `clients_server`
+    INNER JOIN `clients` ON `clients`.`client_database_id` = `clients_server`.`client_database_id`
+    WHERE `server_id` = :serverId LIMIT :offset, :limit
+) AS `clients`
+LEFT JOIN `properties` ON `properties`.serverId = :serverId AND `properties`.key = 'client_description' AND `properties`.`id` = `clients`.`client_database_id`
+)"};
+
+void DatabaseHelper::listDatabaseClients(
+        ServerId server_id,
+        const std::optional<int64_t>& offset,
+        const std::optional<int64_t>& limit,
+        void (* callback)(void *, const DatabaseClient &),
+        void *user_argument) {
+
+    DatabaseClient client;
+    size_t set_index{0};
+    auto sqlResult = sql::command{this->sql, kDBListQuery,
+                                  variable{":serverId", server_id},
+                                  variable{":offset", offset.has_value() ? *offset : 0},
+                                  variable{":limit", limit.has_value() ? *limit : -1}
+    }.query([&](int length, std::string* values, std::string* names) {
+        set_index++;
+
+        auto index{0};
+        try {
+            assert(names[index] == "client_database_id");
+            client.client_database_id = std::stoull(values[index++]);
+
+            assert(names[index] == "client_unique_id");
+            client.client_unique_id = values[index++];
+
+            assert(names[index] == "client_nickname");
+            client.client_nickname = values[index++];
+
+            assert(names[index] == "client_ip");
+            client.client_ip = values[index++];
+
+            assert(names[index] == "client_created");
+            client.client_created = values[index++];
+
+            assert(names[index] == "client_last_connected");
+            client.client_last_connected = values[index++];
+
+            assert(names[index] == "client_total_connections");
+            client.client_total_connections = values[index++];
+
+            assert(names[index] == "client_login_name");
+            client.client_login_name = values[index++];
+
+            assert(names[index] == "client_description");
+            client.client_description = values[index++];
+
+            assert(index == length);
+        } catch (std::exception& ex) {
+            logError(server_id, "Failed to parse client base properties at index {}: {}. Offset: {} Limit: {} Set: {}",
+                     index - 1,
+                     ex.what(),
+                     offset.has_value() ? std::to_string(*offset) : "not given",
+                     limit.has_value() ? std::to_string(*limit) : "not given",
+                     set_index - 1
+            );
+            return;
+        }
+
+        callback(user_argument, client);
+    });
 }

server/src/DatabaseHelper.h

@@ -8,125 +8,127 @@
 #include <Properties.h>
 #include <cstdint>
 
-namespace ts {
-    namespace server {
-        class VirtualServer;
-        class DataClient;
+namespace ts::server {
+    class VirtualServer;
+    class DataClient;
 
-        struct ClientDatabaseInfo {
-            ServerId sid;
-            ClientDbId cldbid;
-            std::chrono::time_point<std::chrono::system_clock> created;
-            std::chrono::time_point<std::chrono::system_clock> lastjoin;
-            std::string uniqueId;
-            std::string lastName;
-            uint32_t connections;
-        };
+    struct ClientDatabaseInfo {
+        ServerId server_id;
 
-        struct CachedPermissionManager {
-            ServerId sid;
-            ClientDbId cldbid;
-            std::weak_ptr<permission::v2::PermissionManager> manager;
-            std::shared_ptr<permission::v2::PermissionManager> ownLock;
-            std::chrono::time_point<std::chrono::system_clock> lastAccess;
-        };
+        ClientDbId client_database_id;
+        std::string client_unique_id;
+        std::string client_nickname;
 
-        struct CachedProperties {
-            ServerId sid;
-            ClientDbId cldbid;
-            std::weak_ptr<Properties> properties;
-            std::shared_ptr<Properties> ownLock;
-            std::chrono::time_point<std::chrono::system_clock> lastAccess;
-        };
+        std::chrono::time_point<std::chrono::system_clock> client_created;
+        std::chrono::time_point<std::chrono::system_clock> client_last_connected;
 
-        /*
-        CREATE_TABLE("properties", "`serverId` INTEGER DEFAULT -1, `type` INTEGER, `id` INTEGER, `key` VARCHAR(" UNKNOWN_KEY_LENGTH "), `value` TEXT", command_append_utf8);
-        CREATE_TABLE("permissions", "`serverId` INT NOT NULL, `type` INT, `id` INT, `channelId` INT, `permId` VARCHAR(" UNKNOWN_KEY_LENGTH "), `value` INT, `grant` INT", command_append_utf8);
-         */
-        struct StartupPermissionEntry {
-            permission::PermissionSqlType type = permission::SQL_PERM_CHANNEL;
-            uint64_t id = 0;
-            ChannelId channelId = 0;
-            std::shared_ptr<permission::PermissionTypeEntry> permission = permission::PermissionTypeEntry::unknown;
-            permission::PermissionValue value = 0;
-            permission::PermissionValue grant = 0;
+        uint32_t client_total_connections;
+    };
 
-            bool flag_skip = false;
-            bool flag_negate = false;
-        };
+    struct StartupPermissionEntry {
+        permission::PermissionSqlType type = permission::SQL_PERM_CHANNEL;
+        uint64_t id = 0;
+        ChannelId channelId = 0;
+        std::shared_ptr<permission::PermissionTypeEntry> permission = permission::PermissionTypeEntry::unknown;
+        permission::PermissionValue value = 0;
+        permission::PermissionValue grant = 0;
 
-        struct StartupPropertyEntry {
-            property::PropertyType type = property::PropertyType::PROP_TYPE_UNKNOWN;
-            uint64_t id = 0;
-            std::shared_ptr<property::PropertyDescription> info = property::PropertyDescription::unknown;
-            std::string value;
-        };
+        bool flag_skip = false;
+        bool flag_negate = false;
+    };
 
-        struct StartupCacheEntry {
-            ServerId sid;
+    struct StartupPropertyEntry {
+        property::PropertyType type = property::PropertyType::PROP_TYPE_UNKNOWN;
+        uint64_t id{0};
+        const property::PropertyDescription* info{&property::undefined_property_description};
+        std::string value;
+    };
 
-            std::deque<std::unique_ptr<StartupPermissionEntry>> permissions;
-            std::deque<std::unique_ptr<StartupPropertyEntry>> properties;
-        };
+    struct DatabaseClient {
+        ClientDbId client_database_id;
+        std::string client_unique_id;
 
-        struct FastPropertyEntry {
-            std::shared_ptr<property::PropertyDescription> type;
-            std::string value;
-        };
+        std::string client_nickname;
+        std::string client_ip;
 
-        class DatabaseHelper {
-            public:
-                static std::shared_ptr<Properties> default_properties_client(std::shared_ptr<Properties> /* properties */, ClientType /* type */);
-                static bool assignDatabaseId(sql::SqlManager *, ServerId id, std::shared_ptr<DataClient>);
+        std::string client_created; /* seconds since epoch */
+        std::string client_last_connected; /* seconds since epoch */
+        std::string client_total_connections;
 
-                explicit DatabaseHelper(sql::SqlManager*);
-                ~DatabaseHelper();
+        std::string client_login_name;
+        std::string client_description; /* optional and only given sometimes */
+    };
 
-                void loadStartupCache();
-                size_t cacheBinarySize();
-                void clearStartupCache(ServerId sid = 0);
+    struct FastPropertyEntry {
+        const property::PropertyDescription* type;
+        std::string value;
+    };
 
-                void deleteClient(const std::shared_ptr<VirtualServer>&,ClientDbId);
-                bool validClientDatabaseId(const std::shared_ptr<VirtualServer>&, ClientDbId);
-                std::deque<std::shared_ptr<ClientDatabaseInfo>> queryDatabaseInfo(const std::shared_ptr<VirtualServer>&, const std::deque<ClientDbId>&);
-                std::deque<std::shared_ptr<ClientDatabaseInfo>> queryDatabaseInfoByUid(const std::shared_ptr<VirtualServer> &, std::deque<std::string>);
+    struct CachedPermissionManager;
+    struct StartupCacheEntry;
+    class DatabaseHelper {
+        public:
+            static std::shared_ptr<Properties> default_properties_client(std::shared_ptr<Properties> /* properties */, ClientType /* type */);
+            static bool assignDatabaseId(sql::SqlManager *, ServerId serverId, std::shared_ptr<DataClient>);
 
-                std::shared_ptr<permission::v2::PermissionManager> loadClientPermissionManager(const std::shared_ptr<VirtualServer>&, ClientDbId);
-                void saveClientPermissions(const std::shared_ptr<VirtualServer>&, ClientDbId , const std::shared_ptr<permission::v2::PermissionManager>& /* permission manager */);
+            explicit DatabaseHelper(sql::SqlManager*);
+            ~DatabaseHelper();
 
-                std::shared_ptr<permission::v2::PermissionManager> loadChannelPermissions(const std::shared_ptr<VirtualServer>&, ChannelId);
-                void saveChannelPermissions(const std::shared_ptr<VirtualServer>&, ChannelId, const std::shared_ptr<permission::v2::PermissionManager>& /* permission manager */);
+            void loadStartupCache();
+            size_t cacheBinarySize();
+            void clearStartupCache(ServerId sid = 0);
 
-                std::shared_ptr<permission::v2::PermissionManager> loadGroupPermissions(const std::shared_ptr<VirtualServer>&, GroupId);
-                void saveGroupPermissions(const std::shared_ptr<VirtualServer>&, GroupId, const std::shared_ptr<permission::v2::PermissionManager>& /* permission manager */);
+            void handleServerDelete(ServerId /* server id */);
 
-                std::shared_ptr<permission::v2::PermissionManager> loadPlaylistPermissions(const std::shared_ptr<VirtualServer>&, PlaylistId /* playlist id */);
-                void savePlaylistPermissions(const std::shared_ptr<VirtualServer>&, PlaylistId, const std::shared_ptr<permission::v2::PermissionManager>& /* permission manager */);
+            void listDatabaseClients(
+                    ServerId /* server id */,
+                    const std::optional<int64_t>& offset,
+                    const std::optional<int64_t>& limit,
+                    void(* /* callback */)(void* /* user argument */, const DatabaseClient& /* client */),
+                    void* /* user argument */);
 
-                std::shared_ptr<Properties> loadServerProperties(const std::shared_ptr<VirtualServer>&); //Read and write
-                std::shared_ptr<Properties> loadPlaylistProperties(const std::shared_ptr<VirtualServer>&, PlaylistId); //Read and write
-                std::shared_ptr<Properties> loadChannelProperties(const std::shared_ptr<VirtualServer>&, ChannelId); //Read and write
-                std::shared_ptr<Properties> loadClientProperties(const std::shared_ptr<VirtualServer>&, ClientDbId, ClientType);
+            void deleteClient(const std::shared_ptr<VirtualServer>&,ClientDbId);
+            bool validClientDatabaseId(const std::shared_ptr<VirtualServer>&, ClientDbId);
+            std::deque<std::shared_ptr<ClientDatabaseInfo>> queryDatabaseInfo(const std::shared_ptr<VirtualServer>&, const std::deque<ClientDbId>&);
+            std::deque<std::shared_ptr<ClientDatabaseInfo>> queryDatabaseInfoByUid(const std::shared_ptr<VirtualServer> &, std::deque<std::string>);
 
-                bool deleteGroupPermissions(const std::shared_ptr<VirtualServer>&, GroupId);
-                bool deleteChannelPermissions(const std::shared_ptr<VirtualServer>&, ChannelId);
-                bool deletePlaylist(const std::shared_ptr<VirtualServer>&, PlaylistId /* playlist id */);
-                std::deque<std::unique_ptr<FastPropertyEntry>> query_properties(ServerId /* server */, property::PropertyType /* type */, uint64_t /* id */); /* required for server snapshots */
+            std::shared_ptr<permission::v2::PermissionManager> loadClientPermissionManager(const std::shared_ptr<VirtualServer>&, ClientDbId);
+            void saveClientPermissions(const std::shared_ptr<VirtualServer>&, ClientDbId , const std::shared_ptr<permission::v2::PermissionManager>& /* permission manager */);
 
-                void tick();
-            private:
-                void loadStartupPermissionCache();
-                void loadStartupPropertyCache();
+            std::shared_ptr<permission::v2::PermissionManager> loadChannelPermissions(const std::shared_ptr<VirtualServer>&, ChannelId);
+            void saveChannelPermissions(const std::shared_ptr<VirtualServer>&, ChannelId, const std::shared_ptr<permission::v2::PermissionManager>& /* permission manager */);
 
-                bool use_startup_cache = false;
-                threads::Mutex startup_lock;
-                std::deque<std::shared_ptr<StartupCacheEntry>> startup_entries;
+            std::shared_ptr<permission::v2::PermissionManager> loadGroupPermissions(const std::shared_ptr<VirtualServer>&, GroupId);
+            void saveGroupPermissions(const std::shared_ptr<VirtualServer>&, GroupId, const std::shared_ptr<permission::v2::PermissionManager>& /* permission manager */);
 
-                sql::SqlManager* sql = nullptr;
-                threads::Mutex permManagerLock;
-                std::deque<CachedPermissionManager*> cachedPermissionManagers;
-                threads::Mutex propsLock;
-                std::deque<CachedProperties*> cachedProperties;
-        };
-    }
+            std::shared_ptr<permission::v2::PermissionManager> loadPlaylistPermissions(const std::shared_ptr<VirtualServer>&, PlaylistId /* playlist id */);
+            void savePlaylistPermissions(const std::shared_ptr<VirtualServer>&, PlaylistId, const std::shared_ptr<permission::v2::PermissionManager>& /* permission manager */);
+
+            std::shared_ptr<Properties> loadServerProperties(const std::shared_ptr<VirtualServer>&); //Read and write
+            std::shared_ptr<Properties> loadPlaylistProperties(const std::shared_ptr<VirtualServer>&, PlaylistId); //Read and write
+            std::shared_ptr<Properties> loadChannelProperties(const std::shared_ptr<VirtualServer>&, ChannelId); //Read and write
+            std::shared_ptr<Properties> loadClientProperties(const std::shared_ptr<VirtualServer>&, ClientDbId, ClientType);
+
+            void deleteGroupArtifacts(ServerId, GroupId);
+            bool deleteChannelPermissions(const std::shared_ptr<VirtualServer>&, ChannelId);
+            bool deletePlaylist(const std::shared_ptr<VirtualServer>&, PlaylistId /* playlist id */);
+            std::deque<std::unique_ptr<FastPropertyEntry>> query_properties(ServerId /* server */, property::PropertyType /* type */, uint64_t /* id */); /* required for server snapshots */
+
+            void tick();
+        private:
+            void loadStartupPermissionCache();
+            void loadStartupPropertyCache();
+
+            bool use_startup_cache = false;
+            threads::Mutex startup_lock;
+            std::deque<std::shared_ptr<StartupCacheEntry>> startup_entries;
+
+            sql::SqlManager* sql = nullptr;
+
+            threads::Mutex cached_permission_manager_lock;
+            std::deque<std::unique_ptr<CachedPermissionManager>> cached_permission_managers;
+
+            /* Attention: cached_permission_manager_lock should be locked! */
+            [[nodiscard]] inline std::shared_ptr<permission::v2::PermissionManager> find_cached_permission_manager(ServerId /* server id */, ClientDbId /* client id */);
+    };
 }

server/src/FileServerHandler.cpp

@@ -0,0 +1,207 @@
+//
+// Created by WolverinDEV on 12/05/2020.
+//
+
+#include <files/FileServer.h>
+#include <files/Config.h>
+
+#include "./client/ConnectedClient.h"
+#include "FileServerHandler.h"
+
+using namespace ts::server;
+using namespace ts::server::file;
+
+FileServerHandler::FileServerHandler(ts::server::InstanceHandler *instance) : instance_{instance} {}
+
+bool FileServerHandler::initialize(std::string &error) {
+    if(!file::initialize(error, serverInstance->properties()[property::SERVERINSTANCE_FILETRANSFER_HOST].as<std::string>(), serverInstance->properties()[property::SERVERINSTANCE_FILETRANSFER_PORT].as<uint16_t>()))
+        return false;
+
+
+    file::config::ssl_option_supplier = [&]{
+        return this->instance_->sslManager()->web_ssl_options();
+    };
+
+    auto server = file::server();
+    assert(server);
+
+    auto& transfer = server->file_transfer();
+    transfer.callback_transfer_registered = std::bind(&FileServerHandler::callback_transfer_registered, this, std::placeholders::_1);
+    transfer.callback_transfer_started = std::bind(&FileServerHandler::callback_transfer_started, this, std::placeholders::_1);
+    transfer.callback_transfer_finished = std::bind(&FileServerHandler::callback_transfer_finished, this, std::placeholders::_1);
+
+    transfer.callback_transfer_aborted = std::bind(&FileServerHandler::callback_transfer_aborted, this, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3);
+    transfer.callback_transfer_statistics = std::bind(&FileServerHandler::callback_transfer_statistics, this, std::placeholders::_1, std::placeholders::_2);
+    return true;
+}
+
+void FileServerHandler::finalize() {
+    file::finalize();
+}
+
+void FileServerHandler::callback_transfer_registered(const std::shared_ptr<transfer::Transfer> &transfer) {
+    auto server = this->instance_->getVoiceServerManager()->findServerById(transfer->server->server_id());
+    if(!server) return; /* well that's bad */
+
+    const auto bytes = transfer->expected_file_size - transfer->file_offset;
+    if(transfer->direction == transfer::Transfer::DIRECTION_UPLOAD) {
+        server->properties()[property::VIRTUALSERVER_TOTAL_BYTES_UPLOADED] += bytes;
+        server->properties()[property::VIRTUALSERVER_MONTH_BYTES_UPLOADED] += bytes;
+    } else {
+        server->properties()[property::VIRTUALSERVER_TOTAL_BYTES_DOWNLOADED] += bytes;
+        server->properties()[property::VIRTUALSERVER_MONTH_BYTES_DOWNLOADED] += bytes;
+    }
+
+    auto client = server->find_client_by_id(transfer->client_id);
+    if(client && client->getUid() == transfer->client_unique_id) {
+        if(transfer->direction == transfer::Transfer::DIRECTION_UPLOAD) {
+            client->properties()[property::CLIENT_TOTAL_BYTES_UPLOADED] += bytes;
+            client->properties()[property::CLIENT_MONTH_BYTES_UPLOADED] += bytes;
+        } else {
+            client->properties()[property::CLIENT_MONTH_BYTES_DOWNLOADED] += bytes;
+            client->properties()[property::CLIENT_MONTH_BYTES_DOWNLOADED] += bytes;
+        }
+    }
+}
+
+void FileServerHandler::callback_transfer_aborted(const std::shared_ptr<transfer::Transfer> &transfer,
+                                                  const transfer::TransferStatistics &statistics,
+                                                  const ts::server::file::transfer::TransferError &error) {
+    auto server = this->instance_->getVoiceServerManager()->findServerById(transfer->server->server_id());
+    if(!server) return; /* well that's bad */
+
+    if(statistics.file_total_size < statistics.file_current_offset)
+        return;
+
+    const int64_t bytes_left = statistics.file_total_size - statistics.file_current_offset;
+    if(transfer->direction == transfer::Transfer::DIRECTION_UPLOAD) {
+        server->properties()[property::VIRTUALSERVER_TOTAL_BYTES_UPLOADED] += -bytes_left;
+        server->properties()[property::VIRTUALSERVER_MONTH_BYTES_UPLOADED] += -bytes_left;
+    } else {
+        server->properties()[property::VIRTUALSERVER_TOTAL_BYTES_DOWNLOADED] += -bytes_left;
+        server->properties()[property::VIRTUALSERVER_MONTH_BYTES_DOWNLOADED] += -bytes_left;
+    }
+
+    auto client = server->find_client_by_id(transfer->client_id);
+    if(client && client->getUid() == transfer->client_unique_id) {
+        if(transfer->direction == transfer::Transfer::DIRECTION_UPLOAD) {
+            client->properties()[property::CLIENT_TOTAL_BYTES_UPLOADED] += -bytes_left;
+            client->properties()[property::CLIENT_MONTH_BYTES_UPLOADED] += -bytes_left;
+        } else {
+            client->properties()[property::CLIENT_MONTH_BYTES_DOWNLOADED] += -bytes_left;
+            client->properties()[property::CLIENT_MONTH_BYTES_DOWNLOADED] += -bytes_left;
+        }
+
+        ts::command_builder notify{"notifystatusfiletransfer"};
+
+        notify.put_unchecked(0, "clientftfid", transfer->client_transfer_id);
+        notify.put(0, "size", 0);
+
+        ts::command_result status{};
+        using ErrorType = ts::server::file::transfer::TransferError::Type;
+        switch (error.error_type) {
+            case ErrorType::TRANSFER_TIMEOUT:
+                status.reset(ts::command_result{error::file_transfer_connection_timeout});
+                break;
+
+            case ErrorType::DISK_IO_ERROR:
+            case ErrorType::DISK_TIMEOUT:
+            case ErrorType::DISK_INITIALIZE_ERROR:
+                status.reset(ts::command_result{error::file_io_error});
+                break;
+
+            case ErrorType::UNKNOWN:
+            case ErrorType::NETWORK_IO_ERROR:
+                status.reset(ts::command_result{error::file_connection_lost});
+                break;
+
+            case ErrorType::UNEXPECTED_CLIENT_DISCONNECT:
+            case ErrorType::UNEXPECTED_DISK_EOF:
+                status.reset(ts::command_result{error::file_transfer_interrupted});
+
+            case ErrorType::USER_REQUEST:
+                status.reset(ts::command_result{error::file_transfer_canceled});
+                break;
+        }
+        client->writeCommandResult(notify, status, "status");
+        client->sendCommand(notify);
+    }
+}
+
+void FileServerHandler::callback_transfer_statistics(const std::shared_ptr<transfer::Transfer> &transfer,
+                                                     const ts::server::file::transfer::TransferStatistics &statistics) {
+    auto server = this->instance_->getVoiceServerManager()->findServerById(transfer->server->server_id());
+    if(!server) return; /* well that's bad */
+
+    auto client = server->find_client_by_id(transfer->client_id);
+    if(!client || client->getUid() != transfer->client_unique_id) {
+        /* client not online anymore, but we could still log this as server traffic */
+        if(transfer->direction == transfer::Transfer::DIRECTION_UPLOAD) {
+            server->getServerStatistics()->logFileTransferIn(statistics.delta_file_bytes_transferred);
+        } else {
+            server->getServerStatistics()->logFileTransferOut(statistics.delta_file_bytes_transferred);
+        }
+        return;
+    }
+
+    if(transfer->direction == transfer::Transfer::DIRECTION_UPLOAD) {
+        client->getConnectionStatistics()->logFileTransferIn(statistics.delta_file_bytes_transferred);
+    } else {
+        client->getConnectionStatistics()->logFileTransferOut(statistics.delta_file_bytes_transferred);
+    }
+
+    if(client->getType() == ClientType::CLIENT_TEAMSPEAK)
+        return; /* TS3 does not know this notify */
+
+    ts::command_builder notify{"notifyfiletransferprogress"};
+    notify.put_unchecked(0, "clientftfid", transfer->client_transfer_id);
+
+    notify.put_unchecked(0, "file_bytes_transferred", statistics.file_bytes_transferred);
+    notify.put_unchecked(0, "network_bytes_send", statistics.network_bytes_send);
+    notify.put_unchecked(0, "network_bytes_received", statistics.network_bytes_received);
+
+    notify.put_unchecked(0, "file_start_offset", statistics.file_start_offset);
+    notify.put_unchecked(0, "file_current_offset", statistics.file_current_offset);
+    notify.put_unchecked(0, "file_total_size", statistics.file_total_size);
+
+    notify.put_unchecked(0, "network_current_speed", statistics.current_speed);
+    notify.put_unchecked(0, "network_average_speed", statistics.average_speed);
+
+    client->sendCommand(notify);
+}
+
+void FileServerHandler::callback_transfer_started(const std::shared_ptr<transfer::Transfer> &transfer) {
+    auto server = this->instance_->getVoiceServerManager()->findServerById(transfer->server->server_id());
+    if(!server) return; /* well that's bad */
+
+    auto client = server->find_client_by_id(transfer->client_id);
+    if(!client || client->getUid() != transfer->client_unique_id) return;
+
+
+    ts::command_builder notify{"notifyfiletransferstarted"};
+    notify.put_unchecked(0, "clientftfid", transfer->client_transfer_id);
+    client->sendCommand(notify);
+}
+
+void FileServerHandler::callback_transfer_finished(const std::shared_ptr<transfer::Transfer> &transfer) {
+    auto server = this->instance_->getVoiceServerManager()->findServerById(transfer->server->server_id());
+    if(!server) return; /* well that's bad */
+
+    auto client = server->find_client_by_id(transfer->client_id);
+    if(!client || client->getUid() != transfer->client_unique_id) return;
+
+
+    if(client->getType() == ClientType::CLIENT_TEAMSPEAK)
+        return;
+
+    ts::command_builder notify{"notifystatusfiletransfer"};
+
+    notify.put_unchecked(0, "clientftfid", transfer->client_transfer_id);
+    notify.put(0, "size", transfer->expected_file_size); /* not sure where TeamSpeak counts from */
+    notify.put_unchecked(0, "status", (int) error::file_transfer_complete);
+    notify.put_unchecked(0, "msg", findError(error::file_transfer_complete).message);
+
+    /* TODO: Some stats? */
+
+    client->sendCommand(notify);
+}

server/src/FileServerHandler.h

@@ -0,0 +1,25 @@
+#pragma once
+
+#include <string>
+#include <files/FileServer.h>
+
+#include "./InstanceHandler.h"
+
+namespace ts::server::file {
+    class FileServerHandler {
+        public:
+            explicit FileServerHandler(InstanceHandler*);
+
+            bool initialize(std::string& /* error */);
+            void finalize();
+        private:
+            InstanceHandler* instance_;
+
+            void callback_transfer_registered(const std::shared_ptr<transfer::Transfer>&);
+            void callback_transfer_started(const std::shared_ptr<transfer::Transfer>&);
+            void callback_transfer_finished(const std::shared_ptr<transfer::Transfer>&);
+
+            void callback_transfer_aborted(const std::shared_ptr<transfer::Transfer>&, const transfer::TransferStatistics&, const transfer::TransferError&);
+            void callback_transfer_statistics(const std::shared_ptr<transfer::Transfer>&, const transfer::TransferStatistics&);
+    };
+}

server/src/Group.cpp

@@ -6,7 +6,6 @@
 #include "VirtualServer.h"
 #include "src/client/ConnectedClient.h"
 #include "InstanceHandler.h"
-#include "src/server/file/FileServer.h"
 
 using namespace std;
 using namespace std::chrono;
@@ -93,6 +92,7 @@ GroupManager::GroupManager(const shared_ptr<VirtualServer> &server, sql::SqlMana
 GroupManager::~GroupManager() {}
 
 bool GroupManager::loadGroupFormDatabase(GroupId id) {
+    std::lock_guard glock{this->group_lock};
     if(id == 0){
         this->groups.clear();
 
@@ -116,8 +116,13 @@ bool GroupManager::loadGroupFormDatabase(GroupId id) {
 
 std::vector<std::shared_ptr<Group>> GroupManager::availableGroups(bool root) {
     std::vector<std::shared_ptr<Group>> response;
-    for(const auto& group : this->groups)
-        response.push_back(group);
+
+    {
+        std::lock_guard glock{this->group_lock};
+        for(const auto& group : this->groups)
+            response.push_back(group);
+    }
+
     if(root && this->root){
         auto elm = this->root->availableGroups();
         for(const auto& e : elm)
@@ -128,9 +133,14 @@ std::vector<std::shared_ptr<Group>> GroupManager::availableGroups(bool root) {
 
 std::vector<std::shared_ptr<Group>> GroupManager::availableServerGroups(bool root){
     std::vector<std::shared_ptr<Group>> response;
-    for(const auto& group : this->groups)
-        if(group->target() == GroupTarget::GROUPTARGET_SERVER)
-            response.push_back(group);
+
+    {
+        std::lock_guard glock{this->group_lock};
+        for(const auto& group : this->groups)
+            if(group->target() == GroupTarget::GROUPTARGET_SERVER)
+                response.push_back(group);
+    }
+
     if(root && this->root){
         auto elm = this->root->availableServerGroups();
         for(const auto& e : elm)
@@ -141,9 +151,12 @@ std::vector<std::shared_ptr<Group>> GroupManager::availableServerGroups(bool roo
 
 std::vector<std::shared_ptr<Group>> GroupManager::availableChannelGroups(bool root) {
     std::vector<std::shared_ptr<Group>> response;
-    for(const auto& group : this->groups)
-        if(group->target() == GroupTarget::GROUPTARGET_CHANNEL)
-            response.push_back(group);
+    {
+        std::lock_guard glock{this->group_lock};
+        for(const auto& group : this->groups)
+            if(group->target() == GroupTarget::GROUPTARGET_CHANNEL)
+                response.push_back(group);
+    }
     if(root && this->root){
         auto elm = this->root->availableChannelGroups(true);
         for(const auto& e : elm)
@@ -167,8 +180,7 @@ int GroupManager::insertGroupFromDb(int count, char **values, char **column) {
             groupId = (GroupType) stoll(values[index]);
         else if(strcmp(column[index], "displayName") == 0)
             targetName = values[index];
-        else if(strcmp(column[index], "serverId") == 0);
-        else cerr << "Invalid group table row " << column[index] << endl;
+        //else cerr << "Invalid group table row " << column[index] << endl;
     }
 
     if((size_t) groupId == 0 || (size_t) target == 0xff || (size_t) type == 0xff || targetName.empty()) {
@@ -217,11 +229,13 @@ int GroupManager::insertGroupFromDb(int count, char **values, char **column) {
     debugMessage(this->getServerId(), "Push back group -> " + to_string(group->groupId()) + " - " + group->name());
     this->groups.push_back(group);
 
+#if 0
     auto iconId = (IconId) group->icon_id();
     if(iconId != 0 && serverInstance && serverInstance->getFileServer() && !serverInstance->getFileServer()->iconExists(this->server.lock(), iconId)) {
         logMessage(this->getServerId(), "[FILE] Missing group icon (" + to_string(iconId) + ").");
         if(config::server::delete_missing_icon_permissions) group->permissions()->set_permission(permission::i_icon_id, {0, 0}, permission::v2::delete_value, permission::v2::do_nothing);
     }
+#endif
     return 0;
 }
 
@@ -237,6 +251,7 @@ void GroupManager::handleChannelDeleted(const ChannelId& channel_id) {
 }
 
 bool GroupManager::isLocalGroup(std::shared_ptr<Group> gr) {
+    std::lock_guard glock{this->group_lock};
     return std::find(this->groups.begin(), this->groups.end(), gr) != this->groups.end();
 }
 
@@ -252,9 +267,12 @@ std::shared_ptr<Group> GroupManager::defaultGroup(GroupTarget type, bool enforce
     auto group = this->findGroupLocal(id);
     if(group || enforce_property) return group;
 
-    for(auto elm : this->groups)
-        if(elm->target() == type)
-            return elm;
+    {
+        std::lock_guard glock{this->group_lock};
+        for(auto elm : this->groups)
+            if(elm->target() == type)
+                return elm;
+    }
 
     return nullptr; //Worst case!
 }
@@ -266,6 +284,7 @@ std::shared_ptr<Group> GroupManager::findGroup(GroupId groupId) {
 }
 
 std::shared_ptr<Group> GroupManager::findGroupLocal(GroupId groupId) {
+    std::lock_guard glock{this->group_lock};
     for(const auto& elm : this->groups)
         if(elm->groupId() == groupId) return elm;
     return nullptr;
@@ -273,8 +292,11 @@ std::shared_ptr<Group> GroupManager::findGroupLocal(GroupId groupId) {
 
 std::vector<std::shared_ptr<Group>> GroupManager::findGroup(GroupTarget target, std::string name) {
     vector<shared_ptr<Group>> res;
-    for(const auto &elm : this->groups)
-        if(elm->name() == name && elm->target() == target) res.push_back(elm);
+    {
+        std::lock_guard glock{this->group_lock};
+        for(const auto &elm : this->groups)
+            if(elm->name() == name && elm->target() == target) res.push_back(elm);
+    }
     if(this->root) {
         auto r = root->findGroup(target, name);
         for(const auto &e : r) res.push_back(e);
@@ -305,6 +327,8 @@ std::shared_ptr<Group> GroupManager::createGroup(GroupTarget target, GroupType t
     std::shared_ptr<Group> group = std::make_shared<Group>(this, target, type, groupId);
     group->properties()[property::GROUP_NAME] = name;
     group->setPermissionManager(serverInstance->databaseHelper()->loadGroupPermissions(this->server.lock(), group->groupId()));
+
+    std::lock_guard glock{this->group_lock};
     this->groups.push_back(group);
     return group;
 }
@@ -394,7 +418,10 @@ bool GroupManager::deleteGroup(std::shared_ptr<Group> group) {
         return false;
     }
 
-    this->groups.erase(std::find(this->groups.begin(), this->groups.end(), group));
+    {
+        std::lock_guard glock{this->group_lock};
+        this->groups.erase(std::find(this->groups.begin(), this->groups.end(), group));
+    }
 
     /* erase the group out of our cache */
     {
@@ -424,7 +451,10 @@ bool GroupManager::deleteGroup(std::shared_ptr<Group> group) {
     LOG_SQL_CMD(res);
     flag_sql |= !res;
 
-    flag_sql &= serverInstance->databaseHelper()->deleteGroupPermissions(this->server.lock(), group->groupId());
+    serverInstance->databaseHelper()->deleteGroupArtifacts(this->getServerId(), group->groupId());
+    if(auto server{this->server.lock()}; server)
+        server->getTokenManager()->handleGroupDelete(group->groupId());
+
     if(flag_sql)
         logError(this->getServerId(), "Could not delete group {} ({}) from database. May leader to invalid data", group->name(), group->groupId());
 
@@ -591,41 +621,65 @@ bool GroupManager::isClientCached(const ClientDbId& client_database_id) {
     return this->resolve_cached_client(client_database_id) == nullptr;
 }
 
+constexpr static auto kGroupMemberListQuery{R"(
+    SELECT
+        assignedGroups.cldbid,
+        clients_server.client_unique_id,
+        clients_server.client_nickname,
+        assignedGroups.channelId,
+        assignedGroups.until
+    FROM assignedGroups
+    INNER JOIN clients_server ON
+        clients_server.client_database_id = assignedGroups.cldbid AND clients_server.server_id = :sid
+    WHERE assignedGroups.`serverId` = :sid AND `groupId` = :gid;
+)"};
+
 typedef std::vector<std::shared_ptr<GroupMember>> ResList;
-std::vector<std::shared_ptr<GroupMember>> GroupManager::listGroupMembers(std::shared_ptr<Group> group, bool names) { //TODO juse inner join only on names = true
+std::deque<GroupMember> GroupManager::listGroupMembers(std::shared_ptr<Group> group, bool names) {
     if(!isLocalGroup(group)){
-        if(this->root) return this->root->listGroupMembers(group, names);
+        if(this->root)
+            return this->root->listGroupMembers(group, names);
         return {};
     }
-    ResList result;
 
-    sql::command(this->sql,
-                 "SELECT assignedGroups.cldbid, assignedGroups.channelId, assignedGroups.until, clients.clientUid, clients.lastName FROM assignedGroups INNER JOIN clients ON (clients.cldbid = assignedGroups.cldbid AND clients.serverId = assignedGroups.serverId) WHERE assignedGroups.`serverId` = :sid AND `groupId` = :gid;",
-                 variable{":sid", this->getServerId()}, variable{":gid", group->groupId()})
-            .query([&](ResList* list, int columnCount, char** values, char** columnName){
-        std::shared_ptr<GroupMember> member = std::make_shared<GroupMember>();
-        member->displayName = "undefined";
-        member->uid = "undefined";
-        for(int index = 0; index < columnCount; index++){
-            if(values[index] == nullptr) {
-                logError(this->getServerId(), string() + "Invalid value at " + columnName[index]);
-                continue;
-            }
-            if(strcmp(columnName[index], "cldbid") == 0)
-                member->cldbId = stoll(values[index]);
-            else if(strcmp(columnName[index], "until") == 0)
-                member->until = time_point<system_clock>() + milliseconds(stoll(values[index]));
-            else if(strcmp(columnName[index], "clientUid") == 0)
-                member->uid = values[index];
-            else if(strcmp(columnName[index], "lastName") == 0)
-                member->displayName = values[index];
-            else if(strcmp(columnName[index], "channelId") == 0)
-                member->channelId = stoll(values[index]);
-            else cerr << "Invalid column name " << columnName[index] << endl;
+    std::deque<GroupMember> result{};
+
+    size_t set_index{0};
+    sql::command{this->sql, std::string{kGroupMemberListQuery}, variable{":sid", this->getServerId()}, variable{":gid", group->groupId()}}
+    .query([&](int length, std::string* values, std::string* names) {
+        set_index++;
+
+        auto index{0};
+        try {
+            auto& member = result.emplace_back();
+            assert(names[index] == "cldbid");
+            member.cldbId = std::stoull(values[index++]);
+
+            assert(names[index] == "client_unique_id");
+            member.uid = values[index++];
+
+            assert(names[index] == "client_nickname");
+            member.displayName = values[index++];
+
+            assert(names[index] == "channelId");
+            member.channelId = std::stoull(values[index++]);
+
+            assert(names[index] == "until");
+            member.until = std::chrono::system_clock::time_point{} + std::chrono::milliseconds{std::stoll(values[index++])};
+
+            assert(index == length);
+        } catch (std::exception& ex) {
+            result.pop_back();
+            logError(this->getServerId(), "Failed to parse client group assignment for group {}: {}. Set index: {}, Column: {}",
+                     group->groupId(),
+                     ex.what(),
+                     set_index - 1,
+                     index - 1
+            );
+            return;
         }
-        list->push_back(member);
-        return 0;
-    }, &result);
+    });
+
     return result;
 }
 

server/src/Group.h

@@ -198,7 +198,7 @@ namespace ts {
             bool renameGroup(std::shared_ptr<Group>, std::string);
             bool deleteGroup(std::shared_ptr<Group>);
             bool deleteAllGroups();
-            std::vector<std::shared_ptr<GroupMember>> listGroupMembers(std::shared_ptr<Group>, bool names = false);
+            std::deque<GroupMember> listGroupMembers(std::shared_ptr<Group>, bool names = false);
             std::vector<std::shared_ptr<GroupAssignment>> listGroupAssignments(ClientDbId client);
 
             void cleanupAssignments(ClientDbId);
@@ -215,7 +215,6 @@ namespace ts {
             bool isClientCached(const ClientDbId& /* client database id */);
             void clearCache();
 
-
             bool isLocalGroup(std::shared_ptr<Group>);
         protected:
             void handleChannelDeleted(const ChannelId& /* channel id */);
@@ -225,7 +224,10 @@ namespace ts {
             ServerId getServerId();
 
             sql::SqlManager* sql;
+
+            std::mutex group_lock{};
             std::vector<std::shared_ptr<Group>> groups;
+
             threads::Mutex cacheLock;
             std::vector<std::shared_ptr<CachedClient>> cachedClients;
 

server/src/InstanceHandler.cpp

@@ -3,15 +3,13 @@
 #define XFREE undefined_free
 #define XREALLOC undefined_realloc
 
-#include <netdb.h>
 #include "src/weblist/WebListManager.h"
 #include <log/LogUtils.h>
 #include "InstanceHandler.h"
 #include "src/client/InternalClient.h"
 #include "src/server/QueryServer.h"
-#include "src/server/file/FileServer.h"
-#include "SignalHandler.h"
 #include "src/manager/PermissionNameMapper.h"
+#include "./FileServerHandler.h"
 #include <ThreadPool/Timer.h>
 #include "ShutdownHelper.h"
 #include <sys/utsname.h>
@@ -21,13 +19,15 @@
 #include <misc/hex.h>
 #include <misc/rnd.h>
 #include <misc/strobf.h>
-#include <jemalloc/jemalloc.h>
 #include <protocol/buffers.h>
 
 #ifndef _POSIX_SOURCE
     #define _POSIX_SOURCE
 #endif
 #include <unistd.h>
+#include <files/FileServer.h>
+#include "./manager/ActionLogger.h"
+
 #undef _POSIX_SOURCE
 
 using namespace std;
@@ -41,8 +41,7 @@ extern bool mainThreadActive;
 InstanceHandler::InstanceHandler(SqlDataManager *sql) : sql(sql) {
     serverInstance = this;
     this->tick_manager = make_shared<threads::Scheduler>(config::threads::ticking, "tick task ");
-    this->statistics = make_shared<stats::ConnectionStatistics>(nullptr, true);
-    this->statistics->measure_bandwidths(true);
+    this->statistics = make_shared<stats::ConnectionStatistics>(nullptr);
 
     std::string error_message{};
     this->license_service_ = std::make_shared<license::LicenseService>();
@@ -51,6 +50,13 @@ InstanceHandler::InstanceHandler(SqlDataManager *sql) : sql(sql) {
     }
     this->dbHelper = new DatabaseHelper(this->getSql());
 
+    this->action_logger_ = std::make_unique<log::ActionLogger>();
+    if(!this->action_logger_->initialize(error_message)) {
+        logCritical(LOG_INSTANCE, "Failed to initialize instance action logs: {}", error_message);
+        logCritical(LOG_INSTANCE, "Action log has been disabled.");
+        this->action_logger_->finalize();
+    }
+
     this->_properties = new Properties();
     this->_properties->register_property_type<property::InstanceProperties>();
     this->properties()[property::SERVERINSTANCE_FILETRANSFER_PORT] = ts::config::binding::DefaultFilePort;
@@ -70,8 +76,8 @@ InstanceHandler::InstanceHandler(SqlDataManager *sql) : sql(sql) {
                     }
                 }
 
-                const auto &info = property::impl::info<property::InstanceProperties>(key);
-                if(*info == property::SERVERINSTANCE_UNDEFINED) {
+                const auto &info = property::find<property::InstanceProperties>(key);
+                if(info == property::SERVERINSTANCE_UNDEFINED) {
                     logError(0, "Got an unknown instance property " + key);
                     return 0;
                 }
@@ -282,31 +288,11 @@ bool InstanceHandler::startInstance() {
     }
 
     this->loadWebCertificate();
-    fileServer = new ts::server::FileServer();
-    {
-        auto bindings_string = this->properties()[property::SERVERINSTANCE_FILETRANSFER_HOST].as<string>();
-        auto port = this->properties()[property::SERVERINSTANCE_FILETRANSFER_PORT].as<uint16_t>();
-        auto ft_bindings = net::resolve_bindings(bindings_string, port);
-        deque<shared_ptr<FileServer::Binding>> bindings;
 
-        for(auto& binding : ft_bindings) {
-            if(!get<2>(binding).empty()) {
-                logError(LOG_FT, "Failed to resolve binding for {}: {}", get<0>(binding), get<2>(binding));
-                continue;
-            }
-            auto entry = make_shared<FileServer::Binding>();
-            memcpy(&entry->address, &get<1>(binding), sizeof(sockaddr_storage));
-
-            entry->file_descriptor = -1;
-            entry->event_accept = nullptr;
-            bindings.push_back(entry);
-        }
-
-        logMessage(LOG_FT, "Starting server on {}:{}", bindings_string, port);
-        if(!fileServer->start(bindings, errorMessage)) {
-            logCritical(LOG_FT, "Failed to start server: {}", errorMessage);
-            return false;
-        }
+    this->file_server_handler_ = new file::FileServerHandler{this};
+    if(!this->file_server_handler_->initialize(errorMessage)) {
+        logCritical(LOG_FT, "Failed to initialize server: {}", errorMessage);
+        return false;
     }
 
     if(config::query::sslMode > 0) {
@@ -414,7 +400,7 @@ FwIDAQAB
     startTimestamp = system_clock::now();
     this->voiceServerManager->executeAutostart();
 
-    this->scheduler()->schedule(INSTANCE_TICK_NAME, bind(&InstanceHandler::tickInstance, this), milliseconds(100));
+    this->scheduler()->schedule(INSTANCE_TICK_NAME, bind(&InstanceHandler::tickInstance, this), milliseconds{500});
     return true;
 }
 
@@ -444,14 +430,17 @@ void InstanceHandler::stopInstance() {
     debugMessage(LOG_QUERY, "Query server stopped");
 
     debugMessage(LOG_FT, "Stopping file server");
-    if (this->fileServer) this->fileServer->stop();
-    delete this->fileServer;
-    this->fileServer = nullptr;
+    file::finalize();
     debugMessage(LOG_FT, "File server stopped");
 
     this->save_channel_permissions();
     this->save_group_permissions();
 
+    if(this->file_server_handler_) {
+        this->file_server_handler_->finalize();
+        delete std::exchange(this->file_server_handler_, nullptr);
+    }
+
     delete this->sslMgr;
     this->sslMgr = nullptr;
 
@@ -482,12 +471,12 @@ void InstanceHandler::tickInstance() {
         ALARM_TIMER(t, "InstanceHandler::tickInstance -> flush", milliseconds(5));
         //logger::flush();
     }
-    if(statisticsUpdateTimestamp + seconds(5) < now) {
+    {
+        ALARM_TIMER(t, "InstanceHandler::tickInstance -> statistics tick", milliseconds(5));
+        this->statistics->tick();
+    }
+    if(statisticsUpdateTimestamp + seconds(1) < now) {
         statisticsUpdateTimestamp = now;
-        {
-            ALARM_TIMER(t, "InstanceHandler::tickInstance -> statistics tick", milliseconds(5));
-            this->statistics->tick();
-        }
 
         {
             ALARM_TIMER(t, "InstanceHandler::tickInstance -> statistics tick [monthly]", milliseconds(2));
@@ -520,10 +509,6 @@ void InstanceHandler::tickInstance() {
             }
         }
     }
-    {
-        ALARM_TIMER(t, "InstanceHandler::tickInstance -> fileserver tick", milliseconds(5));
-        if(this->fileServer) this->fileServer->instanceTick();
-    }
     {
         ALARM_TIMER(t, "InstanceHandler::tickInstance -> sql_test tick", milliseconds(5));
         if(this->sql && this->active) {

server/src/InstanceHandler.h

@@ -23,6 +23,14 @@ namespace ts {
             class LicenseService;
         }
 
+        namespace file {
+            class FileServerHandler;
+        }
+
+        namespace log {
+            class ActionLogger;
+        }
+
         class InstanceHandler {
             public:
                 explicit InstanceHandler(SqlDataManager*);
@@ -42,12 +50,13 @@ namespace ts {
                 std::shared_mutex& getChannelTreeLock() { return this->default_tree_lock; }
 
                 VirtualServerManager* getVoiceServerManager(){ return this->voiceServerManager; }
-                FileServer* getFileServer(){ return fileServer; }
                 QueryServer* getQueryServer(){ return queryServer; }
                 DatabaseHelper* databaseHelper(){ return this->dbHelper; }
                 BanManager* banManager(){ return this->banMgr; }
                 ssl::SSLManager* sslManager(){ return this->sslMgr; }
                 sql::SqlManager* getSql(){ return sql->sql(); }
+                log::ActionLogger* action_logger() { return &*this->action_logger_; }
+                file::FileServerHandler* getFileServerHandler() { return this->file_server_handler_; }
 
                 std::chrono::time_point<std::chrono::system_clock> getStartTimestamp(){ return startTimestamp; }
 
@@ -110,12 +119,13 @@ namespace ts {
                 std::chrono::system_clock::time_point memcleanTimestamp;
                 SqlDataManager* sql;
 
-                FileServer* fileServer = nullptr;
                 QueryServer* queryServer = nullptr;
                 VirtualServerManager* voiceServerManager = nullptr;
                 DatabaseHelper* dbHelper = nullptr;
                 BanManager* banMgr = nullptr;
                 ssl::SSLManager* sslMgr = nullptr;
+                file::FileServerHandler* file_server_handler_{nullptr};
+                std::unique_ptr<log::ActionLogger> action_logger_{nullptr};
 
                 ts::Properties* _properties = nullptr;
 

server/src/InstanceHandlerSetup.cpp

@@ -6,7 +6,6 @@
 #include "InstanceHandler.h"
 #include "src/client/InternalClient.h"
 #include "src/server/QueryServer.h"
-#include "src/server/file/FileServer.h"
 
 using namespace std;
 using namespace std::chrono;
@@ -134,8 +133,8 @@ bool InstanceHandler::setupDefaultGroups() {
         }
 
         for(const auto& property : info->properties) {
-            const auto& prop = property::impl::info<property::InstanceProperties>(property);
-            if(*prop == property::SERVERINSTANCE_UNDEFINED) {
+            const auto& prop = property::find<property::InstanceProperties>(property);
+            if(prop.is_undefined()) {
                 logCritical(LOG_INSTANCE, "Invalid template property name: " + property);
             } else {
                 this->properties()[prop] = group->groupId();

server/src/ServerManagerSnapshotDeploy.cpp

@@ -163,6 +163,7 @@ struct SnapshotPermissionEntry {
     }
 };
 
+#if 0
 std::shared_ptr<VirtualServer> VirtualServerManager::createServerFromSnapshot(shared_ptr<VirtualServer> old, std::string host,
                                                                               uint16_t port, const ts::Command &arguments,
                                                                               std::string &error) {
@@ -201,7 +202,7 @@ std::shared_ptr<VirtualServer> VirtualServerManager::createServerFromSnapshot(sh
     auto snapshot_version = arguments[index].has("snapshot_version") ? arguments[index++]["snapshot_version"] : 0;
     debugMessage(0, "Got server snapshot with version {}", snapshot_version);
     while(true){
-        for(auto &key : arguments[index].keys()){
+        for(const auto &key : arguments[index].keys()){
             if(key == "end_virtualserver") continue;
             if(key == "begin_virtualserver") continue;
             if(snapshot_version == 0) {
@@ -559,8 +560,8 @@ std::shared_ptr<VirtualServer> VirtualServerManager::createServerFromSnapshot(sh
                             if(key == "bot_owner_id") continue;
                             if(key == "bot_id") continue;
 
-                            const auto& property = property::info<property::ClientProperties>(key);
-                            if(property->property_index == property::CLIENT_UNDEFINED) {
+                            const auto& property = property::find<property::ClientProperties>(key);
+                            if(property.is_undefined()) {
                                 debugMessage(log_server_id, PREFIX + "Failed to parse give music bot property {} for bot {} (old: {}). Value: {}", key, new_bot_id, bot_id, arguments[index][key].string());
                                 continue;
                             }
@@ -597,8 +598,8 @@ std::shared_ptr<VirtualServer> VirtualServerManager::createServerFromSnapshot(sh
                             if(key == "begin_playlist") continue;
                             if(key == "playlist_id") continue;
 
-                            const auto& property = property::info<property::ClientProperties>(key);
-                            if(property->property_index == property::CLIENT_UNDEFINED) {
+                            const auto& property = property::find<property::ClientProperties>(key);
+                            if(property.is_undefined()) {
                                 debugMessage(log_server_id, PREFIX + "Failed to parse given playlist property {} for playlist {} (old: {}). Value: {}", key, playlist_index, playlist_id, arguments[index][key].string());
                                 continue;
                             }
@@ -692,6 +693,7 @@ std::shared_ptr<VirtualServer> VirtualServerManager::createServerFromSnapshot(sh
     server->ensureValidDefaultGroups();
     return server;
 }
+#endif
 
 struct CommandTuple {
     Command& cmd;
@@ -771,8 +773,8 @@ bool VirtualServerManager::createServerSnapshot(Command &cmd, shared_ptr<Virtual
 
     int index = 0;
 
-    if(version == -1) version = 2; //Auto versioned
-    if(version < 0 || version > 2) {
+    if(version == -1) version = 3; //Auto versioned
+    if(version < 0 || version > 3) {
         error = "Invalid snapshot version!";
         return false;
     }
@@ -787,12 +789,12 @@ bool VirtualServerManager::createServerSnapshot(Command &cmd, shared_ptr<Virtual
                     case property::VIRTUALSERVER_UPLOAD_QUOTA:
                     case property::VIRTUALSERVER_MAX_DOWNLOAD_TOTAL_BANDWIDTH:
                     case property::VIRTUALSERVER_MAX_UPLOAD_TOTAL_BANDWIDTH:
-                        cmd[index][serverProperty.type().name] = (uint64_t) serverProperty.as_save<int64_t>();
+                        cmd[index][std::string{serverProperty.type().name}] = (uint64_t) serverProperty.as_save<int64_t>();
                     default:
                         break;
                 }
             }
-            cmd[index][serverProperty.type().name] = serverProperty.value();
+            cmd[index][std::string{serverProperty.type().name}] = serverProperty.value();
         }
         cmd[index++]["end_virtualserver"] = "";
     }
@@ -807,7 +809,7 @@ bool VirtualServerManager::createServerSnapshot(Command &cmd, shared_ptr<Virtual
                 else if(channelProperty.type() == property::CHANNEL_PID)
                     cmd[index]["channel_pid"] = channelProperty.as<string>();
                 else
-                    cmd[index][channelProperty.type().name] = channelProperty.as<string>();
+                    cmd[index][std::string{channelProperty.type().name}] = channelProperty.as<string>();
             }
             index++;
         }
@@ -817,44 +819,26 @@ bool VirtualServerManager::createServerSnapshot(Command &cmd, shared_ptr<Virtual
     //Clients
     {
         cmd[index]["begin_clients"] = "";
-        CommandTuple parm{cmd, index, version};
-        auto res = sql::command(server->getSql(), "SELECT `cldbid`, `clientUid`, `firstConnect`, `lastName` FROM `clients` WHERE `serverId` = :sid",
-                                variable{":sid", server->getServerId()}
-        ).query([&](CommandTuple* commandIndex, int length, char** value, char** name) {
-            ClientDbId cldbid = 0;
-            int64_t clientCreated = 0;
-            string clientUid;
-            string lastName;
-            string description; //TODO description
 
-            for(int idx = 0; idx < length; idx++) {
-                try {
-                    if(strcmp(name[idx], "cldbid") == 0)
-                        cldbid = value[idx] && strlen(value[idx]) > 0 ? stoul(value[idx]) : 0;
-                    else if(strcmp(name[idx], "clientUid") == 0)
-                        clientUid = value[idx];
-                    else if(strcmp(name[idx], "firstConnect") == 0)
-                        clientCreated = value[idx] && strlen(value[idx]) > 0 ? stoll(value[idx]) : 0L;
-                    else if(strcmp(name[idx], "lastName") == 0)
-                        lastName = value[idx];
-                } catch (std::exception& ex) {
-                    logError(0, "Failed to write snapshot client (Skipping it)! Message: {} @ {} => {}", ex.what(), name[idx], value[idx]);
-                    return 0;
-                }
-            }
+        struct CallbackArgument {
+            Command& command;
+            int& index;
+            int version;
+        };
 
+        CallbackArgument callback_argument{cmd, index, version};
+        this->handle->databaseHelper()->listDatabaseClients(server->getServerId(), std::nullopt, std::nullopt, [](void* ptr_argument, const DatabaseClient& client) {
+            auto argument = (CallbackArgument*) ptr_argument;
 
-            commandIndex->cmd[commandIndex->index]["client_id"] = cldbid;
-            commandIndex->cmd[commandIndex->index]["client_unique_id"] = clientUid;
-            commandIndex->cmd[commandIndex->index]["client_nickname"] = lastName;
-            commandIndex->cmd[commandIndex->index]["client_created"] = clientCreated;
-            commandIndex->cmd[commandIndex->index]["client_description"] = description;
-            if(commandIndex->version == 0)
-                commandIndex->cmd[commandIndex->index]["client_unread_messages"] = 0;
-            commandIndex->index++;
-            return 0;
-        }, &parm);
-        LOG_SQL_CMD(res);
+            argument->command[argument->index]["client_id"] = client.client_database_id;
+            argument->command[argument->index]["client_unique_id"] = client.client_unique_id;
+            argument->command[argument->index]["client_nickname"] = client.client_nickname;
+            argument->command[argument->index]["client_created"] = client.client_created;
+            argument->command[argument->index]["client_description"] = client.client_description;
+            if(argument->version == 0)
+                argument->command[argument->index]["client_unread_messages"] = 0;
+            argument->index++;
+        }, &callback_argument);
         cmd[index++]["end_clients"] = "";
     }
 
@@ -899,7 +883,7 @@ bool VirtualServerManager::createServerSnapshot(Command &cmd, shared_ptr<Virtual
                     if((property->type->flags & (property::FLAG_SAVE_MUSIC | property::FLAG_SAVE)) == 0) continue;
                     if(property->value == property->type->default_value) continue;
 
-                    cmd[index][property->type->name] = property->value;
+                    cmd[index][std::string{property->type->name}] = property->value;
                 }
 
                 index++;
@@ -931,7 +915,7 @@ bool VirtualServerManager::createServerSnapshot(Command &cmd, shared_ptr<Virtual
                     if((property->type->flags & (property::FLAG_SAVE_MUSIC | property::FLAG_SAVE)) == 0) continue;
                     if(property->value == property->type->default_value) continue;
 
-                    cmd[index][property->type->name] = property->value;
+                    cmd[index][std::string{property->type->name}] = property->value;
                 }
 
                 index++;

server/src/ShutdownHelper.cpp

@@ -1,6 +1,7 @@
 #include <log/LogUtils.h>
 #include <StringVariable.h>
 #include <ThreadPool/ThreadHelper.h>
+#include <src/terminal/PipedTerminal.h>
 #include "ShutdownHelper.h"
 #include "InstanceHandler.h"
 
@@ -21,6 +22,7 @@ void ts::server::shutdownInstance(const std::string& message) {
         logCriticalFmt(true, 0, "Could not shutdown server within 30 seconds! (Hangup!)");
         logCriticalFmt(true, 0, "Killing server!");
 
+        terminal::finalize_pipe();
         auto force_kill = std::thread([]{
             threads::self::sleep_for(chrono::seconds(5));
             logCriticalFmt(true, 0, "Failed to exit normally!");
@@ -31,6 +33,7 @@ void ts::server::shutdownInstance(const std::string& message) {
         force_kill.detach();
 
         exit(2);
+        //kill(0, SIGKILL);
     });
     threads::name(hangup_controller, "stop controller");
     hangup_controller.detach();

server/src/SignalHandler.cpp

@@ -8,6 +8,7 @@
 #include <csignal>
 #include <log/LogUtils.h>
 #include <experimental/filesystem>
+#include <src/terminal/PipedTerminal.h>
 
 using namespace std;
 namespace fs = std::experimental::filesystem;
@@ -44,6 +45,8 @@ static bool dumpCallback(const google_breakpad::MinidumpDescriptor& descriptor,
     logCritical(LOG_GENERAL, "Official issue and bug tracker url: https://github.com/TeaSpeak/TeaSpeak/issues");
     logCritical(LOG_GENERAL, "Any reports of crashes are useless if you not provide the above generated crashlog!");
     logCritical(LOG_GENERAL, "Stopping server");
+
+    terminal::finalize_pipe();
     ts::server::shutdownInstance(ts::config::messages::applicationCrashed);
     while(!mainThreadDone) threads::self::sleep_for(chrono::seconds(1));
     return succeeded;

server/src/TS3ServerClientManager.cpp

@@ -6,6 +6,7 @@
 #include <misc/timer.h>
 #include <log/LogUtils.h>
 #include <misc/sassert.h>
+#include <src/manager/ActionLogger.h>
 #include "InstanceHandler.h"
 
 using namespace std;
@@ -323,7 +324,7 @@ void VirtualServer::notify_client_kick(
  *
  * Note: channel cant be a ref because the channel itself gets deleted!
  */
-void VirtualServer::delete_channel(shared_ptr<ts::ServerChannel> channel, const shared_ptr<ConnectedClient> &invoker, const std::string& kick_message, unique_lock<std::shared_mutex> &tree_lock) {
+void VirtualServer::delete_channel(shared_ptr<ts::ServerChannel> channel, const shared_ptr<ConnectedClient> &invoker, const std::string& kick_message, unique_lock<std::shared_mutex> &tree_lock, bool temp_delete) {
     if(!tree_lock.owns_lock())
         tree_lock.lock();
     if(channel->deleted)
@@ -358,7 +359,12 @@ void VirtualServer::delete_channel(shared_ptr<ts::ServerChannel> channel, const
         tree_lock.lock(); /* no clients left within that tree */
     command_locks.clear();
 
-    auto channel_ids = this->channelTree->delete_channel_root(channel);
+    auto deleted_channels = this->channelTree->delete_channel_root(channel);
+    log::ChannelDeleteReason delete_reason{temp_delete ? log::ChannelDeleteReason::EMPTY : log::ChannelDeleteReason::USER_ACTION};
+    for(const auto& deleted_channel : deleted_channels) {
+        serverInstance->action_logger()->channel_logger.log_channel_delete(this->serverId, invoker, deleted_channel->channelId(), channel == deleted_channel ? delete_reason : log::ChannelDeleteReason::PARENT_DELETED);
+    }
+
     this->forEachClient([&](const shared_ptr<ConnectedClient>& client) {
         unique_lock client_channel_lock(client->channel_lock);
         client->notifyChannelDeleted(client->channels->delete_channel_root(channel), invoker);

server/src/TS3ServerHeartbeat.cpp

@@ -87,8 +87,7 @@ void VirtualServer::executeServerTick() {
             if(clientOnline + queryOnline == 0) //We don't need to tick, when server is empty!
                 return;
             properties()[property::VIRTUALSERVER_CHANNELS_ONLINE] = this->channelTree->channel_count();
-            properties()[property::VIRTUALSERVER_TOTAL_PING] = this->averagePing();
-
+            properties()[property::VIRTUALSERVER_TOTAL_PING] = this->generate_network_report().average_ping;
             END_TIMINGS(timing_update_states);
         }
         
@@ -188,7 +187,7 @@ void VirtualServer::executeServerTick() {
                     if(system_clock::now() - last_left < deleteTimeout) continue; //One second stay
                     if(system_clock::now() - channel_created < deleteTimeout + seconds(1)) continue; //One second stay
 
-                    this->delete_channel(server_channel, this->serverRoot, "temporary autodelete", channel_lock);
+                    this->delete_channel(server_channel, this->serverRoot, "temporary auto delete", channel_lock, true);
                     if(channel_lock.owns_lock())
                         channel_lock.unlock();
                 }
@@ -210,20 +209,6 @@ void VirtualServer::executeServerTick() {
             BEGIN_TIMINGS();
 
             this->serverStatistics->tick();
-
-            if(fileStatisticsTimestamp + seconds(5) < system_clock::now()) {
-                fileStatisticsTimestamp = system_clock::now();
-                auto update = this->serverStatistics->mark_file_bytes();
-                if(update.first > 0) {
-                    this->properties()[property::VIRTUALSERVER_MONTH_BYTES_DOWNLOADED] += update.first;
-                    this->properties()[property::VIRTUALSERVER_TOTAL_BYTES_DOWNLOADED] += update.first;
-                }
-                if(update.second > 0) {
-                    this->properties()[property::VIRTUALSERVER_MONTH_BYTES_UPLOADED] += update.second;
-                    this->properties()[property::VIRTUALSERVER_TOTAL_BYTES_UPLOADED] += update.second;
-                }
-            }
-
             {
                 lock_guard<threads::Mutex> lock(this->join_attempts_lock);
                 if(system_clock::now() > this->join_last_decrease + seconds(5)) {

server/src/VirtualServer.cpp

@@ -10,6 +10,8 @@
 #include <misc/digest.h>
 #include <misc/base64.h>
 
+#include <files/FileServer.h>
+
 #include "weblist/WebListManager.h"
 #include "./client/web/WebClient.h"
 #include "./client/voice/VoiceClient.h"
@@ -18,13 +20,13 @@
 #include "./client/query/QueryClient.h"
 #include "music/MusicBotManager.h"
 #include "server/VoiceServer.h"
-#include "server/file/FileServer.h"
 #include "server/QueryServer.h"
 #include "InstanceHandler.h"
 #include "Configuration.h"
 #include "VirtualServer.h"
 #include "src/manager/ConversationManager.h"
 #include <misc/sassert.h>
+#include <src/manager/ActionLogger.h>
 
 using namespace std;
 using namespace std::chrono;
@@ -39,7 +41,6 @@ using namespace ts::buffer;
 #define BUILD_VERSION "Unknown build"
 #endif
 
-extern ts::server::InstanceHandler* serverInstance;
 VirtualServer::VirtualServer(uint16_t serverId, sql::SqlManager* database) : serverId(serverId), sql(database) {
     memtrack::allocated<VirtualServer>(this);
 }
@@ -52,12 +53,19 @@ bool VirtualServer::initialize(bool test_properties) {
         if(prop.type() == property::VIRTUALSERVER_DISABLE_IP_SAVING) {
             this->_disable_ip_saving = prop.as<bool>();
             return;
-        }
-        if(prop.type() == property::VIRTUALSERVER_CODEC_ENCRYPTION_MODE) {
+        } else if(prop.type() == property::VIRTUALSERVER_CODEC_ENCRYPTION_MODE) {
             this->_voice_encryption_mode = prop.as<int>();
             return;
+        } else if(prop.type() == property::VIRTUALSERVER_MAX_UPLOAD_TOTAL_BANDWIDTH) {
+            auto file_vs = file::server()->find_virtual_server(this->getServerId());
+            if(!file_vs) return;
+            file_vs->max_networking_upload_bandwidth(prop.as_save<int64_t>([]{ return -1; }));
+        } else if(prop.type() == property::VIRTUALSERVER_MAX_DOWNLOAD_TOTAL_BANDWIDTH) {
+            auto file_vs = file::server()->find_virtual_server(this->getServerId());
+            if(!file_vs) return;
+            file_vs->max_networking_download_bandwidth(prop.as_save<int64_t>([]{ return -1; }));
         }
-        std::string sql;
+        std::string sql{};
         if(prop.type() == property::VIRTUALSERVER_HOST)
             sql = "UPDATE `servers` SET `host` = :value WHERE `serverId` = :sid";
         else if(prop.type() == property::VIRTUALSERVER_PORT)
@@ -72,6 +80,60 @@ bool VirtualServer::initialize(bool test_properties) {
     this->properties()[property::VIRTUALSERVER_ID] = serverId;
     this->_disable_ip_saving = this->properties()[property::VIRTUALSERVER_DISABLE_IP_SAVING];
 
+    /* initialize logging */
+    {
+        auto server_id = this->serverId;
+        auto sync_property = [server_id](Property& prop) {
+            log::LoggerGroup action_type;
+            switch (prop.type().property_index) {
+                case property::VIRTUALSERVER_LOG_SERVER:
+                    action_type = log::LoggerGroup::SERVER;
+                    break;
+
+                case property::VIRTUALSERVER_LOG_CHANNEL:
+                    action_type = log::LoggerGroup::CHANNEL;
+                    break;
+
+                case property::VIRTUALSERVER_LOG_CLIENT:
+                    action_type = log::LoggerGroup::CLIENT;
+                    break;
+
+                case property::VIRTUALSERVER_LOG_FILETRANSFER:
+                    action_type = log::LoggerGroup::FILES;
+                    break;
+
+                case property::VIRTUALSERVER_LOG_PERMISSIONS:
+                    action_type = log::LoggerGroup::PERMISSION;
+                    break;
+
+                case property::VIRTUALSERVER_LOG_QUERY:
+                    action_type = log::LoggerGroup::QUERY;
+                    break;
+
+                default:
+                    return;
+            }
+
+            serverInstance->action_logger()->toggle_logging_group(server_id, action_type, prop.as_save<bool>([]{ return true; }));
+        };
+
+        for(const property::VirtualServerProperties& property : {
+            property::VIRTUALSERVER_LOG_SERVER,
+            property::VIRTUALSERVER_LOG_CHANNEL,
+            property::VIRTUALSERVER_LOG_CLIENT,
+            property::VIRTUALSERVER_LOG_FILETRANSFER,
+            property::VIRTUALSERVER_LOG_QUERY,
+            property::VIRTUALSERVER_LOG_PERMISSIONS
+        }) {
+            auto prop = this->_properties->find(property::PROP_TYPE_SERVER, property);
+            sync_property(prop);
+        }
+
+        this->_properties->registerNotifyHandler([sync_property](Property& prop){
+            sync_property(prop);
+        });
+    }
+
     if(!properties()[property::VIRTUALSERVER_KEYPAIR].as<std::string>().empty()){
         debugMessage(this->serverId, "Importing server keypair");
         this->_serverKey = new ecc_key;
@@ -129,7 +191,8 @@ bool VirtualServer::initialize(bool test_properties) {
         return false;
     }
 
-    if(channelTree->channel_count() == 0){
+    channelTree->deleteSemiPermanentChannels();
+    if(channelTree->channel_count() == 0) {
         logMessage(this->serverId, "Creating new channel tree (Copy from server 0)");
         LOG_SQL_CMD(sql::command(this->getSql(), "INSERT INTO `channels` (`serverId`, `channelId`, `type`, `parentId`) SELECT :serverId AS `serverId`, `channelId`, `type`, `parentId` FROM `channels` WHERE `serverId` = 0", variable{":serverId", this->serverId}).execute());
         LOG_SQL_CMD(sql::command(this->getSql(), "INSERT INTO `properties` (`serverId`, `type`, `id`, `key`, `value`) SELECT :serverId AS `serverId`, `type`, `id`, `key`, `value` FROM `properties` WHERE `serverId` = 0 AND `type` = :type",
@@ -141,7 +204,7 @@ bool VirtualServer::initialize(bool test_properties) {
         channelTree->loadChannelsFromDatabase();
         if(channelTree->channel_count() == 0){
             logCritical(this->serverId, "Failed to setup channel tree!");
-            return 0;
+            return false;
         }
         if(!channelTree->getDefaultChannel()) {
             logError(this->serverId, "Missing default channel! Using first one!");
@@ -180,7 +243,7 @@ bool VirtualServer::initialize(bool test_properties) {
 
     letters = new letter::LetterManager(this);
 
-    serverStatistics = make_shared<stats::ConnectionStatistics>(serverInstance->getStatistics(), true);
+    serverStatistics = make_shared<stats::ConnectionStatistics>(serverInstance->getStatistics());
 
     this->serverRoot = std::make_shared<InternalClient>(this->sql, self.lock(), this->properties()[property::VIRTUALSERVER_NAME].as<string>(), false);
     static_pointer_cast<InternalClient>(this->serverRoot)->setSharedLock(this->serverRoot);
@@ -195,8 +258,32 @@ bool VirtualServer::initialize(bool test_properties) {
     this->serverAdmin->server = nullptr;
     this->registerInternalClient(this->serverAdmin); /* lets assign server id 0  */
 
-    if(serverInstance->getFileServer())
-        serverInstance->getFileServer()->setupServer(self.lock());
+    {
+        using ErrorType = file::filesystem::ServerCommandErrorType;
+
+        auto file_vs = file::server()->register_server(this->getServerId());
+        auto initialize_result = file::server()->file_system().initialize_server(file_vs);
+        if(!initialize_result->wait_for(std::chrono::seconds{5})) {
+            logError(this->getServerId(), "Failed to wait for file directory initialisation.");
+        } else if(!initialize_result->succeeded()) {
+            switch (initialize_result->error().error_type) {
+                case ErrorType::FAILED_TO_CREATE_DIRECTORIES:
+                    logError(this->getServerId(), "Failed to create server file directories ({}).", initialize_result->error().error_message);
+                    break;
+
+                case ErrorType::UNKNOWN:
+                case ErrorType::FAILED_TO_DELETE_DIRECTORIES:
+                    logError(this->getServerId(), "Failed to initialize server file directory due to an unknown error: {}/{}",
+                            (int) initialize_result->error().error_type, initialize_result->error().error_message);
+                    break;
+            }
+        }
+
+
+        this->properties()[property::VIRTUALSERVER_FILEBASE] = file::server()->file_base_path();
+        file_vs->max_networking_download_bandwidth(this->properties()[property::VIRTUALSERVER_MAX_DOWNLOAD_TOTAL_BANDWIDTH].as_save<int64_t>([]{ return -1; }));
+        file_vs->max_networking_upload_bandwidth(this->properties()[property::VIRTUALSERVER_MAX_UPLOAD_TOTAL_BANDWIDTH].as_save<int64_t>([]{ return -1; }));
+    }
 
     this->channelTree->printChannelTree([&](std::string msg){ debugMessage(this->serverId, msg); });
     this->musicManager = make_shared<music::MusicBotManager>(self.lock());
@@ -204,11 +291,13 @@ bool VirtualServer::initialize(bool test_properties) {
     this->musicManager->load_playlists();
     this->musicManager->load_bots();
 
+#if 0
     if(this->properties()[property::VIRTUALSERVER_ICON_ID] != (IconId) 0)
         if(!serverInstance->getFileServer()->iconExists(self.lock(), this->properties()[property::VIRTUALSERVER_ICON_ID])) {
             debugMessage(this->getServerId(), "Removing invalid icon id of server");
             this->properties()[property::VIRTUALSERVER_ICON_ID] = 0;
-        }
+       }
+#endif
 
     for(const auto& type : vector<property::VirtualServerProperties>{
         property::VIRTUALSERVER_DOWNLOAD_QUOTA,
@@ -216,18 +305,16 @@ bool VirtualServer::initialize(bool test_properties) {
         property::VIRTUALSERVER_MAX_UPLOAD_TOTAL_BANDWIDTH,
         property::VIRTUALSERVER_MAX_DOWNLOAD_TOTAL_BANDWIDTH,
     }) {
-        auto info = property::impl::info(type);
+        const auto& info = property::describe(type);
         auto prop = this->properties()[type];
         if(prop.default_value() == prop.value()) continue;
-        if(!info->validate_input(this->properties()[type].value())) {
-            this->properties()[type] = info->default_value;
-            logMessage(this->getServerId(), "Server property " + info->name + " contains an invalid value! Resetting it.");
+
+        if(!info.validate_input(this->properties()[type].value())) {
+            this->properties()[type] = info.default_value;
+            logMessage(this->getServerId(), "Server property " + std::string{info.name} + " contains an invalid value! Resetting it.");
         }
     }
 
-    if(this->properties()[property::VIRTUALSERVER_FILEBASE].value().empty())
-        this->properties()[property::VIRTUALSERVER_FILEBASE] = serverInstance->getFileServer()->server_file_base(self.lock());
-
     /* lets cleanup the conversations for not existent channels */
     this->_conversation_manager->synchronize_channels();
     return true;
@@ -711,8 +798,8 @@ bool VirtualServer::notifyServerEdited(std::shared_ptr<ConnectedClient> invoker,
     cmd["invokeruid"] = invoker->getUid();
     cmd["reasonid"] = ViewReasonId::VREASON_EDITED;
     for(const auto& key : keys) {
-        auto info = property::impl::info<property::VirtualServerProperties>(key);
-        if(*info == property::VIRTUALSERVER_UNDEFINED) {
+        const auto& info = property::find<property::VirtualServerProperties>(key);
+        if(info == property::VIRTUALSERVER_UNDEFINED) {
             logError(this->getServerId(), "Tried to broadcast a server update with an unknown info: " + key);
             continue;
         }
@@ -724,10 +811,10 @@ bool VirtualServer::notifyServerEdited(std::shared_ptr<ConnectedClient> invoker,
     return true;
 }
 
-bool VirtualServer::notifyClientPropertyUpdates(std::shared_ptr<ConnectedClient> client, const deque<shared_ptr<property::PropertyDescription>>& keys, bool selfNotify) {
-    if(keys.empty()) return false;
+bool VirtualServer::notifyClientPropertyUpdates(std::shared_ptr<ConnectedClient> client, const deque<const property::PropertyDescription*>& keys, bool selfNotify) {
+    if(keys.empty() || !client) return false;
     this->forEachClient([&](const shared_ptr<ConnectedClient>& cl) {
-        shared_lock client_channel_lock(client->channel_lock);
+        shared_lock client_channel_lock(cl->channel_lock);
         if(cl->isClientVisible(client, false) || (cl == client && selfNotify))
             cl->notifyClientUpdated(client, keys, false);
     });
@@ -798,8 +885,6 @@ vector<pair<ts::permission::PermissionType, ts::permission::v2::PermissionFlagge
     bool have_skip_permission = false;
     int skip_permission_type = -1; /* -1 := unset | 0 := skip, not explicit | 1 := skip, explicit */
 
-    bool have_skip;
-
     /*
      * server_group_data[0] := Server group id
      * server_group_data[1] := Skip flag
@@ -843,10 +928,10 @@ vector<pair<ts::permission::PermissionType, ts::permission::v2::PermissionFlagge
         server_group_data_initialized = true;
         active_server_group = nullptr;
 
-        auto groups = cache->getGroupAssignments(this, client_dbid, client_type);
-        server_group_data.resize(groups.size());
+        auto assigned_groups = cache->getGroupAssignments(this, client_dbid, client_type);
+        server_group_data.resize(assigned_groups.size());
         auto it = server_group_data.begin();
-        for(auto& group : groups) {
+        for(auto& group : assigned_groups) {
             auto group_permissions = group->group->permissions();
             auto permission_flags = group_permissions->permission_flags(permission_type);
 
@@ -896,13 +981,6 @@ vector<pair<ts::permission::PermissionType, ts::permission::v2::PermissionFlagge
     };
 
     for(const auto& permission : permissions) {
-        if(permission == permission::b_client_skip_channelgroup_permissions) {
-            if(skip_permission_type == -1) /* initialize skip flag */
-                calculate_skip();
-            result.push_back({permission, {have_skip_permission, skip_permission_type == 1}});
-            continue;
-        }
-
         server_group_data_initialized = false; /* reset all group data */
         auto client_permission_flags = cache->client_permissions->permission_flags(permission);
         /* lets try to resolve the channel specific permission */
@@ -916,27 +994,28 @@ vector<pair<ts::permission::PermissionType, ts::permission::v2::PermissionFlagge
         }
 
 
-        have_skip = channel_id == 0;
-        if(!have_skip) {
+        bool skip_channel_permissions = channel_id == 0;
+        if(!skip_channel_permissions) {
             /* look if somewhere is the skip permission flag set */
             if(skip_permission_type == -1) /* initialize skip flag */
                 calculate_skip();
-            have_skip = have_skip_permission;
+            skip_channel_permissions = have_skip_permission;
         }
-        if(!have_skip) {
+        if(!skip_channel_permissions) {
             /* okey we've no global skip. Then now lookup the groups and the client permissions */
             if(calculate_granted ? client_permission_flags.grant_set : client_permission_flags.value_set) {
                 /* okey the client has the permission, this counts */
-                have_skip = client_permission_flags.skip;
+                skip_channel_permissions = client_permission_flags.skip;
             } else {
                 if(!server_group_data_initialized)
                     initialize_group_data(permission);
+
                 if(active_server_group)
-                    have_skip = std::get<1>(*active_server_group);
+                    skip_channel_permissions = std::get<1>(*active_server_group);
             }
         }
 
-        if(!have_skip) {
+        if(!skip_channel_permissions) {
             /* lookup the channel group */
             {
                 auto channel_assignment = cache->getChannelAssignment(this, client_dbid, channel_id);
@@ -1017,35 +1096,33 @@ bool VirtualServer::verifyServerPassword(std::string password, bool hashed) {
     return password == this->properties()[property::VIRTUALSERVER_PASSWORD].as<std::string>();
 }
 
-float VirtualServer::averagePacketLoss() {
-    //TODO Average packet loss
-    return 0.f;
-}
+VirtualServer::NetworkReport VirtualServer::generate_network_report() {
+    double total_ping{0}, total_loss{0};
+    size_t pings_counted{0}, loss_counted{0};
 
-float VirtualServer::averagePing() {
-    float count = 0;
-    float sum = 0;
-
-    this->forEachClient([&count, &sum](shared_ptr<ConnectedClient> client) {
-        auto type = client->getType();
-        if(type == ClientType::CLIENT_TEAMSPEAK || type == ClientType::CLIENT_TEASPEAK) {
-            count++;
-            sum += duration_cast<milliseconds>(dynamic_pointer_cast<VoiceClient>(client)->calculatePing()).count();
+    this->forEachClient([&](const std::shared_ptr<ConnectedClient>& client) {
+        if(auto vc = dynamic_pointer_cast<VoiceClient>(client); vc) {
+            total_ping += vc->current_ping().count();
+            total_loss += vc->current_packet_loss();
+            pings_counted++;
+            loss_counted++;
         }
 #ifdef COMPILE_WEB_CLIENT
-        else if(type == ClientType::CLIENT_WEB) {
-            count++;
-            sum += duration_cast<milliseconds>(dynamic_pointer_cast<WebClient>(client)->client_ping()).count();
+        else if(client->getType() == ClientType::CLIENT_WEB) {
+            pings_counted++;
+            total_ping += duration_cast<milliseconds>(dynamic_pointer_cast<WebClient>(client)->client_ping()).count();
         }
 #endif
     });
 
-    if(count == 0) return 0;
-    return sum / count;
+    VirtualServer::NetworkReport result{};
+    if(loss_counted) result.average_loss = total_loss / loss_counted;
+    if(pings_counted) result.average_ping = total_ping / pings_counted;
+    return result;
 }
 
 bool VirtualServer::resetPermissions(std::string& token) {
-    LOG_SQL_CMD(sql::command(this->sql, "DELETE FROM `permissions` WHERE `serverId` = :serverId", variable{":serverId", this->serverId}).execute());
+    LOG_SQL_CMD(sql::command(this->sql, "DELETE FROM `permissions` WHERE `serverId` = :serverId AND `type` != :channel_type", variable{":serverId", this->serverId}, variable{":channel_type", permission::SQL_PERM_CHANNEL}).execute());
     LOG_SQL_CMD(sql::command(this->sql, "DELETE FROM `assignedGroups` WHERE `serverId` = :serverId", variable{":serverId", this->serverId}).execute());
     LOG_SQL_CMD(sql::command(this->sql, "DELETE FROM `groups` WHERE `serverId` = :serverId", variable{":serverId", this->serverId}).execute());
 
@@ -1186,7 +1263,15 @@ void VirtualServer::send_text_message(const std::shared_ptr<BasicChannel> &chann
     auto channel_id = channel->channelId();
     auto now = chrono::system_clock::now();
 
-    bool conversation_private = channel->properties()[property::CHANNEL_FLAG_CONVERSATION_PRIVATE].as<bool>();
+    bool conversation_private;
+    auto conversation_mode = channel->properties()[property::CHANNEL_CONVERSATION_MODE].as<ChannelConversationMode>();
+    if(conversation_mode == ChannelConversationMode::CHANNELCONVERSATIONMODE_NONE) {
+        /* nothing to do */
+        return;
+    } else {
+        conversation_private = conversation_mode == ChannelConversationMode::CHANNELCONVERSATIONMODE_PRIVATE;
+    }
+
     auto flag_password = channel->properties()[property::CHANNEL_FLAG_PASSWORD].as<bool>();
     for(const auto& client : this->getClients()) {
         if(client->connectionState() != ConnectionState::CONNECTED)
@@ -1217,4 +1302,41 @@ void VirtualServer::send_text_message(const std::shared_ptr<BasicChannel> &chann
         auto conversation = conversations->get_or_create(channel->channelId());
         conversation->register_message(sender->getClientDatabaseId(), sender->getUid(), sender->getDisplayName(), now, message);
     }
+}
+
+void VirtualServer::update_channel_from_permissions(const std::shared_ptr<BasicChannel> &channel, const std::shared_ptr<ConnectedClient>& issuer) {
+    bool require_view_update;
+    auto property_updates = channel->update_properties_from_permissions(require_view_update);
+
+    if(!property_updates.empty()) {
+        this->forEachClient([&](const std::shared_ptr<ConnectedClient>& cl) {
+            shared_lock client_channel_lock(cl->channel_lock);
+            cl->notifyChannelEdited(channel, property_updates, issuer, false);
+        });
+    }
+
+    if(require_view_update) {
+        auto l_source = this->channelTree->findLinkedChannel(channel->channelId());
+        this->forEachClient([&](const shared_ptr<ConnectedClient>& cl) {
+            /* server tree read lock still active */
+            auto l_target = !cl->currentChannel ? nullptr : cl->server->channelTree->findLinkedChannel(cl->currentChannel->channelId());
+            sassert(l_source);
+            if(cl->currentChannel) sassert(l_target);
+
+            {
+                unique_lock client_channel_lock(cl->channel_lock);
+
+                deque<ChannelId> deleted;
+                for(const auto& [flag_visible, channel] : cl->channels->update_channel(l_source, l_target)) {
+                    if(flag_visible) {
+                        cl->notifyChannelShow(channel->channel(), channel->previous_channel);
+                    } else {
+                        deleted.push_back(channel->channelId());
+                    }
+                }
+                if(!deleted.empty())
+                    cl->notifyChannelHide(deleted, false);
+            }
+        });
+    }
 }

server/src/VirtualServer.h

@@ -58,7 +58,6 @@ namespace ts {
         class InstanceHandler;
         class VoiceServer;
         class QueryServer;
-        class FileServer;
         class SpeakingClient;
 
         class WebControlServer;
@@ -136,6 +135,11 @@ namespace ts {
                 friend class InstanceHandler;
                 friend class VirtualServerManager;
             public:
+                struct NetworkReport {
+                    float average_ping{0};
+                    float average_loss{0};
+                };
+                
                 VirtualServer(ServerId serverId, sql::SqlManager*);
                 ~VirtualServer();
 
@@ -181,12 +185,16 @@ namespace ts {
                 inline ServerChannelTree* getChannelTree(){ return this->channelTree; }
                 inline GroupManager* getGroupManager() { return this->groups; }
 
+                [[nodiscard]] inline auto getTokenManager() -> token::TokenManager* {
+                    return this->tokenManager;
+                }
+
                 bool notifyServerEdited(std::shared_ptr<ConnectedClient>, std::deque<std::string> keys);
-                bool notifyClientPropertyUpdates(std::shared_ptr<ConnectedClient>, const std::deque<std::shared_ptr<property::PropertyDescription>>& keys, bool selfNotify = true); /* execute only with at least channel tree read lock! */
+                bool notifyClientPropertyUpdates(std::shared_ptr<ConnectedClient>, const std::deque<const property::PropertyDescription*>& keys, bool selfNotify = true); /* execute only with at least channel tree read lock! */
                 inline bool notifyClientPropertyUpdates(const std::shared_ptr<ConnectedClient>& client, const std::deque<property::ClientProperties>& keys, bool selfNotify = true) {
                     if(keys.empty()) return false;
-                    std::deque<std::shared_ptr<property::PropertyDescription>> _keys;
-                    for(const auto& key : keys) _keys.push_back(property::impl::info<property::ClientProperties>(key));
+                    std::deque<const property::PropertyDescription*> _keys{};
+                    for(const auto& key : keys) _keys.push_back(&property::describe(key));
                     return this->notifyClientPropertyUpdates(client, _keys, selfNotify);
                 };
 
@@ -230,8 +238,7 @@ namespace ts {
 
                 void testBanStateChange(const std::shared_ptr<ConnectedClient>& invoker);
 
-                float averagePing();
-                float averagePacketLoss();
+                [[nodiscard]] NetworkReport generate_network_report();
 
                 bool resetPermissions(std::string&);
                 void ensureValidDefaultGroups();
@@ -267,13 +274,18 @@ namespace ts {
                         std::shared_ptr<ServerChannel> /* target channel */,
                         const std::shared_ptr<ConnectedClient>& /* invoker */,
                         const std::string& /* kick message */,
-                        std::unique_lock<std::shared_mutex>& /* tree lock */
+                        std::unique_lock<std::shared_mutex>& /* tree lock */,
+                        bool temporary_auto_delete
                 );
 
                 void send_text_message(const std::shared_ptr<BasicChannel>& /* channel */, const std::shared_ptr<ConnectedClient>& /* sender */, const std::string& /* message */);
 
                 inline int voice_encryption_mode() { return this->_voice_encryption_mode; }
                 inline std::shared_ptr<conversation::ConversationManager> conversation_manager() { return this->_conversation_manager; }
+
+                inline auto& get_channel_tree_lock() { return this->channel_tree_lock; }
+
+                void update_channel_from_permissions(const std::shared_ptr<BasicChannel>& /* channel */, const std::shared_ptr<ConnectedClient>& /* issuer */);
             protected:
                 bool registerClient(std::shared_ptr<ConnectedClient>);
                 bool unregisterClient(std::shared_ptr<ConnectedClient>, std::string, std::unique_lock<std::shared_mutex>& channel_tree_lock);

server/src/VirtualServerManager.cpp

@@ -4,16 +4,16 @@
 #include "src/server/VoiceServer.h"
 #include "src/client/query/QueryClient.h"
 #include "InstanceHandler.h"
-#include "src/server/file/FileServer.h"
 #include "src/client/ConnectedClient.h"
 #include <ThreadPool/ThreadHelper.h>
+#include <files/FileServer.h>
 
 using namespace std;
 using namespace std::chrono;
 using namespace ts::server;
 
 VirtualServerManager::VirtualServerManager(InstanceHandler* handle) : handle(handle) {
-    this->puzzles = new protocol::PuzzleManager();
+    this->puzzles = new udp::PuzzleManager{};
     this->handshakeTickers = new threads::Scheduler(1, "handshake ticker");
     this->execute_loop = new event::EventExecutor("executor #");
     //this->join_loop = new event::EventExecutor("joiner #");
@@ -67,7 +67,8 @@ bool VirtualServerManager::initialize(bool autostart) {
     this->state = State::STARTING;
     logMessage(LOG_INSTANCE, "Generating server puzzles...");
     auto start = system_clock::now();
-    this->puzzles->precomputePuzzles(config::voice::DefaultPuzzlePrecomputeSize);
+    if(!this->puzzles->precompute_puzzles(config::voice::DefaultPuzzlePrecomputeSize))
+        logCritical(LOG_INSTANCE, "Failed to precompute RSA puzzles");
     logMessage(LOG_INSTANCE, "Puzzles generated! Time required: " + to_string(duration_cast<milliseconds>(system_clock::now() - start).count()) + "ms");
 
     size_t serverCount = 0;
@@ -109,6 +110,9 @@ bool VirtualServerManager::initialize(bool autostart) {
         if(id == 0) {
             logError(LOG_INSTANCE, "Failed to load virtual server from database. Server id is zero!");
             return 0;
+        } else if(id == 0xFFFF) {
+            /* snapshot server */
+            return 0;
         }
 
         if(host.empty()) {
@@ -200,9 +204,11 @@ shared_ptr<VirtualServer> VirtualServerManager::findServerByPort(uint16_t port)
     return nullptr;
 }
 
-uint16_t VirtualServerManager::next_available_port() {
+uint16_t VirtualServerManager::next_available_port(const std::string& host_string) {
     auto instances = this->serverInstances();
-    deque<uint16_t> unallowed_ports;
+    std::vector<uint16_t> unallowed_ports{};
+    unallowed_ports.reserve(instances.size());
+
     for(const auto& instance : instances) {
         unallowed_ports.push_back(instance->properties()[property::VIRTUALSERVER_PORT].as<uint16_t>());
 
@@ -213,18 +219,39 @@ uint16_t VirtualServerManager::next_available_port() {
             }
         }
     }
+    auto bindings = net::resolve_bindings(host_string, 0);
 
     uint16_t port = config::voice::default_voice_port;
     while(true) {
-        if(port < 1024) goto c;
+        if(port < 1024) goto next_port;
 
         for(auto& p : unallowed_ports) {
             if(p == port)
-                goto c;
+                goto next_port;
+        }
+
+        for(auto& binding : bindings) {
+            if(!std::get<2>(binding).empty()) continue; /* error on that */
+            auto& baddress = std::get<1>(binding);
+            auto& raw_port = baddress.ss_family == AF_INET ? ((sockaddr_in*) &baddress)->sin_port : ((sockaddr_in6*) &baddress)->sin6_port;
+            raw_port = htons(port);
+
+            switch (net::address_available(baddress, net::binding_type::TCP)) {
+                case net::binding_result::ADDRESS_USED:
+                    goto next_port;
+                default:
+                    break; /* if we've an internal error we ignore it */
+            }
+            switch (net::address_available(baddress, net::binding_type::UDP)) {
+                case net::binding_result::ADDRESS_USED:
+                    goto next_port;
+                default:
+                    break; /* if we've an internal error we ignore it */
+            }
         }
         break;
 
-        c:
+        next_port:
         port++;
     }
     return port;
@@ -232,6 +259,7 @@ uint16_t VirtualServerManager::next_available_port() {
 
 ts::ServerId VirtualServerManager::next_available_server_id(bool& success) {
     auto server_id_base = this->handle->properties()[property::SERVERINSTANCE_VIRTUAL_SERVER_ID_INDEX].as<ServerId>();
+    /* ensure we're not using 0xFFFF (This is the snapshot server) */
     if(server_id_base > 65530) {
         success = false;
         return 0;
@@ -316,8 +344,9 @@ shared_ptr<VirtualServer> VirtualServerManager::create_server(std::string hosts,
     if(!sid_success)
         return nullptr;
 
+    this->delete_server_in_db(serverId, false); /* just to ensure */
     sql::command(this->handle->getSql(), "INSERT INTO `servers` (`serverId`, `host`, `port`) VALUES (:sid, :host, :port)", variable{":sid", serverId}, variable{":host", hosts}, variable{":port", port}).executeLater().waitAndGetLater(LOG_SQL_CMD, {1, "future failed"});
-    //`serverId` INTEGER DEFAULT -1, `type` INTEGER, `id` INTEGER, `key` VARCHAR(" UNKNOWN_KEY_LENGTH "), `value` TEXT
+
     auto prop_copy = sql::command(this->handle->getSql(), "INSERT INTO `properties` (`serverId`, `type`, `id`, `key`, `value`) SELECT :target_sid AS `serverId`, `type`, `id`, `key`, `value` FROM `properties` WHERE `type` = :type AND `id` = 0 AND `serverId` = 0;",
                  variable{":target_sid", serverId},
                  variable{":type", property::PROP_TYPE_SERVER}).execute();
@@ -393,21 +422,29 @@ bool VirtualServerManager::deleteServer(shared_ptr<VirtualServer> server) {
     }
 
     this->handle->properties()[property::SERVERINSTANCE_SPOKEN_TIME_DELETED] += server->properties()[property::VIRTUALSERVER_SPOKEN_TIME].as<uint64_t>();
-    sql::command(this->handle->getSql(), "DELETE FROM `tokens` WHERE `serverId` = :sid", variable{":sid", server->getServerId()}).executeLater().waitAndGetLater(LOG_SQL_CMD, {1, "future failed"});
-    sql::command(this->handle->getSql(), "DELETE FROM `properties` WHERE `serverId` = :sid", variable{":sid", server->getServerId()}).executeLater().waitAndGetLater(LOG_SQL_CMD, {1, "future failed"});
-    sql::command(this->handle->getSql(), "DELETE FROM `permissions` WHERE `serverId` = :sid", variable{":sid", server->getServerId()}).executeLater().waitAndGetLater(LOG_SQL_CMD, {1, "future failed"});
-    sql::command(this->handle->getSql(), "DELETE FROM `groups` WHERE `serverId` = :sid", variable{":sid", server->getServerId()}).executeLater().waitAndGetLater(LOG_SQL_CMD, {1, "future failed"});
-    sql::command(this->handle->getSql(), "DELETE FROM `clients` WHERE `serverId` = :sid", variable{":sid", server->getServerId()}).executeLater().waitAndGetLater(LOG_SQL_CMD, {1, "future failed"});
-    sql::command(this->handle->getSql(), "DELETE FROM `channels` WHERE `serverId` = :sid", variable{":sid", server->getServerId()}).executeLater().waitAndGetLater(LOG_SQL_CMD, {1, "future failed"});
-    sql::command(this->handle->getSql(), "DELETE FROM `bannedClients` WHERE `serverId` = :sid", variable{":sid", server->getServerId()}).executeLater().waitAndGetLater(LOG_SQL_CMD, {1, "future failed"});
-    sql::command(this->handle->getSql(), "DELETE FROM `assignedGroups` WHERE `serverId` = :sid", variable{":sid", server->getServerId()}).executeLater().waitAndGetLater(LOG_SQL_CMD, {1, "future failed"});
-    sql::command(this->handle->getSql(), "DELETE FROM `servers` WHERE `serverId` = :sid", variable{":sid", server->getServerId()}).executeLater().waitAndGetLater(LOG_SQL_CMD, {1, "future failed"});
-    sql::command(this->handle->getSql(), "DELETE FROM `musicbots` WHERE `serverId` = :sid", variable{":sid", server->getServerId()}).executeLater().waitAndGetLater(LOG_SQL_CMD, {1, "future failed"});
+    this->delete_server_in_db(server->serverId, false);
+    this->handle->databaseHelper()->handleServerDelete(server->serverId);
 
-    sql::command(this->handle->getSql(), "DELETE FROM `bannedClients` WHERE `serverId` = :sid", variable{":sid", server->getServerId()}).executeLater().waitAndGetLater(LOG_SQL_CMD, {1, "future failed"});
-    sql::command(this->handle->getSql(), "DELETE FROM `ban_trigger` WHERE `server_id` = :sid", variable{":sid", server->getServerId()}).executeLater().waitAndGetLater(LOG_SQL_CMD, {1, "future failed"});
-    this->handle->getFileServer()->deleteServer(server);
+    if(auto file_vs = file::server()->find_virtual_server(server->getServerId()); file_vs) {
+        using ErrorType = file::filesystem::ServerCommandErrorType;
 
+        auto delete_result = file::server()->file_system().delete_server(file_vs);
+        if(!delete_result->wait_for(std::chrono::seconds{5})) {
+            logError(LOG_INSTANCE, "Failed to wait for file directory initialisation.");
+        } else if(!delete_result->succeeded()) {
+            switch (delete_result->error().error_type) {
+                case ErrorType::FAILED_TO_DELETE_DIRECTORIES:
+                    logError(LOG_INSTANCE, "Failed to delete server {} file directories ({}).", server->getServerId(), delete_result->error().error_message);
+                    break;
+
+                case ErrorType::UNKNOWN:
+                case ErrorType::FAILED_TO_CREATE_DIRECTORIES:
+                    logError(LOG_INSTANCE, "Failed to delete server {} file directory due to an unknown error: {}/{}",
+                             server->getServerId(), (int) delete_result->error().error_type, delete_result->error().error_message);
+                    break;
+            }
+        }
+    }
     return true;
 }
 
@@ -446,4 +483,72 @@ void VirtualServerManager::tickHandshakeClients() {
         if(vserver)
             vserver->tickHandshakingClients();
     }
+}
+
+void VirtualServerManager::delete_server_in_db(ts::ServerId server_id, bool data_only) {
+#define execute_delete(statement) \
+result = sql::command(this->handle->getSql(), statement, variable{":sid", server_id}).execute(); \
+if(!result) { \
+    logWarning(LOG_INSTANCE, "Failed to execute SQL command {}: {}", statement, result.fmtStr()); \
+    result = sql::result{}; \
+}
+
+    sql::result result{};
+
+    if(!data_only) {
+        execute_delete("DELETE FROM `servers` WHERE `serverId` = :sid");
+    }
+
+    execute_delete("DELETE FROM `tokens` WHERE `serverId` = :sid");
+    execute_delete("DELETE FROM `properties` WHERE `serverId` = :sid");
+    execute_delete("DELETE FROM `permissions` WHERE `serverId` = :sid");
+    execute_delete("DELETE FROM `channels` WHERE `serverId` = :sid");
+    execute_delete("DELETE FROM `bannedClients` WHERE `serverId` = :sid");
+    execute_delete("DELETE FROM `groups` WHERE `serverId` = :sid");
+    execute_delete("DELETE FROM `assignedGroups` WHERE `serverId` = :sid");
+    execute_delete("DELETE FROM `complains` WHERE `serverId` = :sid");
+    execute_delete("DELETE FROM `letters` WHERE `serverId` = :sid");
+
+    execute_delete("DELETE FROM `musicbots` WHERE `serverId` = :sid");
+
+    execute_delete("DELETE FROM `playlists` WHERE `serverId` = :sid");
+    execute_delete("DELETE FROM `playlist_songs` WHERE `serverId` = :sid");
+
+    execute_delete("DELETE FROM `conversations` WHERE `server_id` = :sid");
+    execute_delete("DELETE FROM `conversation_blocks` WHERE `server_id` = :sid");
+    execute_delete("DELETE FROM `ban_trigger` WHERE `server_id` = :sid");
+    execute_delete("DELETE FROM `clients_server` WHERE `server_id` = :sid");
+}
+
+#define execute_change(table, column) \
+result = sql::command(this->handle->getSql(), "UPDATE `" table "` SET `" column "` = :nsid WHERE `" column "` = :osid;", \
+                        variable{":osid", old_id}, variable{":nsid", new_id}).execute(); \
+if(!result) { \
+    logWarning(LOG_INSTANCE, "Failed to execute server id change on table {} (column {}): {}", table, column, result.fmtStr()); \
+    result = sql::result{}; \
+}
+
+void VirtualServerManager::change_server_id_in_db(ts::ServerId old_id, ts::ServerId new_id) {
+    sql::result result{};
+
+
+    execute_change("tokens", "serverId");
+    execute_change("properties", "serverId");
+    execute_change("permissions", "serverId");
+    execute_change("channels", "serverId");
+    execute_change("bannedClients", "serverId");
+    execute_change("groups", "serverId");
+    execute_change("assignedGroups", "serverId");
+    execute_change("complains", "serverId");
+    execute_change("letters", "serverId");
+    execute_change("musicbots", "serverId");
+    execute_change("playlists", "serverId");
+    execute_change("playlist_songs", "serverId");
+
+    execute_change("conversations", "server_id");
+    execute_change("conversation_blocks", "server_id");
+    execute_change("ban_trigger", "server_id");
+    execute_change("clients_server", "server_id");
+
+    execute_change("servers", "serverId");
 }

server/src/VirtualServerManager.h

@@ -2,98 +2,114 @@
 
 #include <deque>
 #include <EventLoop.h>
-#include "client/voice/PrecomputedPuzzles.h"
+#include "src/server/PrecomputedPuzzles.h"
 #include "server/VoiceIOManager.h"
 #include "VirtualServer.h"
+#include <query/command3.h>
+#include "snapshots/snapshot.h"
 
-namespace ts {
-    namespace server {
-        class InstanceHandler;
+namespace ts::server {
+    class InstanceHandler;
 
-        struct ServerReport {
-            size_t avariable;
-            size_t online;
+    struct ServerReport {
+        size_t avariable;
+        size_t online;
 
-            size_t slots;
-            size_t onlineClients;
-            size_t onlineChannels;
-        };
-        class VirtualServerManager {
-            public:
-                enum State {
-                    STOPPED,
-                    STARTING,
-                    STARTED,
-                    STOPPING
-                };
+        size_t slots;
+        size_t onlineClients;
+        size_t onlineChannels;
+    };
+    class VirtualServerManager {
+        public:
+            enum State {
+                STOPPED,
+                STARTING,
+                STARTED,
+                STOPPING
+            };
 
-                explicit VirtualServerManager(InstanceHandler*);
-                ~VirtualServerManager();
+            enum struct SnapshotDeployResult {
+                SUCCESS,
 
-                bool initialize(bool execute_autostart = true);
+                REACHED_SOFTWARE_SERVER_LIMIT,
+                REACHED_CONFIG_SERVER_LIMIT,
+                REACHED_SERVER_ID_LIMIT,
 
-                std::shared_ptr<VirtualServer> create_server(std::string hosts, uint16_t port);
-                bool deleteServer(std::shared_ptr<VirtualServer>);
+                CUSTOM_ERROR /* error message set */
+            };
 
-                std::shared_ptr<VirtualServer> findServerById(ServerId);
-                std::shared_ptr<VirtualServer> findServerByPort(uint16_t);
-                uint16_t next_available_port();
-                ServerId next_available_server_id(bool& /* success */);
-                
-                std::deque<std::shared_ptr<VirtualServer>> serverInstances(){
-                    threads::MutexLock l(this->instanceLock);
-                    return instances;
-                }
+            explicit VirtualServerManager(InstanceHandler*);
+            ~VirtualServerManager();
 
-                ServerReport report();
-                OnlineClientReport clientReport();
-                size_t runningServers();
-                size_t usedSlots();
+            bool initialize(bool execute_autostart = true);
 
-                void executeAutostart();
-                void shutdownAll(const std::string&);
+            std::shared_ptr<VirtualServer> create_server(std::string hosts, uint16_t port);
+            bool deleteServer(std::shared_ptr<VirtualServer>);
 
-                //Dotn use shared_ptr references to keep sure that they be hold in memory
-                bool createServerSnapshot(Command &cmd, std::shared_ptr<VirtualServer> server, int version, std::string &error);
-                std::shared_ptr<VirtualServer> createServerFromSnapshot(std::shared_ptr<VirtualServer> old, std::string, uint16_t, const ts::Command &, std::string &);
+            std::shared_ptr<VirtualServer> findServerById(ServerId);
+            std::shared_ptr<VirtualServer> findServerByPort(uint16_t);
+            uint16_t next_available_port(const std::string& /* host string */);
+            ServerId next_available_server_id(bool& /* success */);
 
-                protocol::PuzzleManager* rsaPuzzles() { return this->puzzles; }
+            std::deque<std::shared_ptr<VirtualServer>> serverInstances(){
+                threads::MutexLock l(this->instanceLock);
+                return instances;
+            }
 
-                event::EventExecutor* get_join_loop() { return this->join_loop; }
-                event::EventExecutor* get_executor_loop() { return this->execute_loop; }
+            ServerReport report();
+            OnlineClientReport clientReport();
+            size_t runningServers();
+            size_t usedSlots();
 
-                inline void adjust_executor_threads() {
-                    std::unique_lock instance_lock(this->instanceLock);
-                    auto instance_count = this->instances.size();
-                    instance_lock.unlock();
+            void executeAutostart();
+            void shutdownAll(const std::string&);
 
-                    auto threads = std::min(config::threads::voice::execute_per_server * instance_count, config::threads::voice::execute_limit);
-                    this->execute_loop->threads(threads);
-                }
-                io::VoiceIOManager* ioManager(){ return this->_ioManager; }
+            //Don't use shared_ptr references to keep sure that they be hold in memory
+            bool createServerSnapshot(Command &cmd, std::shared_ptr<VirtualServer> server, int version, std::string &error);
+            SnapshotDeployResult deploy_snapshot(std::string& /* error */, std::shared_ptr<VirtualServer>& /* target server */, const command_parser& /* source */);
 
-                threads::Mutex server_create_lock;
+            udp::PuzzleManager* rsaPuzzles() { return this->puzzles; }
 
-                State getState() { return this->state; }
-            private:
-                State state = State::STOPPED;
-                InstanceHandler* handle;
-                threads::Mutex instanceLock;
-                std::deque<std::shared_ptr<VirtualServer>> instances;
-                protocol::PuzzleManager* puzzles = nullptr;
+            event::EventExecutor* get_join_loop() { return this->join_loop; }
+            event::EventExecutor* get_executor_loop() { return this->execute_loop; }
 
-                event::EventExecutor* execute_loop = nullptr;
-                event::EventExecutor* join_loop = nullptr;
-                threads::Scheduler* handshakeTickers = nullptr;
-                io::VoiceIOManager* _ioManager = nullptr;
+            inline void adjust_executor_threads() {
+                std::unique_lock instance_lock(this->instanceLock);
+                auto instance_count = this->instances.size();
+                instance_lock.unlock();
 
-                struct {
-                    std::thread executor{};
-                    std::condition_variable condition;
-                    std::mutex lock;
-                } acknowledge;
+                auto threads = std::min(config::threads::voice::execute_per_server * instance_count, config::threads::voice::execute_limit);
+                this->execute_loop->threads(threads);
+            }
+            io::VoiceIOManager* ioManager(){ return this->_ioManager; }
 
-                void tickHandshakeClients();
-        };
-    }
+            /* This must be recursive */
+            threads::Mutex server_create_lock;
+
+            State getState() { return this->state; }
+        private:
+            State state = State::STOPPED;
+            InstanceHandler* handle;
+            threads::Mutex instanceLock;
+            std::deque<std::shared_ptr<VirtualServer>> instances;
+            udp::PuzzleManager* puzzles{nullptr};
+
+            event::EventExecutor* execute_loop = nullptr;
+            event::EventExecutor* join_loop = nullptr;
+            threads::Scheduler* handshakeTickers = nullptr;
+            io::VoiceIOManager* _ioManager = nullptr;
+
+            struct {
+                std::thread executor{};
+                std::condition_variable condition;
+                std::mutex lock;
+            } acknowledge;
+
+            void tickHandshakeClients();
+
+            void delete_server_in_db(ServerId /* server id */, bool /* data only */);
+            void change_server_id_in_db(ServerId /* old id */, ServerId /* new id */);
+
+            bool try_deploy_snapshot(std::string& /* error */, ServerId /* target server id */, ServerId /* logging server id */, const command_parser& /* source */);
+    };
 }

server/src/absl/btree/map.h

@@ -0,0 +1,280 @@
+/*
+ * Copyright (c) 2019 German Mendez Bravo (Kronuz)
+ * Copyright (c) 2013 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ *
+ * A btree::map<> implements the STL unique sorted associative container
+ * interface and the pair associative container interface (a.k.a map<>) using a
+ * btree. See btree.h for details of the btree implementation and caveats.
+ */
+
+#ifndef BTREE_MAP_H__
+#define BTREE_MAP_H__
+
+#include "btree.h"
+
+#include <stdexcept>
+
+namespace btree {
+
+// A common base class for map and safe_map.
+template <typename Tree>
+class btree_map_container : public btree_unique_container<Tree> {
+	typedef btree_map_container<Tree> self_type;
+	typedef btree_unique_container<Tree> super_type;
+
+public:
+	typedef typename Tree::key_type key_type;
+	typedef typename Tree::data_type data_type;
+	typedef typename Tree::value_type value_type;
+	typedef typename Tree::mapped_type mapped_type;
+	typedef typename Tree::key_compare key_compare;
+	typedef typename Tree::allocator_type allocator_type;
+	typedef typename Tree::iterator iterator;
+	typedef typename Tree::const_iterator const_iterator;
+
+public:
+	// Default constructor.
+	btree_map_container(const key_compare& comp = key_compare(),
+				  const allocator_type& alloc = allocator_type())
+			: super_type(comp, alloc) {
+	}
+
+	// Copy constructor.
+	btree_map_container(const self_type& x)
+			: super_type(x) {
+	}
+
+	// Range constructor.
+	template <class InputIterator>
+	btree_map_container(InputIterator b, InputIterator e,
+				  const key_compare& comp = key_compare(),
+				  const allocator_type& alloc = allocator_type())
+			: super_type(b, e, comp, alloc) {
+	}
+
+	template <typename... Args>
+	std::pair<iterator, bool> try_emplace(const key_type& key, Args&&... args) {
+		return this->__tree.emplace_unique_key_args(key,
+			std::piecewise_construct,
+			std::forward_as_tuple(key),
+			std::forward_as_tuple(std::forward<Args>(args)...));
+	}
+
+	template <typename... Args>
+	std::pair<iterator, bool> try_emplace(key_type&& key, Args&&... args) {
+		return this->__tree.emplace_unique_key_args(key,
+			std::piecewise_construct,
+			std::forward_as_tuple(std::move(key)),
+			std::forward_as_tuple(std::forward<Args>(args)...));
+	}
+
+	template <typename... Args>
+	iterator try_emplace(const_iterator hint, const key_type& key, Args&&... args) {
+		return this->__tree.emplace_hint_unique_key_args(hint, key,
+			std::piecewise_construct,
+			std::forward_as_tuple(key),
+			std::forward_as_tuple(std::forward<Args>(args)...));
+	}
+
+	template <typename... Args>
+	iterator try_emplace(const_iterator hint, key_type&& key, Args&&... args) {
+		return this->__tree.emplace_hint_unique_key_args(hint, key,
+			std::piecewise_construct,
+			std::forward_as_tuple(std::move(key)),
+			std::forward_as_tuple(std::forward<Args>(args)...));
+	}
+
+	// Access specified element with bounds checking.
+	mapped_type& at(const key_type& key) {
+		auto it = this->find(key);
+		if (it == this->end()) {
+			throw std::out_of_range("map::at:  key not found");
+		}
+		return it->second;
+	}
+	const mapped_type& at(const key_type& key) const {
+		auto it = this->find(key);
+		if (it == this->end()) {
+			throw std::out_of_range("map::at:  key not found");
+		}
+		return it->second;
+	}
+
+	// Insertion routines.
+	data_type& operator[](const key_type& key) {
+		return this->try_emplace(key).first->second;
+	}
+
+	data_type& operator[](key_type&& key) {
+		return this->try_emplace(std::move(key)).first->second;
+	}
+};
+
+// The map class is needed mainly for its constructors.
+template <typename Key, typename Value,
+	typename Compare = std::less<Key>,
+	typename Alloc = std::allocator<std::pair<const Key, Value>>,
+	int TargetNodeSize = 256>
+class map : public btree_map_container<
+	btree<btree_map_params<Key, Value, Compare, Alloc, TargetNodeSize>>> {
+
+	typedef map<Key, Value, Compare, Alloc, TargetNodeSize> self_type;
+	typedef btree_map_params<Key, Value, Compare, Alloc, TargetNodeSize> params_type;
+	typedef btree<params_type> btree_type;
+	typedef btree_map_container<btree_type> super_type;
+
+public:
+	typedef typename btree_type::key_compare key_compare;
+	typedef typename btree_type::allocator_type allocator_type;
+
+public:
+	// Default constructor.
+	map(const key_compare& comp = key_compare(),
+		const allocator_type& alloc = allocator_type())
+		: super_type(comp, alloc) {
+	}
+
+	// Copy constructor.
+	map(const self_type& x)
+		: super_type(x) {
+	}
+
+	// Range constructor.
+	template <class InputIterator>
+	map(InputIterator b, InputIterator e,
+		const key_compare& comp = key_compare(),
+		const allocator_type& alloc = allocator_type())
+		: super_type(b, e, comp, alloc) {
+	}
+};
+
+} // namespace btree
+
+template <typename K, typename V, typename C, typename A, int N>
+bool operator==(const btree::map<K, V, C, A, N>& lhs, const btree::map<K, V, C, A, N>& rhs) {
+	return lhs.size() == rhs.size() && std::equal(lhs.begin(), lhs.end(), rhs.begin());
+}
+
+template <typename K, typename V, typename C, typename A, int N>
+bool operator<(const btree::map<K, V, C, A, N>& lhs, const btree::map<K, V, C, A, N>& rhs) {
+	return std::lexicographical_compare(lhs.begin(), lhs.end(), rhs.begin(), rhs.end());
+}
+
+template <typename K, typename V, typename C, typename A, int N>
+bool operator!=(const btree::map<K, V, C, A, N>& lhs, const btree::map<K, V, C, A, N>& rhs) {
+	return !(lhs == rhs);
+}
+
+template <typename K, typename V, typename C, typename A, int N>
+bool operator>(const btree::map<K, V, C, A, N>& lhs, const btree::map<K, V, C, A, N>& rhs) {
+	return rhs < lhs;
+}
+
+template <typename K, typename V, typename C, typename A, int N>
+bool operator>=(const btree::map<K, V, C, A, N>& lhs, const btree::map<K, V, C, A, N>& rhs) {
+	return !(lhs < rhs);
+}
+
+template <typename K, typename V, typename C, typename A, int N>
+bool operator<=(const btree::map<K, V, C, A, N>& lhs, const btree::map<K, V, C, A, N>& rhs) {
+	return !(rhs < lhs);
+}
+
+template <typename K, typename V, typename C, typename A, int N>
+inline void swap(btree::map<K, V, C, A, N>& x, btree::map<K, V, C, A, N>& y) {
+	x.swap(y);
+}
+
+namespace btree {
+
+// The multimap class is needed mainly for its constructors.
+template <typename Key, typename Value,
+	typename Compare = std::less<Key>,
+	typename Alloc = std::allocator<std::pair<const Key, Value> >,
+	int TargetNodeSize = 256>
+class multimap : public btree_multi_container<
+	btree<btree_map_params<Key, Value, Compare, Alloc, TargetNodeSize> > > {
+
+	typedef multimap<Key, Value, Compare, Alloc, TargetNodeSize> self_type;
+	typedef btree_map_params< Key, Value, Compare, Alloc, TargetNodeSize> params_type;
+	typedef btree<params_type> btree_type;
+	typedef btree_multi_container<btree_type> super_type;
+
+ public:
+	typedef typename btree_type::key_compare key_compare;
+	typedef typename btree_type::allocator_type allocator_type;
+	typedef typename btree_type::data_type data_type;
+	typedef typename btree_type::mapped_type mapped_type;
+
+ public:
+	// Default constructor.
+	multimap(const key_compare& comp = key_compare(),
+			 const allocator_type& alloc = allocator_type())
+		: super_type(comp, alloc) {
+	}
+
+	// Copy constructor.
+	multimap(const self_type& x)
+		: super_type(x) {
+	}
+
+	// Range constructor.
+	template <class InputIterator>
+	multimap(InputIterator b, InputIterator e,
+			 const key_compare& comp = key_compare(),
+			 const allocator_type& alloc = allocator_type())
+		: super_type(b, e, comp, alloc) {
+	}
+};
+
+} // namespace btree
+
+template <typename K, typename V, typename C, typename A, int N>
+bool operator==(const btree::multimap<K, V, C, A, N>& lhs, const btree::multimap<K, V, C, A, N>& rhs) {
+	return lhs.size() == rhs.size() && std::equal(lhs.begin(), lhs.end(), rhs.begin());
+}
+
+template <typename K, typename V, typename C, typename A, int N>
+bool operator<(const btree::multimap<K, V, C, A, N>& lhs, const btree::multimap<K, V, C, A, N>& rhs) {
+	return std::lexicographical_compare(lhs.begin(), lhs.end(), rhs.begin(), rhs.end());
+}
+
+template <typename K, typename V, typename C, typename A, int N>
+bool operator!=(const btree::multimap<K, V, C, A, N>& lhs, const btree::multimap<K, V, C, A, N>& rhs) {
+	return !(lhs == rhs);
+}
+
+template <typename K, typename V, typename C, typename A, int N>
+bool operator>(const btree::multimap<K, V, C, A, N>& lhs, const btree::multimap<K, V, C, A, N>& rhs) {
+	return rhs < lhs;
+}
+
+template <typename K, typename V, typename C, typename A, int N>
+bool operator>=(const btree::multimap<K, V, C, A, N>& lhs, const btree::multimap<K, V, C, A, N>& rhs) {
+	return !(lhs < rhs);
+}
+
+template <typename K, typename V, typename C, typename A, int N>
+bool operator<=(const btree::multimap<K, V, C, A, N>& lhs, const btree::multimap<K, V, C, A, N>& rhs) {
+	return !(rhs < lhs);
+}
+
+template <typename K, typename V, typename C, typename A, int N>
+inline void swap(btree::multimap<K, V, C, A, N>& x, btree::multimap<K, V, C, A, N>& y) {
+	x.swap(y);
+}
+
+#endif  // BTREE_MAP_H__

server/src/absl/btree/set.h

@@ -0,0 +1,183 @@
+/*
+ * Copyright (c) 2019 German Mendez Bravo (Kronuz)
+ * Copyright (c) 2013 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ *
+ * A btree::set<> implements the STL unique sorted associative container
+ * interface (a.k.a set<>) using a btree. See btree.h for details of the btree
+ * implementation and caveats.
+ */
+
+#ifndef BTREE_SET_H__
+#define BTREE_SET_H__
+
+#include "btree.h"
+
+namespace btree {
+
+// The set class is needed mainly for its constructors.
+template <typename Key,
+	typename Compare = std::less<Key>,
+	typename Alloc = std::allocator<Key>,
+	int TargetNodeSize = 256>
+class set : public btree_unique_container<
+	btree<btree_set_params<Key, Compare, Alloc, TargetNodeSize> > > {
+
+	typedef set<Key, Compare, Alloc, TargetNodeSize> self_type;
+	typedef btree_set_params<Key, Compare, Alloc, TargetNodeSize> params_type;
+	typedef btree<params_type> btree_type;
+	typedef btree_unique_container<btree_type> super_type;
+
+ public:
+	typedef typename btree_type::key_compare key_compare;
+	typedef typename btree_type::allocator_type allocator_type;
+
+ public:
+	// Default constructor.
+	set(const key_compare& comp = key_compare(),
+		const allocator_type& alloc = allocator_type())
+		: super_type(comp, alloc) {
+	}
+
+	// Copy constructor.
+	set(const self_type& x)
+		: super_type(x) {
+	}
+
+	// Range constructor.
+	template <class InputIterator>
+	set(InputIterator b, InputIterator e,
+		const key_compare& comp = key_compare(),
+		const allocator_type& alloc = allocator_type())
+		: super_type(b, e, comp, alloc) {
+	}
+};
+
+} // namespace btree
+
+template <typename K, typename C, typename A, int N>
+bool operator==(const btree::set<K, C, A, N>& lhs, const btree::set<K, C, A, N>& rhs) {
+	return lhs.size() == rhs.size() && std::equal(lhs.begin(), lhs.end(), rhs.begin());
+}
+
+template <typename K, typename C, typename A, int N>
+bool operator<(const btree::set<K, C, A, N>& lhs, const btree::set<K, C, A, N>& rhs) {
+	return std::lexicographical_compare(lhs.begin(), lhs.end(), rhs.begin(), rhs.end());
+}
+
+template <typename K, typename C, typename A, int N>
+bool operator!=(const btree::set<K, C, A, N>& lhs, const btree::set<K, C, A, N>& rhs) {
+	return !(lhs == rhs);
+}
+
+template <typename K, typename C, typename A, int N>
+bool operator>(const btree::set<K, C, A, N>& lhs, const btree::set<K, C, A, N>& rhs) {
+	return rhs < lhs;
+}
+
+template <typename K, typename C, typename A, int N>
+bool operator>=(const btree::set<K, C, A, N>& lhs, const btree::set<K, C, A, N>& rhs) {
+	return !(lhs < rhs);
+}
+
+template <typename K, typename C, typename A, int N>
+bool operator<=(const btree::set<K, C, A, N>& lhs, const btree::set<K, C, A, N>& rhs) {
+	return !(rhs < lhs);
+}
+
+template <typename K, typename C, typename A, int N>
+inline void swap(btree::set<K, C, A, N>& x, btree::set<K, C, A, N>& y) {
+	x.swap(y);
+}
+
+namespace btree {
+
+// The multiset class is needed mainly for its constructors.
+template <typename Key,
+					typename Compare = std::less<Key>,
+					typename Alloc = std::allocator<Key>,
+					int TargetNodeSize = 256>
+class multiset : public btree_multi_container<
+	btree<btree_set_params<Key, Compare, Alloc, TargetNodeSize> > > {
+
+	typedef multiset<Key, Compare, Alloc, TargetNodeSize> self_type;
+	typedef btree_set_params<Key, Compare, Alloc, TargetNodeSize> params_type;
+	typedef btree<params_type> btree_type;
+	typedef btree_multi_container<btree_type> super_type;
+
+ public:
+	typedef typename btree_type::key_compare key_compare;
+	typedef typename btree_type::allocator_type allocator_type;
+
+ public:
+	// Default constructor.
+	multiset(const key_compare& comp = key_compare(),
+			 const allocator_type& alloc = allocator_type())
+		: super_type(comp, alloc) {
+	}
+
+	// Copy constructor.
+	multiset(const self_type& x)
+		: super_type(x) {
+	}
+
+	// Range constructor.
+	template <class InputIterator>
+	multiset(InputIterator b, InputIterator e,
+			 const key_compare& comp = key_compare(),
+			 const allocator_type& alloc = allocator_type())
+		: super_type(b, e, comp, alloc) {
+	}
+};
+
+
+} // namespace btree
+
+template <typename K, typename C, typename A, int N>
+bool operator==(const btree::multiset<K, C, A, N>& lhs, const btree::multiset<K, C, A, N>& rhs) {
+	return lhs.size() == rhs.size() && std::equal(lhs.begin(), lhs.end(), rhs.begin());
+}
+
+template <typename K, typename C, typename A, int N>
+bool operator<(const btree::multiset<K, C, A, N>& lhs, const btree::multiset<K, C, A, N>& rhs) {
+	return std::lexicographical_compare(lhs.begin(), lhs.end(), rhs.begin(), rhs.end());
+}
+
+template <typename K, typename C, typename A, int N>
+bool operator!=(const btree::multiset<K, C, A, N>& lhs, const btree::multiset<K, C, A, N>& rhs) {
+	return !(lhs == rhs);
+}
+
+template <typename K, typename C, typename A, int N>
+bool operator>(const btree::multiset<K, C, A, N>& lhs, const btree::multiset<K, C, A, N>& rhs) {
+	return rhs < lhs;
+}
+
+template <typename K, typename C, typename A, int N>
+bool operator>=(const btree::multiset<K, C, A, N>& lhs, const btree::multiset<K, C, A, N>& rhs) {
+	return !(lhs < rhs);
+}
+
+template <typename K, typename C, typename A, int N>
+bool operator<=(const btree::multiset<K, C, A, N>& lhs, const btree::multiset<K, C, A, N>& rhs) {
+	return !(rhs < lhs);
+}
+
+template <typename K, typename C, typename A, int N>
+inline void swap(btree::multiset<K, C, A, N>& x, btree::multiset<K, C, A, N>& y) {
+	x.swap(y);
+}
+
+#endif  // BTREE_SET_H__

server/src/channel/ServerChannel.cpp

@@ -5,7 +5,6 @@
 #include "misc/rnd.h"
 #include "src/VirtualServer.h"
 #include "src/client/ConnectedClient.h"
-#include "src/server/file/FileServer.h"
 #include "src/InstanceHandler.h"
 #include "../manager/ConversationManager.h"
 
@@ -102,7 +101,7 @@ std::shared_ptr<BasicChannel> ServerChannelTree::createChannel(ChannelId parentI
     channel->properties()[property::CHANNEL_LAST_LEFT] = chrono::duration_cast<chrono::milliseconds>(chrono::system_clock::now().time_since_epoch()).count();
 
 
-    auto result = sql::command(this->sql, "INSERT INTO `channels` (`serverId`, `channelId`, `type`, `parentId`) VALUES(:sid, :chid, :type, :parent);",  variable{":sid", this->getServerId()}, variable{":chid", channel->channelId()}, variable{":type", channel->channelType()}, variable{":parent", channel->parent() ? channel->parent()->channelId() : 0}).execute();
+    auto result = sql::command(this->sql, "INSERT INTO `channels` (`serverId`, `channelId`, `parentId`) VALUES(:sid, :chid, :parent);",  variable{":sid", this->getServerId()}, variable{":chid", channel->channelId()}, variable{":parent", channel->parent() ? channel->parent()->channelId() : 0}).execute();
     auto pf = LOG_SQL_CMD;
     pf(result);
 
@@ -481,6 +480,7 @@ bool ServerChannelTree::validateChannelNames() {
 bool ServerChannelTree::validateChannelIcons() {
     for(const auto &channel : this->channels()) {
         auto iconId = (IconId) channel->properties()[property::CHANNEL_ICON_ID];
+#if 0
         if(iconId != 0 && !serverInstance->getFileServer()->iconExists(this->server.lock(), iconId)) {
             logMessage(this->getServerId(), "[FILE] Missing channel icon (" + to_string(iconId) + ").");
             if(config::server::delete_missing_icon_permissions) {
@@ -488,12 +488,13 @@ bool ServerChannelTree::validateChannelIcons() {
                 channel->permissions()->set_permission(permission::i_icon_id, {0, 0}, permission::v2::PermissionUpdateType::set_value, permission::v2::PermissionUpdateType::do_nothing);
             }
         }
+#endif
     }
     return true;
 }
 
 void ServerChannelTree::loadChannelsFromDatabase() {
-    auto res = sql::command(this->sql, "SELECT * FROM `channels` WHERE `serverId` = :sid", variable{":sid", this->getServerId()}).query(&ServerChannelTree::loadChannelFromData, this);
+    auto res = sql::command(this->sql, "SELECT  `channelId`, `parentId` FROM `channels` WHERE `serverId` = :sid", variable{":sid", this->getServerId()}).query(&ServerChannelTree::loadChannelFromData, this);
     (LOG_SQL_CMD)(res);
     if(!res){
         logError(this->getServerId(), "Could not load channel tree from database");
@@ -512,7 +513,6 @@ void ServerChannelTree::loadChannelsFromDatabase() {
 int ServerChannelTree::loadChannelFromData(int argc, char **data, char **column) {
     ChannelId channelId = 0;
     ChannelId parentId = 0;
-    auto type = static_cast<ChannelType::ChannelType>(0xFF);
 
 
     int index = 0;
@@ -520,8 +520,6 @@ int ServerChannelTree::loadChannelFromData(int argc, char **data, char **column)
         for(index = 0; index < argc; index++){
             if(strcmp(column[index], "channelId") == 0) channelId = static_cast<ChannelId>(stoll(data[index]));
             else if(strcmp(column[index], "parentId") == 0) parentId = static_cast<ChannelId>(stoll(data[index]));
-            else if(strcmp(column[index], "type") == 0) type = static_cast<ChannelType::ChannelType>(stoll(data[index]));
-            else if(strcmp(column[index], "serverId") == 0) {}
             else logError(this->getServerId(), "ServerChannelTree::loadChannelFromData called with invalid column from sql \"{}\"", column[index]);
         }
     } catch (std::exception& ex) {
@@ -531,6 +529,8 @@ int ServerChannelTree::loadChannelFromData(int argc, char **data, char **column)
 
     //assert(type != 0xFF);
     assert(channelId != 0);
+    if(channelId == 0)
+        return 0;
 
     auto channel = std::make_shared<ServerChannel>(parentId, channelId);
     auto server = this->server.lock();

server/src/client/ConnectedClient.cpp

@@ -11,7 +11,6 @@
 #include "src/VirtualServer.h"
 #include "voice/VoiceClient.h"
 #include "../server/VoiceServer.h"
-#include "../server/file/FileServer.h"
 #include "../InstanceHandler.h"
 #include "ConnectedClient.h"
 
@@ -29,9 +28,7 @@ ConnectedClient::ConnectedClient(sql::SqlManager* db, const std::shared_ptr<Virt
     memtrack::allocated<ConnectedClient>(this);
     memset(&this->remote_address, 0, sizeof(this->remote_address));
 
-    connectionStatistics = make_shared<stats::ConnectionStatistics>(server ? server->getServerStatistics() : nullptr, false);
-    this->connectionStatistics->measure_bandwidths(false); /* done by the client and we trust this */
-
+    connectionStatistics = make_shared<stats::ConnectionStatistics>(server ? server->getServerStatistics() : nullptr);
     channels = make_shared<ClientChannelView>(this);
 }
 
@@ -39,6 +36,14 @@ ConnectedClient::~ConnectedClient() {
     memtrack::freed<ConnectedClient>(this);
 }
 
+bool ConnectedClient::loadDataForCurrentServer() {
+    auto result = DataClient::loadDataForCurrentServer();
+    if(!result) return false;
+
+    this->properties()[property::CONNECTION_CLIENT_IP] = this->getLoggingPeerIp();
+    return true;
+}
+
 std::shared_ptr<ConnectionInfoData> ConnectedClient::request_connection_info(const std::shared_ptr<ConnectedClient> &receiver, bool& send_temp) {
     auto& info = this->connection_info;
 
@@ -145,7 +150,8 @@ void ConnectedClient::updateChannelClientProperties(bool lock_channel_tree, bool
             iconId = value.value;
     }
     logTrace(this->getServerId(), "[CLIENT] Updating client icon from " + to_string(this->properties()[property::CLIENT_ICON_ID].as<IconId>()) + " to " + to_string(iconId));
-    if(this->properties()[property::CLIENT_ICON_ID].as<IconId>() != iconId){
+    if(this->properties()[property::CLIENT_ICON_ID].as<IconId>() != iconId) {
+#if 0
         if(server_ref && iconId != 0) {
             auto dir = serverInstance->getFileServer()->iconDirectory(server_ref);
             if(!serverInstance->getFileServer()->iconExists(server_ref, iconId)) {
@@ -153,6 +159,7 @@ void ConnectedClient::updateChannelClientProperties(bool lock_channel_tree, bool
                 iconId = 0;
             }
         }
+#endif
         if(this->properties()[property::CLIENT_ICON_ID].as<IconId>() != iconId) {
             this->properties()[property::CLIENT_ICON_ID] = (IconId) iconId;
             notifyList.emplace_back(property::CLIENT_ICON_ID);
@@ -169,7 +176,7 @@ void ConnectedClient::updateChannelClientProperties(bool lock_channel_tree, bool
     block_flood = !permission::v2::permission_granted(1, permission_ignore_antiflood);
     if(server_ref)
         server_ref->notifyClientPropertyUpdates(_this.lock(), notifyList, notify_self);
-    this->updateTalkRights(permission_talk_power.has_value ? permission_talk_power.value : 0);
+    this->updateTalkRights(permission_talk_power);
 
     if((this->channels_view_power != permission_channel_view_power || this->channels_ignore_view != permission_channel_ignore_view_power) && notify_self && this->currentChannel && server_ref) {
         this->channels_view_power = permission_channel_view_power;
@@ -184,22 +191,25 @@ void ConnectedClient::updateChannelClientProperties(bool lock_channel_tree, bool
             client_channel_lock.lock();
         }
 
-        deque<ChannelId> deleted;
-        for(const auto& update_entry : this->channels->update_channel_path(server_ref->channelTree->tree_head(), server_ref->channelTree->find_linked_entry(this->currentChannel->channelId()))) {
-            if(update_entry.first)
-                this->notifyChannelShow(update_entry.second->channel(), update_entry.second->previous_channel);
-            else deleted.push_back(update_entry.second->channelId());
+        /* might have been changed since we locked the tree */
+        if(this->currentChannel) {
+            deque<ChannelId> deleted;
+            for(const auto& update_entry : this->channels->update_channel_path(server_ref->channelTree->tree_head(), server_ref->channelTree->find_linked_entry(this->currentChannel->channelId()))) {
+                if(update_entry.first)
+                    this->notifyChannelShow(update_entry.second->channel(), update_entry.second->previous_channel);
+                else deleted.push_back(update_entry.second->channelId());
+            }
+            if(!deleted.empty())
+                this->notifyChannelHide(deleted, false); /* we've locked the tree before */
         }
-        if(!deleted.empty())
-            this->notifyChannelHide(deleted, false); /* we've locked the tree before */
     }
 }
 
-void ConnectedClient::updateTalkRights(permission::PermissionValue talk_power) {
+void ConnectedClient::updateTalkRights(permission::v2::PermissionFlaggedValue talk_power) {
     bool flag = false;
     flag |= this->properties()[property::CLIENT_IS_TALKER].as<bool>();
     if(!flag && this->currentChannel) {
-        flag = this->currentChannel->talk_power_granted({talk_power, talk_power != permNotGranted});
+        flag = this->currentChannel->talk_power_granted(talk_power);
     }
     this->allowedToTalk = flag;
 }
@@ -381,7 +391,7 @@ bool ConnectedClient::notifyClientLeftView(
         std::shared_ptr<ConnectedClient> invoker,
         bool lock_channel_tree) {
     assert(!lock_channel_tree); /* not supported yet! */
-    assert(client && client->getClientId() != 0);
+    assert(client == this || (client && client->getClientId() != 0));
     assert(client->currentChannel || &*client == this);
 
     if(client != this) {
@@ -573,30 +583,37 @@ bool ConnectedClient::notifyClientNeededPermissions() {
     return true;
 }
 
+inline void write_command_result_error(ts::command_builder_bulk bulk, const command_result& result, const std::string& errorCodeKey) {
+    bulk.put_unchecked(errorCodeKey, (uint32_t) result.error_code());
+    bulk.put_unchecked("msg", findError(result.error_code()).message);
+    if(result.is_permission_error())
+        bulk.put_unchecked("failed_permid", (uint32_t) result.permission_id());
+}
+
+inline void write_command_result_detailed(ts::command_builder_bulk bulk, const command_result& result, const std::string& errorCodeKey) {
+    auto details = result.details();
+    bulk.put_unchecked(errorCodeKey, (uint32_t) details->error_id);
+    bulk.put_unchecked("msg", findError(details->error_id).message);
+
+    for(const auto& extra : details->extra_properties)
+        bulk.put(extra.first, extra.second);
+}
+
 bool ConnectedClient::notifyError(const command_result& result, const std::string& retCode) {
-    Command cmd("error");
+    ts::command_builder command{"error"};
 
-    if(result.is_detailed()) {
-        auto detailed = result.details();
-        cmd["id"] = (int) detailed->error_id;
-        cmd["msg"] = findError(detailed->error_id).message;
+    this->writeCommandResult(command, result);
+    if(!retCode.empty())
+        command.put_unchecked(0, "return_code", retCode);
 
-        for(const auto& extra : detailed->extra_properties)
-            cmd[extra.first] = extra.second;
-    } else {
-        cmd["id"] = (int) result.error_code();
-        cmd["msg"] = findError(result.error_code()).message;
-        if(result.is_permission_error())
-            cmd["failed_permid"] = result.permission_id();
-    }
-
-    if(retCode.length() > 0)
-        cmd["return_code"] = retCode;
-
-    this->sendCommand(cmd);
+    this->sendCommand(command);
     return true;
 }
 
+void ConnectedClient::writeCommandResult(ts::command_builder &cmd_builder, const command_result &result, const std::string& errorCodeKey) {
+    result.build_error_response(cmd_builder, errorCodeKey);
+}
+
 inline std::shared_ptr<ViewEntry> pop_view_entry(std::deque<std::shared_ptr<ViewEntry>>& pool, ChannelId id) {
     for(auto it = pool.begin(); it != pool.end(); it++) {
         if((*it)->channelId() == id) {
@@ -635,12 +652,7 @@ inline void send_channels(ConnectedClient* client, ChannelIT begin, const Channe
             break;
     }
 
-    if(dynamic_cast<VoiceClient*>(client)) {
-        auto vc = dynamic_cast<VoiceClient*>(client);
-        vc->sendCommand0(builder.build(), false, true); /* we need to process this command directly so it will be processed before the channellistfinished stuff */
-    } else {
-        client->sendCommand(builder);
-    }
+    client->sendCommand(builder);
     if(begin != end)
         send_channels(client, begin, end, override_orderid);
 }
@@ -744,17 +756,14 @@ void ConnectedClient::tick(const std::chrono::system_clock::time_point &time) {
     }
 
 
-    if(this->last_statistics_tick + seconds(5) < time) {
-        this->last_statistics_tick = time;
-        this->connectionStatistics->tick();
-    }
+    this->connectionStatistics->tick();
 }
 
 void ConnectedClient::sendServerInit() {
     Command command("initserver");
 
     for(const auto& prop : this->server->properties().list_properties(property::FLAG_SERVER_VIEW, this->getType() == CLIENT_TEAMSPEAK ? property::FLAG_NEW : (uint16_t) 0)) {
-        command[prop.type().name] = prop.value();
+        command[std::string{prop.type().name}] = prop.value();
     }
     command["virtualserver_maxclients"] = 32;
 
@@ -784,11 +793,7 @@ void ConnectedClient::sendServerInit() {
     command["pv"] = 6; //Protocol version
     command["acn"] = this->getDisplayName();
     command["aclid"] = this->getClientId();
-    if(dynamic_cast<VoiceClient*>(this)) {
-        dynamic_cast<VoiceClient*>(this)->sendCommand0(command.build(), false, true); /* process it directly so the order for the channellist entries is ensured. (First serverinit then everything else) */
-    } else {
-        this->sendCommand(command);
-    }
+    this->sendCommand(command);
 }
 
 bool ConnectedClient::handleCommandFull(Command& cmd, bool disconnectOnFail) {
@@ -800,14 +805,38 @@ bool ConnectedClient::handleCommandFull(Command& cmd, bool disconnectOnFail) {
 
     command_result result;
     try {
-        result = this->handleCommand(cmd);
+        result.reset(this->handleCommand(cmd));
+    } catch(command_value_cast_failed& ex){
+        auto message = ex.key() + " at " + std::to_string(ex.index()) + " could not be casted to " + ex.target_type().name();
+        if(disconnectOnFail) {
+            this->disconnect(message);
+            return false;
+        } else {
+            result.reset(command_result{error::parameter_convert, message});
+        }
+    } catch(command_bulk_index_out_of_bounds_exception& ex){
+        auto message = "missing bulk for index " + std::to_string(ex.index());
+        if(disconnectOnFail) {
+            this->disconnect(message);
+            return false;
+        } else {
+            result.reset(command_result{error::parameter_invalid_count, message});
+        }
+    } catch(command_value_missing_exception& ex){
+        auto message = "missing value for " + ex.key() + (ex.bulk_index() == std::string::npos ? "" : " at " + std::to_string(ex.bulk_index()));
+        if(disconnectOnFail) {
+            this->disconnect(message);
+            return false;
+        } else {
+            result.reset(command_result{error::parameter_missing, message});
+        }
     } catch(invalid_argument& ex){
         logWarning(this->getServerId(), "{}[Command] Failed to handle command. Received invalid argument exception: {}", CLIENT_STR_LOG_PREFIX, ex.what());
         if(disconnectOnFail) {
             this->disconnect("Invalid argument (" + string(ex.what()) + ")");
             return false;
         } else {
-            result = command_result{error::parameter_convert};
+            result.reset(command_result{error::parameter_convert, ex.what()});
         }
     } catch (exception& ex) {
         logWarning(this->getServerId(), "{}[Command] Failed to handle command. Received exception with message: {}", CLIENT_STR_LOG_PREFIX, ex.what());
@@ -815,7 +844,7 @@ bool ConnectedClient::handleCommandFull(Command& cmd, bool disconnectOnFail) {
             this->disconnect("Error while command handling (" + string(ex.what()) + ")!");
             return false;
         } else {
-            result = command_result{error::vs_critical};
+            result.reset(command_result{error::vs_critical});
         }
     } catch (...) {
         this->disconnect("Error while command handling! (unknown)");
@@ -823,7 +852,7 @@ bool ConnectedClient::handleCommandFull(Command& cmd, bool disconnectOnFail) {
     }
 
     bool generateReturnStatus = false;
-    if(result.error_code() != error::ok || this->getType() == ClientType::CLIENT_QUERY){
+    if(result.has_error() || this->getType() == ClientType::CLIENT_QUERY){
         generateReturnStatus = true;
     } else if(cmd["return_code"].size() > 0) {
         generateReturnStatus = !cmd["return_code"].string().empty();
@@ -832,7 +861,7 @@ bool ConnectedClient::handleCommandFull(Command& cmd, bool disconnectOnFail) {
     if(generateReturnStatus)
         this->notifyError(result, cmd["return_code"].size() > 0 ? cmd["return_code"].first().as<std::string>() : "");
 
-    if(result.error_code() != error::ok && this->state == ConnectionState::INIT_HIGH)
+    if(result.has_error() && this->state == ConnectionState::INIT_HIGH)
         this->close_connection(system_clock::now()); //Disconnect now
 
     for (const auto& handler : postCommandHandler)
@@ -846,7 +875,7 @@ bool ConnectedClient::handleCommandFull(Command& cmd, bool disconnectOnFail) {
         else
             logWarning(this->getServerId(), "Command handling of command {} needs {}ms.", cmd.command(), duration_cast<milliseconds>(end - start).count());
     }
-    result.release_details();
+    result.release_data();
     return true;
 }
 

server/src/client/ConnectedClient.h

@@ -13,7 +13,11 @@
 #define CLIENT_STR_LOG_PREFIX CLIENT_STR_LOG_PREFIX_(this)
 
 #define CMD_REQ_SERVER   \
-if(!this->server) return command_result{error::server_invalid_id};
+do { \
+    if(!this->server) { \
+        return command_result{error::server_invalid_id}; \
+    } \
+} while(0)
 
 /* TODO: Play lock the server here with read? So the client dosn't get kicked within that moment */
 #define CMD_REF_SERVER(variable_name)   \
@@ -33,8 +37,10 @@ if(!cmd[0].has(parm)) return command_result{error::parameter_not_found};
 
 //the message here is show to the manager!
 #define CMD_CHK_AND_INC_FLOOD_POINTS(num) \
-this->increaseFloodPoints(num); \
-if(this->shouldFloodBlock()) return command_result{error::ban_flooding};
+do {\
+    this->increaseFloodPoints(num); \
+    if(this->shouldFloodBlock()) return command_result{error::ban_flooding}; \
+} while(0)
 
 #define CMD_CHK_PARM_COUNT(count) \
 if(cmd.bulkCount() != count) return command_result{error::parameter_invalid_count};
@@ -116,6 +122,7 @@ namespace ts {
 
                 /** Notifies general stuff **/
                 virtual bool notifyError(const command_result&, const std::string& retCode = "");
+                virtual void writeCommandResult(ts::command_builder&, const command_result&, const std::string& errorCodeKey = "id");
 
                 /** Notifies (after request) */
                 bool sendNeededPermissions(bool /* force an update */); /* invoke this because it dosn't spam the client */
@@ -135,7 +142,7 @@ namespace ts {
                 virtual bool notifyClientPoke(std::shared_ptr<ConnectedClient> invoker, std::string msg);
                 virtual bool notifyClientUpdated(
                         const std::shared_ptr<ConnectedClient> &,
-                        const std::deque<std::shared_ptr<property::PropertyDescription>> &,
+                        const std::deque<const property::PropertyDescription*> &,
                         bool lock_channel_tree
                 ); /* invalid client id causes error: invalid clientID */
 
@@ -247,7 +254,7 @@ namespace ts {
                 std::shared_ptr<ConnectionInfoData> request_connection_info(const std::shared_ptr<ConnectedClient> & /* receiver */, bool& /* send temporary (no client response yet) */);
 
                 virtual void updateChannelClientProperties(bool /* lock channel tree */, bool /* notify our self */);
-                void updateTalkRights(permission::PermissionValue talk_power);
+                void updateTalkRights(permission::v2::PermissionFlaggedValue talk_power);
 
                 virtual std::shared_ptr<BanRecord> resolveActiveBan(const std::string& ip_address);
 
@@ -326,7 +333,6 @@ namespace ts {
                 std::chrono::system_clock::time_point lastOnlineTimestamp;
                 std::chrono::system_clock::time_point lastTransfareTimestamp;
                 std::chrono::system_clock::time_point idleTimestamp;
-                std::chrono::system_clock::time_point last_statistics_tick;
 
                 struct {
                     std::mutex lock;
@@ -369,6 +375,8 @@ namespace ts {
                 std::weak_ptr<MusicClient> subscribed_bot;
                 std::weak_ptr<ts::music::Playlist> _subscribed_playlist{};
 
+                bool loadDataForCurrentServer() override;
+
                 virtual void tick(const std::chrono::system_clock::time_point &time);
                 //Locked by everything who has something todo with command handling
                 threads::Mutex command_lock; /* Note: This mutex must be recursive! */
@@ -411,13 +419,13 @@ namespace ts {
                 command_result handleCommandChannelDelPerm(Command&);
 
                 //Server group manager management
-                command_result handleCommandServerGroupCopy(Command&);
                 command_result handleCommandServerGroupAdd(Command&);
+                command_result handleCommandServerGroupCopy(Command&);
                 command_result handleCommandServerGroupRename(Command&);
                 command_result handleCommandServerGroupDel(Command&);
                 command_result handleCommandServerGroupClientList(Command&);
-                command_result handleCommandServerGroupDelClient(Command&);
                 command_result handleCommandServerGroupAddClient(Command&);
+                command_result handleCommandServerGroupDelClient(Command&);
                 command_result handleCommandServerGroupPermList(Command&);
                 command_result handleCommandServerGroupAddPerm(Command&);
                 command_result handleCommandServerGroupDelPerm(Command&);
@@ -454,11 +462,10 @@ namespace ts {
                 command_result handleCommandFTDeleteFile(Command&);  
                 command_result handleCommandFTInitUpload(Command&);  
                 command_result handleCommandFTInitDownload(Command&); 
-                command_result handleCommandFTGetFileInfo(Command&); 
-                //CMD_TODO handleCommandFTGetFileInfo -> 5 points
-                //CMD_TODO handleCommandFTStop -> 5 points
-                //CMD_TODO handleCommandFTRenameFile -> 5 points
-                //CMD_TODO handleCommandFTList -> 5 points
+                command_result handleCommandFTGetFileInfo(Command&);
+                command_result handleCommandFTRenameFile(Command&);
+                command_result handleCommandFTList(Command&);
+                command_result handleCommandFTStop(Command&);
 
                 command_result handleCommandBanList(Command&);
                 command_result handleCommandBanAdd(Command&);
@@ -593,7 +600,10 @@ namespace ts {
                 command_result handleCommandConversationMessageDelete(Command&);
 
                 command_result handleCommandLogView(Command&);
-                //CMD_TODO handleCommandLogAdd
+                command_result handleCommandLogQuery(Command&);
+                command_result handleCommandLogAdd(Command&);
+
+                command_result handleCommandListFeatureSupport(Command &cmd);
 
                 //handleCommandClientSiteReport() -> return findError(0x00)
                 //handleCommandChannelCreatePrivate() -> return findError(0x02)
@@ -634,18 +644,20 @@ namespace ts {
 
         template <typename T>
         struct ConnectedLockedClient {
+            ConnectedLockedClient() {}
             explicit ConnectedLockedClient(std::shared_ptr<T> client) : client{std::move(client)} {
                 if(this->client)
                     this->connection_lock = this->client->require_connected_state();
             }
+            explicit ConnectedLockedClient(ConnectedLockedClient&& client) : client{std::move(client.client)}, connection_lock{std::move(client.connection_lock)} { }
 
-            inline ConnectedLockedClient<T> &operator=(const ConnectedLockedClient& other) {
+            inline ConnectedLockedClient &operator=(const ConnectedLockedClient& other) {
                 this->client = other.client;
                 if(other)
                     this->connection_lock = std::shared_lock{*other.connection_lock.mutex()}; /* if the other is true (state locked & client) than we could easily acquire a shared lock */
             }
 
-            inline ConnectedLockedClient<T> &operator=(ConnectedLockedClient&& other) {
+            inline ConnectedLockedClient &operator=(ConnectedLockedClient&& other) {
                 this->client = std::move(other.client);
                 this->connection_lock = std::move(other.connection_lock);
             }

server/src/client/ConnectedClientNotifyHandler.cpp

@@ -3,7 +3,6 @@
 #include <algorithm>
 #include "ConnectedClient.h"
 #include "voice/VoiceClient.h"
-#include "../server/file/FileServer.h"
 #include "../server/VoiceServer.h"
 #include "../InstanceHandler.h"
 #include "../server/QueryServer.h"
@@ -21,9 +20,7 @@ using namespace ts;
 using namespace ts::server;
 using namespace ts::token;
 
-extern ts::server::InstanceHandler* serverInstance;
-
-#define INVOKER(command, invoker)                                           \
+# define INVOKER(command, invoker)                                           \
 do {                                                                        \
     if(invoker) {                                                           \
         command["invokerid"] = invoker->getClientId();                      \
@@ -60,7 +57,7 @@ bool ConnectedClient::notifyServerGroupList() {
             cmd[index]["sgid"] = group->groupId();
         }
         for (const auto &prop : group->properties().list_properties(property::FLAG_GROUP_VIEW, this->getType() == CLIENT_TEAMSPEAK ? property::FLAG_NEW : (uint16_t) 0))
-            cmd[index][prop.type().name] = prop.value();
+            cmd[index][std::string{prop.type().name}] = prop.value();
 
 
         auto modify_power = group->permissions()->permission_value_flagged(permission::i_displayed_group_needed_modify_power);
@@ -186,7 +183,7 @@ bool ConnectedClient::notifyChannelGroupList() {
             cmd[index]["sgid"] = group->groupId();
         }
         for (auto &prop : group->properties().list_properties(property::FLAG_GROUP_VIEW, this->getType() == CLIENT_TEAMSPEAK ? property::FLAG_NEW : (uint16_t) 0))
-            cmd[index][prop.type().name] = prop.value();
+            cmd[index][std::string{prop.type().name}] = prop.value();
 
 
         auto modify_power = group->permissions()->permission_value_flagged(permission::i_displayed_group_needed_modify_power);
@@ -271,108 +268,116 @@ bool ConnectedClient::notifyClientChannelGroupChanged(
 }
 
 bool ConnectedClient::notifyConnectionInfo(const shared_ptr<ConnectedClient> &target, const shared_ptr<ConnectionInfoData> &info) {
-    Command notify("notifyconnectioninfo");
-    notify["clid"] = target->getClientId();
+    command_builder notify{"notifyconnectioninfo"};
+    auto bulk = notify.bulk(0);
+    bulk.put_unchecked("clid", target->getClientId());
 
     auto not_set = this->getType() == CLIENT_TEAMSPEAK ? 0 : -1;
     /* we deliver data to the web client as well, because its a bit dump :D */
     if(target->getClientId() != this->getClientId()) {
-        auto report = target->connectionStatistics->full_report();
+        auto file_stats = target->connectionStatistics->file_stats();
 
         /* default values which normally sets the client */
-        notify["connection_bandwidth_received_last_minute_control"] = not_set;
-        notify["connection_bandwidth_received_last_minute_keepalive"] = not_set;
-        notify["connection_bandwidth_received_last_minute_speech"] = not_set;
-        notify["connection_bandwidth_received_last_second_control"] = not_set;
-        notify["connection_bandwidth_received_last_second_keepalive"] = not_set;
-        notify["connection_bandwidth_received_last_second_speech"] = not_set;
+        bulk.put_unchecked(property::CONNECTION_BANDWIDTH_RECEIVED_LAST_MINUTE_CONTROL, not_set);
+        bulk.put_unchecked(property::CONNECTION_BANDWIDTH_RECEIVED_LAST_MINUTE_KEEPALIVE, not_set);
+        bulk.put_unchecked(property::CONNECTION_BANDWIDTH_RECEIVED_LAST_MINUTE_SPEECH, not_set);
+        bulk.put_unchecked(property::CONNECTION_BANDWIDTH_RECEIVED_LAST_SECOND_CONTROL, not_set);
+        bulk.put_unchecked(property::CONNECTION_BANDWIDTH_RECEIVED_LAST_SECOND_KEEPALIVE, not_set);
+        bulk.put_unchecked(property::CONNECTION_BANDWIDTH_RECEIVED_LAST_SECOND_SPEECH, not_set);
 
-        notify["connection_bandwidth_sent_last_minute_control"] = not_set;
-        notify["connection_bandwidth_sent_last_minute_keepalive"] = not_set;
-        notify["connection_bandwidth_sent_last_minute_speech"] = not_set;
-        notify["connection_bandwidth_sent_last_second_control"] = not_set;
-        notify["connection_bandwidth_sent_last_second_keepalive"] = not_set;
-        notify["connection_bandwidth_sent_last_second_speech"] = not_set;
+        bulk.put_unchecked(property::CONNECTION_BANDWIDTH_SENT_LAST_MINUTE_CONTROL, not_set);
+        bulk.put_unchecked(property::CONNECTION_BANDWIDTH_SENT_LAST_MINUTE_KEEPALIVE, not_set);
+        bulk.put_unchecked(property::CONNECTION_BANDWIDTH_SENT_LAST_MINUTE_SPEECH, not_set);
+        bulk.put_unchecked(property::CONNECTION_BANDWIDTH_SENT_LAST_SECOND_CONTROL, not_set);
+        bulk.put_unchecked(property::CONNECTION_BANDWIDTH_SENT_LAST_SECOND_KEEPALIVE, not_set);
+        bulk.put_unchecked(property::CONNECTION_BANDWIDTH_SENT_LAST_SECOND_SPEECH, not_set);
 
         /* its flipped here because the report is out of the clients view */
-        notify["connection_bytes_received_control"] = not_set;
-        notify["connection_bytes_received_keepalive"] = not_set;
-        notify["connection_bytes_received_speech"] = not_set;
-        notify["connection_bytes_sent_control"] = not_set;
-        notify["connection_bytes_sent_keepalive"] = not_set;
-        notify["connection_bytes_sent_speech"] = not_set;
+        bulk.put_unchecked(property::CONNECTION_BYTES_RECEIVED_CONTROL, not_set);
+        bulk.put_unchecked(property::CONNECTION_BYTES_RECEIVED_KEEPALIVE, not_set);
+        bulk.put_unchecked(property::CONNECTION_BYTES_RECEIVED_SPEECH, not_set);
+        bulk.put_unchecked(property::CONNECTION_BYTES_SENT_CONTROL, not_set);
+        bulk.put_unchecked(property::CONNECTION_BYTES_SENT_KEEPALIVE, not_set);
+        bulk.put_unchecked(property::CONNECTION_BYTES_SENT_SPEECH, not_set);
 
         /* its flipped here because the report is out of the clients view */
-        notify["connection_packets_received_control"] = not_set;
-        notify["connection_packets_received_keepalive"] = not_set;
-        notify["connection_packets_received_speech"] = not_set;
-        notify["connection_packets_sent_control"] = not_set;
-        notify["connection_packets_sent_keepalive"] = not_set;
-        notify["connection_packets_sent_speech"] = not_set;
+        bulk.put_unchecked(property::CONNECTION_PACKETS_RECEIVED_CONTROL, not_set);
+        bulk.put_unchecked(property::CONNECTION_PACKETS_RECEIVED_KEEPALIVE, not_set);
+        bulk.put_unchecked(property::CONNECTION_PACKETS_RECEIVED_SPEECH, not_set);
+        bulk.put_unchecked(property::CONNECTION_PACKETS_SENT_CONTROL, not_set);
+        bulk.put_unchecked(property::CONNECTION_PACKETS_SENT_KEEPALIVE, not_set);
+        bulk.put_unchecked(property::CONNECTION_PACKETS_SENT_SPEECH, not_set);
 
-        notify["connection_server2client_packetloss_control"] = not_set;
-        notify["connection_server2client_packetloss_keepalive"] = not_set;
-        notify["connection_server2client_packetloss_speech"] = not_set;
-        notify["connection_server2client_packetloss_total"] = not_set;
+        bulk.put_unchecked(property::CONNECTION_SERVER2CLIENT_PACKETLOSS_CONTROL, not_set);
+        bulk.put_unchecked(property::CONNECTION_SERVER2CLIENT_PACKETLOSS_KEEPALIVE, not_set);
+        bulk.put_unchecked(property::CONNECTION_SERVER2CLIENT_PACKETLOSS_SPEECH, not_set);
+        bulk.put_unchecked(property::CONNECTION_SERVER2CLIENT_PACKETLOSS_TOTAL, not_set);
 
-        notify["connection_ping"] = 0;
-        notify["connection_ping_deviation"] = 0;
+        bulk.put_unchecked(property::CONNECTION_CLIENT2SERVER_PACKETLOSS_SPEECH, not_set);
+        bulk.put_unchecked(property::CONNECTION_CLIENT2SERVER_PACKETLOSS_KEEPALIVE, not_set);
+        bulk.put_unchecked(property::CONNECTION_CLIENT2SERVER_PACKETLOSS_CONTROL, not_set);
+        bulk.put_unchecked(property::CONNECTION_CLIENT2SERVER_PACKETLOSS_TOTAL, not_set);
 
-        notify["connection_connected_time"] = 0;
-        notify["connection_idle_time"] = 0;
+        bulk.put_unchecked(property::CONNECTION_PING, 0);
+        bulk.put_unchecked(property::CONNECTION_PING_DEVIATION, 0);
+
+        bulk.put_unchecked(property::CONNECTION_CONNECTED_TIME, 0);
+        bulk.put_unchecked(property::CONNECTION_IDLE_TIME, 0);
 
         /* its flipped here because the report is out of the clients view */
-        notify["connection_filetransfer_bandwidth_sent"] = report.file_bytes_received;
-        notify["connection_filetransfer_bandwidth_received"] = report.file_bytes_sent;
+        bulk.put_unchecked(property::CONNECTION_FILETRANSFER_BANDWIDTH_SENT, file_stats.bytes_received);
+        bulk.put_unchecked(property::CONNECTION_FILETRANSFER_BANDWIDTH_RECEIVED, file_stats.bytes_sent);
     }
 
     if(info) {
-        for(const auto& elm : info->properties) {
-            notify[elm.first] = elm.second;
-        }
+        for(const auto& [key, value] : info->properties)
+            bulk.put(key, value);
     } else {
-        //Fill in some server stuff
-        if(dynamic_pointer_cast<VoiceClient>(target))
-            notify["connection_ping"] = floor<milliseconds>(dynamic_pointer_cast<VoiceClient>(target)->calculatePing()).count();
+        //Fill in what we can, else we trust the client
+        if(target->getType() == ClientType::CLIENT_TEASPEAK || target->getType() == ClientType::CLIENT_TEAMSPEAK || target->getType() == ClientType::CLIENT_WEB) {
+            auto& stats = target->connectionStatistics->total_stats();
+            /* its flipped here because the report is out of the clients view */
+            bulk.put(property::CONNECTION_BYTES_RECEIVED_CONTROL, stats.connection_bytes_received[stats::ConnectionStatistics::category::COMMAND]);
+            bulk.put(property::CONNECTION_BYTES_RECEIVED_KEEPALIVE, stats.connection_bytes_received[stats::ConnectionStatistics::category::KEEP_ALIVE]);
+            bulk.put(property::CONNECTION_BYTES_RECEIVED_SPEECH, stats.connection_bytes_received[stats::ConnectionStatistics::category::VOICE]);
+            bulk.put(property::CONNECTION_BYTES_SENT_CONTROL, stats.connection_bytes_sent[stats::ConnectionStatistics::category::COMMAND]);
+            bulk.put(property::CONNECTION_BYTES_SENT_KEEPALIVE, stats.connection_bytes_sent[stats::ConnectionStatistics::category::KEEP_ALIVE]);
+            bulk.put(property::CONNECTION_BYTES_SENT_SPEECH, stats.connection_bytes_sent[stats::ConnectionStatistics::category::VOICE]);
+
+            /* its flipped here because the report is out of the clients view */
+            bulk.put(property::CONNECTION_PACKETS_RECEIVED_CONTROL, stats.connection_packets_received[stats::ConnectionStatistics::category::COMMAND]);
+            bulk.put(property::CONNECTION_PACKETS_RECEIVED_KEEPALIVE, stats.connection_packets_received[stats::ConnectionStatistics::category::KEEP_ALIVE]);
+            bulk.put(property::CONNECTION_PACKETS_RECEIVED_SPEECH, stats.connection_packets_received[stats::ConnectionStatistics::category::VOICE]);
+            bulk.put(property::CONNECTION_PACKETS_SENT_CONTROL, stats.connection_packets_sent[stats::ConnectionStatistics::category::COMMAND]);
+            bulk.put(property::CONNECTION_PACKETS_SENT_KEEPALIVE, stats.connection_packets_sent[stats::ConnectionStatistics::category::KEEP_ALIVE]);
+            bulk.put(property::CONNECTION_PACKETS_SENT_SPEECH, stats.connection_packets_sent[stats::ConnectionStatistics::category::VOICE]);
+        }
+    }
+    if(auto vc = dynamic_pointer_cast<VoiceClient>(target); vc) {
+        bulk.put(property::CONNECTION_PING, floor<milliseconds>(vc->current_ping()).count());
+        bulk.put(property::CONNECTION_PING_DEVIATION, vc->current_ping_deviation());
+    }
 #ifdef COMPILE_WEB_CLIENT
-        else if(dynamic_pointer_cast<WebClient>(target))
-            notify["connection_ping"] = floor<milliseconds>(dynamic_pointer_cast<WebClient>(target)->client_ping()).count();
+    else if(dynamic_pointer_cast<WebClient>(target))
+        bulk.put(property::CONNECTION_PING, floor<milliseconds>(dynamic_pointer_cast<WebClient>(target)->client_ping()).count());
 #endif
 
-        if(target->getType() == ClientType::CLIENT_TEASPEAK || target->getType() == ClientType::CLIENT_TEAMSPEAK || target->getType() == ClientType::CLIENT_WEB) {
-            auto report = target->connectionStatistics->full_report();
-
-            /* its flipped here because the report is out of the clients view */
-            notify["connection_bytes_received_control"] = report.connection_bytes_sent[stats::ConnectionStatistics::category::COMMAND];
-            notify["connection_bytes_received_keepalive"] = report.connection_bytes_sent[stats::ConnectionStatistics::category::ACK];
-            notify["connection_bytes_received_speech"] = report.connection_bytes_sent[stats::ConnectionStatistics::category::VOICE];
-            notify["connection_bytes_sent_control"] = report.connection_bytes_sent[stats::ConnectionStatistics::category::COMMAND];
-            notify["connection_bytes_sent_keepalive"] = report.connection_bytes_sent[stats::ConnectionStatistics::category::ACK];
-            notify["connection_bytes_sent_speech"] = report.connection_bytes_sent[stats::ConnectionStatistics::category::VOICE];
-
-            /* its flipped here because the report is out of the clients view */
-            notify["connection_packets_received_control"] = report.connection_packets_sent[stats::ConnectionStatistics::category::COMMAND];
-            notify["connection_packets_received_keepalive"] = report.connection_packets_sent[stats::ConnectionStatistics::category::ACK];
-            notify["connection_packets_received_speech"] = report.connection_packets_sent[stats::ConnectionStatistics::category::VOICE];
-            notify["connection_packets_sent_control"] = report.connection_packets_sent[stats::ConnectionStatistics::category::COMMAND];
-            notify["connection_packets_sent_keepalive"] = report.connection_packets_sent[stats::ConnectionStatistics::category::ACK];
-            notify["connection_packets_sent_speech"] = report.connection_packets_sent[stats::ConnectionStatistics::category::VOICE];
-        }
+    if(auto vc = dynamic_pointer_cast<VoiceClient>(target); vc){
+        auto& calculator = vc->connection->packet_statistics();
+        auto report = calculator.loss_report();
+        bulk.put(property::CONNECTION_CLIENT2SERVER_PACKETLOSS_SPEECH, std::to_string(report.voice_loss()));
+        bulk.put(property::CONNECTION_CLIENT2SERVER_PACKETLOSS_KEEPALIVE, std::to_string(report.keep_alive_loss()));
+        bulk.put(property::CONNECTION_CLIENT2SERVER_PACKETLOSS_CONTROL, std::to_string(report.control_loss()));
+        bulk.put(property::CONNECTION_CLIENT2SERVER_PACKETLOSS_TOTAL, std::to_string(report.total_loss()));
     }
 
     if(target->getClientId() == this->getClientId() || permission::v2::permission_granted(1, this->calculate_permission(permission::b_client_remoteaddress_view, this->getChannelId()))) {
-        notify["connection_client_ip"] = target->getLoggingPeerIp();
-        notify["connection_client_port"] = target->getPeerPort();
+        bulk.put(property::CONNECTION_CLIENT_IP, target->getLoggingPeerIp());
+        bulk.put(property::CONNECTION_CLIENT_PORT, target->getPeerPort());
     }
 
-    //Needs to be filled out
-    notify["connection_client2server_packetloss_speech"] = not_set;
-    notify["connection_client2server_packetloss_keepalive"] = not_set;
-    notify["connection_client2server_packetloss_control"] = not_set;
-    notify["connection_client2server_packetloss_total"] = not_set;
-
-    notify["connection_connected_time"] = chrono::duration_cast<chrono::milliseconds>(chrono::system_clock::now() - target->connectTimestamp).count();
-    notify["connection_idle_time"] = chrono::duration_cast<chrono::milliseconds>(chrono::system_clock::now() - target->idleTimestamp).count();
+    bulk.put(property::CONNECTION_CONNECTED_TIME, chrono::duration_cast<chrono::milliseconds>(chrono::system_clock::now() - target->connectTimestamp).count());
+    bulk.put(property::CONNECTION_IDLE_TIME, chrono::duration_cast<chrono::milliseconds>(chrono::system_clock::now() - target->idleTimestamp).count());
     this->sendCommand(notify);
     return true;
 }
@@ -404,7 +409,7 @@ bool ConnectedClient::notifyClientMoved(const shared_ptr<ConnectedClient> &clien
     return true;
 }
 
-bool ConnectedClient::notifyClientUpdated(const std::shared_ptr<ConnectedClient> &client, const deque<shared_ptr<property::PropertyDescription>> &props, bool lock) {
+bool ConnectedClient::notifyClientUpdated(const std::shared_ptr<ConnectedClient> &client, const deque<const property::PropertyDescription*> &props, bool lock) {
     shared_lock channel_lock(this->channel_lock, defer_lock);
     if(lock)
         channel_lock.lock();
@@ -420,7 +425,7 @@ bool ConnectedClient::notifyClientUpdated(const std::shared_ptr<ConnectedClient>
     Command response("notifyclientupdated");
     response["clid"] = client_id;
     for (const auto &prop : props) {
-        if(lastOnlineTimestamp.time_since_epoch().count() > 0 && (prop->property_index == property::CLIENT_TOTAL_ONLINE_TIME || prop->property_index == property::CLIENT_MONTH_ONLINE_TIME))
+        if(lastOnlineTimestamp.time_since_epoch().count() > 0 && (*prop == property::CLIENT_TOTAL_ONLINE_TIME || *prop == property::CLIENT_MONTH_ONLINE_TIME))
             response[prop->name] = client->properties()[prop].as<int64_t>() + duration_cast<seconds>(system_clock::now() - client->lastOnlineTimestamp).count();
         else
             response[prop->name] = client->properties()[prop].value();
@@ -650,7 +655,7 @@ bool ConnectedClient::notifyClientEnterView(const std::deque<std::shared_ptr<Con
 
         this->visibleClients.push_back(client);
         for (const auto &elm : client->properties()->list_properties(property::FLAG_CLIENT_VIEW, this->getType() == CLIENT_TEAMSPEAK ? property::FLAG_NEW : (uint16_t) 0)) {
-            cmd[index][elm.type().name] = elm.value();
+            cmd[index][std::string{elm.type().name}] = elm.value();
         }
 
         index++;
@@ -678,14 +683,14 @@ bool ConnectedClient::notifyChannelEdited(
 
     Command notify("notifychanneledited");
     for(auto prop : properties) {
-        const auto& prop_info = property::impl::info(prop);
+        const auto& prop_info = property::describe(prop);
 
         if(prop == property::CHANNEL_ORDER)
-            notify[prop_info->name] = v_channel->previous_channel;
+            notify[prop_info.name] = v_channel->previous_channel;
         else if(prop == property::CHANNEL_DESCRIPTION) {
             send_description_change = true;
         } else {
-            notify[prop_info->name] = channel->properties()[prop].as<string>();
+            notify[prop_info.name] = channel->properties()[prop].as<string>();
         }
     }
 

server/src/client/ConnectedClientTextCommandHandler.cpp

@@ -158,8 +158,8 @@ bool ConnectedClient::handle_text_command(
         if (TARG(0, "create")) {
             Command cmd("");
             auto result = this->handleCommandMusicBotCreate(cmd);
-            if(result.error_code()) {
-                result.release_details();
+            if(result.has_error()) {
+                result.release_data();
                 HANDLE_CMD_ERROR("Failed to create music bot");
                 return true;
             }
@@ -220,9 +220,9 @@ bool ConnectedClient::handle_text_command(
             Command cmd("");
             cmd["client_nickname"] = ss.str();
             auto result = this->handleCommandClientEdit(cmd, bot);
-            if(result.error_code()) {
+            if(result.has_error()) {
                 HANDLE_CMD_ERROR("Failed to rename bot");
-                result.release_details();
+                result.release_data();
                 return true;
             }
             send_message(serverInstance->musicRoot(), "Name successfully changed!");
@@ -391,8 +391,8 @@ bool ConnectedClient::handle_text_command(
                 for(const auto& song : bot_playlist->list_songs()) {
                     string invoker = "unknown";
                     for(const auto& e : dbinfo) {
-                        if(e->cldbid == song->invoker) {
-                            invoker = "[URL=client://0/" + e->uniqueId + "~" + e->lastName + "]" + e->lastName + "[/URL]";
+                        if(e->client_database_id == song->invoker) {
+                            invoker = "[URL=client://0/" + e->client_unique_id + "~" + e->client_nickname + "]" + e->client_nickname + "[/URL]";
                             break;
                         }
                     }
@@ -486,7 +486,7 @@ bool ConnectedClient::handle_text_command(
                     for(const auto& property : bot->properties()->list_properties(~0)) {
                         if(find(editable_properties.begin(), editable_properties.end(), property.type().name) == editable_properties.end()) continue;
 
-                        send_message(bot, " - " + property.type().name + " = " + property.value() + " " + (property.default_value() == property.value() ? "(default)" : ""));
+                        send_message(bot, " - " + std::string{property.type().name} + " = " + property.value() + " " + (property.default_value() == property.value() ? "(default)" : ""));
                     }
                 } else if(arguments.size() < 4) {
                     if(find(editable_properties.begin(), editable_properties.end(), arguments[2]) == editable_properties.end()) {
@@ -494,23 +494,23 @@ bool ConnectedClient::handle_text_command(
                         return true;
                     }
 
-                    const std::shared_ptr<property::PropertyDescription> &property_info = property::info<property::ClientProperties>(arguments[2]);
-                    if(!property_info || property_info->type_property == property::PROP_TYPE_UNKNOWN || property_info->property_index == property::CLIENT_UNDEFINED) {
+                    const auto &property_info = property::find<property::ClientProperties>(arguments[2]);
+                    if(property_info.is_undefined()) {
                         send_message(bot, "Unknown property " + arguments[2] + ".");
                         return true;
                     }
 
-                    auto prop = bot->properties()[(property::ClientProperties) property_info->property_index];
-                    send_message(bot, "Bot property " + property_info->name + " = " + prop.value() + " " + (property_info->default_value == prop.value() ? "(default)" : ""));
+                    auto prop = bot->properties()[(property::ClientProperties) property_info.property_index];
+                    send_message(bot, "Bot property " + std::string{property_info.name} + " = " + prop.value() + " " + (property_info.default_value == prop.value() ? "(default)" : ""));
                     return true;
                 } else {
                     Command cmd("");
                     JOIN_ARGS(value, 3);
                     cmd[arguments[2]] = value;
                     auto result = this->handleCommandClientEdit(cmd, bot);
-                    if(result.error_code()) {
+                    if(result.has_error()) {
                         HANDLE_CMD_ERROR("Failed to change bot property");
-                        result.release_details();
+                        result.release_data();
                         return true;
                     }
                     send_message(serverInstance->musicRoot(), "Property successfully changed!");
@@ -527,36 +527,36 @@ bool ConnectedClient::handle_text_command(
                 if(arguments.size() < 3) {
                     send_message(bot, "Playlist properties:");
                     for(const auto& property : playlist->properties().list_properties(property::FLAG_PLAYLIST_VARIABLE)) {
-                        send_message(bot, " - " + property.type().name + " = " + property.value() + " " + (property.default_value() == property.value() ? "(default)" : ""));
+                        send_message(bot, " - " + std::string{property.type().name} + " = " + property.value() + " " + (property.default_value() == property.value() ? "(default)" : ""));
                     }
                 } else if(arguments.size() < 4) {
-                    const std::shared_ptr<property::PropertyDescription> &property_info = property::info<property::PlaylistProperties>(arguments[2]);
-                    if(!property_info || property_info->type_property == property::PROP_TYPE_UNKNOWN || property_info->property_index == property::PLAYLIST_UNDEFINED) {
+                    const auto &property_info = property::find<property::PlaylistProperties>(arguments[2]);
+                    if(property_info.is_undefined()) {
                         send_message(bot, "Unknown property " + arguments[2] + ".");
                         return true;
                     }
 
-                    auto prop = playlist->properties()[(property::PlaylistProperties) property_info->property_index];
-                    send_message(bot, "Bot property " + property_info->name + " = " + prop.value() + " " + (property_info->default_value == prop.value() ? "(default)" : ""));
+                    auto prop = playlist->properties()[(property::PlaylistProperties) property_info.property_index];
+                    send_message(bot, "Bot property " + std::string{property_info.name} + " = " + prop.value() + " " + (property_info.default_value == prop.value() ? "(default)" : ""));
                 } else {
-                    const std::shared_ptr<property::PropertyDescription> &property_info = property::info<property::PlaylistProperties>(arguments[2]);
-                    if(!property_info || property_info->type_property == property::PROP_TYPE_UNKNOWN || property_info->property_index == property::PLAYLIST_UNDEFINED) {
+                    const auto &property_info = property::find<property::PlaylistProperties>(arguments[2]);
+                    if(property_info.is_undefined()) {
                         send_message(bot, "Unknown property " + arguments[2] + ".");
                         return true;
                     }
 
                     JOIN_ARGS(value, 3);
-                    if(!property_info->validate_input(value)) {
+                    if(!property_info.validate_input(value)) {
                         send_message(bot, "Please enter a valid value!");
                         return true;
                     }
 
-                    if((property_info->flags & property::FLAG_USER_EDITABLE) == 0) {
+                    if((property_info.flags & property::FLAG_USER_EDITABLE) == 0) {
                         send_message(bot, "This property isnt changeable!");
                         return true;
                     }
 
-                    playlist->properties()[(property::PlaylistProperties) property_info->property_index] = value;
+                    playlist->properties()[(property::PlaylistProperties) property_info.property_index] = value;
                     send_message(bot, "Property successfully changed");
                     return true;
                 }
@@ -639,13 +639,58 @@ bool ConnectedClient::handle_text_command(
                     protocol::PacketTypeInfo::Ping,
                     protocol::PacketTypeInfo::Pong}) {
 
-                auto id = vc->getConnection()->getPacketIdManager().currentPacketId(type);
-                auto gen = vc->getConnection()->getPacketIdManager().generationId(type);
-                send_message(_this.lock(), " OUT " + type.name() + " => generation: " + to_string(gen) + " id: " + to_string(id));
-                //auto& buffer = vc->getConnection()->packet_buffers()[type.type()];
-                //send_message(_this.lock(), " IN  " + type.name() + " => generation: " + to_string(buffer.generation(0)) + " id: " + to_string(buffer.current_index()));
+                //auto id = vc->getConnection()->getPacketIdManager().currentPacketId(type);
+                //auto gen = vc->getConnection()->getPacketIdManager().generationId(type);
+                //auto& genestis = vc->getConnection()->get_incoming_generation_estimators();
+
+                //send_message(_this.lock(), " OUT " + type.name() + " => generation: " + to_string(gen) + " id: " + to_string(id));
+                //send_message(_this.lock(), " IN  " + type.name() + " => generation: " + to_string(genestis[type.type()].generation()) + " id: " + to_string(genestis[type.type()].current_packet_id()));
             }
             return true;
+        } else if(TARG(0, "rtt")) {
+            auto vc = dynamic_pointer_cast<VoiceClient>(_this.lock());
+            if(!vc) return false;
+
+            auto& ack = vc->connection->packet_encoder().acknowledge_manager();
+            send_message(_this.lock(), "Command retransmission values:");
+            send_message(_this.lock(), " RTO   : " + std::to_string(ack.current_rto()));
+            send_message(_this.lock(), " RTTVAR: " + std::to_string(ack.current_rttvar()));
+            send_message(_this.lock(), " SRTT  : " + std::to_string(ack.current_srtt()));
+            return true;
+        } else if(TARG(0, "sgeneration")) {
+            TLEN(4);
+
+            try {
+                auto type = stol(arguments[1]);
+                auto generation = stol(arguments[2]);
+                auto pid = stol(arguments[3]);
+
+                auto vc = dynamic_pointer_cast<VoiceClient>(_this.lock());
+                if(!vc) return false;
+
+                /*
+                auto& genestis = vc->getConnection()->get_incoming_generation_estimators();
+                if(type >= genestis.size()) {
+                    send_message(_this.lock(), "Invalid type");
+                    return true;
+                }
+                genestis[type].set_last_state(pid, generation);
+                */
+            } catch(std::exception& ex) {
+                send_message(_this.lock(), "Failed to parse argument");
+                return true;
+            }
+            return true;
+        } else if(TARG(0, "ping")) {
+            auto vc = dynamic_pointer_cast<VoiceClient>(_this.lock());
+            if(!vc) return false;
+
+            auto ping_handler = vc->getConnection()->ping_handler();
+            send_message(this->ref(), "Ping: " + std::to_string(ping_handler.current_ping().count()));
+
+            auto last_response_ms = std::chrono::ceil<std::chrono::milliseconds>(std::chrono::system_clock::now() - ping_handler.last_ping_response()).count();
+            send_message(this->ref(), "Last ping response: " + std::to_string(last_response_ms));
+            return true;
         } else if(TARG(0, "disconnect")) {
             auto vc = dynamic_pointer_cast<VoiceClient>(_this.lock());
             if(!vc) return false;
@@ -659,8 +704,8 @@ bool ConnectedClient::handle_text_command(
             if(!vc) return false;
 
             send_message(_this.lock(), "I lost your IP address. I'm so dump :)");
+            vc->connection->reset_remote_address();
             memset(&vc->remote_address, 0, sizeof(vc->remote_address));
-            memset(&vc->address_info, 0, sizeof(vc->address_info));
             send_message(_this.lock(), "Hey, we got the address back");
             return true;
         } else if(TARG(0, "fb")) {

server/src/client/DataClient.cpp

@@ -1,12 +1,10 @@
 #include <log/LogUtils.h>
-#include <src/client/voice/VoiceClient.h>
 #include <misc/sassert.h>
 #include <misc/hex.h>
-#include "DataClient.h"
-#include "ConnectedClient.h"
-#include "src/server/file/FileServer.h"
-#include "src/InstanceHandler.h"
-#include "misc/base64.h"
+#include <misc/base64.h>
+
+#include "./DataClient.h"
+#include "../InstanceHandler.h"
 
 using namespace std;
 using namespace ts;
@@ -23,52 +21,78 @@ DataClient::~DataClient() {
     this->_properties = nullptr;
 }
 
-bool DataClient::loadDataForCurrentServer() { //TODO for query
-    if(this->getUid().empty()) return false;
+constexpr static std::string_view kClientLoadCommand{R"(
+    SELECT
+        `client_database_id`,
+        `client_created`,
+
+        `client_last_connected`,
+        `client_total_connections`,
+
+        `client_month_upload`,
+        `client_month_download`,
+
+        `client_total_upload`,
+        `client_total_download`
+    FROM `clients_server` WHERE `server_id` = :sid AND `client_unique_id` = :uid
+)"};
+
+bool DataClient::loadDataForCurrentServer() {
+    auto uniqueId = this->getUid();
+    if(uniqueId.empty())
+        return false;
+
     auto ref_server = this->server;
     auto server_id = ref_server ? ref_server->getServerId() : 0;
 
     properties()[property::CLIENT_DATABASE_ID] = 0;
-    properties()[property::CLIENT_CREATED] = 0;
-    properties()[property::CLIENT_TOTALCONNECTIONS] = 0;
+    this->clientPermissions = std::make_shared<permission::v2::PermissionManager>();
 
+    auto properties = this->properties();
+    sql::command{this->sql, std::string{kClientLoadCommand}, variable{":uid", uniqueId}, variable{":sid", server_id}}.query([&](int length, std::string* values, std::string* names) {
+        auto index{0};
 
-    ClientDbId client_db_id = 0;
-    sql::command(this->sql, "SELECT `cldbid`,`firstConnect`,`connections` FROM `clients` WHERE `serverId` = :sid AND `clientUid` = :uid LIMIT 1",
-            variable{":sid", server_id},
-            variable{":uid", this->getUid()}
-    ).query([&](DataClient* cl, int length, string* values, string* names){
-        for (int index = 0; index < length; index++) {
-            try {
-                if (names[index] == "cldbid") {
-                    client_db_id = stoull(values[index]);
-                } else if (names[index] == "firstConnect") {
-                    cl->properties()[property::CLIENT_CREATED] = values[index];
-                } else if (names[index] == "connections") {
-                    cl->properties()[property::CLIENT_TOTALCONNECTIONS] = values[index];
-                } else {
-                    logWarning(LOG_INSTANCE, "Received unknown column with name {} within client list", names[index]);
-                }
-            } catch(const std::exception& ex) {
-                logError(server_id, "Failed to load client {} base properties from database. Colum parsing for column {} failed. Value: {}. Message: {}",
-                   this->getUid(),
-                   names[index],
-                   values[index],
-                   ex.what()
-                );
-                return 0;
-            }
+        try {
+            assert(names[index] == "client_database_id");
+            properties[property::CLIENT_DATABASE_ID] = std::stoull(values[index++]);
+
+            assert(names[index] == "client_created");
+            properties[property::CLIENT_CREATED] = std::stoull(values[index++]);
+
+            assert(names[index] == "client_last_connected");
+            properties[property::CLIENT_LASTCONNECTED] = std::stoull(values[index++]);
+
+            assert(names[index] == "client_total_connections");
+            properties[property::CLIENT_TOTALCONNECTIONS] = std::stoull(values[index++]);
+
+            assert(names[index] == "client_month_upload");
+            properties[property::CLIENT_MONTH_BYTES_UPLOADED] = std::stoull(values[index++]);
+
+            assert(names[index] == "client_month_download");
+            properties[property::CLIENT_MONTH_BYTES_DOWNLOADED] = std::stoull(values[index++]);
+
+            assert(names[index] == "client_total_upload");
+            properties[property::CLIENT_TOTAL_BYTES_UPLOADED] = std::stoull(values[index++]);
+
+            assert(names[index] == "client_total_download");
+            properties[property::CLIENT_TOTAL_BYTES_DOWNLOADED] = std::stoull(values[index++]);
+
+            assert(index == length);
+        } catch (std::exception& ex) {
+            logError(server_id, "Failed to load client {} base properties from database: {} (Index {})",
+                     this->getUid(),
+                     ex.what(),
+                     index - 1
+            );
+            properties[property::CLIENT_DATABASE_ID] = 0;
         }
-        return 0;
-    }, this);
+    });
 
-    if(client_db_id == 0)
+    if(this->properties()[property::CLIENT_DATABASE_ID].as<ClientDbId>() == 0)
         return false;
 
-    this->properties()[property::CLIENT_DATABASE_ID] = client_db_id; /* do this before the property saving (it saved the cldbid as well!)*/
-
     //Load general properties
-    deque<ts::PropertyWrapper> copied;
+    std::deque<ts::PropertyWrapper> copied;
     for(const auto& prop : this->_properties->list_properties()){
         if((prop.type().flags & property::FLAG_GLOBAL) == 0) continue;
         if(prop.type().default_value == prop.value()) continue;
@@ -81,28 +105,7 @@ bool DataClient::loadDataForCurrentServer() { //TODO for query
         return false;
     }
 
-    auto client_type = this->getType();
-    if(client_type == CLIENT_TEAMSPEAK || ref_server) {
-        this->_properties = serverInstance->databaseHelper()->loadClientProperties(ref_server, this->getClientDatabaseId(), client_type);
-    } else {
-        this->_properties = DatabaseHelper::default_properties_client(nullptr, client_type);
-
-        this->_properties->registerNotifyHandler([&, server_id, client_db_id](Property& prop){
-            std::string query;
-            if(prop.type() == property::CLIENT_TOTALCONNECTIONS)
-                query = "UPDATE `clients` SET `connections` = :value WHERE `serverId` = :sid AND `cldbid` = :cldbid";
-            else if(prop.type() == property::CLIENT_NICKNAME)
-                query = "UPDATE `clients` SET `lastName` = :value WHERE `serverId` = :sid AND `cldbid` = :cldbid";
-            else if(prop.type() == property::CLIENT_LASTCONNECTED)
-                query = "UPDATE `clients` SET `lastConnect` = :value WHERE `serverId` = :sid AND `cldbid` = :cldbid";
-            else
-                return;
-
-            debugMessage(server_id, "[Property] Updating general client table property for client {}. Key: {} Value: {}", client_db_id, prop.type().name, prop.value());
-            sql::command(this->sql, query, variable{":sid", 0}, variable{":cldbid", client_db_id}, variable{":value", prop.value()}).executeLater()
-            .waitAndGetLater(LOG_SQL_CMD, {1, "failed to update general client properties"});
-        });
-    }
+    this->_properties = serverInstance->databaseHelper()->loadClientProperties(ref_server, this->getClientDatabaseId(), this->getType());
 
     this->_properties->toggleSave(false);
     for(const auto& e : copied) {
@@ -115,6 +118,7 @@ bool DataClient::loadDataForCurrentServer() { //TODO for query
     this->clientPermissions = serverInstance->databaseHelper()->loadClientPermissionManager(ref_server, this->getClientDatabaseId());
 
     //Setup / fix stuff
+#if 0
     if(!this->properties()[property::CLIENT_FLAG_AVATAR].as<string>().empty()){
         if(
                 !ref_server ||
@@ -123,6 +127,7 @@ bool DataClient::loadDataForCurrentServer() { //TODO for query
                 this->properties()[property::CLIENT_FLAG_AVATAR] = "";
         }
     }
+#endif
 
     if(ref_server)
         this->properties()[property::CLIENT_UNREAD_MESSAGES] = ref_server->letters->unread_letter_count(this->getUid());

server/src/client/DataClient.h

@@ -45,10 +45,17 @@ namespace ts {
                         return std::forward<F>(f)(*handle);
                     }
 
+                    /*
                     template <typename T>
                     ts::PropertyWrapper operator[](T type) {
                         return (*handle)[type];
                     }
+                    */
+
+                    template <typename T>
+                    ts::PropertyWrapper operator[](const T& type) {
+                        return (*handle)[type];
+                    }
                 };
 
                 DataClient(sql::SqlManager*, const std::shared_ptr<VirtualServer>&);

server/src/client/InternalClient.h

@@ -2,24 +2,23 @@
 
 #include "ConnectedClient.h"
 
-namespace ts {
-    namespace server {
-        class InternalClient : public ConnectedClient {
-            public:
-                InternalClient(sql::SqlManager*, const std::shared_ptr<server::VirtualServer>&, std::string, bool);
-                ~InternalClient();
+namespace ts::server {
+    class VirtualServer;
+    class InternalClient : public ConnectedClient {
+        public:
+            InternalClient(sql::SqlManager*, const std::shared_ptr<VirtualServer>&, std::string, bool);
+            ~InternalClient();
 
-                void setSharedLock(const std::shared_ptr<ConnectedClient>& _this){
-                    assert(_this.get() == this);
-                    this->_this = _this;
-                }
+            void setSharedLock(const std::shared_ptr<ConnectedClient>& _this){
+                assert(_this.get() == this);
+                this->_this = _this;
+            }
 
-                void sendCommand(const ts::Command &command, bool low) override;
-                void sendCommand(const ts::command_builder &command, bool low) override;
-                bool close_connection(const std::chrono::system_clock::time_point& timeout = std::chrono::system_clock::time_point()) override;
-                bool disconnect(const std::string &reason) override;
-            protected:
-                void tick(const std::chrono::system_clock::time_point &time) override;
-        };
-    }
+            void sendCommand(const ts::Command &command, bool low) override;
+            void sendCommand(const ts::command_builder &command, bool low) override;
+            bool close_connection(const std::chrono::system_clock::time_point& timeout = std::chrono::system_clock::time_point()) override;
+            bool disconnect(const std::string &reason) override;
+        protected:
+            void tick(const std::chrono::system_clock::time_point &time) override;
+    };
 }

server/src/client/SpeakingClient.cpp

@@ -1,17 +1,19 @@
 #include <memory>
 
+#include <PermissionManager.h>
 #include <misc/endianness.h>
 #include <log/LogUtils.h>
 #include <ThreadPool/Timer.h>
 #include <regex>
 #include <src/build.h>
 #include <Properties.h>
+#include <src/client/command_handler/helpers.h>
 #include "src/channel/ClientChannelView.h"
 #include "SpeakingClient.h"
 #include "src/InstanceHandler.h"
 #include "StringVariable.h"
-#include "src/music/MusicBotManager.h"
 #include "misc/timer.h"
+#include "../manager/ActionLogger.h"
 
 using namespace std::chrono;
 using namespace ts;
@@ -20,6 +22,9 @@ using namespace ts::protocol;
 
 //#define PKT_LOG_VOICE
 //#define PKT_LOG_WHISPER
+constexpr static auto kMaxWhisperClientNameLength{30};
+constexpr static auto kWhisperClientUniqueIdLength{28}; /* base64 encoded SHA1 hash */
+constexpr static auto kWhisperMaxHeaderLength{2 + 2 + 1 + 2 + kWhisperClientUniqueIdLength + 1 + kMaxWhisperClientNameLength};
 
 bool SpeakingClient::shouldReceiveVoice(const std::shared_ptr<ConnectedClient> &sender) {
     //if(this->properties()[property::CLIENT_AWAY].as<bool>()) return false;
@@ -52,6 +57,14 @@ void SpeakingClient::handlePacketVoice(const pipes::buffer_view& data, bool head
         return;
     }
 
+#if 0
+    if(rand() % 10 == 0) {
+        logMessage(0, "Dropping audio packet");
+        return;
+    }
+    logMessage(0, "Received voice: Head: {} Fragmented: {}, length: {}", head, fragmented, data.length());
+#endif
+
     auto current_channel = this->currentChannel;
     if(!current_channel) { return; }
     if(!this->allowedToTalk) { return; }
@@ -65,7 +78,6 @@ void SpeakingClient::handlePacketVoice(const pipes::buffer_view& data, bool head
         if(!speaking_client) return true;
 
         return !speaking_client->shouldReceiveVoice(self);
-
     }), target_clients.end());
     if(target_clients.empty()) {
         return;
@@ -100,9 +112,10 @@ void SpeakingClient::handlePacketVoice(const pipes::buffer_view& data, bool head
         memcpy(&buffer[5], &data[3], data.length() - 3);
     }
 
+    auto bview = pipes::buffer_view{buffer, data.length() + 2};
     for (const auto& client : target_clients) {
         auto speaking_client = static_pointer_cast<SpeakingClient>(client);
-        speaking_client->send_voice_packet(pipes::buffer_view{buffer, data.length() + 2}, flags);
+        speaking_client->send_voice_packet(bview, flags);
     }
 }
 
@@ -114,7 +127,9 @@ enum WhisperType {
     SERVER_GROUP = 0,
     CHANNEL_GROUP = 1,
     CHANNEL_COMMANDER = 2,
-    ALL = 3
+    ALL = 3,
+
+    ECHO = 0x10,
 };
 
 enum WhisperTarget {
@@ -140,232 +155,271 @@ inline bool update_whisper_error(std::chrono::system_clock::time_point& last) {
 //Server group => type := SERVER_GROUP and target_id := <server group id>
 //Channel group => type := CHANNEL_GROUP and target_id := <channel group id>
 //Channel commander => type := CHANNEL_COMMANDER and target_id := 0
-void SpeakingClient::handlePacketVoiceWhisper(const pipes::buffer_view& data, bool new_packet) {
-    if(data.length() < 5) {
+void SpeakingClient::handlePacketVoiceWhisper(const pipes::buffer_view& payload, bool new_packet, bool head) {
+    if(payload.length() < 5) {
         this->disconnect("Invalid packet (Voice whisper)");
-        logMessage(this->getServerId(), "{} Tried to send a too short whisper packet. Length: {}", CLIENT_STR_LOG_PREFIX, data.length());
+        logMessage(this->getServerId(), "{} Tried to send a too short whisper packet. Length: {}", CLIENT_STR_LOG_PREFIX, payload.length());
         return;
     }
 
-    uint16_t offset = 0;
-    auto vpacketId = be2le16((char*) data.data_ptr(), offset, &offset);
-    auto codec = (uint8_t) data[offset++];
+    uint16_t payload_offset{0};
+    auto voice_packet_id = be2le16((char*) payload.data_ptr(), payload_offset, &payload_offset);
+    auto voice_codec = (uint8_t) payload[payload_offset++];
 
-    VoicePacketFlags flags{};
-    flags.head = false;
-    flags.fragmented = false;
-    flags.new_protocol = new_packet;
+    std::deque<std::shared_ptr<SpeakingClient>> target_clients;
 
     if(new_packet) {
-        if(data.length() < 7) {
+        if(payload.length() < 7) {
             this->disconnect("Invalid packet (Voice whisper | New)");
-            logMessage(this->getServerId(), "{} Tried to send a too short whisper packet. Length: {}", CLIENT_STR_LOG_PREFIX, data.length());
+            logMessage(this->getServerId(), "{} Tried to send a too short whisper packet. Length: {}", CLIENT_STR_LOG_PREFIX, payload.length());
             return;
         }
 
-        auto type = (WhisperType) data[offset++];
-        auto target = (WhisperTarget) data[offset++];
-        auto type_id = be2le64((char*) data.data_ptr(), offset, &offset);
-        this->resetIdleTime();
+        auto type = (WhisperType) payload[payload_offset++];
+        auto target = (WhisperTarget) payload[payload_offset++];
+        auto type_id = be2le64((char*) payload.data_ptr(), payload_offset, &payload_offset);
 
-        size_t data_length = data.length() - offset;
 #ifdef PKT_LOG_WHISPER
         logTrace(this->getServerId(), "{} Whisper data length: {}. Type: {}. Target: {}. Target ID: {}.", CLIENT_STR_LOG_PREFIX, data_length, type, target, type_id);
 #endif
+        if(type == WhisperType::ECHO) {
+            target_clients.push_back(dynamic_pointer_cast<SpeakingClient>(this->ref()));
+        } else {
+            for(const auto& client : this->server->getClients()) {
+                auto speakingClient = dynamic_pointer_cast<SpeakingClient>(client);
+                if(!speakingClient || client == this) continue;
+                if(!speakingClient->currentChannel) continue;
 
-        deque<shared_ptr<SpeakingClient>> available_clients;
-        for(const auto& client : this->server->getClients()) {
-            auto speakingClient = dynamic_pointer_cast<SpeakingClient>(client);
-            if(!speakingClient || client == this) continue;
-            if(!speakingClient->currentChannel) continue;
-
-            if(type == WhisperType::ALL) {
-                available_clients.push_back(speakingClient);
-            } else if(type == WhisperType::SERVER_GROUP) {
-                if(type_id == 0)
-                    available_clients.push_back(speakingClient);
-                else {
-                    shared_lock client_lock(this->channel_lock);
-                    for(const auto& id : client->cached_server_groups) {
-                        if(id == type_id) {
-                            available_clients.push_back(speakingClient);
-                            break;
+                if(type == WhisperType::ALL) {
+                    target_clients.push_back(speakingClient);
+                } else if(type == WhisperType::SERVER_GROUP) {
+                    if(type_id == 0)
+                        target_clients.push_back(speakingClient);
+                    else {
+                        shared_lock client_lock(this->channel_lock);
+                        for(const auto& id : client->cached_server_groups) {
+                            if(id == type_id) {
+                                target_clients.push_back(speakingClient);
+                                break;
+                            }
                         }
                     }
+                } else if(type == WhisperType::CHANNEL_GROUP) {
+                    if(client->cached_channel_group == type_id)
+                        target_clients.push_back(speakingClient);
+                } else if(type == WhisperType::CHANNEL_COMMANDER) {
+                    if(client->properties()[property::CLIENT_IS_CHANNEL_COMMANDER].as<bool>())
+                        target_clients.push_back(speakingClient);
                 }
-            } else if(type == WhisperType::CHANNEL_GROUP) {
-                if(client->cached_channel_group == type_id)
-                    available_clients.push_back(speakingClient);
-            } else if(type == WhisperType::CHANNEL_COMMANDER) {
-                if(client->properties()[property::CLIENT_IS_CHANNEL_COMMANDER].as<bool>())
-                    available_clients.push_back(speakingClient);
             }
-        }
 
-        if(target == WhisperTarget::CHANNEL_CURRENT) {
-            available_clients.erase(std::remove_if(available_clients.begin(), available_clients.end(), [&](const shared_ptr<SpeakingClient>& target) {
-                return target->currentChannel != this->currentChannel;
-            }), available_clients.end());
-        } else if(target == WhisperTarget::CHANNEL_PARENT) {
-            auto current_parent = this->currentChannel->parent();
-            if(!current_parent) return;
-
-            available_clients.erase(std::remove_if(available_clients.begin(), available_clients.end(), [&](const shared_ptr<SpeakingClient>& target) {
-                return target->currentChannel != current_parent;
-            }), available_clients.end());
-        } else if(target == WhisperTarget::CHANNEL_ALL_PARENT) {
-            shared_ptr<BasicChannel> current_parent;
-            {
-                current_parent = this->currentChannel->parent();
+            if(target == WhisperTarget::CHANNEL_CURRENT) {
+                target_clients.erase(std::remove_if(target_clients.begin(), target_clients.end(), [&](const shared_ptr<SpeakingClient>& target) {
+                    return target->currentChannel != this->currentChannel;
+                }), target_clients.end());
+            } else if(target == WhisperTarget::CHANNEL_PARENT) {
+                auto current_parent = this->currentChannel->parent();
                 if(!current_parent) return;
+
+                target_clients.erase(std::remove_if(target_clients.begin(), target_clients.end(), [&](const shared_ptr<SpeakingClient>& target) {
+                    return target->currentChannel != current_parent;
+                }), target_clients.end());
+            } else if(target == WhisperTarget::CHANNEL_ALL_PARENT) {
+                shared_ptr<BasicChannel> current_parent;
+                {
+                    current_parent = this->currentChannel->parent();
+                    if(!current_parent) return;
+                }
+                target_clients.erase(std::remove_if(target_clients.begin(), target_clients.end(), [&](const shared_ptr<SpeakingClient>& target) {
+                    auto tmp_parent = current_parent;
+                    while(tmp_parent && tmp_parent != target->currentChannel)
+                        tmp_parent = tmp_parent->parent();
+                    return target->currentChannel != tmp_parent;
+                }), target_clients.end());
+            } else if(target == WhisperTarget::CHANNEL_FAMILY) {
+                shared_ptr<BasicChannel> current = this->currentChannel;
+                target_clients.erase(std::remove_if(target_clients.begin(), target_clients.end(), [&](const shared_ptr<SpeakingClient>& target) {
+                    auto tmp_current = target->currentChannel;
+                    while(tmp_current && tmp_current != current)
+                        tmp_current = tmp_current->parent();
+                    return current != tmp_current;
+                }), target_clients.end());
+            } else if(target == WhisperTarget::CHANNEL_COMPLETE_FAMILY) {
+                shared_ptr<BasicChannel> current = this->currentChannel;
+                while(current && current->parent()) current = current->parent();
+                target_clients.erase(std::remove_if(target_clients.begin(), target_clients.end(), [&](const shared_ptr<SpeakingClient>& target) {
+                    auto tmp_current = target->currentChannel;
+                    while(tmp_current && tmp_current != current)
+                        tmp_current = tmp_current->parent();
+                    return current != tmp_current;
+                }), target_clients.end());
+            } else if(target == WhisperTarget::CHANNEL_SUBCHANNELS) {
+                shared_ptr<BasicChannel> current = this->currentChannel;
+                target_clients.erase(std::remove_if(target_clients.begin(), target_clients.end(), [&](const shared_ptr<SpeakingClient>& target) {
+                    return target->currentChannel->parent() != current;
+                }), target_clients.end());
             }
-            available_clients.erase(std::remove_if(available_clients.begin(), available_clients.end(), [&](const shared_ptr<SpeakingClient>& target) {
-                auto tmp_parent = current_parent;
-                while(tmp_parent && tmp_parent != target->currentChannel)
-                    tmp_parent = tmp_parent->parent();
-                return target->currentChannel != tmp_parent;
-            }), available_clients.end());
-        } else if(target == WhisperTarget::CHANNEL_FAMILY) {
-            shared_ptr<BasicChannel> current = this->currentChannel;
-            available_clients.erase(std::remove_if(available_clients.begin(), available_clients.end(), [&](const shared_ptr<SpeakingClient>& target) {
-                auto tmp_current = target->currentChannel;
-                while(tmp_current && tmp_current != current)
-                    tmp_current = tmp_current->parent();
-                return current != tmp_current;
-            }), available_clients.end());
-        } else if(target == WhisperTarget::CHANNEL_COMPLETE_FAMILY) {
-            shared_ptr<BasicChannel> current = this->currentChannel;
-            while(current && current->parent()) current = current->parent();
-            available_clients.erase(std::remove_if(available_clients.begin(), available_clients.end(), [&](const shared_ptr<SpeakingClient>& target) {
-                auto tmp_current = target->currentChannel;
-                while(tmp_current && tmp_current != current)
-                    tmp_current = tmp_current->parent();
-                return current != tmp_current;
-            }), available_clients.end());
-        } else if(target == WhisperTarget::CHANNEL_SUBCHANNELS) {
-            shared_ptr<BasicChannel> current = this->currentChannel;
-            available_clients.erase(std::remove_if(available_clients.begin(), available_clients.end(), [&](const shared_ptr<SpeakingClient>& target) {
-                return target->currentChannel->parent() != current;
-            }), available_clients.end());
-        }
 
-        auto self_lock = this->_this.lock();
-        available_clients.erase(std::remove_if(available_clients.begin(), available_clients.end(), [&](const std::shared_ptr<ConnectedClient>& cl) {
-            auto speakingClient = dynamic_pointer_cast<SpeakingClient>(cl);
-            return !speakingClient->shouldReceiveVoiceWhisper(self_lock);
-        }), available_clients.end());
+            auto self_lock = this->_this.lock();
+            target_clients.erase(std::remove_if(target_clients.begin(), target_clients.end(), [&](const std::shared_ptr<ConnectedClient>& cl) {
+                auto speakingClient = dynamic_pointer_cast<SpeakingClient>(cl);
+                return !speakingClient->shouldReceiveVoiceWhisper(self_lock);
+            }), target_clients.end());
 
-        if(available_clients.empty()) {
-            if(update_whisper_error(this->speak_last_no_whisper_target)) {
-                command_result result{error::whisper_no_targets};
-                this->notifyError(result);
+            if(target_clients.empty()) {
+                if(update_whisper_error(this->speak_last_no_whisper_target)) {
+                    command_result result{error::whisper_no_targets};
+                    this->notifyError(result);
+                }
+                return;
             }
-            return;
-        }
-        if(available_clients.size() > this->server->properties()[property::VIRTUALSERVER_MIN_CLIENTS_IN_CHANNEL_BEFORE_FORCED_SILENCE].as_save<size_t>()) {
-            if(update_whisper_error(this->speak_last_too_many_whisper_targets)) {
-                command_result result{error::whisper_too_many_targets};
-                this->notifyError(result);
+
+            if(target_clients.size() > this->server->properties()[property::VIRTUALSERVER_MIN_CLIENTS_IN_CHANNEL_BEFORE_FORCED_SILENCE].as_save<size_t>()) {
+                if(update_whisper_error(this->speak_last_too_many_whisper_targets)) {
+                    command_result result{error::whisper_too_many_targets};
+                    this->notifyError(result);
+                }
+                return;
             }
-            return;
         }
-
-        //Create the packet data
-        char packet_buffer[OUT_WHISPER_PKT_OFFSET + data_length];
-        if(offset < data.length())
-            memcpy(&packet_buffer[OUT_WHISPER_PKT_OFFSET], &data[offset], data_length);
-
-        le2be16(vpacketId, packet_buffer, 0);
-        le2be16(this->getClientId(), packet_buffer, 2);
-        packet_buffer[4] = codec;
-
-        VoicePacketFlags flags{};
-        auto data = pipes::buffer_view(packet_buffer, OUT_WHISPER_PKT_OFFSET + data_length);
-        for(const auto& cl : available_clients){
-                cl->send_voice_whisper_packet(data, flags);
-        }
-
-        this->updateSpeak(false, system_clock::now());
     } else {
-        auto clientCount = (uint8_t) data[offset++];
-        auto channelCount = (uint8_t) data[offset++];
-        if(data.length() < 5 + channelCount * 2 + clientCount * 8) {
-            logMessage(this->getServerId(), "{} Tried to send a too short whisper packet. Length: {} Required: {}", CLIENT_STR_LOG_PREFIX, data.length(), to_string(5 + channelCount * 2 + clientCount * 8));
+        auto channelCount = (uint8_t) payload[payload_offset++];
+        auto clientCount = (uint8_t) payload[payload_offset++];
+        if(payload.length() < 5 + clientCount * 2 + channelCount * 8) {
+            logMessage(this->getServerId(), "{} Tried to send a too short whisper packet. Length: {} Required: {}", CLIENT_STR_LOG_PREFIX, payload.length(), to_string(5 + channelCount * 2 + clientCount * 8));
             return;
         }
 
-        this->resetIdleTime();
-        ChannelId channelIds[clientCount];
-        ClientId clientIds[channelCount];
+        ChannelId channelIds[channelCount];
+        ClientId clientIds[clientCount];
 
-        for(uint8_t index = 0; index < clientCount; index++)
-            channelIds[index] = be2le64((char*) data.data_ptr(), offset, &offset);
-        for(uint8_t index = 0; index < channelCount; index++)
-            clientIds[index] = be2le16((char*) data.data_ptr(), offset, &offset);
-
-        auto available_clients = this->server->getClients();
-        available_clients.erase(std::remove_if(available_clients.begin(), available_clients.end(), [&](const std::shared_ptr<ConnectedClient>& cl) {
-            auto speakingClient = dynamic_pointer_cast<SpeakingClient>(cl);
-            if(!speakingClient || cl == this || !speakingClient->currentChannel) return true;
-
-            auto clientChannelId = cl->currentChannel->channelId();
-            auto clientId = cl->getClientId();
-
-            for(uint8_t index = 0; index < clientCount; index++)
-                if(channelIds[index] == clientChannelId) return false;
-
-            for(uint8_t index = 0; index < channelCount; index++)
-                if(clientIds[index] == clientId) return false;
-
-            return true;
-        }), available_clients.end());
-
-        auto self_lock = this->_this.lock();
-        available_clients.erase(std::remove_if(available_clients.begin(), available_clients.end(), [&](const std::shared_ptr<ConnectedClient>& cl) {
-            auto speakingClient = dynamic_pointer_cast<SpeakingClient>(cl);
-            return !speakingClient->shouldReceiveVoiceWhisper(self_lock);
-        }), available_clients.end());
-
-        if(available_clients.empty()) {
-            if(update_whisper_error(this->speak_last_no_whisper_target)) {
-                command_result result{error::whisper_no_targets};
-                this->notifyError(result);
-            }
-            return;
-        }
-        if(available_clients.size() > this->server->properties()[property::VIRTUALSERVER_MIN_CLIENTS_IN_CHANNEL_BEFORE_FORCED_SILENCE].as_save<size_t>()) {
-            if(update_whisper_error(this->speak_last_too_many_whisper_targets)) {
-                command_result result{error::whisper_too_many_targets};
-                this->notifyError(result);
-            }
-            return;
+        for(uint8_t index = 0; index < channelCount; index++) {
+            channelIds[index] = be2le64((char*) payload.data_ptr(), payload_offset, &payload_offset);
+        }
+
+        for(uint8_t index = 0; index < clientCount; index++) {
+            clientIds[index] = be2le16((char*) payload.data_ptr(), payload_offset, &payload_offset);
         }
 
-        size_t dataLength = data.length() - offset;
 #ifdef PKT_LOG_WHISPER
         logTrace(this->getServerId(), "{} Whisper data length: {}. Client count: {}. Channel count: {}.", CLIENT_STR_LOG_PREFIX, dataLength, clientCount, channelCount);
 #endif
+
+        for(const auto& client : this->server->getClients()) {
+            auto speaking_client = dynamic_pointer_cast<SpeakingClient>(client);
+            if(!speaking_client || client == this || !speaking_client->currentChannel)
+                continue;
+
+            auto clientChannelId = speaking_client->getChannelId();
+            auto clientId = speaking_client->getClientId();
+
+            for(uint8_t index = 0; index < channelCount; index++) {
+                if(channelIds[index] == clientChannelId) {
+                    goto add_client;
+                }
+            }
+
+            for(uint8_t index = 0; index < clientCount; index++) {
+                if(clientIds[index] == clientId) {
+                    goto add_client;
+                }
+            }
+
+            continue;
+
+            add_client:
+            if(!speaking_client->shouldReceiveVoiceWhisper(this->ref())) {
+                continue;
+            }
+
+            target_clients.push_back(speaking_client);
+        }
+    }
+
+    if(target_clients.empty()) {
+        if(update_whisper_error(this->speak_last_no_whisper_target)) {
+            command_result result{error::whisper_no_targets};
+            this->notifyError(result);
+        }
+        return;
+    }
+
+    if(target_clients.size() > this->server->properties()[property::VIRTUALSERVER_MIN_CLIENTS_IN_CHANNEL_BEFORE_FORCED_SILENCE].as_save<size_t>()) {
+        if(update_whisper_error(this->speak_last_too_many_whisper_targets)) {
+            command_result result{error::whisper_too_many_targets};
+            this->notifyError(result);
+        }
+        return;
+    }
+
+    /* send the packet */
+    {
+        size_t voice_payload_length = payload.length() - payload_offset;
+
         //Create the packet data
-        char packetBuffer[OUT_WHISPER_PKT_OFFSET + dataLength];
-        if(offset < data.length())
-            memcpy(&packetBuffer[OUT_WHISPER_PKT_OFFSET], &data[offset], dataLength);
+        char whisper_packet_buffer[kWhisperMaxHeaderLength + voice_payload_length];
+        size_t whisper_packet_offset{0};
+        size_t whisper_packet_teamspeak_offset{0};
 
-        le2be16(vpacketId, packetBuffer, 0);
-        le2be16(this->getClientId(), packetBuffer, 2);
-        packetBuffer[4] = codec;
+        /* writing the teaspeak header */
+        if(head) {
+            auto uniqueId = this->getUid();
+            auto nickname = this->getDisplayName();
 
-        VoicePacketFlags flags{};
-        auto data = pipes::buffer_view(packetBuffer, OUT_WHISPER_PKT_OFFSET + dataLength);
+            if(uniqueId.length() > kWhisperClientUniqueIdLength) {
+                logCritical(LOG_GENERAL, "Clients unique id is longer than the expected max length of {}. Unique length: {}", kWhisperClientUniqueIdLength, uniqueId.length());
+                return;
+            }
 
-        for(const auto& cl : available_clients){ //Faster?
-            auto speakingClient = dynamic_pointer_cast<SpeakingClient>(cl);
-            assert(speakingClient);
-            if(speakingClient->shouldReceiveVoiceWhisper(_this.lock()))
-                speakingClient->send_voice_whisper_packet(data, flags);
+            if(nickname.length() > kMaxWhisperClientNameLength) {
+                logCritical(LOG_GENERAL, "Clients name is longer than the expected max length of {}. Name length: {}", kMaxWhisperClientNameLength, nickname.length());
+                return;
+            }
+
+            memset(whisper_packet_buffer + whisper_packet_offset, 0, kWhisperClientUniqueIdLength);
+            memcpy(whisper_packet_buffer + whisper_packet_offset, uniqueId.data(), uniqueId.length());
+            whisper_packet_offset += kWhisperClientUniqueIdLength;
+
+            whisper_packet_buffer[whisper_packet_offset++] = nickname.length();
+            memcpy(whisper_packet_buffer + whisper_packet_offset, nickname.data(), nickname.length());
+            whisper_packet_offset += nickname.length();
         }
 
-        this->updateSpeak(false, system_clock::now());
+        /* writing the "normal" header and payload */
+        {
+            whisper_packet_teamspeak_offset = whisper_packet_offset;
+
+            *(uint16_t*) &whisper_packet_buffer[whisper_packet_offset] = htons(voice_packet_id);
+            whisper_packet_offset += 2;
+
+            *(uint16_t*) &whisper_packet_buffer[whisper_packet_offset] = htons(this->getClientId());
+            whisper_packet_offset += 2;
+
+            whisper_packet_buffer[whisper_packet_offset++] = voice_codec;
+
+            if(voice_payload_length > 0) {
+                memcpy(&whisper_packet_buffer[whisper_packet_offset], &payload[payload_offset], voice_payload_length);
+                whisper_packet_offset += voice_payload_length;
+            }
+        }
+
+        VoicePacketFlags flags{};
+        flags.head = head;
+
+        pipes::buffer_view teaspeak_packet{}, teamspeak_packet{};
+        teaspeak_packet = pipes::buffer_view{whisper_packet_buffer, whisper_packet_offset};
+        teamspeak_packet = pipes::buffer_view{whisper_packet_buffer + whisper_packet_teamspeak_offset, whisper_packet_offset - whisper_packet_teamspeak_offset};
+
+        auto self_ref = this->ref();
+        for(const auto& cl : target_clients) {
+            if(cl == self_ref || cl->shouldReceiveVoiceWhisper(self_ref)) {
+                cl->send_voice_whisper_packet(teamspeak_packet, teaspeak_packet, flags);
+            }
+        }
     }
+
+    this->resetIdleTime();
+    this->updateSpeak(false, std::chrono::system_clock::now());
 }
 
 #define TEST_PARM(type) \
@@ -378,7 +432,7 @@ auto regex_wildcard = std::regex(".*");
 
 #define S(x) #x
 #define HWID_REGEX(name, pattern)                           \
-auto regex_hwid_ ##name = [](){                             \
+auto regex_hwid_ ##name = []() noexcept {                   \
     try {                                                   \
         return std::regex(pattern);                         \
     } catch (std::exception& ex) {                          \
@@ -478,15 +532,28 @@ command_result SpeakingClient::handleCommandClientInit(Command& cmd) {
                     logWarning(this->getServerId(), "{} Tried to join within an invalid supplied '{}' ({})", CLIENT_STR_LOG_PREFIX, key,cmd[key].string());
                     return command_result{error::client_hacked};
                 }
+        } else if(key == "client_talk_request_msg") {
+            if(cmd["client_talk_request_msg"].string().length() > ts::config::server::limits::talk_power_request_message_length)
+                return command_result{error::parameter_invalid_size, "client_talk_request_msg"};
+        } else if(key == "client_away_message") {
+            if(cmd["client_away_message"].string().length() > ts::config::server::limits::afk_message_length)
+                return command_result{error::parameter_invalid_size, "client_away_message"};
+        } else if(key == "client_nickname_phonetic") {
+            auto name = cmd["client_nickname_phonetic"].string();
+            if (count_characters(name) > 30) return command_result{error::parameter_invalid, "client_nickname_phonetic"};
+        } else if(key == "client_nickname") {
+            auto name = cmd["client_nickname"].string();
+            if (count_characters(name) < 3) return command_result{error::parameter_invalid, "client_nickname"};
+            if (count_characters(name) > 30) return command_result{error::parameter_invalid, "client_nickname"};
         }
 
-        const auto &info = property::info<property::ClientProperties>(key);
-        if(*info == property::CLIENT_UNDEFINED) {
+        const auto &info = property::find<property::ClientProperties>(key);
+        if(info.is_undefined()) {
             logError(this->getServerId(), "{} Tried to pass a unknown value {}. Please report this, if you're sure that this key should be known!", CLIENT_STR_LOG_PREFIX, key);
             continue;
             //return {findError("parameter_invalid"), "Unknown property " + key};
         }
-        if(!info->validate_input(cmd[key].as<string>()))
+        if(!info.validate_input(cmd[key].as<string>()))
             return command_result{error::parameter_invalid};
 
         this->properties()[info] = cmd[key].as<std::string>();
@@ -533,37 +600,39 @@ command_result SpeakingClient::handleCommandClientInit(Command& cmd) {
         if(!this->server->verifyServerPassword(cmd["client_server_password"].string(), true))
             return command_result{error::server_invalid_password};
 
-    size_t clones_uid = 0;
-    size_t clones_ip = 0;
-    size_t clones_hwid = 0;
+    if(!config::server::clients::ignore_max_clone_permissions) {
+        size_t clones_uid = 0;
+        size_t clones_ip = 0;
+        size_t clones_hwid = 0;
 
-    auto _own_hwid = this->getHardwareId();
-    this->server->forEachClient([&](const shared_ptr<ConnectedClient>& client) {
-        if(client->getExternalType() != CLIENT_TEAMSPEAK) return;
-        if(client->getUid() == this->getUid())
-            clones_uid++;
-        if(client->getPeerIp() == this->getPeerIp())
-            clones_ip++;
-        if(!_own_hwid.empty() && client->getHardwareId() == _own_hwid)
-            clones_hwid++;
-    });
+        auto _own_hwid = this->getHardwareId();
+        this->server->forEachClient([&](const shared_ptr<ConnectedClient>& client) {
+            if(client->getExternalType() != CLIENT_TEAMSPEAK) return;
+            if(client->getUid() == this->getUid())
+                clones_uid++;
+            if(client->getPeerIp() == this->getPeerIp())
+                clones_ip++;
+            if(!_own_hwid.empty() && client->getHardwareId() == _own_hwid)
+                clones_hwid++;
+        });
 
-    if(clones_uid > 0 && permissions[permission::i_client_max_clones_uid].has_value && !permission::v2::permission_granted(clones_uid, permissions[permission::i_client_max_clones_uid])) {
-        logMessage(this->getServerId(), "{} Disconnecting because there are already {} uid clones connected. (Allowed: {})", CLIENT_STR_LOG_PREFIX, clones_uid, permissions[permission::i_client_max_clones_uid]);
-        return command_result{error:: client_too_many_clones_connected, "too many clones connected (uid)"};
+        if(clones_uid > 0 && permissions[permission::i_client_max_clones_uid].has_value && !permission::v2::permission_granted(clones_uid, permissions[permission::i_client_max_clones_uid])) {
+            logMessage(this->getServerId(), "{} Disconnecting because there are already {} uid clones connected. (Allowed: {})", CLIENT_STR_LOG_PREFIX, clones_uid, permissions[permission::i_client_max_clones_uid]);
+            return command_result{error:: client_too_many_clones_connected, "too many clones connected (uid)"};
+        }
+
+        if(clones_ip > 0 && permissions[permission::i_client_max_clones_ip].has_value && !permission::v2::permission_granted(clones_ip, permissions[permission::i_client_max_clones_ip])) {
+            logMessage(this->getServerId(), "{} Disconnecting because there are already {} ip clones connected. (Allowed: {})", CLIENT_STR_LOG_PREFIX, clones_ip, permissions[permission::i_client_max_clones_ip]);
+            return command_result{error:: client_too_many_clones_connected, "too many clones connected (ip)"};
+        }
+
+        if(clones_hwid > 0 && permissions[permission::i_client_max_clones_hwid].has_value && !permission::v2::permission_granted(clones_hwid, permissions[permission::i_client_max_clones_hwid])) {
+            logMessage(this->getServerId(), "{} Disconnecting because there are already {} hwid clones connected. (Allowed: {})", CLIENT_STR_LOG_PREFIX, clones_hwid, permissions[permission::i_client_max_clones_hwid]);
+            return command_result{error:: client_too_many_clones_connected, "too many clones connected (hwid)"};
+        }
+        TIMING_STEP(timings, "max clones ");
     }
 
-    if(clones_ip > 0 && permissions[permission::i_client_max_clones_ip].has_value && !permission::v2::permission_granted(clones_ip, permissions[permission::i_client_max_clones_ip])) {
-        logMessage(this->getServerId(), "{} Disconnecting because there are already {} ip clones connected. (Allowed: {})", CLIENT_STR_LOG_PREFIX, clones_ip, permissions[permission::i_client_max_clones_ip]);
-        return command_result{error:: client_too_many_clones_connected, "too many clones connected (ip)"};
-    }
-
-    if(clones_hwid > 0 && permissions[permission::i_client_max_clones_hwid].has_value && !permission::v2::permission_granted(clones_hwid, permissions[permission::i_client_max_clones_hwid])) {
-        logMessage(this->getServerId(), "{} Disconnecting because there are already {} hwid clones connected. (Allowed: {})", CLIENT_STR_LOG_PREFIX, clones_hwid, permissions[permission::i_client_max_clones_hwid]);
-        return command_result{error:: client_too_many_clones_connected, "too many clones connected (hwid)"};
-    }
-    TIMING_STEP(timings, "max clones ");
-
     auto banEntry = this->resolveActiveBan(this->getPeerIp());
     if(banEntry) {
         logMessage(this->getServerId(), "{} Disconnecting while init because of ban record. Record id {} at server {}",
@@ -722,7 +791,7 @@ void SpeakingClient::processJoin() {
     if(!ref_server->assignDefaultChannel(this->ref(), false)) {
         auto result = command_result{error::vs_critical, "Could not assign default channel!"};
         this->notifyError(result);
-        result.release_details();
+        result.release_data();
         this->close_connection(system_clock::now() + seconds(1));
         return;
     }
@@ -743,7 +812,7 @@ void SpeakingClient::processJoin() {
 
         unique_lock server_channel_lock(this->server->channel_tree_lock);
         this->server->client_move(this->ref(), channel, nullptr, "", ViewReasonId::VREASON_USER_ACTION, false, server_channel_lock);
-        this->subscribeChannel({this->currentChannel}, false, true);
+        if(this->getType() != ClientType::CLIENT_TEAMSPEAK) this->subscribeChannel({this->currentChannel}, false, true); /* su "improve" the TS3 clients join speed we send the channel clients a bit later, when the TS3 client gets his own client variables */
     }
     TIMING_STEP(timings, "join move  ");
 
@@ -761,12 +830,37 @@ void SpeakingClient::processJoin() {
     this->connectTimestamp = chrono::system_clock::now();
     this->idleTimestamp = chrono::system_clock::now();
 
+    TIMING_STEP(timings, "welcome msg");
+    {
+        std::string message{};
+        config::server::clients::WelcomeMessageType type{config::server::clients::WELCOME_MESSAGE_TYPE_NONE};
+        if(this->getType() == ClientType::CLIENT_TEASPEAK) {
+            message = config::server::clients::extra_welcome_message_teaspeak;
+            type = config::server::clients::extra_welcome_message_type_teaspeak;
+        } else if(this->getType() == ClientType::CLIENT_TEAMSPEAK) {
+            message = config::server::clients::extra_welcome_message_teamspeak;
+            type = config::server::clients::extra_welcome_message_type_teamspeak;
+        } else if(this->getType() == ClientType::CLIENT_WEB) {
+            message = config::server::clients::extra_welcome_message_teaweb;
+            type = config::server::clients::extra_welcome_message_type_teaweb;
+        }
+
+        if(type == config::server::clients::WELCOME_MESSAGE_TYPE_POKE) {
+            this->notifyClientPoke(this->server->serverRoot, message);
+        } else if(type == config::server::clients::WELCOME_MESSAGE_TYPE_CHAT) {
+            this->notifyTextMessage(ChatMessageMode::TEXTMODE_SERVER, this->server->serverRoot, 0, 0, std::chrono::system_clock::now(), message);
+        }
+    }
     debugMessage(this->getServerId(), "{} Client join timings: {}", CLIENT_STR_LOG_PREFIX, TIMING_FINISH(timings));
+
+    serverInstance->action_logger()->client_channel_logger.log_client_join(this->getServerId(), this->ref(), this->getChannelId(), this->currentChannel->name());
 }
 
 void SpeakingClient::processLeave() {
     auto ownLock = _this.lock();
     auto server = this->getServer();
+
+    auto channel = this->currentChannel;
     if(server){
         logMessage(this->getServerId(), "Voice client {}/{} ({}) from {} left.", this->getClientDatabaseId(), this->getUid(), this->getDisplayName(), this->getLoggingPeerIp() + ":" + to_string(this->getPeerPort()));
         {
@@ -837,13 +931,13 @@ command_result SpeakingClient::handleCommand(Command &command) {
         if(this->handshake.state == HandshakeState::BEGIN || this->handshake.state == HandshakeState::IDENTITY_PROOF) {
             command_result result;
             if(command.command() == "handshakebegin")
-                result = this->handleCommandHandshakeBegin(command);
+                result.reset(this->handleCommandHandshakeBegin(command));
             else if(command.command() == "handshakeindentityproof")
-                result = this->handleCommandHandshakeIdentityProof(command);
+                result.reset(this->handleCommandHandshakeIdentityProof(command));
             else
-                result = command_result{error::client_not_logged_in};
+                result.reset(command_result{error::client_not_logged_in});
 
-            if(result.error_code())
+            if(result.has_error())
                 this->postCommandHandler.push_back([&]{
                     this->close_connection(system_clock::now() + seconds(1));
                 });

server/src/client/SpeakingClient.h

@@ -3,98 +3,96 @@
 #include <json/json.h>
 #include "ConnectedClient.h"
 
-namespace ts {
-    namespace server {
-        class VirtualServer;
-        class SpeakingClient : public ConnectedClient {
-            public:
-                struct VoicePacketFlags {
-                    bool encrypted      : 1;
-                    bool head           : 1;
-                    bool fragmented     : 1; /* used by MONO. IDK What this is */
-                    bool new_protocol   : 1;
-                    char _unused        : 4;
+namespace ts::server {
+    class VirtualServer;
+    class SpeakingClient : public ConnectedClient {
+        public:
+            struct VoicePacketFlags {
+                bool encrypted      : 1;
+                bool head           : 1;
+                bool fragmented     : 1; /* used by MONO. IDK What this is */
+                bool new_protocol   : 1;
+                char _unused        : 4;
 
-                    VoicePacketFlags() : encrypted{false}, head{false}, fragmented{false}, new_protocol{false}, _unused{0} { }
-                };
-                static_assert(sizeof(VoicePacketFlags) == 1);
+                VoicePacketFlags() : encrypted{false}, head{false}, fragmented{false}, new_protocol{false}, _unused{0} { }
+            };
+            static_assert(sizeof(VoicePacketFlags) == 1);
 
-                enum HandshakeState {
-                    BEGIN,
-                    IDENTITY_PROOF,
-                    SUCCEEDED
-                };
-                enum IdentityType : uint8_t {
-                    TEASPEAK_FORUM,
-                    TEAMSPEAK,
-                    NICKNAME,
+            enum HandshakeState {
+                BEGIN,
+                IDENTITY_PROOF,
+                SUCCEEDED
+            };
+            enum IdentityType : uint8_t {
+                TEASPEAK_FORUM,
+                TEAMSPEAK,
+                NICKNAME,
 
-                    UNSET = 0xff
-                };
+                UNSET = 0xff
+            };
 
-                SpeakingClient(sql::SqlManager* a, const std::shared_ptr<VirtualServer>& b) : ConnectedClient(a, b) {
-                    speak_begin = std::chrono::system_clock::now();
-                    speak_last_packet = std::chrono::system_clock::now();
-                };
-                ~SpeakingClient() override = default;
+            SpeakingClient(sql::SqlManager* a, const std::shared_ptr<VirtualServer>& b) : ConnectedClient(a, b) {
+                speak_begin = std::chrono::system_clock::now();
+                speak_last_packet = std::chrono::system_clock::now();
+            };
+            ~SpeakingClient() override = default;
 
-                //Voice
-                virtual void send_voice_packet(const pipes::buffer_view& /* voice packet data */, const VoicePacketFlags& /* flags */) = 0;
-                virtual bool shouldReceiveVoice(const std::shared_ptr<ConnectedClient> &sender);
+            //Voice
+            virtual void send_voice_packet(const pipes::buffer_view& /* voice packet data */, const VoicePacketFlags& /* flags */) = 0;
+            virtual bool shouldReceiveVoice(const std::shared_ptr<ConnectedClient> &sender);
 
-                //Whisper
-                bool shouldReceiveVoiceWhisper(const std::shared_ptr<ConnectedClient> &sender);
-                virtual void send_voice_whisper_packet(const pipes::buffer_view& /* voice packet data */, const VoicePacketFlags& /* flags */) = 0;
+            //Whisper
+            bool shouldReceiveVoiceWhisper(const std::shared_ptr<ConnectedClient> &sender);
+            virtual void send_voice_whisper_packet(const pipes::buffer_view& /* teamspeak payload */, const pipes::buffer_view& /* teaspeak payload */, const VoicePacketFlags& /* flags */) = 0;
 
-                inline std::chrono::milliseconds takeSpokenTime() {
-                    auto time = this->speak_time;
-                    this->speak_time = std::chrono::milliseconds(0);
-                    return time;
-                }
-            protected:
-                void tick(const std::chrono::system_clock::time_point &time) override;
+            inline std::chrono::milliseconds takeSpokenTime() {
+                auto time = this->speak_time;
+                this->speak_time = std::chrono::milliseconds(0);
+                return time;
+            }
+        protected:
+            void tick(const std::chrono::system_clock::time_point &time) override;
 
-            protected:
-            public:
-                void updateChannelClientProperties(bool channel_lock, bool notify) override;
+        protected:
+        public:
+            void updateChannelClientProperties(bool channel_lock, bool notify) override;
 
-            protected:
-                command_result handleCommand(Command &command) override;
+        protected:
+            command_result handleCommand(Command &command) override;
 
-            public:
-                void handlePacketVoice(const pipes::buffer_view&, bool head, bool fragmented);
-                virtual void handlePacketVoiceWhisper(const pipes::buffer_view&, bool /* new */);
+        public:
+            void handlePacketVoice(const pipes::buffer_view&, bool head, bool fragmented);
+            virtual void handlePacketVoiceWhisper(const pipes::buffer_view&, bool /* new */, bool /* head */);
 
-                void processJoin();
-                void processLeave();
+            void processJoin();
+            void processLeave();
 
-                virtual command_result handleCommandHandshakeBegin(Command&);
-                virtual command_result handleCommandHandshakeIdentityProof(Command &);
-                virtual command_result handleCommandClientInit(Command&);
+            virtual command_result handleCommandHandshakeBegin(Command&);
+            virtual command_result handleCommandHandshakeIdentityProof(Command &);
+            virtual command_result handleCommandClientInit(Command&);
 
-                void triggerVoiceEnd();
-                inline void updateSpeak(bool onlyUpdate, const std::chrono::system_clock::time_point &time);
-                std::chrono::milliseconds speak_accuracy = std::chrono::seconds{1};
+            void triggerVoiceEnd();
+            inline void updateSpeak(bool onlyUpdate, const std::chrono::system_clock::time_point &time);
+            std::chrono::milliseconds speak_accuracy = std::chrono::seconds{1};
 
-                threads::Mutex speak_lock;
-                std::chrono::milliseconds speak_time = std::chrono::milliseconds{0};
-                std::chrono::system_clock::time_point speak_begin;
-                std::chrono::system_clock::time_point speak_last_packet;
+            threads::Mutex speak_lock;
+            std::chrono::milliseconds speak_time = std::chrono::milliseconds{0};
+            std::chrono::system_clock::time_point speak_begin;
+            std::chrono::system_clock::time_point speak_last_packet;
 
-                std::chrono::system_clock::time_point speak_last_no_whisper_target;
-                std::chrono::system_clock::time_point speak_last_too_many_whisper_targets;
+            std::chrono::system_clock::time_point speak_last_no_whisper_target;
+            std::chrono::system_clock::time_point speak_last_too_many_whisper_targets;
 
-                permission::v2::PermissionFlaggedValue max_idle_time{permission::v2::empty_permission_flagged_value};
-                struct {
-                    HandshakeState state{HandshakeState::BEGIN};
+            permission::v2::PermissionFlaggedValue max_idle_time{permission::v2::empty_permission_flagged_value};
+            struct {
+                HandshakeState state{HandshakeState::BEGIN};
 
-                    IdentityType identityType{IdentityType::UNSET};
-                    std::string proof_message;
-                    //TeamSpeak
-                    std::shared_ptr<ecc_key> identityKey;
-                    //TeaSpeak
-                    std::shared_ptr<Json::Value> identityData;
-                } handshake;
-        };
-    }
+                IdentityType identityType{IdentityType::UNSET};
+                std::string proof_message;
+                //TeamSpeak
+                std::shared_ptr<ecc_key> identityKey;
+                //TeaSpeak
+                std::shared_ptr<Json::Value> identityData;
+            } handshake;
+    };
 }

server/src/client/SpeakingClientHandshake.cpp

@@ -35,7 +35,7 @@ command_result SpeakingClient::handleCommandHandshakeBegin(Command& cmd) { //If
     if(authenticationMethod == IdentityType::TEAMSPEAK) {
         this->handshake.identityType = IdentityType::TEAMSPEAK;
         
-        auto identity = base64::decode(cmd["publicKey"]);
+        auto identity = base64::decode(cmd["publicKey"].string());
         this->properties()[property::CLIENT_UNIQUE_IDENTIFIER] = base64::encode(digest::sha1(cmd["publicKey"].string()));
 
         this->handshake.identityKey = shared_ptr<ecc_key>(new ecc_key{}, free_ecc);
@@ -144,7 +144,7 @@ command_result SpeakingClient::handleCommandHandshakeIdentityProof(Command& cmd)
         this->properties()[property::CLIENT_TEAFORO_NAME] = (*this->handshake.identityData)["user_name"].asString();
         this->handshake.state = HandshakeState::SUCCEEDED;
     } else if(this->handshake.identityType == IdentityType::TEAMSPEAK) {
-        auto proof = base64::decode(cmd["proof"]);
+        auto proof = base64::decode(cmd["proof"].string());
 
         int result;
         if(ecc_verify_hash((u_char*) proof.data(), proof.length(), (u_char*) this->handshake.proof_message.data(), this->handshake.proof_message.length(), &result, this->handshake.identityKey.get()) != CRYPT_OK)

server/src/client/command_handler/bulk_parsers.cpp

@@ -0,0 +1,5 @@
+//
+// Created by WolverinDEV on 07/05/2020.
+//
+
+#include "bulk_parsers.h"

server/src/client/command_handler/bulk_parsers.h

@@ -0,0 +1,295 @@
+#pragma once
+
+#include <vector>
+#include <query/Command.h>
+#include <Error.h>
+#include <PermissionManager.h>
+#include <src/client/ConnectedClient.h>
+#include "./helpers.h"
+#include "../../manager/ActionLogger.h"
+
+namespace ts::command::bulk_parser {
+    template <bool kParseValue>
+    class PermissionBulkParser {
+            friend class PermissionBulkParser<false>;
+        public:
+            explicit PermissionBulkParser(ts::ParameterBulk& bulk) {
+                if(bulk.has("permid")) {
+                    auto type = bulk["permid"].as<permission::PermissionType>();
+                    if ((type & PERM_ID_GRANT) != 0) {
+                        type &= ~PERM_ID_GRANT;
+                    }
+
+                    this->permission_ = permission::resolvePermissionData(type);
+                } else if(bulk.has("permsid")) {
+                    auto permission_name = bulk["permsid"].string();
+                    this->permission_ = permission::resolvePermissionData(permission_name);
+                    this->grant_ = this->permission_->grantName() == permission_name;;
+                } else {
+                    this->error_.reset(ts::command_result{error::parameter_missing, "permid"});
+                    return;
+                }
+
+                if(this->permission_->is_invalid()) {
+                    this->error_.reset(ts::command_result{error::parameter_invalid});
+                    return;
+                }
+
+                if(kParseValue) {
+                    if(!bulk.has("permvalue")) {
+                        this->error_.reset(ts::command_result{error::parameter_missing, "permvalue"});
+                        return;
+                    }
+
+                    this->value_ = bulk["permvalue"].as<permission::PermissionValue>();
+                    this->flag_skip_ = bulk.has("permskip") && bulk["permskip"].as<bool>();
+                    this->flag_negated_ = bulk.has("permnegated") && bulk["permnegated"].as<bool>();
+                }
+            }
+            PermissionBulkParser(const PermissionBulkParser&) = delete;
+            PermissionBulkParser(PermissionBulkParser&&)  noexcept = default;
+
+            ~PermissionBulkParser() {
+                this->error_.release_data();
+            }
+
+            [[nodiscard]] inline bool has_error() const {
+                assert(!this->error_released_);
+                return this->error_.has_error();
+            }
+
+            [[nodiscard]] inline bool has_value() const { return this->value_ != permission::undefined; }
+            [[nodiscard]] inline bool is_grant_permission() const { return this->grant_; }
+
+            [[nodiscard]] inline const auto& permission() const { return this->permission_; }
+            [[nodiscard]] inline auto permission_type() const { return this->permission_->type; }
+
+            [[nodiscard]] inline auto value() const { return this->value_; }
+            [[nodiscard]] inline auto flag_skip() const { return this->flag_skip_; }
+            [[nodiscard]] inline auto flag_negated() const { return this->flag_negated_; }
+
+            [[nodiscard]] inline ts::command_result release_error() {
+                assert(!std::exchange(this->error_released_, true));
+                return std::move(this->error_);
+            }
+
+            inline void emplace_custom_error(ts::command_result&& result) {
+                assert(!this->error_released_);
+                this->error_.reset(std::forward<ts::command_result>(result));
+            }
+
+            inline void apply_to(const std::shared_ptr<permission::v2::PermissionManager>& manager, permission::v2::PermissionUpdateType mode) const {
+                if(this->is_grant_permission()) {
+                    this->old_value = manager->set_permission(this->permission_type(), { 0, this->value() }, permission::v2::PermissionUpdateType::do_nothing, mode);
+                } else {
+                    this->old_value = manager->set_permission(
+                            this->permission_type(),
+                            { this->value(), 0 },
+                            mode,
+                            permission::v2::PermissionUpdateType::do_nothing,
+                            this->flag_skip(),
+                            this->flag_negated()
+                    );
+                }
+            }
+
+            inline void log_update(
+                    server::log::PermissionActionLogger& logger,
+                    ServerId sid,
+                    const std::shared_ptr<server::ConnectedClient>& issuer,
+                    server::log::PermissionTarget target,
+                    permission::v2::PermissionUpdateType mode,
+                    uint64_t id1, const std::string& id1_name,
+                    uint64_t id2, const std::string& id2_name
+            ) const {
+                if(this->is_grant_permission()) {
+                    switch (mode) {
+                        case permission::v2::delete_value:
+                            if(!this->old_value.flags.grant_set)
+                                return;
+
+                            logger.log_permission_remove_grant(sid, issuer, target, id1, id1_name, id2, id2_name, *this->permission_, this->old_value.values.grant);
+                            break;
+
+                        case permission::v2::set_value:
+                            if(this->old_value.flags.grant_set) {
+                                logger.log_permission_edit_grant(sid, issuer, target, id1, id1_name, id2, id2_name, *this->permission_, this->old_value.values.grant, this->value_);
+                            } else {
+                                logger.log_permission_add_grant(sid, issuer, target, id1, id1_name, id2, id2_name, *this->permission_, this->value_);
+                            }
+                            break;
+                        case permission::v2::do_nothing:
+                            break;
+                    }
+                } else {
+                    switch (mode) {
+                        case permission::v2::delete_value:
+                            if(!this->old_value.flags.value_set)
+                                return;
+
+                            logger.log_permission_remove_value(sid, issuer, target, id1, id1_name, id2, id2_name, *this->permission_, this->old_value.values.value, this->old_value.flags.negate, this->old_value.flags.skip);
+                            break;
+
+                        case permission::v2::set_value:
+                            if(this->old_value.flags.value_set) {
+                                logger.log_permission_edit_value(sid, issuer, target, id1, id1_name, id2, id2_name, *this->permission_, this->old_value.values.value, this->old_value.flags.negate, this->old_value.flags.skip, this->value_, this->flag_negated_, this->flag_skip_);
+                            } else {
+                                logger.log_permission_add_value(sid, issuer, target, id1, id1_name, id2, id2_name, *this->permission_, this->old_value.values.value, this->old_value.flags.negate, this->old_value.flags.skip);
+                            }
+                            break;
+                        case permission::v2::do_nothing:
+                            break;
+                    }
+                }
+            }
+
+            inline void apply_to_channel(const std::shared_ptr<permission::v2::PermissionManager>& manager, permission::v2::PermissionUpdateType mode, ChannelId channel_id) const {
+                if(this->is_grant_permission()) {
+                    this->old_value = manager->set_channel_permission(this->permission_type(), channel_id, { this->value(), true }, permission::v2::PermissionUpdateType::do_nothing, mode);
+                } else {
+                    this->old_value = manager->set_channel_permission(
+                            this->permission_type(),
+                            channel_id,
+                            { this->value(), true },
+                            mode,
+                            permission::v2::PermissionUpdateType::do_nothing
+                    );
+                }
+            }
+
+            [[nodiscard]] inline bool is_group_property() const {
+                return permission_is_group_property(this->permission_type());
+            }
+
+            [[nodiscard]] inline bool is_client_view_property() const {
+                return permission_is_client_property(this->permission_type());
+            }
+        private:
+            std::shared_ptr<permission::PermissionTypeEntry> permission_{nullptr};
+            bool grant_{false};
+
+            bool flag_skip_{false};
+            bool flag_negated_{false};
+
+            permission::PermissionValue value_{0};
+            ts::command_result error_{error::ok};
+
+            mutable permission::v2::PermissionContainer old_value{};
+#ifndef NDEBUG
+            bool error_released_{false};
+#endif
+    };
+
+    template <bool kParseValue>
+    class PermissionBulksParser {
+        public:
+            PermissionBulksParser(const PermissionBulksParser&) = delete;
+            PermissionBulksParser(PermissionBulksParser&&)  noexcept = default;
+
+            template <typename base_iterator>
+            struct FilteredPermissionIterator : public base_iterator {
+                public:
+                    FilteredPermissionIterator() = default;
+                    explicit FilteredPermissionIterator(base_iterator position, base_iterator end = {}) : base_iterator{position}, end_{end} {
+                        if(*this != this->end_) {
+                            const auto& entry = **this;
+                            if(entry.has_error())
+                                this->operator++();
+                        }
+                    }
+
+                    FilteredPermissionIterator& operator++() {
+                        while(true) {
+                            base_iterator::operator++();
+                            if(*this == this->end_) break;
+
+                            const auto& entry = **this;
+                            if(!entry.has_error()) break;
+                        }
+                        return *this;
+                    }
+
+                    [[nodiscard]] FilteredPermissionIterator operator++(int) const {
+                        FilteredPermissionIterator copy = *this;
+                        ++*this;
+                        return copy;
+                    }
+
+                private:
+                    base_iterator end_;
+            };
+
+            struct FilteredPermissionListIterable {
+                typedef typename std::vector<PermissionBulkParser<kParseValue>>::const_iterator const_iterator;
+                public:
+                    FilteredPermissionListIterable(const_iterator begin, const_iterator end) noexcept : begin_{begin}, end_{end} {}
+
+                    FilteredPermissionIterator<const_iterator> begin() const {
+                        return FilteredPermissionIterator{this->begin_, this->end_};
+                    }
+
+                    FilteredPermissionIterator<const_iterator> end() const {
+                        return FilteredPermissionIterator{this->end_, this->end_};
+                    }
+                private:
+                    const_iterator begin_;
+                    const_iterator end_;
+            };
+
+            explicit PermissionBulksParser(ts::Command& command) {
+                this->permissions_.reserve(command.bulkCount());
+                for(size_t index{0}; index < command.bulkCount(); index++)
+                    this->permissions_.emplace_back(command[index]);
+            }
+
+            [[nodiscard]] inline bool validate(const std::shared_ptr<server::ConnectedClient>& issuer, ChannelId channel_id) {
+                auto ignore_granted_values = permission::v2::permission_granted(1, issuer->calculate_permission(permission::b_permission_modify_power_ignore, channel_id));
+                if(!ignore_granted_values) {
+                    auto max_value = issuer->calculate_permission(permission::i_permission_modify_power, channel_id, false);
+                    if(!max_value.has_value) {
+                        for(PermissionBulkParser<kParseValue>& permission : this->permissions_) {
+                            if(permission.has_error()) continue;
+
+                            permission.emplace_custom_error(ts::command_result{permission::i_permission_modify_power});
+                        }
+                        return false;
+                    }
+
+                    for(size_t index{0}; index < this->permissions_.size(); index++) {
+                        PermissionBulkParser<kParseValue>& permission = this->permissions_[index];
+                        if(permission.has_error()) continue;
+
+                        if(kParseValue && permission_require_granted_value(permission.permission_type()) && !permission::v2::permission_granted(permission.value(), max_value)) {
+                            permission.emplace_custom_error(ts::command_result{permission::i_permission_modify_power});
+                            continue;
+                        }
+
+                        if(!permission::v2::permission_granted(1, issuer->calculate_permission(permission.permission_type(), channel_id, true))) {
+                            permission.emplace_custom_error(ts::command_result{permission::i_permission_modify_power});
+                            continue;
+                        }
+                    }
+                }
+                return true;
+            }
+
+            [[nodiscard]] inline FilteredPermissionListIterable iterate_valid_permissions() const {
+                return FilteredPermissionListIterable{this->permissions_.begin(), this->permissions_.end()};
+            }
+
+            [[nodiscard]] inline ts::command_result build_command_result() {
+                assert(!std::exchange(this->result_created_, true));
+                ts::command_result_bulk result{};
+
+                for(auto& permission : this->permissions_)
+                    result.insert_result(std::forward<ts::command_result>(permission.release_error()));
+
+                return ts::command_result{std::move(result)};
+            }
+        private:
+            std::vector<PermissionBulkParser<kParseValue>> permissions_{};
+#ifndef NDEBUG
+            bool result_created_{false};
+#endif
+    };
+}