Fixed declaration

.gitmodules

@@ -4,7 +4,7 @@
 	branch = master
 [submodule "shared"]
 	path = shared
-	url = https://git.did.science/WolverinDEV/TeaSpeak-SharedLib.git
+	url = https://git.did.science/TeaSpeak/TeaSpeakLibrary.git
 [submodule "music"]
 	path = music
 	url = https://github.com/TeaSpeak/TeaMusic-Providers.git

CMakeLists.txt

@@ -103,4 +103,5 @@ add_definitions(-DINET -DINET6)
 add_subdirectory(shared/)
 add_subdirectory(server/)
 add_subdirectory(license/)
-add_subdirectory(MusicBot/)
+add_subdirectory(MusicBot/)
+add_subdirectory(file/)

MusicBot/src/MusicPlayer.cpp

@@ -78,11 +78,16 @@ void manager::loadProviders(const std::string& path) {
     }
 
     deque<fs::path> paths;
-    for(const auto& entry : fs::directory_iterator(dir)){
+    error_code error_code{};
+    for(const auto& entry : fs::directory_iterator(dir, error_code)){
         if(!entry.path().has_extension()) continue;
         if(entry.path().extension().string() == ".so")
             paths.push_back(entry.path());
     }
+    if(error_code) {
+        log::log(log::err, "Failed to scan the target directory (" + dir.string() + "): " + error_code.message());
+        return;
+    }
     std::sort(paths.begin(), paths.end(), [](const fs::path& a, const fs::path& b){ return a.filename().string() < b.filename().string(); });
 
     int index = 0;

file/CMakeLists.txt

@@ -0,0 +1,35 @@
+cmake_minimum_required(VERSION 3.6)
+project(TeaSpeak-Files)
+
+#set(CMAKE_CXX_STANDARD 17)
+#set(CMAKE_CXX_STANDARD_REQUIRED ON)
+
+add_library(TeaSpeak-FileServer STATIC
+        local_server/LocalFileProvider.cpp
+        local_server/LocalFileSystem.cpp
+        local_server/LocalFileTransfer.cpp
+        local_server/LocalFileTransferClientWorker.cpp
+        local_server/LocalFileTransferDisk.cpp
+        local_server/LocalFileTransferNetwork.cpp
+        local_server/clnpath.cpp
+)
+
+target_link_libraries(TeaSpeak-FileServer PUBLIC TeaSpeak ${StringVariable_LIBRARIES_STATIC} stdc++fs
+        libevent::core libevent::pthreads
+        DataPipes::core::static
+        openssl::ssl::shared
+        openssl::crypto::shared
+)
+
+target_include_directories(TeaSpeak-FileServer PUBLIC include/)
+
+add_executable(TeaSpeak-FileServerTest test/main.cpp)
+target_link_libraries(TeaSpeak-FileServerTest PUBLIC TeaSpeak-FileServer
+        TeaMusic            #Static (Must be in here, so we link against TeaMusic which uses C++11. That forbids GCC to use the newer glibc version)
+        CXXTerminal::static  #Static
+        stdc++fs
+)
+target_compile_options(TeaSpeak-FileServerTest PUBLIC -static-libgcc -static-libstdc++)
+
+add_executable(FileServer-CLNText local_server/clnpath.cpp)
+target_compile_definitions(FileServer-CLNText PUBLIC -DCLN_EXEC)

file/include/files/FileServer.h

@@ -0,0 +1,307 @@
+#pragma once
+
+#include <string>
+#include <chrono>
+#include <Definitions.h>
+#include <condition_variable>
+#include <variant>
+#include <deque>
+#include <functional>
+
+#define TRANSFER_KEY_LENGTH (32)
+
+namespace ts::server::file {
+    enum struct ExecuteStatus {
+        UNKNOWN,
+        WAITING,
+        SUCCESS,
+        ERROR
+    };
+
+    template<typename VariantType, typename T, std::size_t index = 0>
+    constexpr std::size_t variant_index() {
+        if constexpr (index == std::variant_size_v<VariantType>) {
+            return index;
+        } else if constexpr (std::is_same_v<std::variant_alternative_t<index, VariantType>, T>) {
+            return index;
+        } else {
+            return variant_index<VariantType, T, index + 1>();
+        }
+    }
+
+    struct EmptyExecuteResponse { };
+    template <class error_t, class response_t = EmptyExecuteResponse>
+    class ExecuteResponse {
+            typedef std::variant<EmptyExecuteResponse, error_t, response_t> variant_t;
+        public:
+            ExecuteStatus status{ExecuteStatus::WAITING};
+
+            [[nodiscard]] inline const auto& response() const { return std::get<response_t>(this->response_); }
+
+            template <typename = std::enable_if_t<!std::is_void<error_t>::value>>
+            [[nodiscard]] inline const error_t& error() const { return std::get<error_t>(this->response_); }
+
+            inline void wait() const {
+                std::unique_lock nlock{this->notify_mutex};
+                this->notify_cv.wait(nlock, [&]{ return this->status != ExecuteStatus::WAITING; });
+            }
+
+            template<typename _Rep, typename _Period>
+            [[nodiscard]] inline bool wait_for(const std::chrono::duration<_Rep, _Period>& time) const {
+                std::unique_lock nlock{this->notify_mutex};
+                return this->notify_cv.wait_for(nlock, time, [&]{ return this->status != ExecuteStatus::WAITING; });
+            }
+
+            template <typename... Args>
+            inline void emplace_success(Args&&... args) {
+                constexpr auto success_index = variant_index<variant_t, response_t>();
+
+                std::lock_guard rlock{this->notify_mutex};
+                this->response_.template emplace<success_index, Args...>(std::forward<Args>(args)...);
+                this->status = ExecuteStatus::SUCCESS;
+                this->notify_cv.notify_all();
+            }
+
+            template <typename... Args>
+            inline void emplace_fail(Args&&... args) {
+                constexpr auto error_index = variant_index<variant_t, error_t>();
+
+                std::lock_guard rlock{this->notify_mutex};
+                this->response_.template emplace<error_index, Args...>(std::forward<Args>(args)...);
+                this->status = ExecuteStatus::ERROR;
+                this->notify_cv.notify_all();
+            }
+
+            [[nodiscard]] inline bool succeeded() const {
+                return this->status == ExecuteStatus::SUCCESS;
+            }
+
+            ExecuteResponse(std::mutex& notify_mutex, std::condition_variable& notify_cv)
+                : notify_mutex{notify_mutex}, notify_cv{notify_cv} {}
+        private:
+            variant_t response_{}; /* void* as default value so we don't initialize error_t or response_t */
+
+            std::mutex& notify_mutex;
+            std::condition_variable& notify_cv;
+    };
+
+    namespace filesystem {
+        template <typename ErrorCodes>
+        struct DetailedError {
+            ErrorCodes error_type{ErrorCodes::UNKNOWN};
+            std::string error_message{};
+
+            DetailedError(ErrorCodes type, std::string extraMessage) : error_type{type}, error_message{std::move(extraMessage)} {}
+        };
+
+        enum struct DirectoryQueryErrorType {
+            UNKNOWN,
+            PATH_EXCEEDS_ROOT_PATH,
+            PATH_IS_A_FILE,
+            PATH_DOES_NOT_EXISTS,
+            FAILED_TO_LIST_FILES,
+
+            MAX
+        };
+        constexpr std::array<std::string_view, (int) DirectoryQueryErrorType::MAX> directory_query_error_messages = {
+            "unknown error",
+            "path exceeds base path",
+            "path is a file",
+            "path does not exists",
+            "failed to list files"
+        };
+
+        typedef DetailedError<DirectoryQueryErrorType> DirectoryQueryError;
+
+        struct DirectoryEntry {
+            enum Type {
+                UNKNOWN,
+                DIRECTORY,
+                FILE
+            };
+
+            Type type{Type::UNKNOWN};
+            std::string name{};
+            std::chrono::system_clock::time_point modified_at{};
+
+            size_t size{0};
+        };
+
+        enum struct DirectoryModifyErrorType {
+            UNKNOWN,
+            PATH_EXCEEDS_ROOT_PATH,
+            PATH_ALREADY_EXISTS,
+            FAILED_TO_CREATE_DIRECTORIES
+        };
+        typedef DetailedError<DirectoryModifyErrorType> DirectoryModifyError;
+
+        enum struct FileModifyErrorType {
+            UNKNOWN,
+            PATH_EXCEEDS_ROOT_PATH,
+            TARGET_PATH_EXCEEDS_ROOT_PATH,
+            PATH_DOES_NOT_EXISTS,
+            TARGET_PATH_ALREADY_EXISTS,
+            FAILED_TO_DELETE_FILES,
+            FAILED_TO_RENAME_FILE,
+
+            SOME_FILES_ARE_LOCKED
+        };
+        typedef DetailedError<FileModifyErrorType> FileModifyError;
+
+        enum struct ServerCommandErrorType {
+            UNKNOWN,
+            FAILED_TO_CREATE_DIRECTORIES,
+            FAILED_TO_DELETE_DIRECTORIES
+        };
+        typedef DetailedError<ServerCommandErrorType> ServerCommandError;
+
+        class AbstractProvider {
+            public:
+                typedef ExecuteResponse<DirectoryQueryError, std::deque<DirectoryEntry>> directory_query_response_t;
+
+                /* server */
+                [[nodiscard]] virtual std::shared_ptr<ExecuteResponse<ServerCommandError>> initialize_server(ServerId /* server */) = 0;
+                [[nodiscard]] virtual std::shared_ptr<ExecuteResponse<ServerCommandError>> delete_server(ServerId /* server */) = 0;
+
+                /* channels */
+                [[nodiscard]] virtual std::shared_ptr<directory_query_response_t> query_channel_directory(ServerId /* server */, ChannelId /* channel */, const std::string& /* path */) = 0;
+                [[nodiscard]] virtual std::shared_ptr<ExecuteResponse<DirectoryModifyError>> create_channel_directory(ServerId /* server */, ChannelId /* channel */, const std::string& /* path */) = 0;
+                [[nodiscard]] virtual std::shared_ptr<ExecuteResponse<FileModifyError>> delete_channel_file(ServerId /* server */, ChannelId /* channel */, const std::string& /* path */) = 0;
+                [[nodiscard]] virtual std::shared_ptr<ExecuteResponse<FileModifyError>> rename_channel_file(ServerId /* server */, ChannelId /* channel */, const std::string& /* path */, const std::string& /* target */) = 0;
+
+                /* icons */
+                [[nodiscard]] virtual std::shared_ptr<directory_query_response_t> query_icon_directory(ServerId /* server */) = 0;
+                [[nodiscard]] virtual std::shared_ptr<ExecuteResponse<FileModifyError>> delete_icon(ServerId /* server */, const std::string& /* name */) = 0;
+
+                /* avatars */
+                [[nodiscard]] virtual std::shared_ptr<directory_query_response_t> query_avatar_directory(ServerId /* server */) = 0;
+                [[nodiscard]] virtual std::shared_ptr<ExecuteResponse<FileModifyError>> delete_avatar(ServerId /* server */, const std::string& /* name */) = 0;
+            private:
+        };
+    }
+
+    namespace transfer {
+        typedef uint32_t transfer_id;
+
+        struct Transfer {
+            transfer_id server_transfer_id{0};
+            transfer_id client_transfer_id{0};
+
+            ServerId server_id{0};
+            ClientId client_id{0};
+            ChannelId channel_id{0};
+
+            char transfer_key[TRANSFER_KEY_LENGTH]{};
+            std::chrono::system_clock::time_point initialized_timestamp{};
+            enum Direction {
+                DIRECTION_UNKNOWN,
+                DIRECTION_UPLOAD,
+                DIRECTION_DOWNLOAD
+            } direction{DIRECTION_UNKNOWN};
+
+            struct Address {
+                std::string hostname{};
+                uint16_t port{0};
+            };
+            std::vector<Address> server_addresses{};
+
+            enum TargetType {
+                TARGET_TYPE_UNKNOWN,
+                TARGET_TYPE_CHANNEL_FILE,
+                TARGET_TYPE_ICON,
+                TARGET_TYPE_AVATAR
+            } target_type{TARGET_TYPE_UNKNOWN};
+            std::string target_file_path{};
+
+            int64_t max_bandwidth{-1};
+            size_t expected_file_size{0}; /* incl. the offset! */
+            size_t file_offset{0};
+            bool override_exiting{false};
+        };
+
+        struct TransferStatistics {
+            uint64_t network_bytes_send{0};
+            uint64_t network_bytes_received{0};
+
+            uint64_t delta_network_bytes_send{0};
+            uint64_t delta_network_bytes_received{0};
+
+            uint64_t file_bytes_transferred{0};
+            uint64_t delta_file_bytes_transferred{0};
+
+            size_t file_start_offset{0};
+            size_t file_current_offset{0};
+            size_t file_total_size{0};
+        };
+
+        struct TransferInitError {
+            enum Type {
+                UNKNOWN
+            } error_type{UNKNOWN};
+            std::string error_message{};
+        };
+
+        struct TransferActionError {
+            enum Type {
+                UNKNOWN,
+
+                UNKNOWN_TRANSFER
+            } error_type{UNKNOWN};
+            std::string error_message{};
+        };
+
+        struct TransferError {
+            enum Type {
+                UNKNOWN,
+
+                TRANSFER_TIMEOUT,
+
+                DISK_IO_ERROR,
+                DISK_TIMEOUT,
+                DISK_INITIALIZE_ERROR,
+
+                NETWORK_IO_ERROR,
+
+                UNEXPECTED_CLIENT_DISCONNECT,
+                UNEXPECTED_DISK_EOF
+            } error_type{UNKNOWN};
+            std::string error_message{};
+        };
+
+        class AbstractProvider {
+            public:
+                struct TransferInfo {
+                    std::string file_path{};
+
+                    bool override_exiting{false}; /* only for upload valid */
+                    size_t file_offset{0};
+                    size_t expected_file_size{0};
+                    int64_t max_bandwidth{-1};
+                };
+
+                virtual std::shared_ptr<ExecuteResponse<TransferInitError, std::shared_ptr<Transfer>>> initialize_channel_transfer(Transfer::Direction /* direction */, ServerId /* server */, ChannelId /* channel */, const TransferInfo& /* info */) = 0;
+                virtual std::shared_ptr<ExecuteResponse<TransferInitError, std::shared_ptr<Transfer>>> initialize_icon_transfer(Transfer::Direction /* direction */, ServerId /* server */, const TransferInfo& /* info */) = 0;
+                virtual std::shared_ptr<ExecuteResponse<TransferInitError, std::shared_ptr<Transfer>>> initialize_avatar_transfer(Transfer::Direction /* direction */, ServerId /* server */, const TransferInfo& /* info */) = 0;
+
+                virtual std::shared_ptr<ExecuteResponse<TransferActionError>> stop_transfer(transfer_id /* id */, bool /* flush */) = 0;
+
+                std::function<void(const std::shared_ptr<Transfer>&)> callback_transfer_registered{}; /* transfer has been registered */
+                std::function<void(const std::shared_ptr<Transfer>&)> callback_transfer_started{}; /* transfer has been started */
+                std::function<void(const std::shared_ptr<Transfer>&)> callback_transfer_finished{}; /* transfer has been finished */
+                std::function<void(const std::shared_ptr<Transfer>&, const TransferError&)> callback_transfer_aborted{}; /* an error happened while transferring the data  */
+                std::function<void(const std::shared_ptr<Transfer>&, const TransferStatistics&)> callback_transfer_statistics{};
+        };
+    }
+
+    class AbstractFileServer {
+        public:
+            [[nodiscard]] virtual filesystem::AbstractProvider& file_system() = 0;
+            [[nodiscard]] virtual transfer::AbstractProvider& file_transfer() = 0;
+        private:
+    };
+
+    extern bool initialize(std::string& /* error */);
+    extern void finalize();
+
+    extern std::shared_ptr<AbstractFileServer> server();
+}

file/local_server/LocalFileProvider.cpp

@@ -0,0 +1,121 @@
+//
+// Created by WolverinDEV on 28/04/2020.
+//
+
+#include <netinet/in.h>
+#include "LocalFileProvider.h"
+
+using namespace ts::server;
+using LocalFileServer = file::LocalFileProvider;
+
+EVP_PKEY* ssl_generate_key() {
+    auto key = std::unique_ptr<EVP_PKEY, decltype(&EVP_PKEY_free)>(EVP_PKEY_new(), ::EVP_PKEY_free);
+
+    auto rsa = RSA_new();
+    auto e = std::unique_ptr<BIGNUM, decltype(&BN_free)>(BN_new(), ::BN_free);
+    BN_set_word(e.get(), RSA_F4);
+    if(!RSA_generate_key_ex(rsa, 2048, e.get(), nullptr)) return nullptr;
+    EVP_PKEY_assign_RSA(key.get(), rsa);
+    return key.release();
+}
+
+X509* ssl_generate_certificate(EVP_PKEY* key) {
+    auto cert = X509_new();
+    X509_set_pubkey(cert, key);
+
+    ASN1_INTEGER_set(X509_get_serialNumber(cert), 3);
+    X509_gmtime_adj(X509_get_notBefore(cert), 0);
+    X509_gmtime_adj(X509_get_notAfter(cert), 31536000L);
+
+    X509_NAME* name = nullptr;
+    name = X509_get_subject_name(cert);
+    //for(const auto& subject : this->subjects)
+    //    X509_NAME_add_entry_by_txt(name, subject.first.c_str(),  MBSTRING_ASC, (unsigned char *) subject.second.c_str(), subject.second.length(), -1, 0);
+    X509_set_subject_name(cert, name);
+
+    name = X509_get_issuer_name(cert);
+    //for(const auto& subject : this->issues)
+    //    X509_NAME_add_entry_by_txt(name, subject.first.c_str(),  MBSTRING_ASC, (unsigned char *) subject.second.c_str(), subject.second.length(), -1, 0);
+
+    X509_set_issuer_name(cert, name);
+
+    X509_sign(cert, key, EVP_sha512());
+    return cert;
+}
+
+std::shared_ptr<LocalFileServer> server_instance{};
+bool file::initialize(std::string &error) {
+    server_instance = std::make_shared<LocalFileProvider>();
+
+    auto options = std::make_shared<pipes::SSL::Options>();
+    options->verbose_io = true;
+    options->context_method = SSLv23_method();
+    options->free_unused_keypairs = false;
+
+    {
+        std::shared_ptr<EVP_PKEY> pkey{ssl_generate_key(), ::EVP_PKEY_free};
+        std::shared_ptr<X509> cert{ssl_generate_certificate(&*pkey), ::X509_free};
+
+        options->default_keypair({pkey, cert});
+    }
+
+    if(!server_instance->initialize(error, options)) {
+        server_instance = nullptr;
+        return false;
+    }
+    return true;
+}
+
+void file::finalize() {
+    auto server = std::exchange(server_instance, nullptr);
+    if(!server) return;
+
+    server->finalize();
+}
+
+std::shared_ptr<file::AbstractFileServer> file::server() {
+    return server_instance;
+}
+
+LocalFileServer::LocalFileProvider() : file_system_{}, file_transfer_{this->file_system_} {}
+LocalFileServer::~LocalFileProvider() {}
+
+bool LocalFileServer::initialize(std::string &error, const std::shared_ptr<pipes::SSL::Options>& ssl_options) {
+    if(!this->file_system_.initialize(error, "file-root/"))
+        return false;
+
+
+    std::deque<std::shared_ptr<transfer::NetworkBinding>> bindings{};
+    {
+        auto binding = std::make_shared<transfer::NetworkBinding>();
+
+        binding->hostname = "localhost";
+
+        auto& iaddr = *(sockaddr_in*) &binding->address;
+        iaddr.sin_family = AF_INET;
+        iaddr.sin_port = htons(1112);
+        iaddr.sin_addr.s_addr = INADDR_ANY;
+
+        bindings.push_back(std::move(binding));
+    }
+
+    if(!this->file_transfer_.start(bindings, ssl_options)) {
+        error = "transfer server startup failed";
+        this->file_system_.finalize();
+        return false;
+    }
+    return true;
+}
+
+void LocalFileServer::finalize() {
+    this->file_transfer_.stop();
+    this->file_system_.finalize();
+}
+
+file::filesystem::AbstractProvider &LocalFileServer::file_system() {
+    return this->file_system_;
+}
+
+file::transfer::AbstractProvider & LocalFileServer::file_transfer() {
+    return this->file_transfer_;
+}

file/local_server/LocalFileProvider.h

@@ -0,0 +1,547 @@
+#pragma once
+
+#include <files/FileServer.h>
+#include <deque>
+#include <utility>
+#include <thread>
+#include <shared_mutex>
+#include <sys/socket.h>
+#include <pipes/ws.h>
+#include <pipes/ssl.h>
+#include <misc/net.h>
+#include <random>
+
+#define TRANSFER_MAX_CACHED_BYTES (1024 * 1024 * 1) // Buffer up to 1mb
+
+namespace ts::server::file {
+    namespace filesystem {
+#ifdef FS_INCLUDED
+        namespace fs = std::experimental::filesystem;
+#endif
+
+        class LocalFileSystem : public filesystem::AbstractProvider {
+                using FileModifyError = filesystem::FileModifyError;
+                using DirectoryModifyError = filesystem::DirectoryModifyError;
+            public:
+                enum struct FileCategory {
+                    ICON,
+                    AVATAR,
+                    CHANNEL
+                };
+
+                virtual ~LocalFileSystem();
+
+                bool initialize(std::string & /* error */, const std::string & /* root path */);
+                void finalize();
+
+                void lock_file(const std::string& /* absolute path */);
+                void unlock_file(const std::string& /* absolute path */);
+
+                [[nodiscard]] inline const auto &root_path() const { return this->root_path_; }
+
+                [[nodiscard]] std::string absolute_avatar_path(ServerId, const std::string&);
+                [[nodiscard]] std::string absolute_icon_path(ServerId, const std::string&);
+                [[nodiscard]] std::string absolute_channel_path(ServerId, ChannelId, const std::string&);
+
+                std::shared_ptr<ExecuteResponse<ServerCommandError>> initialize_server(ServerId /* server */) override;
+                std::shared_ptr<ExecuteResponse<ServerCommandError>> delete_server(ServerId /* server */) override;
+
+                std::shared_ptr<directory_query_response_t>
+                query_channel_directory(ServerId id, ChannelId channelId, const std::string &string) override;
+
+                std::shared_ptr<ExecuteResponse<DirectoryModifyError>>
+                create_channel_directory(ServerId id, ChannelId channelId, const std::string &string) override;
+
+                std::shared_ptr<ExecuteResponse<FileModifyError>>
+                delete_channel_file(ServerId id, ChannelId channelId, const std::string &string) override;
+
+                std::shared_ptr<ExecuteResponse<FileModifyError>>
+                rename_channel_file(ServerId id, ChannelId channelId, const std::string &, const std::string &) override;
+
+                std::shared_ptr<directory_query_response_t> query_icon_directory(ServerId id) override;
+
+                std::shared_ptr<ExecuteResponse<FileModifyError>>
+                delete_icon(ServerId id, const std::string &string) override;
+
+                std::shared_ptr<directory_query_response_t> query_avatar_directory(ServerId id) override;
+
+                std::shared_ptr<ExecuteResponse<FileModifyError>>
+                delete_avatar(ServerId id, const std::string &string) override;
+
+            private:
+#ifdef FS_INCLUDED
+                [[nodiscard]] fs::path server_path(ServerId);
+                [[nodiscard]] fs::path server_channel_path(ServerId, ChannelId);
+                [[nodiscard]] static bool exceeds_base_path(const fs::path& /* base */, const fs::path& /* target */);
+                [[nodiscard]] bool is_any_file_locked(const fs::path& /* base */, const std::string& /* path */, std::string& /* file (relative to the base) */);
+
+                [[nodiscard]] std::shared_ptr<ExecuteResponse<FileModifyError>>
+                delete_file(const fs::path& /* base */, const std::string &string);
+
+                [[nodiscard]] std::shared_ptr<directory_query_response_t>
+                query_directory(const fs::path& /* base */, const std::string &string, bool);
+#endif
+
+                template <typename error_t, typename result_t = EmptyExecuteResponse>
+                std::shared_ptr<ExecuteResponse<error_t, result_t>> create_execute_response() {
+                    return std::make_shared<ExecuteResponse<error_t, result_t>>(this->result_notify_mutex, this->result_notify_cv);
+                }
+                std::string target_file_path(FileCategory type, ts::ServerId sid, ts::ChannelId cid, const std::string &path);
+
+                std::mutex result_notify_mutex{};
+                std::condition_variable result_notify_cv{};
+
+                std::string root_path_{};
+
+                std::mutex locked_files_mutex{};
+                std::deque<std::string> locked_files_{};
+        };
+    }
+
+    namespace transfer {
+        class LocalFileTransfer;
+
+        struct Buffer {
+            Buffer* next{nullptr};
+
+            size_t capacity{0};
+            size_t length{0};
+            size_t offset{0};
+
+            char data[1]{};
+        };
+        [[nodiscard]] extern Buffer* allocate_buffer(size_t);
+        extern void free_buffer(Buffer*);
+
+        struct NetworkThrottle {
+            constexpr static auto kThrottleTimespanMs{250};
+            typedef uint8_t span_t;
+
+            ssize_t max_bytes{0};
+
+            span_t current_index{0};
+            size_t bytes_send{0};
+
+            inline bool increase_bytes(size_t bytes) {
+                auto current_ms = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
+                auto current_span = (span_t) (current_ms / kThrottleTimespanMs);
+                if(this->current_index != current_span) {
+                    this->current_index = current_span;
+                    this->bytes_send = bytes;
+                } else {
+                    this->bytes_send += bytes;
+                }
+                return this->max_bytes > 0 && this->bytes_send >= this->max_bytes;
+            }
+
+            inline void set_max_bandwidth(ssize_t bytes_per_second) {
+                if(bytes_per_second <= 0)
+                    this->max_bytes = -1;
+                else
+                    this->max_bytes = bytes_per_second * kThrottleTimespanMs / 1000;
+            }
+
+            [[nodiscard]] inline bool should_throttle(timeval& next_timestamp) {
+                if(this->max_bytes <= 0) return false;
+
+                auto current_ms = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
+                auto current_span = (span_t) (current_ms / kThrottleTimespanMs);
+                if(this->current_index != current_span) return false;
+                if(this->bytes_send < this->max_bytes) return false;
+
+                next_timestamp.tv_usec = (kThrottleTimespanMs - current_ms % kThrottleTimespanMs) * 1000;
+                next_timestamp.tv_sec =  next_timestamp.tv_usec / 1000000;
+                next_timestamp.tv_usec -= next_timestamp.tv_sec * 1000000;
+                return true;
+            }
+
+            [[nodiscard]] inline size_t bytes_left() const {
+                if(this->max_bytes <= 0) return (size_t) -1;
+
+                auto current_ms = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
+                auto current_span = (span_t) (current_ms / kThrottleTimespanMs);
+                if(this->current_index != current_span) return this->max_bytes;
+                if(this->bytes_send < this->max_bytes) return this->max_bytes - this->bytes_send;
+                return 0;
+            }
+
+            [[nodiscard]] inline std::chrono::milliseconds expected_writing_time(size_t bytes) const {
+                if(this->max_bytes <= 0) return std::chrono::milliseconds{0};
+
+                return std::chrono::seconds{bytes / (this->max_bytes * (1000 / kThrottleTimespanMs))};
+            }
+        };
+
+        /* all variables are locked via the state_mutex */
+        struct FileClient : std::enable_shared_from_this<FileClient> {
+            LocalFileTransfer* handle;
+            std::shared_ptr<Transfer> transfer{nullptr};
+
+            std::shared_mutex state_mutex{};
+            enum {
+                STATE_AWAITING_KEY, /* includes SSL/HTTP init */
+                STATE_TRANSFERRING,
+                STATE_DISCONNECTING,
+                STATE_DISCONNECTED
+            } state{STATE_AWAITING_KEY};
+
+            enum NetworkingProtocol {
+                PROTOCOL_UNKNOWN,
+                PROTOCOL_HTTPS,
+                PROTOCOL_TS_V1
+            };
+
+            enum HTTPUploadState {
+                HTTP_AWAITING_HEADER,
+                HTTP_STATE_AWAITING_BOUNDARY,
+                HTTP_STATE_UPLOADING,
+                HTTP_STATE_DOWNLOADING
+            };
+
+            struct {
+                bool file_locked{false};
+                std::string absolute_path{};
+
+                int file_descriptor{0};
+
+                bool currently_processing{false};
+                FileClient* next_client{nullptr};
+            } file{};
+
+            struct {
+                size_t provided_bytes{0};
+                char key[TRANSFER_KEY_LENGTH]{0};
+            } transfer_key{};
+
+            /* will be used for both directions */
+            struct {
+                std::mutex mutex{};
+                size_t bytes{0};
+
+                bool buffering_stopped{false};
+
+                Buffer* buffer_head{nullptr};
+                Buffer** buffer_tail{&buffer_head};
+            } buffer{};
+
+            struct {
+                sockaddr_storage address{};
+                int file_descriptor{0};
+
+                NetworkingProtocol protocol{PROTOCOL_UNKNOWN};
+
+                struct event* event_read{nullptr};
+                struct event* event_write{nullptr};
+                struct event* event_throttle{nullptr};
+
+                bool add_event_write{false}, add_event_read{false};
+
+                std::chrono::system_clock::time_point disconnect_timeout{};
+
+                NetworkThrottle throttle;
+
+                pipes::SSL pipe_ssl{};
+                bool pipe_ssl_init{false};
+                std::unique_ptr<Buffer, decltype(free_buffer)*> http_header_buffer{nullptr, free_buffer};
+                HTTPUploadState http_state{HTTPUploadState::HTTP_AWAITING_HEADER};
+
+                /* Only read the transfer key length at the beginning. We than have the actual limit which will be set via throttle */
+                size_t max_read_buffer_size{TRANSFER_KEY_LENGTH};
+            } networking{};
+
+            struct {
+                size_t network_bytes_send{0};
+                size_t network_bytes_received{0};
+
+                size_t file_bytes_transferred{0};
+
+                /* used for delta statistics */
+                size_t last_network_bytes_send{0};
+                size_t last_network_bytes_received{0};
+
+                /* used for delta statistics */
+                size_t last_file_bytes_transferred{0};
+
+                size_t disk_bytes_read{0};
+                size_t disk_bytes_write{0};
+            } statistics{};
+
+            struct {
+                std::chrono::system_clock::time_point last_write{};
+                std::chrono::system_clock::time_point last_read{};
+
+                std::chrono::system_clock::time_point connected{};
+                std::chrono::system_clock::time_point key_received{};
+                std::chrono::system_clock::time_point disconnecting{};
+            } timings;
+
+            explicit FileClient(LocalFileTransfer* handle) : handle{handle} {}
+            ~FileClient();
+
+            void add_network_write_event(bool /* ignore bandwidth limits */);
+            void add_network_write_event_nolock(bool /* ignore bandwidth limits */);
+
+            /* will check if we've enough space in out read buffer again */
+            void add_network_read_event(bool /* ignore bandwidth limits */);
+
+            bool send_file_bytes(const void* /* buffer */, size_t /* length */);
+            bool enqueue_buffer_bytes(const void* /* buffer */, size_t /* length */);
+
+            [[nodiscard]] inline std::string log_prefix() const { return "[" + net::to_string(this->networking.address) + "]"; }
+        };
+
+        enum struct DiskIOStartResult {
+            SUCCESS,
+            OUT_OF_MEMORY
+        };
+
+        enum struct NetworkingStartResult {
+            SUCCESS,
+            OUT_OF_MEMORY,
+            NO_BINDINGS
+        };
+
+        enum struct ClientWorkerStartResult {
+            SUCCESS
+        };
+
+        enum struct NetworkInitializeResult {
+            SUCCESS,
+            OUT_OF_MEMORY
+        };
+
+        enum struct FileInitializeResult {
+            SUCCESS,
+
+            INVALID_TRANSFER_DIRECTION,
+            OUT_OF_MEMORY,
+
+            PROCESS_FILE_LIMIT_REACHED,
+            SYSTEM_FILE_LIMIT_REACHED,
+
+            FILE_IS_BUSY,
+            FILE_DOES_NOT_EXISTS,
+            FILE_SYSTEM_ERROR,
+            FILE_IS_A_DIRECTORY,
+
+            FILE_TOO_LARGE,
+            DISK_IS_READ_ONLY,
+
+            FILE_SIZE_MISMATCH,
+            FILE_SEEK_FAILED,
+
+            FILE_IS_NOT_ACCESSIBLE,
+
+            MAX
+        };
+
+        constexpr static std::array<std::string_view, (size_t) FileInitializeResult::MAX> kFileInitializeResultMessages{
+                /* SUCCESS */                       "success",
+
+                /* INVALID_TRANSFER_DIRECTION */    "invalid file transfer direction",
+                /* OUT_OF_MEMORY */                 "out of memory",
+
+                /* PROCESS_FILE_LIMIT_REACHED */    "process file limit reached",
+                /* SYSTEM_FILE_LIMIT_REACHED */     "system file limit reached",
+
+                /* FILE_IS_BUSY */                  "target file is busy",
+                /* FILE_DOES_NOT_EXISTS */          "target file does not exists",
+                /* FILE_SYSTEM_ERROR */             "internal file system error",
+                /* FILE_IS_A_DIRECTORY */           "target file is a directory",
+
+                /* FILE_TOO_LARGE */                "file is too large",
+                /* DISK_IS_READ_ONLY */             "disk is in read only mode",
+
+                /* FILE_SIZE_MISMATCH */            "file size mismatch",
+                /* FILE_SEEK_FAILED */              "failed to seek to target file offset",
+
+                /* FILE_IS_NOT_ACCESSIBLE */        "file is not accessible"
+        };
+
+        enum struct TransferKeyApplyResult {
+            SUCCESS,
+            FILE_ERROR,
+            UNKNOWN_KEY,
+
+            INTERNAL_ERROR
+        };
+
+        enum struct TransferUploadRawResult {
+            MORE_DATA_TO_RECEIVE,
+            FINISH,
+
+            /* UNKNOWN ERROR */
+        };
+
+        enum struct TransferUploadHTTPResult {
+            MORE_DATA_TO_RECEIVE,
+            FINISH,
+
+            BOUNDARY_MISSING,
+            BOUNDARY_TOKEN_INVALID,
+            BOUNDARY_INVALID,
+
+            MISSING_CONTENT_TYPE,
+            INVALID_CONTENT_TYPE
+            /* UNKNOWN ERROR */
+        };
+
+        struct NetworkBinding : std::enable_shared_from_this<NetworkBinding> {
+            std::string hostname{};
+            sockaddr_storage address{};
+
+            int file_descriptor{-1};
+            struct event* accept_event{nullptr};
+
+            LocalFileTransfer* handle{nullptr};
+        };
+
+        class LocalFileTransfer : public AbstractProvider {
+            public:
+                explicit LocalFileTransfer(filesystem::LocalFileSystem&);
+                ~LocalFileTransfer();
+
+                [[nodiscard]] bool start(const std::deque<std::shared_ptr<NetworkBinding>>& /* bindings */, const std::shared_ptr<pipes::SSL::Options>& /* ssl options */);
+                void stop();
+
+                [[nodiscard]] inline const auto& ssl_options() const {
+                    return this->ssl_options_;
+                }
+
+                inline void set_ssl_options(const std::shared_ptr<pipes::SSL::Options>& options) {
+                    this->ssl_options_ = options;
+                }
+
+                std::shared_ptr<ExecuteResponse<TransferInitError, std::shared_ptr<Transfer>>>
+                initialize_channel_transfer(Transfer::Direction direction, ServerId id, ChannelId channelId,
+                                            const TransferInfo &info) override;
+
+                std::shared_ptr<ExecuteResponse<TransferInitError, std::shared_ptr<Transfer>>>
+                initialize_icon_transfer(Transfer::Direction direction, ServerId id, const TransferInfo &info) override;
+
+                std::shared_ptr<ExecuteResponse<TransferInitError, std::shared_ptr<Transfer>>>
+                initialize_avatar_transfer(Transfer::Direction direction, ServerId id, const TransferInfo &info) override;
+
+                std::shared_ptr<ExecuteResponse<TransferActionError>> stop_transfer(transfer_id id, bool) override;
+            private:
+                enum struct DiskIOLoopState {
+                    STOPPED,
+                    RUNNING,
+
+                    STOPPING,
+                    FORCE_STOPPING
+                };
+                filesystem::LocalFileSystem& file_system_;
+
+                std::atomic<transfer_id> current_transfer_id{0};
+                std::mt19937 transfer_random_token_generator{std::random_device{}()};
+
+                std::mutex result_notify_mutex{};
+                std::condition_variable result_notify_cv{};
+
+                std::mutex transfers_mutex{};
+                std::deque<std::shared_ptr<FileClient>> transfers_{};
+                std::deque<std::shared_ptr<Transfer>> pending_transfers{};
+
+                std::shared_ptr<pipes::SSL::Options> ssl_options_{};
+
+                enum ServerState {
+                    STOPPED,
+                    RUNNING
+                } state{ServerState::STOPPED};
+
+                struct {
+                    bool active{false};
+
+                    std::thread dispatch_thread{};
+                    std::mutex mutex{};
+                    std::condition_variable notify_cv{};
+                } disconnect;
+
+                struct {
+                    bool active{false};
+                    std::thread dispatch_thread{};
+                    struct event_base* event_base{nullptr};
+
+                    std::deque<std::shared_ptr<NetworkBinding>> bindings{};
+                } network{};
+
+                struct {
+                    DiskIOLoopState state{DiskIOLoopState::STOPPED};
+                    std::thread dispatch_thread{};
+                    std::mutex queue_lock{};
+                    std::condition_variable notify_work_awaiting{};
+                    std::condition_variable notify_client_processed{};
+
+                    FileClient* queue_head{nullptr};
+                    FileClient** queue_tail{&queue_head};
+                } disk_io{};
+
+                template <typename error_t, typename result_t = EmptyExecuteResponse>
+                std::shared_ptr<ExecuteResponse<error_t, result_t>> create_execute_response() {
+                    return std::make_shared<ExecuteResponse<error_t, result_t>>(this->result_notify_mutex, this->result_notify_cv);
+                }
+
+                std::shared_ptr<ExecuteResponse<TransferInitError, std::shared_ptr<Transfer>>>
+                initialize_transfer(Transfer::Direction, ServerId, ChannelId, Transfer::TargetType, const TransferInfo &info);
+
+                [[nodiscard]] NetworkingStartResult start_networking();
+                [[nodiscard]] DiskIOStartResult start_disk_io();
+                [[nodiscard]] ClientWorkerStartResult start_client_worker();
+
+                void shutdown_networking();
+                void shutdown_disk_io();
+                void shutdown_client_worker();
+
+                void disconnect_client(const std::shared_ptr<FileClient>& /* client */, std::unique_lock<std::shared_mutex>& /* state lock */, bool /* flush */);
+
+                [[nodiscard]] NetworkInitializeResult initialize_networking(const std::shared_ptr<FileClient>& /* client */, int /* file descriptor */);
+                /* might block 'till all IO operations have been succeeded */
+                void finalize_networking(const std::shared_ptr<FileClient>& /* client */, std::unique_lock<std::shared_mutex>& /* state lock */);
+
+                [[nodiscard]] FileInitializeResult initialize_file_io(const std::shared_ptr<FileClient>& /* client */);
+                void finalize_file_io(const std::shared_ptr<FileClient>& /* client */, std::unique_lock<std::shared_mutex>& /* state lock */);
+
+                [[nodiscard]] bool initialize_client_ssl(const std::shared_ptr<FileClient>& /* client */);
+                void finalize_client_ssl(const std::shared_ptr<FileClient>& /* client */);
+
+                void enqueue_disk_io(const std::shared_ptr<FileClient>& /* client */);
+                void execute_disk_io(const std::shared_ptr<FileClient>& /* client */);
+
+                void report_transfer_statistics(const std::shared_ptr<FileClient>& /* client */);
+                [[nodiscard]] TransferUploadRawResult handle_transfer_upload_raw(const std::shared_ptr<FileClient>& /* client */, const char * /* buffer */, size_t /* length */);
+                [[nodiscard]] TransferUploadHTTPResult handle_transfer_upload_http(const std::shared_ptr<FileClient>& /* client */, const char * /* buffer */, size_t /* length */);
+
+                void send_http_response(const std::shared_ptr<FileClient>& /* client */, http::HttpResponse& /* response */);
+
+                static void callback_transfer_network_write(int, short, void*);
+                static void callback_transfer_network_read(int, short, void*);
+                static void callback_transfer_network_throttle(int, short, void*);
+                static void callback_transfer_network_accept(int, short, void*);
+
+                static void dispatch_loop_client_worker(void*);
+                static void dispatch_loop_network(void*);
+                static void dispatch_loop_disk_io(void*);
+
+                size_t handle_transfer_read(const std::shared_ptr<FileClient>& /* client */, const char* /* buffer */, size_t /* bytes */);
+                size_t handle_transfer_read_raw(const std::shared_ptr<FileClient>& /* client */, const char* /* buffer */, size_t /* bytes */);
+                [[nodiscard]] TransferKeyApplyResult handle_transfer_key_provided(const std::shared_ptr<FileClient>& /* client */, std::string& /* error */);
+        };
+    }
+
+    class LocalFileProvider : public AbstractFileServer {
+        public:
+            LocalFileProvider();
+            virtual ~LocalFileProvider();
+
+            [[nodiscard]] bool initialize(std::string& /* error */, const std::shared_ptr<pipes::SSL::Options>& /* ssl options */);
+            void finalize();
+
+            filesystem::AbstractProvider &file_system() override;
+            transfer::AbstractProvider &file_transfer() override;
+
+        private:
+            filesystem::LocalFileSystem file_system_;
+            transfer::LocalFileTransfer file_transfer_;
+    };
+}

file/local_server/LocalFileSystem.cpp

@@ -0,0 +1,350 @@
+//
+// Created by WolverinDEV on 29/04/2020.
+//
+#include <experimental/filesystem>
+#define FS_INCLUDED
+
+#include <log/LogUtils.h>
+#include "LocalFileProvider.h"
+#include "clnpath.h"
+
+using namespace ts::server::file;
+using namespace ts::server::file::filesystem;
+namespace fs = std::experimental::filesystem;
+using directory_query_response_t = AbstractProvider::directory_query_response_t;
+
+LocalFileSystem::~LocalFileSystem() = default;
+
+bool LocalFileSystem::initialize(std::string &error_message, const std::string &root_path_string) {
+    auto root_path = fs::u8path(root_path_string);
+    std::error_code error{};
+
+    if(!fs::exists(root_path, error)) {
+        if(error)
+            logWarning(LOG_FT, "Failed to check root path existence. Assuming it does not exist. ({}/{})", error.value(), error.message());
+        if(!fs::create_directories(root_path, error) || error) {
+            error_message = "Failed to create root file system at " + root_path_string + ": " + std::to_string(error.value()) + "/" + error.message();
+            return false;
+        }
+    }
+
+    auto croot = clnpath(fs::absolute(root_path).string());
+    logMessage(LOG_FT, "Started file system root at {}", croot);
+    this->root_path_ = croot;
+    return true;
+}
+
+void LocalFileSystem::finalize() {}
+
+fs::path LocalFileSystem::server_path(ts::ServerId id) {
+    return fs::u8path(this->root_path_) / fs::u8path("server_" + std::to_string(id));
+}
+
+fs::path LocalFileSystem::server_channel_path(ts::ServerId sid, ts::ChannelId cid) {
+    return this->server_path(sid) / fs::u8path("channel_" + std::to_string(cid));
+}
+
+bool LocalFileSystem::exceeds_base_path(const fs::path &base, const fs::path &target) {
+    auto rel_target = clnpath(target.string());
+    if(rel_target.starts_with("..")) return true;
+
+    auto base_string = clnpath(fs::absolute(base).string());
+    auto target_string = clnpath(fs::absolute(target).string());
+    return !target_string.starts_with(base_string);
+}
+
+bool LocalFileSystem::is_any_file_locked(const fs::path &base, const std::string &path, std::string &locked_file) {
+    auto c_path = clnpath(fs::absolute(base / fs::u8path(path)).string());
+
+    std::lock_guard lock{this->locked_files_mutex};
+    for(const auto& lfile : this->locked_files_) {
+        if(lfile.starts_with(c_path)) {
+            locked_file = lfile.substr(base.string().length());
+            return true;
+        }
+    }
+
+    return false;
+}
+
+std::string LocalFileSystem::target_file_path(FileCategory type, ts::ServerId sid, ts::ChannelId cid, const std::string &path) {
+    fs::path target_path{};
+    switch (type) {
+        case FileCategory::CHANNEL:
+            target_path = this->server_channel_path(sid, cid) / path;
+            break;
+        case FileCategory::ICON:
+            target_path = this->server_path(sid) / "icons" / path;
+            break;
+        case FileCategory::AVATAR:
+            target_path = this->server_path(sid) / "avatars" / path;
+            break;
+    }
+
+    return clnpath(fs::absolute(target_path).string());
+}
+
+std::string LocalFileSystem::absolute_avatar_path(ServerId sid, const std::string &path) {
+    return this->target_file_path(FileCategory::AVATAR, sid, 0, path);
+}
+
+std::string LocalFileSystem::absolute_icon_path(ServerId sid, const std::string &path) {
+    return this->target_file_path(FileCategory::ICON, sid, 0, path);
+}
+
+std::string LocalFileSystem::absolute_channel_path(ServerId sid, ChannelId cid, const std::string &path) {
+    return this->target_file_path(FileCategory::CHANNEL, sid, cid, path);
+}
+
+void LocalFileSystem::lock_file(const std::string &c_path) {
+    std::lock_guard lock{this->locked_files_mutex};
+    this->locked_files_.push_back(c_path);
+}
+
+void LocalFileSystem::unlock_file(const std::string &c_path) {
+    std::lock_guard lock{this->locked_files_mutex};
+
+    this->locked_files_.erase(std::remove_if(this->locked_files_.begin(), this->locked_files_.end(), [&](const auto& p) { return p == c_path; }), this->locked_files_.end());
+}
+
+std::shared_ptr<ExecuteResponse<ServerCommandError>> LocalFileSystem::initialize_server(ServerId id) {
+    auto path = this->server_path(id);
+    std::error_code error{};
+
+    auto response = this->create_execute_response<ServerCommandError>();
+
+    if(!fs::exists(path, error)) {
+        if(!fs::create_directories(path, error) || error) {
+            response->emplace_fail(ServerCommandErrorType::FAILED_TO_CREATE_DIRECTORIES, std::to_string(error.value()) + "/" + error.message());
+            return response;
+        }
+    }
+
+    //TODO: Copy the default icon
+
+    response->emplace_success();
+    return response;
+}
+
+std::shared_ptr<ExecuteResponse<ServerCommandError>> LocalFileSystem::delete_server(ServerId id) {
+    auto path = this->server_path(id);
+    std::error_code error{};
+
+    auto response = this->create_execute_response<ServerCommandError>();
+
+    //TODO: Stop all running file transfers!
+
+    if(fs::exists(path, error)) {
+        if(!fs::remove_all(path, error) || error) {
+            response->emplace_fail(ServerCommandErrorType::FAILED_TO_DELETE_DIRECTORIES, std::to_string(error.value()) + "/" + error.message());
+            return response;
+        }
+    }
+
+    response->emplace_success();
+    return response;
+}
+
+std::shared_ptr<directory_query_response_t> LocalFileSystem::query_directory(const fs::path &base,
+                                                                             const std::string &path,
+                                                                             bool allow_non_existance) {
+    std::error_code error{};
+    auto response = this->create_execute_response<DirectoryQueryError, std::deque<DirectoryEntry>>();
+    auto target_path = base / fs::u8path(path);
+    if(this->exceeds_base_path(base, target_path)) {
+        response->emplace_fail(DirectoryQueryErrorType::PATH_EXCEEDS_ROOT_PATH, "");
+        return response;
+    }
+
+    if(!fs::exists(target_path, error)) {
+        if(allow_non_existance)
+            response->emplace_success();
+        else
+            response->emplace_fail(DirectoryQueryErrorType::PATH_DOES_NOT_EXISTS, "");
+        return response;
+    } else if(error) {
+        logWarning(LOG_FT, "Failed to check for file at {}: {}. Assuming it does not exists.", target_path.string(), error.value(), error.message());
+        response->emplace_fail(DirectoryQueryErrorType::PATH_DOES_NOT_EXISTS, "");
+        return response;
+    }
+
+    if(!fs::is_directory(target_path, error)) {
+        response->emplace_fail(DirectoryQueryErrorType::PATH_IS_A_FILE, "");
+        return response;
+    } else if(error) {
+        logWarning(LOG_FT, "Failed to check for directory at {}: {}. Assuming its not a directory.", target_path.string(), error.value(), error.message());
+        response->emplace_fail(DirectoryQueryErrorType::PATH_IS_A_FILE, "");
+        return response;
+    }
+
+    std::deque<DirectoryEntry> entries{};
+    for(auto& entry : fs::directory_iterator(target_path, error)) {
+        auto status = entry.status(error);
+        if(error) {
+            logWarning(LOG_FT, "Failed to query file status for {} ({}/{}). Skipping entry for directory query.", entry.path().string(), error.value(), error.message());
+            continue;
+        }
+
+        if(status.type() == fs::file_type::directory) {
+            auto& dentry = entries.emplace_back();
+            dentry.type = DirectoryEntry::DIRECTORY;
+            dentry.name = entry.path().filename();
+
+            dentry.modified_at = fs::last_write_time(entry.path(), error);
+            if(error)
+                logWarning(LOG_FT, "Failed to query last write time for directory {} ({}/{})", entry.path().string(), error.value(), error.message());
+            dentry.size = 0;
+        } else if(status.type() == fs::file_type::regular) {
+            auto& dentry = entries.emplace_back();
+            dentry.type = DirectoryEntry::FILE;
+            dentry.name = entry.path().filename();
+
+            dentry.modified_at = fs::last_write_time(entry.path(), error);
+            if(error)
+                logWarning(LOG_FT, "Failed to query last write time for file {} ({}/{}).", entry.path().string(), error.value(), error.message());
+            dentry.size = fs::file_size(entry.path(), error);
+            if(error)
+                logWarning(LOG_FT, "Failed to query size for file {} ({}/{}).", entry.path().string(), error.value(), error.message());
+        } else {
+            logWarning(LOG_FT, "Directory query listed an unknown file type for file {} ({}).", entry.path().string(), (int) status.type());
+        }
+    }
+    if(error && entries.empty()) {
+        response->emplace_fail(DirectoryQueryErrorType::FAILED_TO_LIST_FILES, std::to_string(error.value()) + "/" + error.message());
+        return response;
+    }
+    response->emplace_success(std::forward<decltype(entries)>(entries));
+    return response;
+}
+
+std::shared_ptr<directory_query_response_t> LocalFileSystem::query_icon_directory(ServerId id) {
+    return this->query_directory(this->server_path(id) / fs::u8path("icons"), "/", true);
+}
+
+std::shared_ptr<directory_query_response_t> LocalFileSystem::query_avatar_directory(ServerId id) {
+    return this->query_directory(this->server_path(id) / fs::u8path("avatars"), "/", true);
+}
+
+std::shared_ptr<directory_query_response_t> LocalFileSystem::query_channel_directory(ServerId id, ChannelId channelId, const std::string &path) {
+    return this->query_directory(this->server_channel_path(id, channelId), path, false);
+}
+
+std::shared_ptr<ExecuteResponse<DirectoryModifyError>> LocalFileSystem::create_channel_directory(ServerId id, ChannelId channelId, const std::string &path) {
+    auto channel_path_root = this->server_channel_path(id, channelId);
+    std::error_code error{};
+
+    auto response = this->create_execute_response<DirectoryModifyError>();
+    auto target_path = channel_path_root / fs::u8path(path);
+    if(this->exceeds_base_path(channel_path_root, target_path)) {
+        response->emplace_fail(DirectoryModifyErrorType::PATH_EXCEEDS_ROOT_PATH, "");
+        return response;
+    }
+
+    if(fs::exists(target_path, error)) {
+        response->emplace_fail(DirectoryModifyErrorType::PATH_ALREADY_EXISTS, "");
+        return response;
+    } else if(error) {
+        logWarning(LOG_FT, "Failed to check for file at {}: {}. Assuming it does not exists.", target_path.string(), error.value(), error.message());
+    }
+
+    if(!fs::create_directories(target_path, error) || error) {
+        response->emplace_fail(DirectoryModifyErrorType::FAILED_TO_CREATE_DIRECTORIES, std::to_string(error.value()) + "/" + error.message());
+        return response;
+    }
+
+    response->emplace_success();
+    return response;
+}
+
+std::shared_ptr<ExecuteResponse<FileModifyError>> LocalFileSystem::rename_channel_file(ServerId id, ChannelId channelId, const std::string &current_path_string, const std::string &new_path_string) {
+    auto channel_path_root = this->server_channel_path(id, channelId);
+    std::error_code error{};
+    std::string locked_file{};
+
+    auto response = this->create_execute_response<FileModifyError>();
+    auto current_path = channel_path_root / fs::u8path(current_path_string);
+    auto target_path = channel_path_root / fs::u8path(new_path_string);
+
+    if(this->exceeds_base_path(channel_path_root, current_path)) {
+        response->emplace_fail(FileModifyErrorType::PATH_EXCEEDS_ROOT_PATH, "");
+        return response;
+    }
+    if(this->exceeds_base_path(channel_path_root, target_path)) {
+        response->emplace_fail(FileModifyErrorType::TARGET_PATH_EXCEEDS_ROOT_PATH, "");
+        return response;
+    }
+
+    if(!fs::exists(current_path, error)) {
+        response->emplace_fail(FileModifyErrorType::PATH_DOES_NOT_EXISTS, "");
+        return response;
+    } else if(error) {
+        logWarning(LOG_FT, "Failed to check for file at {}: {}. Assuming it does not exists.", current_path.string(), error.value(), error.message());
+        response->emplace_fail(FileModifyErrorType::PATH_DOES_NOT_EXISTS, "");
+        return response;
+    }
+
+    if(fs::exists(target_path, error)) {
+        response->emplace_fail(FileModifyErrorType::TARGET_PATH_ALREADY_EXISTS, "");
+        return response;
+    } else if(error) {
+        logWarning(LOG_FT, "Failed to check for file at {}: {}. Assuming it does exists.", current_path.string(), error.value(), error.message());
+        response->emplace_fail(FileModifyErrorType::TARGET_PATH_ALREADY_EXISTS, "");
+        return response;
+    }
+
+    if(this->is_any_file_locked(channel_path_root, current_path, locked_file)) {
+        response->emplace_fail(FileModifyErrorType::SOME_FILES_ARE_LOCKED, locked_file);
+        return response;
+    }
+
+    fs::rename(current_path, target_path, error);
+    if(error) {
+        response->emplace_fail(FileModifyErrorType::FAILED_TO_RENAME_FILE, std::to_string(error.value()) + "/" + error.message());
+        return response;
+    }
+
+    response->emplace_success();
+    return response;
+}
+
+std::shared_ptr<ExecuteResponse<FileModifyError>> LocalFileSystem::delete_file(const fs::path &base,
+                                                                               const std::string &path) {
+    std::error_code error{};
+    std::string locked_file{};
+    auto response = this->create_execute_response<FileModifyError>();
+    auto target_path = base / fs::u8path(path);
+
+    if(fs::exists(target_path, error)) {
+        response->emplace_fail(FileModifyErrorType::TARGET_PATH_ALREADY_EXISTS, "");
+        return response;
+    } else if(error) {
+        logWarning(LOG_FT, "Failed to check for file at {}: {}. Assuming it does exists.", target_path.string(), error.value(), error.message());
+        response->emplace_fail(FileModifyErrorType::TARGET_PATH_ALREADY_EXISTS, "");
+        return response;
+    }
+
+    if(this->is_any_file_locked(base, path, locked_file)) {
+        response->emplace_fail(FileModifyErrorType::SOME_FILES_ARE_LOCKED, locked_file);
+        return response;
+    }
+
+    if(!fs::remove(target_path, error) || error) {
+        response->emplace_fail(FileModifyErrorType::FAILED_TO_DELETE_FILES, std::to_string(error.value()) + "/" + error.message());
+        return response;
+    }
+
+    response->emplace_success();
+    return response;
+}
+
+std::shared_ptr<ExecuteResponse<FileModifyError>> LocalFileSystem::delete_channel_file(ServerId id, ChannelId channelId, const std::string &path) {
+    return this->delete_file(this->server_channel_path(id, channelId), path);
+}
+
+std::shared_ptr<ExecuteResponse<FileModifyError>> LocalFileSystem::delete_icon(ServerId id, const std::string &icon) {
+    return this->delete_file(this->server_path(id) / fs::u8path("icons"), icon);
+}
+
+std::shared_ptr<ExecuteResponse<FileModifyError>> LocalFileSystem::delete_avatar(ServerId id, const std::string &avatar) {
+    return this->delete_file(this->server_path(id) / fs::u8path("avatars"), avatar);
+}

file/local_server/LocalFileTransfer.cpp

@@ -0,0 +1,212 @@
+//
+// Created by WolverinDEV on 04/05/2020.
+//
+
+#include <cassert>
+#include <event2/event.h>
+#include <log/LogUtils.h>
+#include <random>
+#include "./LocalFileProvider.h"
+#include "LocalFileProvider.h"
+
+using namespace ts::server::file;
+using namespace ts::server::file::transfer;
+
+Buffer* transfer::allocate_buffer(size_t size) {
+    auto total_size = sizeof(Buffer) + size;
+    auto buffer = (Buffer*) malloc(total_size);
+    new (buffer) Buffer{};
+    buffer->capacity = size;
+    return buffer;
+}
+
+void transfer::free_buffer(Buffer* buffer) {
+    buffer->~Buffer();
+    free(buffer);
+}
+
+FileClient::~FileClient() {
+    auto head = this->buffer.buffer_head;
+    while (head) {
+        auto next = head->next;
+        free_buffer(head);
+        head = next;
+    }
+
+    assert(!this->file.file_descriptor);
+    assert(!this->file.currently_processing);
+    assert(!this->file.next_client);
+
+    assert(!this->networking.event_read);
+    assert(!this->networking.event_write);
+
+    assert(this->state == STATE_DISCONNECTED);
+}
+
+LocalFileTransfer::LocalFileTransfer(filesystem::LocalFileSystem &fs) : file_system_{fs} {}
+LocalFileTransfer::~LocalFileTransfer() = default;
+
+bool LocalFileTransfer::start(const std::deque<std::shared_ptr<NetworkBinding>>& bindings, const std::shared_ptr<pipes::SSL::Options>& ssl_options) {
+    assert(ssl_options);
+    this->ssl_options_ = ssl_options;
+
+    (void) this->start_client_worker();
+
+    {
+        auto start_result = this->start_disk_io();
+        switch (start_result) {
+            case DiskIOStartResult::SUCCESS:
+                break;
+            case DiskIOStartResult::OUT_OF_MEMORY:
+                logError(LOG_FT, "Failed to start disk worker (Out of memory)");
+                goto error_exit_disk;
+            default:
+                logError(LOG_FT, "Failed to start disk worker ({})", (int) start_result);
+                goto error_exit_disk;
+        }
+    }
+
+
+    {
+        this->network.bindings = bindings;
+        auto start_result = this->start_networking();
+        switch (start_result) {
+            case NetworkingStartResult::SUCCESS:
+                break;
+            case NetworkingStartResult::OUT_OF_MEMORY:
+                logError(LOG_FT, "Failed to start networking (Out of memory)");
+                goto error_exit_network;
+            case NetworkingStartResult::NO_BINDINGS:
+                logError(LOG_FT, "Failed to start networking (No address could be bound)");
+                goto error_exit_network;
+            default:
+                logError(LOG_FT, "Failed to start networking ({})", (int) start_result);
+                goto error_exit_network;
+        }
+    }
+
+    return true;
+    error_exit_network:
+    this->shutdown_networking();
+
+    error_exit_disk:
+    this->shutdown_disk_io();
+    this->shutdown_client_worker();
+    return false;
+}
+
+void LocalFileTransfer::stop() {
+    this->shutdown_networking();
+    this->shutdown_disk_io();
+    this->shutdown_client_worker();
+}
+
+std::shared_ptr<ExecuteResponse<TransferInitError, std::shared_ptr<Transfer>>> LocalFileTransfer::initialize_icon_transfer(Transfer::Direction direction, ServerId sid, const TransferInfo &info) {
+    return this->initialize_transfer(direction, sid, 0, Transfer::TARGET_TYPE_ICON, info);
+}
+
+std::shared_ptr<ExecuteResponse<TransferInitError, std::shared_ptr<Transfer>>> LocalFileTransfer::initialize_avatar_transfer(Transfer::Direction direction, ServerId sid, const TransferInfo &info) {
+    return this->initialize_transfer(direction, sid, 0, Transfer::TARGET_TYPE_AVATAR, info);
+}
+
+std::shared_ptr<ExecuteResponse<TransferInitError, std::shared_ptr<Transfer>>> LocalFileTransfer::initialize_channel_transfer(Transfer::Direction direction, ServerId sid, ChannelId cid, const TransferInfo &info) {
+    return this->initialize_transfer(direction, sid, cid, Transfer::TARGET_TYPE_CHANNEL_FILE, info);
+}
+
+std::shared_ptr<ExecuteResponse<TransferInitError, std::shared_ptr<Transfer>>> LocalFileTransfer::initialize_transfer(
+        Transfer::Direction direction, ServerId sid, ChannelId cid,
+        Transfer::TargetType ttype,
+        const TransferInfo &info) {
+    auto response = this->create_execute_response<TransferInitError, std::shared_ptr<Transfer>>();
+
+    /* TODO: test for a transfer limit */
+
+    auto transfer = std::make_shared<Transfer>();
+    transfer->server_transfer_id = ++this->current_transfer_id;
+    transfer->server_id = sid;
+    transfer->channel_id = cid;
+    transfer->target_type = ttype;
+    transfer->direction = direction;
+
+    transfer->client_id = 0; /* must be provided externally */
+    transfer->client_transfer_id = 0; /* must be provided externally */
+
+    transfer->server_addresses.reserve(this->network.bindings.size());
+    for(auto& binding : this->network.bindings) {
+        if(!binding->file_descriptor) continue;
+
+        transfer->server_addresses.emplace_back(Transfer::Address{binding->hostname, net::port(binding->address)});
+    }
+
+    transfer->target_file_path = info.file_path;
+    transfer->file_offset = info.file_offset;
+    transfer->expected_file_size = info.expected_file_size;
+    transfer->max_bandwidth = info.max_bandwidth;
+
+    constexpr static std::string_view kTokenCharacters{"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"};
+    for(auto& c : transfer->transfer_key)
+        c = kTokenCharacters[transfer_random_token_generator() % kTokenCharacters.length()];
+    transfer->transfer_key[0] = (char) 'r'; /* (114) */ /* a non valid SSL header type to indicate that we're using a file transfer key and not doing a SSL handshake */
+    transfer->transfer_key[1] = (char) 'a'; /* ( 97) */
+    transfer->transfer_key[2] = (char) 'w'; /* (119) */
+
+    transfer->initialized_timestamp = std::chrono::system_clock::now();
+
+    {
+        std::lock_guard tlock{this->transfers_mutex};
+        this->pending_transfers.push_back(transfer);
+    }
+
+    if(auto callback{this->callback_transfer_registered}; callback)
+        callback(transfer);
+
+    response->emplace_success(std::move(transfer));
+    return response;
+}
+
+std::shared_ptr<ExecuteResponse<TransferActionError>> LocalFileTransfer::stop_transfer(transfer_id id, bool flush) {
+    auto response = this->create_execute_response<TransferActionError>();
+
+    std::shared_ptr<Transfer> transfer{};
+    std::shared_ptr<FileClient> connected_transfer{};
+
+    {
+        std::lock_guard tlock{this->transfers_mutex};
+
+        auto ct_it = std::find_if(this->transfers_.begin(), this->transfers_.end(), [&](const std::shared_ptr<FileClient>& t) {
+            return t->transfer && t->transfer->server_transfer_id == id;
+        });
+        if(ct_it != this->transfers_.end())
+            connected_transfer = *ct_it;
+        else {
+            auto t_it = std::find_if(this->pending_transfers.begin(), this->pending_transfers.end(), [&](const std::shared_ptr<Transfer>& t) {
+                return t->server_transfer_id == id;
+            });
+            if(t_it != this->pending_transfers.end()) {
+                transfer = *t_it;
+                this->pending_transfers.erase(t_it);
+            }
+        }
+    }
+
+    if(!transfer) {
+        if(connected_transfer)
+            transfer = connected_transfer->transfer;
+        else {
+            response->emplace_fail(TransferActionError{TransferActionError::UNKNOWN_TRANSFER, ""});
+            return response;
+        }
+    }
+
+    if(connected_transfer) {
+        logMessage(LOG_FT, "{} Stopping transfer due to an user request.", connected_transfer->log_prefix());
+
+        std::unique_lock slock{connected_transfer->state_mutex};
+        this->disconnect_client(connected_transfer, slock, flush);
+    } else {
+        logMessage(LOG_FT, "Removing pending file transfer for id {}", id);
+    }
+
+    response->emplace_success();
+    return response;
+}

file/local_server/LocalFileTransferClientWorker.cpp

@@ -0,0 +1,222 @@
+//
+// Created by WolverinDEV on 04/05/2020.
+//
+
+#include <cassert>
+#include <event2/event.h>
+#include <log/LogUtils.h>
+#include "./LocalFileProvider.h"
+#include "LocalFileProvider.h"
+
+using namespace ts::server::file;
+using namespace ts::server::file::transfer;
+
+ClientWorkerStartResult LocalFileTransfer::start_client_worker() {
+    assert(!this->disconnect.active);
+    this->disconnect.active = true;
+
+    this->disconnect.dispatch_thread = std::thread(&LocalFileTransfer::dispatch_loop_client_worker, this);
+    return ClientWorkerStartResult::SUCCESS;
+}
+
+void LocalFileTransfer::shutdown_client_worker() {
+    if(!this->disconnect.active) return;
+    this->disconnect.active = false;
+
+    this->disconnect.notify_cv.notify_all();
+    if(this->disconnect.dispatch_thread.joinable())
+        this->disconnect.dispatch_thread.join();
+
+    {
+        std::unique_lock tlock{this->transfers_mutex};
+        if(!this->transfers_.empty())
+            logWarning(LOG_FT, "Shutting down disconnect worker even thou we still have some active clients. This could cause memory leaks.");
+    }
+}
+
+void LocalFileTransfer::disconnect_client(const std::shared_ptr<FileClient> &client, std::unique_lock<std::shared_mutex>& state_lock, bool flush) {
+    assert(state_lock.owns_lock());
+
+    if(client->state == FileClient::STATE_DISCONNECTED || (client->state == FileClient::STATE_DISCONNECTING && flush)) {
+        return; /* shall NOT happen */
+    }
+
+#define del_ev_noblock(event) if(event) event_del_noblock(event)
+
+    client->state = flush ? FileClient::STATE_DISCONNECTING : FileClient::STATE_DISCONNECTED;
+    client->timings.disconnecting = std::chrono::system_clock::now();
+    if(flush) {
+        const auto network_flush_time = client->networking.throttle.expected_writing_time(client->buffer.bytes) + std::chrono::seconds{10};
+
+        if(!client->transfer) {
+            del_ev_noblock(client->networking.event_read);
+            del_ev_noblock(client->networking.event_throttle);
+            client->add_network_write_event_nolock(false);
+
+            /* max flush 10 seconds */
+            client->networking.disconnect_timeout = std::chrono::system_clock::now() + network_flush_time;
+            debugMessage(LOG_FT, "{} Disconnecting client. Flushing pending bytes (max {} seconds)", client->log_prefix(), std::chrono::floor<std::chrono::seconds>(network_flush_time).count());
+        } else if(client->transfer->direction == Transfer::DIRECTION_UPLOAD) {
+            del_ev_noblock(client->networking.event_read);
+            del_ev_noblock(client->networking.event_write);
+            del_ev_noblock(client->networking.event_throttle);
+
+            /* no direct timeout needed here, we're just flushing the disk */
+            this->enqueue_disk_io(client);
+        } else if(client->transfer->direction == Transfer::DIRECTION_DOWNLOAD) {
+            del_ev_noblock(client->networking.event_read);
+            client->add_network_write_event_nolock(false);
+
+            /* max flush 10 seconds */
+            client->networking.disconnect_timeout = std::chrono::system_clock::now() + network_flush_time;
+            debugMessage(LOG_FT, "{} Disconnecting client. Flushing pending bytes (max {} seconds)", client->log_prefix(), std::chrono::floor<std::chrono::seconds>(network_flush_time).count());
+        }
+    } else {
+        del_ev_noblock(client->networking.event_read);
+        del_ev_noblock(client->networking.event_write);
+        del_ev_noblock(client->networking.event_throttle);
+
+        this->disconnect.notify_cv.notify_one();
+    }
+
+#undef del_ev_noblock
+}
+
+void LocalFileTransfer::dispatch_loop_client_worker(void *ptr_transfer) {
+    auto provider = reinterpret_cast<LocalFileTransfer*>(ptr_transfer);
+
+    while(provider->disconnect.active) {
+        {
+            std::unique_lock dlock{provider->disconnect.mutex};
+            provider->disconnect.notify_cv.wait_for(dlock, std::chrono::seconds{1});
+        }
+        /* run the disconnect worker at least once before exiting */
+
+        /* transfer statistics */
+        {
+            std::unique_lock tlock{provider->transfers_mutex};
+            auto transfers = provider->transfers_;
+            tlock.unlock();
+            for(const auto& transfer : transfers) {
+                switch(transfer->state) {
+                    case FileClient::STATE_TRANSFERRING:
+                        break;
+                    case FileClient::STATE_DISCONNECTING:
+                        if(transfer->transfer && transfer->transfer->direction == Transfer::DIRECTION_DOWNLOAD)
+                            break; /* we're still transferring (sending data) */
+                    default:
+                        continue;
+                }
+
+                provider->report_transfer_statistics(transfer->shared_from_this());
+            }
+        }
+
+        {
+            std::deque<std::shared_ptr<Transfer>> timeouted_transfers{};
+
+            {
+                std::unique_lock tlock{provider->transfers_mutex};
+
+                auto now = std::chrono::system_clock::now();
+                std::copy_if(provider->pending_transfers.begin(), provider->pending_transfers.end(), std::back_inserter(timeouted_transfers), [&](const std::shared_ptr<Transfer>& t) {
+                    return t->initialized_timestamp + std::chrono::seconds{100} < now; //FIXME: Decrease to 10 again!
+                });
+                provider->pending_transfers.erase(std::remove_if(provider->pending_transfers.begin(), provider->pending_transfers.end(), [&](const auto& t) {
+                    return std::find(timeouted_transfers.begin(), timeouted_transfers.end(), t) != timeouted_transfers.end();
+                }), provider->pending_transfers.end());
+            }
+
+            for(const auto& pt : timeouted_transfers) {
+                if(auto callback{provider->callback_transfer_aborted}; callback)
+                    callback(pt, { TransferError::TRANSFER_TIMEOUT, "" });
+            }
+
+            if(!timeouted_transfers.empty())
+                logMessage(LOG_FT, "Removed {} pending transfers because no request has been made for them.", timeouted_transfers.size());
+        }
+
+
+        {
+            std::deque<std::shared_ptr<FileClient>> disconnected_clients{};
+            {
+                std::unique_lock tlock{provider->transfers_mutex};
+
+                auto now = std::chrono::system_clock::now();
+                std::copy_if(provider->transfers_.begin(), provider->transfers_.end(), std::back_inserter(disconnected_clients), [&](const std::shared_ptr<FileClient>& t) {
+                    std::shared_lock slock{t->state_mutex};
+                    if(t->state == FileClient::STATE_DISCONNECTED) {
+                        return true;
+                    } else if(t->state == FileClient::STATE_AWAITING_KEY) {
+                        return t->timings.connected + std::chrono::seconds{10} < now;
+                    } else if(t->state == FileClient::STATE_TRANSFERRING) {
+                        assert(t->transfer);
+                        if(t->transfer->direction == Transfer::DIRECTION_UPLOAD) {
+                            return false; //FIXME: Due to debugging reasons
+                            return t->timings.last_read + std::chrono::seconds{5} < now;
+                        } else if(t->transfer->direction == Transfer::DIRECTION_DOWNLOAD) {
+                            return t->timings.last_write + std::chrono::seconds{5} < now;
+                        }
+                    } else if(t->state == FileClient::STATE_DISCONNECTING) {
+                        if(t->networking.disconnect_timeout.time_since_epoch().count() > 0)
+                            return t->networking.disconnect_timeout + std::chrono::seconds{5} < now;
+                        return t->timings.disconnecting + std::chrono::seconds{30} < now;
+                    }
+                    return false;
+                });
+                provider->transfers_.erase(std::remove_if(provider->transfers_.begin(), provider->transfers_.end(), [&](const auto& t) {
+                    return std::find(disconnected_clients.begin(), disconnected_clients.end(), t) != disconnected_clients.end();
+                }), provider->transfers_.end());
+            }
+
+            for(auto& client : disconnected_clients) {
+                switch(client->state) {
+                    case FileClient::STATE_AWAITING_KEY:
+                        logMessage(LOG_FT, "{} Received no key. Dropping client.", client->log_prefix());
+                        break;
+                    case FileClient::STATE_TRANSFERRING:
+                        logMessage(LOG_FT, "{} Networking timeout. Dropping client", client->log_prefix());
+                        if(auto callback{provider->callback_transfer_aborted}; callback)
+                            callback(client->transfer, { TransferError::TRANSFER_TIMEOUT, "" });
+                        break;
+                    case FileClient::STATE_DISCONNECTING:
+                        logMessage(LOG_FT, "{} Failed to flush connection. Dropping client", client->log_prefix());
+                        break;
+                    default:
+                        break;
+                }
+                {
+                    std::unique_lock slock{client->state_mutex};
+                    client->state = FileClient::STATE_DISCONNECTED;
+                    provider->finalize_file_io(client, slock);
+                    provider->finalize_client_ssl(client);
+                    provider->finalize_networking(client, slock);
+                }
+
+                debugMessage(LOG_FT, "{} Destroying transfer.", client->log_prefix());
+            }
+        }
+    }
+}
+
+void LocalFileTransfer::report_transfer_statistics(const std::shared_ptr<FileClient> &client) {
+    auto callback{this->callback_transfer_statistics};
+    if(!callback) return;
+
+    TransferStatistics stats{};
+
+    stats.network_bytes_send = client->statistics.network_bytes_send;
+    stats.network_bytes_received = client->statistics.network_bytes_received;
+    stats.file_bytes_transferred = client->statistics.file_bytes_transferred;
+
+    stats.delta_network_bytes_received = stats.network_bytes_received - std::exchange(client->statistics.last_network_bytes_received, stats.network_bytes_received);
+    stats.delta_network_bytes_send = stats.network_bytes_send - std::exchange(client->statistics.last_network_bytes_send, stats.network_bytes_send);
+
+    stats.delta_file_bytes_transferred = stats.file_bytes_transferred - std::exchange(client->statistics.last_file_bytes_transferred, stats.file_bytes_transferred);
+
+    stats.file_start_offset = client->transfer->file_offset;
+    stats.file_current_offset = client->statistics.file_bytes_transferred + client->transfer->file_offset;
+    stats.file_total_size = client->transfer->expected_file_size;
+
+    callback(client->transfer, stats);
+}

file/local_server/LocalFileTransferDisk.cpp

@@ -0,0 +1,475 @@
+//
+// Created by WolverinDEV on 04/05/2020.
+//
+
+#include <cassert>
+#include <event.h>
+#include <experimental/filesystem>
+#include <log/LogUtils.h>
+#include "./LocalFileProvider.h"
+#include "./duration_utils.h"
+#include "LocalFileProvider.h"
+
+using namespace ts::server::file;
+using namespace ts::server::file::transfer;
+namespace fs = std::experimental::filesystem;
+
+DiskIOStartResult LocalFileTransfer::start_disk_io() {
+    assert(this->disk_io.state == DiskIOLoopState::STOPPED);
+
+    this->disk_io.state = DiskIOLoopState::RUNNING;
+    this->disk_io.dispatch_thread = std::thread(&LocalFileTransfer::dispatch_loop_disk_io, this);
+    return DiskIOStartResult::SUCCESS;
+}
+
+void LocalFileTransfer::shutdown_disk_io() {
+    if(this->disk_io.state == DiskIOLoopState::STOPPED) return;
+
+    this->disk_io.state = DiskIOLoopState::STOPPING;
+    {
+        std::unique_lock qlock{this->disk_io.queue_lock};
+        this->disk_io.notify_work_awaiting.notify_all();
+        while(this->disk_io.queue_head)
+            this->disk_io.notify_client_processed.wait_for(qlock, std::chrono::seconds{10});
+
+        if(this->disk_io.queue_head) {
+            logWarning(0, "Failed to flush disk IO. Force aborting.");
+            this->disk_io.state = DiskIOLoopState::FORCE_STOPPING;
+            this->disk_io.notify_work_awaiting.notify_all();
+            this->disk_io.notify_client_processed.wait(qlock);
+        }
+    }
+
+    if(this->disk_io.dispatch_thread.joinable())
+        this->disk_io.dispatch_thread.join();
+
+    this->disk_io.state = DiskIOLoopState::STOPPED;
+}
+
+void LocalFileTransfer::dispatch_loop_disk_io(void *provider_ptr) {
+    auto provider = reinterpret_cast<LocalFileTransfer*>(provider_ptr);
+
+    std::shared_ptr<FileClient> client{};
+    while(true) {
+        {
+            std::unique_lock qlock{provider->disk_io.queue_lock};
+            if(client) {
+                client->file.currently_processing = false;
+                provider->disk_io.notify_client_processed.notify_all();
+                client = nullptr;
+            }
+
+            provider->disk_io.notify_work_awaiting.wait(qlock, [&]{ return provider->disk_io.state != DiskIOLoopState::RUNNING || provider->disk_io.queue_head != nullptr; });
+
+            if(provider->disk_io.queue_head) {
+                client = provider->disk_io.queue_head->shared_from_this();
+
+                provider->disk_io.queue_head = provider->disk_io.queue_head->file.next_client;
+                if(!provider->disk_io.queue_head)
+                    provider->disk_io.queue_tail = &provider->disk_io.queue_head;
+            }
+
+            if(provider->disk_io.state != DiskIOLoopState::RUNNING) {
+                if(provider->disk_io.state == DiskIOLoopState::STOPPING) {
+                    if(!client)
+                        break;
+                    /* break only if all clients have been flushed */
+                } else {
+                    /* force stopping without any flushing */
+                    auto fclient = &*client;
+                    while(fclient)
+                        fclient = std::exchange(fclient->file.next_client, nullptr);
+
+                    provider->disk_io.queue_head = nullptr;
+                    provider->disk_io.queue_tail = &provider->disk_io.queue_head;
+                    break;
+                }
+            }
+
+            if(!client)
+                continue;
+
+            client->file.currently_processing = true;
+            client->file.next_client = nullptr;
+        }
+
+        provider->execute_disk_io(client);
+    }
+    provider->disk_io.notify_client_processed.notify_all();
+}
+
+FileInitializeResult LocalFileTransfer::initialize_file_io(const std::shared_ptr<FileClient> &transfer) {
+    FileInitializeResult result{FileInitializeResult::SUCCESS};
+    assert(transfer->transfer);
+
+    std::shared_lock slock{transfer->state_mutex};
+    auto& file_data = transfer->file;
+    assert(!file_data.file_descriptor);
+    assert(!file_data.next_client);
+
+    {
+        unsigned int open_flags{0};
+        if(transfer->transfer->direction == Transfer::DIRECTION_DOWNLOAD) {
+            open_flags = O_RDONLY;
+
+            std::error_code fs_error{};
+            if(file_data.absolute_path.empty() || !fs::exists(file_data.absolute_path, fs_error)) {
+                result = FileInitializeResult::FILE_DOES_NOT_EXISTS;
+                goto error_exit;
+            } else if(fs_error) {
+                logWarning(LOG_FT, "{} Failed to check for file existence of {}: {}/{}", transfer->log_prefix(), file_data.absolute_path, fs_error.value(), fs_error.message());
+                result = FileInitializeResult::FILE_SYSTEM_ERROR;
+                goto error_exit;
+            }
+        } else if(transfer->transfer->direction == Transfer::DIRECTION_UPLOAD) {
+            open_flags = (unsigned) O_WRONLY | (unsigned) O_CREAT;
+            if(transfer->transfer->override_exiting)
+                open_flags |= (unsigned) O_TRUNC;
+        } else {
+            return FileInitializeResult::INVALID_TRANSFER_DIRECTION;
+        }
+
+        file_data.file_descriptor = open(file_data.absolute_path.c_str(), (int) open_flags, 0644);
+        if(file_data.file_descriptor <= 0) {
+            const auto errno_ = errno;
+            switch (errno_) {
+                case EACCES:
+                    result = FileInitializeResult::FILE_IS_NOT_ACCESSIBLE;
+                    break;
+                case EDQUOT:
+                    logWarning(LOG_FT, "{} Disk inode limit has been reached. Failed to start file transfer for file {}", transfer->log_prefix(), file_data.absolute_path);
+                    result = FileInitializeResult::FILE_SYSTEM_ERROR;
+                    break;
+                case EISDIR:
+                    result = FileInitializeResult::FILE_IS_A_DIRECTORY;
+                    break;
+                case EMFILE:
+                    result = FileInitializeResult::PROCESS_FILE_LIMIT_REACHED;
+                    break;
+                case ENFILE:
+                    result = FileInitializeResult::SYSTEM_FILE_LIMIT_REACHED;
+                    break;
+                case ETXTBSY:
+                    result = FileInitializeResult::FILE_IS_BUSY;
+                    break;
+                case EROFS:
+                    result = FileInitializeResult::DISK_IS_READ_ONLY;
+                    break;
+                default:
+                    logWarning(LOG_FT, "{} Failed to start file transfer for file {}: {}/{}", transfer->log_prefix(), file_data.absolute_path, errno_, strerror(errno_));
+                    result = FileInitializeResult::FILE_SYSTEM_ERROR;
+                    break;
+            }
+            goto error_exit;
+        }
+    }
+
+    this->file_system_.lock_file(transfer->file.absolute_path);
+    transfer->file.file_locked = true;
+
+    if(transfer->transfer->direction == Transfer::DIRECTION_UPLOAD) {
+        if(ftruncate(file_data.file_descriptor, transfer->transfer->expected_file_size) != 0) {
+            const auto errno_ = errno;
+            switch (errno_) {
+                case EACCES:
+                    logWarning(LOG_FT, "{} File {} got inaccessible on truncating, but not on opening.", transfer->log_prefix(), file_data.absolute_path);
+                    result = FileInitializeResult::FILE_IS_NOT_ACCESSIBLE;
+                    goto error_exit;
+
+                case EFBIG:
+                    result = FileInitializeResult::FILE_TOO_LARGE;
+                    goto error_exit;
+
+                case EIO:
+                    logWarning(LOG_FT, "{} A disk IO error occurred while resizing file {}.", transfer->log_prefix(), file_data.absolute_path);
+                    result = FileInitializeResult::FILE_IS_NOT_ACCESSIBLE;
+                    goto error_exit;
+
+                case EROFS:
+                    logWarning(LOG_FT, "{} Failed to resize file {} because disk is in read only mode.", transfer->log_prefix(), file_data.absolute_path);
+                    result = FileInitializeResult::FILE_IS_NOT_ACCESSIBLE;
+                    goto error_exit;
+                default:
+                    debugMessage(LOG_FT, "{} Failed to truncate file {}: {}/{}. Trying to upload file anyways.", transfer->log_prefix(), file_data.absolute_path, errno_, strerror(errno_));
+            }
+        }
+    } else if(transfer->transfer->direction == Transfer::DIRECTION_DOWNLOAD) {
+        auto file_size = lseek(file_data.file_descriptor, 0, SEEK_END);
+        if(file_size != transfer->transfer->expected_file_size) {
+            logWarning(LOG_FT, "{} Expected target file to be of size {}, but file is actually of size {}", transfer->log_prefix(), transfer->transfer->expected_file_size, file_size);
+            result = FileInitializeResult::FILE_SIZE_MISMATCH;
+            goto error_exit;
+        }
+    }
+    {
+        auto new_pos = lseek(file_data.file_descriptor, transfer->transfer->file_offset, SEEK_SET);
+        if(new_pos < 0) {
+            logWarning(LOG_FT, "{} Failed to seek to target file offset ({}): {}/{}", transfer->log_prefix(), transfer->transfer->file_offset, errno, strerror(errno));
+            result = FileInitializeResult::FILE_SEEK_FAILED;
+            goto error_exit;
+        } else if(new_pos != transfer->transfer->file_offset) {
+            logWarning(LOG_FT, "{} File rw offset mismatch after seek. Expected {} but received {}", transfer->log_prefix(), transfer->transfer->file_offset, new_pos);
+            result = FileInitializeResult::FILE_SEEK_FAILED;
+            goto error_exit;
+        }
+        debugMessage(LOG_FT, "{} Seek to file offset {}. New actual offset is {}", transfer->log_prefix(), transfer->transfer->file_offset, new_pos);
+    }
+
+    return FileInitializeResult::SUCCESS;
+    error_exit:
+    if(std::exchange(transfer->file.file_locked, false))
+        this->file_system_.unlock_file(transfer->file.absolute_path);
+
+    if(file_data.file_descriptor > 0)
+        ::close(file_data.file_descriptor);
+    file_data.file_descriptor = 0;
+
+    return result;
+}
+
+void LocalFileTransfer::finalize_file_io(const std::shared_ptr<FileClient> &transfer,
+                                         std::unique_lock<std::shared_mutex> &state_lock) {
+    assert(state_lock.owns_lock());
+
+    auto& file_data = transfer->file;
+
+    state_lock.unlock();
+    {
+        std::unique_lock dlock{this->disk_io.queue_lock};
+        while(true) {
+            if(file_data.currently_processing) {
+                this->disk_io.notify_client_processed.wait(dlock);
+                continue;
+            }
+
+            if(file_data.next_client) {
+                if(this->disk_io.queue_head == &*transfer) {
+                    this->disk_io.queue_head = file_data.next_client;
+                    if(!this->disk_io.queue_head)
+                        this->disk_io.queue_tail = &this->disk_io.queue_head;
+                } else {
+                    FileClient* head{this->disk_io.queue_head};
+                    while(head->file.next_client != &*transfer) {
+                        assert(head->file.next_client);
+                        head = head->file.next_client;
+                    }
+
+                    head->file.next_client = file_data.next_client;
+                    if(!file_data.next_client)
+                        this->disk_io.queue_tail = &head->file.next_client;
+
+                }
+                file_data.next_client = nullptr;
+            }
+
+            break;
+        }
+    }
+    state_lock.lock();
+
+    if(std::exchange(file_data.file_locked, false))
+        this->file_system_.unlock_file(file_data.absolute_path);
+
+    if(file_data.file_descriptor > 0)
+        ::close(file_data.file_descriptor);
+    file_data.file_descriptor = 0;
+}
+
+void LocalFileTransfer::enqueue_disk_io(const std::shared_ptr<FileClient> &client) {
+    if(!client->file.file_descriptor)
+        return;
+
+    if(!client->transfer)
+        return;
+
+    if(client->transfer->direction == Transfer::DIRECTION_DOWNLOAD) {
+        if(client->state != FileClient::STATE_TRANSFERRING)
+            return;
+
+        if(client->buffer.bytes > TRANSFER_MAX_CACHED_BYTES)
+            return;
+    } else if(client->transfer->direction == Transfer::DIRECTION_UPLOAD) {
+        /* we don't do this check because this might be a flush instruction, where the buffer is actually zero bytes filled */
+        /*
+        if(client->buffer.bytes == 0)
+            return;
+        */
+    }
+
+    std::lock_guard dlock{this->disk_io.queue_lock};
+    if(client->file.next_client || this->disk_io.queue_tail == &client->file.next_client)
+        return;
+
+    *this->disk_io.queue_tail = &*client;
+    this->disk_io.queue_tail = &client->file.next_client;
+
+    this->disk_io.notify_work_awaiting.notify_all();
+}
+
+void LocalFileTransfer::execute_disk_io(const std::shared_ptr<FileClient> &client) {
+    if(!client->transfer) return;
+
+    if(client->transfer->direction == Transfer::DIRECTION_UPLOAD) {
+        Buffer* buffer{nullptr};
+        size_t buffer_left_size{0};
+
+        while(true) {
+            {
+                std::lock_guard block{client->buffer.mutex};
+                buffer = client->buffer.buffer_head;
+                buffer_left_size = client->buffer.bytes;
+            }
+            if(!buffer) {
+                assert(buffer_left_size == 0);
+                break;
+            }
+
+            assert(buffer->offset < buffer->length);
+            auto written = ::write(client->file.file_descriptor, buffer->data + buffer->offset, buffer->length - buffer->offset);
+            if(written <= 0) {
+                if(written == 0) {
+                    /* EOF, how the hell is this event possible?! */
+                    auto offset_written = client->statistics.disk_bytes_write + client->transfer->file_offset;
+                    auto aoffset = lseek(client->file.file_descriptor, 0, SEEK_CUR);
+                    logError(LOG_FT, "{} Received unexpected file write EOF. EOF received at {} but expected {}. Actual file offset: {}. Closing transfer.",
+                            client->log_prefix(), offset_written, client->transfer->expected_file_size, aoffset);
+
+                    this->report_transfer_statistics(client);
+                    if(auto callback{client->handle->callback_transfer_aborted}; callback)
+                        callback(client->transfer, { TransferError::UNEXPECTED_DISK_EOF, strerror(errno) });
+
+                    {
+                        std::unique_lock slock{client->state_mutex};
+                        client->handle->disconnect_client(client, slock, true);
+                    }
+                } else {
+                    if(errno == EAGAIN) {
+                        //TODO: Timeout?
+                        this->enqueue_disk_io(client);
+                        break;
+                    }
+
+                    auto offset_written = client->statistics.disk_bytes_write + client->transfer->file_offset;
+                    auto aoffset = lseek(client->file.file_descriptor, 0, SEEK_CUR);
+                    logError(LOG_FT, "{} Received write to disk IO error. Write pointer is at {} of {}. Actual file offset: {}. Closing transfer.",
+                            client->log_prefix(), offset_written, client->transfer->expected_file_size, aoffset);
+
+                    this->report_transfer_statistics(client);
+                    if(auto callback{client->handle->callback_transfer_aborted}; callback)
+                        callback(client->transfer, { TransferError::DISK_IO_ERROR, strerror(errno) });
+
+                    {
+                        std::unique_lock slock{client->state_mutex};
+                        client->handle->disconnect_client(client, slock, true);
+                    }
+                }
+                return;
+            } else {
+                buffer->offset += written;
+                assert(buffer->offset <= buffer->length);
+                if(buffer->length == buffer->offset) {
+                    {
+                        std::lock_guard block{client->buffer.mutex};
+                        client->buffer.buffer_head = buffer->next;
+                        if(!buffer->next)
+                            client->buffer.buffer_tail = &client->buffer.buffer_head;
+
+                        assert(client->buffer.bytes >= written);
+                        client->buffer.bytes -= written;
+                        buffer_left_size = client->buffer.bytes;
+                        (void) buffer_left_size; /* trick my IDE here a bit */
+                    }
+
+                    free_buffer(buffer);
+                } else {
+                    std::lock_guard block{client->buffer.mutex};
+                    assert(client->buffer.bytes >= written);
+                    client->buffer.bytes -= written;
+                    buffer_left_size = client->buffer.bytes;
+                    (void) buffer_left_size; /* trick my IDE here a bit */
+                }
+
+                client->statistics.disk_bytes_write += written;
+            }
+        }
+
+        if(buffer_left_size > 0)
+            this->enqueue_disk_io(client);
+        else if(client->state == FileClient::STATE_DISCONNECTING) {
+            debugMessage(LOG_FT, "{} Disk IO has been flushed.", client->log_prefix());
+
+            std::unique_lock slock{client->state_mutex};
+            client->handle->disconnect_client(client->shared_from_this(), slock, false);
+        }
+
+        if(client->state == FileClient::STATE_TRANSFERRING && buffer_left_size < TRANSFER_MAX_CACHED_BYTES / 2) {
+            if(client->buffer.buffering_stopped)
+                logMessage(LOG_FT, "{} Starting network read, buffer is capable for reading again.", client->log_prefix());
+            client->add_network_read_event(false);
+        }
+    } else if(client->transfer->direction == Transfer::DIRECTION_DOWNLOAD) {
+        if(client->state == FileClient::STATE_DISCONNECTING) return;
+
+        while(true) {
+            constexpr auto buffer_capacity{4096};
+            char buffer[buffer_capacity];
+
+            auto read = ::read(client->file.file_descriptor, buffer, buffer_capacity);
+            if(read <= 0) {
+                if(read == 0) {
+                    /* EOF */
+                    auto offset_send = client->statistics.disk_bytes_read + client->transfer->file_offset;
+                    if(client->transfer->expected_file_size == offset_send) {
+                        debugMessage(LOG_FT, "{} Finished file reading. Flushing and disconnecting transfer. Reading took {} seconds.",
+                                client->log_prefix(), duration_to_string(std::chrono::system_clock::now() - client->timings.key_received));
+                    } else {
+                        auto aoffset = lseek(client->file.file_descriptor, 0, SEEK_CUR);
+                        logError(LOG_FT, "{} Received unexpected read EOF. EOF received at {} but expected {}. Actual file offset: {}. Disconnecting client.",
+                                client->log_prefix(), offset_send, client->transfer->expected_file_size, aoffset);
+
+                        this->report_transfer_statistics(client);
+                        if(auto callback{client->handle->callback_transfer_aborted}; callback)
+                            callback(client->transfer, { TransferError::UNEXPECTED_DISK_EOF, strerror(errno) });
+                    }
+
+                    {
+                        std::unique_lock slock{client->state_mutex};
+                        client->handle->disconnect_client(client, slock, true);
+                    }
+                } else {
+                    if(errno == EAGAIN) {
+                        this->enqueue_disk_io(client);
+                        return;
+                    }
+
+                    logWarning(LOG_FT, "{} Failed to read from file {} ({}/{}). Aborting transfer.", client->log_prefix(), client->file.absolute_path, errno, strerror(errno));
+
+                    this->report_transfer_statistics(client);
+                    if(auto callback{client->handle->callback_transfer_aborted}; callback)
+                        callback(client->transfer, { TransferError::DISK_IO_ERROR, strerror(errno) });
+
+                    {
+                        std::unique_lock slock{client->state_mutex};
+                        client->handle->disconnect_client(client, slock, true);
+                    }
+                }
+                return;
+            } else {
+                auto buffer_full = client->send_file_bytes(buffer, read);
+                client->statistics.disk_bytes_read += read;
+                client->statistics.file_bytes_transferred += read;
+
+                std::shared_lock slock{client->state_mutex};
+                if(buffer_full) {
+                    logMessage(LOG_FT, "{} Stopping buffering from disk. Buffer full ({}bytes)", client->log_prefix(), client->buffer.bytes);
+                    break;
+                }
+
+                /* we've stuff to write again, yeahr */
+                client->add_network_write_event(false);
+            }
+        }
+    } else {
+        logError(LOG_FT, "{} Disk IO scheduled, but transfer direction is unknown.", client->log_prefix());
+    }
+}

file/local_server/clnpath.cpp

@@ -0,0 +1,283 @@
+//
+// Created by WolverinDEV on 29/04/2020.
+//
+
+#include "clnpath.h"
+#include <cstring>
+
+#define MAX_PATH_ELEMENTS   128  /* Number of levels of directory */
+void ts_clnpath(char *path)
+{
+    char           *src;
+    char           *dst;
+    char            c;
+    int             slash = 0;
+
+    /* Convert multiple adjacent slashes to single slash */
+    src = dst = path;
+    while ((c = *dst++ = *src++) != '\0')
+    {
+        if (c == '/')
+        {
+            slash = 1;
+            while (*src == '/')
+                src++;
+        }
+    }
+
+    if (slash == 0)
+        return;
+
+    /* Remove "./" from "./xxx" but leave "./" alone. */
+    /* Remove "/." from "xxx/." but reduce "/." to "/". */
+    /* Reduce "xxx/./yyy" to "xxx/yyy" */
+    src = dst = (*path == '/') ? path + 1 : path;
+    while (src[0] == '.' && src[1] == '/' && src[2] != '\0')
+        src += 2;
+    while ((c = *dst++ = *src++) != '\0')
+    {
+        if (c == '/' && src[0] == '.' && (src[1] == '\0' || src[1] == '/'))
+        {
+            src++;
+            dst--;
+        }
+    }
+    if (path[0] == '/' && path[1] == '.' &&
+        (path[2] == '\0' || (path[2] == '/' && path[3] == '\0')))
+        path[1] = '\0';
+
+    /* Remove trailing slash, if any.  There is at most one! */
+    /* dst is pointing one beyond terminating null */
+    if ((dst -= 2) > path && *dst == '/')
+        *dst++ = '\0';
+}
+
+bool ts_strequal(const char* a, const char* b) {
+    return strcmp(a, b) == 0;
+}
+
+int ts_tokenise(char* ostring, const char* del, char** result, int max_tokens) {
+    int num_tokens{0};
+
+    char* token, *string, *tofree;
+    tofree = string = strdup(ostring);
+    while ((token = strsep(&string, del)) != nullptr) {
+        result[num_tokens++] = strdup(token);
+        if(num_tokens > max_tokens)
+            break;
+    }
+
+    free(tofree);
+    return num_tokens;
+}
+
+/*
+** clnpath2() is not part of the basic clnpath() function because it can
+** change the meaning of a path name if there are symbolic links on the
+** system.  For example, suppose /usr/tmp is a symbolic link to /var/tmp.
+** If the user supplies /usr/tmp/../abcdef as the directory name, clnpath
+** would transform that to /usr/abcdef, not to /var/abcdef which is what
+** the kernel would interpret it as.
+*/
+void ts_clnpath2(char *path)
+{
+    char *token[MAX_PATH_ELEMENTS], *otoken[MAX_PATH_ELEMENTS];
+    int   ntok, ontok;
+
+    ts_clnpath(path);
+
+    /* Reduce "<name>/.." to "/" */
+    ntok = ontok = ts_tokenise(path, "/", otoken, MAX_PATH_ELEMENTS);
+    memcpy(token, otoken, sizeof(char*) * ntok);
+
+    if (ntok > 1) {
+        for (int i = 0; i < ntok - 1; i++)
+        {
+            if (!ts_strequal(token[i], "..") && ts_strequal(token[i + 1], ".."))
+            {
+                if (*token[i] == '\0')
+                    continue;
+                while (i < ntok - 1)
+                {
+                    token[i] = token[i + 2];
+                    i++;
+                }
+                ntok -= 2;
+                i = -1;     /* Restart enclosing for loop */
+            }
+        }
+    }
+
+    /* Reassemble string */
+    char *dst = path;
+    if (ntok == 0)
+    {
+        *dst++ = '.';
+        *dst = '\0';
+    }
+    else
+    {
+        if (token[0][0] == '\0')
+        {
+            int   i;
+            for (i = 1; i < ntok && ts_strequal(token[i], ".."); i++)
+                ;
+            if (i > 1)
+            {
+                int j;
+                for (j = 1; i < ntok; i++)
+                    token[j++] = token[i];
+                ntok = j;
+            }
+        }
+        if (ntok == 1 && token[0][0] == '\0')
+        {
+            *dst++ = '/';
+            *dst = '\0';
+        }
+        else
+        {
+            for (int i = 0; i < ntok; i++)
+            {
+                char *src = token[i];
+                while ((*dst++ = *src++) != '\0')
+                    ;
+                *(dst - 1) = '/';
+            }
+            *(dst - 1) = '\0';
+        }
+    }
+
+    for(int i{0}; i < ontok; i++)
+        ::free(otoken[i]);
+}
+
+std::string clnpath(const std::string_view& data) {
+    std::string result{data};
+    ts_clnpath2(result.data());
+    auto index = result.find((char) 0);
+    if(index != std::string::npos)
+        result.resize(index);
+    return result;
+}
+
+#ifdef CLN_EXEC
+typedef struct p1_test_case
+{
+    const char *input;
+    const char *output;
+} p1_test_case;
+
+/* This stress tests the cleaning, concentrating on the boundaries. */
+static const p1_test_case p1_tests[] =
+        {
+                { "/",                                  "/",            },
+                { "//",                                 "/",            },
+                { "///",                                "/",            },
+                { "/.",                                 "/",            },
+                { "/./",                                "/",            },
+                { "/./.",                               "/",            },
+                { "/././.profile",                      "/.profile",    },
+                { "./",                                 ".",            },
+                { "./.",                                ".",            },
+                { "././",                               ".",            },
+                { "./././.profile",                     ".profile",     },
+                { "abc/.",                              "abc",          },
+                { "abc/./def",                          "abc/def",      },
+                { "./abc",                              "abc",          },
+
+                { "//abcd///./abcd////",                "/abcd/abcd",                   },
+                { "//abcd///././../defg///ddd//.",      "/abcd/../defg/ddd",            },
+                { "/abcd/./../././defg/./././ddd",      "/abcd/../defg/ddd",            },
+                { "//abcd//././../defg///ddd//.///",    "/abcd/../defg/ddd",            },
+
+                /* Most of these are minimal interest in phase 1 */
+                { "/usr/tmp/clnpath.c",                 "/usr/tmp/clnpath.c",           },
+                { "/usr/tmp/",                          "/usr/tmp",                     },
+                { "/bin/..",                            "/bin/..",                      },
+                { "bin/..",                             "bin/..",                       },
+                { "/bin/.",                             "/bin",                         },
+                { "sub/directory",                      "sub/directory",                },
+                { "sub/directory/file",                 "sub/directory/file",           },
+                { "/part1/part2/../.././../",           "/part1/part2/../../..",        },
+                { "/.././../usr//.//bin/./cc",          "/../../usr/bin/cc",            },
+        };
+
+static void p1_tester(const void *data)
+{
+    const p1_test_case *test = (const p1_test_case *)data;
+    char  buffer[256];
+
+    strcpy(buffer, test->input);
+    ts_clnpath(buffer);
+    if (strcmp(buffer, test->output) == 0)
+        printf("<<%s>> cleans to <<%s>>\n", test->input, buffer);
+    else
+    {
+        fprintf(stderr, "<<%s>> - unexpected output from clnpath()\n", test->input);
+        fprintf(stderr, "Wanted <<%s>>\n", test->output);
+        fprintf(stderr, "Actual <<%s>>\n", buffer);
+    }
+}
+
+typedef struct p2_test_case
+{
+    const char *input;
+    const char *output;
+} p2_test_case;
+
+static const p2_test_case p2_tests[] =
+{
+    { "/abcd/../defg/ddd",              "/defg/ddd"         },
+    { "/bin/..",                        "/"                 },
+    { "bin/..",                         "."                 },
+    { "/usr/bin/..",                    "/usr"              },
+    { "/usr/bin/../..",                 "/"                 },
+    { "usr/bin/../..",                  "."                 },
+    { "../part/of/../the/way",          "../part/the/way"   },
+    { "/../part/of/../the/way",         "/part/the/way"     },
+    { "part1/part2/../../part3",        "part3"             },
+    { "part1/part2/../../../part3",     "../part3"          },
+    { "/part1/part2/../../../part3",    "/part3"            },
+    { "/part1/part2/../../../",         "/"                 },
+    { "/../../usr/bin/cc",              "/usr/bin/cc"       },
+    { "../../usr/bin/cc",               "../../usr/bin/cc"  },
+    { "part1/./part2/../../part3",      "part3"             },
+    { "./part1/part2/../../../part3",   "../part3"          },
+    { "/part1/part2/.././../../part3",  "/part3"            },
+    { "/part1/part2/../.././../",       "/"                 },
+    { "/.././..//./usr///bin/cc/",      "/usr/bin/cc"       },
+    {nullptr, nullptr}
+};
+
+static void p2_tester(const void *data)
+{
+    auto test = (const p2_test_case *)data;
+    char  buffer[256];
+
+    strcpy(buffer, test->input);
+    ts_clnpath2(buffer);
+    if (strcmp(buffer, test->output) == 0)
+        printf("<<%s>> cleans to <<%s>>\n", test->input, buffer);
+    else
+    {
+        fprintf(stderr, "<<%s>> - unexpected output from clnpath2()\n", test->input);
+        fprintf(stderr, "Wanted <<%s>>\n", test->output);
+        fprintf(stderr, "Actual <<%s>>\n", buffer);
+    }
+}
+
+int main() {
+    for(const auto& test : p1_tests) {
+        if(!test.input) break;
+        p1_tester(&test);
+    }
+
+    printf("------------------------------\n");
+    for(const auto& test : p2_tests) {
+        if(!test.input) break;
+        p2_tester(&test);
+    }
+}
+
+#endif

file/local_server/clnpath.h

@@ -0,0 +1,6 @@
+#pragma once
+
+#include <string>
+#include <string_view>
+
+extern std::string clnpath(const std::string_view&);

file/local_server/duration_utils.h

@@ -0,0 +1,35 @@
+#pragma once
+
+template <typename Rep, typename Per>
+inline std::string duration_to_string(std::chrono::duration<Rep, Per> ms) {
+    std::string result{};
+
+    {
+        auto hours = std::chrono::duration_cast<std::chrono::hours>(ms);
+        if(hours.count())
+            result += std::to_string(hours.count()) + " hours ";
+        ms -= hours;
+    }
+    {
+        auto minutes = std::chrono::duration_cast<std::chrono::minutes>(ms);
+        if(minutes.count())
+            result += std::to_string(minutes.count()) + " minutes ";
+        ms -= minutes;
+    }
+    {
+        auto seconds = std::chrono::duration_cast<std::chrono::seconds>(ms);
+        if(seconds.count())
+            result += std::to_string(seconds.count()) + " seconds ";
+        ms -= seconds;
+    }
+    if(result.empty()) {
+        auto milliseconds = std::chrono::duration_cast<std::chrono::milliseconds>(ms);
+        if(milliseconds.count())
+            result = std::to_string(milliseconds.count()) + " milliseconds ";
+    }
+    if(result.empty()) {
+        auto microseconds = std::chrono::duration_cast<std::chrono::microseconds>(ms);
+        result = std::to_string(microseconds.count()) + " microseconds ";
+    }
+    return result.substr(0, result.length() - 1);
+}

file/test/main.cpp

@@ -0,0 +1,181 @@
+//
+// Created by WolverinDEV on 29/04/2020.
+//
+
+#include <files/FileServer.h>
+#include <log/LogUtils.h>
+
+#include <experimental/filesystem>
+#include <local_server/clnpath.h>
+#include <event2/thread.h>
+
+namespace fs = std::experimental::filesystem;
+
+using namespace ts::server;
+
+struct Nothing {};
+
+template <typename ErrorType, typename ResponseType>
+inline void print_fs_response(const std::string& message, const std::shared_ptr<file::ExecuteResponse<file::filesystem::DetailedError<ErrorType>, ResponseType>>& response) {
+    if(response->status == file::ExecuteStatus::ERROR)
+        logError(LOG_FT, "{}: {} => {}", message, (int) response->error().error_type, response->error().error_message);
+    else if(response->status == file::ExecuteStatus::SUCCESS)
+        logMessage(LOG_FT, "{}: success", message);
+    else
+        logWarning(LOG_FT, "Unknown response state ({})!", (int) response->status);
+}
+
+template <typename ErrorType, typename ResponseType>
+inline void print_ft_response(const std::string& message, const std::shared_ptr<file::ExecuteResponse<ErrorType, ResponseType>>& response) {
+    if(response->status == file::ExecuteStatus::ERROR)
+        logError(LOG_FT, "{}: {} => {}", message, (int) response->error().error_type, response->error().error_message);
+    else if(response->status == file::ExecuteStatus::SUCCESS)
+        logMessage(LOG_FT, "{}: success", message);
+    else
+        logWarning(LOG_FT, "Unknown response state ({})!", (int) response->status);
+}
+
+inline void print_query(const std::string& message, const file::filesystem::AbstractProvider::directory_query_response_t& response) {
+    if(response.status == file::ExecuteStatus::ERROR)
+        logError(LOG_FT, "{}: {} => {}", message, (int) response.error().error_type, response.error().error_message);
+    else if(response.status == file::ExecuteStatus::SUCCESS) {
+        const auto& entries = response.response();
+        logMessage(LOG_FT, "{}: Found {} entries", message, entries.size());
+        for(auto& entry : entries) {
+            if(entry.type == file::filesystem::DirectoryEntry::FILE)
+                logMessage(LOG_FT, "  - File {}", entry.name);
+            else if(entry.type == file::filesystem::DirectoryEntry::DIRECTORY)
+                logMessage(LOG_FT, "  - Directory {}", entry.name);
+            else
+                logMessage(LOG_FT, "  - Unknown {}", entry.name);
+            logMessage(LOG_FT, "    Write timestamp: {}", std::chrono::floor<std::chrono::seconds>(entry.modified_at.time_since_epoch()).count());
+            logMessage(LOG_FT, "    Size: {}", entry.size);
+        }
+    } else
+        logWarning(LOG_FT, "{}: Unknown response state ({})!", message, (int) response.status);
+}
+
+int main() {
+    evthread_use_pthreads();
+
+    auto log_config = std::make_shared<logger::LoggerConfig>();
+    log_config->terminalLevel = spdlog::level::trace;
+    logger::setup(log_config);
+
+    std::string error{};
+    if(!file::initialize(error)) {
+        logError(LOG_FT, "Failed to initialize file server: {}", error);
+        return 0;
+    }
+    logMessage(LOG_FT, "File server started");
+    auto instance = file::server();
+
+
+#if 0
+    auto& fs = instance->file_system();
+    {
+        auto response = fs.initialize_server(0);
+        response->wait();
+        print_fs_response("Server init result", response);
+        if(response->status != file::ExecuteStatus::SUCCESS)
+            return 0;
+    }
+
+
+    {
+        auto response = fs.create_channel_directory(0, 2, "/");
+        response->wait();
+        print_fs_response("Channel dir create A", response);
+    }
+
+
+    {
+        auto response = fs.create_channel_directory(0, 2, "/test-folder/");
+        response->wait();
+        print_fs_response("Channel dir create B", response);
+    }
+
+
+    {
+        auto response = fs.create_channel_directory(0, 2, "../test-folder/");
+        response->wait();
+        print_fs_response("Channel dir create C", response);
+    }
+
+
+    {
+        auto response = fs.create_channel_directory(0, 2, "./test-folder/../test-folder-2");
+        response->wait();
+        print_fs_response("Channel dir create D", response);
+    }
+
+    {
+        auto response = fs.query_channel_directory(0, 2, "/");
+        response->wait();
+        print_query("Channel query", *response);
+    }
+
+    {
+        auto response = fs.query_icon_directory(0);
+        response->wait();
+        print_query("Icons", *response);
+    }
+
+    {
+        auto response = fs.query_avatar_directory(0);
+        response->wait();
+        print_query("Avatars", *response);
+    }
+
+    {
+        auto response = fs.rename_channel_file(0, 2, "./test-folder/../test-folder-2", "./test-folder/../test-folder-3");
+        response->wait();
+        print_fs_response("Folder rename A", response);
+    }
+
+    {
+        auto response = fs.rename_channel_file(0, 2, "./test-folder/../test-folder-3", "./test-folder/../test-folder-2");
+        response->wait();
+        print_fs_response("Folder rename B", response);
+    }
+#endif
+
+#if 1
+    auto& ft = instance->file_transfer();
+
+    ft.callback_transfer_finished = [](const std::shared_ptr<file::transfer::Transfer>& transfer) {
+        logMessage(0, "Transfer finished");
+    };
+
+    ft.callback_transfer_started = [](const std::shared_ptr<file::transfer::Transfer>& transfer) {
+        logMessage(0, "Transfer started");
+    };
+
+    ft.callback_transfer_aborted = [](const std::shared_ptr<file::transfer::Transfer>& transfer, const file::transfer::TransferError& error) {
+        logMessage(0, "Transfer aborted ({}/{})", (int) error.error_type, error.error_message);
+    };
+
+    ft.callback_transfer_statistics = [](const std::shared_ptr<file::transfer::Transfer>& transfer, const file::transfer::TransferStatistics& stats) {
+        logMessage(0, "Transfer stats. New file bytes: {}, delta bytes send {}", stats.delta_file_bytes_transferred, stats.delta_network_bytes_send);
+    };
+
+    {
+        auto response = ft.initialize_channel_transfer(file::transfer::Transfer::DIRECTION_UPLOAD, 0, 2, {
+                "test2.txt",
+                false,
+                4,
+                120,
+                32
+        });
+        response->wait();
+        print_ft_response("Download test.txt", response);
+        if(response->succeeded())
+            logMessage(LOG_FT, "Download key: {}", std::string{response->response()->transfer_key, TRANSFER_KEY_LENGTH});
+    }
+#endif
+
+    std::this_thread::sleep_for(std::chrono::seconds{120});
+    //TODO: Test file locking
+    file::finalize();
+    return 0;
+}

file/test/upload-content.bin

@@ -0,0 +1,2 @@
+Hello World
+xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx

license/server/LicenseServer.cpp

@@ -134,16 +134,19 @@ void LicenseServer::handleEventWrite(int fd, short, void* ptrServer) {
     if(!client) return;
 
     buffer::RawBuffer* write_buffer{nullptr};
+    std::unique_lock write_lock(client->network.write_queue_lock);
     while(true) { //TODO: May add some kind of timeout?
-        std::lock_guard lock(client->network.write_queue_lock);
         write_buffer = TAILQ_FIRST(&client->network.write_queue);
         if(!write_buffer) return;
 
         auto writtenBytes = send(fd, &write_buffer->buffer[write_buffer->index], write_buffer->length - write_buffer->index, 0);
         if(writtenBytes <= 0) {
+            write_lock.unlock();
         	if(writtenBytes == -1 && errno == EAGAIN)
         		return;
 	        logError(LOG_LICENSE_CONTROLL, "Invalid write. Disconnecting remote client. Message: {}/{}", errno, strerror(errno));
+	        server->unregisterClient(client);
+	        return;
         } else {
             write_buffer->index += writtenBytes;
         }
@@ -187,7 +190,7 @@ void ConnectedClient::init() {
 	TAILQ_INIT(&network.write_queue);
 }
 
-void ConnectedClient::uninit() {
+void ConnectedClient::uninit(bool blocking) {
     {
         lock_guard queue_lock{this->network.write_queue_lock};
         ts::buffer::RawBuffer* buffer;
@@ -201,11 +204,21 @@ void ConnectedClient::uninit() {
         close(this->network.fileDescriptor);
 	    network.fileDescriptor = 0;
     }
-    if(this->network.readEvent) event_del(this->network.readEvent);
-    this->network.readEvent = nullptr;
 
-    if(this->network.writeEvent) event_del(this->network.writeEvent);
-    this->network.writeEvent = nullptr;
+    std::unique_lock elock{this->network.event_mutex};
+    auto read_event = std::exchange(this->network.readEvent, nullptr);
+    auto write_event = std::exchange(this->network.writeEvent, nullptr);
+    elock.unlock();
+
+    if(blocking) {
+        if(read_event) event_del_block(read_event);
+        if(write_event) event_del_block(write_event);
+    } else {
+        if(read_event) event_del_noblock(read_event);
+        if(write_event) event_del_noblock(write_event);
+    }
+    if(read_event) event_free(read_event);
+    if(write_event) event_free(write_event);
 }
 
 void LicenseServer::handleEventRead(int fd, short, void* ptrServer) {
@@ -221,12 +234,20 @@ void LicenseServer::handleEventRead(int fd, short, void* ptrServer) {
     if(read < 0){
         if(errno == EWOULDBLOCK) return;
         logError(LOG_LICENSE_CONTROLL, "Invalid read. Disconnecting remote client. Message: {}/{}", errno, strerror(errno));
-        event_del_noblock(client->network.readEvent);
+        {
+            std::lock_guard elock{client->network.event_mutex};
+            if(client->network.readEvent)
+                event_del_noblock(client->network.readEvent);
+        }
 	    server->closeConnection(client);
         return;
     } else if(read == 0) {
         logMessage(LOG_LICENSE_CONTROLL, "[CLIENT][" + client->address() + "] Received EOF for client. Removing client.");
-        event_del_noblock(client->network.readEvent);
+        {
+            std::lock_guard elock{client->network.event_mutex};
+            if(client->network.readEvent)
+                event_del_noblock(client->network.readEvent);
+        }
 	    server->closeConnection(client);
         return;
     }
@@ -312,7 +333,7 @@ void LicenseServer::unregisterClient(const std::shared_ptr<ConnectedClient> &cli
         std::lock_guard state_lock{client->protocol.state_lock};
         client->protocol.state = protocol::UNCONNECTED;
     }
-    client->uninit();
+    client->uninit(this_thread::get_id() != this->event_base_dispatch.get_id());
 }
 
 void LicenseServer::cleanup_clients() {

license/server/LicenseServer.h

@@ -31,6 +31,8 @@ namespace license {
 		    struct {
 			    sockaddr_in remoteAddr;
 			    int fileDescriptor = 0;
+
+			    std::mutex event_mutex{};
 			    event* readEvent = nullptr; /* protected via state_lock (check state and the use these variables) */
 			    event* writeEvent = nullptr;
 
@@ -57,7 +59,7 @@ namespace license {
 		    bool invalid_license = false;
 
             void init();
-            void uninit();
+            void uninit(bool /* blocking */);
             void sendPacket(const protocol::packet&);
 
 		    inline std::string address() { return inet_ntoa(network.remoteAddr.sin_addr); }

license/server/LicenseServerHandler.cpp

@@ -188,7 +188,7 @@ bool LicenseServer::handleServerValidation(shared_ptr<ConnectedClient> &client,
 		        logMessage(LOG_GENERAL, "[CLIENT][{}] Remote license has been deleted! Shutting down server!", client->address());
 	        } else {
 		        fill_info(response.mutable_license_info(), info, remote_license->data.licenceKey);
-		        auto is_invalid = !info->isValid();
+		        auto is_invalid = !info->isNotExpired();
 		        if(is_invalid) {
                     response.set_invalid_reason("license is invalid");
                     response.set_valid(false);

license/server/WebAPI.cpp

@@ -297,17 +297,21 @@ void WebStatistics::handleEventWrite(int file_descriptor, short, void* ptr_serve
 	}
 
 
-	std::lock_guard<std::recursive_mutex> lock(client->execute_lock);
+	std::unique_lock elock(client->execute_lock);
 	if(client->buffer_write.empty()) return;
 	auto& buffer = client->buffer_write.front();
 
 	auto written = send(file_descriptor, buffer.data(), buffer.length(), MSG_DONTWAIT | MSG_NOSIGNAL);
 	if(written < 0){
+		elock.unlock();
+
 		if(errno == EWOULDBLOCK) return;
 		logError(LOG_LICENSE_WEB, "[{}] Invalid write: {}/{}. Closing connection.", client->client_prefix(), errno, strerror(errno));
 		server->close_connection(client);
 		return;
 	} else if(written == 0) {
+		elock.unlock();
+
 		logError(LOG_LICENSE_WEB, "[{}] Invalid write (eof). Closing connection", client->client_prefix());
 		server->close_connection(client);
 		return;
@@ -334,17 +338,20 @@ void WebStatistics::close_connection(const std::shared_ptr<license::web::WebStat
 		else; //TODO Error handling?
 	}
 
-	std::lock_guard<std::recursive_mutex> lock(client->execute_lock);
-	if(client->event_read) {
-		event_del(client->event_read);
-		event_free(client->event_read);
-		client->event_read = nullptr;
+	std::unique_lock elock(client->execute_lock);
+	auto event_read = std::exchange(client->event_read, nullptr);
+	auto event_write = std::exchange(client->event_write, nullptr);
+	elock.unlock();
+	if(event_read) {
+		event_del(event_read);
+		event_free(event_read);
 	}
-	if(client->event_write) {
-		event_del(client->event_write);
-		event_free(client->event_write);
-		client->event_write = nullptr;
+	if(event_write) {
+		event_del(event_write);
+		event_free(event_write);
 	}
+
+	elock.lock();
 	if(client->file_descriptor > 0) {
 		if(shutdown(client->file_descriptor, SHUT_RDWR) < 0); //TODO error handling
 		if(close(client->file_descriptor) < 0); //TODO error handling
@@ -353,6 +360,7 @@ void WebStatistics::close_connection(const std::shared_ptr<license::web::WebStat
 
 	if(client->pipe_websocket)
 		client->pipe_websocket = nullptr;
+
 	if(client->pipe_ssl) {
 		client->pipe_ssl->finalize();
 		client->pipe_ssl = nullptr;

license/server/WebAPI.h

@@ -92,7 +92,6 @@ namespace license {
 				std::deque<std::shared_ptr<Client>> clients;
 
 				threads::ThreadPool scheduler{1, "WebStatistics #"};
-
 			protected:
 				static void handleEventAccept(int, short, void*);
 				static void handleEventRead(int, short, void*);

license/shared/include/license/license.h

@@ -348,7 +348,7 @@ namespace license {
 	    bool deleted{false};
 	    uint32_t upgrade_id{0};
 
-        inline bool isValid() { return (end.time_since_epoch().count() == 0 || std::chrono::system_clock::now() < this->end); }
+        inline bool isNotExpired() { return (end.time_since_epoch().count() == 0 || std::chrono::system_clock::now() < this->end); }
     };
 
     extern std::shared_ptr<License> readLocalLicence(const std::string &, std::string &);

license/shared/src/client.cpp

@@ -94,13 +94,21 @@ bool LicenseServerClient::start_connection(std::string &error) {
     this->network.event_base = event_base_new();
     this->network.event_read = event_new(this->network.event_base, this->network.file_descriptor, EV_READ | EV_PERSIST, [](int, short e, void* _this) {
         auto client = reinterpret_cast<LicenseServerClient*>(_this);
-        auto client_ref = client->shared_from_this(); /* We're not allowed to delete outself while hading data. This will lead to dangling pointers */
+        auto client_ref = client->weak_from_this().lock();
+        if(!client_ref) {
+            logCritical(0, "Network callback for expired client (E011)!");
+            return;
+        }
         client->callback_read(e);
         client_ref.reset();
     }, this);
     this->network.event_write = event_new(this->network.event_base, this->network.file_descriptor, EV_WRITE, [](int, short e, void* _this) {
         auto client = reinterpret_cast<LicenseServerClient*>(_this);
-        auto client_ref = client->shared_from_this(); /* We're not allowed to delete outself while hading data. This will lead to dangling pointers */
+        auto client_ref = client->weak_from_this().lock();
+        if(!client_ref) {
+            logCritical(0, "Network callback for expired client (E012)!");
+            return;
+        }
         client->callback_write(e);
         client_ref.reset();
     }, this);
@@ -169,7 +177,7 @@ void LicenseServerClient::cleanup_network_resources() {
         auto buffer = TAILQ_FIRST(&this->buffers.write);
         while(buffer) {
             auto next = TAILQ_NEXT(buffer, tail);
-            Buffer::free(next);
+            Buffer::free(buffer);
             buffer = next;
         }
         TAILQ_INIT(&this->buffers.write);

server/CMakeLists.txt

@@ -62,10 +62,11 @@ set(SERVER_SOURCE_FILES
         src/client/command_handler/client.cpp
         src/client/command_handler/server.cpp
         src/client/command_handler/misc.cpp
+        src/client/command_handler/bulk_parsers.cpp
 
         src/client/ConnectedClientNotifyHandler.cpp
         src/VirtualServerManager.cpp
-        src/server/file/FileServer.cpp
+        src/server/file/LocalFileServer.cpp
         src/channel/ServerChannel.cpp
         src/channel/ClientChannelView.cpp
         src/client/file/FileClient.cpp
@@ -82,7 +83,6 @@ set(SERVER_SOURCE_FILES
         src/client/query/QueryClientCommands.cpp
         src/client/query/QueryClientNotify.cpp
 
-
         src/manager/IpListManager.cpp
 
         src/ConnectionStatistics.cpp
@@ -243,7 +243,7 @@ target_link_libraries(PermMapHelper
 
 SET(CPACK_PACKAGE_VERSION_MAJOR "1")
 SET(CPACK_PACKAGE_VERSION_MINOR "4")
-SET(CPACK_PACKAGE_VERSION_PATCH "12")
+SET(CPACK_PACKAGE_VERSION_PATCH "14")
 if (BUILD_TYPE_NAME EQUAL OFF)
     SET(CPACK_PACKAGE_VERSION_DATA "beta")
 elseif (BUILD_TYPE_NAME STREQUAL "")
@@ -289,7 +289,13 @@ target_link_libraries(TeaSpeakServer
 )
 
 if (COMPILE_WEB_CLIENT)
-    target_link_libraries(TeaSpeakServer ${glib20_DIR}/lib/x86_64-linux-gnu/libffi.so.7 ${nice_DIR}/lib/libnice.so.10)
+    file(GLOB GLIB20_ARCHS ${glib20_DIR}/lib/*)
+    list(LENGTH GLIB20_ARCHS GLIB20_ARCHS_LENGTH)
+    if (NOT ${GLIB20_ARCHS_LENGTH} EQUAL 1)
+        message(FATAL_ERROR "Missing arch specific folder for glib2.0 in ${glib20_DIR}. Found ${GLIB20_ARCHS_LENGTH} directories, expected 1.")
+    endif ()
+    list(GET GLIB20_ARCHS 0 GLIB20_ARCH_DIR)
+    target_link_libraries(TeaSpeakServer ${GLIB20_ARCH_DIR}/libffi.so.7 ${nice_DIR}/lib/libnice.so.10)
 endif ()
 
 # include_directories(${LIBRARY_PATH}/boringssl/include/)

server/environment/geoloc

@@ -1 +0,0 @@
-../repro/env/geoloc/

server/environment/providers

@@ -1 +0,0 @@
-../../music/bin/providers/

server/environment/resources

@@ -1 +0,0 @@
-../repro/env/resources/

server/main.cpp

@@ -10,7 +10,7 @@
 #include "src/VirtualServer.h"
 #include "src/InstanceHandler.h"
 #include "src/server/QueryServer.h"
-#include "src/server/file/FileServer.h"
+#include "src/server/file/LocalFileServer.h"
 #include "src/terminal/CommandHandler.h"
 #include "src/client/InternalClient.h"
 #include "src/SignalHandler.h"
@@ -44,11 +44,10 @@ extern void testTomMath();
     #define DB_NAME "TeaData.sqlite"
 #endif
 
-#include <regex>
 #include <codecvt>
 #include "src/client/music/internal_provider/channel_replay/ChannelProvider.h"
 
-class CLIParser{
+class CLIParser {
     public:
         CLIParser (int &argc, char **argv){
             for (int i = 1; i < argc; i++)
@@ -264,7 +263,7 @@ int main(int argc, char** argv) {
     if(!cfgErrors.empty()){
         logErrorFmt(true, LOG_GENERAL, "Could not load configuration. Errors: (" + to_string(cfgErrors.size()) + ")");
         for(const auto& entry : cfgErrors)
-            logError(true, LOG_GENERAL, " - {}", entry);
+            logErrorFmt(true, LOG_GENERAL, " - {}", entry);
         logErrorFmt(true, LOG_GENERAL, "Stopping server...");
         goto stopApp;
     }

server/repro/build.sh

@@ -6,6 +6,39 @@ if [[ -z "${BUILD_PATH}" ]]; then
     exit 1
 fi
 
+rm buildVersion.txt
+cp env/buildVersion.txt .
+rm -r env
+mkdir env
+cd env
+[[ $? -ne 0 ]] && {
+  echo "Failed to create the env"
+  exit 1
+}
+cp -rf ../../../git-teaspeak/default_files/{certs,commanddocs,geoloc,resources,*.sh} .
+[[ $? -ne 0 ]] && {
+  echo "Failed to copy env"
+  exit 1
+}
+
+cp -rf ../../../music/bin/providers .
+[[ $? -ne 0 ]] && {
+  echo "Failed to copy providers"
+  exit 1
+}
+#
+cp ../../environment/TeaSpeakServer .
+[[ $? -ne 0 ]] && {
+  echo "Failed to copy server"
+  exit 1
+}
+cd ..
+mv buildVersion.txt env/buildVersion.txt
+[[ $? -ne 0 ]] && {
+  echo "Failed to move the build version back"
+  exit 1
+}
+
 ./generate_version.sh "${BUILD_PATH}" || {
     echo "Failed to generate version! ($?)"
     exit 1
@@ -24,4 +57,4 @@ fi
 ./deploy_build.sh "${BUILD_PATH}" || {
     echo "Failed to deploy package! ($?)"
     exit 1
-}
+}

server/repro/env/TeaSpeakServer

@@ -1 +0,0 @@
-../../environment/TeaSpeakServer

server/repro/env/certs

@@ -1 +0,0 @@
-../../../git-teaspeak/default_files/certs/

server/repro/env/commanddocs

@@ -1 +0,0 @@
-../../../git-teaspeak/default_files/commanddocs/

server/repro/env/geoloc

@@ -1 +0,0 @@
-../../../git-teaspeak/default_files/geoloc/

server/repro/env/install_music.sh

@@ -1 +0,0 @@
-../../../git-teaspeak/default_files/install_music.sh

server/repro/env/providers

@@ -1 +0,0 @@
-../../../music/bin/providers/

server/repro/env/resources

@@ -1 +0,0 @@
-../../../git-teaspeak/default_files/resources/

server/repro/env/tealoop.sh

@@ -1 +0,0 @@
-../../../git-teaspeak/default_files/tealoop.sh

server/repro/env/teastart.sh

@@ -1 +0,0 @@
-../../../git-teaspeak/default_files/teastart.sh

server/repro/env/teastart_autorestart.sh

@@ -1 +0,0 @@
-../../../git-teaspeak/default_files/teastart_autorestart.sh

server/repro/env/teastart_minimal.sh

@@ -1 +0,0 @@
-../../../git-teaspeak/default_files/teastart_minimal.sh

server/src/Configuration.cpp

@@ -63,6 +63,8 @@ bool config::server::clients::teaspeak;
 std::string config::server::clients::extra_welcome_message_teaspeak;
 config::server::clients::WelcomeMessageType config::server::clients::extra_welcome_message_type_teaspeak;
 
+bool config::server::clients::ignore_max_clone_permissions;
+
 uint16_t config::voice::default_voice_port;
 size_t config::voice::DefaultPuzzlePrecomputeSize;
 bool config::server::delete_missing_icon_permissions;
@@ -123,8 +125,12 @@ bool config::web::activated;
 std::deque<std::tuple<std::string, std::string, std::string>> config::web::ssl::certificates;
 uint16_t config::web::webrtc_port_max;
 uint16_t config::web::webrtc_port_min;
-deque<string> config::web::ice_servers;
+bool config::web::stun_enabled;
+std::string config::web::stun_host;
+uint16_t config::web::stun_port;
 bool config::web::enable_upnp;
+bool config::web::udp_enabled;
+bool config::web::tcp_enabled;
 
 size_t config::log::vs_size;
 std::string config::log::path;
@@ -476,7 +482,6 @@ vector<string> config::parseConfig(const std::string& path) {
                 goto license_parsing;
             }
 
-            /*
             if(!config::license->isValid()) {
                 if(config::license->data.type == license::LicenseType::INVALID) {
                     errors.emplace_back(strobf("Give license isn't valid!").string());
@@ -486,35 +491,15 @@ vector<string> config::parseConfig(const std::string& path) {
                 logErrorFmt(true, LOG_GENERAL, strobf("The given license isn't valid!").string());
                 logErrorFmt(true, LOG_GENERAL, strobf("Falling back to the default license.").string());
                 teaspeak_license = "none";
+
                 goto license_parsing;
             }
-             */
         }
 
         {
             auto bindings = create_bindings();
             read_bindings(config, bindings);
             build_comments(comments, bindings);
-
-            for(const auto& entry : config::web::ice_servers) {
-                auto dp = entry.find(':');
-                if(dp == string::npos) {
-                    errors.emplace_back("Invalid ice server entry! Missing port");
-                    continue;
-                }
-                auto host = entry.substr(0, dp);
-                auto port = entry.substr(dp + 1);
-                if(port.find_last_not_of("0123456789") != string::npos)  {
-                    errors.push_back("Invalid ice server entry! Invalid port (" + port + ")");
-                    continue;
-                }
-
-                try {
-                    stoi(port);
-                } catch(std::exception& ex) {
-                    errors.push_back("Invalid ice server entry! Invalid port (" + port + ")");
-                }
-            }
         }
 
         auto currentVersion = config::server::default_version();
@@ -1115,17 +1100,18 @@ std::deque<std::shared_ptr<EntryBinding>> config::create_bindings() {
         BIND_GROUP(query);
         {
             CREATE_BINDING("nl_char", 0);
-            BIND_STRING(config::query::newlineCharacter, "\r\n");
+            BIND_STRING(config::query::newlineCharacter, "\n");
             ADD_DESCRIPTION("Change the query newline character");
+            ADD_NOTE("NOTE: TS3 - Compatible bots may require \"\n\r\"");
         }
         {
             CREATE_BINDING("motd", 0);
-            BIND_STRING(config::query::motd, "TeaSpeak\r\nWelcome on the TeaSpeak ServerQuery interface.\r\n");
+            BIND_STRING(config::query::motd, "TeaSpeak\nWelcome on the TeaSpeak ServerQuery interface.\n");
             ADD_DESCRIPTION("The query welcome message");
 
             ADD_NOTE("If not like TeamSpeak then some applications may not recognize the Query");
             ADD_NOTE("Default TeamSpeak 3 MOTD:");
-            ADD_NOTE("TS3\r\nWelcome to the TeamSpeak 3 ServerQuery interface, type \"help\" for a list of commands and \"help <command>\" for information on a specific command.\r\n");
+            ADD_NOTE("TS3\n\rWelcome to the TeamSpeak 3 ServerQuery interface, type \"help\" for a list of commands and \"help <command>\" for information on a specific command.\n\r");
             ADD_NOTE("NOTE: Sometimes you have to append one \r\n more!");
         }
         {
@@ -1400,6 +1386,14 @@ std::deque<std::shared_ptr<EntryBinding>> config::create_bindings() {
                 ADD_DESCRIPTION(std::to_string(WelcomeMessageType::WELCOME_MESSAGE_TYPE_POKE) + " - Message, pokes the client with the message when he enters the server");
                 ADD_NOTE_RELOADABLE();
             }
+
+            {
+                CREATE_BINDING("ignore_max_clone_permissions", FLAG_RELOADABLE);
+                BIND_BOOL(config::server::clients::ignore_max_clone_permissions, false);
+                ADD_DESCRIPTION("Allows you to disable the permission checks for i_client_max_clones_uid, i_client_max_clones_ip and i_client_max_clones_hwid");
+                ADD_SENSITIVE();
+                ADD_NOTE_RELOADABLE();
+            }
         }
     }
     {
@@ -1490,9 +1484,34 @@ std::deque<std::shared_ptr<EntryBinding>> config::create_bindings() {
             ADD_NOTE("These ports must opened to use the voice bridge (Protocol: UDP)");
         }
         {
-            CREATE_BINDING("webrtc.ice", 0);
-            BIND_VECTOR(config::web::ice_servers, {"stun.l.google.com:19302"});
-            ADD_DESCRIPTION("A list of possible offered ice servers");
+            CREATE_BINDING("webrtc.stun.enabled", 0);
+            BIND_INTEGRAL(config::web::stun_enabled, false, false, true);
+            ADD_DESCRIPTION("Whatever to use a STUN server");
+            ADD_NOTE_RELOADABLE();
+        }
+        {
+            CREATE_BINDING("webrtc.stun.host", 0);
+            BIND_STRING(config::web::stun_host, "stun.l.google.com");
+            ADD_DESCRIPTION("Stun hostname");
+            ADD_NOTE_RELOADABLE();
+        }
+        {
+            CREATE_BINDING("webrtc.stun.port", 0);
+            BIND_INTEGRAL(config::web::stun_port, 19302, 1, 0xFFFF);
+            ADD_DESCRIPTION("Port of the stun server");
+            ADD_NOTE_RELOADABLE();
+        }
+        {
+            CREATE_BINDING("webrtc.udp", 0);
+            BIND_INTEGRAL(config::web::udp_enabled, true, false, true);
+            ADD_DESCRIPTION("Enable UDP for theweb client");
+            ADD_NOTE_RELOADABLE();
+        }
+        {
+            CREATE_BINDING("webrtc.tcp", 0);
+            BIND_INTEGRAL(config::web::tcp_enabled, true, false, true);
+            ADD_DESCRIPTION("Enable TCP for theweb client");
+            ADD_NOTE_RELOADABLE();
         }
     }
     {

server/src/Configuration.h

@@ -107,6 +107,8 @@ namespace ts::config {
             extern bool teaweb;
             extern std::string extra_welcome_message_teaweb;
             extern WelcomeMessageType extra_welcome_message_type_teaweb;
+
+            extern bool ignore_max_clone_permissions;
         }
 
         extern ssize_t max_virtual_server;
@@ -208,8 +210,15 @@ namespace ts::config {
 
         extern uint16_t webrtc_port_min;
         extern uint16_t webrtc_port_max;
-        extern std::deque<std::string> ice_servers;
+
         extern bool enable_upnp;
+
+        extern bool stun_enabled;
+        extern std::string stun_host;
+        extern uint16_t stun_port;
+
+        extern bool tcp_enabled;
+        extern bool udp_enabled;
     }
 
     namespace threads {

server/src/ConnectionStatistics.h

@@ -142,7 +142,7 @@ namespace ts {
 
                 void tick();
 
-                [[nodiscard]] inline const BandwidthEntry<uint32_t>& total_stats() const { return this->total_statistics; }
+                [[nodiscard]] inline const BandwidthEntry<uint64_t>& total_stats() const { return this->total_statistics; }
                 [[nodiscard]] inline BandwidthEntry<uint32_t> second_stats() const { return this->statistics_second; }
                 [[nodiscard]] BandwidthEntry<uint32_t> minute_stats() const;
 
@@ -151,7 +151,7 @@ namespace ts {
             private:
                 std::shared_ptr<ConnectionStatistics> handle;
 
-                BandwidthEntry<uint32_t> total_statistics{};
+                BandwidthEntry<uint64_t> total_statistics{};
 
                 BandwidthEntry<std::atomic<uint64_t>> statistics_second_current{};
                 BandwidthEntry<uint32_t> statistics_second{}; /* will be updated every second by the stats from the "current_second" */

server/src/DatabaseHelper.cpp

@@ -560,14 +560,16 @@ std::shared_ptr<Properties> DatabaseHelper::default_properties_client(std::share
     return properties;
 }
 
+std::mutex DatabaseHelper::database_id_mutex{};
+
 bool DatabaseHelper::assignDatabaseId(sql::SqlManager *sql, ServerId id, std::shared_ptr<DataClient> cl) {
     cl->loadDataForCurrentServer();
     if(cl->getClientDatabaseId() == 0){ //Client does not exist
-        ClientDbId cldbid = 0;
+        ClientDbId new_client_database_id{0};
         auto res = sql::command(sql, "SELECT `cldbid` FROM `clients` WHERE `serverId` = 0 AND `clientUid` = :cluid", variable{":cluid", cl->getUid()}).query([](ClientDbId* ptr, int length, char** values, char** names){
             *ptr = static_cast<ClientDbId>(stoll(values[0]));
             return 0;
-        }, &cldbid);
+        }, &new_client_database_id);
         auto pf = LOG_SQL_CMD;
         pf(res);
         if(!res) return false;
@@ -576,25 +578,26 @@ bool DatabaseHelper::assignDatabaseId(sql::SqlManager *sql, ServerId id, std::sh
                                          variable{":cluid", cl->getUid()}, variable{":name", cl->getDisplayName()},
                                          variable{":fconnect", duration_cast<seconds>(system_clock::now().time_since_epoch()).count()}, variable{":lconnect", 0},
                                          variable{":connections", 0});
-        if(cldbid == 0){ //Completly new user
-            res = sql::command(sql, "SELECT `cldbid` FROM `clients` WHERE `serverId` = 0 ORDER BY `cldbid` DESC LIMIT 1").query([](ClientDbId* ptr, int length, char** values, char** names){
-                *ptr = static_cast<ClientDbId>(stoll(values[0]));
-                return 0;
-            }, &cldbid);
-            pf(res);
+        if(new_client_database_id == 0) { /* we've a completely new user */
+            std::lock_guard db_id_lock{DatabaseHelper::database_id_mutex};
+            res = sql::command(sql, "SELECT `cldbid` FROM `clients` WHERE `serverId` = 0 ORDER BY `cldbid` DESC LIMIT 1").query([&](int length, std::string* values, std::string* names) {
+                assert(length == 1);
+                new_client_database_id = (ClientDbId) stoll(values[0]);
+            });
+            LOG_SQL_CMD(res);
             if(!res) return false;
 
-            cldbid += 1;
-            res = insertTemplate.command().values(variable{":serverId", 0}, variable{":cldbid", cldbid}).execute(); //Insert global
-            pf(res);
+            new_client_database_id += 1;
+            res = insertTemplate.command().values(variable{":serverId", 0}, variable{":cldbid", new_client_database_id}).execute(); //Insert global
+            LOG_SQL_CMD(res);
             if(!res) return false;
-            debugMessage(id, "Having new instance user on server {}. (Database ID: {} Name: {})", id, cldbid, cl->getDisplayName());
+            debugMessage(LOG_INSTANCE, "Registered a new client. Unique id: {}, First server: {}, Database ID: {}", cl->getUid(), id, new_client_database_id);
         } else {
-            debugMessage(id, "Having new server user on server {}. (Database ID: {} Name: {})", id, cldbid, cl->getDisplayName());
+            debugMessage(id, "Having new client, which is already known on this instance. Unique id: {}, First server: {}, Database ID: {}", cl->getUid(), id, new_client_database_id);
         }
 
         if(id != 0){ //Else already inserted
-            res = insertTemplate.command().values(variable{":serverId", id}, variable{":cldbid", cldbid}).execute();
+            res = insertTemplate.command().values(variable{":serverId", id}, variable{":cldbid", new_client_database_id}).execute();
             pf(res);
             if(!res) return false;
         }
@@ -602,7 +605,7 @@ bool DatabaseHelper::assignDatabaseId(sql::SqlManager *sql, ServerId id, std::sh
         return assignDatabaseId(sql, id, cl);
     }
 
-    logTrace(id, "Loaded client from database. Database id: {} Unique id: {}", cl->getClientDatabaseId(), cl->getUid());
+    logTrace(id, "Loaded client successfully from database. Database id: {} Unique id: {}", cl->getClientDatabaseId(), cl->getUid());
     return true;
 }
 

server/src/DatabaseHelper.h

@@ -78,6 +78,7 @@ namespace ts {
             public:
                 static std::shared_ptr<Properties> default_properties_client(std::shared_ptr<Properties> /* properties */, ClientType /* type */);
                 static bool assignDatabaseId(sql::SqlManager *, ServerId id, std::shared_ptr<DataClient>);
+                static std::mutex database_id_mutex;
 
                 explicit DatabaseHelper(sql::SqlManager*);
                 ~DatabaseHelper();

server/src/Group.cpp

@@ -6,7 +6,7 @@
 #include "VirtualServer.h"
 #include "src/client/ConnectedClient.h"
 #include "InstanceHandler.h"
-#include "src/server/file/FileServer.h"
+#include "src/server/file/LocalFileServer.h"
 
 using namespace std;
 using namespace std::chrono;
@@ -93,6 +93,7 @@ GroupManager::GroupManager(const shared_ptr<VirtualServer> &server, sql::SqlMana
 GroupManager::~GroupManager() {}
 
 bool GroupManager::loadGroupFormDatabase(GroupId id) {
+    std::lock_guard glock{this->group_lock};
     if(id == 0){
         this->groups.clear();
 
@@ -116,8 +117,13 @@ bool GroupManager::loadGroupFormDatabase(GroupId id) {
 
 std::vector<std::shared_ptr<Group>> GroupManager::availableGroups(bool root) {
     std::vector<std::shared_ptr<Group>> response;
-    for(const auto& group : this->groups)
-        response.push_back(group);
+
+    {
+        std::lock_guard glock{this->group_lock};
+        for(const auto& group : this->groups)
+            response.push_back(group);
+    }
+
     if(root && this->root){
         auto elm = this->root->availableGroups();
         for(const auto& e : elm)
@@ -128,9 +134,14 @@ std::vector<std::shared_ptr<Group>> GroupManager::availableGroups(bool root) {
 
 std::vector<std::shared_ptr<Group>> GroupManager::availableServerGroups(bool root){
     std::vector<std::shared_ptr<Group>> response;
-    for(const auto& group : this->groups)
-        if(group->target() == GroupTarget::GROUPTARGET_SERVER)
-            response.push_back(group);
+
+    {
+        std::lock_guard glock{this->group_lock};
+        for(const auto& group : this->groups)
+            if(group->target() == GroupTarget::GROUPTARGET_SERVER)
+                response.push_back(group);
+    }
+
     if(root && this->root){
         auto elm = this->root->availableServerGroups();
         for(const auto& e : elm)
@@ -141,9 +152,12 @@ std::vector<std::shared_ptr<Group>> GroupManager::availableServerGroups(bool roo
 
 std::vector<std::shared_ptr<Group>> GroupManager::availableChannelGroups(bool root) {
     std::vector<std::shared_ptr<Group>> response;
-    for(const auto& group : this->groups)
-        if(group->target() == GroupTarget::GROUPTARGET_CHANNEL)
-            response.push_back(group);
+    {
+        std::lock_guard glock{this->group_lock};
+        for(const auto& group : this->groups)
+            if(group->target() == GroupTarget::GROUPTARGET_CHANNEL)
+                response.push_back(group);
+    }
     if(root && this->root){
         auto elm = this->root->availableChannelGroups(true);
         for(const auto& e : elm)
@@ -167,8 +181,7 @@ int GroupManager::insertGroupFromDb(int count, char **values, char **column) {
             groupId = (GroupType) stoll(values[index]);
         else if(strcmp(column[index], "displayName") == 0)
             targetName = values[index];
-        else if(strcmp(column[index], "serverId") == 0);
-        else cerr << "Invalid group table row " << column[index] << endl;
+        //else cerr << "Invalid group table row " << column[index] << endl;
     }
 
     if((size_t) groupId == 0 || (size_t) target == 0xff || (size_t) type == 0xff || targetName.empty()) {
@@ -237,6 +250,7 @@ void GroupManager::handleChannelDeleted(const ChannelId& channel_id) {
 }
 
 bool GroupManager::isLocalGroup(std::shared_ptr<Group> gr) {
+    std::lock_guard glock{this->group_lock};
     return std::find(this->groups.begin(), this->groups.end(), gr) != this->groups.end();
 }
 
@@ -252,9 +266,12 @@ std::shared_ptr<Group> GroupManager::defaultGroup(GroupTarget type, bool enforce
     auto group = this->findGroupLocal(id);
     if(group || enforce_property) return group;
 
-    for(auto elm : this->groups)
-        if(elm->target() == type)
-            return elm;
+    {
+        std::lock_guard glock{this->group_lock};
+        for(auto elm : this->groups)
+            if(elm->target() == type)
+                return elm;
+    }
 
     return nullptr; //Worst case!
 }
@@ -266,6 +283,7 @@ std::shared_ptr<Group> GroupManager::findGroup(GroupId groupId) {
 }
 
 std::shared_ptr<Group> GroupManager::findGroupLocal(GroupId groupId) {
+    std::lock_guard glock{this->group_lock};
     for(const auto& elm : this->groups)
         if(elm->groupId() == groupId) return elm;
     return nullptr;
@@ -273,8 +291,11 @@ std::shared_ptr<Group> GroupManager::findGroupLocal(GroupId groupId) {
 
 std::vector<std::shared_ptr<Group>> GroupManager::findGroup(GroupTarget target, std::string name) {
     vector<shared_ptr<Group>> res;
-    for(const auto &elm : this->groups)
-        if(elm->name() == name && elm->target() == target) res.push_back(elm);
+    {
+        std::lock_guard glock{this->group_lock};
+        for(const auto &elm : this->groups)
+            if(elm->name() == name && elm->target() == target) res.push_back(elm);
+    }
     if(this->root) {
         auto r = root->findGroup(target, name);
         for(const auto &e : r) res.push_back(e);
@@ -305,6 +326,8 @@ std::shared_ptr<Group> GroupManager::createGroup(GroupTarget target, GroupType t
     std::shared_ptr<Group> group = std::make_shared<Group>(this, target, type, groupId);
     group->properties()[property::GROUP_NAME] = name;
     group->setPermissionManager(serverInstance->databaseHelper()->loadGroupPermissions(this->server.lock(), group->groupId()));
+
+    std::lock_guard glock{this->group_lock};
     this->groups.push_back(group);
     return group;
 }
@@ -394,7 +417,10 @@ bool GroupManager::deleteGroup(std::shared_ptr<Group> group) {
         return false;
     }
 
-    this->groups.erase(std::find(this->groups.begin(), this->groups.end(), group));
+    {
+        std::lock_guard glock{this->group_lock};
+        this->groups.erase(std::find(this->groups.begin(), this->groups.end(), group));
+    }
 
     /* erase the group out of our cache */
     {

server/src/Group.h

@@ -224,7 +224,10 @@ namespace ts {
             ServerId getServerId();
 
             sql::SqlManager* sql;
+
+            std::mutex group_lock{};
             std::vector<std::shared_ptr<Group>> groups;
+
             threads::Mutex cacheLock;
             std::vector<std::shared_ptr<CachedClient>> cachedClients;
 

server/src/InstanceHandler.cpp

@@ -9,7 +9,7 @@
 #include "InstanceHandler.h"
 #include "src/client/InternalClient.h"
 #include "src/server/QueryServer.h"
-#include "src/server/file/FileServer.h"
+#include "src/server/file/LocalFileServer.h"
 #include "SignalHandler.h"
 #include "src/manager/PermissionNameMapper.h"
 #include <ThreadPool/Timer.h>
@@ -281,19 +281,19 @@ bool InstanceHandler::startInstance() {
     }
 
     this->loadWebCertificate();
-    fileServer = new ts::server::FileServer();
+    fileServer = new ts::server::LocalFileServer();
     {
         auto bindings_string = this->properties()[property::SERVERINSTANCE_FILETRANSFER_HOST].as<string>();
         auto port = this->properties()[property::SERVERINSTANCE_FILETRANSFER_PORT].as<uint16_t>();
         auto ft_bindings = net::resolve_bindings(bindings_string, port);
-        deque<shared_ptr<FileServer::Binding>> bindings;
+        deque<shared_ptr<LocalFileServer::Binding>> bindings;
 
         for(auto& binding : ft_bindings) {
             if(!get<2>(binding).empty()) {
                 logError(LOG_FT, "Failed to resolve binding for {}: {}", get<0>(binding), get<2>(binding));
                 continue;
             }
-            auto entry = make_shared<FileServer::Binding>();
+            auto entry = make_shared<LocalFileServer::Binding>();
             memcpy(&entry->address, &get<1>(binding), sizeof(sockaddr_storage));
 
             entry->file_descriptor = -1;

server/src/InstanceHandler.h

@@ -42,7 +42,7 @@ namespace ts {
                 std::shared_mutex& getChannelTreeLock() { return this->default_tree_lock; }
 
                 VirtualServerManager* getVoiceServerManager(){ return this->voiceServerManager; }
-                FileServer* getFileServer(){ return fileServer; }
+                LocalFileServer* getFileServer(){ return fileServer; }
                 QueryServer* getQueryServer(){ return queryServer; }
                 DatabaseHelper* databaseHelper(){ return this->dbHelper; }
                 BanManager* banManager(){ return this->banMgr; }
@@ -110,7 +110,7 @@ namespace ts {
                 std::chrono::system_clock::time_point memcleanTimestamp;
                 SqlDataManager* sql;
 
-                FileServer* fileServer = nullptr;
+                LocalFileServer* fileServer = nullptr;
                 QueryServer* queryServer = nullptr;
                 VirtualServerManager* voiceServerManager = nullptr;
                 DatabaseHelper* dbHelper = nullptr;

server/src/InstanceHandlerSetup.cpp

@@ -6,7 +6,7 @@
 #include "InstanceHandler.h"
 #include "src/client/InternalClient.h"
 #include "src/server/QueryServer.h"
-#include "src/server/file/FileServer.h"
+#include "src/server/file/LocalFileServer.h"
 
 using namespace std;
 using namespace std::chrono;

server/src/ShutdownHelper.cpp

@@ -31,6 +31,7 @@ void ts::server::shutdownInstance(const std::string& message) {
         force_kill.detach();
 
         exit(2);
+        //kill(0, SIGKILL);
     });
     threads::name(hangup_controller, "stop controller");
     hangup_controller.detach();

server/src/VirtualServer.cpp

@@ -18,7 +18,7 @@
 #include "./client/query/QueryClient.h"
 #include "music/MusicBotManager.h"
 #include "server/VoiceServer.h"
-#include "server/file/FileServer.h"
+#include "src/server/file/LocalFileServer.h"
 #include "server/QueryServer.h"
 #include "InstanceHandler.h"
 #include "Configuration.h"
@@ -57,7 +57,7 @@ bool VirtualServer::initialize(bool test_properties) {
             this->_voice_encryption_mode = prop.as<int>();
             return;
         }
-        std::string sql;
+        std::string sql{};
         if(prop.type() == property::VIRTUALSERVER_HOST)
             sql = "UPDATE `servers` SET `host` = :value WHERE `serverId` = :sid";
         else if(prop.type() == property::VIRTUALSERVER_PORT)
@@ -129,6 +129,7 @@ bool VirtualServer::initialize(bool test_properties) {
         return false;
     }
 
+    channelTree->deleteSemiPermanentChannels();
     if(channelTree->channel_count() == 0){
         logMessage(this->serverId, "Creating new channel tree (Copy from server 0)");
         LOG_SQL_CMD(sql::command(this->getSql(), "INSERT INTO `channels` (`serverId`, `channelId`, `type`, `parentId`) SELECT :serverId AS `serverId`, `channelId`, `type`, `parentId` FROM `channels` WHERE `serverId` = 0", variable{":serverId", this->serverId}).execute());
@@ -141,7 +142,7 @@ bool VirtualServer::initialize(bool test_properties) {
         channelTree->loadChannelsFromDatabase();
         if(channelTree->channel_count() == 0){
             logCritical(this->serverId, "Failed to setup channel tree!");
-            return 0;
+            return false;
         }
         if(!channelTree->getDefaultChannel()) {
             logError(this->serverId, "Missing default channel! Using first one!");
@@ -726,9 +727,9 @@ bool VirtualServer::notifyServerEdited(std::shared_ptr<ConnectedClient> invoker,
 }
 
 bool VirtualServer::notifyClientPropertyUpdates(std::shared_ptr<ConnectedClient> client, const deque<const property::PropertyDescription*>& keys, bool selfNotify) {
-    if(keys.empty()) return false;
+    if(keys.empty() || !client) return false;
     this->forEachClient([&](const shared_ptr<ConnectedClient>& cl) {
-        shared_lock client_channel_lock(client->channel_lock);
+        shared_lock client_channel_lock(cl->channel_lock);
         if(cl->isClientVisible(client, false) || (cl == client && selfNotify))
             cl->notifyClientUpdated(client, keys, false);
     });
@@ -1216,4 +1217,41 @@ void VirtualServer::send_text_message(const std::shared_ptr<BasicChannel> &chann
         auto conversation = conversations->get_or_create(channel->channelId());
         conversation->register_message(sender->getClientDatabaseId(), sender->getUid(), sender->getDisplayName(), now, message);
     }
+}
+
+void VirtualServer::update_channel_from_permissions(const std::shared_ptr<BasicChannel> &channel, const std::shared_ptr<ConnectedClient>& issuer) {
+    bool require_view_update;
+    auto property_updates = channel->update_properties_from_permissions(require_view_update);
+
+    if(!property_updates.empty()) {
+        this->forEachClient([&](const std::shared_ptr<ConnectedClient>& cl) {
+            shared_lock client_channel_lock(cl->channel_lock);
+            cl->notifyChannelEdited(channel, property_updates, issuer, false);
+        });
+    }
+
+    if(require_view_update) {
+        auto l_source = this->channelTree->findLinkedChannel(channel->channelId());
+        this->forEachClient([&](const shared_ptr<ConnectedClient>& cl) {
+            /* server tree read lock still active */
+            auto l_target = !cl->currentChannel ? nullptr : cl->server->channelTree->findLinkedChannel(cl->currentChannel->channelId());
+            sassert(l_source);
+            if(cl->currentChannel) sassert(l_target);
+
+            {
+                unique_lock client_channel_lock(cl->channel_lock);
+
+                deque<ChannelId> deleted;
+                for(const auto& [flag_visible, channel] : cl->channels->update_channel(l_source, l_target)) {
+                    if(flag_visible) {
+                        cl->notifyChannelShow(channel->channel(), channel->previous_channel);
+                    } else {
+                        deleted.push_back(channel->channelId());
+                    }
+                }
+                if(!deleted.empty())
+                    cl->notifyChannelHide(deleted, false);
+            }
+        });
+    }
 }

server/src/VirtualServer.h

@@ -58,7 +58,7 @@ namespace ts {
         class InstanceHandler;
         class VoiceServer;
         class QueryServer;
-        class FileServer;
+        class LocalFileServer;
         class SpeakingClient;
 
         class WebControlServer;
@@ -278,6 +278,9 @@ namespace ts {
 
                 inline int voice_encryption_mode() { return this->_voice_encryption_mode; }
                 inline std::shared_ptr<conversation::ConversationManager> conversation_manager() { return this->_conversation_manager; }
+
+
+                void update_channel_from_permissions(const std::shared_ptr<BasicChannel>& /* channel */, const std::shared_ptr<ConnectedClient>& /* issuer */);
             protected:
                 bool registerClient(std::shared_ptr<ConnectedClient>);
                 bool unregisterClient(std::shared_ptr<ConnectedClient>, std::string, std::unique_lock<std::shared_mutex>& channel_tree_lock);

server/src/VirtualServerManager.cpp

@@ -4,7 +4,7 @@
 #include "src/server/VoiceServer.h"
 #include "src/client/query/QueryClient.h"
 #include "InstanceHandler.h"
-#include "src/server/file/FileServer.h"
+#include "src/server/file/LocalFileServer.h"
 #include "src/client/ConnectedClient.h"
 #include <ThreadPool/ThreadHelper.h>
 
@@ -201,9 +201,11 @@ shared_ptr<VirtualServer> VirtualServerManager::findServerByPort(uint16_t port)
     return nullptr;
 }
 
-uint16_t VirtualServerManager::next_available_port() {
+uint16_t VirtualServerManager::next_available_port(const std::string& host_string) {
     auto instances = this->serverInstances();
-    deque<uint16_t> unallowed_ports;
+    std::vector<uint16_t> unallowed_ports{};
+    unallowed_ports.reserve(instances.size());
+
     for(const auto& instance : instances) {
         unallowed_ports.push_back(instance->properties()[property::VIRTUALSERVER_PORT].as<uint16_t>());
 
@@ -214,18 +216,39 @@ uint16_t VirtualServerManager::next_available_port() {
             }
         }
     }
+    auto bindings = net::resolve_bindings(host_string, 0);
 
     uint16_t port = config::voice::default_voice_port;
     while(true) {
-        if(port < 1024) goto c;
+        if(port < 1024) goto next_port;
 
         for(auto& p : unallowed_ports) {
             if(p == port)
-                goto c;
+                goto next_port;
+        }
+
+        for(auto& binding : bindings) {
+            if(!std::get<2>(binding).empty()) continue; /* error on that */
+            auto& baddress = std::get<1>(binding);
+            auto& raw_port = baddress.ss_family == AF_INET ? ((sockaddr_in*) &baddress)->sin_port : ((sockaddr_in6*) &baddress)->sin6_port;
+            raw_port = htons(port);
+
+            switch (net::address_available(baddress, net::binding_type::TCP)) {
+                case net::binding_result::ADDRESS_USED:
+                    goto next_port;
+                default:
+                    break; /* if we've an internal error we ignore it */
+            }
+            switch (net::address_available(baddress, net::binding_type::UDP)) {
+                case net::binding_result::ADDRESS_USED:
+                    goto next_port;
+                default:
+                    break; /* if we've an internal error we ignore it */
+            }
         }
         break;
 
-        c:
+        next_port:
         port++;
     }
     return port;
@@ -317,6 +340,7 @@ shared_ptr<VirtualServer> VirtualServerManager::create_server(std::string hosts,
     if(!sid_success)
         return nullptr;
 
+    this->delete_server_in_db(serverId); /* just to ensure */
     sql::command(this->handle->getSql(), "INSERT INTO `servers` (`serverId`, `host`, `port`) VALUES (:sid, :host, :port)", variable{":sid", serverId}, variable{":host", hosts}, variable{":port", port}).executeLater().waitAndGetLater(LOG_SQL_CMD, {1, "future failed"});
 
     auto prop_copy = sql::command(this->handle->getSql(), "INSERT INTO `properties` (`serverId`, `type`, `id`, `key`, `value`) SELECT :target_sid AS `serverId`, `type`, `id`, `key`, `value` FROM `properties` WHERE `type` = :type AND `id` = 0 AND `serverId` = 0;",
@@ -453,6 +477,7 @@ if(!result) { \
     execute_delete("DELETE FROM `channels` WHERE `serverId` = :sid");
     execute_delete("DELETE FROM `bannedClients` WHERE `serverId` = :sid");
     execute_delete("DELETE FROM `ban_trigger` WHERE `server_id` = :sid");
+    execute_delete("DELETE FROM `groups` WHERE `serverId` = :sid");
     execute_delete("DELETE FROM `assignedGroups` WHERE `serverId` = :sid");
     execute_delete("DELETE FROM `servers` WHERE `serverId` = :sid");
 

server/src/VirtualServerManager.h

@@ -38,7 +38,7 @@ namespace ts::server {
 
             std::shared_ptr<VirtualServer> findServerById(ServerId);
             std::shared_ptr<VirtualServer> findServerByPort(uint16_t);
-            uint16_t next_available_port();
+            uint16_t next_available_port(const std::string& /* host string */);
             ServerId next_available_server_id(bool& /* success */);
 
             std::deque<std::shared_ptr<VirtualServer>> serverInstances(){

server/src/channel/ServerChannel.cpp

@@ -5,7 +5,7 @@
 #include "misc/rnd.h"
 #include "src/VirtualServer.h"
 #include "src/client/ConnectedClient.h"
-#include "src/server/file/FileServer.h"
+#include "src/server/file/LocalFileServer.h"
 #include "src/InstanceHandler.h"
 #include "../manager/ConversationManager.h"
 
@@ -102,7 +102,7 @@ std::shared_ptr<BasicChannel> ServerChannelTree::createChannel(ChannelId parentI
     channel->properties()[property::CHANNEL_LAST_LEFT] = chrono::duration_cast<chrono::milliseconds>(chrono::system_clock::now().time_since_epoch()).count();
 
 
-    auto result = sql::command(this->sql, "INSERT INTO `channels` (`serverId`, `channelId`, `type`, `parentId`) VALUES(:sid, :chid, :type, :parent);",  variable{":sid", this->getServerId()}, variable{":chid", channel->channelId()}, variable{":type", channel->channelType()}, variable{":parent", channel->parent() ? channel->parent()->channelId() : 0}).execute();
+    auto result = sql::command(this->sql, "INSERT INTO `channels` (`serverId`, `channelId`, `parentId`) VALUES(:sid, :chid, :parent);",  variable{":sid", this->getServerId()}, variable{":chid", channel->channelId()}, variable{":parent", channel->parent() ? channel->parent()->channelId() : 0}).execute();
     auto pf = LOG_SQL_CMD;
     pf(result);
 
@@ -493,7 +493,7 @@ bool ServerChannelTree::validateChannelIcons() {
 }
 
 void ServerChannelTree::loadChannelsFromDatabase() {
-    auto res = sql::command(this->sql, "SELECT * FROM `channels` WHERE `serverId` = :sid", variable{":sid", this->getServerId()}).query(&ServerChannelTree::loadChannelFromData, this);
+    auto res = sql::command(this->sql, "SELECT  `channelId`, `parentId` FROM `channels` WHERE `serverId` = :sid", variable{":sid", this->getServerId()}).query(&ServerChannelTree::loadChannelFromData, this);
     (LOG_SQL_CMD)(res);
     if(!res){
         logError(this->getServerId(), "Could not load channel tree from database");
@@ -512,7 +512,6 @@ void ServerChannelTree::loadChannelsFromDatabase() {
 int ServerChannelTree::loadChannelFromData(int argc, char **data, char **column) {
     ChannelId channelId = 0;
     ChannelId parentId = 0;
-    auto type = static_cast<ChannelType::ChannelType>(0xFF);
 
 
     int index = 0;
@@ -520,8 +519,6 @@ int ServerChannelTree::loadChannelFromData(int argc, char **data, char **column)
         for(index = 0; index < argc; index++){
             if(strcmp(column[index], "channelId") == 0) channelId = static_cast<ChannelId>(stoll(data[index]));
             else if(strcmp(column[index], "parentId") == 0) parentId = static_cast<ChannelId>(stoll(data[index]));
-            else if(strcmp(column[index], "type") == 0) type = static_cast<ChannelType::ChannelType>(stoll(data[index]));
-            else if(strcmp(column[index], "serverId") == 0) {}
             else logError(this->getServerId(), "ServerChannelTree::loadChannelFromData called with invalid column from sql \"{}\"", column[index]);
         }
     } catch (std::exception& ex) {

server/src/client/ConnectedClient.cpp

@@ -11,7 +11,7 @@
 #include "src/VirtualServer.h"
 #include "voice/VoiceClient.h"
 #include "../server/VoiceServer.h"
-#include "../server/file/FileServer.h"
+#include "src/server/file/LocalFileServer.h"
 #include "../InstanceHandler.h"
 #include "ConnectedClient.h"
 
@@ -182,14 +182,17 @@ void ConnectedClient::updateChannelClientProperties(bool lock_channel_tree, bool
             client_channel_lock.lock();
         }
 
-        deque<ChannelId> deleted;
-        for(const auto& update_entry : this->channels->update_channel_path(server_ref->channelTree->tree_head(), server_ref->channelTree->find_linked_entry(this->currentChannel->channelId()))) {
-            if(update_entry.first)
-                this->notifyChannelShow(update_entry.second->channel(), update_entry.second->previous_channel);
-            else deleted.push_back(update_entry.second->channelId());
+        /* might have been changed since we locked the tree */
+        if(this->currentChannel) {
+            deque<ChannelId> deleted;
+            for(const auto& update_entry : this->channels->update_channel_path(server_ref->channelTree->tree_head(), server_ref->channelTree->find_linked_entry(this->currentChannel->channelId()))) {
+                if(update_entry.first)
+                    this->notifyChannelShow(update_entry.second->channel(), update_entry.second->previous_channel);
+                else deleted.push_back(update_entry.second->channelId());
+            }
+            if(!deleted.empty())
+                this->notifyChannelHide(deleted, false); /* we've locked the tree before */
         }
-        if(!deleted.empty())
-            this->notifyChannelHide(deleted, false); /* we've locked the tree before */
     }
 }
 
@@ -571,27 +574,66 @@ bool ConnectedClient::notifyClientNeededPermissions() {
     return true;
 }
 
+inline void write_command_result_error(ts::command_builder_bulk bulk, const command_result& result) {
+    bulk.put_unchecked("id", (uint32_t) result.error_code());
+    bulk.put_unchecked("msg", findError(result.error_code()).message);
+    if(result.is_permission_error())
+        bulk.put_unchecked("failed_permid", (uint32_t) result.permission_id());
+}
+
+inline void write_command_result_detailed(ts::command_builder_bulk bulk, const command_result& result) {
+    auto details = result.details();
+    bulk.put_unchecked("id", (uint32_t) details->error_id);
+    bulk.put_unchecked("msg", findError(details->error_id).message);
+
+    for(const auto& extra : details->extra_properties)
+        bulk.put(extra.first, extra.second);
+}
+
 bool ConnectedClient::notifyError(const command_result& result, const std::string& retCode) {
-    Command cmd("error");
+    ts::command_builder command{"error"};
 
-    if(result.is_detailed()) {
-        auto detailed = result.details();
-        cmd["id"] = (int) detailed->error_id;
-        cmd["msg"] = findError(detailed->error_id).message;
+    switch(result.type()) {
+        case command_result_type::error:
+            write_command_result_error(command.bulk(0), result);
+            break;
+        case command_result_type::detailed:
+            write_command_result_detailed(command.bulk(0), result);
+            break;
 
-        for(const auto& extra : detailed->extra_properties)
-            cmd[extra.first] = extra.second;
-    } else {
-        cmd["id"] = (int) result.error_code();
-        cmd["msg"] = findError(result.error_code()).message;
-        if(result.is_permission_error())
-            cmd["failed_permid"] = result.permission_id();
+        case command_result_type::bulked: {
+            auto bulks = result.bulks();
+            command.reserve_bulks(bulks->size());
+            for(size_t index{0}; index < bulks->size(); index++) {
+                auto& entry = bulks->at(index);
+                switch (entry.type()) {
+                    case command_result_type::error:
+                        write_command_result_error(command.bulk(index), entry);
+                        break;
+                    case command_result_type::detailed:
+                        write_command_result_detailed(command.bulk(index), entry);
+                        break;
+                    case command_result_type::bulked:
+                        assert(false);
+                        break;
+                }
+            }
+            if(bulks->empty()) {
+                logWarning(this->getServerId(), "{} Trying to send empty error bulk.", CLIENT_STR_LOG_PREFIX_(this));
+                command.put_unchecked(0, "id", (uint32_t) error::ok);
+                command.put_unchecked(0, "msg", findError(error::ok).message);
+            }
+            break;
+        }
+        default:
+            assert(false);
+            break;
     }
 
-    if(retCode.length() > 0)
-        cmd["return_code"] = retCode;
+    if(!retCode.empty())
+        command.put_unchecked(0, "return_code", retCode);
 
-    this->sendCommand(cmd);
+    this->sendCommand(command);
     return true;
 }
 
@@ -633,12 +675,7 @@ inline void send_channels(ConnectedClient* client, ChannelIT begin, const Channe
             break;
     }
 
-    if(dynamic_cast<VoiceClient*>(client)) {
-        auto vc = dynamic_cast<VoiceClient*>(client);
-        vc->sendCommand0(builder.build(), false, true); /* we need to process this command directly so it will be processed before the channellistfinished stuff */
-    } else {
-        client->sendCommand(builder);
-    }
+    client->sendCommand(builder);
     if(begin != end)
         send_channels(client, begin, end, override_orderid);
 }
@@ -779,11 +816,7 @@ void ConnectedClient::sendServerInit() {
     command["pv"] = 6; //Protocol version
     command["acn"] = this->getDisplayName();
     command["aclid"] = this->getClientId();
-    if(dynamic_cast<VoiceClient*>(this)) {
-        dynamic_cast<VoiceClient*>(this)->sendCommand0(command.build(), false, true); /* process it directly so the order for the channellist entries is ensured. (First serverinit then everything else) */
-    } else {
-        this->sendCommand(command);
-    }
+    this->sendCommand(command);
 }
 
 bool ConnectedClient::handleCommandFull(Command& cmd, bool disconnectOnFail) {
@@ -795,14 +828,14 @@ bool ConnectedClient::handleCommandFull(Command& cmd, bool disconnectOnFail) {
 
     command_result result;
     try {
-        result = this->handleCommand(cmd);
+        result.reset(this->handleCommand(cmd));
     } catch(invalid_argument& ex){
         logWarning(this->getServerId(), "{}[Command] Failed to handle command. Received invalid argument exception: {}", CLIENT_STR_LOG_PREFIX, ex.what());
         if(disconnectOnFail) {
             this->disconnect("Invalid argument (" + string(ex.what()) + ")");
             return false;
         } else {
-            result = command_result{error::parameter_convert};
+            result.reset(command_result{error::parameter_convert, ex.what()});
         }
     } catch (exception& ex) {
         logWarning(this->getServerId(), "{}[Command] Failed to handle command. Received exception with message: {}", CLIENT_STR_LOG_PREFIX, ex.what());
@@ -810,7 +843,7 @@ bool ConnectedClient::handleCommandFull(Command& cmd, bool disconnectOnFail) {
             this->disconnect("Error while command handling (" + string(ex.what()) + ")!");
             return false;
         } else {
-            result = command_result{error::vs_critical};
+            result.reset(command_result{error::vs_critical});
         }
     } catch (...) {
         this->disconnect("Error while command handling! (unknown)");
@@ -818,7 +851,7 @@ bool ConnectedClient::handleCommandFull(Command& cmd, bool disconnectOnFail) {
     }
 
     bool generateReturnStatus = false;
-    if(result.error_code() != error::ok || this->getType() == ClientType::CLIENT_QUERY){
+    if(result.has_error() || this->getType() == ClientType::CLIENT_QUERY){
         generateReturnStatus = true;
     } else if(cmd["return_code"].size() > 0) {
         generateReturnStatus = !cmd["return_code"].string().empty();
@@ -827,7 +860,7 @@ bool ConnectedClient::handleCommandFull(Command& cmd, bool disconnectOnFail) {
     if(generateReturnStatus)
         this->notifyError(result, cmd["return_code"].size() > 0 ? cmd["return_code"].first().as<std::string>() : "");
 
-    if(result.error_code() != error::ok && this->state == ConnectionState::INIT_HIGH)
+    if(result.has_error() && this->state == ConnectionState::INIT_HIGH)
         this->close_connection(system_clock::now()); //Disconnect now
 
     for (const auto& handler : postCommandHandler)
@@ -841,7 +874,7 @@ bool ConnectedClient::handleCommandFull(Command& cmd, bool disconnectOnFail) {
         else
             logWarning(this->getServerId(), "Command handling of command {} needs {}ms.", cmd.command(), duration_cast<milliseconds>(end - start).count());
     }
-    result.release_details();
+    result.release_data();
     return true;
 }
 

server/src/client/ConnectedClient.h

@@ -633,18 +633,20 @@ namespace ts {
 
         template <typename T>
         struct ConnectedLockedClient {
+            ConnectedLockedClient() {}
             explicit ConnectedLockedClient(std::shared_ptr<T> client) : client{std::move(client)} {
                 if(this->client)
                     this->connection_lock = this->client->require_connected_state();
             }
+            explicit ConnectedLockedClient(ConnectedLockedClient&& client) : client{std::move(client.client)}, connection_lock{std::move(client.connection_lock)} { }
 
-            inline ConnectedLockedClient<T> &operator=(const ConnectedLockedClient& other) {
+            inline ConnectedLockedClient &operator=(const ConnectedLockedClient& other) {
                 this->client = other.client;
                 if(other)
                     this->connection_lock = std::shared_lock{*other.connection_lock.mutex()}; /* if the other is true (state locked & client) than we could easily acquire a shared lock */
             }
 
-            inline ConnectedLockedClient<T> &operator=(ConnectedLockedClient&& other) {
+            inline ConnectedLockedClient &operator=(ConnectedLockedClient&& other) {
                 this->client = std::move(other.client);
                 this->connection_lock = std::move(other.connection_lock);
             }

server/src/client/ConnectedClientNotifyHandler.cpp

@@ -3,7 +3,7 @@
 #include <algorithm>
 #include "ConnectedClient.h"
 #include "voice/VoiceClient.h"
-#include "../server/file/FileServer.h"
+#include "src/server/file/LocalFileServer.h"
 #include "../server/VoiceServer.h"
 #include "../InstanceHandler.h"
 #include "../server/QueryServer.h"

server/src/client/ConnectedClientTextCommandHandler.cpp

@@ -158,8 +158,8 @@ bool ConnectedClient::handle_text_command(
         if (TARG(0, "create")) {
             Command cmd("");
             auto result = this->handleCommandMusicBotCreate(cmd);
-            if(result.error_code()) {
-                result.release_details();
+            if(result.has_error()) {
+                result.release_data();
                 HANDLE_CMD_ERROR("Failed to create music bot");
                 return true;
             }
@@ -220,9 +220,9 @@ bool ConnectedClient::handle_text_command(
             Command cmd("");
             cmd["client_nickname"] = ss.str();
             auto result = this->handleCommandClientEdit(cmd, bot);
-            if(result.error_code()) {
+            if(result.has_error()) {
                 HANDLE_CMD_ERROR("Failed to rename bot");
-                result.release_details();
+                result.release_data();
                 return true;
             }
             send_message(serverInstance->musicRoot(), "Name successfully changed!");
@@ -508,9 +508,9 @@ bool ConnectedClient::handle_text_command(
                     JOIN_ARGS(value, 3);
                     cmd[arguments[2]] = value;
                     auto result = this->handleCommandClientEdit(cmd, bot);
-                    if(result.error_code()) {
+                    if(result.has_error()) {
                         HANDLE_CMD_ERROR("Failed to change bot property");
-                        result.release_details();
+                        result.release_data();
                         return true;
                     }
                     send_message(serverInstance->musicRoot(), "Property successfully changed!");

server/src/client/DataClient.cpp

@@ -4,7 +4,7 @@
 #include <misc/hex.h>
 #include "DataClient.h"
 #include "ConnectedClient.h"
-#include "src/server/file/FileServer.h"
+#include "src/server/file/LocalFileServer.h"
 #include "src/InstanceHandler.h"
 #include "misc/base64.h"
 

server/src/client/SpeakingClient.cpp

@@ -10,7 +10,6 @@
 #include "SpeakingClient.h"
 #include "src/InstanceHandler.h"
 #include "StringVariable.h"
-#include "src/music/MusicBotManager.h"
 #include "misc/timer.h"
 
 using namespace std::chrono;
@@ -52,6 +51,14 @@ void SpeakingClient::handlePacketVoice(const pipes::buffer_view& data, bool head
         return;
     }
 
+#if 0
+    if(rand() % 10 == 0) {
+        logMessage(0, "Dropping audio packet");
+        return;
+    }
+    logMessage(0, "Received voice: Head: {} Fragmented: {}, length: {}", head, fragmented, data.length());
+#endif
+
     auto current_channel = this->currentChannel;
     if(!current_channel) { return; }
     if(!this->allowedToTalk) { return; }
@@ -99,9 +106,10 @@ void SpeakingClient::handlePacketVoice(const pipes::buffer_view& data, bool head
         memcpy(&buffer[5], &data[3], data.length() - 3);
     }
 
+    auto bview = pipes::buffer_view{buffer, data.length() + 2};
     for (const auto& client : target_clients) {
         auto speaking_client = static_pointer_cast<SpeakingClient>(client);
-        speaking_client->send_voice_packet(pipes::buffer_view{buffer, data.length() + 2}, flags);
+        speaking_client->send_voice_packet(bview, flags);
     }
 }
 
@@ -113,7 +121,9 @@ enum WhisperType {
     SERVER_GROUP = 0,
     CHANNEL_GROUP = 1,
     CHANNEL_COMMANDER = 2,
-    ALL = 3
+    ALL = 3,
+
+    ECHO_TEXT = 0x10,
 };
 
 enum WhisperTarget {
@@ -173,87 +183,93 @@ void SpeakingClient::handlePacketVoiceWhisper(const pipes::buffer_view& data, bo
 #endif
 
         deque<shared_ptr<SpeakingClient>> available_clients;
-        for(const auto& client : this->server->getClients()) {
-            auto speakingClient = dynamic_pointer_cast<SpeakingClient>(client);
-            if(!speakingClient || client == this) continue;
-            if(!speakingClient->currentChannel) continue;
+        if(type == WhisperType::ECHO_TEXT) {
+            available_clients.push_back(dynamic_pointer_cast<SpeakingClient>(this->ref()));
+        } else {
+            for(const auto& client : this->server->getClients()) {
+                auto speakingClient = dynamic_pointer_cast<SpeakingClient>(client);
+                if(!speakingClient || client == this) continue;
+                if(!speakingClient->currentChannel) continue;
 
-            if(type == WhisperType::ALL) {
-                available_clients.push_back(speakingClient);
-            } else if(type == WhisperType::SERVER_GROUP) {
-                if(type_id == 0)
+                if(type == WhisperType::ALL) {
                     available_clients.push_back(speakingClient);
-                else {
-                    shared_lock client_lock(this->channel_lock);
-                    for(const auto& id : client->cached_server_groups) {
-                        if(id == type_id) {
-                            available_clients.push_back(speakingClient);
-                            break;
+                } else if(type == WhisperType::SERVER_GROUP) {
+                    if(type_id == 0)
+                        available_clients.push_back(speakingClient);
+                    else {
+                        shared_lock client_lock(this->channel_lock);
+                        for(const auto& id : client->cached_server_groups) {
+                            if(id == type_id) {
+                                available_clients.push_back(speakingClient);
+                                break;
+                            }
                         }
                     }
+                } else if(type == WhisperType::CHANNEL_GROUP) {
+                    if(client->cached_channel_group == type_id)
+                        available_clients.push_back(speakingClient);
+                } else if(type == WhisperType::CHANNEL_COMMANDER) {
+                    if(client->properties()[property::CLIENT_IS_CHANNEL_COMMANDER].as<bool>())
+                        available_clients.push_back(speakingClient);
                 }
-            } else if(type == WhisperType::CHANNEL_GROUP) {
-                if(client->cached_channel_group == type_id)
-                    available_clients.push_back(speakingClient);
-            } else if(type == WhisperType::CHANNEL_COMMANDER) {
-                if(client->properties()[property::CLIENT_IS_CHANNEL_COMMANDER].as<bool>())
-                    available_clients.push_back(speakingClient);
             }
-        }
 
-        if(target == WhisperTarget::CHANNEL_CURRENT) {
-            available_clients.erase(std::remove_if(available_clients.begin(), available_clients.end(), [&](const shared_ptr<SpeakingClient>& target) {
-                return target->currentChannel != this->currentChannel;
-            }), available_clients.end());
-        } else if(target == WhisperTarget::CHANNEL_PARENT) {
-            auto current_parent = this->currentChannel->parent();
-            if(!current_parent) return;
 
-            available_clients.erase(std::remove_if(available_clients.begin(), available_clients.end(), [&](const shared_ptr<SpeakingClient>& target) {
-                return target->currentChannel != current_parent;
-            }), available_clients.end());
-        } else if(target == WhisperTarget::CHANNEL_ALL_PARENT) {
-            shared_ptr<BasicChannel> current_parent;
-            {
-                current_parent = this->currentChannel->parent();
+
+            if(target == WhisperTarget::CHANNEL_CURRENT) {
+                available_clients.erase(std::remove_if(available_clients.begin(), available_clients.end(), [&](const shared_ptr<SpeakingClient>& target) {
+                    return target->currentChannel != this->currentChannel;
+                }), available_clients.end());
+            } else if(target == WhisperTarget::CHANNEL_PARENT) {
+                auto current_parent = this->currentChannel->parent();
                 if(!current_parent) return;
+
+                available_clients.erase(std::remove_if(available_clients.begin(), available_clients.end(), [&](const shared_ptr<SpeakingClient>& target) {
+                    return target->currentChannel != current_parent;
+                }), available_clients.end());
+            } else if(target == WhisperTarget::CHANNEL_ALL_PARENT) {
+                shared_ptr<BasicChannel> current_parent;
+                {
+                    current_parent = this->currentChannel->parent();
+                    if(!current_parent) return;
+                }
+                available_clients.erase(std::remove_if(available_clients.begin(), available_clients.end(), [&](const shared_ptr<SpeakingClient>& target) {
+                    auto tmp_parent = current_parent;
+                    while(tmp_parent && tmp_parent != target->currentChannel)
+                        tmp_parent = tmp_parent->parent();
+                    return target->currentChannel != tmp_parent;
+                }), available_clients.end());
+            } else if(target == WhisperTarget::CHANNEL_FAMILY) {
+                shared_ptr<BasicChannel> current = this->currentChannel;
+                available_clients.erase(std::remove_if(available_clients.begin(), available_clients.end(), [&](const shared_ptr<SpeakingClient>& target) {
+                    auto tmp_current = target->currentChannel;
+                    while(tmp_current && tmp_current != current)
+                        tmp_current = tmp_current->parent();
+                    return current != tmp_current;
+                }), available_clients.end());
+            } else if(target == WhisperTarget::CHANNEL_COMPLETE_FAMILY) {
+                shared_ptr<BasicChannel> current = this->currentChannel;
+                while(current && current->parent()) current = current->parent();
+                available_clients.erase(std::remove_if(available_clients.begin(), available_clients.end(), [&](const shared_ptr<SpeakingClient>& target) {
+                    auto tmp_current = target->currentChannel;
+                    while(tmp_current && tmp_current != current)
+                        tmp_current = tmp_current->parent();
+                    return current != tmp_current;
+                }), available_clients.end());
+            } else if(target == WhisperTarget::CHANNEL_SUBCHANNELS) {
+                shared_ptr<BasicChannel> current = this->currentChannel;
+                available_clients.erase(std::remove_if(available_clients.begin(), available_clients.end(), [&](const shared_ptr<SpeakingClient>& target) {
+                    return target->currentChannel->parent() != current;
+                }), available_clients.end());
             }
-            available_clients.erase(std::remove_if(available_clients.begin(), available_clients.end(), [&](const shared_ptr<SpeakingClient>& target) {
-                auto tmp_parent = current_parent;
-                while(tmp_parent && tmp_parent != target->currentChannel)
-                    tmp_parent = tmp_parent->parent();
-                return target->currentChannel != tmp_parent;
-            }), available_clients.end());
-        } else if(target == WhisperTarget::CHANNEL_FAMILY) {
-            shared_ptr<BasicChannel> current = this->currentChannel;
-            available_clients.erase(std::remove_if(available_clients.begin(), available_clients.end(), [&](const shared_ptr<SpeakingClient>& target) {
-                auto tmp_current = target->currentChannel;
-                while(tmp_current && tmp_current != current)
-                    tmp_current = tmp_current->parent();
-                return current != tmp_current;
-            }), available_clients.end());
-        } else if(target == WhisperTarget::CHANNEL_COMPLETE_FAMILY) {
-            shared_ptr<BasicChannel> current = this->currentChannel;
-            while(current && current->parent()) current = current->parent();
-            available_clients.erase(std::remove_if(available_clients.begin(), available_clients.end(), [&](const shared_ptr<SpeakingClient>& target) {
-                auto tmp_current = target->currentChannel;
-                while(tmp_current && tmp_current != current)
-                    tmp_current = tmp_current->parent();
-                return current != tmp_current;
-            }), available_clients.end());
-        } else if(target == WhisperTarget::CHANNEL_SUBCHANNELS) {
-            shared_ptr<BasicChannel> current = this->currentChannel;
-            available_clients.erase(std::remove_if(available_clients.begin(), available_clients.end(), [&](const shared_ptr<SpeakingClient>& target) {
-                return target->currentChannel->parent() != current;
+
+            auto self_lock = this->_this.lock();
+            available_clients.erase(std::remove_if(available_clients.begin(), available_clients.end(), [&](const std::shared_ptr<ConnectedClient>& cl) {
+                auto speakingClient = dynamic_pointer_cast<SpeakingClient>(cl);
+                return !speakingClient->shouldReceiveVoiceWhisper(self_lock);
             }), available_clients.end());
         }
 
-        auto self_lock = this->_this.lock();
-        available_clients.erase(std::remove_if(available_clients.begin(), available_clients.end(), [&](const std::shared_ptr<ConnectedClient>& cl) {
-            auto speakingClient = dynamic_pointer_cast<SpeakingClient>(cl);
-            return !speakingClient->shouldReceiveVoiceWhisper(self_lock);
-        }), available_clients.end());
-
         if(available_clients.empty()) {
             if(update_whisper_error(this->speak_last_no_whisper_target)) {
                 command_result result{error::whisper_no_targets};
@@ -281,7 +297,7 @@ void SpeakingClient::handlePacketVoiceWhisper(const pipes::buffer_view& data, bo
         VoicePacketFlags flags{};
         auto data = pipes::buffer_view(packet_buffer, OUT_WHISPER_PKT_OFFSET + data_length);
         for(const auto& cl : available_clients){
-                cl->send_voice_whisper_packet(data, flags);
+            cl->send_voice_whisper_packet(data, flags);
         }
 
         this->updateSpeak(false, system_clock::now());
@@ -377,7 +393,7 @@ auto regex_wildcard = std::regex(".*");
 
 #define S(x) #x
 #define HWID_REGEX(name, pattern)                           \
-auto regex_hwid_ ##name = [](){                             \
+auto regex_hwid_ ##name = []() noexcept {                   \
     try {                                                   \
         return std::regex(pattern);                         \
     } catch (std::exception& ex) {                          \
@@ -532,37 +548,39 @@ command_result SpeakingClient::handleCommandClientInit(Command& cmd) {
         if(!this->server->verifyServerPassword(cmd["client_server_password"].string(), true))
             return command_result{error::server_invalid_password};
 
-    size_t clones_uid = 0;
-    size_t clones_ip = 0;
-    size_t clones_hwid = 0;
+    if(!config::server::clients::ignore_max_clone_permissions) {
+        size_t clones_uid = 0;
+        size_t clones_ip = 0;
+        size_t clones_hwid = 0;
 
-    auto _own_hwid = this->getHardwareId();
-    this->server->forEachClient([&](const shared_ptr<ConnectedClient>& client) {
-        if(client->getExternalType() != CLIENT_TEAMSPEAK) return;
-        if(client->getUid() == this->getUid())
-            clones_uid++;
-        if(client->getPeerIp() == this->getPeerIp())
-            clones_ip++;
-        if(!_own_hwid.empty() && client->getHardwareId() == _own_hwid)
-            clones_hwid++;
-    });
+        auto _own_hwid = this->getHardwareId();
+        this->server->forEachClient([&](const shared_ptr<ConnectedClient>& client) {
+            if(client->getExternalType() != CLIENT_TEAMSPEAK) return;
+            if(client->getUid() == this->getUid())
+                clones_uid++;
+            if(client->getPeerIp() == this->getPeerIp())
+                clones_ip++;
+            if(!_own_hwid.empty() && client->getHardwareId() == _own_hwid)
+                clones_hwid++;
+        });
 
-    if(clones_uid > 0 && permissions[permission::i_client_max_clones_uid].has_value && !permission::v2::permission_granted(clones_uid, permissions[permission::i_client_max_clones_uid])) {
-        logMessage(this->getServerId(), "{} Disconnecting because there are already {} uid clones connected. (Allowed: {})", CLIENT_STR_LOG_PREFIX, clones_uid, permissions[permission::i_client_max_clones_uid]);
-        return command_result{error:: client_too_many_clones_connected, "too many clones connected (uid)"};
+        if(clones_uid > 0 && permissions[permission::i_client_max_clones_uid].has_value && !permission::v2::permission_granted(clones_uid, permissions[permission::i_client_max_clones_uid])) {
+            logMessage(this->getServerId(), "{} Disconnecting because there are already {} uid clones connected. (Allowed: {})", CLIENT_STR_LOG_PREFIX, clones_uid, permissions[permission::i_client_max_clones_uid]);
+            return command_result{error:: client_too_many_clones_connected, "too many clones connected (uid)"};
+        }
+
+        if(clones_ip > 0 && permissions[permission::i_client_max_clones_ip].has_value && !permission::v2::permission_granted(clones_ip, permissions[permission::i_client_max_clones_ip])) {
+            logMessage(this->getServerId(), "{} Disconnecting because there are already {} ip clones connected. (Allowed: {})", CLIENT_STR_LOG_PREFIX, clones_ip, permissions[permission::i_client_max_clones_ip]);
+            return command_result{error:: client_too_many_clones_connected, "too many clones connected (ip)"};
+        }
+
+        if(clones_hwid > 0 && permissions[permission::i_client_max_clones_hwid].has_value && !permission::v2::permission_granted(clones_hwid, permissions[permission::i_client_max_clones_hwid])) {
+            logMessage(this->getServerId(), "{} Disconnecting because there are already {} hwid clones connected. (Allowed: {})", CLIENT_STR_LOG_PREFIX, clones_hwid, permissions[permission::i_client_max_clones_hwid]);
+            return command_result{error:: client_too_many_clones_connected, "too many clones connected (hwid)"};
+        }
+        TIMING_STEP(timings, "max clones ");
     }
 
-    if(clones_ip > 0 && permissions[permission::i_client_max_clones_ip].has_value && !permission::v2::permission_granted(clones_ip, permissions[permission::i_client_max_clones_ip])) {
-        logMessage(this->getServerId(), "{} Disconnecting because there are already {} ip clones connected. (Allowed: {})", CLIENT_STR_LOG_PREFIX, clones_ip, permissions[permission::i_client_max_clones_ip]);
-        return command_result{error:: client_too_many_clones_connected, "too many clones connected (ip)"};
-    }
-
-    if(clones_hwid > 0 && permissions[permission::i_client_max_clones_hwid].has_value && !permission::v2::permission_granted(clones_hwid, permissions[permission::i_client_max_clones_hwid])) {
-        logMessage(this->getServerId(), "{} Disconnecting because there are already {} hwid clones connected. (Allowed: {})", CLIENT_STR_LOG_PREFIX, clones_hwid, permissions[permission::i_client_max_clones_hwid]);
-        return command_result{error:: client_too_many_clones_connected, "too many clones connected (hwid)"};
-    }
-    TIMING_STEP(timings, "max clones ");
-
     auto banEntry = this->resolveActiveBan(this->getPeerIp());
     if(banEntry) {
         logMessage(this->getServerId(), "{} Disconnecting while init because of ban record. Record id {} at server {}",
@@ -721,7 +739,7 @@ void SpeakingClient::processJoin() {
     if(!ref_server->assignDefaultChannel(this->ref(), false)) {
         auto result = command_result{error::vs_critical, "Could not assign default channel!"};
         this->notifyError(result);
-        result.release_details();
+        result.release_data();
         this->close_connection(system_clock::now() + seconds(1));
         return;
     }
@@ -857,13 +875,13 @@ command_result SpeakingClient::handleCommand(Command &command) {
         if(this->handshake.state == HandshakeState::BEGIN || this->handshake.state == HandshakeState::IDENTITY_PROOF) {
             command_result result;
             if(command.command() == "handshakebegin")
-                result = this->handleCommandHandshakeBegin(command);
+                result.reset(this->handleCommandHandshakeBegin(command));
             else if(command.command() == "handshakeindentityproof")
-                result = this->handleCommandHandshakeIdentityProof(command);
+                result.reset(this->handleCommandHandshakeIdentityProof(command));
             else
-                result = command_result{error::client_not_logged_in};
+                result.reset(command_result{error::client_not_logged_in});
 
-            if(result.error_code())
+            if(result.has_error())
                 this->postCommandHandler.push_back([&]{
                     this->close_connection(system_clock::now() + seconds(1));
                 });

server/src/client/command_handler/bulk_parsers.cpp

@@ -0,0 +1,5 @@
+//
+// Created by WolverinDEV on 07/05/2020.
+//
+
+#include "bulk_parsers.h"

server/src/client/command_handler/bulk_parsers.h

@@ -0,0 +1,242 @@
+#pragma once
+
+#include <vector>
+#include <query/Command.h>
+#include <Error.h>
+#include <PermissionManager.h>
+#include <src/client/ConnectedClient.h>
+#include "./helpers.h"
+
+namespace ts::command::bulk_parser {
+    template <bool kParseValue>
+    class PermissionBulkParser {
+        public:
+            explicit PermissionBulkParser(ts::ParameterBulk& bulk) {
+                if(bulk.has("permid")) {
+                    auto type = bulk["permid"].as<permission::PermissionType>();
+                    if ((type & PERM_ID_GRANT) != 0) {
+                        type &= ~PERM_ID_GRANT;
+                    }
+
+                    this->permission_ = permission::resolvePermissionData(type);
+                } else if(bulk.has("permsid")) {
+                    auto permission_name = bulk["permsid"].string();
+                    this->permission_ = permission::resolvePermissionData(permission_name);
+                    this->grant_ = this->permission_->grantName() == permission_name;;
+                } else {
+                    this->error_.reset(ts::command_result{error::parameter_missing, "permid"});
+                    return;
+                }
+
+                if(this->permission_->is_invalid()) {
+                    this->error_.reset(ts::command_result{error::parameter_invalid});
+                    return;
+                }
+
+                if(kParseValue) {
+                    if(!bulk.has("permvalue")) {
+                        this->error_.reset(ts::command_result{error::parameter_missing, "permvalue"});
+                        return;
+                    }
+
+                    this->value_ = bulk["permvalue"].as<permission::PermissionValue>();
+                    this->flag_skip_ = bulk.has("permskip") && bulk["permskip"].as<bool>();
+                    this->flag_negated_ = bulk.has("permnegated") && bulk["permnegated"].as<bool>();
+                }
+            }
+            PermissionBulkParser(const PermissionBulkParser&) = delete;
+            PermissionBulkParser(PermissionBulkParser&&) = default;
+
+            ~PermissionBulkParser() {
+                this->error_.release_data();
+            }
+
+            [[nodiscard]] inline bool has_error() const {
+                assert(!this->error_released_);
+                return this->error_.has_error();
+            }
+
+            [[nodiscard]] inline bool has_value() const { return this->value_ != permission::undefined; }
+            [[nodiscard]] inline bool is_grant_permission() const { return this->grant_; }
+
+            [[nodiscard]] inline const auto& permission() const { return this->permission_; }
+            [[nodiscard]] inline auto permission_type() const { return this->permission_->type; }
+
+            [[nodiscard]] inline auto value() const { return this->value_; }
+            [[nodiscard]] inline auto flag_skip() const { return this->flag_skip_; }
+            [[nodiscard]] inline auto flag_negated() const { return this->flag_negated_; }
+
+            [[nodiscard]] inline ts::command_result release_error() {
+                assert(!std::exchange(this->error_released_, true));
+                return std::move(this->error_);
+            }
+
+            inline void emplace_custom_error(ts::command_result&& result) {
+                assert(!this->error_released_);
+                this->error_.reset(std::forward<ts::command_result>(result));
+            }
+
+            inline void apply_to(const std::shared_ptr<permission::v2::PermissionManager>& manager, permission::v2::PermissionUpdateType mode) const {
+                if(this->is_grant_permission()) {
+                    manager->set_permission(this->permission_type(), { 0, this->value() }, permission::v2::PermissionUpdateType::do_nothing, mode);
+                } else {
+                    manager->set_permission(
+                            this->permission_type(),
+                            { this->value(), 0 },
+                            mode,
+                            permission::v2::PermissionUpdateType::do_nothing,
+                            this->flag_skip(),
+                            this->flag_negated()
+                    );
+                }
+            }
+
+            inline void apply_to_channel(const std::shared_ptr<permission::v2::PermissionManager>& manager, permission::v2::PermissionUpdateType mode, ChannelId channel_id) const {
+                if(this->is_grant_permission()) {
+                    manager->set_channel_permission(this->permission_type(), channel_id, { this->value(), true }, permission::v2::PermissionUpdateType::do_nothing, mode);
+                } else {
+                    manager->set_channel_permission(
+                            this->permission_type(),
+                            channel_id,
+                            { this->value(), true },
+                            mode,
+                            permission::v2::PermissionUpdateType::do_nothing
+                    );
+                }
+            }
+
+            [[nodiscard]] inline bool is_group_property() const {
+                return permission_is_group_property(this->permission_type());
+            }
+
+            [[nodiscard]] inline bool is_client_view_property() const {
+                return permission_is_client_property(this->permission_type());
+            }
+        private:
+            std::shared_ptr<permission::PermissionTypeEntry> permission_{nullptr};
+            bool grant_{false};
+
+            bool flag_skip_{false};
+            bool flag_negated_{false};
+
+            permission::PermissionValue value_{0};
+            ts::command_result error_{error::ok};
+
+#ifndef NDEBUG
+            bool error_released_{false};
+#endif
+    };
+
+    template <bool kParseValue>
+    class PermissionBulksParser {
+        public:
+            PermissionBulksParser(const PermissionBulksParser&) = delete;
+            PermissionBulksParser(PermissionBulksParser&&) = default;
+
+            template <typename base_iterator>
+            struct FilteredPermissionIterator : public base_iterator {
+                public:
+                    FilteredPermissionIterator() = default;
+                    explicit FilteredPermissionIterator(base_iterator position, base_iterator end = {}) : base_iterator{position}, end_{end} {
+                        if(*this != this->end_) {
+                            const auto& entry = **this;
+                            if(entry.has_error())
+                                this->operator++();
+                        }
+                    }
+
+                    FilteredPermissionIterator& operator++() {
+                        while(true) {
+                            base_iterator::operator++();
+                            if(*this == this->end_) break;
+
+                            const auto& entry = **this;
+                            if(!entry.has_error()) break;
+                        }
+                        return *this;
+                    }
+
+                    [[nodiscard]] FilteredPermissionIterator operator++(int) const {
+                        FilteredPermissionIterator copy = *this;
+                        ++*this;
+                        return copy;
+                    }
+
+                private:
+                    base_iterator end_;
+            };
+
+            struct FilteredPermissionListIterable {
+                typedef typename std::vector<PermissionBulkParser<kParseValue>>::const_iterator const_iterator;
+                public:
+                    FilteredPermissionListIterable(const_iterator begin, const_iterator end) noexcept : begin_{begin}, end_{end} {}
+
+                    FilteredPermissionIterator<const_iterator> begin() const {
+                        return FilteredPermissionIterator{this->begin_, this->end_};
+                    }
+
+                    FilteredPermissionIterator<const_iterator> end() const {
+                        return FilteredPermissionIterator{this->end_, this->end_};
+                    }
+                private:
+                    const_iterator begin_;
+                    const_iterator end_;
+            };
+
+            explicit PermissionBulksParser(ts::Command& command) {
+                this->permissions_.reserve(command.bulkCount());
+                for(size_t index{0}; index < command.bulkCount(); index++)
+                    this->permissions_.emplace_back(command[index]);
+            }
+
+            [[nodiscard]] inline bool validate(const std::shared_ptr<server::ConnectedClient>& issuer, ChannelId channel_id) {
+                auto ignore_granted_values = permission::v2::permission_granted(1, issuer->calculate_permission(permission::b_permission_modify_power_ignore, channel_id));
+                if(!ignore_granted_values) {
+                    auto max_value = issuer->calculate_permission(permission::i_permission_modify_power, channel_id, false);
+                    if(!max_value.has_value) {
+                        for(PermissionBulkParser<kParseValue>& permission : this->permissions_) {
+                            if(permission.has_error()) continue;
+
+                            permission.emplace_custom_error(ts::command_result{permission::i_permission_modify_power});
+                        }
+                        return false;
+                    }
+
+                    for(size_t index{0}; index < this->permissions_.size(); index++) {
+                        PermissionBulkParser<kParseValue>& permission = this->permissions_[index];
+                        if(permission.has_error()) continue;
+
+                        if(kParseValue && permission_require_granted_value(permission.permission_type()) && !permission::v2::permission_granted(permission.value(), max_value)) {
+                            permission.emplace_custom_error(ts::command_result{permission::i_permission_modify_power});
+                            continue;
+                        }
+
+                        if(!permission::v2::permission_granted(1, issuer->calculate_permission(permission.permission_type(), channel_id, true))) {
+                            permission.emplace_custom_error(ts::command_result{permission::i_permission_modify_power});
+                            continue;
+                        }
+                    }
+                }
+                return true;
+            }
+
+            [[nodiscard]] inline FilteredPermissionListIterable iterate_valid_permissions() const {
+                return FilteredPermissionListIterable{this->permissions_.begin(), this->permissions_.end()};
+            }
+
+            [[nodiscard]] inline ts::command_result build_command_result() {
+                assert(!std::exchange(this->result_created_, true));
+                ts::command_result_bulk result{};
+
+                for(auto& permission : this->permissions_)
+                    result.insert_result(std::forward<ts::command_result>(permission.release_error()));
+
+                return ts::command_result{std::move(result)};
+            }
+        private:
+            std::vector<PermissionBulkParser<kParseValue>> permissions_{};
+#ifndef NDEBUG
+            bool result_created_{false};
+#endif
+    };
+}

server/src/client/command_handler/channel.cpp

@@ -5,7 +5,7 @@
 #include "../../build.h"
 #include "../ConnectedClient.h"
 #include "../InternalClient.h"
-#include "../../server/file/FileServer.h"
+#include "src/server/file/LocalFileServer.h"
 #include "../../server/VoiceServer.h"
 #include "../voice/VoiceClient.h"
 #include "PermissionManager.h"
@@ -20,6 +20,7 @@
 #include <cstdint>
 
 #include "helpers.h"
+#include "./bulk_parsers.h"
 
 #include <Properties.h>
 #include <log/LogUtils.h>
@@ -407,46 +408,16 @@ command_result ConnectedClient::handleCommandChannelGroupAddPerm(Command &cmd) {
     if (!channelGroup || channelGroup->target() != GROUPTARGET_CHANNEL) return command_result{error::parameter_invalid, "invalid channel group id"};
     ACTION_REQUIRES_GROUP_PERMISSION(channelGroup, permission::i_channel_group_needed_modify_power, permission::i_channel_group_modify_power, true);
 
-    auto max_value = this->calculate_permission(permission::i_permission_modify_power, 0, true);
-    if(!max_value.has_value) return command_result{permission::i_permission_modify_power};
+    command::bulk_parser::PermissionBulksParser<true> pparser{cmd};
+    if(!pparser.validate(this->ref(), 0))
+        return pparser.build_command_result();
 
-    auto ignore_granted_values = permission::v2::permission_granted(1, this->calculate_permission(permission::b_permission_modify_power_ignore, 0));
-
-    auto conOnError = cmd[0].has("continueonerror");
-    bool updateList = false;
-
-    auto permission_manager = channelGroup->permissions();
-    for (int index = 0; index < cmd.bulkCount(); index++) {
-        PARSE_PERMISSION(cmd);
-
-        auto val = cmd[index]["permvalue"].as<permission::PermissionValue>();
-        if(permission_require_granted_value(permType) && !permission::v2::permission_granted(val, max_value)) {
-            if(conOnError) continue;
-            return command_result{permission::i_permission_modify_power};
-        }
-
-        if(!ignore_granted_values && !permission::v2::permission_granted(1, this->calculate_permission(permType, 0, true))) {
-            if(conOnError) continue;
-            return command_result{permission::i_permission_modify_power};
-        }
-
-        if (grant) {
-            permission_manager->set_permission(permType, {0, cmd[index]["permvalue"]}, permission::v2::PermissionUpdateType::do_nothing, permission::v2::PermissionUpdateType::set_value);
-        } else {
-            permission_manager->set_permission(
-                    permType,
-                    {cmd[index]["permvalue"], 0},
-                    permission::v2::PermissionUpdateType::set_value,
-                    permission::v2::PermissionUpdateType::do_nothing,
-
-                    cmd[index]["permskip"].as<bool>() ? 1 : 0,
-                    cmd[index]["permnegated"].as<bool>() ? 1 : 0
-            );
-            updateList |= permission_is_group_property(permType);
-        }
+    bool updateList{false};
+    for(const auto& ppermission : pparser.iterate_valid_permissions()) {
+        ppermission.apply_to(channelGroup->permissions(), permission::v2::PermissionUpdateType::set_value);
+        updateList |= ppermission.is_group_property();
     }
 
-
     if(updateList)
         channelGroup->apply_properties_from_permissions();
 
@@ -465,7 +436,8 @@ command_result ConnectedClient::handleCommandChannelGroupAddPerm(Command &cmd) {
             }
         });
     }
-    return command_result{error::ok};
+
+    return pparser.build_command_result();
 }
 
 command_result ConnectedClient::handleCommandChannelGroupDelPerm(Command &cmd) {
@@ -475,31 +447,14 @@ command_result ConnectedClient::handleCommandChannelGroupDelPerm(Command &cmd) {
     if (!channelGroup || channelGroup->target() != GROUPTARGET_CHANNEL) return command_result{error::parameter_invalid, "invalid channel group id"};
     ACTION_REQUIRES_GROUP_PERMISSION(channelGroup, permission::i_channel_group_needed_modify_power, permission::i_channel_group_modify_power, true);
 
-    auto ignore_granted_values = permission::v2::permission_granted(1, this->calculate_permission(permission::b_permission_modify_power_ignore, 0));
+    command::bulk_parser::PermissionBulksParser<false> pparser{cmd};
+    if(!pparser.validate(this->ref(), 0))
+        return pparser.build_command_result();
 
-    bool updateList = false;
-    bool conOnError = cmd[0].has("continueonerror");
-
-    auto permission_manager = channelGroup->permissions();
-    for (int index = 0; index < cmd.bulkCount(); index++) {
-        PARSE_PERMISSION(cmd)
-
-        if(!ignore_granted_values && !permission::v2::permission_granted(0, this->calculate_permission(permType, 0, true))) {
-            if(conOnError) continue;
-            return command_result{permission::i_permission_modify_power};
-        }
-
-        if (grant) {
-            permission_manager->set_permission(permType, permission::v2::empty_permission_values, permission::v2::PermissionUpdateType::do_nothing, permission::v2::PermissionUpdateType::delete_value);
-        } else {
-            permission_manager->set_permission(
-                    permType,
-                    permission::v2::empty_permission_values,
-                    permission::v2::PermissionUpdateType::delete_value,
-                    permission::v2::PermissionUpdateType::do_nothing
-            );
-            updateList |= permission_is_group_property(permType);
-        }
+    bool updateList{false};
+    for(const auto& ppermission : pparser.iterate_valid_permissions()) {
+        ppermission.apply_to(channelGroup->permissions(), permission::v2::PermissionUpdateType::delete_value);
+        updateList |= ppermission.is_group_property();
     }
 
     if(updateList)
@@ -520,7 +475,8 @@ command_result ConnectedClient::handleCommandChannelGroupDelPerm(Command &cmd) {
             }
         });
     }
-    return command_result{error::ok};
+
+    return pparser.build_command_result();
 }
 
 //TODO: Test if parent or previous is deleted!
@@ -1481,101 +1437,33 @@ command_result ConnectedClient::handleCommandChannelAddPerm(Command &cmd) {
 
     ACTION_REQUIRES_CHANNEL_PERMISSION(channel, permission::i_channel_needed_permission_modify_power, permission::i_channel_permission_modify_power, true);
 
-    auto max_value = this->calculate_permission(permission::i_permission_modify_power, channel_id, true);
-    if(!max_value.has_value) return command_result{permission::i_permission_modify_power};
-
-    auto ignore_granted_values = permission::v2::permission_granted(1, this->calculate_permission(permission::b_permission_modify_power_ignore, channel_id));
-    auto updateClients = false, update_view = false, update_channel_properties = false;
-
-    bool conOnError = cmd[0].has("continueonerror");
+    command::bulk_parser::PermissionBulksParser<true> pparser{cmd};
+    if(!pparser.validate(this->ref(), channel->channelId()))
+        return pparser.build_command_result();
 
     auto permission_manager = channel->permissions();
-    for (int index = 0; index < cmd.bulkCount(); index++) {
-        PARSE_PERMISSION(cmd);
+    auto updateClients = false, update_join_permissions = false, update_channel_properties = false;
+    for(const auto& ppermission : pparser.iterate_valid_permissions()) {
+        ppermission.apply_to(permission_manager, permission::v2::PermissionUpdateType::set_value);
 
-        auto val = cmd[index]["permvalue"].as<permission::PermissionValue>();
-        if(permission_require_granted_value(permType) && !permission::v2::permission_granted(val, max_value)) {
-            if(conOnError) continue;
-            return command_result{permission::i_permission_modify_power};
-        }
-
-        if(!ignore_granted_values && !permission::v2::permission_granted(1, this->calculate_permission(permType, channel_id, true))) {
-            if(conOnError) continue;
-            return command_result{permission::i_permission_modify_power};
-        }
-
-
-        if (grant) {
-            permission_manager->set_permission(permType, {0, cmd[index]["permvalue"]}, permission::v2::PermissionUpdateType::do_nothing, permission::v2::PermissionUpdateType::set_value);
-        } else {
-            permission_manager->set_permission(
-                    permType,
-                    {cmd[index]["permvalue"], 0},
-                    permission::v2::PermissionUpdateType::set_value,
-                    permission::v2::PermissionUpdateType::do_nothing,
-
-                    cmd[index]["permskip"].as<bool>() ? 1 : 0,
-                    cmd[index]["permnegated"].as<bool>() ? 1 : 0
-            );
-            updateClients |= permission_is_client_property(permType);
-            update_view |= permType == permission::i_channel_needed_view_power;
-            update_channel_properties |= channel->permission_require_property_update(permType);
-
-            if (permType == permission::i_icon_id) {
-                if(this->server) {
-                    auto self_ref = this->ref();
-                    this->server->forEachClient([&](std::shared_ptr<ConnectedClient> cl) {
-                        shared_lock client_channel_lock(cl->channel_lock);
-                        cl->notifyChannelEdited(channel, {property::CHANNEL_ICON_ID}, self_ref, false);
-                    });
-                }
-                continue;
-            }
-        }
+        updateClients |= ppermission.is_client_view_property();
+        update_join_permissions = ppermission.permission_type() == permission::i_channel_needed_join_power;
+        update_channel_properties |= channel->permission_require_property_update(ppermission.permission_type());
     }
 
-    /* broadcast the updated channel properties */
-    if(update_channel_properties) {
-        auto updates = channel->update_properties_from_permissions();
-        if(!updates.empty() && this->server){
-            auto self_ref = this->ref();
-            this->server->forEachClient([&](std::shared_ptr<ConnectedClient> cl) {
-                shared_lock client_channel_lock(cl->channel_lock);
-                cl->notifyChannelEdited(channel, updates, self_ref, false);
-            });
-        }
-    }
+    if(update_channel_properties && this->server)
+        this->server->update_channel_from_permissions(channel, this->ref());
 
-    if(updateClients && this->server)
-        for(const auto& client : this->server->getClientsByChannel(channel)) {
-            /* let them lock the server channel tree as well (read lock so does not matter) */
-            client->updateChannelClientProperties(true, true);
-            client->join_state_id++; /* join permission may changed, all channels need to be recalculate dif needed */
-        }
-
-    if(update_view && this->server) {
-        auto l_source = this->server->channelTree->findLinkedChannel(channel->channelId());
-        this->server->forEachClient([&](const shared_ptr<ConnectedClient>& cl) {
-            /* server tree read lock still active */
-            auto l_target = !cl->currentChannel ? nullptr : cl->server->channelTree->findLinkedChannel(cl->currentChannel->channelId());
-            sassert(l_source);
-            if(cl->currentChannel) sassert(l_target);
-
-            {
-                unique_lock client_channel_lock(cl->channel_lock);
-
-                deque<ChannelId> deleted;
-                for(const auto& update_entry : cl->channels->update_channel(l_source, l_target)) {
-                    if(update_entry.first)
-                        cl->notifyChannelShow(update_entry.second->channel(), update_entry.second->previous_channel);
-                    else deleted.push_back(update_entry.second->channelId());
-                }
-                if(!deleted.empty())
-                    cl->notifyChannelHide(deleted, false);
-            }
+    if((updateClients || update_join_permissions) && this->server) {
+        this->server->forEachClient([&](std::shared_ptr<ConnectedClient> cl) {
+            if(updateClients && cl->currentChannel == channel)
+                cl->updateChannelClientProperties(true, true);
+            if(update_join_permissions)
+                cl->join_state_id++;
         });
     }
-    return command_result{error::ok};;
+
+    return pparser.build_command_result();
 }
 
 command_result ConnectedClient::handleCommandChannelDelPerm(Command &cmd) {
@@ -1587,71 +1475,33 @@ command_result ConnectedClient::handleCommandChannelDelPerm(Command &cmd) {
 
     ACTION_REQUIRES_CHANNEL_PERMISSION(channel, permission::i_channel_needed_permission_modify_power, permission::i_channel_permission_modify_power, true);
 
-    auto ignore_granted_values = permission::v2::permission_granted(1, this->calculate_permission(permission::b_permission_modify_power_ignore, channel_id));
-    bool conOnError = cmd[0].has("continueonerror");
-    auto updateClients = false, update_view = false, update_channel_properties = false;
+    command::bulk_parser::PermissionBulksParser<false> pparser{cmd};
+    if(!pparser.validate(this->ref(), channel->channelId()))
+        return pparser.build_command_result();
 
     auto permission_manager = channel->permissions();
-    for (int index = 0; index < cmd.bulkCount(); index++) {
-        PARSE_PERMISSION(cmd);
+    auto updateClients = false, update_join_permissions = false, update_channel_properties = false;
+    for(const auto& ppermission : pparser.iterate_valid_permissions()) {
+        ppermission.apply_to(permission_manager, permission::v2::PermissionUpdateType::delete_value);
 
-        if(!ignore_granted_values && !permission::v2::permission_granted(0, this->calculate_permission(permType, channel_id, true))) {
-            if(conOnError) continue;
-            return command_result{permission::i_permission_modify_power};
-        }
-
-        if (grant) {
-            permission_manager->set_permission(permType, permission::v2::empty_permission_values, permission::v2::PermissionUpdateType::do_nothing, permission::v2::PermissionUpdateType::delete_value);
-        } else {
-            permission_manager->set_permission(permType, permission::v2::empty_permission_values, permission::v2::PermissionUpdateType::delete_value, permission::v2::PermissionUpdateType::do_nothing);
-            updateClients |= permission_is_client_property(permType);
-            update_view |= permType == permission::i_channel_needed_view_power;
-            update_channel_properties |= channel->permission_require_property_update(permType);
-        }
+        updateClients |= ppermission.is_client_view_property();
+        update_join_permissions = ppermission.permission_type() == permission::i_channel_needed_join_power;
+        update_channel_properties |= channel->permission_require_property_update(ppermission.permission_type());
     }
 
-    /* broadcast the updated channel properties */
-    if(update_channel_properties) {
-        auto updates = channel->update_properties_from_permissions();
-        if(!updates.empty() && this->server){
-            auto self_ref = this->ref();
-            this->server->forEachClient([&](std::shared_ptr<ConnectedClient> cl) {
-                shared_lock client_channel_lock(cl->channel_lock);
-                cl->notifyChannelEdited(channel, updates, self_ref, false);
-            });
-        }
-    }
+    if(update_channel_properties && this->server)
+        this->server->update_channel_from_permissions(channel, this->ref());
 
-    if(updateClients && this->server)
+    if((updateClients || update_join_permissions) && this->server) {
         this->server->forEachClient([&](std::shared_ptr<ConnectedClient> cl) {
-            if(cl->currentChannel == channel) {
+            if(updateClients && cl->currentChannel == channel)
                 cl->updateChannelClientProperties(true, true);
-                cl->join_state_id++; /* join permission may changed, all channels need to be recalculate dif needed */
-            }
-        });
-    if(update_view && this->server) {
-        this->server->forEachClient([&](std::shared_ptr<ConnectedClient> cl) {
-            /* server tree read lock still active */
-            auto l_source = cl->server->channelTree->findLinkedChannel(channel->channelId());
-            auto l_target = !cl->currentChannel ? nullptr : cl->server->channelTree->findLinkedChannel(cl->currentChannel->channelId());
-            sassert(l_source);
-            if(cl->currentChannel) sassert(l_target);
-
-            {
-                unique_lock client_channel_lock(cl->channel_lock);
-
-                deque<ChannelId> deleted;
-                for(const auto& update_entry : cl->channels->update_channel(l_source, l_target)) {
-                    if(update_entry.first)
-                        cl->notifyChannelShow(update_entry.second->channel(), update_entry.second->previous_channel);
-                    else deleted.push_back(update_entry.second->channelId());
-                }
-                if(!deleted.empty())
-                    cl->notifyChannelHide(deleted, false);
-            }
+            if(update_join_permissions)
+                cl->join_state_id++;
         });
     }
-    return command_result{error::ok};
+
+    return pparser.build_command_result();
 }
 
 command_result ConnectedClient::handleCommandChannelClientPermList(Command &cmd) {
@@ -1730,29 +1580,21 @@ command_result ConnectedClient::handleCommandChannelClientDelPerm(Command &cmd)
         ACTION_REQUIRES_PERMISSION(permission::i_client_permission_modify_power, required_permissions, channel_id);
     }
 
-    auto ignore_granted_values = permission::v2::permission_granted(1, this->calculate_permission(permission::b_permission_modify_power_ignore, channel_id));
+    command::bulk_parser::PermissionBulksParser<false> pparser{cmd};
+    if(!pparser.validate(this->ref(), channel->channelId()))
+        return pparser.build_command_result();
 
-    bool conOnError = cmd[0].has("continueonerror"), update_view = false;
-    auto cll = this->server->findClientsByCldbId(cldbid);
-    for (int index = 0; index < cmd.bulkCount(); index++) {
-        PARSE_PERMISSION(cmd);
-
-        if(!ignore_granted_values && !permission::v2::permission_granted(0, this->calculate_permission(permType, channel_id, true))) {
-            if(conOnError) continue;
-            return command_result{permission::i_permission_modify_power};
-        }
-
-        if (grant) {
-            mgr->set_channel_permission(permType, channel->channelId(), permission::v2::empty_permission_values, permission::v2::do_nothing, permission::v2::delete_value);
-        } else {
-            mgr->set_channel_permission(permType, channel->channelId(), permission::v2::empty_permission_values, permission::v2::delete_value, permission::v2::do_nothing);
-            update_view = permType == permission::b_channel_ignore_view_power || permType == permission::i_channel_view_power;
-        }
+    bool update_view{false};
+    for(const auto& ppermission : pparser.iterate_valid_permissions()) {
+        ppermission.apply_to_channel(mgr, permission::v2::PermissionUpdateType::delete_value, channel->channelId());
+        update_view |= ppermission.is_client_view_property();
     }
 
     serverInstance->databaseHelper()->saveClientPermissions(this->server, cldbid, mgr);
-    if (!cll.empty()) {
-        for (const auto &elm : cll) {
+
+    auto onlineClients = this->server->findClientsByCldbId(cldbid);
+    if (!onlineClients.empty()) {
+        for (const auto &elm : onlineClients) {
             if(elm->update_cached_permissions()) /* update cached calculated permissions */
                 elm->sendNeededPermissions(false); /* cached permissions had changed, notify the client */
 
@@ -1778,7 +1620,7 @@ command_result ConnectedClient::handleCommandChannelClientDelPerm(Command &cmd)
         }
     }
 
-    return command_result{error::ok};
+    return pparser.build_command_result();
 }
 
 command_result ConnectedClient::handleCommandChannelClientAddPerm(Command &cmd) {
@@ -1795,45 +1637,25 @@ command_result ConnectedClient::handleCommandChannelClientAddPerm(Command &cmd)
     if(!channel) return command_result{error::vs_critical};
 
     auto mgr = serverInstance->databaseHelper()->loadClientPermissionManager(this->server, cldbid);
-    {
-        auto required_permissions = this->server->calculate_permission(permission::i_client_needed_permission_modify_power, cmd["cldbid"], ClientType::CLIENT_TEAMSPEAK, channel_id);
-        ACTION_REQUIRES_PERMISSION(permission::i_client_permission_modify_power, required_permissions, channel_id);
-    }
+    auto required_permissions = this->server->calculate_permission(permission::i_client_needed_permission_modify_power, cmd["cldbid"], ClientType::CLIENT_TEAMSPEAK, channel_id);
+    ACTION_REQUIRES_PERMISSION(permission::i_client_permission_modify_power, required_permissions, channel_id);
 
-    auto max_value = this->calculate_permission(permission::i_permission_modify_power, channel_id, true);
-    if(!max_value.has_value) return command_result{permission::i_permission_modify_power};
+    command::bulk_parser::PermissionBulksParser<true> pparser{cmd};
+    if(!pparser.validate(this->ref(), channel->channelId()))
+        return pparser.build_command_result();
 
-    auto ignore_granted_values = permission::v2::permission_granted(1, this->calculate_permission(permission::b_permission_modify_power_ignore, channel_id));
-    auto update_view = false;
-
-    bool conOnError = cmd[0].has("continueonerror");
-    auto onlineClientInstances = this->server->findClientsByCldbId(cldbid);
-
-    for (int index = 0; index < cmd.bulkCount(); index++) {
-        PARSE_PERMISSION(cmd);
-
-        auto val = cmd[index]["permvalue"].as<permission::PermissionValue>();
-        if(permission_require_granted_value(permType) && !permission::v2::permission_granted(val, max_value)) {
-            if(conOnError) continue;
-            return command_result{permission::i_permission_modify_power};
-        }
-
-        if(!ignore_granted_values && !permission::v2::permission_granted(1, this->calculate_permission(permType, channel_id, true))) {
-            if(conOnError) continue;
-            return command_result{permission::i_permission_modify_power};
-        }
-
-        if (grant) {
-            mgr->set_channel_permission(permType, channel->channelId(), {0, cmd[index]["permvalue"]}, permission::v2::do_nothing, permission::v2::set_value);
-        } else {
-            mgr->set_channel_permission(permType, channel->channelId(), {cmd[index]["permvalue"], 0}, permission::v2::set_value, permission::v2::do_nothing,  cmd[index]["permskip"] ? 1 : 0, cmd[index]["permnegated"] ? 1 : 0);
-            update_view = permType == permission::b_channel_ignore_view_power || permType == permission::i_channel_view_power;
-        }
+    bool update_view{false};
+    for(const auto& ppermission : pparser.iterate_valid_permissions()) {
+        ppermission.apply_to_channel(mgr, permission::v2::PermissionUpdateType::set_value, channel->channelId());
+        update_view |= ppermission.is_client_view_property();
     }
 
     serverInstance->databaseHelper()->saveClientPermissions(this->server, cldbid, mgr);
-    if (!onlineClientInstances.empty())
-        for (const auto &elm : onlineClientInstances) {
+
+
+    auto onlineClients = this->server->findClientsByCldbId(cldbid);
+    if (!onlineClients.empty())
+        for (const auto &elm : onlineClients) {
             if (elm->update_cached_permissions()) /* update cached calculated permissions */
                 elm->sendNeededPermissions(false); /* cached permissions had changed, notify the client */
 
@@ -1857,7 +1679,7 @@ command_result ConnectedClient::handleCommandChannelClientAddPerm(Command &cmd)
             elm->join_state_id++; /* join permission may changed, all channels need to be recalculate dif needed */
         }
 
-    return command_result{error::ok};
+    return pparser.build_command_result();
 }
 
 

server/src/client/command_handler/client.cpp

@@ -1,11 +1,12 @@
 #include <memory>
 
+#include <vector>
 #include <bitset>
 #include <algorithm>
 #include "../../build.h"
 #include "../ConnectedClient.h"
 #include "../InternalClient.h"
-#include "../../server/file/FileServer.h"
+#include "src/server/file/LocalFileServer.h"
 #include "../voice/VoiceClient.h"
 #include "PermissionManager.h"
 #include "../../InstanceHandler.h"
@@ -19,6 +20,7 @@
 #include <cstdint>
 
 #include "helpers.h"
+#include "./bulk_parsers.h"
 
 #include <Properties.h>
 #include <log/LogUtils.h>
@@ -61,29 +63,56 @@ command_result ConnectedClient::handleCommandClientKick(Command &cmd) {
     CMD_REQ_SERVER;
     CMD_CHK_AND_INC_FLOOD_POINTS(25);
 
-    ConnectedLockedClient client{this->server->find_client_by_id(cmd["clid"].as<ClientId>())};
-    if (!client) return command_result{error::client_invalid_id};
-    if (client->getType() == CLIENT_MUSIC) return command_result{error::client_invalid_type, "You cant kick a music bot!"};
-    std::shared_ptr<BasicChannel> targetChannel = nullptr;
-    auto type = cmd["reasonid"].as<ViewReasonId>();
-    if (type == ViewReasonId::VREASON_CHANNEL_KICK) {
-        auto channel = client->getChannel();
-        ACTION_REQUIRES_PERMISSION(permission::i_client_kick_from_channel_power, client->calculate_permission(permission::i_client_needed_kick_from_channel_power, client->getChannelId()), client->getChannelId());
-        targetChannel = this->server->channelTree->getDefaultChannel();
-    } else if (type == ViewReasonId::VREASON_SERVER_KICK) {
-        auto channel = client->getChannel();
-        ACTION_REQUIRES_GLOBAL_PERMISSION(permission::i_client_kick_from_server_power, client->calculate_permission(permission::i_client_needed_kick_from_server_power, client->getChannelId()));
-        targetChannel = nullptr;
-    } else return command_result{error::not_implemented};
+    command_result_bulk result{};
+    result.reserve(cmd.bulkCount());
 
-    if (targetChannel) {
-        this->server->notify_client_kick(client.client, this->ref(), cmd["reasonmsg"].as<std::string>(), targetChannel);
-    } else {
-        this->server->notify_client_kick(client.client, this->ref(), cmd["reasonmsg"].as<std::string>(), nullptr);
-        client->close_connection(system_clock::now() + seconds(1));
+    std::vector<ConnectedLockedClient<ConnectedClient>> clients{};
+    clients.reserve(cmd.bulkCount());
+
+    auto type = cmd["reasonid"].as<ViewReasonId>();
+    auto target_channel = type == ViewReasonId::VREASON_CHANNEL_KICK ? this->server->channelTree->getDefaultChannel() : nullptr;
+    auto kick_power = type == ViewReasonId::VREASON_CHANNEL_KICK ?
+            this->calculate_permission(permission::i_client_kick_from_channel_power, target_channel->channelId()) :
+            this->calculate_permission(permission::i_client_kick_from_server_power, 0);
+
+    for(size_t index = 0; index < cmd.bulkCount(); index++) {
+        ConnectedLockedClient<ConnectedClient> client{this->server->find_client_by_id(cmd[index]["clid"].as<ClientId>())};
+
+        if (!client) {
+            result.emplace_result(error::client_invalid_id);
+            continue;
+        }
+        if (client->getType() == CLIENT_MUSIC) {
+            result.emplace_result(error::client_invalid_type);
+            continue;
+        }
+
+        if(type == ViewReasonId::VREASON_CHANNEL_KICK) {
+            if(!permission::v2::permission_granted(client->calculate_permission(permission::i_client_needed_kick_from_channel_power, client->getChannelId()), kick_power)) {
+                result.emplace_result(permission::i_client_needed_kick_from_channel_power);
+                continue;
+            }
+        } else {
+            if(!permission::v2::permission_granted(client->calculate_permission(permission::i_client_needed_kick_from_server_power, client->getChannelId()), kick_power)) {
+                result.emplace_result(permission::i_client_needed_kick_from_server_power);
+                continue;
+            }
+        }
+
+        clients.emplace_back(std::move(client));
+        result.emplace_result(error::ok);
     }
 
-    return command_result{error::ok};
+    for(auto& client : clients) {
+        if (target_channel) {
+            this->server->notify_client_kick(client.client, this->ref(), cmd["reasonmsg"].as<std::string>(), target_channel);
+        } else {
+            this->server->notify_client_kick(client.client, this->ref(), cmd["reasonmsg"].as<std::string>(), nullptr);
+            client->close_connection(system_clock::now() + seconds(1));
+        }
+    }
+
+    return command_result{std::forward<command_result_bulk>(result)};
 }
 
 command_result ConnectedClient::handleCommandClientGetIds(Command &cmd) {
@@ -130,16 +159,7 @@ command_result ConnectedClient::handleCommandClientMove(Command &cmd) {
     CMD_CHK_AND_INC_FLOOD_POINTS(10);
 
     shared_lock server_channel_r_lock(this->server->channel_tree_lock);
-    auto target_client_id = cmd["clid"].as<ClientId>();
-    ConnectedLockedClient target_client{target_client_id == 0 ? this->ref() : this->server->find_client_by_id(target_client_id)};
-    if(!target_client) {
-        return command_result{error::client_invalid_id, "Invalid target clid"};
-    }
 
-    if(!target_client->getChannel()) {
-        if(target_client.client != this)
-            return command_result{error::client_invalid_id, "Invalid target clid"};
-    }
     auto channel = this->server->channelTree->findChannel(cmd["cid"].as<ChannelId>());
     if (!channel) {
         return command_result{error::channel_invalid_id};
@@ -148,6 +168,7 @@ command_result ConnectedClient::handleCommandClientMove(Command &cmd) {
     auto permission_cache = make_shared<CalculateCache>();
     if(!cmd[0].has("cpw"))
         cmd["cpw"] = "";
+
     if (!channel->passwordMatch(cmd["cpw"], true))
         if (!permission::v2::permission_granted(1, this->calculate_permission(permission::b_channel_join_ignore_password, channel->channelId())))
             return command_result{error::channel_invalid_password};
@@ -155,11 +176,49 @@ command_result ConnectedClient::handleCommandClientMove(Command &cmd) {
     auto permission_error = this->calculate_and_get_join_state(channel);
     if(permission_error != permission::unknown) return command_result{permission_error};
 
+    command_result_bulk result{};
+    result.reserve(cmd.bulkCount());
+
+    std::vector<ConnectedLockedClient<ConnectedClient>> clients{};
+
+    for(size_t index{0}; index < cmd.bulkCount(); index++) {
+        auto target_client_id = cmd[index]["clid"].as<ClientId>();
+        ConnectedLockedClient target_client{target_client_id == 0 ? this->ref() : this->server->find_client_by_id(target_client_id)};
+        if(!target_client) {
+            result.emplace_result(error::client_invalid_id);
+            continue;
+        }
+
+        if(!target_client->getChannel()) {
+            if(target_client.client != this) {
+                result.emplace_result(error::client_invalid_id);
+                continue;
+            }
+        }
+        if(target_client->getChannel() == channel) {
+            result.emplace_result(error::ok);
+            continue;
+        }
+
+        if(target_client.client != this) {
+            if(!permission::v2::permission_granted(target_client->calculate_permission(permission::i_client_needed_move_power, target_client->getChannelId()), this->calculate_permission(permission::i_client_move_power, target_client->getChannelId()))) {
+                result.emplace_result(permission::i_client_move_power);
+                continue;
+            }
+            if(!permission::v2::permission_granted(target_client->calculate_permission(permission::i_client_needed_move_power, channel->channelId()), this->calculate_permission(permission::i_client_move_power, channel->channelId()))) {
+                result.emplace_result(permission::i_client_move_power);
+                continue;
+            }
+        }
+        clients.emplace_back(std::move(target_client));
+        result.emplace_result(error::ok);
+    }
+
     if (!channel->properties()[property::CHANNEL_FLAG_MAXCLIENTS_UNLIMITED].as<bool>() || !channel->properties()[property::CHANNEL_FLAG_MAXFAMILYCLIENTS_UNLIMITED].as<bool>()) {
         if(!permission::v2::permission_granted(1, this->calculate_permission(permission::b_channel_join_ignore_maxclients, channel->channelId()))) {
             if(!channel->properties()[property::CHANNEL_FLAG_MAXCLIENTS_UNLIMITED].as<bool>()) {
                 auto maxClients = channel->properties()[property::CHANNEL_MAXCLIENTS].as<int32_t>();
-                if (maxClients >= 0 && maxClients <= this->server->getClientsByChannel(channel).size())
+                if (maxClients >= 0 && maxClients < this->server->getClientsByChannel(channel).size() + clients.size())
                     return command_result{error::channel_maxclients_reached};
             }
             if(!channel->properties()[property::CHANNEL_FLAG_MAXFAMILYCLIENTS_UNLIMITED].as<bool>()) {
@@ -167,36 +226,45 @@ command_result ConnectedClient::handleCommandClientMove(Command &cmd) {
 
                 if(channel->properties()[property::CHANNEL_FLAG_MAXFAMILYCLIENTS_INHERITED].as<bool>()) {
                     family_root = channel;
-                    while(family_root && family_root->properties()[property::CHANNEL_FLAG_MAXFAMILYCLIENTS_INHERITED].as<bool>()) family_root = family_root->parent();
+                    while(family_root && family_root->properties()[property::CHANNEL_FLAG_MAXFAMILYCLIENTS_INHERITED].as<bool>())
+                        family_root = family_root->parent();
                 }
                 if(family_root && !family_root->properties()[property::CHANNEL_FLAG_MAXFAMILYCLIENTS_UNLIMITED]) { //Could not be CHANNEL_FLAG_MAXFAMILYCLIENTS_INHERITED
                     auto maxClients = family_root->properties()[property::CHANNEL_MAXFAMILYCLIENTS].as<int32_t>();
-                    auto clients = 0;
-                    for(const auto& entry : this->server->getClientsByChannelRoot(channel, false)) if(entry.get() != this) clients++; //Dont count the client itself
-                    if (maxClients >= 0 && maxClients <= clients)
+                    auto client_count = 0;
+                    for(const auto& entry : this->server->getClientsByChannelRoot(channel, false))
+                        if(entry.get() != this) client_count++; //Dont count the client itself
+
+                    if (maxClients >= 0 && maxClients < client_count + clients.size())
                         return command_result{error::channel_maxfamily_reached};
                 }
             }
         }
     }
 
-    if (target_client.client != this)
-        ACTION_REQUIRES_PERMISSION(permission::i_client_move_power, target_client->calculate_permission(permission::i_client_needed_move_power, target_client->getChannelId()), target_client->getChannelId());
-
     server_channel_r_lock.unlock();
     unique_lock server_channel_w_lock(this->server->channel_tree_lock);
-    auto oldChannel = target_client->getChannel();
-    this->server->client_move(
-            target_client.client,
-            channel,
-            target_client.client == this ? nullptr : _this.lock(),
-            "",
-            target_client.client == this ? ViewReasonId::VREASON_USER_ACTION : ViewReasonId::VREASON_MOVED,
-            true,
-            server_channel_w_lock
-    );
+    std::vector<std::shared_ptr<BasicChannel>> channels{};
+    channels.reserve(clients.size());
 
-    if(oldChannel) {
+    for(auto& client : clients) {
+        auto oldChannel = client->getChannel();
+        if(!oldChannel) continue;
+
+        this->server->client_move(
+                client.client,
+                channel,
+                client.client == this ? nullptr : _this.lock(),
+                "",
+                client.client == this ? ViewReasonId::VREASON_USER_ACTION : ViewReasonId::VREASON_MOVED,
+                true,
+                server_channel_w_lock
+        );
+        if(std::find_if(channels.begin(), channels.end(), [&](const std::shared_ptr<BasicChannel>& channel) { return &*channel == &*oldChannel; }) == channels.end())
+            channels.push_back(oldChannel);
+    }
+
+    for(const auto& oldChannel : channels) {
         if(!server_channel_w_lock.owns_lock())
             server_channel_w_lock.lock();
         if(oldChannel->channelType() == ChannelType::temporary && oldChannel->properties()[property::CHANNEL_DELETE_DELAY].as<int64_t>() == 0)
@@ -205,7 +273,7 @@ command_result ConnectedClient::handleCommandClientMove(Command &cmd) {
         if(server_channel_w_lock.owns_lock())
             server_channel_w_lock.unlock();
     }
-    return command_result{error::ok};
+    return command_result{std::forward<command_result_bulk>(result)};
 }
 
 command_result ConnectedClient::handleCommandClientPoke(Command &cmd) {
@@ -213,13 +281,45 @@ command_result ConnectedClient::handleCommandClientPoke(Command &cmd) {
     CMD_RESET_IDLE;
     CMD_CHK_AND_INC_FLOOD_POINTS(25);
 
-    ConnectedLockedClient client{ this->server->find_client_by_id(cmd["clid"].as<ClientId>())};
-    if (!client) return command_result{error::client_invalid_id};
-    if (client->getType() == CLIENT_MUSIC) return command_result{error::client_invalid_type};
-    ACTION_REQUIRES_PERMISSION(permission::i_client_poke_power, client->calculate_permission(permission::i_client_needed_poke_power, client->getChannelId()), client->getChannelId());
+    command_result_bulk result{};
+    result.reserve(cmd.bulkCount());
 
-    client->notifyClientPoke(_this.lock(), cmd["msg"]);
-    return command_result{error::ok};
+    std::vector<ConnectedLockedClient<ConnectedClient>> clients{};
+    clients.reserve(cmd.bulkCount());
+
+    for(size_t index{0}; index < cmd.bulkCount(); index++) {
+        ConnectedLockedClient client{ this->server->find_client_by_id(cmd[index]["clid"].as<ClientId>())};
+        if (!client) {
+            result.emplace_result(error::client_invalid_id);
+            continue;
+        }
+        if (client->getType() == CLIENT_MUSIC) {
+            result.emplace_result(error::client_invalid_type);
+            continue;
+        }
+
+        auto own_permission = this->calculate_permission(permission::i_client_poke_power, client->getChannelId());
+        if(!permission::v2::permission_granted(client->calculate_permission(permission::i_client_needed_poke_power, client->getChannelId()), own_permission)) {
+            result.emplace_result(permission::i_client_poke_power);
+            continue;
+        }
+
+        clients.push_back(std::move(client));
+        result.emplace_result(error::ok);
+    }
+
+    /* clients might be empty ;) */
+
+    if(clients.size() > 1) {
+        auto max_clients = this->calculate_permission(permission::i_client_poke_max_clients, 0);
+        if(!permission::v2::permission_granted(clients.size(), max_clients))
+            return command_result{permission::i_client_poke_max_clients};
+    }
+
+    for(auto& client : clients)
+        client->notifyClientPoke(_this.lock(), cmd["msg"]);
+
+    return command_result{std::forward<command_result_bulk>(result)};
 }
 
 
@@ -874,33 +974,17 @@ command_result ConnectedClient::handleCommandClientAddPerm(Command &cmd) {
     auto mgr = serverInstance->databaseHelper()->loadClientPermissionManager(this->server, cldbid);
 
     ACTION_REQUIRES_GLOBAL_PERMISSION(permission::i_client_permission_modify_power, this->server->calculate_permission(permission::i_client_needed_permission_modify_power, cldbid, ClientType::CLIENT_TEAMSPEAK, 0));
-    auto max_value = this->calculate_permission(permission::i_permission_modify_power, 0, true);
-    if(!max_value.has_value) return command_result{permission::i_permission_modify_power};
 
-    auto ignore_granted_values = permission::v2::permission_granted(1, this->calculate_permission(permission::b_permission_modify_power_ignore, 0));
-    bool conOnError = cmd[0].has("continueonerror");
-    auto update_channels = false;
-    for (int index = 0; index < cmd.bulkCount(); index++) {
-        PARSE_PERMISSION(cmd);
+    command::bulk_parser::PermissionBulksParser<true> pparser{cmd};
+    if(!pparser.validate(this->ref(), 0))
+        return pparser.build_command_result();
 
-        auto val = cmd[index]["permvalue"].as<permission::PermissionValue>();
-        if(permission_require_granted_value(permType) && !permission::v2::permission_granted(val, max_value)) {
-            if(conOnError) continue;
-            return command_result{permission::i_permission_modify_power};
-        }
-
-        if(!ignore_granted_values && !permission::v2::permission_granted(1, this->calculate_permission(permType, 0, true))) {
-            if(conOnError) continue;
-            return command_result{permission::i_permission_modify_power};
-        }
-
-        if (grant) {
-            mgr->set_permission(permType, {0, cmd[index]["permvalue"]}, permission::v2::do_nothing, permission::v2::set_value);
-        } else {
-            mgr->set_permission(permType, {cmd[index]["permvalue"], 0}, permission::v2::set_value, permission::v2::do_nothing,  cmd[index]["permskip"] ? 1 : 0, cmd[index]["permnegated"] ? 1 : 0);
-            update_channels |= permission_is_client_property(permType);
-        }
+    bool update_channels{false};
+    for(const auto& ppermission : pparser.iterate_valid_permissions()) {
+        ppermission.apply_to(mgr, permission::v2::PermissionUpdateType::set_value);
+        update_channels |= ppermission.is_client_view_property();
     }
+
     serverInstance->databaseHelper()->saveClientPermissions(this->server, cldbid, mgr);
     auto onlineClients = this->server->findClientsByCldbId(cldbid);
     if (!onlineClients.empty())
@@ -912,7 +996,7 @@ command_result ConnectedClient::handleCommandClientAddPerm(Command &cmd) {
             elm->join_state_id++; /* join permission may changed, all channels need to be recalculate dif needed */
         }
 
-    return command_result{error::ok};
+    return pparser.build_command_result();
 }
 
 command_result ConnectedClient::handleCommandClientDelPerm(Command &cmd) {
@@ -926,37 +1010,28 @@ command_result ConnectedClient::handleCommandClientDelPerm(Command &cmd) {
     auto mgr = serverInstance->databaseHelper()->loadClientPermissionManager(this->server, cldbid);
     ACTION_REQUIRES_GLOBAL_PERMISSION(permission::i_client_permission_modify_power, this->server->calculate_permission(permission::i_client_needed_permission_modify_power, cldbid, ClientType::CLIENT_TEAMSPEAK, 0));
 
-    auto ignore_granted_values = permission::v2::permission_granted(1, this->calculate_permission(permission::b_permission_modify_power_ignore, 0));
-    bool conOnError = cmd[0].has("continueonerror");
-    auto onlineClients = this->server->findClientsByCldbId(cmd["cldbid"]);
-    auto update_channel = false;
-    for (int index = 0; index < cmd.bulkCount(); index++) {
-        PARSE_PERMISSION(cmd)
+    command::bulk_parser::PermissionBulksParser<false> pparser{cmd};
+    if(!pparser.validate(this->ref(), 0))
+        return pparser.build_command_result();
 
-        if(!ignore_granted_values && !permission::v2::permission_granted(0, this->calculate_permission(permType, 0, true))) {
-            if(conOnError) continue;
-            return command_result{permission::i_permission_modify_power};
-        }
-
-
-        if (grant) {
-            mgr->set_permission(permType, permission::v2::empty_permission_values, permission::v2::do_nothing, permission::v2::delete_value);
-        } else {
-            mgr->set_permission(permType, permission::v2::empty_permission_values, permission::v2::delete_value, permission::v2::do_nothing);
-            update_channel |= permission_is_client_property(permType);
-        }
+    bool update_channels{false};
+    for(const auto& ppermission : pparser.iterate_valid_permissions()) {
+        ppermission.apply_to(mgr, permission::v2::PermissionUpdateType::delete_value);
+        update_channels |= ppermission.is_client_view_property();
     }
 
     serverInstance->databaseHelper()->saveClientPermissions(this->server, cldbid, mgr);
+    auto onlineClients = this->server->findClientsByCldbId(cldbid);
     if (!onlineClients.empty())
         for (const auto &elm : onlineClients) {
             if(elm->update_cached_permissions()) /* update cached calculated permissions */
                 elm->sendNeededPermissions(false); /* cached permissions had changed, notify the client */
-            if(update_channel)
+            if(update_channels)
                 elm->updateChannelClientProperties(true, true);
             elm->join_state_id++; /* join permission may changed, all channels need to be recalculate dif needed */
         }
-    return command_result{error::ok};
+
+    return pparser.build_command_result();
 }
 
 command_result ConnectedClient::handleCommandClientPermList(Command &cmd) {

server/src/client/command_handler/file.cpp

@@ -13,7 +13,7 @@
 #include "../../build.h"
 #include "../ConnectedClient.h"
 #include "../InternalClient.h"
-#include "../../server/file/FileServer.h"
+#include "src/server/file/LocalFileServer.h"
 #include "../../server/VoiceServer.h"
 #include "../voice/VoiceClient.h"
 #include "PermissionManager.h"
@@ -118,14 +118,9 @@ command_result ConnectedClient::handleCommandFTGetFileList(Command &cmd) {
     }
 
     if (fileList[0].has("name")) {
-        if(dynamic_cast<VoiceClient*>(this)) {
-            dynamic_cast<VoiceClient*>(this)->sendCommand0(fileList.build(), false, true); /* We need to process this directly else the order could get shuffled up! */
+        this->sendCommand(fileList);
+        if(this->getType() != CLIENT_QUERY)
             this->sendCommand(fileListFinished);
-        } else {
-            this->sendCommand(fileList);
-            if(this->getType() != CLIENT_QUERY)
-                this->sendCommand(fileListFinished);
-        }
         return command_result{error::ok};
     } else {
         return command_result{error::database_empty_result};
@@ -234,13 +229,11 @@ command_result ConnectedClient::handleCommandFTInitUpload(Command &cmd) {
             directory = serverInstance->getFileServer()->avatarDirectory(this->server);
             cmd["name"] = "/avatar_" + this->getAvatarId();
         } else {
-            cerr << "Unknown requested directory: " << cmd["path"].as<std::string>() << endl;
             return command_result{error::not_implemented};
         }
     }
 
     if (!directory || directory->type != file::FileType::DIRECTORY) { //Should not happen
-        cerr << "Invalid upload file path!" << endl;
         return command_result{error::file_invalid_path, "could not resolve directory"};
     }
 

server/src/client/command_handler/helpers.h

@@ -150,6 +150,8 @@ inline bool permission_require_granted_value(ts::permission::PermissionType type
         case permission::i_max_playlist_size:
         case permission::i_max_playlists:
 
+        case permission::i_client_poke_max_clients:
+
         case permission::i_client_ban_max_bantime:
         case permission::i_client_max_idletime:
         case permission::i_group_sort_id:

server/src/client/command_handler/misc.cpp

@@ -13,7 +13,7 @@
 #include "../../build.h"
 #include "../ConnectedClient.h"
 #include "../InternalClient.h"
-#include "../../server/file/FileServer.h"
+#include "src/server/file/LocalFileServer.h"
 #include "../../server/VoiceServer.h"
 #include "../voice/VoiceClient.h"
 #include "PermissionManager.h"

server/src/client/command_handler/music.cpp

@@ -13,7 +13,7 @@
 #include "../../build.h"
 #include "../ConnectedClient.h"
 #include "../InternalClient.h"
-#include "../../server/file/FileServer.h"
+#include "src/server/file/LocalFileServer.h"
 #include "../../server/VoiceServer.h"
 #include "../voice/VoiceClient.h"
 #include "PermissionManager.h"
@@ -30,6 +30,7 @@
 #include <StringVariable.h>
 
 #include "helpers.h"
+#include "./bulk_parsers.h"
 
 #include <Properties.h>
 #include <log/LogUtils.h>
@@ -558,35 +559,14 @@ command_result ConnectedClient::handleCommandPlaylistAddPerm(ts::Command &cmd) {
     if(auto perr = playlist->client_has_permissions(this->ref(), permission::i_playlist_needed_permission_modify_power, permission::i_playlist_permission_modify_power); perr)
         return command_result{perr};
 
-    auto max_value = this->calculate_permission(permission::i_permission_modify_power, 0, true);
-    if(!max_value.has_value) return command_result{permission::i_permission_modify_power};
+    command::bulk_parser::PermissionBulksParser<true> pparser{cmd};
+    if(!pparser.validate(this->ref(), 0))
+        return pparser.build_command_result();
 
-    auto ignore_granted_values = permission::v2::permission_granted(1, this->calculate_permission(permission::b_permission_modify_power_ignore, 0));
+    for(const auto& ppermission : pparser.iterate_valid_permissions())
+        ppermission.apply_to(playlist->permission_manager(), permission::v2::PermissionUpdateType::set_value);
 
-
-    bool conOnError = cmd[0].has("continueonerror");
-    for (int index = 0; index < cmd.bulkCount(); index++) {
-        PARSE_PERMISSION(cmd);
-
-        auto val = cmd[index]["permvalue"].as<permission::PermissionValue>();
-        if(permission_require_granted_value(permType) && !permission::v2::permission_granted(val, max_value)) {
-            if(conOnError) continue;
-            return command_result{permission::i_permission_modify_power};
-        }
-
-        if(!ignore_granted_values && !permission::v2::permission_granted(1, this->calculate_permission(permType, 0, true))) {
-            if(conOnError) continue;
-            return command_result{permission::i_permission_modify_power};
-        }
-
-        if (grant) {
-            playlist->permission_manager()->set_permission(permType, {0, cmd[index]["permvalue"]}, permission::v2::do_nothing, permission::v2::set_value);
-        } else {
-            playlist->permission_manager()->set_permission(permType, {cmd[index]["permvalue"],0}, permission::v2::set_value, permission::v2::do_nothing, cmd[index]["permskip"].as<bool>(), cmd[index]["permnegated"].as<bool>());
-        }
-    }
-
-    return command_result{error::ok};
+    return pparser.build_command_result();
 }
 
 command_result ConnectedClient::handleCommandPlaylistDelPerm(ts::Command &cmd) {
@@ -600,26 +580,14 @@ command_result ConnectedClient::handleCommandPlaylistDelPerm(ts::Command &cmd) {
     if(auto perr = playlist->client_has_permissions(this->ref(), permission::i_playlist_needed_permission_modify_power, permission::i_playlist_permission_modify_power); perr)
         return command_result{perr};
 
-    auto ignore_granted_values = permission::v2::permission_granted(1, this->calculate_permission(permission::b_permission_modify_power_ignore, 0));
-    bool conOnError = cmd[0].has("continueonerror");
+    command::bulk_parser::PermissionBulksParser<false> pparser{cmd};
+    if(!pparser.validate(this->ref(), 0))
+        return pparser.build_command_result();
 
-    for (int index = 0; index < cmd.bulkCount(); index++) {
-        PARSE_PERMISSION(cmd);
+    for(const auto& ppermission : pparser.iterate_valid_permissions())
+        ppermission.apply_to(playlist->permission_manager(), permission::v2::PermissionUpdateType::delete_value);
 
-        if(!ignore_granted_values && !permission::v2::permission_granted(0, this->calculate_permission(permType, 0, true))) {
-            if(conOnError) continue;
-            return command_result{permission::i_permission_modify_power};
-        }
-
-
-        if (grant) {
-            playlist->permission_manager()->set_permission(permType, {0, 0}, permission::v2::do_nothing, permission::v2::delete_value);
-        } else {
-            playlist->permission_manager()->set_permission(permType, {0, 0}, permission::v2::delete_value, permission::v2::do_nothing);
-        }
-    }
-
-    return command_result{error::ok};
+    return pparser.build_command_result();
 }
 
 command_result ConnectedClient::handleCommandPlaylistClientList(ts::Command &cmd) {
@@ -735,34 +703,14 @@ command_result ConnectedClient::handleCommandPlaylistClientAddPerm(ts::Command &
     if(auto perr = playlist->client_has_permissions(this->ref(), permission::i_playlist_needed_permission_modify_power, permission::i_playlist_permission_modify_power); perr)
         return command_result{perr};
 
-    auto max_value = this->calculate_permission(permission::i_permission_modify_power, 0, true);
-    if(!max_value.has_value) return command_result{permission::i_permission_modify_power};
+    command::bulk_parser::PermissionBulksParser<true> pparser{cmd};
+    if(!pparser.validate(this->ref(), this->getClientDatabaseId()))
+        return pparser.build_command_result();
 
-    auto ignore_granted_values = permission::v2::permission_granted(1, this->calculate_permission(permission::b_permission_modify_power_ignore, 0));
+    for(const auto& ppermission : pparser.iterate_valid_permissions())
+        ppermission.apply_to_channel(playlist->permission_manager(), permission::v2::PermissionUpdateType::set_value, client_id);
 
-    bool conOnError = cmd[0].has("continueonerror");
-    for (int index = 0; index < cmd.bulkCount(); index++) {
-        PARSE_PERMISSION(cmd);
-
-        auto val = cmd[index]["permvalue"].as<permission::PermissionValue>();
-        if(permission_require_granted_value(permType) && !permission::v2::permission_granted(val, max_value)) {
-            if(conOnError) continue;
-            return command_result{permission::i_permission_modify_power};
-        }
-
-        if(!ignore_granted_values && !permission::v2::permission_granted(1, this->calculate_permission(permType, 0, true))) {
-            if(conOnError) continue;
-            return command_result{permission::i_permission_modify_power};
-        }
-
-        if (grant) {
-            playlist->permission_manager()->set_channel_permission(permType, client_id, {0, cmd[index]["permvalue"]}, permission::v2::do_nothing, permission::v2::set_value);
-        } else {
-            playlist->permission_manager()->set_channel_permission(permType, client_id, {cmd[index]["permvalue"], 0}, permission::v2::set_value, permission::v2::do_nothing, cmd[index]["permskip"].as<bool>(), cmd[index]["permnegated"].as<bool>());
-        }
-    }
-
-    return command_result{error::ok};
+    return pparser.build_command_result();
 }
 
 command_result ConnectedClient::handleCommandPlaylistClientDelPerm(ts::Command &cmd) {
@@ -779,26 +727,15 @@ command_result ConnectedClient::handleCommandPlaylistClientDelPerm(ts::Command &
     if(auto perr = playlist->client_has_permissions(this->ref(), permission::i_playlist_needed_permission_modify_power, permission::i_playlist_permission_modify_power); perr)
         return command_result{perr};
 
-    auto ignore_granted_values = permission::v2::permission_granted(1, this->calculate_permission(permission::b_permission_modify_power_ignore, 0));
-    bool conOnError = cmd[0].has("continueonerror");
 
-    for (int index = 0; index < cmd.bulkCount(); index++) {
-        PARSE_PERMISSION(cmd);
+    command::bulk_parser::PermissionBulksParser<false> pparser{cmd};
+    if(!pparser.validate(this->ref(), this->getClientDatabaseId()))
+        return pparser.build_command_result();
 
-        if(!ignore_granted_values && !permission::v2::permission_granted(0, this->calculate_permission(permType, 0, true))) {
-            if(conOnError) continue;
-            return command_result{permission::i_permission_modify_power};
-        }
+    for(const auto& ppermission : pparser.iterate_valid_permissions())
+        ppermission.apply_to_channel(playlist->permission_manager(), permission::v2::PermissionUpdateType::delete_value, client_id);
 
-
-        if (grant) {
-            playlist->permission_manager()->set_channel_permission(permType, client_id, {0, 0}, permission::v2::do_nothing, permission::v2::delete_value);
-        } else {
-            playlist->permission_manager()->set_channel_permission(permType, client_id, {0, 0}, permission::v2::delete_value, permission::v2::do_nothing);
-        }
-    }
-
-    return command_result{error::ok};
+    return pparser.build_command_result();
 }
 
 constexpr auto max_song_meta_info = 1024 * 512;

server/src/client/command_handler/server.cpp

@@ -4,19 +4,15 @@
 
 #include <memory>
 
-#include <spdlog/sinks/rotating_file_sink.h>
-
-#include <iostream>
 #include <bitset>
 #include <algorithm>
 #include <openssl/sha.h>
 #include "../../build.h"
 #include "../ConnectedClient.h"
 #include "../InternalClient.h"
-#include "../../server/file/FileServer.h"
+#include "src/server/file/LocalFileServer.h"
 #include "../../server/VoiceServer.h"
 #include "../voice/VoiceClient.h"
-#include "PermissionManager.h"
 #include "../../InstanceHandler.h"
 #include "../../server/QueryServer.h"
 #include "../file/FileClient.h"
@@ -25,25 +21,17 @@
 #include "../../weblist/WebListManager.h"
 #include "../../manager/ConversationManager.h"
 #include "../../manager/PermissionNameMapper.h"
-#include <experimental/filesystem>
-#include <cstdint>
-#include <StringVariable.h>
 
 #include "helpers.h"
+#include "./bulk_parsers.h"
 
-#include <Properties.h>
 #include <log/LogUtils.h>
-#include <misc/sassert.h>
 #include <misc/base64.h>
-#include <misc/hex.h>
 #include <misc/digest.h>
 #include <misc/rnd.h>
-#include <misc/timer.h>
-#include <misc/strobf.h>
 #include <misc/scope_guard.h>
 #include <bbcode/bbcodes.h>
 
-namespace fs = std::experimental::filesystem;
 using namespace std::chrono;
 using namespace std;
 using namespace ts;
@@ -279,16 +267,15 @@ command_result ConnectedClient::handleCommandServerRequestConnectionInfo(Command
     first_bulk.put_unchecked("connection_filetransfer_bytes_sent_month", this->server->properties()[property::VIRTUALSERVER_MONTH_BYTES_DOWNLOADED].as<string>());
     first_bulk.put_unchecked("connection_filetransfer_bytes_received_month", this->server->properties()[property::VIRTUALSERVER_MONTH_BYTES_UPLOADED].as<string>());
 
-    first_bulk.put_unchecked(property::CONNECTION_PACKETS_SENT_TOTAL, std::accumulate(total_stats.connection_packets_sent.begin(), total_stats.connection_packets_sent.end(), 0U));
-    first_bulk.put_unchecked(property::CONNECTION_BYTES_SENT_TOTAL, std::accumulate(total_stats.connection_bytes_sent.begin(), total_stats.connection_bytes_sent.end(), 0U));
-    first_bulk.put_unchecked(property::CONNECTION_PACKETS_RECEIVED_TOTAL, std::accumulate(total_stats.connection_packets_received.begin(), total_stats.connection_packets_received.end(), 0U));
-    first_bulk.put_unchecked(property::CONNECTION_BYTES_RECEIVED_TOTAL, std::accumulate(total_stats.connection_bytes_received.begin(), total_stats.connection_bytes_received.end(), 0U));
+    first_bulk.put_unchecked(property::CONNECTION_PACKETS_SENT_TOTAL, std::accumulate(total_stats.connection_packets_sent.begin(), total_stats.connection_packets_sent.end(), (size_t) 0U));
+    first_bulk.put_unchecked(property::CONNECTION_BYTES_SENT_TOTAL, std::accumulate(total_stats.connection_bytes_sent.begin(), total_stats.connection_bytes_sent.end(), (size_t) 0U));
+    first_bulk.put_unchecked(property::CONNECTION_PACKETS_RECEIVED_TOTAL, std::accumulate(total_stats.connection_packets_received.begin(), total_stats.connection_packets_received.end(), (size_t) 0U));
+    first_bulk.put_unchecked(property::CONNECTION_BYTES_RECEIVED_TOTAL, std::accumulate(total_stats.connection_bytes_received.begin(), total_stats.connection_bytes_received.end(), (size_t) 0U));
 
-
-    first_bulk.put_unchecked(property::CONNECTION_BANDWIDTH_SENT_LAST_SECOND_TOTAL, std::accumulate(second_report.connection_bytes_sent.begin(), second_report.connection_bytes_sent.end(), 0U));
-    first_bulk.put_unchecked(property::CONNECTION_BANDWIDTH_SENT_LAST_MINUTE_TOTAL, std::accumulate(minute_report.connection_bytes_sent.begin(), minute_report.connection_bytes_sent.end(), 0U));
-    first_bulk.put_unchecked(property::CONNECTION_BANDWIDTH_RECEIVED_LAST_SECOND_TOTAL, std::accumulate(second_report.connection_bytes_received.begin(), second_report.connection_bytes_received.end(), 0U));
-    first_bulk.put_unchecked(property::CONNECTION_BANDWIDTH_RECEIVED_LAST_MINUTE_TOTAL, std::accumulate(minute_report.connection_bytes_received.begin(), minute_report.connection_bytes_received.end(), 0U));
+    first_bulk.put_unchecked(property::CONNECTION_BANDWIDTH_SENT_LAST_SECOND_TOTAL, std::accumulate(second_report.connection_bytes_sent.begin(), second_report.connection_bytes_sent.end(), (size_t) 0U));
+    first_bulk.put_unchecked(property::CONNECTION_BANDWIDTH_SENT_LAST_MINUTE_TOTAL, std::accumulate(minute_report.connection_bytes_sent.begin(), minute_report.connection_bytes_sent.end(), (size_t) 0U));
+    first_bulk.put_unchecked(property::CONNECTION_BANDWIDTH_RECEIVED_LAST_SECOND_TOTAL, std::accumulate(second_report.connection_bytes_received.begin(), second_report.connection_bytes_received.end(), (size_t) 0U));
+    first_bulk.put_unchecked(property::CONNECTION_BANDWIDTH_RECEIVED_LAST_MINUTE_TOTAL, std::accumulate(minute_report.connection_bytes_received.begin(), minute_report.connection_bytes_received.end(), (size_t) 0U));
 
     first_bulk.put_unchecked(property::CONNECTION_CONNECTED_TIME, this->server->properties()[property::VIRTUALSERVER_UPTIME].as<string>());
     first_bulk.put_unchecked(property::CONNECTION_PACKETLOSS_TOTAL, network_report.average_loss);
@@ -551,7 +538,7 @@ command_result ConnectedClient::handleCommandServerGroupAddClient(Command &cmd)
     auto continue_on_error = cmd.hasParm("continueonerror");
     {
         auto permission_add_power = this->calculate_permission(permission::i_server_group_member_add_power, -1);
-        auto permission_self_add_power = this->calculate_permission(permission::i_server_group_member_add_power, -1);
+        auto permission_self_add_power = this->calculate_permission(permission::i_server_group_self_add_power, -1);
 
         for(auto index = 0; index < cmd.bulkCount(); index++) {
             auto group_id = cmd[index]["sgid"];
@@ -667,7 +654,7 @@ command_result ConnectedClient::handleCommandServerGroupDelClient(Command &cmd)
     auto continue_on_error = cmd.hasParm("continueonerror");
     {
         auto permission_remove_power = this->calculate_permission(permission::i_server_group_member_remove_power, -1);
-        auto permission_self_remove_power = this->calculate_permission(permission::i_server_group_member_remove_power, -1);
+        auto permission_self_remove_power = this->calculate_permission(permission::i_server_group_self_remove_power, -1);
 
         for(auto index = 0; index < cmd.bulkCount(); index++) {
             auto group_id = cmd[index]["sgid"];
@@ -755,6 +742,9 @@ command_result ConnectedClient::handleCommandServerGroupDelClient(Command &cmd)
 
     for(const auto& _server : target_server ? std::deque<shared_ptr<VirtualServer>>{target_server} : serverInstance->getVoiceServerManager()->serverInstances()) {
         for (const auto &targetClient : _server->findClientsByCldbId(target_cldbid)) {
+            ConnectedLockedClient clock{targetClient};
+            if(!clock) continue;
+            
             if (_server->notifyClientPropertyUpdates(targetClient, _server->groups->update_server_group_property(targetClient, true, targetClient->getChannel()))) {
                 for (const auto &client : _server->getClients()) {
                     if(client->isClientVisible(targetClient, true) || client == targetClient)
@@ -803,73 +793,43 @@ command_result ConnectedClient::handleCommandServerGroupAddPerm(Command &cmd) {
         ACTION_REQUIRES_GLOBAL_PERMISSION(permission::b_serverinstance_modify_templates, 1);
     }
 
+    command::bulk_parser::PermissionBulksParser<true> pparser{cmd};
+    if(!pparser.validate(this->ref(), 0))
+        return pparser.build_command_result();
 
-    auto max_value = this->calculate_permission(permission::i_permission_modify_power, 0, true);
-    if(!max_value.has_value) return command_result{permission::i_permission_modify_power};
-
-    auto ignore_granted_values = permission::v2::permission_granted(1, this->calculate_permission(permission::b_permission_modify_power_ignore, 0));
-
-    bool conOnError = cmd[0].has("continueonerror");
-    bool checkTp = false;
-    bool sgroupUpdate = false;
-
+    bool update_talk_power{false}, update_server_group_list{false};
     auto permissions = serverGroup->permissions();
-    for (int index = 0; index < cmd.bulkCount(); index++) {
-        PARSE_PERMISSION(cmd);
-        //permvalue='1' permnegated='0' permskip='0'
-
-        auto val = cmd[index]["permvalue"].as<permission::PermissionValue>();
-        if(permission_require_granted_value(permType) && !permission::v2::permission_granted(val, max_value)) {
-            if(conOnError) continue;
-            return command_result{permission::i_permission_modify_power};
-        }
-
-        if(!ignore_granted_values && !permission::v2::permission_granted(1, this->calculate_permission(permType, 0, true))) {
-            if(conOnError) continue;
-            return command_result{permission::i_permission_modify_power};
-        }
-
-        if (grant) {
-            permissions->set_permission(permType, {0, cmd[index]["permvalue"]}, permission::v2::PermissionUpdateType::do_nothing, permission::v2::PermissionUpdateType::set_value);
-        } else {
-            permissions->set_permission(
-                    permType,
-                    {cmd[index]["permvalue"], 0},
-                    permission::v2::PermissionUpdateType::set_value,
-                    permission::v2::PermissionUpdateType::do_nothing,
-
-                    cmd[index]["permskip"].as<bool>() ? 1 : 0,
-                    cmd[index]["permnegated"].as<bool>() ? 1 : 0
-            );
-            sgroupUpdate |= permission_is_group_property(permType);
-            checkTp |= permission_is_client_property(permType);
-        }
+    for(const auto& ppermission : pparser.iterate_valid_permissions()) {
+        ppermission.apply_to(serverGroup->permissions(), permission::v2::PermissionUpdateType::set_value);
+        update_server_group_list |= ppermission.is_group_property();
+        update_talk_power |= ppermission.is_client_view_property();
     }
 
-    if(sgroupUpdate)
+    if(update_server_group_list)
         serverGroup->apply_properties_from_permissions();
 
     //TODO may update for every server?
     if(this->server) {
         auto lock = this->_this.lock();
         auto server = this->server;
-        threads::Thread([checkTp, sgroupUpdate, serverGroup, lock, server]() {
-            if(sgroupUpdate)
+        threads::Thread([update_talk_power, update_server_group_list, serverGroup, lock, server]() {
+            if(update_server_group_list)
                 server->forEachClient([](shared_ptr<ConnectedClient> cl) {
                     cl->notifyServerGroupList();
                 });
-            server->forEachClient([serverGroup, checkTp](shared_ptr<ConnectedClient> cl) {
+            server->forEachClient([serverGroup, update_talk_power](shared_ptr<ConnectedClient> cl) {
                 if (cl->serverGroupAssigned(serverGroup)) {
                     if(cl->update_cached_permissions()) /* update cached calculated permissions */
                         cl->sendNeededPermissions(false); /* cached permissions had changed, notify the client */
-                    if (checkTp)
+                    if (update_talk_power)
                         cl->updateChannelClientProperties(true, true);
                     cl->join_state_id++; /* join permission may changed, all channels need to be recalculate dif needed */
                 }
             });
         }).detach();
     }
-    return command_result{error::ok};
+
+    return pparser.build_command_result();
 }
 
 command_result ConnectedClient::handleCommandServerGroupDelPerm(Command &cmd) {
@@ -888,49 +848,34 @@ command_result ConnectedClient::handleCommandServerGroupDelPerm(Command &cmd) {
         ACTION_REQUIRES_GLOBAL_PERMISSION(permission::b_serverinstance_modify_templates, 1);
     }
 
-    auto ignore_granted_values = permission::v2::permission_granted(1, this->calculate_permission(permission::b_permission_modify_power_ignore, 0));
-    bool conOnError = cmd[0].has("continueonerror");
-    bool checkTp = false;
-    auto sgroupUpdate = false;
-    for (int index = 0; index < cmd.bulkCount(); index++) {
-        PARSE_PERMISSION(cmd);
+    command::bulk_parser::PermissionBulksParser<false> pparser{cmd};
+    if(!pparser.validate(this->ref(), 0))
+        return pparser.build_command_result();
 
-        if(!ignore_granted_values && !permission::v2::permission_granted(0, this->calculate_permission(permType, 0, true))) {
-            if(conOnError) continue;
-            return command_result{permission::i_permission_modify_power};
-        }
-
-        if (grant) {
-            serverGroup->permissions()->set_permission(permType, permission::v2::empty_permission_values, permission::v2::PermissionUpdateType::do_nothing, permission::v2::PermissionUpdateType::delete_value);
-        } else {
-            serverGroup->permissions()->set_permission(
-                    permType,
-                    permission::v2::empty_permission_values,
-                    permission::v2::PermissionUpdateType::delete_value,
-                    permission::v2::PermissionUpdateType::do_nothing
-            );
-            sgroupUpdate |= permission_is_group_property(permType);
-            checkTp |= permission_is_client_property(permType);
-        }
+    bool update_talk_power{false}, update_server_group_list{false};
+    for(const auto& ppermission : pparser.iterate_valid_permissions()) {
+        ppermission.apply_to(serverGroup->permissions(), permission::v2::PermissionUpdateType::delete_value);
+        update_server_group_list |= ppermission.is_group_property();
+        update_talk_power |= ppermission.is_client_view_property();
     }
 
-    if(sgroupUpdate)
+    if(update_server_group_list)
         serverGroup->apply_properties_from_permissions();
 
     if(this->server) {
         auto lock = this->_this.lock();
         auto server = this->server;
-        threads::Thread([checkTp, sgroupUpdate, serverGroup, lock, server]() {
-            if(sgroupUpdate)
+        threads::Thread([update_talk_power, update_server_group_list, serverGroup, lock, server]() {
+            if(update_server_group_list)
                 server->forEachClient([](shared_ptr<ConnectedClient> cl) {
                     cl->notifyServerGroupList();
                 });
-            server->forEachClient([serverGroup, checkTp](shared_ptr<ConnectedClient> cl) {
+            server->forEachClient([serverGroup, update_talk_power](shared_ptr<ConnectedClient> cl) {
                 if (cl->serverGroupAssigned(serverGroup)) {
 
                     if(cl->update_cached_permissions()) /* update cached calculated permissions */
                         cl->sendNeededPermissions(false); /* cached permissions had changed, notify the client */
-                    if (checkTp)
+                    if (update_talk_power)
                         cl->updateChannelClientProperties(true, true);
                     cl->join_state_id++; /* join permission may changed, all channels need to be recalculate dif needed */
                 }
@@ -938,7 +883,7 @@ command_result ConnectedClient::handleCommandServerGroupDelPerm(Command &cmd) {
         }).detach();
     }
 
-    return command_result{error::ok};
+    return pparser.build_command_result();
 }
 
 command_result ConnectedClient::handleCommandServerGroupAutoAddPerm(ts::Command& cmd) {
@@ -968,67 +913,37 @@ command_result ConnectedClient::handleCommandServerGroupAutoAddPerm(ts::Command&
     if(groups.empty())
         return command_result{error::ok};
 
-    auto max_value = this->calculate_permission(permission::i_permission_modify_power, 0, true);
-    if(!max_value.has_value) return command_result{permission::i_permission_modify_power};
+    command::bulk_parser::PermissionBulksParser<true> pparser{cmd};
+    if(!pparser.validate(this->ref(), 0))
+        return pparser.build_command_result();
 
-    auto ignore_granted_values = permission::v2::permission_granted(1, this->calculate_permission(permission::b_permission_modify_power_ignore, 0));
+    bool update_clients{false}, update_server_group_list{false};
+    for(const auto& ppermission : pparser.iterate_valid_permissions()) {
+        for(const auto& serverGroup : groups)
+            ppermission.apply_to(serverGroup->permissions(), permission::v2::PermissionUpdateType::set_value);
 
-    bool conOnError = cmd[0].has("continueonerror");
-    bool checkTp = false;
-    bool sgroupUpdate = false;
-
-    for (int index = 0; index < cmd.bulkCount(); index++) {
-        PARSE_PERMISSION(cmd);
-        //permvalue='1' permnegated='0' permskip='0'end
-
-        auto val = cmd[index]["permvalue"].as<permission::PermissionValue>();
-        if(permission_require_granted_value(permType) && !permission::v2::permission_granted(val, max_value)) {
-            if(conOnError) continue;
-            return command_result{permission::i_permission_modify_power};
-        }
-
-        if(!ignore_granted_values && !permission::v2::permission_granted(1, this->calculate_permission(permType, 0, true))) {
-            if(conOnError) continue;
-            return command_result{permission::i_permission_modify_power};
-        }
-
-        for(const auto& serverGroup : groups) {
-            if (grant) {
-                serverGroup->permissions()->set_permission(permType, {0, cmd[index]["permvalue"]}, permission::v2::PermissionUpdateType::do_nothing, permission::v2::PermissionUpdateType::set_value);
-            } else {
-                serverGroup->permissions()->set_permission(
-                        permType,
-                        {cmd[index]["permvalue"], 0},
-                        permission::v2::PermissionUpdateType::set_value,
-                        permission::v2::PermissionUpdateType::do_nothing,
-
-                        cmd[index]["permskip"].as<bool>() ? 1 : 0,
-                        cmd[index]["permnegated"].as<bool>() ? 1 : 0
-                );
-            }
-        }
-        sgroupUpdate |= permission_is_group_property(permType);
-        checkTp |= permission_is_client_property(permType);
+        update_server_group_list |= ppermission.is_group_property();
+        update_clients |= ppermission.is_client_view_property();
     }
 
-    if(sgroupUpdate)
+    if(update_server_group_list)
         for(auto& group : groups)
             group->apply_properties_from_permissions();
 
     auto lock = this->_this.lock();
     if(ref_server) {
-        threads::Thread([checkTp, sgroupUpdate, groups, lock, ref_server]() {
-            if(sgroupUpdate)
+        threads::Thread([update_clients, update_server_group_list, groups, lock, ref_server]() {
+            if(update_server_group_list)
                 ref_server->forEachClient([](shared_ptr<ConnectedClient> cl) {
                     cl->notifyServerGroupList();
                 });
-            ref_server->forEachClient([groups, checkTp](shared_ptr<ConnectedClient> cl) {
+            ref_server->forEachClient([groups, update_clients](shared_ptr<ConnectedClient> cl) {
                 for(const auto& serverGroup : groups) {
                     if (cl->serverGroupAssigned(serverGroup)) {
                         if(cl->update_cached_permissions()) {/* update cached calculated permissions */
                             cl->sendNeededPermissions(false); /* cached permissions had changed, notify the client */
                         }
-                        if (checkTp) {
+                        if (update_clients) {
                             cl->updateChannelClientProperties(true, true);
                         }
                         cl->join_state_id++; /* join permission may changed, all channels need to be recalculate if needed */
@@ -1038,7 +953,8 @@ command_result ConnectedClient::handleCommandServerGroupAutoAddPerm(ts::Command&
             });
         }).detach();
     }
-    return command_result{error::ok};
+
+    return pparser.build_command_result();
 }
 
 command_result ConnectedClient::handleCommandServerGroupAutoDelPerm(ts::Command& cmd) {
@@ -1067,54 +983,37 @@ command_result ConnectedClient::handleCommandServerGroupAutoDelPerm(ts::Command&
 
     if(groups.empty()) return command_result{error::ok};
 
-    auto ignore_granted_values = permission::v2::permission_granted(1, this->calculate_permission(permission::b_permission_modify_power_ignore, 0));
+    command::bulk_parser::PermissionBulksParser<false> pparser{cmd};
+    if(!pparser.validate(this->ref(), 0))
+        return pparser.build_command_result();
 
-    bool conOnError = cmd[0].has("continueonerror");
-    bool checkTp = false;
-    auto sgroupUpdate = false;
-    for (int index = 0; index < cmd.bulkCount(); index++) {
-        PARSE_PERMISSION(cmd);
+    bool update_clients{false}, update_server_group_list{false};
+    for(const auto& ppermission : pparser.iterate_valid_permissions()) {
+        for(const auto& serverGroup : groups)
+            ppermission.apply_to(serverGroup->permissions(), permission::v2::PermissionUpdateType::delete_value);
 
-        if(!ignore_granted_values && !permission::v2::permission_granted(0, this->calculate_permission(permType, 0, true))) {
-            if(conOnError) continue;
-            return command_result{permission::i_permission_modify_power};
-        }
-
-        for(const auto& serverGroup : groups) {
-            if (grant) {
-                serverGroup->permissions()->set_permission(permType, permission::v2::empty_permission_values, permission::v2::PermissionUpdateType::do_nothing, permission::v2::PermissionUpdateType::delete_value);
-            } else {
-                serverGroup->permissions()->set_permission(
-                        permType,
-                        permission::v2::empty_permission_values,
-                        permission::v2::PermissionUpdateType::delete_value,
-                        permission::v2::PermissionUpdateType::do_nothing
-                );
-                sgroupUpdate |= permission_is_group_property(permType);
-            }
-        }
-        checkTp |= permission_is_client_property(permType);
+        update_server_group_list |= ppermission.is_group_property();
+        update_clients |= ppermission.is_client_view_property();
     }
 
-
-    if(sgroupUpdate) {
+    if(update_server_group_list) {
         for(auto& group : groups)
             group->apply_properties_from_permissions();
     }
 
     if(ref_server) {
         auto lock = this->_this.lock();
-        threads::Thread([checkTp, sgroupUpdate, groups, lock, ref_server]() {
-            if(sgroupUpdate)
+        threads::Thread([update_clients, update_server_group_list, groups, lock, ref_server]() {
+            if(update_server_group_list)
                 ref_server->forEachClient([](shared_ptr<ConnectedClient> cl) {
                     cl->notifyServerGroupList();
                 });
-            ref_server->forEachClient([groups, checkTp](shared_ptr<ConnectedClient> cl) {
+            ref_server->forEachClient([groups, update_clients](shared_ptr<ConnectedClient> cl) {
                 for(const auto& serverGroup : groups) {
                     if (cl->serverGroupAssigned(serverGroup)) {
                         if(cl->update_cached_permissions()) /* update cached calculated permissions */
                             cl->sendNeededPermissions(false); /* cached permissions had changed, notify the client */
-                        if (checkTp)
+                        if (update_clients)
                             cl->updateChannelClientProperties(true, true);
                         cl->join_state_id++; /* join permission may changed, all channels need to be recalculate dif needed */
                         break;
@@ -1124,7 +1023,7 @@ command_result ConnectedClient::handleCommandServerGroupAutoDelPerm(ts::Command&
         }).detach();
     }
 
-    return command_result{error::ok};
+    return pparser.build_command_result();
 }
 
 command_result ConnectedClient::handleCommandServerGroupsByClientId(Command &cmd) {

server/src/client/file/FileClient.cpp

@@ -1,5 +1,5 @@
 #include <algorithm>
-#include <src/server/file/FileServer.h>
+#include <src/server/file/LocalFileServer.h>
 #include <log/LogUtils.h>
 #include "FileClient.h"
 #include <src/InstanceHandler.h>
@@ -15,7 +15,7 @@ using namespace ts::server;
 namespace fs = std::experimental::filesystem;
 
 #define BUFFER_SIZE (size_t) 2048
-FileClient::FileClient(FileServer* handle, int socketFd) : handle(handle), clientFd(socketFd) {
+FileClient::FileClient(LocalFileServer* handle, int socketFd) : handle(handle), clientFd(socketFd) {
     memtrack::allocated<FileClient>(this);
     this->last_io_action = system_clock::now();
 

server/src/client/file/FileClient.h

@@ -3,7 +3,7 @@
 #include <protocol/buffers.h>
 #include <poll.h>
 #include <fstream>
-#include <src/server/file/FileServer.h>
+#include <src/server/file/LocalFileServer.h>
 #include <event.h>
 #include <pipes/ws.h>
 #include <pipes/ssl.h>
@@ -14,7 +14,7 @@ namespace ts {
         class ConnectedClient;
 
         class ConnectedClient;
-        class FileServer;
+        class LocalFileServer;
 
         enum FTType {
                 Unknown,
@@ -28,7 +28,7 @@ namespace ts {
         };
 
         class FileClient {
-                friend class FileServer;
+                friend class LocalFileServer;
             public:
                 enum TransferState {
                     T_INITIALIZE,
@@ -45,7 +45,7 @@ namespace ts {
                     uint16_t length = 0;
                 };
 
-                FileClient(FileServer* handle, int socketFd);
+                FileClient(LocalFileServer* handle, int socketFd);
                 ~FileClient();
 
                 void disconnect(std::chrono::milliseconds = std::chrono::milliseconds(5000));
@@ -83,7 +83,7 @@ namespace ts {
                 void handle_ws_message(const pipes::WSMessage&);
                 bool handle_ts_message();
             private:
-                FileServer* handle;
+                LocalFileServer* handle;
                 std::weak_ptr<FileClient> _this;
 
                 std::recursive_mutex bandwidth_lock;

server/src/client/file/FileClientIO.cpp

@@ -1,6 +1,6 @@
 #include <algorithm>
 #include <memory>
-#include <src/server/file/FileServer.h>
+#include <src/server/file/LocalFileServer.h>
 #include <log/LogUtils.h>
 #include <misc/std_unique_ptr.h>
 #include <pipes/buffer.h>

server/src/client/music/internal_provider/channel_replay/ChannelProvider.cpp

@@ -1,7 +1,7 @@
 #include "ChannelProvider.h"
 #include "../../MusicClient.h"
 #include "../../../../InstanceHandler.h"
-#include "../../../../server/file/FileServer.h"
+#include "src/server/file/LocalFileServer.h"
 #include "../../../../../../music/providers/ffmpeg/FFMpegProvider.h"
 
 

server/src/client/query/QueryClient.cpp

@@ -77,7 +77,7 @@ void QueryClient::postInitialize() {
     if(ts::config::query::sslMode == 1 && this->connectionType != ConnectionType::SSL_ENCRIPTED) {
         command_result error{error::failed_connection_initialisation,  "Please use a SSL encryption!"};
         this->notifyError(error);
-        error.release_details();
+        error.release_data();
         this->disconnect("Please us a SSL encryption for more security.\nThe server denies also all other connections!");
         return;
     }
@@ -501,25 +501,25 @@ bool QueryClient::handleMessage(const pipes::buffer_view& message) {
         cmd = make_unique<Command>(Command::parse(pipes::buffer_view{(void*) command.data(), command.length()}, true, !ts::config::server::strict_ut8_mode));
     } catch(std::invalid_argument& ex) {
         logTrace(LOG_QUERY, "[{}:{}] Failed to parse command (invalid argument): {}", this->getLoggingPeerIp(), this->getPeerPort(), command);
-        error = command_result{error::parameter_convert};
+        error.reset(command_result{error::parameter_convert});
         goto handle_error;
     } catch(std::exception& ex) {
         logTrace(LOG_QUERY, "[{}:{}] Failed to parse command (exception: {}): {}", this->getLoggingPeerIp(), this->getPeerPort(), ex.what(), command);
-        error = command_result{error::vs_critical, std::string{ex.what()}};
+        error.reset(command_result{error::vs_critical, std::string{ex.what()}});
         goto handle_error;
     }
 
     try {
         this->handleCommandFull(*cmd);
     } catch(std::exception& ex) {
-        error = command_result{error::vs_critical, std::string{ex.what()}};
+        error.reset(command_result{error::vs_critical, std::string{ex.what()}});
         goto handle_error;
     }
     return true;
 
     handle_error:
     this->notifyError(error);
-    error.release_details();
+    error.release_data();
     return false;
 }
 

server/src/client/query/QueryClientCommands.cpp

@@ -101,12 +101,15 @@ command_result QueryClient::handleCommand(Command& cmd) {
         case string_hash("bindinglist"):
             return this->handleCommandBindingList(cmd);
         case string_hash("serversnapshotdeploy"): {
-            //return this->handleCommandServerSnapshotDeploy(cmd);
+#if 1
+            return this->handleCommandServerSnapshotDeploy(cmd);
+#else
             auto cmd_str = cmd.build();
             ts::command_parser parser{cmd_str};
             if(!parser.parse(true)) return command_result{error::vs_critical};
 
             return this->handleCommandServerSnapshotDeployNew(parser);
+#endif
         }
         case string_hash("serversnapshotcreate"):
             return this->handleCommandServerSnapshotCreate(cmd);
@@ -171,8 +174,8 @@ command_result QueryClient::handleCommandLogin(Command& cmd) {
     if(!this->properties()[property::CLIENT_LOGIN_NAME].as<string>().empty()) {
         Command log("logout");
         auto result = this->handleCommandLogout(log);
-        if(result.error_code()) {
-            result.release_details();
+        if(result.has_error()) {
+            result.release_data();
             logError(this->getServerId(), "Query client failed to login from old login.");
             return command_result{error::vs_critical};
         }
@@ -406,13 +409,15 @@ command_result QueryClient::handleCommandServerInfo(Command &) {
         auto total_stats = this->server->getServerStatistics()->total_stats();
         auto report_second = this->server->serverStatistics->second_stats();
         auto report_minute = this->server->serverStatistics->minute_stats();
-        cmd["connection_bandwidth_sent_last_second_total"] = std::accumulate(report_second.connection_bytes_sent.begin(), report_second.connection_bytes_sent.end(), 0U);
-        cmd["connection_bandwidth_sent_last_minute_total"] = std::accumulate(report_minute.connection_bytes_sent.begin(), report_minute.connection_bytes_sent.end(), 0U);
-        cmd["connection_bandwidth_received_last_second_total"] = std::accumulate(report_second.connection_bytes_received.begin(), report_second.connection_bytes_received.end(), 0U);
-        cmd["connection_bandwidth_received_last_minute_total"] = std::accumulate(report_minute.connection_bytes_received.begin(), report_minute.connection_bytes_received.end(), 0U);
+        cmd["connection_bandwidth_sent_last_second_total"] = std::accumulate(report_second.connection_bytes_sent.begin(), report_second.connection_bytes_sent.end(), (size_t) 0U);
+        cmd["connection_bandwidth_sent_last_minute_total"] = std::accumulate(report_minute.connection_bytes_sent.begin(), report_minute.connection_bytes_sent.end(), (size_t) 0U);
+        cmd["connection_bandwidth_received_last_second_total"] = std::accumulate(report_second.connection_bytes_received.begin(), report_second.connection_bytes_received.end(), (size_t) 0U);
+        cmd["connection_bandwidth_received_last_minute_total"] = std::accumulate(report_minute.connection_bytes_received.begin(), report_minute.connection_bytes_received.end(), (size_t) 0U);
 
         cmd["connection_filetransfer_bandwidth_sent"] = report_minute.file_bytes_sent;
         cmd["connection_filetransfer_bandwidth_received"] = report_minute.file_bytes_received;
+        cmd["connection_filetransfer_bytes_sent_total"] = total_stats.file_bytes_sent;
+        cmd["connection_filetransfer_bytes_received_total"] = total_stats.file_bytes_received;
 
         cmd["connection_packets_sent_speech"] = total_stats.connection_packets_sent[stats::ConnectionStatistics::category::VOICE];
         cmd["connection_bytes_sent_speech"] = total_stats.connection_bytes_sent[stats::ConnectionStatistics::category::VOICE];
@@ -429,10 +434,10 @@ command_result QueryClient::handleCommandServerInfo(Command &) {
         cmd["connection_packets_received_control"] = total_stats.connection_packets_received[stats::ConnectionStatistics::category::COMMAND];
         cmd["connection_bytes_received_control"] = total_stats.connection_bytes_received[stats::ConnectionStatistics::category::COMMAND];
 
-        cmd["connection_packets_sent_total"] = std::accumulate(report_second.connection_packets_sent.begin(), report_second.connection_packets_sent.end(), 0U);
-        cmd["connection_bytes_sent_total"] = std::accumulate(report_second.connection_bytes_sent.begin(), report_second.connection_bytes_sent.end(), 0U);
-        cmd["connection_packets_received_total"] = std::accumulate(report_second.connection_packets_received.begin(), report_second.connection_packets_received.end(), 0U);
-        cmd["connection_bytes_received_total"] = std::accumulate(report_second.connection_bytes_received.begin(), report_second.connection_bytes_received.end(), 0U);
+        cmd["connection_packets_sent_total"] = std::accumulate(total_stats.connection_packets_sent.begin(), total_stats.connection_packets_sent.end(), (size_t) 0U);
+        cmd["connection_bytes_sent_total"] = std::accumulate(total_stats.connection_bytes_sent.begin(), total_stats.connection_bytes_sent.end(), (size_t) 0U);
+        cmd["connection_packets_received_total"] = std::accumulate(total_stats.connection_packets_received.begin(), total_stats.connection_packets_received.end(), (size_t) 0U);
+        cmd["connection_bytes_received_total"] = std::accumulate(total_stats.connection_bytes_received.begin(), total_stats.connection_bytes_received.end(), (size_t) 0U);
     } else {
         cmd["connection_bandwidth_sent_last_second_total"] = "0";
         cmd["connection_bandwidth_sent_last_minute_total"] = "0";
@@ -481,11 +486,13 @@ command_result QueryClient::handleCommandChannelList(Command& cmd) {
     command_builder result{"", 1024, entries.size()};
     for(const auto& channel : entries){
         if(!channel) continue;
+
+        const auto channel_clients = this->server ? this->server->getClientsByChannel(channel).size() : 0;
         result.put_unchecked(index, "cid", channel->channelId());
         result.put_unchecked(index, "pid", channel->properties()[property::CHANNEL_PID].as<string>());
         result.put_unchecked(index, "channel_name", channel->name());
         result.put_unchecked(index, "channel_order", channel->channelOrder());
-        result.put_unchecked(index, "total_clients", this->server ? this->server->getClientsByChannel(channel).size() : 0);
+        result.put_unchecked(index, "total_clients", channel_clients);
         /* result.put_unchecked(index, "channel_needed_subscribe_power", channel->permissions()->getPermissionValue(permission::i_channel_needed_subscribe_power, channel, 0)); */
 
         if(cmd.hasParm("flags")){
@@ -517,8 +524,8 @@ command_result QueryClient::handleCommandChannelList(Command& cmd) {
         if(cmd.hasParm("topic")) {
             result.put_unchecked(index, "channel_topic", channel->properties()[property::CHANNEL_TOPIC].as<string>());
         }
-        if(cmd.hasParm("times")){
-            result.put_unchecked(index, "seconds_empty", channel->emptySince());
+        if(cmd.hasParm("times") || cmd.hasParm("secondsempty")){
+            result.put_unchecked(index, "seconds_empty", channel_clients == 0 ? channel->emptySince() : 0);
         }
         index++;
     }
@@ -587,9 +594,9 @@ command_result QueryClient::handleCommandServerCreate(Command& cmd) {
             time_wait = duration_cast<milliseconds>(end - start);
         }
 
-        uint16_t freePort = serverInstance->getVoiceServerManager()->next_available_port();
-
         std::string host = cmd[0].has("virtualserver_host") ? cmd["virtualserver_host"].as<string>() : config::binding::DefaultVoiceHost;
+        uint16_t freePort = serverInstance->getVoiceServerManager()->next_available_port(host);
+
         uint16_t port = cmd[0].has("virtualserver_port") ? cmd["virtualserver_port"].as<uint16_t>() : freePort;
         {
             auto _start = system_clock::now();
@@ -764,17 +771,17 @@ command_result QueryClient::handleCommandHostInfo(Command &) {
 
 
     auto total_stats = serverInstance->getStatistics()->total_stats();
-    res["connection_packets_sent_total"] = std::accumulate(total_stats.connection_packets_sent.begin(), total_stats.connection_packets_sent.end(), 0U);
-    res["connection_bytes_sent_total"] = std::accumulate(total_stats.connection_bytes_sent.begin(), total_stats.connection_bytes_sent.end(), 0U);
-    res["connection_packets_received_total"] = std::accumulate(total_stats.connection_packets_received.begin(), total_stats.connection_packets_received.end(), 0U);
-    res["connection_bytes_received_total"] = std::accumulate(total_stats.connection_bytes_received.begin(), total_stats.connection_bytes_received.end(), 0U);
+    res["connection_packets_sent_total"] = std::accumulate(total_stats.connection_packets_sent.begin(), total_stats.connection_packets_sent.end(), (size_t) 0U);
+    res["connection_bytes_sent_total"] = std::accumulate(total_stats.connection_bytes_sent.begin(), total_stats.connection_bytes_sent.end(), (size_t) 0U);
+    res["connection_packets_received_total"] = std::accumulate(total_stats.connection_packets_received.begin(), total_stats.connection_packets_received.end(), (size_t) 0U);
+    res["connection_bytes_received_total"] = std::accumulate(total_stats.connection_bytes_received.begin(), total_stats.connection_bytes_received.end(), (size_t) 0U);
 
     auto report_second = serverInstance->getStatistics()->second_stats();
     auto report_minute = serverInstance->getStatistics()->minute_stats();
-    res["connection_bandwidth_sent_last_second_total"] = std::accumulate(report_second.connection_bytes_sent.begin(), report_second.connection_bytes_sent.end(), 0U);
-    res["connection_bandwidth_sent_last_minute_total"] = std::accumulate(report_minute.connection_bytes_sent.begin(), report_minute.connection_bytes_sent.end(), 0U);
-    res["connection_bandwidth_received_last_second_total"] = std::accumulate(report_second.connection_bytes_received.begin(), report_second.connection_bytes_received.end(), 0U);
-    res["connection_bandwidth_received_last_minute_total"] = std::accumulate(report_minute.connection_bytes_received.begin(), report_minute.connection_bytes_received.end(), 0U);
+    res["connection_bandwidth_sent_last_second_total"] = std::accumulate(report_second.connection_bytes_sent.begin(), report_second.connection_bytes_sent.end(), (size_t) 0U);
+    res["connection_bandwidth_sent_last_minute_total"] = std::accumulate(report_minute.connection_bytes_sent.begin(), report_minute.connection_bytes_sent.end(), (size_t) 0U);
+    res["connection_bandwidth_received_last_second_total"] = std::accumulate(report_second.connection_bytes_received.begin(), report_second.connection_bytes_received.end(), (size_t) 0U);
+    res["connection_bandwidth_received_last_minute_total"] = std::accumulate(report_minute.connection_bytes_received.begin(), report_minute.connection_bytes_received.end(), (size_t) 0U);
 
 
     res["connection_filetransfer_bandwidth_sent"] = report_minute.file_bytes_sent;
@@ -844,7 +851,7 @@ command_result QueryClient::handleCommandServerSnapshotDeploy(Command& cmd) {
 
     unique_lock server_create_lock(serverInstance->getVoiceServerManager()->server_create_lock);
     if(port == 0)
-        port = serverInstance->getVoiceServerManager()->next_available_port();
+        port = serverInstance->getVoiceServerManager()->next_available_port(host);
     auto result = serverInstance->getVoiceServerManager()->createServerFromSnapshot(this->server, host, port, cmd, error);
     server_create_lock.unlock();
     auto end = system_clock::now();

server/src/client/voice/PacketStatistics.h

@@ -2,7 +2,7 @@
 
 #include <protocol/PacketLossCalculator.h>
 #include <protocol/Packet.h>
-#include <misc/spin_lock.h>
+#include <misc/spin_mutex.h>
 
 namespace ts::server::client {
     class PacketStatistics {
@@ -53,7 +53,7 @@ namespace ts::server::client {
         private:
             std::chrono::system_clock::time_point last_short{};
 
-            spin_lock data_mutex{};
+            spin_mutex data_mutex{};
             protocol::UnorderedPacketLossCalculator calculator_voice_whisper{};
             protocol::UnorderedPacketLossCalculator calculator_voice{};
 

server/src/client/voice/PrecomputedPuzzles.cpp

@@ -24,8 +24,20 @@ bool PuzzleManager::precompute_puzzles(size_t amount) {
 }
 
 std::shared_ptr<Puzzle> PuzzleManager::next_puzzle() {
-    std::lock_guard lock{this->cache_lock};
-    return this->cached_puzzles[this->cache_index++ % this->cached_puzzles.size()];
+    {
+        std::lock_guard lock{this->cache_lock};
+        auto it = this->cached_puzzles.begin() + (this->cache_index++ % this->cached_puzzles.size());
+        if((*it)->fail_count > 2) {
+            this->cached_puzzles.erase(it);
+        } else {
+            return *it;
+        }
+    }
+
+    std::random_device rd{};
+    std::mt19937 mt{rd()};
+    this->generate_puzzle(mt);
+    return this->next_puzzle();
 }
 
 inline void random_number(std::mt19937& generator, mp_int *result, int length){
@@ -74,8 +86,18 @@ void PuzzleManager::generate_puzzle(std::mt19937& random_generator) {
     mp_init_multi(&puzzle->x, &puzzle->n, &puzzle->result, nullptr);
 
     generate_new:
-    random_number(random_generator, &puzzle->x, 64);
-    random_number(random_generator, &puzzle->n, 64);
+    static int x{0};
+    if(x++ == 0 || true) {
+        mp_set(&puzzle->x, 1);
+        mp_set(&puzzle->n, 1);
+        //random_number(random_generator, &puzzle->x, 64);
+        //random_number(random_generator, &puzzle->n, 64);
+    } else {
+        const static std::string_view n_{"\x01\x7a\xc5\x8d\x28\x7a\x61\x58\xf6\xe3\x98\x60\x2f\x81\x9c\x8a\x48\xc9\x20\xd1\x59\xe0\x24\x75\x91\x27\x9f\x52\x1e\x2c\x24\x85\xa9\xdc\x74\xfa\x0b\x36\xf9\x6c\x77\xa3\x7c\xf9\xbb\xf7\x04\xad\xa3\x84\x0d\x97\x25\x54\x19\x72\x4f\x8f\xfc\x66\xbe\x41\xda\x95"};
+        const static std::string_view x_{"\xd1\xef\xf0\x16\x34\x48\x56\x53\x15\x97\xa0\x28\xbd\x13\xce\xbf\xc2\xd6\x79\x9d\x21\x81\x83\x37\x8c\xe8\xee\xee\xa1\x22\xa4\xf5\x63\x33\x53\x0c\x38\x2f\x0a\x00\x53\x20\xc7\x93\x52\xa9\xd0\xc2\xfb\xbc\xc5\xc4\xc3\x54\xad\xcb\x49\x52\xc0\xd8\x97\x32\x94\xee"};
+        mp_read_unsigned_bin(&puzzle->x, (unsigned char*) x_.data(), x_.length());
+        mp_read_unsigned_bin(&puzzle->n, (unsigned char*) n_.data(), n_.length());
+    }
 
     if(!solve_puzzle(puzzle))
         goto generate_new;

server/src/client/voice/PrecomputedPuzzles.h

@@ -3,20 +3,22 @@
 #include <tommath.h>
 #include <memory>
 #include <vector>
-#include <misc/spin_lock.h>
+#include <misc/spin_mutex.h>
 #include <random>
 
 namespace ts::server::udp {
     struct Puzzle {
-        mp_int x;
-        mp_int n;
-        int level;
+        mp_int x{};
+        mp_int n{};
+        int level{0};
 
-        mp_int result;
+        mp_int result{};
 
-        uint8_t data_x[64];
-        uint8_t data_n[64];
-        uint8_t data_result[64];
+        uint8_t data_x[64]{0};
+        uint8_t data_n[64]{0};
+        uint8_t data_result[64]{0};
+
+        size_t fail_count{0};
     };
 
     class PuzzleManager {
@@ -33,7 +35,7 @@ namespace ts::server::udp {
             void generate_puzzle(std::mt19937&);
 
             size_t cache_index{0};
-            spin_lock cache_lock{};
+            spin_mutex cache_lock{};
             std::vector<std::shared_ptr<Puzzle>> cached_puzzles{};
     };
 }

server/src/client/voice/VoiceClient.cpp

@@ -47,21 +47,8 @@ VoiceClient::~VoiceClient() {
     memtrack::freed<VoiceClient>(this);
 }
 
-void VoiceClient::sendCommand0(const std::string_view& cmd, bool low, bool direct, std::unique_ptr<threads::Future<bool>> listener) {
-    if(cmd.empty()) {
-        logCritical(this->getServerId(), "{} Attempted to send an empty command!", CLIENT_STR_LOG_PREFIX);
-        return;
-    }
-
-    auto packet = make_shared<protocol::ServerPacket>(
-            low ? protocol::PacketTypeInfo::CommandLow : protocol::PacketTypeInfo::Command,
-            pipes::buffer_view{(void*) cmd.data(), cmd.length()}
-    );
-    if(low) {
-        packet->enable_flag(protocol::PacketFlag::NewProtocol);
-    }
-    packet->setListener(std::move(listener));
-    this->connection->sendPacket(packet, false, direct);
+void VoiceClient::sendCommand0(const std::string_view& cmd, bool low, std::unique_ptr<threads::Future<bool>> listener) {
+    this->connection->send_command(cmd, low, std::move(listener));
 
 #ifdef PKT_LOG_CMD
     logTrace(this->getServerId(), "{}[Command][Server -> Client] Sending command {}. Command low: {}. Full command: {}", CLIENT_STR_LOG_PREFIX, cmd.substr(0, cmd.find(' ')), low, cmd);
@@ -71,10 +58,8 @@ void VoiceClient::sendAcknowledge(uint16_t packetId, bool low) {
     char buffer[2];
     le2be16(packetId, buffer);
 
-    auto packet = make_shared<protocol::ServerPacket>(low ? protocol::PacketTypeInfo::AckLow : protocol::PacketTypeInfo::Ack, pipes::buffer_view{buffer, 2});
-    packet->enable_flag(PacketFlag::Unencrypted);
-    if(!low) packet->enable_flag(protocol::PacketFlag::NewProtocol);
-    this->connection->sendPacket(packet);
+    auto pflags = PacketFlag::Unencrypted | PacketFlag::NewProtocol;
+    this->connection->send_packet(low ? protocol::PacketType::ACK_LOW : protocol::PacketType::ACK, (PacketFlag::PacketFlag) pflags, buffer, 2);
 #ifdef PKT_LOG_ACK
     logTrace(this->getServerId(), "{}[Acknowledge][Server -> Client] Sending acknowledge for {}", CLIENT_STR_LOG_PREFIX, packetId);
 #endif
@@ -186,7 +171,7 @@ bool VoiceClient::disconnect(ts::ViewReasonId reason_id, const std::string &reas
 
             self->close_connection(chrono::system_clock::time_point{}); /* we received the ack, we do not need to flush anything */
         }, system_clock::now() + seconds(5));
-        this->sendCommand0(cmd.build(), false, false, std::move(listener));
+        this->sendCommand0(cmd.build(), false, std::move(listener));
     } else {
         //TODO: Extra case for INIT_HIGH?
         this->close_connection(chrono::system_clock::now() + chrono::seconds{5});
@@ -284,33 +269,23 @@ void VoiceClient::execute_handle_packet(const std::chrono::system_clock::time_po
 }
 
 void VoiceClient::send_voice_packet(const pipes::buffer_view &voice_buffer, const SpeakingClient::VoicePacketFlags &flags) {
-    auto packet = make_shared<ServerPacket>(PacketTypeInfo::Voice, voice_buffer.length());
-    {
-        PacketFlag::PacketFlags packet_flags = PacketFlag::None;
-        packet_flags |= flags.encrypted ? 0 : PacketFlag::Unencrypted;
-        packet_flags |= flags.head ? PacketFlag::Compressed : 0;
-        packet_flags |= flags.fragmented ? PacketFlag::Fragmented : 0;
-        packet_flags |= flags.new_protocol ? PacketFlag::NewProtocol : 0;
-        packet->set_flags(packet_flags);
-    }
+    PacketFlag::PacketFlags packet_flags{PacketFlag::None};
+    packet_flags |= flags.encrypted ? 0U : PacketFlag::Unencrypted;
+    packet_flags |= flags.head ? PacketFlag::Compressed : 0U;
+    packet_flags |= flags.fragmented ? PacketFlag::Fragmented : 0U;
+    packet_flags |= flags.new_protocol ? PacketFlag::NewProtocol : 0U;
 
-    memcpy(packet->data().data_ptr<void>(), voice_buffer.data_ptr<void>(), voice_buffer.length());
-    this->connection->sendPacket(packet, false, false);
+    this->connection->send_packet(PacketType::VOICE, packet_flags, voice_buffer.data_ptr<void>(), voice_buffer.length());
 }
 
 void VoiceClient::send_voice_whisper_packet(const pipes::buffer_view &voice_buffer, const SpeakingClient::VoicePacketFlags &flags) {
-    auto packet = make_shared<ServerPacket>(PacketTypeInfo::VoiceWhisper, voice_buffer.length());
-    {
-        PacketFlag::PacketFlags packet_flags = PacketFlag::None;
-        packet_flags |= flags.encrypted ? 0 : PacketFlag::Unencrypted;
-        packet_flags |= flags.head ? PacketFlag::Compressed : 0;
-        packet_flags |= flags.fragmented ? PacketFlag::Fragmented : 0;
-        packet_flags |= flags.new_protocol ? PacketFlag::NewProtocol : 0;
-        packet->set_flags(packet_flags);
-    }
+    PacketFlag::PacketFlags packet_flags{PacketFlag::None};
+    packet_flags |= flags.encrypted ? 0U : PacketFlag::Unencrypted;
+    packet_flags |= flags.head ? PacketFlag::Compressed : 0U;
+    packet_flags |= flags.fragmented ? PacketFlag::Fragmented : 0U;
+    packet_flags |= flags.new_protocol ? PacketFlag::NewProtocol : 0U;
 
-    memcpy(packet->data().data_ptr<void>(), voice_buffer.data_ptr<void>(), voice_buffer.length());
-    this->connection->sendPacket(packet, false, false);
+    this->connection->send_packet(PacketType::VOICE_WHISPER, packet_flags, voice_buffer.data_ptr<void>(), voice_buffer.length());
 }
 
 float VoiceClient::current_ping_deviation() {

server/src/client/voice/VoiceClient.h

@@ -58,7 +58,7 @@ namespace ts {
                 virtual void sendCommand(const ts::command_builder &command, bool low) { return this->sendCommand0(command.build(), low); }
 
                 /* Note: Order is only guaranteed if progressDirectly is on! */
-                virtual void sendCommand0(const std::string_view& /* data */, bool low = false, bool progressDirectly = false, std::unique_ptr<threads::Future<bool>> listener = nullptr);
+                virtual void sendCommand0(const std::string_view& /* data */, bool low = false, std::unique_ptr<threads::Future<bool>> listener = nullptr);
                 virtual void sendAcknowledge(uint16_t packetId, bool low = false);
 
                 connection::VoiceClientConnection* getConnection(){ return connection; }
@@ -80,6 +80,7 @@ namespace ts {
                 void handlePacketCommand(const pipes::buffer_view&);
                 void handlePacketAck(const protocol::ClientPacketParser&);
                 void handlePacketVoice(const protocol::ClientPacketParser&);
+                void handlePacketVoiceWhisper(const protocol::ClientPacketParser&);
                 void handlePacketPing(const protocol::ClientPacketParser&);
                 void handlePacketInit(const protocol::ClientPacketParser&);
 

server/src/client/voice/VoiceClientConnection.cpp

@@ -32,23 +32,16 @@ using namespace ts::server;
 VoiceClientConnection::VoiceClientConnection(VoiceClient* client) : client(client) {
     memtrack::allocated<VoiceClientConnection>(this);
 
+    this->acknowledge_handler.destroy_packet = [](void* packet) {
+        reinterpret_cast<OutgoingServerPacket*>(packet)->unref();
+    };
+
     this->crypt_handler.reset();
     debugMessage(client->getServer()->getServerId(), "Allocated new voice client connection at {}", (void*) this);
 }
 
 VoiceClientConnection::~VoiceClientConnection() {
-    /* locking here should be useless, but just to ensure! */
-    {
-        lock_guard write_queue_lock(this->write_queue_lock);
-        this->write_queue.clear();
-    }
-
-    for(auto& category : this->write_preprocess_queues) {
-        lock_guard work_lock{category.work_lock};
-        lock_guard queue_lock{category.queue_lock};
-
-        category.queue.clear();
-    }
+    this->reset();
     this->client = nullptr;
     memtrack::freed<VoiceClientConnection>(this);
 }
@@ -96,6 +89,12 @@ void VoiceClientConnection::handle_incoming_datagram(const pipes::buffer_view& b
 #endif
 
     auto is_command = packet_parser.type() == protocol::COMMAND || packet_parser.type() == protocol::COMMAND_LOW;
+
+    /* in previous versions we checked if the arrived packet is "worth decoding".
+     * But since in general a command buffer underflow is much more unlikely, especially because most packets are not even command packets,
+     * it's better we just skip that step and decode it anyways */
+
+#if 0
     /* pretest if the packet is worth the effort of decoding it */
     if(is_command) {
         /* handle the order stuff */
@@ -120,6 +119,7 @@ void VoiceClientConnection::handle_incoming_datagram(const pipes::buffer_view& b
             return;
         }
     }
+#endif
 
     //NOTICE I found out that the Compressed flag is set if the packet contains an audio header
 
@@ -194,7 +194,19 @@ void VoiceClientConnection::handle_incoming_datagram(const pipes::buffer_view& b
             unique_lock queue_lock(fragment_buffer.buffer_lock);
 
             if(!fragment_buffer.insert_index(packet_parser.packet_id(), std::move(fragment_entry))) {
-                logTrace(this->client->getServerId(), "{} Failed to insert command packet into command packet buffer.", CLIENT_STR_LOG_PREFIX_(this->client));
+                auto ignore_type = fragment_buffer.accept_index(packet_parser.packet_id());
+                debugMessage(this->client->getServerId(), "{} Dropping command packet because command assembly buffer has an {} ({}|{}|{})",
+                             CLIENT_STR_LOG_PREFIX_(this->client),
+                             ignore_type == -1 ? "underflow" : "overflow",
+                             fragment_buffer.capacity(),
+                             fragment_buffer.current_index(),
+                             packet_parser.packet_id()
+                );
+
+                if(ignore_type == -1) { /* underflow */
+                    /* we've already got the packet, but the client dosn't know that so we've to send the acknowledge again */
+                    this->client->sendAcknowledge(packet_parser.packet_id(), packet_parser.type() == protocol::COMMAND_LOW);
+                }
                 return;
             }
         }
@@ -204,8 +216,10 @@ void VoiceClientConnection::handle_incoming_datagram(const pipes::buffer_view& b
         if(voice_server)
             voice_server->schedule_command_handling(this->client);
     } else {
-        if(packet_parser.type() == protocol::VOICE || packet_parser.type() == protocol::VOICE_WHISPER)
+        if(packet_parser.type() == protocol::VOICE)
             this->client->handlePacketVoice(packet_parser);
+        else if(packet_parser.type() == protocol::VOICE_WHISPER)
+            this->client->handlePacketVoiceWhisper(packet_parser);
         else if(packet_parser.type() == protocol::ACK || packet_parser.type() == protocol::ACK_LOW)
             this->client->handlePacketAck(packet_parser);
         else if(packet_parser.type() == protocol::PING || packet_parser.type() == protocol::PONG)
@@ -369,273 +383,163 @@ bool VoiceClientConnection::next_reassembled_command(unique_lock<std::recursive_
         std::string error{};
 
         auto decompressed_size = compression::qlz_decompressed_size(payload.data_ptr(), payload.length());
-        auto buffer = buffer::allocate_buffer(decompressed_size);
-        if(!compression::qlz_decompress_payload(payload.data_ptr(), buffer.data_ptr(), &decompressed_size)) {
+        if(decompressed_size == 0) {
+            logTrace(this->client->getServerId(), "{} Failed to calculate decompressed size for received command. Dropping packet.", CLIENT_STR_LOG_PREFIX_(this->client));
+            return false;
+        } else if(decompressed_size > 20 * 1024 * 1024) { /* max 20MB */
+            logTrace(this->client->getServerId(), "{} Command packet has a too large compressed size. Dropping packet.", CLIENT_STR_LOG_PREFIX_(this->client));
+            return false;
+        }
+        auto decompress_buffer = buffer::allocate_buffer(decompressed_size);
+        if(!compression::qlz_decompress_payload(payload.data_ptr(), decompress_buffer.data_ptr(), &decompressed_size)) {
             logTrace(this->client->getServerId(), "{} Failed to decompress received command. Dropping packet.", CLIENT_STR_LOG_PREFIX_(this->client));
             return false;
         }
 
-        payload = buffer.range(0, decompressed_size);
+        payload = decompress_buffer.range(0, decompressed_size);
     }
 
     result = std::move(payload);
     return have_more;
 }
 
-
-void VoiceClientConnection::sendPacket(const shared_ptr<protocol::ServerPacket>& original_packet, bool copy, bool prepare_directly) {
-    if(this->client->state == ConnectionState::DISCONNECTED)
-        return;
-
-    shared_ptr<protocol::ServerPacket> packet;
-    if(copy) {
-        packet = protocol::ServerPacket::from_buffer(original_packet->buffer().dup(buffer::allocate_buffer(original_packet->buffer().length())));
-        if(original_packet->getListener())
-            packet->setListener(std::move(original_packet->getListener()));
-        packet->memory_state.flags = original_packet->memory_state.flags;
+bool VoiceClientConnection::prepare_outgoing_packet(ts::protocol::OutgoingServerPacket *packet) {
+    if(packet->type_and_flags & PacketFlag::Unencrypted) {
+        this->crypt_handler.write_default_mac(packet->mac);
     } else {
-        packet = original_packet;
-    }
+        CryptHandler::key_t crypt_key{};
+        CryptHandler::nonce_t crypt_nonce{};
+        std::string error{};
 
-    auto type = WritePreprocessCategory::from_type(packet->type().type());
-    auto& queue = this->write_preprocess_queues[type];
-    if(prepare_directly) {
-        vector<pipes::buffer> buffers;
-        this->prepare_process_count++;
-
-        {
-            unique_lock work_lock{queue.work_lock};
-            if(!this->prepare_packet_for_write(buffers, packet, work_lock)) {
-                logError(this->client->getServerId(), "{} Dropping packet!", CLIENT_STR_LOG_PREFIX_(this->client));
-                this->prepare_process_count--;
-                return;
-            }
-        }
-
-        /* enqueue buffers for write */
-        {
-            lock_guard write_queue_lock(this->write_queue_lock);
-            this->write_queue.insert(this->write_queue.end(), buffers.begin(), buffers.end());
-        }
-        this->prepare_process_count--; /* we're now done preparing */
-    } else {
-        lock_guard queue_lock{queue.queue_lock};
-        queue.queue.push_back(packet);
-        queue.has_work = true;
-    }
-    this->triggerWrite();
-}
-
-bool VoiceClientConnection::prepare_packet_for_write(vector<pipes::buffer> &result, const shared_ptr<ServerPacket> &packet, std::unique_lock<std::mutex>& work_lock) {
-    assert(work_lock.owns_lock());
-
-    string error = "success";
-
-    if(packet->type().compressable() && !packet->memory_state.fragment_entry) {
-        packet->enable_flag(PacketFlag::Compressed);
-        if(!this->compress_handler.progressPacketOut(packet.get(), error)) {
-            logError(this->getClient()->getServerId(), "{} Could not compress outgoing packet.\nThis could cause fatal failed for the client.\nError: {}", error);
-            return false;
-        }
-    }
-
-    std::vector<shared_ptr<ServerPacket>> fragments;
-    fragments.reserve((size_t) (packet->data().length() / packet->type().max_length()) + 1);
-
-    if(packet->data().length() > packet->type().max_length()) {
-        if(!packet->type().fragmentable()) {
-            logError(this->client->getServerId(), "{} We've tried to send a too long, not fragmentable, packet. Dropping packet of type {} with length {}", CLIENT_STR_LOG_PREFIX_(this->client), packet->type().name(), packet->data().length());
-            return false;
-        }
-
-        { //Split packets
-            auto buffer = packet->data();
-
-            const auto max_length = packet->type().max_length();
-            while(buffer.length() > max_length * 2) {
-                fragments.push_back(make_shared<ServerPacket>(packet->type(), buffer.view(0, max_length).dup(buffer::allocate_buffer(max_length))));
-                buffer = buffer.range((size_t) max_length);
-            }
-
-            if(buffer.length() > max_length) { //Divide rest by 2
-                fragments.push_back(make_shared<ServerPacket>(packet->type(), buffer.view(0, buffer.length() / 2).dup(buffer::allocate_buffer(buffer.length() / 2))));
-                buffer = buffer.range(buffer.length() / 2);
-            }
-            fragments.push_back(make_shared<ServerPacket>(packet->type(), buffer));
-
-            for(const auto& frag : fragments) {
-                frag->setFragmentedEntry(true);
-                frag->enable_flag(PacketFlag::NewProtocol);
-            }
-        }
-
-        assert(fragments.size() >= 2);
-        fragments.front()->enable_flag(PacketFlag::Fragmented);
-        if(packet->has_flag(PacketFlag::Compressed))
-            fragments.front()->enable_flag(PacketFlag::Compressed);
-
-        fragments.back()->enable_flag(PacketFlag::Fragmented);
-
-        if(packet->getListener())
-            fragments.back()->setListener(std::move(packet->getListener())); //Move the listener to the last :)
-    } else {
-        fragments.push_back(packet);
-    }
-
-    result.reserve(fragments.size());
-
-    /* apply packet ids */
-    for(const auto& fragment : fragments) {
-        if(!fragment->memory_state.id_branded)
-            fragment->applyPacketId(this->packet_id_manager);
-
-        if(fragment->type().type() == protocol::PacketType::COMMAND_LOW || fragment->type().type() == protocol::PacketType::COMMAND)
-            this->packet_statistics().send_command(fragment->type().type(), fragment->packetId() | fragment->generationId() << 16U);
-    }
-
-    work_lock.unlock(); /* the rest could be unordered */
-
-
-    CryptHandler::key_t crypt_key{};
-    CryptHandler::nonce_t crypt_nonce{};
-    auto statistics = this->client ? this->client->connectionStatistics : nullptr;
-    for(const auto& fragment : fragments) {
-        if(fragment->has_flag(PacketFlag::Unencrypted)) {
-            this->crypt_handler.write_default_mac(fragment->mac().data_ptr());
+        if(!this->client->crypto.protocol_encrypted) {
+            crypt_key = CryptHandler::default_key;
+            crypt_nonce = CryptHandler::default_nonce;
         } else {
-            if(!this->client->crypto.protocol_encrypted) {
-                crypt_key = CryptHandler::default_key;
-                crypt_nonce = CryptHandler::default_nonce;
-            } else {
-                if(!this->crypt_handler.generate_key_nonce(false, fragment->type().type(), fragment->packetId(), fragment->generationId(), crypt_key, crypt_nonce)) {
-                    logError(this->client->getServerId(), "{} Failed to generate crypt key/nonce for sending a packet. This should never happen! Dropping packet.", CLIENT_STR_LOG_PREFIX_(this->client));
-                    return false;
-                }
-            }
-
-            auto crypt_result = this->crypt_handler.encrypt(fragment->header().data_ptr(), fragment->header().length(),
-                                                            fragment->data().data_ptr(), fragment->data().length(),
-                                                            fragment->mac().data_ptr(),
-                                                            crypt_key, crypt_nonce, error);
-            if(!crypt_result){
-                logError(this->client->getServerId(), "{} Failed to encrypt packet. Error: {}", CLIENT_STR_LOG_PREFIX_(this->client), error);
+            if(!this->crypt_handler.generate_key_nonce(false, (uint8_t) packet->packet_type(), packet->packet_id(), packet->generation, crypt_key, crypt_nonce)) {
+                logError(this->client->getServerId(), "{} Failed to generate crypt key/nonce for sending a packet. This should never happen! Dropping packet.", CLIENT_STR_LOG_PREFIX_(this->client));
                 return false;
             }
         }
 
-#ifndef CONNECTION_NO_STATISTICS
-        if(statistics) {
-            auto category = stats::ConnectionStatistics::category::from_type(fragment->type());
-            statistics->logOutgoingPacket(category, fragment->length() + 96); /* 96 for the UDP packet overhead */
+        auto crypt_result = this->crypt_handler.encrypt((char*) packet->packet_data() + ServerPacketP::kHeaderOffset, ServerPacketP::kHeaderLength,
+                                                        packet->payload, packet->payload_size,
+                                                        packet->mac,
+                                                        crypt_key, crypt_nonce, error);
+        if(!crypt_result){
+            logError(this->client->getServerId(), "{} Failed to encrypt packet. Error: {}", CLIENT_STR_LOG_PREFIX_(this->client), error);
+            return false;
         }
-#endif
-        this->acknowledge_handler.process_packet(*fragment);
-        result.push_back(fragment->buffer());
     }
-
     return true;
 }
 
-bool VoiceClientConnection::preprocess_write_packets() {
-    std::shared_ptr<ServerPacket> packet{nullptr};
-    vector<pipes::buffer> buffers{};
-    bool flag_more{false};
+VoiceClientConnection::WBufferPopResult VoiceClientConnection::pop_write_buffer(protocol::OutgoingServerPacket *&result) {
+    if(this->client->state == ConnectionState::DISCONNECTED)
+        return WBufferPopResult::DRAINED;
 
-    prepare_process_count++; /* we're not preparing a packet */
-    for(auto& category : this->write_preprocess_queues) {
-        if(!category.has_work) continue;
-        else if(packet) {
-            flag_more = true;
-            break;
-        }
-
-        unique_lock work_lock{category.work_lock, try_to_lock};
-        if(!work_lock) continue; /* This particular category will already be processed */
-
-        {
-            lock_guard buffer_lock{category.queue_lock};
-            if(category.queue.empty()) {
-                category.has_work = false;
-                continue;
+    bool need_prepare_packet{false}, more_packets{false};
+    {
+        std::lock_guard wlock{this->write_queue_mutex};
+        if(this->resend_queue_head) {
+            result = this->resend_queue_head;
+            if(result->next) {
+                assert(this->resend_queue_tail != &result->next);
+                this->resend_queue_head = result->next;
+            } else {
+                assert(this->resend_queue_tail == &result->next);
+                this->resend_queue_head = nullptr;
+                this->resend_queue_tail = &this->resend_queue_head;
             }
-
-            packet = std::move(category.queue.front());
-            category.queue.pop_front();
-            category.has_work = !category.queue.empty();
-            flag_more = category.has_work;
+        } else if(this->write_queue_head) {
+            result = this->write_queue_head;
+            if(result->next) {
+                assert(this->write_queue_tail != &result->next);
+                this->write_queue_head = result->next;
+            } else {
+                assert(this->write_queue_tail == &result->next);
+                this->write_queue_head = nullptr;
+                this->write_queue_tail = &this->write_queue_head;
+            }
+            need_prepare_packet = true;
+        } else {
+            return WBufferPopResult::DRAINED;
         }
-
-        if(!this->prepare_packet_for_write(buffers, packet, work_lock)) {
-            logError(this->client->getServerId(), "{} Dropping packet!", CLIENT_STR_LOG_PREFIX_(this->client));
-            if(flag_more)
-                break;
-            else
-                continue; /* find out if we have more */
-        }
-
-        if(flag_more)
-            break;
-        else
-            continue; /* find out if we have more */
+        result->next = nullptr;
+        more_packets = this->resend_queue_head != nullptr || this->write_queue_head != nullptr;
     }
 
-    /* enqueue buffers for write */
-    if(!buffers.empty()) {
-        lock_guard write_queue_lock(this->write_queue_lock);
-        this->write_queue.insert(this->write_queue.end(), buffers.begin(), buffers.end());
-    }
-    this->prepare_process_count--; /* we're now done preparing */
-
-    return flag_more;
+    if(need_prepare_packet)
+        this->prepare_outgoing_packet(result);
+    return more_packets ? WBufferPopResult::MORE_AVAILABLE : WBufferPopResult::DRAINED;
 }
 
-int VoiceClientConnection::pop_write_buffer(pipes::buffer& target) {
-    if(this->client->state == DISCONNECTED)
-        return 2;
+void VoiceClientConnection::execute_resend(const std::chrono::system_clock::time_point &now, std::chrono::system_clock::time_point &next) {
+    std::deque<std::shared_ptr<connection::AcknowledgeManager::Entry>> buffers{};
+    std::string error{};
 
-    lock_guard wqlock{this->write_queue_lock};
-    size_t size = this->write_queue.size();
-    if(size == 0)
-        return 2;
+    if (this->acknowledge_handler.execute_resend(now, next, buffers, error) < 0) {
+        debugMessage(client->getServerId(), "{} Failed to execute packet resend: {}", CLIENT_STR_LOG_PREFIX_(this->client), error);
 
-    target = std::move(this->write_queue.front());
-    this->write_queue.pop_front();
-
-#ifdef FUZZING_TESTING_OUTGOING
-    #ifdef FIZZING_TESTING_DISABLE_HANDSHAKE
-    if (this->client->state == ConnectionState::CONNECTED) {
-        #endif
-        if ((rand() % FUZZING_TESTING_DROP_MAX) < FUZZING_TESTING_DROP) {
-            debugMessage(this->client->getServerId(), "{}[FUZZING] Dropping outgoing packet", CLIENT_STR_LOG_PREFIX_(this->client));
-            return 0;
+        if(this->client->state == ConnectionState::CONNECTED) {
+            this->client->disconnect(ViewReasonId::VREASON_TIMEOUT, config::messages::timeout::packet_resend_failed, nullptr, true);
+        } else {
+            this->client->close_connection(system_clock::now() + seconds(1));
         }
-        #ifdef FIZZING_TESTING_DISABLE_HANDSHAKE
+    } else if(!buffers.empty()) {
+        size_t send_count{0};
+        {
+            lock_guard wlock{this->write_queue_mutex};
+            for(auto& buffer : buffers) {
+                auto packet = (protocol::OutgoingServerPacket*) buffer->packet_ptr;
+                if(packet->next) continue; /* still in write queue (this shall not happen very often) */
+                if(&packet->next == this->write_queue_tail || &packet->next == this->resend_queue_tail) continue;
+
+                packet->ref(); /* for the write queue again */
+                *this->resend_queue_tail = packet;
+                this->resend_queue_tail = &packet->next;
+
+                send_count++;
+                buffer->resend_count++;
+                this->packet_statistics().send_command((protocol::PacketType) buffer->packet_type, buffer->packet_full_id);
+            }
+        }
+        logTrace(client->getServerId(), "{} Resending {} packets. Send actually {} packets.", CLIENT_STR_LOG_PREFIX_(client), buffers.size(), send_count);
+        this->triggerWrite();
+    }
+}
+
+void VoiceClientConnection::encrypt_write_queue() {
+    OutgoingServerPacket* packets_head, *packets_tail;
+    {
+        std::lock_guard wlock{this->write_queue_mutex};
+        packets_head = this->write_queue_head;
+        this->write_queue_head = nullptr;
+        this->write_queue_tail = &this->write_queue_head;
+    }
+    if(!packets_head) return;
+
+    auto packet = packets_head;
+    while(packet) {
+        this->prepare_outgoing_packet(packet);
+        packets_tail = packet;
+        packet = packet->next;
+    }
+
+    {
+        std::lock_guard wlock{this->write_queue_mutex};
+        *this->resend_queue_tail = packets_head;
+        this->resend_queue_tail = &packets_tail->next;
     }
-    #endif
-#endif
-    return size > 1;
 }
 
 bool VoiceClientConnection::wait_empty_write_and_prepare_queue(chrono::time_point<chrono::system_clock> until) {
     while(true) {
-        for(auto& queue : this->write_preprocess_queues) {
-            {
-                lock_guard lock{queue.queue_lock};
-                if(!queue.queue.empty())
-                    goto _wait;
-            }
-
-            {
-                unique_lock lock{queue.work_lock, try_to_lock};
-                if(!lock.owns_lock())
-                    goto _wait;
-            }
-        }
         {
-            lock_guard buffer_lock{this->write_queue_lock};
-            if(!this->write_queue.empty())
+            std::lock_guard wlock{this->write_queue_mutex};
+            if(this->write_queue_head)
                 goto _wait;
-            if(this->prepare_process_count != 0)
+
+            if(this->resend_queue_head)
                 goto _wait;
         }
         break;
@@ -650,11 +554,25 @@ bool VoiceClientConnection::wait_empty_write_and_prepare_queue(chrono::time_poin
 }
 
 void VoiceClientConnection::reset() {
-    for(auto& queue : this->write_preprocess_queues) {
-        {
-            lock_guard lock{queue.queue_lock};
-            queue.queue.clear();
+    {
+        std::lock_guard wlock{this->write_queue_mutex};
+        auto head = this->write_queue_head;
+        while(head) {
+            auto next = head->next;
+            head->unref();
+            head = next;
         }
+        this->write_queue_head = nullptr;
+        this->write_queue_tail = &this->write_queue_head;
+
+        head = this->resend_queue_head;
+        while(head) {
+            auto next = head->next;
+            head->unref();
+            head = next;
+        }
+        this->resend_queue_head = nullptr;
+        this->resend_queue_tail = &this->resend_queue_head;
     }
 
     this->acknowledge_handler.reset();
@@ -694,4 +612,163 @@ void VoiceClientConnection::register_initiv_packet() {
     auto& fragment_buffer = this->_command_fragment_buffers[command_fragment_buffer_index(protocol::COMMAND)];
     unique_lock buffer_lock(fragment_buffer.buffer_lock);
     fragment_buffer.set_full_index_to(1); /* the first packet (0) is already the clientinitiv packet */
+}
+
+void VoiceClientConnection::send_packet(protocol::OutgoingServerPacket *packet) {
+    uint32_t full_id;
+    {
+        std::lock_guard id_lock{this->packet_id_mutex};
+        full_id = this->packet_id_manager.generate_full_id(packet->packet_type());
+    }
+    packet->set_packet_id(full_id & 0xFFFFU);
+    packet->generation = full_id >> 16U;
+
+    {
+        std::lock_guard qlock{this->write_queue_mutex};
+        *this->write_queue_tail = packet;
+        this->write_queue_tail = &packet->next;
+    }
+
+    auto statistics = this->client ? this->client->connectionStatistics : nullptr;
+    if(statistics) {
+        auto category = stats::ConnectionStatistics::category::from_type(packet->packet_type());
+        statistics->logOutgoingPacket(category, packet->packet_length() + 96); /* 96 for the UDP packet overhead */
+    }
+
+    this->triggerWrite();
+}
+
+void VoiceClientConnection::send_packet(protocol::PacketType type, protocol::PacketFlag::PacketFlags flag, const void *payload, size_t payload_size) {
+    auto packet = protocol::allocate_outgoing_packet(payload_size);
+
+    packet->type_and_flags = (uint8_t) type | (uint8_t) flag;
+    memcpy(packet->payload, payload, payload_size);
+
+    this->send_packet(packet);
+}
+
+#define MAX_COMMAND_PACKET_PAYLOAD_LENGTH (487)
+void VoiceClientConnection::send_command(const std::string_view &command, bool low, std::unique_ptr<threads::Future<bool>> ack_listener) {
+    bool own_data_buffer{false};
+    void* own_data_buffer_ptr; /* imutable! */
+
+    const char* data_buffer{command.data()};
+    size_t data_length{command.length()};
+
+    uint8_t head_pflags{0};
+    PacketType ptype{low ? PacketType::COMMAND_LOW : PacketType::COMMAND};
+    protocol::OutgoingServerPacket *packets_head{nullptr};
+    protocol::OutgoingServerPacket **packets_tail{&packets_head};
+
+    /* only compress "long" commands */
+    if(command.size() > 100) {
+        size_t max_compressed_payload_size = compression::qlz_compressed_size(command.data(), command.length());
+        auto compressed_buffer = ::malloc(max_compressed_payload_size);
+
+        size_t compressed_size{max_compressed_payload_size};
+        if(!compression::qlz_compress_payload(command.data(), command.length(), compressed_buffer, &compressed_size)) {
+            logCritical(0, "Failed to compress command packet. Dropping packet");
+            ::free(compressed_buffer);
+            return;
+        }
+
+        /* we don't need to make the command longer than it is */
+        if(compressed_size < command.length() || this->client->getType() == ClientType::CLIENT_TEAMSPEAK) { /* TS3 requires each splituped packet to be compressed (Update: Not 100% sure since there was another bug when discovering this but I've kept it since) */
+            own_data_buffer = true;
+            data_buffer = (char*) compressed_buffer;
+            own_data_buffer_ptr = compressed_buffer;
+            data_length = compressed_size;
+            head_pflags |= PacketFlag::Compressed;
+        } else {
+            ::free(compressed_buffer);
+        }
+    }
+
+    uint8_t ptype_and_flags{(uint8_t) ((uint8_t) ptype | (uint8_t) PacketFlag::NewProtocol)};
+    if(data_length > MAX_COMMAND_PACKET_PAYLOAD_LENGTH) {
+        auto chunk_count = (size_t) ceil((float) data_length / (float) MAX_COMMAND_PACKET_PAYLOAD_LENGTH);
+        auto chunk_size = (size_t) ceil((float) data_length / (float) chunk_count);
+
+        while(true) {
+            auto bytes = min(chunk_size, data_length);
+            auto packet = protocol::allocate_outgoing_packet(bytes);
+            packet->type_and_flags = ptype_and_flags;
+            memcpy(packet->payload, data_buffer, bytes);
+
+            *packets_tail = packet;
+            packets_tail = &packet->next;
+
+            data_length -= bytes;
+            if(data_length == 0) {
+                packet->type_and_flags |= PacketFlag::Fragmented;
+                break;
+            }
+            data_buffer += bytes;
+        }
+        packets_head->type_and_flags |= PacketFlag::Fragmented;
+    } else {
+        auto packet = protocol::allocate_outgoing_packet(data_length);
+        packet->type_and_flags = ptype_and_flags;
+
+        memcpy(packet->payload, data_buffer, data_length);
+
+        packets_head = packet;
+        packets_tail = &packet->next;
+    }
+
+
+    {
+        std::lock_guard id_lock{this->packet_id_mutex};
+        uint32_t full_id;
+        auto head = packets_head;
+        while(head) {
+            full_id = this->packet_id_manager.generate_full_id(ptype);
+
+            head->set_packet_id(full_id & 0xFFFFU);
+            head->generation = full_id >> 16U;
+            head = head->next;
+        }
+    }
+    packets_head->type_and_flags |= head_pflags;
+
+    /* do this before the next ptr might get modified due to the write queue */
+    auto statistics = this->client ? this->client->connectionStatistics : nullptr;
+    /* general stats */
+    if(statistics) {
+        auto head = packets_head;
+        while(head) {
+            statistics->logOutgoingPacket(stats::ConnectionStatistics::category::COMMAND, head->packet_length() + 96); /* 96 for the UDP overhead */
+            head = head->next;
+        }
+    }
+
+    /* loss stats */
+    {
+        auto head = packets_head;
+        while(head) {
+            auto full_packet_id = (uint32_t) (head->generation << 16U) | head->packet_id();
+            this->packet_statistics_.send_command(head->packet_type(), full_packet_id);
+
+            /* increase a reference for the ack handler */
+            head->ref();
+
+            /* Even thou the packet is yet unencrypted, it will be encrypted with the next write. The next write will be before the next resend because the next ptr must be null in order to resend a packet */
+            if(&head->next == packets_tail)
+                this->acknowledge_handler.process_packet(ptype, full_packet_id, head, std::move(ack_listener));
+            else
+                this->acknowledge_handler.process_packet(ptype, full_packet_id, head, nullptr);
+
+            head = head->next;
+        }
+    }
+
+    {
+        std::lock_guard qlock{this->write_queue_mutex};
+        *this->write_queue_tail = packets_head;
+        this->write_queue_tail = packets_tail;
+    }
+    this->triggerWrite();
+
+    if(own_data_buffer)
+        ::free(own_data_buffer_ptr);
 }

server/src/client/voice/VoiceClientConnection.h

@@ -39,6 +39,11 @@ namespace ts {
                 friend class server::VoiceClient;
                 friend class server::POWHandler;
             public:
+                enum struct WBufferPopResult {
+                    DRAINED,
+                    MORE_AVAILABLE
+                };
+
                 struct CommandFragment {
                     uint16_t packet_id{0};
                     uint16_t packet_generation{0};
@@ -63,7 +68,10 @@ namespace ts {
                 explicit VoiceClientConnection(server::VoiceClient*);
                 virtual ~VoiceClientConnection();
 
-                void sendPacket(const std::shared_ptr<protocol::ServerPacket>& original_packet, bool copy = false, bool prepare_directly = false);
+                /* Do not send command packets via send_packet! The send_packet will take ownership of the packet! */
+                void send_packet(protocol::OutgoingServerPacket* /* packet */);
+                void send_packet(protocol::PacketType /* type */, protocol::PacketFlag::PacketFlags /* flags */, const void* /* payload */, size_t /* payload length */);
+                void send_command(const std::string_view& /* build command command */, bool /* command low */, std::unique_ptr<threads::Future<bool>> /* acknowledge listener */);
 
                 CryptHandler* getCryptHandler(){ return &crypt_handler; }
 
@@ -72,14 +80,11 @@ namespace ts {
 #ifdef VC_USE_READ_QUEUE
                 bool handleNextDatagram();
 #endif
-                /*
-                 * Split packets waiting in write_process_queue and moves the final buffers to writeQueue.
-                 * @returns true when there are more packets to prepare
-                 */
-                bool preprocess_write_packets();
-                /* return 2 => Nothing | 1 => More and buffer is set | 0 => Buffer is set, nothing more */
-                int pop_write_buffer(pipes::buffer& /* buffer */);
+                /* if the result is true, ownership has been transferred */
+                WBufferPopResult pop_write_buffer(protocol::OutgoingServerPacket*& /* packet */);
+                void execute_resend(const std::chrono::system_clock::time_point &now, std::chrono::system_clock::time_point &next);
 
+                void encrypt_write_queue();
                 bool wait_empty_write_and_prepare_queue(std::chrono::time_point<std::chrono::system_clock> until = std::chrono::time_point<std::chrono::system_clock>());
 
                 protocol::PacketIdManager& getPacketIdManager() { return this->packet_id_manager; }
@@ -105,71 +110,29 @@ namespace ts {
                 CompressionHandler compress_handler;
                 AcknowledgeManager acknowledge_handler;
 
-                std::atomic_bool should_reassembled_reschedule; /* this get checked as soon the command handle lock has been released so trylock will succeed */
+                std::atomic_bool should_reassembled_reschedule{}; /* this get checked as soon the command handle lock has been released so trylock will succeed */
 
                 //Handle stuff
                 void execute_handle_command_packets(const std::chrono::system_clock::time_point& /* scheduled */);
                 bool next_reassembled_command(std::unique_lock<std::recursive_timed_mutex> &buffer_execute_lock /* packet channel execute lock */, pipes::buffer & /* buffer*/, uint16_t& /* packet id */);
 
 
-                /* ---------- Write declarations ---------- */
-                spin_lock write_queue_lock; /* queue access isn't for long in general */
-                std::deque<pipes::buffer> write_queue;
+                /* ---------- Write ---------- */
+                spin_mutex write_queue_mutex{};
+                protocol::OutgoingServerPacket* resend_queue_head{nullptr};
+                protocol::OutgoingServerPacket** resend_queue_tail{&resend_queue_head};
 
-                struct WritePreprocessCategory {
-                    enum value {
-                        PING_PONG = 0, //Ping/Pongs
-                        ACK = 2,
-                        VOICE_WHISPER = 1, //Voice/Whisper
-                        COMMAND = 3,
-                        INIT = 4,
-
-                        MAX = INIT
-                    };
-
-                    inline static value from_type(protocol::PacketType type) {
-                        switch(type) {
-                            case protocol::PING:
-                            case protocol::PONG:
-                                return value::PING_PONG;
-
-                            case protocol::VOICE:
-                            case protocol::VOICE_WHISPER:
-                                return value::VOICE_WHISPER;
-
-                            case protocol::ACK:
-                            case protocol::ACK_LOW:
-                                return value::ACK;
-
-                            case protocol::COMMAND:
-                            case protocol::COMMAND_LOW:
-                                return value::COMMAND;
-
-                            default:
-                                return value::INIT;
-                        }
-                    }
-                };
-
-                struct WritePreprocessQueue {
-                    int _zero1{0};
-                    bool has_work{false};
-                    std::mutex work_lock{};
-
-                    spin_lock queue_lock{};
-                    std::deque<std::shared_ptr<protocol::ServerPacket>> queue{};
-
-                    int _zero{0};
-                };
-                std::array<WritePreprocessQueue, WritePreprocessCategory::MAX> write_preprocess_queues{};
+                protocol::OutgoingServerPacket* write_queue_head{nullptr};
+                protocol::OutgoingServerPacket** write_queue_tail{&write_queue_head};
 
                 /* ---------- Processing ---------- */
                 /* automatically locked because packets of the same kind should be lock their "work_lock" from their WritePreprocessQueue object */
                 protocol::PacketIdManager packet_id_manager;
+                spin_mutex packet_id_mutex{};
 
                 /* this function is thread save :) */
                 std::atomic<uint8_t> prepare_process_count{0}; /* current thread count preparing a packet */
-                bool prepare_packet_for_write(std::vector<pipes::buffer> &/* buffers which need to be transferred */, const std::shared_ptr<protocol::ServerPacket> &/* the packet */, std::unique_lock<std::mutex>& /* work lock */);
+                bool prepare_outgoing_packet(protocol::OutgoingServerPacket* /* packet */);
 
                 std::array<protocol::generation_estimator, 9> incoming_generation_estimators{}; /* implementation is thread save */
                 std::recursive_mutex packet_buffer_lock;

server/src/client/voice/VoiceClientHandschake.cpp

@@ -167,7 +167,8 @@ ts::command_result VoiceClient::handleCommandClientInitIv(Command& command) {
             } else {
                 this->handshake.state = HandshakeState::SUCCEEDED; /* we're doing the verify via TeamSpeak */
             }
-            this->sendCommand0(initivexpand.build(), false, true); //If we setup the encryption now
+            this->sendCommand0(initivexpand.build()); //If we setup the encryption now
+            this->connection->encrypt_write_queue();
         }
 
         {

server/src/client/voice/VoiceClientPacketHandler.cpp

@@ -21,25 +21,25 @@ void VoiceClient::handlePacketCommand(const pipes::buffer_view& command_string)
     try {
         command = make_unique<Command>(Command::parse(command_string, true, !ts::config::server::strict_ut8_mode));
     } catch(std::invalid_argument& ex) {
-        result = command_result{error::parameter_convert, std::string{ex.what()}};
+        result.reset(command_result{error::parameter_convert, std::string{ex.what()}});
         goto handle_error;
     } catch(std::exception& ex) {
-        result = command_result{error::parameter_convert, std::string{ex.what()}};
+        result.reset(command_result{error::parameter_convert, std::string{ex.what()}});
         goto handle_error;
     }
 
     if(command->command() == "clientek") {
-        result = this->handleCommandClientEk(*command);
-        if(result.error_code()) goto handle_error;
+        result.reset(this->handleCommandClientEk(*command));
+        if(result.has_error()) goto handle_error;
     } else if(command->command() == "clientinitiv") {
-        result = this->handleCommandClientInitIv(*command);
-        if(result.error_code()) goto handle_error;
+        result.reset(this->handleCommandClientInitIv(*command));
+        if(result.has_error()) goto handle_error;
     } else this->handleCommandFull(*command, true);
 
     return;
     handle_error:
     this->notifyError(result);
-    result.release_details();
+    result.release_data();
 }
 
 void VoiceClient::handlePacketPing(const protocol::ClientPacketParser& packet) {
@@ -67,17 +67,15 @@ void VoiceClient::handlePacketPing(const protocol::ClientPacketParser& packet) {
 #endif
     char buffer[2];
     le2be16(packet.packet_id(), buffer);
-    auto pkt = make_shared<ServerPacket>(PacketTypeInfo::Pong, pipes::buffer_view{buffer, 2});
-    pkt->enable_flag(PacketFlag::Unencrypted);
-    this->connection->sendPacket(pkt);
+    this->connection->send_packet(PacketType::PONG, PacketFlag::Unencrypted, buffer, 2);
 }
 
 void VoiceClient::handlePacketVoice(const protocol::ClientPacketParser& packet) {
-    if (packet.type() == protocol::VOICE) {
-        SpeakingClient::handlePacketVoice(packet.payload(), (packet.flags() & PacketFlag::Compressed) > 0, (packet.flags() & PacketFlag::Fragmented) > 0);
-    } else if(packet.type() == protocol::VOICE_WHISPER) {
-        SpeakingClient::handlePacketVoiceWhisper(packet.payload(), (packet.flags() & PacketFlag::NewProtocol) > 0);
-    }
+    SpeakingClient::handlePacketVoice(packet.payload(), (packet.flags() & PacketFlag::Compressed) > 0, (packet.flags() & PacketFlag::Fragmented) > 0);
+}
+
+void VoiceClient::handlePacketVoiceWhisper(const ts::protocol::ClientPacketParser &packet) {
+    SpeakingClient::handlePacketVoiceWhisper(packet.payload(), (packet.flags() & PacketFlag::NewProtocol) > 0);
 }
 
 void VoiceClient::handlePacketAck(const protocol::ClientPacketParser& packet) {

server/src/client/voice/VoiceClientView.cpp

@@ -7,16 +7,17 @@ using namespace std;
 using namespace ts::server;
 using namespace ts::protocol;
 
-extern InstanceHandler* serverInstance;
-
 void VoiceClient::sendPingRequest() {
     this->lastPingRequest = std::chrono::system_clock::now();
 
-    auto packet = make_shared<ServerPacket>(PacketTypeInfo::Ping, pipes::buffer_view{});
-    packet->enable_flag(PacketFlag::Unencrypted);
-    this->connection->sendPacket(packet, false, true); /* prepare directly so the packet gets a packet id */
+    auto packet = protocol::allocate_outgoing_packet(0);
+    packet->ref(); /* extra ref for ourself */
 
-    this->lastPingId = packet->packetId();
+    packet->type_and_flags = (uint8_t) PacketType::PING | (uint8_t) PacketFlag::Unencrypted;
+    this->connection->send_packet(packet);
+
+    this->lastPingId = packet->packet_id();
+    packet->unref();
 
 #ifdef PKT_LOG_PING
     logMessage(this->getServerId(), "{}[Ping] Sending a ping request with it {}", CLIENT_STR_LOG_PREFIX, this->lastPingId);

server/src/client/web/VoiceBridge.cpp

@@ -69,28 +69,15 @@ VoiceBridge::VoiceBridge(const shared_ptr<WebClient>& owner) : _owner(owner) {
     config->nice_config = make_shared<rtc::NiceWrapper::Config>();
 
     config->nice_config->ice_port_range = {config::web::webrtc_port_min, config::web::webrtc_port_max};
-    for(const auto& entry : config::web::ice_servers) {
-        auto dp = entry.find(':');
-        if(dp == string::npos) continue;
-        auto host = entry.substr(0, dp);
-        auto port = entry.substr(dp + 1);
-        if(port.find_last_not_of("0123456789") != string::npos) continue;
+    if(config::web::stun_enabled)
+        config->nice_config->stun_server = { config::web::stun_host, config::web::stun_port };
 
-        if(host == "stun.l.google.com" && port == "9302")
-            port = "19302"; /* fix for the invalid config value until 1.3.14beta1 :) */
-
-        try {
-            config->nice_config->ice_servers.push_back({host,(uint16_t) stoi(port)});
-        } catch(std::exception& ex) {}
-    }
-    config->nice_config->ice_servers.push_back({"stun.l.google.com", 19302});
-
-    config->nice_config->allow_ice_udp = true;
-    config->nice_config->allow_ice_tcp = false;
+    config->nice_config->allow_ice_udp = config::web::udp_enabled;
+    config->nice_config->allow_ice_tcp = config::web::tcp_enabled;
     config->nice_config->use_upnp = config::web::enable_upnp;
 
     gioloop::initialize();
-    config->nice_config->main_loop = gioloop::loop();
+    config->event_loop = gioloop::loop();
     /*
     config->nice_config->main_loop = std::shared_ptr<GMainLoop>(g_main_loop_new(nullptr, false), g_main_loop_unref);
     std::thread(g_main_loop_run, config->nice_config->main_loop.get()).detach();
@@ -101,7 +88,7 @@ VoiceBridge::VoiceBridge(const shared_ptr<WebClient>& owner) : _owner(owner) {
     config->logger->callback_argument = this;
     //config->sctp.local_port = 5202; //Fire Fox don't support a different port :D
 
-    this->connection = make_unique<rtc::PeerConnection>(config);
+    this->connection = make_unique<rtc::PeerConnection>(config) ;
 }
 
 VoiceBridge::~VoiceBridge() {
@@ -126,7 +113,7 @@ bool VoiceBridge::initialize(std::string &error) {
                 callback(candidate);
         } else {
             if(auto callback{this->callback_ice_candidate_finished}; callback)
-                callback();
+                callback(candidate.sdpMid, candidate.sdpMLineIndex);
         }
     };
 

server/src/client/web/VoiceBridge.h

@@ -13,7 +13,7 @@ namespace ts {
             public:
                 typedef std::function<void(const pipes::buffer_view&, bool, bool)> cb_voice_data;
                 typedef std::function<void(const rtc::IceCandidate&)> cb_ice_candidate;
-                typedef std::function<void()> cb_ice_candidate_finish;
+                typedef std::function<void(const std::string& /* sdpMid */, int /* sdpMLineIndex */)> cb_ice_candidate_finish;
                 typedef std::function<void()> cb_initialized;
                 typedef std::function<void()> cb_failed;
 

server/src/client/web/WebClient.cpp

@@ -249,7 +249,7 @@ command_result WebClient::handleCommand(Command &command) {
     if(this->connectionState() == ConnectionState::INIT_HIGH && this->handshake.state == HandshakeState::SUCCEEDED){
         if(command.command() == "clientinit") {
             auto result = this->handleCommandClientInit(command);
-            if(result.error_code())
+            if(result.has_error())
                 this->close_connection(system_clock::now() + seconds(1));
             return result;
         }
@@ -505,10 +505,13 @@ void WebClient::handleMessage(const std::string &message) {
 
                     this->sendJson(jsonCandidate);
                 };
-                this->voice_bridge->callback_ice_candidate_finished = [&]{
+                this->voice_bridge->callback_ice_candidate_finished = [&](const std::string& sdpMid, int sdpMLineIndex){
                     Json::Value jsonCandidate;
                     jsonCandidate["type"] = "WebRTC";
                     jsonCandidate["request"] = "ice_finish";
+                    jsonCandidate["msg"]["candidate"] = "";
+                    jsonCandidate["msg"]["sdpMid"] = sdpMid;
+                    jsonCandidate["msg"]["sdpMLineIndex"] = sdpMLineIndex;
 
                     this->sendJson(jsonCandidate);
                 };

server/src/lincense/LicenseService.cpp

@@ -320,7 +320,7 @@ void LicenseService::send_license_validate_request() {
 void LicenseService::handle_message_license_info(const void *buffer, size_t buffer_length) {
     std::lock_guard rlock{this->request_lock};
     if(this->request_state_ != request_state::license_validate) {
-        this->handle_check_fail(strobf("finvalid request state for license response packet").string());
+        this->handle_check_fail(strobf("invalid request state for license response packet").string());
         return;
     }
 
@@ -539,7 +539,7 @@ void LicenseService::execute_tick() {
     auto now = std::chrono::system_clock::now();
     if(this->request_state_ != request_state::empty) {
         if(this->timings.last_request + std::chrono::minutes{5} < now) {
-            this->handle_check_fail(strobf("Scheduling next check at {}").string());
+            this->handle_check_fail(strobf("check timeout").string());
         } else {
             return;
         }

server/src/lincense/LicenseService.h

@@ -15,6 +15,7 @@ namespace google::protobuf {
 
 namespace ts::server::license {
     struct InstanceLicenseInfo {
+        bool is_old_license{false};
         std::shared_ptr<::license::License> license{nullptr};
         std::string web_certificate_revision{};
 

server/src/manager/LetterManager.cpp

@@ -47,7 +47,7 @@ std::vector<std::shared_ptr<LetterHeader>> LetterManager::avariableLetters(Clien
                     if(strcmp(columns[index], "sender") == 0)
                         letter->sender = values[index];
                     else if(strcmp(columns[index], "created") == 0)
-                        letter->created = system_clock::now() + milliseconds(stoull(values[index]));
+                        letter->created = std::chrono::system_clock::time_point{} + seconds{stoull(values[index])};
                     else if(strcmp(columns[index], "letterId") == 0)
                         letter->id = static_cast<LetterId>(stoull(values[index]));
                     else if(strcmp(columns[index], "subject") == 0)
@@ -79,7 +79,7 @@ shared_ptr<Letter> LetterManager::getFullLetter(LetterId letter) {
             if(strcmp(columns[index], "sender") == 0)
                 letter->sender = values[index];
             else if(strcmp(columns[index], "created") == 0)
-                letter->created = system_clock::now() + milliseconds(stoull(values[index]));
+                letter->created = system_clock::now() + std::chrono::seconds{stoull(values[index])};
             else if(strcmp(columns[index], "letterId") == 0)
                 letter->id = static_cast<LetterId>(stoull(values[index]));
             else if(strcmp(columns[index], "subject") == 0)
@@ -96,12 +96,12 @@ shared_ptr<Letter> LetterManager::getFullLetter(LetterId letter) {
     return res;
 }
 
-void LetterManager::createLetter(ClientUid sender, ClientUid reciver, std::string subject, std::string message) {
+void LetterManager::createLetter(const ClientUid& sender, const ClientUid& reciver, const std::string& subject, const std::string& message) {
     sql::command(this->server->getSql(), "INSERT INTO `letters` (`serverId`, `sender`, `receiver`, `created`, `subject`, `message`, `read`) VALUES (:sid, :sender, :receiver, :created, :subject, :message, :read)",
                 variable{":sid", this->server->getServerId()},
                 variable{":sender", sender},
                 variable{":receiver", reciver},
-                variable{":created", duration_cast<milliseconds>(system_clock::now().time_since_epoch()).count()},
+                variable{":created", std::chrono::floor<std::chrono::seconds>(system_clock::now().time_since_epoch()).count()},
                 variable{":subject", subject},
                 variable{":message", message},
                 variable{":read", false})

server/src/manager/LetterManager.h

@@ -37,7 +37,7 @@ namespace ts {
                 void updateReadFlag(LetterId, bool);
                 void deleteLetter(LetterId);
 
-                void createLetter(ClientUid sender, ClientUid reciver, std::string subject, std::string message);
+                void createLetter(const ClientUid& sender, const ClientUid& reciver, const std::string& subject, const std::string& message);
             private:
                 server::VirtualServer* server;
         };

server/src/manager/SqlDataManager.cpp

@@ -45,7 +45,7 @@ if(!result && result.msg().find(ignore) == string::npos){
 #define RESIZE_COLUMN(tblName, rowName, size) up vote EXECUTE("Could not change column size", "ALTER TABLE " tblName " ALTER COLUMN " rowName " varchar(" size ")");
 
 #define CURRENT_DATABASE_VERSION 12
-#define CURRENT_PERMISSION_VERSION 3
+#define CURRENT_PERMISSION_VERSION 4
 
 #define CLIENT_UID_LENGTH "64"
 #define CLIENT_NAME_LENGTH "128"
@@ -271,38 +271,38 @@ bool SqlDataManager::update_database(std::string &error) {
 
             case 7:
                 /* recreate permission table */
-/*
-DROP TABLE `permissions`, `properties`;
-ALTER TABLE permissions_v6 RENAME permissions;
-ALTER TABLE properties_v6 RENAME properties;
-*/
+                /*
+                DROP TABLE `permissions`, `properties`;
+                ALTER TABLE permissions_v6 RENAME permissions;
+                ALTER TABLE properties_v6 RENAME properties;
+                */
 
                 /* MySQL command
 
-START TRANSACTION;
-SET SESSION sql_mode='STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';
+                START TRANSACTION;
+                SET SESSION sql_mode='STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';
 
--- Modify the permissions
-ALTER TABLE permissions RENAME TO permissions_v6;
-CREATE TABLE `permissions`(`serverId` INT NOT NULL, `type` INT, `id` INT, `channelId` INT, `permId` VARCHAR(256), `value` INT, `grant` INT, `flag_skip` TINYINT(1), `flag_negate` TINYINT(1), CONSTRAINT PK PRIMARY KEY(`serverId`, `type`, `id`, `channelId`, `permId`)) CHARACTER SET=utf8;
-CREATE INDEX `idx_permissions_serverId` ON `permissions` (`serverId`);
-CREATE INDEX `idx_permissions_serverId_channelId` ON `permissions` (`serverId`, `channelId`);
-INSERT INTO `permissions` SELECT * FROM `permissions_v6` GROUP BY `serverId`, `type`, `id`, `channelId`, `permId`;
+                -- Modify the permissions
+                ALTER TABLE permissions RENAME TO permissions_v6;
+                CREATE TABLE `permissions`(`serverId` INT NOT NULL, `type` INT, `id` INT, `channelId` INT, `permId` VARCHAR(256), `value` INT, `grant` INT, `flag_skip` TINYINT(1), `flag_negate` TINYINT(1), CONSTRAINT PK PRIMARY KEY(`serverId`, `type`, `id`, `channelId`, `permId`)) CHARACTER SET=utf8;
+                CREATE INDEX `idx_permissions_serverId` ON `permissions` (`serverId`);
+                CREATE INDEX `idx_permissions_serverId_channelId` ON `permissions` (`serverId`, `channelId`);
+                INSERT INTO `permissions` SELECT * FROM `permissions_v6` GROUP BY `serverId`, `type`, `id`, `channelId`, `permId`;
 
--- Modify the properties
-ALTER TABLE properties RENAME TO properties_v6;
-CREATE TABLE properties(`serverId` INTEGER DEFAULT -1, `type` INTEGER, `id` INTEGER, `key` VARCHAR(256), `value` TEXT, CONSTRAINT PK PRIMARY KEY (`serverId`, `type`, `id`, `key`)) CHARACTER SET=utf8;
-CREATE INDEX `idx_properties_serverId` ON `properties` (`serverId`);
-CREATE INDEX `idx_properties_serverId_id` ON `properties` (`serverId`, `id`);
-INSERT INTO `properties` SELECT * FROM `properties_v6` GROUP BY `serverId`, `type`, `id`, `key`;
+                -- Modify the properties
+                ALTER TABLE properties RENAME TO properties_v6;
+                CREATE TABLE properties(`serverId` INTEGER DEFAULT -1, `type` INTEGER, `id` INTEGER, `key` VARCHAR(256), `value` TEXT, CONSTRAINT PK PRIMARY KEY (`serverId`, `type`, `id`, `key`)) CHARACTER SET=utf8;
+                CREATE INDEX `idx_properties_serverId` ON `properties` (`serverId`);
+                CREATE INDEX `idx_properties_serverId_id` ON `properties` (`serverId`, `id`);
+                INSERT INTO `properties` SELECT * FROM `properties_v6` GROUP BY `serverId`, `type`, `id`, `key`;
 
--- Delete orphaned permissions and properties
-DELETE FROM `permissions` WHERE (`type` = 1 OR `type` = 2) AND NOT ((`serverId`, `channelId`) IN (SELECT `serverId`, `channelId` FROM `channels`) OR `channelId` = 0);
-DELETE FROM `permissions` WHERE `type` = 0 AND NOT (`serverId`, `id`) IN (SELECT `serverId`, `groupId` FROM `groups`);
-DELETE FROM `permissions` WHERE `type` = 2 AND NOT (`serverId`, `id`) IN (SELECT `serverId`, `cldbid` FROM `clients`);
-DELETE FROM `properties` WHERE `type` = 1 AND NOT (`serverId` IN (SELECT `serverId` FROM servers) OR `serverId` = 0);
+                -- Delete orphaned permissions and properties
+                DELETE FROM `permissions` WHERE (`type` = 1 OR `type` = 2) AND NOT ((`serverId`, `channelId`) IN (SELECT `serverId`, `channelId` FROM `channels`) OR `channelId` = 0);
+                DELETE FROM `permissions` WHERE `type` = 0 AND NOT (`serverId`, `id`) IN (SELECT `serverId`, `groupId` FROM `groups`);
+                DELETE FROM `permissions` WHERE `type` = 2 AND NOT (`serverId`, `id`) IN (SELECT `serverId`, `cldbid` FROM `clients`);
+                DELETE FROM `properties` WHERE `type` = 1 AND NOT (`serverId` IN (SELECT `serverId` FROM servers) OR `serverId` = 0);
 
-ROLLBACK;
+                ROLLBACK;
                  */
                 if(manager->getType() == sql::TYPE_MYSQL) {
                     /*
@@ -412,6 +412,7 @@ ROLLBACK;
                 CREATE_INDEX2R("conversation_blocks", "server_id", "conversation_id");
                 db_version(11);
 
+#if 0 /* not yet needed */
             case 11:
                 /* update the group table */
                 {
@@ -445,6 +446,7 @@ ROLLBACK;
                         return false;
                     }
                 }
+
                 /* update the client table */
                 {
                     result = sql::command(this->sql(), "CREATE TABLE `clients_v2` (`serverId` INT NOT NULL, `cldbid` INTEGER, `original_client_id` INTEGER DEFAULT 0, `clientUid` VARCHAR(64) NOT NULL, `firstConnect` BIGINT DEFAULT 0, `lastConnect` BIGINT DEFAULT 0, `connections` INT DEFAULT 0, `lastName` VARCHAR(128) DEFAULT '', UNIQUE(`serverId`, `clientUid`));").execute();
@@ -468,7 +470,20 @@ ROLLBACK;
                         error = "failed to rename new clients table to the old clients table (" + result.fmtStr() + ")";
                         return false;
                     }
+
+                    CREATE_INDEX("clients", "serverId");
+                    CREATE_INDEX2R("clients", "serverId", "clientUid");
+                    CREATE_INDEX2R("clients", "serverId", "cldbid");
                 }
+                db_version(12);
+#endif
+            case 11:
+                result = sql::command(this->sql(), "UPDATE `letters` SET `created` = 0 WHERE 1;").execute();
+                if(!result) {
+                    error = "Failed to reset offline messages timestamps (" + result.fmtStr() + ")";
+                    return false;
+                }
+
                 db_version(12);
             default:
                 break;
@@ -624,6 +639,16 @@ bool SqlDataManager::update_permissions(std::string &error) {
                     return false;
 
                 perm_version(3);
+
+            case 3:
+                if(!auto_update(permission::update::SERVER_ADMIN, "i_client_poke_max_clients", {20, true}, false, false, {75, true}))
+                    return false;
+                if(!auto_update(permission::update::QUERY_ADMIN, "i_client_poke_max_clients", {50, true}, false, false, {100, true}))
+                    return false;
+                if(!auto_update(permission::update::SERVER_NORMAL, "i_client_poke_max_clients", {5, true}, false, false, {0, false}))
+                    return false;
+
+                perm_version(4);
             default:
                 break;
         }

server/src/manager/SqlDataManager.h

@@ -10,7 +10,7 @@ namespace ts {
                 virtual ~SqlDataManager();
 
                 [[nodiscard]] inline int get_database_version() const { return this->_database_version; }
-                [[nodiscard]] inline int get_permissions_version() const { return this->_database_version; }
+                [[nodiscard]] inline int get_permissions_version() const { return this->_permissions_version; }
                 bool initialize(std::string&);
                 void finalize();
 

server/src/server/POWHandler.cpp

@@ -252,8 +252,11 @@ void POWHandler::handle_puzzle_solve(const std::shared_ptr<ts::server::POWHandle
         /* should we validate x,n and level as well? */
         if(memcmp(client->rsa_challenge->data_result, &buffer[4 + 1 + 2 * 64 + 04 + 100], 64) != 0) {
             #ifdef POW_ERROR
-            debugMessage(this->get_server_id(), "[POW][{}][Puzzle] Received an invalid puzzle solution! Resetting client", net::to_string(client->address));
+            debugMessage(this->get_server_id(), "[POW][{}][Puzzle] Received an invalid puzzle solution! Resetting client & puzzle", net::to_string(client->address));
             #endif
+            constexpr static uint8_t empty_result[64]{0};
+            if(memcmp(empty_result, &buffer[4 + 1 + 2 * 64 + 04 + 100], 64) == 0)
+                client->rsa_challenge->fail_count++;
             client->rsa_challenge.reset(); /* get another RSA challenge */
             this->reset_client(client);
             return;

server/src/server/VoiceIOManager.h

@@ -5,7 +5,7 @@
 #include <ThreadPool/Thread.h>
 #include <condition_variable>
 #include <pipes/buffer.h>
-#include <misc/spin_lock.h>
+#include <misc/spin_mutex.h>
 #include <ThreadPool/Mutex.h>
 
 namespace ts {
@@ -80,7 +80,7 @@ namespace ts {
             ::event* event_read = nullptr;
             ::event* event_write = nullptr;
 
-            spin_lock write_queue_lock;
+            spin_mutex write_queue_lock;
             datagram_packet_t dg_write_queue_head = nullptr;
             datagram_packet_t dg_write_queue_tail = nullptr;