diff --git a/README.md b/README.md index a5855be..a079bfd 100644 --- a/README.md +++ b/README.md @@ -498,7 +498,9 @@ The Percona Binary Log Server configuration file has the following format. "checkpoint_interval": "30s", "encryption": { "format": "generic", - "keyring_uri": "file:///var/lib/pbs/keyring/keyring_data.json" + "keyring_uri": "file:///var/lib/pbs/keyring/keyring_data.json", + "kek_id": "alpha", + "cipher": "AES-256-CTR" } } } @@ -620,6 +622,8 @@ Please note that S3 API does not provide a way to append a portion of data to an If this section is present, then all the binlog data files will be encrypted before written to the storage. - `` - specifies the encryption format (currently only `generic` is supported). - `` - specifies location of the keyring JSON data file (currently only 'file://' scheme is supported meaning that the file should be taken from the local file sytem from the path specified in this URI, e.g. `file:///var/lib/pbs/keyring/keyring_data.json`). +- `` - specifies the ID of the key that must be used as a key-encryption-key (KEK). This ID must be present in the keyring. +- `` - specifies the data-encryption cipher name used binlog data file encryption (e.g. `AES-256-CTR`). ##### Keyring file format ```json @@ -628,12 +632,12 @@ If this section is present, then all the binlog data files will be encrypted bef "keys": [ { "id": "alpha", - "algorithm": "AES-128-ECB", + "cipher": "AES-128-ECB", "data_hex": "00112233445566778899AABBCCDDEEFF" }, { "id": "beta", - "algorithm": "AES-256-GCM", + "cipher": "AES-256-GCM", "data_hex": "00112233445566778899AABBCCDDEEFFFFEEDDCCBBAA998877665544332211" } ] @@ -643,7 +647,7 @@ Keyring JSON file should represent a top-level JSON object with the following ke - `version` - currently should always be equal to `1`. - `keys` - should be an array of objects tith the following keys - `id` - a unique string identifier of the key in the keyring. - - `algorithm` - the name of the symmetric cypher which should be used with this key (e.g `AES-256-GCM`). + - `cipher` - the name of the symmetric cypher which should be used with this key (e.g `AES-256-GCM`). - `data_hex` - key bytes in hex format (typically `16`, `24`, or `32` bytes, meaning `32`, `48`, or `64` characters) ### Resuming previous operation diff --git a/keyring_data.json b/keyring_data.json index 4340b49..9a7b4ee 100644 --- a/keyring_data.json +++ b/keyring_data.json @@ -3,12 +3,12 @@ "keys": [ { "id": "alpha", - "algorithm": "AES-128-ECB", + "cipher": "AES-128-ECB", "data_hex": "00112233445566778899AABBCCDDEEFF" }, { "id": "beta", - "algorithm": "AES-256-GCM", + "cipher": "AES-256-GCM", "data_hex": "00112233445566778899AABBCCDDEEFFFFEEDDCCBBAA998877665544332211" } ] diff --git a/main_config.json b/main_config.json index dce9160..ccb7037 100644 --- a/main_config.json +++ b/main_config.json @@ -44,7 +44,9 @@ "checkpoint_interval": "30s", "encryption": { "format": "generic", - "keyring_uri": "file:///home/user/keyring/keyring/keyring_data.json" + "keyring_uri": "file:///home/user/keyring/keyring/keyring_data.json", + "kek_id": "alpha", + "cipher": "AES-256-CTR" } } } diff --git a/src/app.cpp b/src/app.cpp index 54cbc08..ff36af8 100644 --- a/src/app.cpp +++ b/src/app.cpp @@ -39,12 +39,10 @@ #include "app_version.hpp" -#include "binsrv/basic_keyring.hpp" #include "binsrv/basic_logger.hpp" // needed for encryption_format_type's operator << #include "binsrv/encryption_format_type.hpp" // IWYU pragma: keep #include "binsrv/exception_handling_helpers.hpp" -#include "binsrv/keyring_factory.hpp" #include "binsrv/log_severity.hpp" #include "binsrv/logger_factory.hpp" #include "binsrv/main_config.hpp" @@ -304,6 +302,10 @@ void log_storage_info(binsrv::basic_logger &logger, msg += std::to_string(storage.get_current_position()); } logger.log(binsrv::log_severity::info, msg); + logger.log(binsrv::log_severity::info, + "keyring status: " + storage.get_keyring_description()); + logger.log(binsrv::log_severity::info, + "active KEK: " + storage.get_active_kek_description()); } void log_library_info(binsrv::basic_logger &logger, @@ -1327,15 +1329,6 @@ int main(int argc, char *argv[]) { const auto verify_checksum{replication_config.get<"verify_checksum">()}; const auto replication_mode{replication_config.get<"mode">()}; const auto optional_rewrite_config{replication_config.get<"rewrite">()}; - const auto &optional_encryption_config{storage_config.get<"encryption">()}; - - binsrv::basic_keyring_ptr keyring; - if (optional_encryption_config.has_value()) { - keyring = binsrv::keyring_factory::create( - optional_encryption_config->get<"keyring_uri">()); - logger->log(binsrv::log_severity::info, - "initialized keyring: " + keyring->get_description()); - } binsrv::storage storage{storage_config, binsrv::storage_construction_mode_type::streaming, diff --git a/src/binsrv/basic_keyring_fwd.hpp b/src/binsrv/basic_keyring_fwd.hpp index 88bc2dc..ce91326 100644 --- a/src/binsrv/basic_keyring_fwd.hpp +++ b/src/binsrv/basic_keyring_fwd.hpp @@ -22,7 +22,7 @@ namespace binsrv { class basic_keyring; -using basic_keyring_ptr = std::shared_ptr; +using basic_keyring_ptr = std::unique_ptr; } // namespace binsrv diff --git a/src/binsrv/encryption_config.hpp b/src/binsrv/encryption_config.hpp index 4b54375..d15b97f 100644 --- a/src/binsrv/encryption_config.hpp +++ b/src/binsrv/encryption_config.hpp @@ -30,7 +30,9 @@ struct [[nodiscard]] encryption_config : util::nv_tuple< // clang-format off util::nv<"format", encryption_format_type>, - util::nv<"keyring_uri", std::string> + util::nv<"keyring_uri", std::string>, + util::nv<"kek_id", std::string>, + util::nv<"cipher", std::string> // clang-format on > { diff --git a/src/binsrv/encryption_format_type_fwd.hpp b/src/binsrv/encryption_format_type_fwd.hpp index 70753b6..4088f38 100644 --- a/src/binsrv/encryption_format_type_fwd.hpp +++ b/src/binsrv/encryption_format_type_fwd.hpp @@ -19,12 +19,14 @@ #include #include #include +#include #include "util/nv_tuple_json_support.hpp" namespace binsrv { enum class encryption_format_type : std::uint8_t; +using optional_encryption_format_type = std::optional; template requires std::same_as diff --git a/src/binsrv/keyring_factory.cpp b/src/binsrv/keyring_factory.cpp index 50b32f8..82bb85f 100644 --- a/src/binsrv/keyring_factory.cpp +++ b/src/binsrv/keyring_factory.cpp @@ -37,7 +37,7 @@ basic_keyring_ptr keyring_factory::create(std::string_view keyring_uri) { const auto &uri{*uri_parse_result}; if (uri.scheme() == file_keyring::uri_schema) { - return std::make_shared(keyring_uri); + return std::make_unique(keyring_uri); } util::exception_location().raise( diff --git a/src/binsrv/keyring_record.hpp b/src/binsrv/keyring_record.hpp index 9d74302..281149c 100644 --- a/src/binsrv/keyring_record.hpp +++ b/src/binsrv/keyring_record.hpp @@ -25,14 +25,23 @@ namespace binsrv { -// clang-format off struct [[nodiscard]] keyring_record : util::nv_tuple< + // clang-format off util::nv<"id", std::string>, - util::nv<"algorithm", std::string>, + util::nv<"cipher", std::string>, util::nv<"data_hex", util::hex_value> - > {}; -// clang-format on + // clang-format on + > { + [[nodiscard]] std::string get_description() const { + std::string result; + result += get<"id">(); + result += '('; + result += get<"cipher">(); + result += ')'; + return result; + } +}; } // namespace binsrv diff --git a/src/binsrv/keyring_record_collection.cpp b/src/binsrv/keyring_record_collection.cpp index 63c680d..c5a8b9d 100644 --- a/src/binsrv/keyring_record_collection.cpp +++ b/src/binsrv/keyring_record_collection.cpp @@ -50,7 +50,8 @@ keyring_record_collection::get_key(std::string_view key_id) const { return key.get<"id">() == key_id; }); if (key_it == std::end(keys)) { - util::exception_location().raise("key not found"); + util::exception_location().raise( + "key not found in the keyring record collection"); } return *key_it; } @@ -69,10 +70,7 @@ void keyring_record_collection::validate() const { result += " key(s):"; for (const auto &key : keys) { result += ' '; - result += key.get<"id">(); - result += '('; - result += key.get<"algorithm">(); - result += ')'; + result += key.get_description(); } return result; } diff --git a/src/binsrv/storage.cpp b/src/binsrv/storage.cpp index 2c52951..0c59172 100644 --- a/src/binsrv/storage.cpp +++ b/src/binsrv/storage.cpp @@ -30,8 +30,11 @@ #include #include +#include "binsrv/basic_keyring.hpp" #include "binsrv/basic_storage_backend.hpp" #include "binsrv/binlog_file_metadata.hpp" +#include "binsrv/encryption_format_type_fwd.hpp" +#include "binsrv/keyring_factory.hpp" #include "binsrv/replication_mode_type.hpp" #include "binsrv/storage_backend_factory.hpp" #include "binsrv/storage_config.hpp" @@ -66,6 +69,16 @@ storage::storage(const storage_config &config, std::chrono::seconds{checkpoint_interval_opt->get_value()}; } + const auto &encryption_config{config.get<"encryption">()}; + optional_encryption_format_type encryption_format{}; + if (encryption_config.has_value()) { + encryption_format = encryption_config->get<"format">(); + encryption_format_ = encryption_format; + keyring_ = keyring_factory::create(encryption_config->get<"keyring_uri">()); + active_kek_ = keyring_->get_key(encryption_config->get<"kek_id">()); + active_data_cipher_ = encryption_config->get<"cipher">(); + } + backend_ = storage_backend_factory::create(config); auto storage_objects{backend_->list_objects()}; @@ -85,7 +98,7 @@ storage::storage(const storage_config &config, storage_objects.erase(metadata_it); load_metadata(); - validate_metadata(replication_mode); + validate_metadata(replication_mode, encryption_format); // if after metadata erasure 'storage_objects' is empty, then this mean // that it has only metadata in it that passes validation and we can @@ -406,6 +419,16 @@ storage::purge_binlogs(const events::composite_binlog_name &target) { return backend_->get_object_uri(binlog_name.str()); } +[[nodiscard]] std::string storage::get_keyring_description() const { + return is_encryption_enabled() ? keyring_->get_description() + : "keyring is not initialized"; +} + +[[nodiscard]] std::string storage::get_active_kek_description() const { + return is_encryption_enabled() ? active_kek_.get_description() + : "active KEK is not set"; +} + void storage::ensure_streaming_mode() const { if (construction_mode_ != storage_construction_mode_type::streaming) { util::exception_location().raise( @@ -584,19 +607,29 @@ void storage::load_metadata() { const auto metadata_content{backend_->get_object(metadata_name)}; const storage_metadata metadata{metadata_content}; replication_mode_ = metadata.root().get<"mode">(); + encryption_format_ = metadata.root().get<"encryption">(); } -void storage::validate_metadata(replication_mode_type replication_mode) const { +void storage::validate_metadata( + replication_mode_type replication_mode, + const optional_encryption_format_type &encryption_format) const { if (replication_mode != replication_mode_) { util::exception_location().raise( "replication mode provided to initialize storage differs from the one " "stored in metadata"); } + + if (encryption_format != encryption_format_) { + util::exception_location().raise( + "storage encryption format provided to initialize storage differs from " + "the one stored in metadata"); + } } void storage::save_metadata() const { storage_metadata metadata{}; metadata.root().get<"mode">() = replication_mode_; + metadata.root().get<"encryption">() = encryption_format_; const auto content{metadata.str()}; backend_->put_object(metadata_name, util::as_const_byte_span(content)); } diff --git a/src/binsrv/storage.hpp b/src/binsrv/storage.hpp index 192d41c..3dcd685 100644 --- a/src/binsrv/storage.hpp +++ b/src/binsrv/storage.hpp @@ -24,7 +24,10 @@ #include #include +#include "binsrv/basic_keyring_fwd.hpp" #include "binsrv/basic_storage_backend_fwd.hpp" +#include "binsrv/encryption_format_type_fwd.hpp" +#include "binsrv/keyring_record.hpp" #include "binsrv/replication_mode_type_fwd.hpp" #include "binsrv/storage_config_fwd.hpp" @@ -170,8 +173,18 @@ class [[nodiscard]] storage { [[nodiscard]] std::string get_binlog_uri(const events::composite_binlog_name &binlog_name) const; + [[nodiscard]] bool is_encryption_enabled() const noexcept { + return static_cast(keyring_); + } + [[nodiscard]] std::string get_keyring_description() const; + [[nodiscard]] std::string get_active_kek_description() const; + private: storage_construction_mode_type construction_mode_; + basic_keyring_ptr keyring_; + optional_encryption_format_type encryption_format_; + keyring_record active_kek_; + std::string active_data_cipher_{}; basic_storage_backend_ptr backend_; replication_mode_type replication_mode_; @@ -238,7 +251,9 @@ class [[nodiscard]] storage { void save_binlog_index() const; void load_metadata(); - void validate_metadata(replication_mode_type replication_mode) const; + void validate_metadata( + replication_mode_type replication_mode, + const optional_encryption_format_type &encryption_format) const; void save_metadata() const; [[nodiscard]] static std::string generate_binlog_metadata_name( diff --git a/src/binsrv/storage_metadata.cpp b/src/binsrv/storage_metadata.cpp index 3b68336..eac64ee 100644 --- a/src/binsrv/storage_metadata.cpp +++ b/src/binsrv/storage_metadata.cpp @@ -23,6 +23,8 @@ #include #include +// Needed for encryption_format_type's operator << +#include "binsrv/encryption_format_type.hpp" // IWYU pragma: keep // Needed for replication_mode_type's operator << #include "binsrv/replication_mode_type.hpp" // IWYU pragma: keep @@ -32,7 +34,7 @@ namespace binsrv { storage_metadata::storage_metadata() - : impl_{{expected_storage_metadata_version}, {}} {} + : impl_{{expected_storage_metadata_version}, {}, {}} {} storage_metadata::storage_metadata(std::string_view data) : impl_{} { auto json_value = boost::json::parse(data); diff --git a/src/binsrv/storage_metadata.hpp b/src/binsrv/storage_metadata.hpp index 7c02a92..952349a 100644 --- a/src/binsrv/storage_metadata.hpp +++ b/src/binsrv/storage_metadata.hpp @@ -21,6 +21,7 @@ #include #include +#include "binsrv/encryption_format_type_fwd.hpp" #include "binsrv/replication_mode_type_fwd.hpp" #include "util/nv_tuple.hpp" @@ -32,7 +33,8 @@ class [[nodiscard]] storage_metadata { using impl_type = util::nv_tuple< // clang-format off util::nv<"version", std::uint32_t>, - util::nv<"mode", replication_mode_type> + util::nv<"mode", replication_mode_type>, + util::nv<"encryption", optional_encryption_format_type> // clang-format on >;