From 86b47bf41419f34143e2531f954e4a0581119a59 Mon Sep 17 00:00:00 2001 From: atarekra Date: Sun, 9 Nov 2025 17:36:56 +0200 Subject: [PATCH 1/6] add compile-time configuration for kvs maximum storage size --- src/cpp/src/kvs.cpp | 51 ++++++++++++++++++++++++++++++++++++++ src/cpp/src/kvs.hpp | 4 +++ src/cpp/tests/test_kvs.cpp | 42 +++++++++++++++++++++++++++++++ 3 files changed, 97 insertions(+) diff --git a/src/cpp/src/kvs.cpp b/src/cpp/src/kvs.cpp index 9cfd8a3d2..6087f52a5 100644 --- a/src/cpp/src/kvs.cpp +++ b/src/cpp/src/kvs.cpp @@ -15,6 +15,7 @@ #include #include "internal/kvs_helper.hpp" #include "kvs.hpp" +#include //TODO Default Value Handling TBD //TODO Add Score Logging @@ -375,10 +376,60 @@ score::ResultBlank Kvs::remove_key(const std::string_view key) { return result; } +score::Result Kvs::get_file_size(const score::filesystem::Path& file_path) { + std::error_code ec; + const auto size = std::filesystem::file_size(file_path.CStr(), ec); + + if (ec) { + // Check if the error is "file not found" + if (ec == std::errc::no_such_file_or_directory) { + // File does not exist, its size is 0. This is not an error. + return 0; + } + logger->LogError() << "Error: Could not get size of file " << file_path << ": " << ec.message(); + return score::MakeUnexpected(ErrorCode::PhysicalStorageFailure); + } + + return size; +} + +/* Helper Function to get current storage size */ +score::Result Kvs::get_current_storage_size() { + size_t total_size = 0; + const std::array file_extensions = {".json", ".hash"}; + + for (size_t snapshot_index = 0; snapshot_index <= KVS_MAX_SNAPSHOTS; ++snapshot_index) { + const std::string snapshot_suffix = "_" + to_string(snapshot_index); + + for (const char* extension : file_extensions) { + const score::filesystem::Path file_path = filename_prefix.Native() + snapshot_suffix + extension; + auto size_result = get_file_size(file_path); + if (!size_result) { + return size_result; // Propagate error directly + } + total_size += size_result.value(); + } + } + return total_size; +} + /* Helper Function to write JSON data to a file for flush process (also adds Hash file)*/ score::ResultBlank Kvs::write_json_data(const std::string& buf) { score::ResultBlank result = score::MakeUnexpected(ErrorCode::UnmappedError); + + auto current_size_res = get_current_storage_size(); + if (!current_size_res) { + return score::MakeUnexpected(static_cast(*current_size_res.error())); + } + + const size_t total_size = current_size_res.value() + buf.size(); + + if (total_size > KVS_MAX_STORAGE_BYTES) { + logger->LogError() << "error: KVS storage limit exceeded. total_size:" << total_size << " KVS_MAX_STORAGE_BYTES:" << KVS_MAX_STORAGE_BYTES; + return score::MakeUnexpected(ErrorCode::OutOfStorageSpace); + } + score::filesystem::Path json_path{filename_prefix.Native() + "_0.json"}; score::filesystem::Path dir = json_path.ParentPath(); if (!dir.Empty()) { diff --git a/src/cpp/src/kvs.hpp b/src/cpp/src/kvs.hpp index eeb6cf9b5..ee9b0bd40 100644 --- a/src/cpp/src/kvs.hpp +++ b/src/cpp/src/kvs.hpp @@ -29,6 +29,8 @@ #include "score/mw/log/logger.h" #define KVS_MAX_SNAPSHOTS 3 +#define KVS_MAX_STORAGE_BYTES (100) /* Max total storage size for all snapshots including hash files in bytes */ + namespace score::mw::per::kvs { @@ -367,6 +369,8 @@ class Kvs final { std::unique_ptr logger; /* Private Methods */ + score::Result get_file_size(const score::filesystem::Path& file_path); + score::Result get_current_storage_size(); score::ResultBlank snapshot_rotate(); score::Result> parse_json_data(const std::string& data); score::Result> open_json(const score::filesystem::Path& prefix, OpenJsonNeedFile need_file); diff --git a/src/cpp/tests/test_kvs.cpp b/src/cpp/tests/test_kvs.cpp index fda2db003..265ae252d 100644 --- a/src/cpp/tests/test_kvs.cpp +++ b/src/cpp/tests/test_kvs.cpp @@ -1216,3 +1216,45 @@ TEST(kvs_get_filename, get_hashname_failure){ cleanup_environment(); } + + +TEST(kvs, flush_fails_when_storage_limit_exceeded) { + // Use a unique directory for this test to avoid conflicts + const std::string test_dir = "./kvs_storage_test/"; + // Ensure the directory is clean before starting + std::filesystem::remove_all(test_dir); + + // 1. Setup KVS + KvsBuilder builder(instance_id); + builder.dir("./kvs_storage_test/"); + auto open_res = builder.build(); + ASSERT_TRUE(open_res); + Kvs kvs = std::move(open_res.value()); + + // 2. Add data that is close to the limit. + // There is overhead for the JSON structure (key, type info, braces, etc.) and the hash file (4 bytes). + // We will make the data payload a bit smaller than the max to account for this. + size_t overhead_estimate = 100; + size_t data_size = KVS_MAX_STORAGE_BYTES - overhead_estimate; + std::string large_data(data_size, 'a'); + + auto set_res1 = kvs.set_value("large_data", KvsValue(large_data.c_str())); + ASSERT_TRUE(set_res1); + + // 3. Flush the first batch of data and assert it succeeds. + auto flush_res1 = kvs.flush(); + ASSERT_TRUE(flush_res1); + + // 4. Add a little more data, which should push the total size over the limit. + auto set_res2 = kvs.set_value("extra_data", KvsValue("this should not fit")); + ASSERT_TRUE(set_res2); + + // 5. The second flush should fail because the storage limit is exceeded. + auto flush_res2 = kvs.flush(); + ASSERT_FALSE(flush_res2); + ASSERT_EQ(static_cast(*flush_res2.error()), ErrorCode::OutOfStorageSpace); + + // Cleanup the test directory + std::filesystem::remove_all(test_dir); +} + \ No newline at end of file From 39988267a783855b3d02012dfed10e5e1a883d3b Mon Sep 17 00:00:00 2001 From: atarekra Date: Thu, 18 Dec 2025 14:07:55 +0200 Subject: [PATCH 2/6] Add calculate_potential_size for storage pre-check --- src/cpp/src/kvs.cpp | 123 +++++++++++++++++++++++-------------- src/cpp/src/kvs.hpp | 21 ++++++- src/cpp/tests/test_kvs.cpp | 48 ++++++++++++++- 3 files changed, 142 insertions(+), 50 deletions(-) diff --git a/src/cpp/src/kvs.cpp b/src/cpp/src/kvs.cpp index 0da563143..30b9d1a86 100644 --- a/src/cpp/src/kvs.cpp +++ b/src/cpp/src/kvs.cpp @@ -265,6 +265,7 @@ score::Result Kvs::key_exists(const std::string_view key) { return result; } + /* Retrieve the value associated with a key*/ score::Result Kvs::get_value(const std::string_view key) { score::Result result = score::MakeUnexpected(ErrorCode::UnmappedError); @@ -393,12 +394,26 @@ score::Result Kvs::get_file_size(const score::filesystem::Path& file_pat return size; } -/* Helper Function to get current storage size */ +/* Helper Function to get current storage size of all persisted files (defaults and historical snapshots) */ score::Result Kvs::get_current_storage_size() { size_t total_size = 0; const std::array file_extensions = {".json", ".hash"}; - for (size_t snapshot_index = 0; snapshot_index <= KVS_MAX_SNAPSHOTS; ++snapshot_index) { + // Add the size of the default files + const std::string default_suffix = "_default"; + for (const char* extension : file_extensions) { + const score::filesystem::Path file_path = filename_prefix.Native() + default_suffix + extension; + auto size_result = get_file_size(file_path); + if (!size_result) { + return size_result; // Propagate error directly + } + total_size += size_result.value(); + } + + // Add the size of historical snapshots (1 to N). + // The loop starts at 1 to intentionally exclude the current working snapshot (index 0), + // allowing the caller to add its size manually for a final check. + for (size_t snapshot_index = 1; snapshot_index <= KVS_MAX_SNAPSHOTS; ++snapshot_index) { const std::string snapshot_suffix = "_" + to_string(snapshot_index); for (const char* extension : file_extensions) { @@ -417,19 +432,6 @@ score::Result Kvs::get_current_storage_size() { score::ResultBlank Kvs::write_json_data(const std::string& buf) { score::ResultBlank result = score::MakeUnexpected(ErrorCode::UnmappedError); - - auto current_size_res = get_current_storage_size(); - if (!current_size_res) { - return score::MakeUnexpected(static_cast(*current_size_res.error())); - } - - const size_t total_size = current_size_res.value() + buf.size(); - - if (total_size > KVS_MAX_STORAGE_BYTES) { - logger->LogError() << "error: KVS storage limit exceeded. total_size:" << total_size << " KVS_MAX_STORAGE_BYTES:" << KVS_MAX_STORAGE_BYTES; - return score::MakeUnexpected(ErrorCode::OutOfStorageSpace); - } - score::filesystem::Path json_path{filename_prefix.Native() + "_0.json"}; score::filesystem::Path dir = json_path.ParentPath(); if (!dir.Empty()) { @@ -460,53 +462,80 @@ score::ResultBlank Kvs::write_json_data(const std::string& buf) return result; } -/* Flush the key-value store*/ -score::ResultBlank Kvs::flush() { - score::ResultBlank result = score::MakeUnexpected(ErrorCode::UnmappedError); - /* Create JSON Object */ +score::Result Kvs::serialize_and_check() { score::json::Object root_obj; - bool error = false; + + // 1. Serialize the current KVS map to a buffer { std::unique_lock lock(kvs_mutex, std::try_to_lock); if (lock.owns_lock()) { for (auto const& [key, value] : kvs) { auto conv = kvsvalue_to_any(value); if (!conv) { - result = score::MakeUnexpected(static_cast(*conv.error())); - error = true; - break; - }else{ - root_obj.emplace( - key, - std::move(conv.value()) /*emplace in map uses move operator*/ - ); + return score::MakeUnexpected(static_cast(*conv.error())); } + root_obj.emplace(key, std::move(conv.value())); } } else { - result = score::MakeUnexpected(ErrorCode::MutexLockFailed); - error = true; + return score::MakeUnexpected(ErrorCode::MutexLockFailed); } } - if(!error){ - /* Serialize Buffer */ - auto buf_res = writer->ToBuffer(root_obj); - if (!buf_res) { - result = score::MakeUnexpected(ErrorCode::JsonGeneratorError); - }else{ - /* Rotate Snapshots */ - auto rotate_result = snapshot_rotate(); - if (!rotate_result) { - result = rotate_result; - }else{ - /* Write JSON Data */ - std::string buf = std::move(buf_res.value()); - result = write_json_data(buf); - } - } + auto buf_res = writer->ToBuffer(root_obj); + if (!buf_res) { + return score::MakeUnexpected(ErrorCode::JsonGeneratorError); } + const std::string& buf = buf_res.value(); - return result; + // 2. Get the size of all other persisted files + auto current_size_res = get_current_storage_size(); + if (!current_size_res) { + return score::MakeUnexpected(static_cast(*current_size_res.error())); + } + + // 3. Calculate the potential total size + const size_t total_size = current_size_res.value() + buf.size() + HASH_FILE_SIZE; + + // 4. Check against the limit + if (total_size > KVS_MAX_STORAGE_BYTES) { + logger->LogError() << "error: KVS storage limit would be exceeded. total_size:" << total_size + << " KVS_MAX_STORAGE_BYTES:" << KVS_MAX_STORAGE_BYTES; + return score::MakeUnexpected(ErrorCode::OutOfStorageSpace); + } + + return buf; +} + +/* Flush the key-value store*/ +score::ResultBlank Kvs::flush() { + auto result = serialize_and_check(); + if (!result) { + return score::MakeUnexpected(static_cast(*result.error())); + } + + auto rotate_result = snapshot_rotate(); + if (!rotate_result) { + return rotate_result; + } + + return write_json_data(result.value()); +} + +/* Performs a 'dry run' to check the potential storage size */ +score::Result Kvs::calculate_potential_size() { + auto result = serialize_and_check(); + if (!result) { + return score::MakeUnexpected(static_cast(*result.error())); + } + + // Re-calculate size to return it, as serialize_and_check only returns the buffer + const std::string& buf = result.value(); + auto current_size_res = get_current_storage_size(); + if (!current_size_res) { + return score::MakeUnexpected(static_cast(*current_size_res.error())); + } + + return current_size_res.value() + buf.size() + HASH_FILE_SIZE; } /* Retrieve the snapshot count*/ diff --git a/src/cpp/src/kvs.hpp b/src/cpp/src/kvs.hpp index f028cbc20..dd6b2cff0 100644 --- a/src/cpp/src/kvs.hpp +++ b/src/cpp/src/kvs.hpp @@ -29,8 +29,8 @@ #include "score/mw/log/logger.h" #define KVS_MAX_SNAPSHOTS 3 -#define KVS_MAX_STORAGE_BYTES (100) /* Max total storage size for all snapshots including hash files in bytes */ - +#define KVS_MAX_STORAGE_BYTES (10000) /* Max total storage size for all snapshots including hash files in bytes */ +static constexpr size_t HASH_FILE_SIZE = 4; namespace score::mw::per::kvs { @@ -283,6 +283,22 @@ class Kvs final { score::ResultBlank flush(); + /** + * @brief Performs a 'dry run' to check if the current in-memory store would + * exceed the storage limit upon flushing. + * + * This function serializes the current key-value data to a temporary buffer + * and calculates the potential total storage size. It checks this size against + * the compile-time `KVS_MAX_STORAGE_BYTES` limit. + * + * @return A score::Result object containing either: + * - On success: The estimated total size (size_t) that the KVS would occupy after a flush. + * - On failure: An `OutOfStorageSpace` error if the limit would be exceeded, + * or another ErrorCode for other failures (e.g., serialization). + */ + score::Result calculate_potential_size(); + + /** * @brief Retrieves the number of snapshots currently stored in the key-value store. * @@ -369,6 +385,7 @@ class Kvs final { std::unique_ptr logger; /* Private Methods */ + score::Result serialize_and_check(); score::Result get_file_size(const score::filesystem::Path& file_path); score::Result get_current_storage_size(); score::ResultBlank snapshot_rotate(); diff --git a/src/cpp/tests/test_kvs.cpp b/src/cpp/tests/test_kvs.cpp index 95d68bf07..357274323 100644 --- a/src/cpp/tests/test_kvs.cpp +++ b/src/cpp/tests/test_kvs.cpp @@ -1257,4 +1257,50 @@ TEST(kvs, flush_fails_when_storage_limit_exceeded) { // Cleanup the test directory std::filesystem::remove_all(test_dir); } - \ No newline at end of file + + +TEST(kvs_check_size, check_size_scenarios) { + const std::string test_dir = "./kvs_check_size_test/"; + std::filesystem::remove_all(test_dir); + + // --- SCENARIO 1: Success on data within limits --- + { + KvsBuilder builder(InstanceId(1)); + builder.dir(std::string(test_dir)); + auto open_res = builder.build(); + ASSERT_TRUE(open_res); + Kvs kvs = std::move(open_res.value()); + + // Add a small amount of data + auto set_res = kvs.set_value("key", KvsValue("some_data")); + ASSERT_TRUE(set_res); + + // Check the size + auto check_res = kvs.check_size(); + ASSERT_TRUE(check_res) << "check_size should succeed for data within limits"; + // Check if the returned size is plausible + EXPECT_GT(check_res.value(), 0); + EXPECT_LT(check_res.value(), KVS_MAX_STORAGE_BYTES); + } + + // --- SCENARIO 2: Failure on data exceeding limits --- + { + KvsBuilder builder(InstanceId(2)); + builder.dir(std::string(test_dir)); + auto open_res = builder.build(); + ASSERT_TRUE(open_res); + Kvs kvs = std::move(open_res.value()); + + // Add data that will certainly exceed the storage limit + std::string large_data(KVS_MAX_STORAGE_BYTES, 'x'); + auto set_res = kvs.set_value("oversized_key", KvsValue(large_data.c_str())); + ASSERT_TRUE(set_res); + + // Check the size, expecting a failure + auto check_res = kvs.check_size(); + ASSERT_FALSE(check_res) << "check_size should fail when storage limit is exceeded"; + EXPECT_EQ(static_cast(*check_res.error()), ErrorCode::OutOfStorageSpace); + } + + std::filesystem::remove_all(test_dir); +} From 40ff6f95a026bc752f83471253c5cf30622a8a59 Mon Sep 17 00:00:00 2001 From: atarekra Date: Thu, 18 Dec 2025 14:53:24 +0200 Subject: [PATCH 3/6] Fixing Test API naming --- src/cpp/tests/test_kvs.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cpp/tests/test_kvs.cpp b/src/cpp/tests/test_kvs.cpp index 59e21df96..57feeeb5d 100644 --- a/src/cpp/tests/test_kvs.cpp +++ b/src/cpp/tests/test_kvs.cpp @@ -1271,7 +1271,7 @@ TEST(kvs_check_size, check_size_scenarios) { ASSERT_TRUE(set_res); // Check the size - auto check_res = kvs.check_size(); + auto check_res = kvs.calculate_potential_size(); ASSERT_TRUE(check_res) << "check_size should succeed for data within limits"; // Check if the returned size is plausible EXPECT_GT(check_res.value(), 0); @@ -1292,7 +1292,7 @@ TEST(kvs_check_size, check_size_scenarios) { ASSERT_TRUE(set_res); // Check the size, expecting a failure - auto check_res = kvs.check_size(); + auto check_res = kvs.calculate_potential_size(); ASSERT_FALSE(check_res) << "check_size should fail when storage limit is exceeded"; EXPECT_EQ(static_cast(*check_res.error()), ErrorCode::OutOfStorageSpace); } From 5182f6b3681ee2268470af2f6090a2468c524725 Mon Sep 17 00:00:00 2001 From: atarekra Date: Mon, 2 Feb 2026 13:14:43 +0200 Subject: [PATCH 4/6] Fixing build errors --- src/cpp/src/kvs.hpp | 120 +++++++------------------------------------- 1 file changed, 17 insertions(+), 103 deletions(-) diff --git a/src/cpp/src/kvs.hpp b/src/cpp/src/kvs.hpp index 320a4beca..c02a76c67 100644 --- a/src/cpp/src/kvs.hpp +++ b/src/cpp/src/kvs.hpp @@ -33,8 +33,8 @@ static constexpr size_t HASH_FILE_SIZE = 4; namespace score::mw::per::kvs -{ +{ struct InstanceId { size_t id; @@ -354,106 +354,20 @@ class Kvs final */ score::Result get_hash_filename(const SnapshotId& snapshot_id) const; - private: - /* Private constructor to prevent direct instantiation */ - Kvs(); - - /* Internal storage and configuration details.*/ - std::mutex kvs_mutex; - std::unordered_map kvs; - - /* Optional default values */ - std::unordered_map default_values; - - /* Filename prefix */ - score::filesystem::Path filename_prefix; - - /* Filesystem handling */ - std::unique_ptr filesystem; - - /* Json handling */ - std::unique_ptr parser; - std::unique_ptr writer; - - /* Logging */ - std::unique_ptr logger; - - /** - * @brief Performs a 'dry run' to check if the current in-memory store would - * exceed the storage limit upon flushing. - * - * This function serializes the current key-value data to a temporary buffer - * and calculates the potential total storage size. It checks this size against - * the compile-time `KVS_MAX_STORAGE_BYTES` limit. - * - * @return A score::Result object containing either: - * - On success: The estimated total size (size_t) that the KVS would occupy after a flush. - * - On failure: An `OutOfStorageSpace` error if the limit would be exceeded, - * or another ErrorCode for other failures (e.g., serialization). - */ - score::Result calculate_potential_size(); - - - /** - * @brief Retrieves the number of snapshots currently stored in the key-value store. - * - * @return A score::Result object that indicates the success or failure of the operation. - * - On success: The total count of snapshots as a size_t value. - * - On failure: Returns an ErrorCode describing the error. - */ - score::Result snapshot_count() const; - - - /** - * @brief Retrieves the maximum number of snapshots that can be stored. - * - * This function returns the upper limit on the number of snapshots - * that the key-value store can maintain at any given time. - * - * @return The maximum count of snapshots as a size_t value. - */ - size_t snapshot_max_count() const; - - - /** - * @brief Restores the state of the key-value store from a specified snapshot. - * - * This function attempts to restore the key-value store to the state - * captured in the snapshot identified by the given snapshot ID. If the - * restoration process fails, an appropriate error code is returned. - * - * @param snapshot_id The identifier of the snapshot to restore from. - * @return score::ResultBlank - * - On success: An empty score::Result indicating the restoration was successful. - * - On failure: An error code describing the reason for the failure. - */ - score::ResultBlank snapshot_restore(const SnapshotId& snapshot_id); - - - /** - * @brief Retrieves the filename associated with a given snapshot ID in the key-value store. - * - * @param snapshot_id The identifier of the snapshot for which the filename is to be retrieved. - * @return score::ResultBlank - * - On success: A score::filesystem::Path with the filename (path) associated with the snapshot ID. - * - On failure: An error code describing the reason for the failure. - */ - score::Result get_kvs_filename(const SnapshotId& snapshot_id) const; - - - /** - * @brief Retrieves the filename of the hash file associated with a given snapshot ID. - * - * This function returns a string view representing the filename of the hash file - * corresponding to the provided snapshot ID. The hash file is typically used to - * store metadata or integrity information for the snapshot. - * - * @param snapshot_id The identifier of the snapshot for which the hash filename is requested. - * @return score::ResultBlank - * - On success: A score::filesystem::Path with the filename (path) of the hash file associated with the snapshot ID. - * - On failure: An error code describing the reason for the failure. - */ - score::Result get_hash_filename(const SnapshotId& snapshot_id) const; + /** + * @brief Performs a 'dry run' to check if the current in-memory store would + * exceed the storage limit upon flushing. + * + * This function serializes the current key-value data to a temporary buffer + * and calculates the potential total storage size. It checks this size against + * the compile-time `KVS_MAX_STORAGE_BYTES` limit. + * + * @return A score::Result object containing either: + * - On success: The estimated total size (size_t) that the KVS would occupy after a flush. + * - On failure: An `OutOfStorageSpace` error if the limit would be exceeded, + * or another ErrorCode for other failures (e.g., serialization). + */ + score::Result calculate_potential_size(); private: /* Private constructor to prevent direct instantiation */ @@ -488,8 +402,8 @@ class Kvs final score::Result> open_json(const score::filesystem::Path& prefix, OpenJsonNeedFile need_file); score::ResultBlank write_json_data(const std::string& buf); -}; + }; -} /* namespace score::mw::per::kvs */ +}/* namespace score::mw::per::kvs */ #endif /* SCORE_LIB_KVS_KVS_HPP */ From 72f9fc09b76ed6e2c60acc36d36b531af601bf03 Mon Sep 17 00:00:00 2001 From: atarekra Date: Sun, 7 Jun 2026 13:06:23 +0300 Subject: [PATCH 5/6] Updating the parameter to be init-time configurable --- src/cpp/src/kvs.cpp | 18 ++--- src/cpp/src/kvs.hpp | 15 ++-- src/cpp/src/kvsbuilder.cpp | 12 +++- src/cpp/src/kvsbuilder.hpp | 12 ++++ src/cpp/tests/BUILD | 0 src/cpp/tests/bm_kvs.cpp | 0 src/cpp/tests/test_kvs.cpp | 110 ++++++++++++++--------------- src/cpp/tests/test_kvs_builder.cpp | 0 src/cpp/tests/test_kvs_error.cpp | 0 src/cpp/tests/test_kvs_general.cpp | 0 src/cpp/tests/test_kvs_general.hpp | 0 src/cpp/tests/test_kvs_helper.cpp | 0 12 files changed, 98 insertions(+), 69 deletions(-) mode change 100644 => 100755 src/cpp/tests/BUILD mode change 100644 => 100755 src/cpp/tests/bm_kvs.cpp mode change 100644 => 100755 src/cpp/tests/test_kvs.cpp mode change 100644 => 100755 src/cpp/tests/test_kvs_builder.cpp mode change 100644 => 100755 src/cpp/tests/test_kvs_error.cpp mode change 100644 => 100755 src/cpp/tests/test_kvs_general.cpp mode change 100644 => 100755 src/cpp/tests/test_kvs_general.hpp mode change 100644 => 100755 src/cpp/tests/test_kvs_helper.cpp diff --git a/src/cpp/src/kvs.cpp b/src/cpp/src/kvs.cpp index 7432e0d61..f8d82db58 100644 --- a/src/cpp/src/kvs.cpp +++ b/src/cpp/src/kvs.cpp @@ -48,7 +48,8 @@ Kvs::Kvs(Kvs&& other) noexcept object would also be okay*/ , writer(std::move(other.writer)), - logger(std::move(other.logger)) + logger(std::move(other.logger)), + max_storage_bytes(other.max_storage_bytes) { { std::lock_guard lock(other.kvs_mutex); @@ -82,6 +83,7 @@ Kvs& Kvs::operator=(Kvs&& other) noexcept parser = std::move(other.parser); writer = std::move(other.writer); logger = std::move(other.logger); + max_storage_bytes = other.max_storage_bytes; } return *this; } @@ -219,7 +221,8 @@ score::Result> Kvs::open_json(const score:: score::Result Kvs::open(const InstanceId& instance_id, OpenNeedDefaults need_defaults, OpenNeedKvs need_kvs, - const std::string&& dir) + const std::string&& dir, + std::optional max_storage_bytes) { score::Result result = score::MakeUnexpected(ErrorCode::UnmappedError); /* Redundant initialization needed, since Resul would call @@ -231,6 +234,7 @@ score::Result Kvs::open(const InstanceId& instance_id, const score::filesystem::Path filename_kvs = filename_prefix.Native() + "_0"; Kvs kvs; /* Create KVS instance */ + kvs.max_storage_bytes = max_storage_bytes; /* Store maximum storage limit */ auto default_res = kvs.open_json( filename_default, need_defaults == OpenNeedDefaults::Required ? OpenJsonNeedFile::Required : OpenJsonNeedFile::Optional); @@ -507,10 +511,8 @@ score::Result Kvs::get_current_storage_size() { total_size += size_result.value(); } - // Add the size of historical snapshots (1 to N). - // The loop starts at 1 to intentionally exclude the current working snapshot (index 0), - // allowing the caller to add its size manually for a final check. - for (size_t snapshot_index = 1; snapshot_index <= KVS_MAX_SNAPSHOTS; ++snapshot_index) { + // Add the size of current and historical snapshots that will be kept after rotation (0 to N-1). + for (size_t snapshot_index = 0; snapshot_index < KVS_MAX_SNAPSHOTS; ++snapshot_index) { const std::string snapshot_suffix = "_" + to_string(snapshot_index); for (const char* extension : file_extensions) { @@ -639,9 +641,9 @@ score::Result Kvs::serialize_and_check() { const size_t total_size = current_size_res.value() + buf.size() + HASH_FILE_SIZE; // 4. Check against the limit - if (total_size > KVS_MAX_STORAGE_BYTES) { + if (this->max_storage_bytes.has_value() && total_size > this->max_storage_bytes.value()) { logger->LogError() << "error: KVS storage limit would be exceeded. total_size:" << total_size - << " KVS_MAX_STORAGE_BYTES:" << KVS_MAX_STORAGE_BYTES; + << " max_storage_bytes:" << this->max_storage_bytes.value(); return score::MakeUnexpected(ErrorCode::OutOfStorageSpace); } diff --git a/src/cpp/src/kvs.hpp b/src/cpp/src/kvs.hpp index 936e28e7e..9b9bc1c85 100644 --- a/src/cpp/src/kvs.hpp +++ b/src/cpp/src/kvs.hpp @@ -27,9 +27,9 @@ #include #include #include +#include #define KVS_MAX_SNAPSHOTS 3 -#define KVS_MAX_STORAGE_BYTES (10000) /* Max total storage size for all snapshots including hash files in bytes */ static constexpr size_t HASH_FILE_SIZE = 4; namespace score::mw::per::kvs @@ -174,7 +174,8 @@ class Kvs final static score::Result open(const InstanceId& instance_id, OpenNeedDefaults need_defaults, OpenNeedKvs need_kvs, - const std::string&& dir); + const std::string&& dir, + std::optional max_storage_bytes); /** * @brief Resets a key-value-storage to its initial state @@ -360,7 +361,7 @@ class Kvs final * * This function serializes the current key-value data to a temporary buffer * and calculates the potential total storage size. It checks this size against - * the compile-time `KVS_MAX_STORAGE_BYTES` limit. + * the configured `max_storage_bytes` limit. * * @return A score::Result object containing either: * - On success: The estimated total size (size_t) that the KVS would occupy after a flush. @@ -392,6 +393,9 @@ class Kvs final /* Logging */ std::unique_ptr logger; + + /* Maximum storage limit in bytes */ + std::optional max_storage_bytes; /* Private Methods */ score::ResultBlank snapshot_rotate(); score::Result> parse_json_data(const std::string& data); @@ -399,9 +403,12 @@ class Kvs final OpenJsonNeedFile need_file); score::ResultBlank write_json_data(const std::string& buf); score::ResultBlank write_and_sync(const std::string& path, const void* data, std::size_t size); + score::Result serialize_and_check(); score::Result get_file_size(const score::filesystem::Path& file_path); score::Result get_current_storage_size(); -};/* namespace score::mw::per::kvs */ +}; + +} /* namespace score::mw::per::kvs */ #endif /* SCORE_LIB_KVS_KVS_HPP */ diff --git a/src/cpp/src/kvsbuilder.cpp b/src/cpp/src/kvsbuilder.cpp index a248ebfea..c1c4b138d 100644 --- a/src/cpp/src/kvsbuilder.cpp +++ b/src/cpp/src/kvsbuilder.cpp @@ -20,7 +20,8 @@ KvsBuilder::KvsBuilder(const InstanceId& instance_id) : instance_id(instance_id), need_defaults(false), need_kvs(false), - directory("./data_folder/") /* Default Directory */ + directory("./data_folder/"), /* Default Directory */ + maximum_size(KVS_DEFAULT_MAX_SIZE_BYTES) /* Default max size in bytes */ { } @@ -42,6 +43,12 @@ KvsBuilder& KvsBuilder::dir(std::string&& dir_path) return *this; } +KvsBuilder& KvsBuilder::max_size(std::optional max_bytes) +{ + this->maximum_size = max_bytes; + return *this; +} + score::Result KvsBuilder::build() { score::Result result = score::MakeUnexpected(ErrorCode::UnmappedError); @@ -55,7 +62,8 @@ score::Result KvsBuilder::build() result = Kvs::open(instance_id, need_defaults ? OpenNeedDefaults::Required : OpenNeedDefaults::Optional, need_kvs ? OpenNeedKvs::Required : OpenNeedKvs::Optional, - std::move(directory)); + std::move(directory), + maximum_size); return result; } diff --git a/src/cpp/src/kvsbuilder.hpp b/src/cpp/src/kvsbuilder.hpp index 2f6e1f6ef..35d69cb53 100644 --- a/src/cpp/src/kvsbuilder.hpp +++ b/src/cpp/src/kvsbuilder.hpp @@ -15,10 +15,13 @@ #include "kvs.hpp" #include +#include namespace score::mw::per::kvs { +constexpr size_t KVS_DEFAULT_MAX_SIZE_BYTES = 1000; + /** * @class KvsBuilder * @brief Builder for opening a KVS object. @@ -89,6 +92,13 @@ class KvsBuilder final */ KvsBuilder& dir(std::string&& dir_path); + /** + * @brief Configure the maximum storage size for the KVS in bytes. + * @param max_bytes The maximum allowed total storage size in bytes. + * @return Reference to this builder (for chaining). + */ + KvsBuilder& max_size(std::optional max_bytes); + /** * @brief Builds and opens the Kvs instance with the configured options. * @@ -103,8 +113,10 @@ class KvsBuilder final bool need_defaults; ///< Whether default values are required bool need_kvs; ///< Whether an existing KVS is required std::string directory; ///< Directory where to store the KVS Files + std::optional maximum_size; ///< Maximum total storage size for the KVS in bytes }; } /* namespace score::mw::per::kvs */ #endif /* SCORE_LIB_KVS_KVSBUILDER_HPP */ + diff --git a/src/cpp/tests/BUILD b/src/cpp/tests/BUILD old mode 100644 new mode 100755 diff --git a/src/cpp/tests/bm_kvs.cpp b/src/cpp/tests/bm_kvs.cpp old mode 100644 new mode 100755 diff --git a/src/cpp/tests/test_kvs.cpp b/src/cpp/tests/test_kvs.cpp old mode 100644 new mode 100755 index afef81d86..edcca5754 --- a/src/cpp/tests/test_kvs.cpp +++ b/src/cpp/tests/test_kvs.cpp @@ -18,12 +18,12 @@ TEST(kvs_constructor, move_constructor) /* create object A */ auto result_a = - Kvs::open(InstanceId(instance_b), OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + Kvs::open(InstanceId(instance_b), OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result_a); Kvs kvs_a = std::move(result_a.value()); /* create object B */ - auto result_b = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + auto result_b = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result_b); Kvs kvs_b = std::move(result_b.value()); @@ -72,7 +72,7 @@ TEST(kvs_TEST, parse_json_data_sucess) prepare_environment(); auto kvs = - Kvs::open(InstanceId(instance_id), OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + Kvs::open(InstanceId(instance_id), OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(kvs); auto mock_parser = std::make_unique(); @@ -101,7 +101,7 @@ TEST(kvs_TEST, parse_json_data_failure) /* Json Parser Failure */ auto kvs = - Kvs::open(InstanceId(instance_id), OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + Kvs::open(InstanceId(instance_id), OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(kvs); auto mock_parser = std::make_unique(); @@ -156,7 +156,7 @@ TEST(kvs_open_json, open_json_success) prepare_environment(); auto kvs = - Kvs::open(InstanceId(instance_id), OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + Kvs::open(InstanceId(instance_id), OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(kvs); auto result = kvs->open_json(score::filesystem::Path(kvs_prefix), OpenJsonNeedFile::Required); @@ -240,7 +240,7 @@ TEST(kvs_open_json, open_json_hash_invalid) TEST(kvs_reset, reset_success) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); /* Check Data existing */ @@ -259,7 +259,7 @@ TEST(kvs_reset, reset_failure) prepare_environment(); /* Mutex locked */ - auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); std::unique_lock lock(result.value().kvs_mutex); @@ -274,7 +274,7 @@ TEST(kvs_get_all_keys, get_all_keys_success) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); /* Check Data existing */ @@ -301,7 +301,7 @@ TEST(kvs_get_all_keys, get_all_keys_failure) prepare_environment(); /* Mutex locked */ - auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); std::unique_lock lock(result.value().kvs_mutex); @@ -316,7 +316,7 @@ TEST(kvs_key_exists, key_exists_success) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); /* Check Data existing */ @@ -339,7 +339,7 @@ TEST(kvs_key_exists, key_exists_failure) prepare_environment(); /* Mutex locked */ - auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); std::unique_lock lock(result.value().kvs_mutex); @@ -354,7 +354,7 @@ TEST(kvs_get_value, get_value_success) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); /* Check Data existing */ @@ -383,7 +383,7 @@ TEST(kvs_get_value, get_value_failure) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); /* Check if non-existing key returns error */ @@ -392,7 +392,7 @@ TEST(kvs_get_value, get_value_failure) EXPECT_EQ(get_value_result.error(), ErrorCode::KeyNotFound); /* Mutex locked */ - result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir)); + result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); std::unique_lock lock(result.value().kvs_mutex); get_value_result = result.value().get_value("kvs"); @@ -406,7 +406,7 @@ TEST(kvs_get_default_value, get_default_value_success) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); /* Check Data existing */ @@ -426,7 +426,7 @@ TEST(kvs_get_default_value, get_default_value_failure) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); /* Check if non-existing key returns error */ @@ -441,7 +441,7 @@ TEST(kvs_reset_key, reset_key_success) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); ASSERT_TRUE(result.value().kvs.count("kvs")); /* Check Data existing */ @@ -466,7 +466,7 @@ TEST(kvs_reset_key, reset_key_failure) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); /* Reset a non-existing key */ @@ -475,7 +475,7 @@ TEST(kvs_reset_key, reset_key_failure) EXPECT_EQ(reset_key_result.error(), ErrorCode::KeyDefaultNotFound); /* Reset a key without default value */ - result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir)); + result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); result.value().default_values.clear(); // Clear default values to ensure no default value // exists for "kvs" @@ -484,7 +484,7 @@ TEST(kvs_reset_key, reset_key_failure) EXPECT_EQ(reset_key_result.error(), ErrorCode::KeyDefaultNotFound); /* Mutex locked */ - result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir)); + result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); std::unique_lock lock(result.value().kvs_mutex); reset_key_result = result.value().reset_key("kvs"); @@ -498,7 +498,7 @@ TEST(kvs_is_value_default, is_value_default) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); auto kvs{std::move(result.value())}; @@ -533,7 +533,7 @@ TEST(kvs_is_value_default, is_value_default) TEST(kvs_set_value, set_value_success) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); /* Set a new value */ @@ -557,7 +557,7 @@ TEST(kvs_set_value, set_value_failure) prepare_environment(); /* Mutex locked */ - auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); std::unique_lock lock(result.value().kvs_mutex); @@ -571,7 +571,7 @@ TEST(kvs_set_value, set_value_failure) TEST(kvs_remove_key, remove_key_success) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); /* Check Data existing */ @@ -589,7 +589,7 @@ TEST(kvs_remove_key, remove_key_failure) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); /* Remove a non-existing key */ @@ -598,7 +598,7 @@ TEST(kvs_remove_key, remove_key_failure) EXPECT_EQ(remove_key_result.error(), ErrorCode::KeyNotFound); /* Mutex locked */ - result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir)); + result = Kvs::open(instance_id, OpenNeedDefaults::Required, OpenNeedKvs::Required, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); std::unique_lock lock(result.value().kvs_mutex); remove_key_result = result.value().remove_key("kvs"); @@ -622,7 +622,7 @@ TEST(kvs_write_json_data, write_json_data_success) system(("rm -rf " + kvs_prefix + ".hash").c_str()); auto kvs = - Kvs::open(InstanceId(instance_id), OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + Kvs::open(InstanceId(instance_id), OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(kvs); kvs->filename_prefix = score::filesystem::Path(filename_prefix); /* Set the filename prefix to the test prefix */ @@ -658,7 +658,7 @@ TEST(kvs_write_json_data, write_json_data_filesystem_failure) system(("rm -rf " + kvs_prefix + ".hash").c_str()); auto kvs = - Kvs::open(InstanceId(instance_id), OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + Kvs::open(InstanceId(instance_id), OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(kvs); /* Mock Filesystem */ @@ -691,7 +691,7 @@ TEST(kvs_write_json_data, write_json_data_permissions_failure) system(("rm -rf " + kvs_prefix + ".hash").c_str()); auto kvs = - Kvs::open(InstanceId(instance_id), OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + Kvs::open(InstanceId(instance_id), OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(kvs); /* Test writing to a non-writable hash file */ @@ -723,7 +723,7 @@ TEST(kvs_snapshot_rotate, snapshot_rotate_success) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); /* Create empty Test-Snapshot Files */ @@ -753,7 +753,7 @@ TEST(kvs_snapshot_rotate, snapshot_rotate_max_snapshots) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); /* Create empty Test-Snapshot Files */ @@ -779,7 +779,7 @@ TEST(kvs_snapshot_rotate, snapshot_rotate_failure_renaming_json) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); /* Create empty Test-Snapshot Files */ @@ -804,7 +804,7 @@ TEST(kvs_snapshot_rotate, snapshot_rotate_failure_renaming_hash) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); /* Create empty Test-Snapshot Files */ @@ -828,7 +828,7 @@ TEST(kvs_snapshot_rotate, snapshot_rotate_failure_mutex) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); /* Mutex locked */ @@ -847,7 +847,7 @@ TEST(kvs_flush, flush_success_data) system(("rm -rf " + kvs_prefix + ".json").c_str()); system(("rm -rf " + kvs_prefix + ".hash").c_str()); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); result.value().kvs.clear(); /* Clear KVS to ensure no data is written */ @@ -873,7 +873,7 @@ TEST(kvs_flush, flush_success_snapshot_rotate) system(("rm -rf " + kvs_prefix + ".json").c_str()); system(("rm -rf " + kvs_prefix + ".hash").c_str()); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); EXPECT_FALSE(std::filesystem::exists(filename_prefix + "_1.json")); EXPECT_FALSE(std::filesystem::exists(filename_prefix + "_1.hash")); @@ -894,7 +894,7 @@ TEST(kvs_flush, flush_failure_mutex) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); std::unique_lock lock(result.value().kvs_mutex); @@ -912,7 +912,7 @@ TEST(kvs_flush, flush_failure_rotate_snapshots) std::string permissions_dir = data_dir + "permissions/"; std::filesystem::create_directories(permissions_dir); auto result = - Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(permissions_dir)); + Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(permissions_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); std::filesystem::permissions( @@ -929,7 +929,7 @@ TEST(kvs_flush, flush_failure_kvsvalue_invalid) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); BrokenKvsValue invalid; @@ -946,7 +946,7 @@ TEST(kvs_flush, flush_failure_json_writer) { prepare_environment(); - auto kvs = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + auto kvs = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(kvs); auto mock_writer = std::make_unique(); /* Force error in writer.ToBuffer */ @@ -966,7 +966,7 @@ TEST(kvs_snapshot_count, snapshot_count_success) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); /* Create empty Test-Snapshot Files */ @@ -990,7 +990,7 @@ TEST(kvs_snapshot_count, snapshot_count_invalid) { prepare_environment(); - auto kvs = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + auto kvs = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(kvs); /* Mock Filesystem */ @@ -1011,7 +1011,7 @@ TEST(kvs_snapshot_restore, snapshot_restore_success) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); /* Create empty Test-Snapshot Files -> Data received by the JsonParser should be the data listed @@ -1051,7 +1051,7 @@ TEST(kvs_snapshot_restore, snapshot_restore_failure_invalid_snapshot_id) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); /* Restore Snapshot ID 0 -> Current KVS*/ @@ -1071,7 +1071,7 @@ TEST(kvs_snapshot_restore, snapshot_restore_failure_open_json) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); /* Create empty Test-Snapshot Files */ @@ -1089,7 +1089,7 @@ TEST(kvs_snapshot_restore, snapshot_restore_failure_mutex) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); std::unique_lock lock(result.value().kvs_mutex); @@ -1104,7 +1104,7 @@ TEST(kvs_snapshot_restore, snapshot_restore_failure_snapshot_count) { prepare_environment(); - auto kvs = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + auto kvs = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(kvs); /* Mock Filesystem */ @@ -1126,7 +1126,7 @@ TEST(kvs_snapshot_max_count, snapshot_max_count) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); EXPECT_EQ(result.value().snapshot_max_count(), KVS_MAX_SNAPSHOTS); @@ -1137,7 +1137,7 @@ TEST(kvs_get_filename, get_kvs_filename_success) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); /* Generate Testfiles */ @@ -1160,7 +1160,7 @@ TEST(kvs_get_filename, get_kvs_filename_failure) { prepare_environment(); - auto kvs = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + auto kvs = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(kvs); /* Testfiles not available */ @@ -1188,7 +1188,7 @@ TEST(kvs_get_filename, get_hashname_success) { prepare_environment(); - auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + auto result = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(result); /* Generate Testfiles */ @@ -1212,7 +1212,7 @@ TEST(kvs_get_filename, get_hashname_failure) { prepare_environment(); - auto kvs = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir)); + auto kvs = Kvs::open(instance_id, OpenNeedDefaults::Optional, OpenNeedKvs::Optional, std::string(data_dir), KVS_DEFAULT_MAX_SIZE_BYTES); ASSERT_TRUE(kvs); /* Testfiles not available */ @@ -1254,7 +1254,7 @@ TEST(kvs, flush_fails_when_storage_limit_exceeded) { // There is overhead for the JSON structure (key, type info, braces, etc.) and the hash file (4 bytes). // We will make the data payload a bit smaller than the max to account for this. size_t overhead_estimate = 100; - size_t data_size = KVS_MAX_STORAGE_BYTES - overhead_estimate; + size_t data_size = KVS_DEFAULT_MAX_SIZE_BYTES - overhead_estimate; std::string large_data(data_size, 'a'); auto set_res1 = kvs.set_value("large_data", KvsValue(large_data.c_str())); @@ -1299,7 +1299,7 @@ TEST(kvs_check_size, check_size_scenarios) { ASSERT_TRUE(check_res) << "check_size should succeed for data within limits"; // Check if the returned size is plausible EXPECT_GT(check_res.value(), 0); - EXPECT_LT(check_res.value(), KVS_MAX_STORAGE_BYTES); + EXPECT_LT(check_res.value(), KVS_DEFAULT_MAX_SIZE_BYTES); } // --- SCENARIO 2: Failure on data exceeding limits --- @@ -1311,7 +1311,7 @@ TEST(kvs_check_size, check_size_scenarios) { Kvs kvs = std::move(open_res.value()); // Add data that will certainly exceed the storage limit - std::string large_data(KVS_MAX_STORAGE_BYTES, 'x'); + std::string large_data(KVS_DEFAULT_MAX_SIZE_BYTES, 'x'); auto set_res = kvs.set_value("oversized_key", KvsValue(large_data.c_str())); ASSERT_TRUE(set_res); diff --git a/src/cpp/tests/test_kvs_builder.cpp b/src/cpp/tests/test_kvs_builder.cpp old mode 100644 new mode 100755 diff --git a/src/cpp/tests/test_kvs_error.cpp b/src/cpp/tests/test_kvs_error.cpp old mode 100644 new mode 100755 diff --git a/src/cpp/tests/test_kvs_general.cpp b/src/cpp/tests/test_kvs_general.cpp old mode 100644 new mode 100755 diff --git a/src/cpp/tests/test_kvs_general.hpp b/src/cpp/tests/test_kvs_general.hpp old mode 100644 new mode 100755 diff --git a/src/cpp/tests/test_kvs_helper.cpp b/src/cpp/tests/test_kvs_helper.cpp old mode 100644 new mode 100755 From d569e36b604e0890589452fe5c8dff01ff08dfb9 Mon Sep 17 00:00:00 2001 From: atarekra Date: Mon, 15 Jun 2026 15:01:18 +0300 Subject: [PATCH 6/6] Restoring file modes --- src/cpp/tests/BUILD | 0 src/cpp/tests/bm_kvs.cpp | 0 src/cpp/tests/test_kvs.cpp | 0 src/cpp/tests/test_kvs_builder.cpp | 0 src/cpp/tests/test_kvs_error.cpp | 0 src/cpp/tests/test_kvs_general.cpp | 0 src/cpp/tests/test_kvs_general.hpp | 0 src/cpp/tests/test_kvs_helper.cpp | 0 8 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 src/cpp/tests/BUILD mode change 100755 => 100644 src/cpp/tests/bm_kvs.cpp mode change 100755 => 100644 src/cpp/tests/test_kvs.cpp mode change 100755 => 100644 src/cpp/tests/test_kvs_builder.cpp mode change 100755 => 100644 src/cpp/tests/test_kvs_error.cpp mode change 100755 => 100644 src/cpp/tests/test_kvs_general.cpp mode change 100755 => 100644 src/cpp/tests/test_kvs_general.hpp mode change 100755 => 100644 src/cpp/tests/test_kvs_helper.cpp diff --git a/src/cpp/tests/BUILD b/src/cpp/tests/BUILD old mode 100755 new mode 100644 diff --git a/src/cpp/tests/bm_kvs.cpp b/src/cpp/tests/bm_kvs.cpp old mode 100755 new mode 100644 diff --git a/src/cpp/tests/test_kvs.cpp b/src/cpp/tests/test_kvs.cpp old mode 100755 new mode 100644 diff --git a/src/cpp/tests/test_kvs_builder.cpp b/src/cpp/tests/test_kvs_builder.cpp old mode 100755 new mode 100644 diff --git a/src/cpp/tests/test_kvs_error.cpp b/src/cpp/tests/test_kvs_error.cpp old mode 100755 new mode 100644 diff --git a/src/cpp/tests/test_kvs_general.cpp b/src/cpp/tests/test_kvs_general.cpp old mode 100755 new mode 100644 diff --git a/src/cpp/tests/test_kvs_general.hpp b/src/cpp/tests/test_kvs_general.hpp old mode 100755 new mode 100644 diff --git a/src/cpp/tests/test_kvs_helper.cpp b/src/cpp/tests/test_kvs_helper.cpp old mode 100755 new mode 100644