From 2c6de8d874b88c87de5c5d973b18b9e96ad9414a Mon Sep 17 00:00:00 2001 From: Vaibhav Pratap Date: Fri, 24 Jul 2026 08:46:20 +0000 Subject: [PATCH] fix: migrate parallel_upload.h and benchmarks to use unified checksum options --- .../benchmarks/throughput_experiment.cc | 33 ++++++++++++++----- google/cloud/storage/client.h | 12 +++---- google/cloud/storage/parallel_upload.h | 19 ++++++----- 3 files changed, 41 insertions(+), 23 deletions(-) diff --git a/google/cloud/storage/benchmarks/throughput_experiment.cc b/google/cloud/storage/benchmarks/throughput_experiment.cc index aced424858d18..538939ad776ce 100644 --- a/google/cloud/storage/benchmarks/throughput_experiment.cc +++ b/google/cloud/storage/benchmarks/throughput_experiment.cc @@ -81,9 +81,14 @@ class ResumableUpload : public ThroughputExperiment { auto const start = std::chrono::system_clock::now(); auto timer = Timer::PerThread(); auto writer = - client_.WriteObject(bucket_name, object_name, - gcs::DisableCrc32cChecksum(!config.enable_crc32c), - gcs::DisableMD5Hash(!config.enable_md5)); + client_.WriteObject( + bucket_name, object_name, + google::cloud::Options{}.set( + config.enable_crc32c + ? (config.enable_md5 ? gcs::ChecksumAlgorithm::kCrc32cAndMD5 + : gcs::ChecksumAlgorithm::kCrc32c) + : (config.enable_md5 ? gcs::ChecksumAlgorithm::kMD5 + : gcs::ChecksumAlgorithm::kNone))); auto upload_id = ExtractUploadId(writer.resumable_session_id()); for (std::int64_t offset = 0; offset < config.object_size; offset += config.app_buffer_size) { @@ -152,9 +157,14 @@ class SimpleUpload : public ThroughputExperiment { auto data = absl::string_view{*random_data_}.substr( 0, static_cast(config.object_size)); auto object_metadata = - client_.InsertObject(bucket_name, object_name, data, - gcs::DisableCrc32cChecksum(!config.enable_crc32c), - gcs::DisableMD5Hash(!config.enable_md5)); + client_.InsertObject( + bucket_name, object_name, data, + google::cloud::Options{}.set( + config.enable_crc32c + ? (config.enable_md5 ? gcs::ChecksumAlgorithm::kCrc32cAndMD5 + : gcs::ChecksumAlgorithm::kCrc32c) + : (config.enable_md5 ? gcs::ChecksumAlgorithm::kMD5 + : gcs::ChecksumAlgorithm::kNone))); auto const usage = timer.Sample(); auto generation = object_metadata ? std::to_string(object_metadata->generation()) @@ -210,9 +220,14 @@ class DownloadObject : public ThroughputExperiment { ? gcs::ReadRange(offset, offset + config.read_range->second) : gcs::ReadRange(); auto reader = - client_.ReadObject(bucket_name, object_name, read_range, - gcs::DisableCrc32cChecksum(!config.enable_crc32c), - gcs::DisableMD5Hash(!config.enable_md5)); + client_.ReadObject( + bucket_name, object_name, read_range, + google::cloud::Options{}.set( + config.enable_crc32c + ? (config.enable_md5 ? gcs::ChecksumAlgorithm::kCrc32cAndMD5 + : gcs::ChecksumAlgorithm::kCrc32c) + : (config.enable_md5 ? gcs::ChecksumAlgorithm::kMD5 + : gcs::ChecksumAlgorithm::kNone))); std::int64_t transfer_size = 0; while (!reader.eof() && !reader.bad()) { reader.read(buffer.data(), buffer.size()); diff --git a/google/cloud/storage/client.h b/google/cloud/storage/client.h index 9b62a8be5d088..cdf08ec2cc2b1 100644 --- a/google/cloud/storage/client.h +++ b/google/cloud/storage/client.h @@ -946,8 +946,8 @@ class Client { * @param contents the contents (media) for the new object. * @param options a list of optional query parameters and/or request headers. * Valid types for this operation include `ContentEncoding`, - * `ContentType`, `Crc32cChecksumValue`, `DisableCrc32cChecksum`, - * `DisableMD5Hash`, `EncryptionKey`, `IfGenerationMatch`, + * `ContentType`, `Crc32cChecksumValue`, + * `UploadChecksumValidationOption`, `EncryptionKey`, `IfGenerationMatch`, * `IfGenerationNotMatch`, `IfMetagenerationMatch`, * `IfMetagenerationNotMatch`, `KmsKeyName`, `MD5HashValue`, * `PredefinedAcl`, `Projection`, `UserProject`, and `WithObjectMetadata`. @@ -1176,8 +1176,8 @@ class Client { * @param bucket_name the name of the bucket that contains the object. * @param object_name the name of the object to be read. * @param options a list of optional query parameters and/or request headers. - * Valid types for this operation include `DisableCrc32cChecksum`, - * `DisableMD5Hash`, `EncryptionKey`, `Generation`, `IfGenerationMatch`, + * Valid types for this operation include `DownloadChecksumValidationOption`, + * `EncryptionKey`, `Generation`, `IfGenerationMatch`, * `IfGenerationNotMatch`, `IfMetagenerationMatch`, * `IfMetagenerationNotMatch`, `ReadFromOffset`, `ReadRange`, `ReadLast`, * `UserProject`, and `AcceptEncoding`. @@ -1240,7 +1240,7 @@ class Client { * @param object_name the name of the object to be read. * @param options a list of optional query parameters and/or request headers. * Valid types for this operation include `ContentEncoding`, `ContentType`, - * `Crc32cChecksumValue`, `DisableCrc32cChecksum`, `DisableMD5Hash`, + * `Crc32cChecksumValue`, `UploadChecksumValidationOption`, * `EncryptionKey`, `IfGenerationMatch`, `IfGenerationNotMatch`, * `IfMetagenerationMatch`, `IfMetagenerationNotMatch`, `KmsKeyName`, * `MD5HashValue`, `PredefinedAcl`, `Projection`, @@ -1295,7 +1295,7 @@ class Client { * @param object_name the name of the object to be read. * @param options a list of optional query parameters and/or request headers. * Valid types for this operation include `ContentEncoding`, `ContentType`, - * `Crc32cChecksumValue`, `DisableCrc32cChecksum`, `DisableMD5Hash`, + * `Crc32cChecksumValue`, `UploadChecksumValidationOption`, * `EncryptionKey`, `IfGenerationMatch`, `IfGenerationNotMatch`, * `IfMetagenerationMatch`, `IfMetagenerationNotMatch`, `KmsKeyName`, * `MD5HashValue`, `PredefinedAcl`, `Projection`, `UserProject`, diff --git a/google/cloud/storage/parallel_upload.h b/google/cloud/storage/parallel_upload.h index 5b0f09c6c87fe..d5ebb05684a85 100644 --- a/google/cloud/storage/parallel_upload.h +++ b/google/cloud/storage/parallel_upload.h @@ -701,8 +701,9 @@ NonResumableParallelUploadState::Create(Client client, auto upload_options = StaticTupleFilter< Among::TPred>(std::move(options)); + UploadChecksumValidationOption, EncryptionKey, KmsKeyName, + PredefinedAcl, UserProject, WithObjectMetadata>::TPred>( + std::move(options)); for (std::size_t i = 0; i < num_shards; ++i) { ResumableUploadRequest request( bucket_name, prefix + ".upload_shard_" + std::to_string(i)); @@ -806,8 +807,9 @@ StatusOr ResumableParallelUploadState::CreateNew( auto upload_options = std::tuple_cat( StaticTupleFilter< Among::TPred>(options), + DisableMD5Hash, UploadChecksumValidationOption, EncryptionKey, + KmsKeyName, PredefinedAcl, UserProject, + WithObjectMetadata>::TPred>(options), std::make_tuple(UseResumableUploadSession(""))); for (std::size_t i = 0; i < num_shards; ++i) { ResumableUploadRequest request( @@ -864,8 +866,8 @@ StatusOr ResumableParallelUploadState::Resume( auto read_options = std::tuple_cat( StaticTupleFilter::TPred>( - options), + UploadChecksumValidationOption, EncryptionKey, + Generation, UserProject>::TPred>(options), std::make_tuple(IfGenerationMatch(state_and_gen->second))); auto state_stream = google::cloud::internal::apply( @@ -915,8 +917,9 @@ StatusOr ResumableParallelUploadState::Resume( auto upload_options = StaticTupleFilter< Among::TPred>(std::move(options)); + UploadChecksumValidationOption, EncryptionKey, KmsKeyName, + PredefinedAcl, UserProject, WithObjectMetadata>::TPred>( + std::move(options)); for (auto& stream_desc : persistent_state->streams) { ResumableUploadRequest request(bucket_name, std::move(stream_desc.object_name));