From 51aebc8445fece5e0e7daf62603bf004b9dfb8ce Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Mar 2026 14:18:15 +0000 Subject: [PATCH 001/326] chore(website deps): bump undici from 7.18.2 to 7.24.1 in /website (#24925) Bumps [undici](https://github.com/nodejs/undici) from 7.18.2 to 7.24.1. - [Release notes](https://github.com/nodejs/undici/releases) - [Commits](https://github.com/nodejs/undici/compare/v7.18.2...v7.24.1) --- updated-dependencies: - dependency-name: undici dependency-version: 7.24.1 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- website/yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/website/yarn.lock b/website/yarn.lock index 3aa10cd2672e1..6e031634c03ef 100644 --- a/website/yarn.lock +++ b/website/yarn.lock @@ -3756,9 +3756,9 @@ undici-types@~7.16.0: integrity sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw== undici@^7.12.0: - version "7.18.2" - resolved "https://registry.yarnpkg.com/undici/-/undici-7.18.2.tgz#6cf724ef799a67d94fd55adf66b1e184176efcdf" - integrity sha512-y+8YjDFzWdQlSE9N5nzKMT3g4a5UBX1HKowfdXh0uvAnTaqqwqB92Jt4UXBAeKekDs5IaDKyJFR4X1gYVCgXcw== + version "7.24.1" + resolved "https://registry.yarnpkg.com/undici/-/undici-7.24.1.tgz#3fd0fe40e67388860810ad3275f9a23b322de650" + integrity sha512-5xoBibbmnjlcR3jdqtY2Lnx7WbrD/tHlT01TmvqZUFVc9Q1w4+j5hbnapTqbcXITMH1ovjq/W7BkqBilHiVAaA== unicode-canonical-property-names-ecmascript@^2.0.0: version "2.0.1" From e3225205fa88d04c4d76d0e485f27dc771900693 Mon Sep 17 00:00:00 2001 From: Benjamin Dornel Date: Mon, 16 Mar 2026 22:31:01 +0800 Subject: [PATCH 002/326] enhancement(clickhouse sink): add UUID type support for ArrowStream (#24856) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit enhancement(clickhouse sink): add UUID type support for ArrowStream format Map ClickHouse `UUID` columns to Arrow `Utf8`, letting ClickHouse handle the String→UUID cast on insert. Removes UUID from the unsupported types list in documentation. Co-authored-by: Pavlos Rontidis --- ...ickhouse_arrow_uuid_support.enhancement.md | 3 +++ src/sinks/clickhouse/arrow/parser.rs | 25 ++++++++++++++++++- src/sinks/clickhouse/integration_tests.rs | 18 +++++++++++-- .../reference/components/sinks/clickhouse.cue | 1 - 4 files changed, 43 insertions(+), 4 deletions(-) create mode 100644 changelog.d/clickhouse_arrow_uuid_support.enhancement.md diff --git a/changelog.d/clickhouse_arrow_uuid_support.enhancement.md b/changelog.d/clickhouse_arrow_uuid_support.enhancement.md new file mode 100644 index 0000000000000..e6dfc806b6775 --- /dev/null +++ b/changelog.d/clickhouse_arrow_uuid_support.enhancement.md @@ -0,0 +1,3 @@ +Added support for the ClickHouse `UUID` type in the ArrowStream format for the `clickhouse` sink. UUID columns are now automatically mapped to Arrow `Utf8` and cast by ClickHouse on insert. + +authors: benjamin-awd diff --git a/src/sinks/clickhouse/arrow/parser.rs b/src/sinks/clickhouse/arrow/parser.rs index ddafd0d0dc373..94286c09759ca 100644 --- a/src/sinks/clickhouse/arrow/parser.rs +++ b/src/sinks/clickhouse/arrow/parser.rs @@ -42,6 +42,9 @@ pub enum ClickHouseType { String, FixedString(u32), + // UUID type + Uuid, + // Date/time types Date, DateTime, @@ -93,7 +96,13 @@ impl ClickHouseType { }), // String types - Self::String | Self::FixedString(_) => Ok(DataType::Utf8), + // Note: UUID is mapped to Utf8 for two reasons: + // 1. Vector has no native UUID type — UUIDs are represented as strings in the event model + // 2. ClickHouse does not support UUID in Arrow format: + // https://github.com/ClickHouse/ClickHouse/blob/master/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp + // https://github.com/ClickHouse/ClickHouse/blob/master/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp + // ClickHouse handles the String → UUID cast implicitly on insert. + Self::String | Self::FixedString(_) | Self::Uuid => Ok(DataType::Utf8), // Date/time types Self::Date => Ok(DataType::Date32), @@ -230,6 +239,7 @@ fn ch_type(input: &str) -> IResult<&str, ClickHouseType> { // String types "String" => Ok((rest, ClickHouseType::String)), + "UUID" => Ok((rest, ClickHouseType::Uuid)), "FixedString" => parens(parse_u32) .map(ClickHouseType::FixedString) .parse(rest), @@ -465,6 +475,19 @@ mod tests { assert!(matches!(data_type, DataType::Map(_, _))); } + #[test] + fn test_uuid_type_mapping() { + assert_eq!(convert_type("UUID").unwrap(), (DataType::Utf8, false)); + assert_eq!( + convert_type("Nullable(UUID)").unwrap(), + (DataType::Utf8, true) + ); + assert_eq!( + convert_type("LowCardinality(Nullable(UUID))").unwrap(), + (DataType::Utf8, true) + ); + } + #[test] fn test_unknown_type_fails() { let result = convert_type("UnknownType"); diff --git a/src/sinks/clickhouse/integration_tests.rs b/src/sinks/clickhouse/integration_tests.rs index f86aeba915db9..a32771bf55730 100644 --- a/src/sinks/clickhouse/integration_tests.rs +++ b/src/sinks/clickhouse/integration_tests.rs @@ -565,7 +565,7 @@ async fn insert_events_arrow_with_schema_fetching() { client .create_table( &table, - "host String, timestamp DateTime64(3), message String, id Int64, name String, score Float64, active Bool", + "host String, timestamp DateTime64(3), message String, id Int64, name String, score Float64, active Bool, request_id UUID", ) .await; @@ -595,6 +595,10 @@ async fn insert_events_arrow_with_schema_fetching() { event.insert("name", format!("user_{}", i)); event.insert("score", 95.5 + i as f64); event.insert("active", i % 2 == 0); + event.insert( + "request_id", + format!("550e8400-e29b-41d4-a716-44665544000{}", i), + ); events.push(event.into()); } @@ -604,7 +608,7 @@ async fn insert_events_arrow_with_schema_fetching() { assert_eq!(3, output.rows); // Verify all fields exist and have the correct types - for row in output.data.iter() { + for (i, row) in output.data.iter().enumerate() { // Check standard Vector fields exist assert!(row.get("host").and_then(|v| v.as_str()).is_some()); assert!(row.get("message").and_then(|v| v.as_str()).is_some()); @@ -620,6 +624,16 @@ async fn insert_events_arrow_with_schema_fetching() { assert!(row.get("name").and_then(|v| v.as_str()).is_some()); assert!(row.get("score").and_then(|v| v.as_f64()).is_some()); assert!(row.get("active").and_then(|v| v.as_bool()).is_some()); + + // Check UUID field + let request_id = row + .get("request_id") + .and_then(|v| v.as_str()) + .expect("request_id should be present"); + assert_eq!( + request_id, + format!("550e8400-e29b-41d4-a716-44665544000{}", i) + ); } } diff --git a/website/cue/reference/components/sinks/clickhouse.cue b/website/cue/reference/components/sinks/clickhouse.cue index c0d80d1444388..320608e26858d 100644 --- a/website/cue/reference/components/sinks/clickhouse.cue +++ b/website/cue/reference/components/sinks/clickhouse.cue @@ -153,7 +153,6 @@ components: sinks: clickhouse: { types are unsupported: - `FIXED_SIZE_BINARY` - `JSON` - - `UUID` - `ENUM` #### Timezone Handling From 7390fe0bd4922b6f804b7c59e56dac36747af86d Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Mon, 16 Mar 2026 11:08:19 -0400 Subject: [PATCH 003/326] chore(external docs): document Cargo feature placement rules (#24933) * docs(style): document Cargo feature placement rules in RUST_STYLE.md Co-Authored-By: Claude Sonnet 4.6 * Apply suggestions from code review Co-authored-by: Thomas --------- Co-authored-by: Claude Sonnet 4.6 Co-authored-by: Thomas --- docs/RUST_STYLE.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/docs/RUST_STYLE.md b/docs/RUST_STYLE.md index 5ee34496af7b8..5d51af6855704 100644 --- a/docs/RUST_STYLE.md +++ b/docs/RUST_STYLE.md @@ -90,6 +90,8 @@ Code in Vector should **NOT** panic under normal circumstances. ## Feature Flags +### Component Feature Flags + New components (sources, sinks, transforms) must be behind feature flags: ```bash @@ -98,3 +100,26 @@ cargo test --lib --no-default-features --features sinks-console sinks::console ``` See `features` section in `Cargo.toml` for examples. + +### Cargo Dependency Feature Placement + +[Cargo features are additive](https://doc.rust-lang.org/cargo/reference/features.html#feature-unification): once any crate in the dependency graph enables a feature, it is enabled for the entire build. This can be surprising — enabling a feature in one crate silently turns it on everywhere. + +Always set `default-features = false` in `[workspace.dependencies]`. For feature declarations, the rule is simple: + +- If only one crate needs a feature, declare it in that crate's own `Cargo.toml`. +- If multiple crates need it, declare it in `[workspace.dependencies]`. + +Add a short comment near the dependency when the feature setup is non-obvious. + +**Auditing:** when unsure whether a feature is truly needed or only transitively enabled, verify with: + +```bash +# Show the full feature tree for the workspace +cargo tree -e features + +# Narrow down to a specific dependency +cargo tree -e features -i +``` + +The `-i` flag shows which crates depend on `` and which features they activate, useful for tracing where a feature is coming from. If a feature only appears because another crate enables it, and your crate relies on that feature being present, declare it explicitly — otherwise your crate silently breaks if the other crate ever stops enabling it. From c27f6d6dd8acc01f57e93c53af4fad0138a07aa1 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 16 Mar 2026 12:05:44 -0400 Subject: [PATCH 004/326] chore(deps): update heim fork to remove unmaintained crates (#24924) * chore(deps): update heim fork to remove unmaintained crates * Update licenses * Fix merge conflicts * Update licenses/Cargo.toml --- Cargo.lock | 276 +++++++++++-------------------------------- Cargo.toml | 2 +- LICENSE-3rdparty.csv | 6 +- deny.toml | 1 - 4 files changed, 71 insertions(+), 214 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e35cfde496f83..db997157c76f8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -583,17 +583,6 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "async-channel" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" -dependencies = [ - "concurrent-queue", - "event-listener 2.5.3", - "futures-core", -] - [[package]] name = "async-channel" version = "2.3.1" @@ -639,22 +628,21 @@ checksum = "c96bf972d85afc50bf5ab8fe2d54d1586b4e0b46c97c50a0c9e71e2f7bcd812a" dependencies = [ "async-task", "concurrent-queue", - "fastrand 2.3.0", - "futures-lite 2.6.0", + "fastrand", + "futures-lite", "pin-project-lite", "slab", ] [[package]] name = "async-fs" -version = "1.6.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "279cf904654eeebfa37ac9bb1598880884924aab82e290aa65c9e77a0e142e06" +checksum = "8034a681df4aed8b8edbd7fbe472401ecf009251c8b40556b304567052e294c5" dependencies = [ - "async-lock 2.8.0", - "autocfg", + "async-lock 3.4.0", "blocking", - "futures-lite 1.13.0", + "futures-lite", ] [[package]] @@ -663,11 +651,11 @@ version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13f937e26114b93193065fd44f507aa2e9169ad0cdabbb996920b1fe1ddea7ba" dependencies = [ - "async-channel 2.3.1", + "async-channel", "async-executor", "async-lock 3.4.0", "blocking", - "futures-lite 2.6.0", + "futures-lite", "tokio", ] @@ -755,26 +743,6 @@ dependencies = [ "warp", ] -[[package]] -name = "async-io" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fc5b45d93ef0529756f812ca52e44c221b35341892d3dcc34132ac02f3dd2af" -dependencies = [ - "async-lock 2.8.0", - "autocfg", - "cfg-if", - "concurrent-queue", - "futures-lite 1.13.0", - "log", - "parking", - "polling 2.8.0", - "rustix 0.37.27", - "slab", - "socket2 0.4.10", - "waker-fn", -] - [[package]] name = "async-io" version = "2.4.0" @@ -785,9 +753,9 @@ dependencies = [ "cfg-if", "concurrent-queue", "futures-io", - "futures-lite 2.6.0", + "futures-lite", "parking", - "polling 3.7.4", + "polling", "rustix 0.38.40", "slab", "tracing 0.1.44", @@ -864,30 +832,31 @@ dependencies = [ [[package]] name = "async-net" -version = "1.8.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0434b1ed18ce1cf5769b8ac540e33f01fa9471058b5e89da9e06f3c882a8c12f" +checksum = "b948000fad4873c1c9339d60f2623323a0cfd3816e5181033c6a5cb68b2accf7" dependencies = [ - "async-io 1.13.0", + "async-io", "blocking", - "futures-lite 1.13.0", + "futures-lite", ] [[package]] name = "async-process" -version = "1.8.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea6438ba0a08d81529c69b36700fa2f95837bfe3e776ab39cde9c14d9149da88" +checksum = "fc50921ec0055cdd8a16de48773bfeec5c972598674347252c0399676be7da75" dependencies = [ - "async-io 1.13.0", - "async-lock 2.8.0", + "async-channel", + "async-io", + "async-lock 3.4.0", "async-signal", + "async-task", "blocking", "cfg-if", - "event-listener 3.0.1", - "futures-lite 1.13.0", - "rustix 0.38.40", - "windows-sys 0.48.0", + "event-listener 5.3.1", + "futures-lite", + "rustix 1.0.1", ] [[package]] @@ -924,7 +893,7 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e47d90f65a225c4527103a8d747001fc56e375203592b25ad103e1ca13124c5" dependencies = [ - "async-io 2.4.0", + "async-io", "async-lock 2.8.0", "atomic-waker", "cfg-if", @@ -1015,7 +984,7 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes 1.11.1", - "fastrand 2.3.0", + "fastrand", "hex", "http 1.3.1", "ring", @@ -1055,7 +1024,7 @@ dependencies = [ "aws-types", "bytes 1.11.1", "bytes-utils", - "fastrand 2.3.0", + "fastrand", "http 0.2.12", "http 1.3.1", "http-body 0.4.6", @@ -1084,7 +1053,7 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", - "fastrand 2.3.0", + "fastrand", "flate2", "http 0.2.12", "http 1.3.1", @@ -1112,7 +1081,7 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes 1.11.1", - "fastrand 2.3.0", + "fastrand", "http 0.2.12", "http 1.3.1", "regex-lite", @@ -1136,7 +1105,7 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes 1.11.1", - "fastrand 2.3.0", + "fastrand", "http 0.2.12", "http 1.3.1", "regex-lite", @@ -1160,7 +1129,7 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes 1.11.1", - "fastrand 2.3.0", + "fastrand", "http 0.2.12", "http 1.3.1", "regex-lite", @@ -1185,7 +1154,7 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes 1.11.1", - "fastrand 2.3.0", + "fastrand", "http 0.2.12", "http 1.3.1", "regex-lite", @@ -1209,7 +1178,7 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes 1.11.1", - "fastrand 2.3.0", + "fastrand", "http 0.2.12", "http 1.3.1", "regex-lite", @@ -1237,7 +1206,7 @@ dependencies = [ "aws-smithy-xml", "aws-types", "bytes 1.11.1", - "fastrand 2.3.0", + "fastrand", "hex", "hmac", "http 0.2.12", @@ -1268,7 +1237,7 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes 1.11.1", - "fastrand 2.3.0", + "fastrand", "http 0.2.12", "http 1.3.1", "regex-lite", @@ -1293,7 +1262,7 @@ dependencies = [ "aws-smithy-types", "aws-smithy-xml", "aws-types", - "fastrand 2.3.0", + "fastrand", "http 0.2.12", "http 1.3.1", "regex-lite", @@ -1317,7 +1286,7 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes 1.11.1", - "fastrand 2.3.0", + "fastrand", "http 0.2.12", "http 1.3.1", "regex-lite", @@ -1341,7 +1310,7 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes 1.11.1", - "fastrand 2.3.0", + "fastrand", "http 0.2.12", "http 1.3.1", "regex-lite", @@ -1365,7 +1334,7 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes 1.11.1", - "fastrand 2.3.0", + "fastrand", "http 0.2.12", "http 1.3.1", "regex-lite", @@ -1390,7 +1359,7 @@ dependencies = [ "aws-smithy-types", "aws-smithy-xml", "aws-types", - "fastrand 2.3.0", + "fastrand", "http 0.2.12", "http 1.3.1", "regex-lite", @@ -1576,7 +1545,7 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "bytes 1.11.1", - "fastrand 2.3.0", + "fastrand", "http 0.2.12", "http 1.3.1", "http-body 0.4.6", @@ -1818,7 +1787,7 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" dependencies = [ - "fastrand 2.3.0", + "fastrand", "gloo-timers", "tokio", ] @@ -1986,10 +1955,10 @@ version = "1.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e83f8d02be6967315521be875afa792a316e28d57b5a2d401897e2a7921b7f21" dependencies = [ - "async-channel 2.3.1", + "async-channel", "async-task", "futures-io", - "futures-lite 2.6.0", + "futures-lite", "piper", ] @@ -3969,17 +3938,6 @@ version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" -[[package]] -name = "event-listener" -version = "3.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01cec0252c2afff729ee6f00e903d479fba81784c8e2bd77447673471fdfaea1" -dependencies = [ - "concurrent-queue", - "parking", - "pin-project-lite", -] - [[package]] name = "event-listener" version = "5.3.1" @@ -4067,15 +4025,6 @@ dependencies = [ "regex-syntax", ] -[[package]] -name = "fastrand" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" -dependencies = [ - "instant", -] - [[package]] name = "fastrand" version = "2.3.0" @@ -4357,28 +4306,13 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" -[[package]] -name = "futures-lite" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce" -dependencies = [ - "fastrand 1.9.0", - "futures-core", - "futures-io", - "memchr", - "parking", - "pin-project-lite", - "waker-fn", -] - [[package]] name = "futures-lite" version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f5edaec856126859abb19ed65f39e90fea3a9574b9707f13539acf4abf7eb532" dependencies = [ - "fastrand 2.3.0", + "fastrand", "futures-core", "futures-io", "parking", @@ -4838,7 +4772,7 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "heim" version = "0.1.0-rc.1" -source = "git+https://github.com/vectordotdev/heim.git?branch=update-nix#f3537d9b32e69a2a8ab19a0d42a1e6f5577a5a45" +source = "git+https://github.com/vectordotdev/heim.git?branch=update-deps#b62e0420231552ef079e86ecff1e6b98def6a553" dependencies = [ "heim-common", "heim-cpu", @@ -4852,7 +4786,7 @@ dependencies = [ [[package]] name = "heim-common" version = "0.1.0-rc.1" -source = "git+https://github.com/vectordotdev/heim.git?branch=update-nix#f3537d9b32e69a2a8ab19a0d42a1e6f5577a5a45" +source = "git+https://github.com/vectordotdev/heim.git?branch=update-deps#b62e0420231552ef079e86ecff1e6b98def6a553" dependencies = [ "cfg-if", "core-foundation 0.9.3", @@ -4860,7 +4794,7 @@ dependencies = [ "futures-util", "lazy_static", "libc", - "mach", + "mach2", "nix 0.23.2", "pin-utils", "uom", @@ -4870,7 +4804,7 @@ dependencies = [ [[package]] name = "heim-cpu" version = "0.1.0-rc.1" -source = "git+https://github.com/vectordotdev/heim.git?branch=update-nix#f3537d9b32e69a2a8ab19a0d42a1e6f5577a5a45" +source = "git+https://github.com/vectordotdev/heim.git?branch=update-deps#b62e0420231552ef079e86ecff1e6b98def6a553" dependencies = [ "cfg-if", "futures 0.3.31", @@ -4879,7 +4813,7 @@ dependencies = [ "heim-runtime", "lazy_static", "libc", - "mach", + "mach2", "ntapi 0.3.7", "smol", "winapi", @@ -4888,7 +4822,7 @@ dependencies = [ [[package]] name = "heim-disk" version = "0.1.0-rc.1" -source = "git+https://github.com/vectordotdev/heim.git?branch=update-nix#f3537d9b32e69a2a8ab19a0d42a1e6f5577a5a45" +source = "git+https://github.com/vectordotdev/heim.git?branch=update-deps#b62e0420231552ef079e86ecff1e6b98def6a553" dependencies = [ "bitflags 1.3.2", "cfg-if", @@ -4896,7 +4830,7 @@ dependencies = [ "heim-common", "heim-runtime", "libc", - "mach", + "mach2", "widestring 0.4.3", "winapi", ] @@ -4904,7 +4838,7 @@ dependencies = [ [[package]] name = "heim-host" version = "0.1.0-rc.1" -source = "git+https://github.com/vectordotdev/heim.git?branch=update-nix#f3537d9b32e69a2a8ab19a0d42a1e6f5577a5a45" +source = "git+https://github.com/vectordotdev/heim.git?branch=update-deps#b62e0420231552ef079e86ecff1e6b98def6a553" dependencies = [ "cfg-if", "heim-common", @@ -4912,7 +4846,7 @@ dependencies = [ "lazy_static", "libc", "log", - "mach", + "mach2", "ntapi 0.3.7", "platforms", "winapi", @@ -4921,21 +4855,21 @@ dependencies = [ [[package]] name = "heim-memory" version = "0.1.0-rc.1" -source = "git+https://github.com/vectordotdev/heim.git?branch=update-nix#f3537d9b32e69a2a8ab19a0d42a1e6f5577a5a45" +source = "git+https://github.com/vectordotdev/heim.git?branch=update-deps#b62e0420231552ef079e86ecff1e6b98def6a553" dependencies = [ "cfg-if", "heim-common", "heim-runtime", "lazy_static", "libc", - "mach", + "mach2", "winapi", ] [[package]] name = "heim-net" version = "0.1.0-rc.1" -source = "git+https://github.com/vectordotdev/heim.git?branch=update-nix#f3537d9b32e69a2a8ab19a0d42a1e6f5577a5a45" +source = "git+https://github.com/vectordotdev/heim.git?branch=update-deps#b62e0420231552ef079e86ecff1e6b98def6a553" dependencies = [ "bitflags 1.3.2", "cfg-if", @@ -4951,7 +4885,7 @@ dependencies = [ [[package]] name = "heim-runtime" version = "0.1.0-rc.1" -source = "git+https://github.com/vectordotdev/heim.git?branch=update-nix#f3537d9b32e69a2a8ab19a0d42a1e6f5577a5a45" +source = "git+https://github.com/vectordotdev/heim.git?branch=update-deps#b62e0420231552ef079e86ecff1e6b98def6a553" dependencies = [ "futures 0.3.31", "futures-timer", @@ -5722,15 +5656,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "instant" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" -dependencies = [ - "cfg-if", -] - [[package]] name = "inventory" version = "0.3.22" @@ -5740,17 +5665,6 @@ dependencies = [ "rustversion", ] -[[package]] -name = "io-lifetimes" -version = "1.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" -dependencies = [ - "hermit-abi 0.3.9", - "libc", - "windows-sys 0.48.0", -] - [[package]] name = "iovec" version = "0.1.4" @@ -6426,12 +6340,6 @@ dependencies = [ "linked-hash-map", ] -[[package]] -name = "linux-raw-sys" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" - [[package]] name = "linux-raw-sys" version = "0.4.14" @@ -6570,10 +6478,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baee0bbc17ce759db233beb01648088061bf678383130602a298e6998eedb2d8" [[package]] -name = "mach" -version = "0.3.2" +name = "mach2" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa" +checksum = "6a1b95cd5421ec55b445b5ae102f5ea0e768de1f82bd3001e11f426c269c3aea" dependencies = [ "libc", ] @@ -7996,7 +7904,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "668d31b1c4eba19242f2088b2bf3316b82ca31082a8335764db4e083db7485d4" dependencies = [ "atomic-waker", - "fastrand 2.3.0", + "fastrand", "futures-io", ] @@ -8061,22 +7969,6 @@ dependencies = [ "plotters-backend", ] -[[package]] -name = "polling" -version = "2.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b2d323e8ca7996b3e23126511a523f7e62924d93ecd5ae73b333815b0eb3dce" -dependencies = [ - "autocfg", - "bitflags 1.3.2", - "cfg-if", - "concurrent-queue", - "libc", - "log", - "pin-project-lite", - "windows-sys 0.48.0", -] - [[package]] name = "polling" version = "3.7.4" @@ -8617,7 +8509,7 @@ version = "6.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6cee616af00383c461f9ceb0067d15dee68e7d313ae47dbd7f8543236aed7ee9" dependencies = [ - "async-channel 2.3.1", + "async-channel", "async-trait", "bytes 1.11.1", "chrono", @@ -9610,20 +9502,6 @@ dependencies = [ "semver", ] -[[package]] -name = "rustix" -version = "0.37.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fea8ca367a3a01fe35e6943c400addf443c0f57670e6ec51196f71a4b8762dd2" -dependencies = [ - "bitflags 1.3.2", - "errno", - "io-lifetimes", - "libc", - "linux-raw-sys 0.3.8", - "windows-sys 0.48.0", -] - [[package]] name = "rustix" version = "0.38.40" @@ -10428,19 +10306,19 @@ dependencies = [ [[package]] name = "smol" -version = "1.3.0" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13f2b548cd8447f8de0fdf1c592929f70f4fc7039a05e47404b0d096ec6987a1" +checksum = "a33bd3e260892199c3ccfc487c88b2da2265080acb316cd920da72fdfd7c599f" dependencies = [ - "async-channel 1.9.0", + "async-channel", "async-executor", "async-fs", - "async-io 1.13.0", - "async-lock 2.8.0", + "async-io", + "async-lock 3.4.0", "async-net", "async-process", "blocking", - "futures-lite 1.13.0", + "futures-lite", ] [[package]] @@ -10510,16 +10388,6 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" -[[package]] -name = "socket2" -version = "0.4.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "socket2" version = "0.5.10" @@ -11121,7 +10989,7 @@ version = "3.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16" dependencies = [ - "fastrand 2.3.0", + "fastrand", "getrandom 0.3.4", "once_cell", "rustix 1.0.1", @@ -13266,12 +13134,6 @@ dependencies = [ "libc", ] -[[package]] -name = "waker-fn" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c4517f54858c779bbcbf228f4fca63d121bf85fbecb2dc578cdf4a39395690" - [[package]] name = "walkdir" version = "2.4.0" diff --git a/Cargo.toml b/Cargo.toml index a43905a1dc00a..dbc8c97a192d2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -448,7 +448,7 @@ arr_macro = { version = "0.2.1" } # depending on fork for bumped nix dependency # https://github.com/heim-rs/heim/pull/360 -heim = { git = "https://github.com/vectordotdev/heim.git", branch = "update-nix", default-features = false, features = ["disk"] } +heim = { git = "https://github.com/vectordotdev/heim.git", branch = "update-deps", default-features = false, features = ["disk"] } # make sure to update the external docs when the Lua version changes mlua = { version = "0.10.5", default-features = false, features = ["lua54", "send", "vendored", "macros"], optional = true } diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 7c616f5b55d3e..f360fe49e53a1 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -393,9 +393,7 @@ inotify,https://github.com/hannobraun/inotify,ISC,"Hanno Braun inout,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers instability,https://github.com/ratatui-org/instability,MIT,"Stephen M. Coakley , Joshka" -instant,https://github.com/sebcrozet/instant,BSD-3-Clause,sebcrozet inventory,https://github.com/dtolnay/inventory,MIT OR Apache-2.0,David Tolnay -io-lifetimes,https://github.com/sunfishcode/io-lifetimes,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,Dan Gohman iovec,https://github.com/carllerche/iovec,MIT OR Apache-2.0,Carl Lerche ipconfig,https://github.com/liranringel/ipconfig,MIT OR Apache-2.0,Liran Ringel ipcrypt-rs,https://github.com/jedisct1/rust-ipcrypt2,ISC,Frank Denis @@ -456,7 +454,7 @@ lz4,https://github.com/10xGenomics/lz4-rs,MIT,"Jens Heyens , Artem V. Navrotskiy , Patrick Marks " lz4_flex,https://github.com/pseitz/lz4_flex,MIT,"Pascal Seitz , Arthur Silva , ticki " macaddr,https://github.com/svartalf/rust-macaddr,Apache-2.0 OR MIT,svartalf -mach,https://github.com/fitzgen/mach,BSD-2-Clause,"Nick Fitzgerald , David Cuddeback , Gonzalo Brito Gadeschi " +mach2,https://github.com/JohnTitor/mach2,BSD-2-Clause OR MIT OR Apache-2.0,The mach2 Authors macro_magic,https://github.com/sam0x17/macro_magic,MIT,sam0x17 macro_magic_core,https://github.com/sam0x17/macro_magic,MIT,The macro_magic_core Authors macro_magic_core_macros,https://github.com/sam0x17/macro_magic,MIT,The macro_magic_core_macros Authors @@ -568,7 +566,6 @@ piper,https://github.com/notgull/piper,MIT OR Apache-2.0,"Stjepan Glavina -polling,https://github.com/smol-rs/polling,Apache-2.0 OR MIT,Stjepan Glavina polling,https://github.com/smol-rs/polling,Apache-2.0 OR MIT,"Stjepan Glavina , John Nunley " poly1305,https://github.com/RustCrypto/universal-hashes,Apache-2.0 OR MIT,RustCrypto Developers portable-atomic,https://github.com/taiki-e/portable-atomic,Apache-2.0 OR MIT,The portable-atomic Authors @@ -859,7 +856,6 @@ vrl,https://github.com/vectordotdev/vrl,MPL-2.0,Vector Contributors , Christian Duerr " wait-timeout,https://github.com/alexcrichton/wait-timeout,MIT OR Apache-2.0,Alex Crichton -waker-fn,https://github.com/smol-rs/waker-fn,Apache-2.0 OR MIT,Stjepan Glavina walkdir,https://github.com/BurntSushi/walkdir,Unlicense OR MIT,Andrew Gallant want,https://github.com/seanmonstar/want,MIT,Sean McArthur warp,https://github.com/seanmonstar/warp,MIT,Sean McArthur diff --git a/deny.toml b/deny.toml index c9a83079d192f..17ad08e1a8a5d 100644 --- a/deny.toml +++ b/deny.toml @@ -46,7 +46,6 @@ ignore = [ "RUSTSEC-2023-0071", { id = "RUSTSEC-2024-0388", reason = "derivative is unmaintained" }, { id = "RUSTSEC-2024-0384", reason = "instant is unmaintained" }, - { id = "RUSTSEC-2020-0168", reason = "mach is unmaintained" }, { id = "RUSTSEC-2025-0012", reason = "backoff is unmaintained" }, # rustls-pemfile is unmaintained. Blocked by both async-nats and http 1.0.0 upgrade. { id = "RUSTSEC-2025-0134", reason = "rustls-pemfile is unmaintained" }, From 0d8bb9bea3418bd6993e08b456fc495050a42930 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Mar 2026 17:07:42 +0000 Subject: [PATCH 005/326] chore(deps): bump pulsar from 6.3.1 to 6.7.0 (#24746) * chore(deps): bump pulsar from 6.3.1 to 6.7.0 Bumps [pulsar](https://github.com/streamnative/pulsar-rs) from 6.3.1 to 6.7.0. - [Release notes](https://github.com/streamnative/pulsar-rs/releases) - [Commits](https://github.com/streamnative/pulsar-rs/compare/v6.3.1...v6.7.0) --- updated-dependencies: - dependency-name: pulsar dependency-version: 6.7.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Update Cargo.toml * Update licenses * Add Default::default to new fields * Remove TODO comment --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Thomas --- Cargo.lock | 40 +++++++++++++++++--------------------- Cargo.toml | 2 +- LICENSE-3rdparty.csv | 1 + src/sinks/pulsar/config.rs | 4 ++++ 4 files changed, 24 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index db997157c76f8..8f526e6c4bb7b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6920,6 +6920,12 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" +[[package]] +name = "murmur3" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9252111cf132ba0929b6f8e030cac2a24b507f3a4d6db6fb2896f27b354c714b" + [[package]] name = "native-tls" version = "0.2.14" @@ -7356,16 +7362,16 @@ dependencies = [ [[package]] name = "oauth2" -version = "4.4.2" +version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c38841cdd844847e3e7c8d29cef9dcfed8877f8f56f9071f77843ecf3baf937f" +checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d" dependencies = [ - "base64 0.13.1", + "base64 0.22.1", "chrono", "getrandom 0.2.15", - "http 0.2.12", + "http 1.3.1", "rand 0.8.5", - "reqwest 0.11.26", + "reqwest 0.12.28", "serde", "serde_json", "serde_path_to_error", @@ -7494,16 +7500,16 @@ dependencies = [ [[package]] name = "openidconnect" -version = "3.5.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f47e80a9cfae4462dd29c41e987edd228971d6565553fbc14b8a11e666d91590" +checksum = "0d8c6709ba2ea764bbed26bce1adf3c10517113ddea6f2d4196e4851757ef2b2" dependencies = [ - "base64 0.13.1", + "base64 0.21.7", "chrono", "dyn-clone", "ed25519-dalek", "hmac", - "http 0.2.12", + "http 1.3.1", "itertools 0.10.5", "log", "oauth2", @@ -7513,7 +7519,6 @@ dependencies = [ "rsa", "serde", "serde-value", - "serde_derive", "serde_json", "serde_path_to_error", "serde_plain", @@ -8505,9 +8510,9 @@ dependencies = [ [[package]] name = "pulsar" -version = "6.3.1" +version = "6.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cee616af00383c461f9ceb0067d15dee68e7d313ae47dbd7f8543236aed7ee9" +checksum = "24c00856d6c64af2078a74c2de029842cdaece2f35b1223f5166809ea49cf5cd" dependencies = [ "async-channel", "async-trait", @@ -8519,6 +8524,7 @@ dependencies = [ "futures 0.3.31", "log", "lz4", + "murmur3", "native-tls", "nom 7.1.3", "oauth2", @@ -9155,7 +9161,6 @@ dependencies = [ "http 0.2.12", "http-body 0.4.6", "hyper 0.14.32", - "hyper-rustls 0.24.2", "hyper-tls 0.5.0", "ipnet", "js-sys", @@ -9165,7 +9170,6 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls 0.21.12", "rustls-pemfile 1.0.4", "serde", "serde_json", @@ -9174,13 +9178,11 @@ dependencies = [ "system-configuration 0.5.1", "tokio", "tokio-native-tls", - "tokio-rustls 0.24.1", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "webpki-roots 0.25.2", "winreg", ] @@ -13323,12 +13325,6 @@ dependencies = [ "web-sys", ] -[[package]] -name = "webpki-roots" -version = "0.25.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14247bb57be4f377dfb94c72830b8ce8fc6beac03cf4bf7b9732eadd414123fc" - [[package]] name = "webpki-roots" version = "0.26.1" diff --git a/Cargo.toml b/Cargo.toml index dbc8c97a192d2..ddeb29d9d765b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -414,7 +414,7 @@ openssl-probe = { version = "0.1.6", default-features = false } ordered-float.workspace = true percent-encoding = { version = "2.3.1", default-features = false } postgres-openssl = { version = "0.5.1", default-features = false, features = ["runtime"], optional = true } -pulsar = { version = "6.3.1", default-features = false, features = ["tokio-runtime", "auth-oauth2", "flate2", "lz4", "snap", "zstd"], optional = true } +pulsar = { version = "6.7.0", default-features = false, features = ["tokio-runtime", "auth-oauth2", "flate2", "lz4", "snap", "zstd"], optional = true } quick-junit = { version = "0.5.1" } rand.workspace = true rand_distr.workspace = true diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index f360fe49e53a1..1f02718d93477 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -487,6 +487,7 @@ mongocrypt-sys,https://github.com/mongodb/libmongocrypt-rust,Apache-2.0,"Abraham mongodb,https://github.com/mongodb/mongo-rust-driver,Apache-2.0,"Saghm Rossi , Patrick Freed , Isabel Atkinson , Abraham Egnor , Kaitlin Mahar , Patrick Meredith " mongodb-internal-macros,https://github.com/mongodb/mongo-rust-driver,Apache-2.0,The mongodb-internal-macros Authors multer,https://github.com/rousan/multer-rs,MIT,Rousan Ali +murmur3,https://github.com/stusmall/murmur3,MIT OR Apache-2.0,Stu Small native-tls,https://github.com/sfackler/rust-native-tls,MIT OR Apache-2.0,Steven Fackler ndk-context,https://github.com/rust-windowing/android-ndk-rs,MIT OR Apache-2.0,The Rust Windowing contributors newtype-uuid,https://github.com/oxidecomputer/newtype-uuid,MIT OR Apache-2.0,The newtype-uuid Authors diff --git a/src/sinks/pulsar/config.rs b/src/sinks/pulsar/config.rs index b006b6212d417..6b27035715226 100644 --- a/src/sinks/pulsar/config.rs +++ b/src/sinks/pulsar/config.rs @@ -310,6 +310,7 @@ impl PulsarSinkConfig { .map_or(default_retry_options.keep_alive, |secs| { Duration::from_secs(secs) }), + connection_max_idle: Default::default(), } }); @@ -338,6 +339,9 @@ impl PulsarSinkConfig { batch_size: self.batch.max_events, batch_byte_size: self.batch.max_bytes, compression: None, + batch_timeout: Default::default(), + block_queue_if_full: Default::default(), + routing_policy: Default::default(), }; match &self.compression { From fd766a0738802e9dee2eae1e5d6f243ed9c69229 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 16 Mar 2026 13:10:58 -0400 Subject: [PATCH 006/326] chore(deps): make rdkafka use gssapi-vendored when publishing/testing (#24912) * chore(deps): make rdkafka use gssapi-vendored when publishing/testing * update licenses * cargo fmt * Remove gssapi from docs * Re-add krb5-src to license file --------- Co-authored-by: Pavlos Rontidis --- Cargo.toml | 15 +++++++++------ scripts/cross/bootstrap-ubuntu.sh | 4 +++- vdev/src/testing/build.rs | 5 ++++- vdev/src/testing/integration.rs | 4 ++-- 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index ddeb29d9d765b..f2bfd2b21f48c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -505,11 +505,14 @@ ntapi = { git = "https://github.com/MSxDOS/ntapi.git", rev = "24fc1e47677fc9f6e3 [features] # Default features for *-unknown-linux-gnu and *-apple-darwin -default = ["enable-unix", "rdkafka?/gssapi-vendored"] -# Default features for `cargo docs`. The same as `default` but without `rdkafka?/gssapi-vendored` which would require installing libsasl in our doc build environment. +default = ["enable-unix", "rdkafka?/gssapi"] +# Default features for `cargo docs`. We're not using `gssapi` which would require installing libsasl2 in our doc environment. docs = ["enable-unix"] # Default features for *-unknown-linux-* which make use of `cmake` for dependencies -default-cmake = ["enable-unix", "rdkafka?/gssapi-vendored", "rdkafka?/cmake_build"] +default-cmake = ["enable-unix", "vendored", "rdkafka?/cmake_build"] + +vendored = ["rdkafka?/gssapi-vendored"] + # Default features for *-pc-windows-msvc # TODO: Enable SASL https://github.com/vectordotdev/vector/pull/3081#issuecomment-659298042 base = ["api", "enrichment-tables", "sinks", "sources", "transforms", "secrets", "vrl/stdlib"] @@ -517,8 +520,8 @@ enable-api-client = ["base", "api-client"] enable-unix = ["enable-api-client", "sources-dnstap", "unix"] default-msvc = ["enable-api-client", "rdkafka?/cmake_build"] -default-musl = ["enable-unix", "rdkafka?/gssapi-vendored", "rdkafka?/cmake_build"] -default-no-api-client = ["base", "sources-dnstap", "unix", "rdkafka?/gssapi-vendored"] +default-musl = ["enable-unix", "vendored", "rdkafka?/cmake_build"] +default-no-api-client = ["base", "sources-dnstap", "unix", "vendored"] tokio-console = ["dep:console-subscriber", "tokio/tracing"] @@ -544,7 +547,7 @@ target-armv7-unknown-linux-gnueabihf = ["target-unix"] target-armv7-unknown-linux-musleabihf = ["target-base"] target-arm-unknown-linux-gnueabi = ["target-unix"] target-arm-unknown-linux-musleabi = ["target-base"] -target-x86_64-unknown-linux-gnu = ["target-unix", "rdkafka?/gssapi-vendored"] +target-x86_64-unknown-linux-gnu = ["target-unix", "vendored"] target-x86_64-unknown-linux-musl = ["target-unix"] # Enables features that work only on systems providing `cfg(unix)` diff --git a/scripts/cross/bootstrap-ubuntu.sh b/scripts/cross/bootstrap-ubuntu.sh index 88e208ae2ea56..8068d1bcd3eb5 100755 --- a/scripts/cross/bootstrap-ubuntu.sh +++ b/scripts/cross/bootstrap-ubuntu.sh @@ -26,4 +26,6 @@ apt-get install -y \ libclang1-9 \ llvm-9 \ clang \ - unzip + unzip \ + libsasl2-dev + diff --git a/vdev/src/testing/build.rs b/vdev/src/testing/build.rs index 7b15f0c513d5f..9d564394d346a 100644 --- a/vdev/src/testing/build.rs +++ b/vdev/src/testing/build.rs @@ -67,7 +67,10 @@ pub fn build_integration_image() -> Result<()> { let mut cmd = prepare_build_command( &image, &dockerfile, - Some(&[ALL_INTEGRATIONS_FEATURE_FLAG.to_string()]), + Some(&[ + ALL_INTEGRATIONS_FEATURE_FLAG.to_string(), + "vendored".to_string(), + ]), &Environment::default(), false, // Integration tests don't pre-build Vector tests. ); diff --git a/vdev/src/testing/integration.rs b/vdev/src/testing/integration.rs index 213d7245abf8c..127346cb3206f 100644 --- a/vdev/src/testing/integration.rs +++ b/vdev/src/testing/integration.rs @@ -194,9 +194,9 @@ impl ComposeTest { // If using shared runner: use 'all-integration-tests' or 'all-e2e-tests' // If using per-test runner: use test-specific features from test.yaml args.push(if self.runner.is_shared_runner() { - self.local_config.feature_flag.to_string() + format!("{},vendored", self.local_config.feature_flag) } else { - self.config.features.join(",") + format!("{},vendored", self.config.features.join(",")) }); // If the test field is not present then use the --lib flag From d38425ad1302489d7420a9d24447224559a5a446 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Mar 2026 17:16:13 +0000 Subject: [PATCH 007/326] chore(deps): bump regex from 1.11.2 to 1.12.3 (#24796) * chore(deps): bump regex from 1.11.2 to 1.12.3 Bumps [regex](https://github.com/rust-lang/regex) from 1.11.2 to 1.12.3. - [Release notes](https://github.com/rust-lang/regex/releases) - [Changelog](https://github.com/rust-lang/regex/blob/master/CHANGELOG.md) - [Commits](https://github.com/rust-lang/regex/compare/1.11.2...1.12.3) --- updated-dependencies: - dependency-name: regex dependency-version: 1.12.3 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Update licenses --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Thomas --- Cargo.lock | 20 ++++++++++---------- Cargo.toml | 2 +- LICENSE-3rdparty.csv | 2 +- lib/vector-core/Cargo.toml | 2 +- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8f526e6c4bb7b..e97720b69c1ba 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2140,7 +2140,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab" dependencies = [ "memchr", - "regex-automata 0.4.8", + "regex-automata 0.4.14", "serde", ] @@ -4021,7 +4021,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72cf461f865c862bb7dc573f643dd6a2b6842f7c30b07882b56bd148cc2761b8" dependencies = [ "bit-set", - "regex-automata 0.4.8", + "regex-automata 0.4.14", "regex-syntax", ] @@ -6123,7 +6123,7 @@ version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "feee752d43abd0f4807a921958ab4131f692a44d4d599733d4419c5d586176ce" dependencies = [ - "regex-automata 0.4.8", + "regex-automata 0.4.14", "rustversion", ] @@ -6555,7 +6555,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" dependencies = [ - "regex-automata 0.4.8", + "regex-automata 0.4.14", ] [[package]] @@ -9069,13 +9069,13 @@ dependencies = [ [[package]] name = "regex" -version = "1.11.2" +version = "1.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23d7fd106d8c02486a8d64e778353d1cffe08ce79ac2e82f540c86d0facf6912" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.8", + "regex-automata 0.4.14", "regex-syntax", ] @@ -9087,9 +9087,9 @@ checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" [[package]] name = "regex-automata" -version = "0.4.8" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" dependencies = [ "aho-corasick", "memchr", @@ -11871,7 +11871,7 @@ dependencies = [ "matchers", "nu-ansi-term", "once_cell", - "regex-automata 0.4.8", + "regex-automata 0.4.14", "serde", "serde_json", "sharded-slab", diff --git a/Cargo.toml b/Cargo.toml index f2bfd2b21f48c..f4dec4497b1c8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -187,7 +187,7 @@ prost-reflect = { version = "0.14", features = ["serde"], default-features = fal prost-types = { version = "0.12", default-features = false } rand = { version = "0.9.2", default-features = false, features = ["small_rng", "thread_rng"] } rand_distr = { version = "0.5.1", default-features = false } -regex = { version = "1.11.2", default-features = false, features = ["std", "perf"] } +regex = { version = "1.12.3", default-features = false, features = ["std", "perf"] } reqwest = { version = "0.11", features = ["json"] } rust_decimal = { version = "1.37.0", default-features = false, features = ["std"] } semver = { version = "1.0.27", default-features = false, features = ["serde", "std"] } diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 1f02718d93477..523769af4238a 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -634,7 +634,7 @@ ref-cast,https://github.com/dtolnay/ref-cast,MIT OR Apache-2.0,David Tolnay referencing,https://github.com/Stranger6667/jsonschema,MIT,Dmitry Dygalo regex,https://github.com/rust-lang/regex,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant " -regex-automata,https://github.com/rust-lang/regex/tree/master/regex-automata,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant " +regex-automata,https://github.com/rust-lang/regex,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant " regex-filtered,https://github.com/ua-parser/uap-rust,BSD-3-Clause,The regex-filtered Authors regex-lite,https://github.com/rust-lang/regex,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant " regex-syntax,https://github.com/rust-lang/regex/tree/master/regex-syntax,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant " diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml index d416d018d3fbb..1147ce8b63e1a 100644 --- a/lib/vector-core/Cargo.toml +++ b/lib/vector-core/Cargo.toml @@ -41,7 +41,7 @@ proptest = { version = "1.10", optional = true } prost-types.workspace = true prost.workspace = true quanta = { version = "0.12.6", default-features = false } -regex = { version = "1.11.2", default-features = false, features = ["std", "perf"] } +regex = { version = "1.12.3", default-features = false, features = ["std", "perf"] } ryu = { version = "1", default-features = false } serde.workspace = true serde_json.workspace = true From c2fc606306e10e02263d2982f20c8c17f69a143b Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 16 Mar 2026 14:05:29 -0400 Subject: [PATCH 008/326] feat(vdev): check Cargo.lock for changes after cargo check/clippy (#24935) * feat(vdev): check Cargo.lock for changes after cargo check/clippy * Only commit if no lockfile changes are detected --- Cargo.lock | 26 +++++++++++++++++++-- Cargo.toml | 1 + lib/codecs/Cargo.toml | 2 +- lib/vector-vrl/category/Cargo.toml | 2 +- vdev/Cargo.toml | 1 + vdev/src/commands/check/rust.rs | 37 +++++++++++++++++++++++------- 6 files changed, 57 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e97720b69c1ba..635320a783e62 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2546,7 +2546,7 @@ dependencies = [ "similar-asserts", "smallvec", "snafu 0.8.9", - "strum 0.26.3", + "strum 0.28.0", "syslog_loose 0.23.0", "tokio", "tokio-util", @@ -10736,6 +10736,15 @@ dependencies = [ "strum_macros 0.27.2", ] +[[package]] +name = "strum" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" +dependencies = [ + "strum_macros 0.28.0", +] + [[package]] name = "strum_macros" version = "0.25.3" @@ -10774,6 +10783,18 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "strum_macros" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664" +dependencies = [ + "heck 0.5.0", + "proc-macro2 1.0.106", + "quote 1.0.44", + "syn 2.0.117", +] + [[package]] name = "subtle" version = "2.5.0" @@ -12362,6 +12383,7 @@ dependencies = [ "serde_json", "serde_yaml", "sha2", + "strum 0.28.0", "tempfile", "toml 0.9.8", "toml_edit 0.23.9", @@ -12914,7 +12936,7 @@ dependencies = [ name = "vector-vrl-category" version = "0.1.0" dependencies = [ - "strum 0.27.2", + "strum 0.28.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index f4dec4497b1c8..f5639daa32ca8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -219,6 +219,7 @@ vector-vrl-functions = { path = "lib/vector-vrl/functions", default-features = f vrl = { git = "https://github.com/vectordotdev/vrl.git", branch = "main", default-features = false, features = ["arbitrary", "cli", "test", "test_framework", "stdlib-base"] } mock_instant = { version = "0.6" } serial_test = { version = "3.2" } +strum = { version = "0.28", features = ["derive"] } [dependencies] cfg-if.workspace = true diff --git a/lib/codecs/Cargo.toml b/lib/codecs/Cargo.toml index 0ed26d0b16a41..90a5fd2780109 100644 --- a/lib/codecs/Cargo.toml +++ b/lib/codecs/Cargo.toml @@ -42,7 +42,7 @@ serde_json.workspace = true serde-aux = { version = "4.5", optional = true } smallvec = { version = "1", default-features = false, features = ["union"] } snafu.workspace = true -strum = { version = "0.26.3", features = ["derive"], optional = true } +strum = { workspace = true, optional = true } syslog_loose = { version = "0.23", default-features = false, optional = true } tokio-util = { version = "0.7", default-features = false, features = ["codec"] } tokio = { workspace = true, features = ["full"] } diff --git a/lib/vector-vrl/category/Cargo.toml b/lib/vector-vrl/category/Cargo.toml index dafb938de89fa..886b7dce5e72f 100644 --- a/lib/vector-vrl/category/Cargo.toml +++ b/lib/vector-vrl/category/Cargo.toml @@ -7,4 +7,4 @@ publish = false license = "MPL-2.0" [dependencies] -strum = { version = "0.27", features = ["derive"] } +strum.workspace = true diff --git a/vdev/Cargo.toml b/vdev/Cargo.toml index 959362252e3cb..8fea6eb281d0a 100644 --- a/vdev/Cargo.toml +++ b/vdev/Cargo.toml @@ -42,6 +42,7 @@ tempfile.workspace = true toml.workspace = true toml_edit = "0.23" semver.workspace = true +strum.workspace = true indoc.workspace = true git2 = { version = "0.20.4" } cfg-if.workspace = true diff --git a/vdev/src/commands/check/rust.rs b/vdev/src/commands/check/rust.rs index cdd6e45381683..e7597550962df 100644 --- a/vdev/src/commands/check/rust.rs +++ b/vdev/src/commands/check/rust.rs @@ -1,9 +1,9 @@ -use anyhow::Result; -use std::ffi::OsString; +use anyhow::{Result, bail}; +use std::{ffi::OsString, fs}; use crate::{ app, - utils::{command::ChainArgs as _, git}, + utils::{command::ChainArgs as _, git, paths}, }; /// Check the Rust code for errors @@ -20,11 +20,16 @@ pub struct Cli { fix: bool, } +#[derive(strum::Display, strum::AsRefStr, Clone, Copy, Debug)] +#[strum(serialize_all = "lowercase")] +enum Tool { + Clippy, + Check, +} + impl Cli { /// Build the argument vector for cargo invocation. - fn build_args(&self) -> Vec { - let tool = if self.clippy { "clippy" } else { "check" }; - + fn build_args(&self, tool: Tool) -> Vec { let pre_args = if self.fix { vec!["--fix"] } else { @@ -47,14 +52,30 @@ impl Cli { ] }; - [tool, "--workspace", "--all-targets"] + [tool.as_ref(), "--workspace", "--all-targets"] .chain_args(feature_args) .chain_args(pre_args) .chain_args(clippy_args) } pub fn exec(self) -> Result<()> { - app::exec("cargo", self.build_args(), true)?; + let lock_file = paths::find_repo_root()?.join("Cargo.lock"); + let lock_before = fs::read(&lock_file)?; + + let tool = if self.clippy { + Tool::Clippy + } else { + Tool::Check + }; + + app::exec("cargo", self.build_args(tool), true)?; + + let lock_after = fs::read(&lock_file)?; + if lock_before != lock_after { + bail!( + "Cargo.lock was modified by `cargo {tool}`. Please commit the updated Cargo.lock." + ); + } // If --fix was used, check for changes and commit them. if self.fix { From 0573297e8ffcc1ad7a096f50f012b2efd0518c27 Mon Sep 17 00:00:00 2001 From: Jesper Date: Mon, 16 Mar 2026 19:54:36 +0100 Subject: [PATCH 009/326] feat(nats source,nats sink): Added feature flag for websockets in async-nats (#24291) * Added feature flag for websockets in async-nats * Add changelog * Add websockets to expect.txt --------- Co-authored-by: Thomas --- .github/actions/spelling/expect.txt | 1 + Cargo.toml | 2 +- changelog.d/24291_async_nats_websockets.feature.md | 4 ++++ 3 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 changelog.d/24291_async_nats_websockets.feature.md diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt index edc77d73e5612..cfc0fe68ee854 100644 --- a/.github/actions/spelling/expect.txt +++ b/.github/actions/spelling/expect.txt @@ -646,6 +646,7 @@ wayfor wday webgraphviz websites +websockets wensite whostheboss willreturn diff --git a/Cargo.toml b/Cargo.toml index f5639daa32ca8..86f06951af994 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -406,7 +406,7 @@ lru = { version = "0.16.3", default-features = false } maxminddb = { version = "0.27.0", default-features = false, optional = true, features = ["simdutf8"] } md-5 = { version = "0.10", default-features = false, optional = true } mongodb = { version = "3.3.0", default-features = false, optional = true, features = ["compat-3-0-0", "dns-resolver", "rustls-tls"] } -async-nats = { version = "0.42.0", default-features = false, optional = true, features = ["ring"] } +async-nats = { version = "0.42.0", default-features = false, optional = true, features = ["ring", "websockets"] } nkeys = { version = "0.4.5", default-features = false, optional = true } nom = { workspace = true, optional = true } notify = { version = "8.1.0", default-features = false, features = ["macos_fsevent"] } diff --git a/changelog.d/24291_async_nats_websockets.feature.md b/changelog.d/24291_async_nats_websockets.feature.md new file mode 100644 index 0000000000000..c11099ce6daf4 --- /dev/null +++ b/changelog.d/24291_async_nats_websockets.feature.md @@ -0,0 +1,4 @@ +Added websocket support to the `nats` source and sink. The `url` field now supports both `ws://` and +`wss://` protocols. + +authors: gedemagt From 3905b4f12067fb2508f0bcb031607da84623f17f Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 16 Mar 2026 15:07:53 -0400 Subject: [PATCH 010/326] chore(vdev): Update README.md and cleanup unused commands (#24936) * chore(vdev): Update README.md and cleanup unused commands * Update spelling allow/expect --- .github/actions/spelling/allow.txt | 1 + .github/actions/spelling/expect.txt | 2 + vdev/README.md | 84 ++++++++++++++--------------- vdev/src/commands/exec.rs | 24 --------- vdev/src/commands/meta/mod.rs | 2 - vdev/src/commands/meta/starship.rs | 23 -------- vdev/src/commands/mod.rs | 2 - 7 files changed, 43 insertions(+), 95 deletions(-) delete mode 100644 vdev/src/commands/exec.rs delete mode 100644 vdev/src/commands/meta/starship.rs diff --git a/.github/actions/spelling/allow.txt b/.github/actions/spelling/allow.txt index b82a12c395bd2..2672ed4c357b7 100644 --- a/.github/actions/spelling/allow.txt +++ b/.github/actions/spelling/allow.txt @@ -60,6 +60,7 @@ Bedove Benss bigendian bindir +binstall binfmt bitcast bitcode diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt index cfc0fe68ee854..60715b9c96c0e 100644 --- a/.github/actions/spelling/expect.txt +++ b/.github/actions/spelling/expect.txt @@ -170,11 +170,13 @@ duser dynamicwireless dyno ebfcee +ector edenhill edns eeyun efgh Elhage +elopment emerg endianess endler diff --git a/vdev/README.md b/vdev/README.md index f8dfeeab5d969..c036bc3c891fd 100644 --- a/vdev/README.md +++ b/vdev/README.md @@ -1,81 +1,77 @@ # vdev ------ +--- -This is the command line tooling for Vector development. +The V(ector) Dev(elopment) tool. + +This is the command line tooling for Vector development. You don't need to use or install this +unless you are doing development in the Vector repo. Table of Contents: +- [Pre-requisites](#pre-requisites) - [Installation](#installation) -- [Configuration](#configuration) - - [Repository](#repository) - - [Starship](#starship) -- [CLI](#cli) - [Running Tests](#running-tests) + - [Running Integration tests](#running-integration-tests) +- [Developing vdev](#developing-vdev) ## Pre-requisites This assumes that you have the following tools installed: -- cargo-nextest - https://nexte.st/ +- [git](https://git-scm.com/) +- [cargo](https://rustup.rs/) +- [docker](https://www.docker.com/) +- [npm](https://www.npmjs.com/) +- [ruby](https://www.ruby-lang.org/en/) + +Some other tools may need to be installed depending on the command you are running. All other +dependencies can be installed by running + +```sh +./scripts/environment/prepare.sh +``` ## Installation +Note that installation is not required, since `cargo vdev` commands will invoke vdev from within the +repository. + Run the following command from the root of the Vector repository: -```text +```sh cargo install -f --path vdev ``` -## Configuration - -### Repository +You can also install vdev from [crates.io](https://crates.io/crates/vdev) -Setting the path to the repository explicitly allows the application to be used at any time no matter the current working directory. - -```text -vdev config set repo . +```sh +cargo install vdev ``` -To test, enter your home directory and then run: +Or using [binstall](https://github.com/cargo-bins/cargo-binstall), which will download a +pre-compiled binary (fastest) -```text -vdev exec ls +```sh +cargo binstall vdev ``` -### Starship - -A custom command for the [Starship](https://starship.rs) prompt is available. -```toml -format = """ -... -${custom.vdev}\ -... -$line_break\ -... -$character""" +## Running Tests -# +Unit tests can be run by calling `make test`. -[custom.vdev] -command = "vdev meta starship" -when = true -# Windows -# shell = ["cmd", "/C"] -# Other -# shell = ["sh", "--norc"] -``` +### Running Integration tests -## CLI +Integration tests require docker or podman to run. -The CLI uses [Clap](https://github.com/clap-rs/clap) with the `derive` construction mechanism and is stored in the [commands](src/commands) directory. +Integration tests are not run by default when running `make test`. Instead, they are accessible via the integration subcommand `cargo vdev int` (example: `cargo vdev int test aws` runs aws-related integration tests). -Every command group/namespace has its own directory with a `cli` module, including the root `vdev` command group. All commands have an `exec` method that provides the actual implementation, which in the case of command groups will be calling sub-commands. +You should use `./scripts/run-integration-test.sh`, which is the wrapper used by CI and which suits most development needs. Integration tests require a `cargo vdev int start`, `cargo vdev int test`, and `cargo vdev int stop`, which the script handles automatically. You can find the list of available integration tests using `cargo vdev int show`. -## Running Tests +## Developing vdev -Unit tests can be run by calling `cargo vdev test`. +The CLI uses [Clap](https://github.com/clap-rs/clap) with the `derive` construction mechanism and is stored in the [commands](src/commands) directory. -Integration tests are not run by default when running `cargo vdev test`. Instead, they are accessible via the integration subcommand (example: `cargo vdev int test aws` runs aws-related integration tests). You can find the list of available integration tests using `cargo vdev int show`. Integration tests require docker or podman to run. +Every command group/namespace has its own directory with a `cli` module, including the root `vdev` command group. All commands have an `exec` method that provides the actual implementation, which in the case of command groups will be calling sub-commands. diff --git a/vdev/src/commands/exec.rs b/vdev/src/commands/exec.rs deleted file mode 100644 index f7e6a11f6af6d..0000000000000 --- a/vdev/src/commands/exec.rs +++ /dev/null @@ -1,24 +0,0 @@ -use std::process::Command; - -use anyhow::Result; -use clap::Args; - -use crate::app::CommandExt as _; - -/// Execute a command within the repository -#[derive(Args, Debug)] -#[command()] -pub struct Cli { - #[arg(required = true, trailing_var_arg = true, allow_hyphen_values = true)] - args: Vec, -} - -impl Cli { - pub fn exec(self) -> Result<()> { - let status = Command::new(&self.args[0]) - .in_repo() - .args(&self.args[1..]) - .run()?; - std::process::exit(status.code().unwrap_or(1)); - } -} diff --git a/vdev/src/commands/meta/mod.rs b/vdev/src/commands/meta/mod.rs index cc67a0680bf5b..3a6abb7796870 100644 --- a/vdev/src/commands/meta/mod.rs +++ b/vdev/src/commands/meta/mod.rs @@ -1,8 +1,6 @@ mod install_git_hooks; -mod starship; crate::cli_subcommands! { "Collection of meta-utilities..." - starship, install_git_hooks, } diff --git a/vdev/src/commands/meta/starship.rs b/vdev/src/commands/meta/starship.rs deleted file mode 100644 index 5cc663f79da91..0000000000000 --- a/vdev/src/commands/meta/starship.rs +++ /dev/null @@ -1,23 +0,0 @@ -use anyhow::Result; -use clap::Args; - -use crate::utils::cargo::CargoToml; - -/// Custom Starship prompt plugin -#[derive(Args, Debug)] -#[command(hide = true)] -pub struct Cli {} - -impl Cli { - pub fn exec(self) -> Result<()> { - let mut contexts = vec![]; - - if let Ok(cargo_toml) = CargoToml::load() { - contexts.push(format!("version: {}", cargo_toml.package.version)); - } - - println!("vector{{ {} }}", contexts.join(", ")); - - Ok(()) - } -} diff --git a/vdev/src/commands/mod.rs b/vdev/src/commands/mod.rs index 807ed268b2fba..d80b6132e9ae4 100644 --- a/vdev/src/commands/mod.rs +++ b/vdev/src/commands/mod.rs @@ -72,7 +72,6 @@ mod check; mod complete; mod crate_versions; mod e2e; -mod exec; mod features; mod fmt; mod info; @@ -92,7 +91,6 @@ cli_commands! { complete, crate_versions, e2e, - exec, features, fmt, info, From 1eca246d5bf297afcc153b555c19ac5f9e9ff372 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 16 Mar 2026 16:49:29 -0400 Subject: [PATCH 011/326] chore(dev): replace derivative macro with standard Default derives on enums (#24938) chore: replace derivative macro with standard Default derives on enums --- lib/codecs/src/decoding/format/vrl.rs | 4 +-- .../src/decoding/framing/chunked_gelf.rs | 5 ++- .../src/decoding/framing/length_delimited.rs | 4 +-- .../src/decoding/framing/newline_delimited.rs | 4 +-- .../src/decoding/framing/octet_counting.rs | 4 +-- .../framing/varint_length_delimited.rs | 4 +-- .../src/encoding/framing/length_delimited.rs | 4 +-- .../framing/varint_length_delimited.rs | 4 +-- src/kafka.rs | 5 ++- src/sinks/clickhouse/config.rs | 5 ++- src/sinks/console/config.rs | 5 ++- src/sinks/elasticsearch/mod.rs | 4 +-- src/sinks/gcp/stackdriver/logs/config.rs | 3 +- src/sinks/gcp_chronicle/compression.rs | 5 ++- src/sinks/gcs_common/config.rs | 10 +++--- src/sinks/greptimedb/metrics/config.rs | 2 +- src/sinks/http/config.rs | 5 ++- src/sinks/loki/config.rs | 5 ++- src/sinks/mqtt/config.rs | 5 ++- src/sinks/new_relic/config.rs | 10 +++--- src/sinks/pulsar/config.rs | 5 ++- src/sinks/redis/config.rs | 32 ++++++++----------- src/sinks/s3_common/config.rs | 10 +++--- src/sinks/util/adaptive_concurrency/tests.rs | 5 ++- src/sinks/util/buffer/compression.rs | 5 ++- src/sinks/util/service/concurrency.rs | 2 +- src/sources/aws_kinesis_firehose/mod.rs | 5 ++- src/sources/aws_s3/mod.rs | 10 +++--- src/sources/demo_logs.rs | 5 ++- src/sources/redis/mod.rs | 12 +++---- src/sources/util/body_decoding.rs | 5 ++- 31 files changed, 74 insertions(+), 119 deletions(-) diff --git a/lib/codecs/src/decoding/format/vrl.rs b/lib/codecs/src/decoding/format/vrl.rs index c67f60fdd8952..6b6a20be3be55 100644 --- a/lib/codecs/src/decoding/format/vrl.rs +++ b/lib/codecs/src/decoding/format/vrl.rs @@ -1,5 +1,4 @@ use bytes::Bytes; -use derivative::Derivative; use smallvec::{SmallVec, smallvec}; use vector_config_macros::configurable_component; use vector_core::{ @@ -26,8 +25,7 @@ pub struct VrlDeserializerConfig { /// VRL-specific decoding options. #[configurable_component] -#[derive(Debug, Clone, PartialEq, Eq, Derivative)] -#[derivative(Default)] +#[derive(Debug, Clone, PartialEq, Eq, Default)] pub struct VrlDeserializerOptions { /// The [Vector Remap Language][vrl] (VRL) program to execute for each event. /// Note that the final contents of the `.` target will be used as the decoding result. diff --git a/lib/codecs/src/decoding/framing/chunked_gelf.rs b/lib/codecs/src/decoding/framing/chunked_gelf.rs index 9a82333c87960..3ba3c9bd33d33 100644 --- a/lib/codecs/src/decoding/framing/chunked_gelf.rs +++ b/lib/codecs/src/decoding/framing/chunked_gelf.rs @@ -85,11 +85,10 @@ pub struct ChunkedGelfDecoderOptions { /// Decompression options for ChunkedGelfDecoder. #[configurable_component] -#[derive(Clone, Copy, Debug, PartialEq, Eq, Derivative)] -#[derivative(Default)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Default)] pub enum ChunkedGelfDecompressionConfig { /// Automatically detect the decompression method based on the magic bytes of the message. - #[derivative(Default)] + #[default] Auto, /// Use Gzip decompression. Gzip, diff --git a/lib/codecs/src/decoding/framing/length_delimited.rs b/lib/codecs/src/decoding/framing/length_delimited.rs index 07a958dce1c52..16c562f653f2d 100644 --- a/lib/codecs/src/decoding/framing/length_delimited.rs +++ b/lib/codecs/src/decoding/framing/length_delimited.rs @@ -1,5 +1,4 @@ use bytes::{Bytes, BytesMut}; -use derivative::Derivative; use tokio_util::codec::Decoder; use vector_config::configurable_component; @@ -8,8 +7,7 @@ use crate::common::length_delimited::LengthDelimitedCoderOptions; /// Config used to build a `LengthDelimitedDecoder`. #[configurable_component] -#[derive(Debug, Clone, Derivative)] -#[derivative(Default)] +#[derive(Debug, Clone, Default)] pub struct LengthDelimitedDecoderConfig { /// Options for the length delimited decoder. #[serde(skip_serializing_if = "vector_core::serde::is_default")] diff --git a/lib/codecs/src/decoding/framing/newline_delimited.rs b/lib/codecs/src/decoding/framing/newline_delimited.rs index 7bdc3a60881e5..90177031c31c9 100644 --- a/lib/codecs/src/decoding/framing/newline_delimited.rs +++ b/lib/codecs/src/decoding/framing/newline_delimited.rs @@ -1,5 +1,4 @@ use bytes::{Bytes, BytesMut}; -use derivative::Derivative; use tokio_util::codec::Decoder; use vector_config::configurable_component; @@ -16,8 +15,7 @@ pub struct NewlineDelimitedDecoderConfig { /// Options for building a `NewlineDelimitedDecoder`. #[configurable_component] -#[derive(Clone, Debug, Derivative, PartialEq, Eq)] -#[derivative(Default)] +#[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct NewlineDelimitedDecoderOptions { /// The maximum length of the byte buffer. /// diff --git a/lib/codecs/src/decoding/framing/octet_counting.rs b/lib/codecs/src/decoding/framing/octet_counting.rs index 749d66b8fffc9..80f5a28ee4aa4 100644 --- a/lib/codecs/src/decoding/framing/octet_counting.rs +++ b/lib/codecs/src/decoding/framing/octet_counting.rs @@ -1,7 +1,6 @@ use std::io; use bytes::{Buf, Bytes, BytesMut}; -use derivative::Derivative; use tokio_util::codec::{LinesCodec, LinesCodecError}; use tracing::trace; use vector_config::configurable_component; @@ -30,8 +29,7 @@ impl OctetCountingDecoderConfig { /// Options for building a `OctetCountingDecoder`. #[configurable_component] -#[derive(Clone, Debug, Derivative, PartialEq, Eq)] -#[derivative(Default)] +#[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct OctetCountingDecoderOptions { /// The maximum length of the byte buffer. #[serde(skip_serializing_if = "vector_core::serde::is_default")] diff --git a/lib/codecs/src/decoding/framing/varint_length_delimited.rs b/lib/codecs/src/decoding/framing/varint_length_delimited.rs index 7b9d26f0a0596..a1b870fe1bb51 100644 --- a/lib/codecs/src/decoding/framing/varint_length_delimited.rs +++ b/lib/codecs/src/decoding/framing/varint_length_delimited.rs @@ -1,5 +1,4 @@ use bytes::{Buf, Bytes, BytesMut}; -use derivative::Derivative; use snafu::Snafu; use tokio_util::codec::Decoder; use vector_config::configurable_component; @@ -38,8 +37,7 @@ impl FramingError for VarintFramingError { /// Config used to build a `VarintLengthDelimitedDecoder`. #[configurable_component] -#[derive(Debug, Clone, Derivative)] -#[derivative(Default)] +#[derive(Debug, Clone, Default)] pub struct VarintLengthDelimitedDecoderConfig { /// Maximum frame length #[serde(default = "default_max_frame_length")] diff --git a/lib/codecs/src/encoding/framing/length_delimited.rs b/lib/codecs/src/encoding/framing/length_delimited.rs index 660049ce5b305..152b3baf73673 100644 --- a/lib/codecs/src/encoding/framing/length_delimited.rs +++ b/lib/codecs/src/encoding/framing/length_delimited.rs @@ -1,5 +1,4 @@ use bytes::BytesMut; -use derivative::Derivative; use tokio_util::codec::{Encoder, LengthDelimitedCodec}; use vector_config::configurable_component; @@ -8,8 +7,7 @@ use crate::common::length_delimited::LengthDelimitedCoderOptions; /// Config used to build a `LengthDelimitedEncoder`. #[configurable_component] -#[derive(Debug, Clone, Derivative, Eq, PartialEq)] -#[derivative(Default)] +#[derive(Debug, Clone, Default, Eq, PartialEq)] pub struct LengthDelimitedEncoderConfig { /// Options for the length delimited decoder. #[serde(skip_serializing_if = "vector_core::serde::is_default")] diff --git a/lib/codecs/src/encoding/framing/varint_length_delimited.rs b/lib/codecs/src/encoding/framing/varint_length_delimited.rs index 87f3737fa3d86..f874fb3d04e48 100644 --- a/lib/codecs/src/encoding/framing/varint_length_delimited.rs +++ b/lib/codecs/src/encoding/framing/varint_length_delimited.rs @@ -1,5 +1,4 @@ use bytes::{BufMut, BytesMut}; -use derivative::Derivative; use snafu::Snafu; use tokio_util::codec::Encoder; use vector_config::configurable_component; @@ -17,8 +16,7 @@ impl FramingError for VarintFramingError {} /// Config used to build a `VarintLengthDelimitedEncoder`. #[configurable_component] -#[derive(Debug, Clone, PartialEq, Eq, Derivative)] -#[derivative(Default)] +#[derive(Debug, Clone, PartialEq, Eq, Default)] pub struct VarintLengthDelimitedEncoderConfig { /// Maximum frame length #[serde(default = "default_max_frame_length")] diff --git a/src/kafka.rs b/src/kafka.rs index ee652cc4f3f90..0c9315288e313 100644 --- a/src/kafka.rs +++ b/src/kafka.rs @@ -19,12 +19,11 @@ enum KafkaError { /// Supported compression types for Kafka. #[configurable_component] -#[derive(Clone, Copy, Debug, Derivative)] -#[derivative(Default)] +#[derive(Clone, Copy, Debug, Default)] #[serde(rename_all = "lowercase")] pub enum KafkaCompression { /// No compression. - #[derivative(Default)] + #[default] None, /// Gzip. diff --git a/src/sinks/clickhouse/config.rs b/src/sinks/clickhouse/config.rs index 827b684f2234b..58ed0a0ac3445 100644 --- a/src/sinks/clickhouse/config.rs +++ b/src/sinks/clickhouse/config.rs @@ -26,12 +26,11 @@ use crate::{ /// /// [formats]: https://clickhouse.com/docs/en/interfaces/formats #[configurable_component] -#[derive(Clone, Copy, Debug, Derivative, Eq, PartialEq, Hash)] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Hash)] #[serde(rename_all = "snake_case")] -#[derivative(Default)] #[allow(clippy::enum_variant_names)] pub enum Format { - #[derivative(Default)] + #[default] /// JSONEachRow. JsonEachRow, diff --git a/src/sinks/console/config.rs b/src/sinks/console/config.rs index 9175cb3d0b516..c7a8717c0a933 100644 --- a/src/sinks/console/config.rs +++ b/src/sinks/console/config.rs @@ -18,14 +18,13 @@ use crate::{ /// /// [standard_streams]: https://en.wikipedia.org/wiki/Standard_streams #[configurable_component] -#[derive(Clone, Debug, Derivative)] -#[derivative(Default)] +#[derive(Clone, Debug, Default)] #[serde(rename_all = "lowercase")] pub enum Target { /// Write output to [STDOUT][stdout]. /// /// [stdout]: https://en.wikipedia.org/wiki/Standard_streams#Standard_output_(stdout) - #[derivative(Default)] + #[default] Stdout, /// Write output to [STDERR][stderr]. diff --git a/src/sinks/elasticsearch/mod.rs b/src/sinks/elasticsearch/mod.rs index fddae4eb3caf9..1466c415dbd1b 100644 --- a/src/sinks/elasticsearch/mod.rs +++ b/src/sinks/elasticsearch/mod.rs @@ -80,7 +80,7 @@ pub enum ElasticsearchMode { /// Bulk API actions. #[configurable_component] -#[derive(Clone, Copy, Debug, Derivative, Eq, Hash, PartialEq)] +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] #[serde(deny_unknown_fields, rename_all = "snake_case")] pub enum BulkAction { /// The `index` action. @@ -127,7 +127,7 @@ impl TryFrom<&str> for BulkAction { /// Elasticsearch version types. #[configurable_component] -#[derive(Clone, Copy, Debug, Derivative, Eq, Hash, PartialEq)] +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] #[serde(deny_unknown_fields, rename_all = "snake_case")] pub enum VersionType { /// The `internal` type. diff --git a/src/sinks/gcp/stackdriver/logs/config.rs b/src/sinks/gcp/stackdriver/logs/config.rs index 94be249e1dfff..9b66eea4ac214 100644 --- a/src/sinks/gcp/stackdriver/logs/config.rs +++ b/src/sinks/gcp/stackdriver/logs/config.rs @@ -162,8 +162,7 @@ pub(super) enum StackdriverLogName { /// Label Configuration. #[configurable_component] -#[derive(Clone, Debug, Derivative)] -#[derivative(Default)] +#[derive(Clone, Debug, Default)] pub(super) struct StackdriverLabelConfig { /// The value of this field is used to retrieve the associated labels from the `jsonPayload` /// and extract their values to set as LogEntry labels. diff --git a/src/sinks/gcp_chronicle/compression.rs b/src/sinks/gcp_chronicle/compression.rs index 3002d934f18dd..b58478eb6d04f 100644 --- a/src/sinks/gcp_chronicle/compression.rs +++ b/src/sinks/gcp_chronicle/compression.rs @@ -20,11 +20,10 @@ use crate::sinks::util::{ }; /// Compression configuration. -#[derive(Clone, Copy, Debug, Derivative, Eq, PartialEq)] -#[derivative(Default)] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] pub enum ChronicleCompression { /// No compression. - #[derivative(Default)] + #[default] None, /// [Gzip][gzip] compression. diff --git a/src/sinks/gcs_common/config.rs b/src/sinks/gcs_common/config.rs index 9f148190a72b3..dcebb6bea9e77 100644 --- a/src/sinks/gcs_common/config.rs +++ b/src/sinks/gcs_common/config.rs @@ -26,8 +26,7 @@ pub fn default_endpoint() -> String { /// /// [predefined_acls]: https://cloud.google.com/storage/docs/access-control/lists#predefined-acl #[configurable_component] -#[derive(Clone, Copy, Debug, Derivative)] -#[derivative(Default)] +#[derive(Clone, Copy, Debug, Default)] #[serde(rename_all = "kebab-case")] pub enum GcsPredefinedAcl { /// Bucket/object can be read by authenticated users. @@ -65,7 +64,7 @@ pub enum GcsPredefinedAcl { /// part of the project team is granted the `READER` permission. /// /// This is the default. - #[derivative(Default)] + #[default] ProjectPrivate, /// Bucket/object can be read publicly. @@ -81,14 +80,13 @@ pub enum GcsPredefinedAcl { /// /// [storage_classes]: https://cloud.google.com/storage/docs/storage-classes #[configurable_component] -#[derive(Clone, Copy, Debug, Derivative, PartialEq, Eq)] -#[derivative(Default)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] #[serde(rename_all = "SCREAMING_SNAKE_CASE")] pub enum GcsStorageClass { /// Standard storage. /// /// This is the default. - #[derivative(Default)] + #[default] Standard, /// Nearline storage. diff --git a/src/sinks/greptimedb/metrics/config.rs b/src/sinks/greptimedb/metrics/config.rs index a4003ae4292a0..4261486a05706 100644 --- a/src/sinks/greptimedb/metrics/config.rs +++ b/src/sinks/greptimedb/metrics/config.rs @@ -18,7 +18,7 @@ use crate::sinks::{ #[configurable(metadata( deprecated = "The `greptimedb` sink has been renamed. Please use `greptimedb_metrics` instead." ))] -#[derive(Clone, Debug, Derivative)] +#[derive(Clone, Debug, Default)] pub struct GreptimeDBConfig(GreptimeDBMetricsConfig); impl GenerateConfig for GreptimeDBConfig { diff --git a/src/sinks/http/config.rs b/src/sinks/http/config.rs index 2c9f257fae5e1..81df73add1efb 100644 --- a/src/sinks/http/config.rs +++ b/src/sinks/http/config.rs @@ -115,9 +115,8 @@ pub struct HttpSinkConfig { /// /// [rfc9110]: https://datatracker.ietf.org/doc/html/rfc9110#section-9.1 #[configurable_component] -#[derive(Clone, Copy, Debug, Derivative, Eq, PartialEq)] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] #[serde(rename_all = "snake_case")] -#[derivative(Default)] pub enum HttpMethod { /// GET. Get, @@ -126,7 +125,7 @@ pub enum HttpMethod { Head, /// POST. - #[derivative(Default)] + #[default] Post, /// PUT. diff --git a/src/sinks/loki/config.rs b/src/sinks/loki/config.rs index 06f827b353bab..16f25dc3d554b 100644 --- a/src/sinks/loki/config.rs +++ b/src/sinks/loki/config.rs @@ -171,8 +171,7 @@ impl SinkBatchSettings for LokiDefaultBatchSettings { /// any necessary sorting/reordering. If you're using an earlier version, then you must use `Drop` /// or `RewriteTimestamp` depending on which option makes the most sense for your use case. #[configurable_component] -#[derive(Copy, Clone, Debug, Derivative)] -#[derivative(Default)] +#[derive(Copy, Clone, Debug, Default)] #[serde(rename_all = "snake_case")] pub enum OutOfOrderAction { /// Accept the event. @@ -180,7 +179,7 @@ pub enum OutOfOrderAction { /// The event is not dropped and is sent without modification. /// /// Requires Loki 2.4.0 or newer. - #[derivative(Default)] + #[default] Accept, /// Rewrite the timestamp of the event to the timestamp of the latest event seen by the sink. diff --git a/src/sinks/mqtt/config.rs b/src/sinks/mqtt/config.rs index 4b6e589d3beb5..af0e4a3aaf645 100644 --- a/src/sinks/mqtt/config.rs +++ b/src/sinks/mqtt/config.rs @@ -53,13 +53,12 @@ pub struct MqttSinkConfig { /// Supported Quality of Service types for MQTT. #[configurable_component] -#[derive(Clone, Copy, Debug, Derivative)] -#[derivative(Default)] +#[derive(Clone, Copy, Debug, Default)] #[serde(rename_all = "lowercase")] #[allow(clippy::enum_variant_names)] pub enum MqttQoS { /// AtLeastOnce. - #[derivative(Default)] + #[default] AtLeastOnce, /// AtMostOnce. diff --git a/src/sinks/new_relic/config.rs b/src/sinks/new_relic/config.rs index f0f6c1d7695a3..fa1e6622d6d61 100644 --- a/src/sinks/new_relic/config.rs +++ b/src/sinks/new_relic/config.rs @@ -12,12 +12,11 @@ use crate::{http::HttpClient, sinks::prelude::*}; /// New Relic region. #[configurable_component] -#[derive(Clone, Copy, Debug, Derivative, Eq, PartialEq)] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] #[serde(rename_all = "snake_case")] -#[derivative(Default)] pub enum NewRelicRegion { /// US region. - #[derivative(Default)] + #[default] Us, /// EU region. @@ -26,12 +25,11 @@ pub enum NewRelicRegion { /// New Relic API endpoint. #[configurable_component] -#[derive(Clone, Copy, Derivative, Debug, Eq, PartialEq)] +#[derive(Clone, Copy, Default, Debug, Eq, PartialEq)] #[serde(rename_all = "snake_case")] -#[derivative(Default)] pub enum NewRelicApi { /// Events API. - #[derivative(Default)] + #[default] Events, /// Metrics API. diff --git a/src/sinks/pulsar/config.rs b/src/sinks/pulsar/config.rs index 6b27035715226..e149c9c5acfe5 100644 --- a/src/sinks/pulsar/config.rs +++ b/src/sinks/pulsar/config.rs @@ -162,12 +162,11 @@ pub struct OAuth2Config { /// Supported compression types for Pulsar. #[configurable_component] -#[derive(Clone, Copy, Debug, Derivative)] -#[derivative(Default)] +#[derive(Clone, Copy, Debug, Default)] #[serde(rename_all = "lowercase")] pub enum PulsarCompression { /// No compression. - #[derivative(Default)] + #[default] None, /// LZ4. diff --git a/src/sinks/redis/config.rs b/src/sinks/redis/config.rs index 17939e679da7a..69dfbdf44e6a3 100644 --- a/src/sinks/redis/config.rs +++ b/src/sinks/redis/config.rs @@ -22,8 +22,7 @@ impl TowerRequestConfigDefaults for RedisTowerRequestConfigDefaults { /// Redis data type to store messages in. #[configurable_component] -#[derive(Clone, Copy, Debug, Derivative)] -#[derivative(Default)] +#[derive(Clone, Copy, Debug, Default)] #[serde(rename_all = "lowercase")] pub enum DataTypeConfig { /// The Redis `list` type. @@ -31,7 +30,7 @@ pub enum DataTypeConfig { /// This resembles a deque, where messages can be popped and pushed from either end. /// /// This is the default. - #[derivative(Default)] + #[default] List, /// The Redis `sorted set` type. @@ -48,7 +47,7 @@ pub enum DataTypeConfig { /// List-specific options. #[configurable_component] -#[derive(Clone, Copy, Debug, Derivative, Eq, PartialEq)] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] #[serde(rename_all = "lowercase")] pub struct ListOption { /// The method to use for pushing messages into a `list`. @@ -57,8 +56,7 @@ pub struct ListOption { /// Method for pushing messages into a `list`. #[configurable_component] -#[derive(Clone, Copy, Debug, Derivative, Eq, PartialEq)] -#[derivative(Default)] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] #[serde(rename_all = "lowercase")] pub enum ListMethod { /// Use the `rpush` method. @@ -66,7 +64,7 @@ pub enum ListMethod { /// This pushes messages onto the tail of the list. /// /// This is the default. - #[derivative(Default)] + #[default] RPush, /// Use the `lpush` method. @@ -77,7 +75,7 @@ pub enum ListMethod { /// Sorted Set-specific options #[configurable_component] -#[derive(Clone, Debug, Derivative, Eq, PartialEq)] +#[derive(Clone, Debug, Default, Eq, PartialEq)] #[serde(rename_all = "lowercase")] pub struct SortedSetOption { /// The method to use for pushing messages into a `sorted set`. @@ -96,8 +94,7 @@ pub struct SortedSetOption { /// Method for pushing messages into a `sorted set`. #[configurable_component] -#[derive(Clone, Copy, Debug, Derivative, Eq, PartialEq)] -#[derivative(Default)] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] #[serde(rename_all = "lowercase")] pub enum SortedSetMethod { /// Use the `zadd` method. @@ -105,7 +102,7 @@ pub enum SortedSetMethod { /// This adds messages onto a queue with a score. /// /// This is the default. - #[derivative(Default)] + #[default] ZAdd, } @@ -281,14 +278,13 @@ impl From for SentinelNodeConnectionInfo { /// How/if TLS should be established. #[configurable_component] -#[derive(Clone, Copy, Debug, Derivative, Eq, PartialEq)] -#[derivative(Default)] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] #[serde(rename_all = "lowercase")] pub enum MaybeTlsMode { /// Don't use TLS. /// /// This is the default. - #[derivative(Default)] + #[default] None, /// Enable TLS with certificate verification. @@ -311,8 +307,7 @@ impl From for Option { /// Connection independent information used to establish a connection /// to a redis instance sentinel owns. #[configurable_component] -#[derive(Clone, Debug, Derivative)] -#[derivative(Default)] +#[derive(Clone, Debug, Default)] pub struct RedisConnectionSettings { /// The database number to use. Usually `0`. pub db: i64, @@ -340,13 +335,12 @@ impl From for RedisConnectionInfo { /// The communication protocol to use with the redis server. #[configurable_component] -#[derive(Clone, Copy, Debug, Derivative, Eq, PartialEq)] -#[derivative(Default)] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] pub enum RedisProtocolVersion { /// Use RESP2. /// /// This is the default. - #[derivative(Default)] + #[default] RESP2, /// Use RESP3. diff --git a/src/sinks/s3_common/config.rs b/src/sinks/s3_common/config.rs index aeb7d4b93113b..f78c649c6f667 100644 --- a/src/sinks/s3_common/config.rs +++ b/src/sinks/s3_common/config.rs @@ -141,12 +141,11 @@ fn example_tags() -> HashMap { /// /// [aws_docs]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/storage-class-intro.html #[configurable_component] -#[derive(Clone, Copy, Debug, Derivative, PartialEq, Eq)] -#[derivative(Default)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] #[serde(rename_all = "SCREAMING_SNAKE_CASE")] pub enum S3StorageClass { /// Standard Redundancy. - #[derivative(Default)] + #[default] Standard, /// Reduced Redundancy. @@ -228,8 +227,7 @@ impl From for ServerSideEncryption { /// /// [canned_acl]: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl #[configurable_component] -#[derive(Clone, Copy, Debug, Derivative)] -#[derivative(Default)] +#[derive(Clone, Copy, Debug, Default)] #[serde(rename_all = "kebab-case")] pub enum S3CannedAcl { /// Bucket/object are private. @@ -238,7 +236,7 @@ pub enum S3CannedAcl { /// access. /// /// This is the default. - #[derivative(Default)] + #[default] Private, /// Bucket/object can be read publicly. diff --git a/src/sinks/util/adaptive_concurrency/tests.rs b/src/sinks/util/adaptive_concurrency/tests.rs index bce42b64ca156..740477d6de144 100644 --- a/src/sinks/util/adaptive_concurrency/tests.rs +++ b/src/sinks/util/adaptive_concurrency/tests.rs @@ -50,11 +50,10 @@ use crate::{ /// Request handling action when the request limit has been exceeded. #[configurable_component] -#[derive(Clone, Copy, Debug, Derivative)] -#[derivative(Default)] +#[derive(Clone, Copy, Debug, Default)] #[serde(rename_all = "lowercase")] enum Action { - #[derivative(Default)] + #[default] /// Additional requests will return with an error. Defer, diff --git a/src/sinks/util/buffer/compression.rs b/src/sinks/util/buffer/compression.rs index c990025c853d4..38e87108f7f9a 100644 --- a/src/sinks/util/buffer/compression.rs +++ b/src/sinks/util/buffer/compression.rs @@ -16,11 +16,10 @@ use vector_lib::configurable::{ use crate::sinks::util::zstd::ZstdCompressionLevel; /// Compression configuration. -#[derive(Copy, Clone, Debug, Derivative, Eq, PartialEq)] -#[derivative(Default)] +#[derive(Copy, Clone, Debug, Default, Eq, PartialEq)] pub enum Compression { /// No compression. - #[derivative(Default)] + #[default] None, /// [Gzip][gzip] compression. diff --git a/src/sinks/util/service/concurrency.rs b/src/sinks/util/service/concurrency.rs index 44c982caff0f9..43c2643807a99 100644 --- a/src/sinks/util/service/concurrency.rs +++ b/src/sinks/util/service/concurrency.rs @@ -18,7 +18,7 @@ use vector_lib::configurable::{ /// /// This can be set either to one of the below enum values or to a positive integer, which denotes /// a fixed concurrency limit. -#[derive(Clone, Copy, Debug, Derivative, Eq, PartialEq, Default)] +#[derive(Clone, Copy, Debug, Eq, PartialEq, Default)] pub enum Concurrency { /// A fixed concurrency of 1. /// diff --git a/src/sources/aws_kinesis_firehose/mod.rs b/src/sources/aws_kinesis_firehose/mod.rs index 4c86b74165e9b..24809473cdfb9 100644 --- a/src/sources/aws_kinesis_firehose/mod.rs +++ b/src/sources/aws_kinesis_firehose/mod.rs @@ -113,9 +113,8 @@ const fn access_keys_example() -> [&'static str; 2] { /// Compression scheme for records in a Firehose message. #[configurable_component] #[configurable(metadata(docs::advanced))] -#[derive(Clone, Copy, Debug, Derivative, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] #[serde(rename_all = "lowercase")] -#[derivative(Default)] pub enum Compression { /// Automatically attempt to determine the compression scheme. /// @@ -127,7 +126,7 @@ pub enum Compression { /// set `gzip` in this field so that any records that are not-gzipped are rejected. /// /// [magic_bytes]: https://en.wikipedia.org/wiki/List_of_file_signatures - #[derivative(Default)] + #[default] Auto, /// Uncompressed. diff --git a/src/sources/aws_s3/mod.rs b/src/sources/aws_s3/mod.rs index 1f2532bd51a24..38ca6e4cb6901 100644 --- a/src/sources/aws_s3/mod.rs +++ b/src/sources/aws_s3/mod.rs @@ -34,9 +34,8 @@ pub mod sqs; /// Compression scheme for objects retrieved from S3. #[configurable_component] #[configurable(metadata(docs::advanced))] -#[derive(Clone, Copy, Debug, Derivative, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] #[serde(rename_all = "lowercase")] -#[derivative(Default)] pub enum Compression { /// Automatically attempt to determine the compression scheme. /// @@ -44,7 +43,7 @@ pub enum Compression { /// `Content-Type` metadata, as well as the key suffix (for example, `.gz`). /// /// It is set to `none` if the compression scheme cannot be determined. - #[derivative(Default)] + #[default] Auto, /// Uncompressed. @@ -59,14 +58,13 @@ pub enum Compression { /// Strategies for consuming objects from AWS S3. #[configurable_component] -#[derive(Clone, Copy, Debug, Derivative)] +#[derive(Clone, Copy, Debug, Default)] #[serde(rename_all = "lowercase")] -#[derivative(Default)] enum Strategy { /// Consumes objects by processing bucket notification events sent to an [AWS SQS queue][aws_sqs]. /// /// [aws_sqs]: https://aws.amazon.com/sqs/ - #[derivative(Default)] + #[default] Sqs, } diff --git a/src/sources/demo_logs.rs b/src/sources/demo_logs.rs index 861a51a957d44..fe02da4574c80 100644 --- a/src/sources/demo_logs.rs +++ b/src/sources/demo_logs.rs @@ -94,8 +94,7 @@ pub enum DemoLogsConfigError { /// Output format configuration. #[configurable_component] -#[derive(Clone, Debug, Derivative)] -#[derivative(Default)] +#[derive(Clone, Debug, Default)] #[serde(tag = "format", rename_all = "snake_case")] #[configurable(metadata( docs::enum_tag_description = "The format of the randomly generated output." @@ -136,7 +135,7 @@ pub enum OutputFormat { /// Randomly generated HTTP server logs in [JSON][json] format. /// /// [json]: https://en.wikipedia.org/wiki/JSON - #[derivative(Default)] + #[default] Json, } diff --git a/src/sources/redis/mod.rs b/src/sources/redis/mod.rs index 0f71009701026..eb2545059ecf8 100644 --- a/src/sources/redis/mod.rs +++ b/src/sources/redis/mod.rs @@ -35,12 +35,11 @@ enum BuildError { /// Data type to use for reading messages from Redis. #[configurable_component] -#[derive(Copy, Clone, Debug, Derivative)] -#[derivative(Default)] +#[derive(Copy, Clone, Debug, Default)] #[serde(rename_all = "lowercase")] pub enum DataTypeConfig { /// The `list` data type. - #[derivative(Default)] + #[default] List, /// The `channel` data type. @@ -51,7 +50,7 @@ pub enum DataTypeConfig { /// Options for the Redis `list` data type. #[configurable_component] -#[derive(Copy, Clone, Debug, Default, Derivative, Eq, PartialEq)] +#[derive(Copy, Clone, Debug, Default, Eq, PartialEq)] #[serde(deny_unknown_fields, rename_all = "lowercase")] pub struct ListOption { #[configurable(derived)] @@ -60,12 +59,11 @@ pub struct ListOption { /// Method for getting events from the `list` data type. #[configurable_component] -#[derive(Clone, Copy, Debug, Derivative, Eq, PartialEq)] -#[derivative(Default)] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] #[serde(rename_all = "lowercase")] pub enum Method { /// Pop messages from the head of the list. - #[derivative(Default)] + #[default] Lpop, /// Pop messages from the tail of the list. diff --git a/src/sources/util/body_decoding.rs b/src/sources/util/body_decoding.rs index 6a99da3240792..d7947940e9ab4 100644 --- a/src/sources/util/body_decoding.rs +++ b/src/sources/util/body_decoding.rs @@ -2,12 +2,11 @@ use vector_lib::configurable::configurable_component; /// Content encoding. #[configurable_component] -#[derive(Clone, Copy, Debug, Derivative, Eq, PartialEq)] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] #[serde(rename_all = "snake_case")] -#[derivative(Default)] pub enum Encoding { /// Plaintext. - #[derivative(Default)] + #[default] Text, /// Newline-delimited JSON. From a8730894c366cd5232353530fef4a4970cd6c307 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 17 Mar 2026 11:00:09 -0400 Subject: [PATCH 012/326] chore(deps): bump lz4_flex from 0.11.5 to 0.11.6 (#24939) Bumps [lz4_flex](https://github.com/pseitz/lz4_flex) from 0.11.5 to 0.11.6. - [Release notes](https://github.com/pseitz/lz4_flex/releases) - [Changelog](https://github.com/PSeitz/lz4_flex/blob/main/CHANGELOG.md) - [Commits](https://github.com/pseitz/lz4_flex/compare/0.11.5...0.11.6) --- updated-dependencies: - dependency-name: lz4_flex dependency-version: 0.11.6 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 635320a783e62..610fc0628a731 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6464,9 +6464,9 @@ dependencies = [ [[package]] name = "lz4_flex" -version = "0.11.5" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08ab2867e3eeeca90e844d1940eab391c9dc5228783db2ed999acbc0a9ed375a" +checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a" dependencies = [ "twox-hash", ] From 9dc3d3e9f7d486ce363975f21b1e1b3b537cb46d Mon Sep 17 00:00:00 2001 From: Thomas Date: Tue, 17 Mar 2026 11:11:36 -0400 Subject: [PATCH 013/326] fix(aggregate transform): reduce memory when not in `Diff` mode (#24943) * Separate prev_map into conditional enum struct * Do the same with multi_map * Add changelog * Update changelog.d/24943_aggregate_transform_reduce_memory.fix.md Co-authored-by: Pavlos Rontidis * Mode -> InnerMode --------- Co-authored-by: Pavlos Rontidis --- ...3_aggregate_transform_reduce_memory.fix.md | 4 + src/transforms/aggregate.rs | 109 ++++++++++++++---- 2 files changed, 89 insertions(+), 24 deletions(-) create mode 100644 changelog.d/24943_aggregate_transform_reduce_memory.fix.md diff --git a/changelog.d/24943_aggregate_transform_reduce_memory.fix.md b/changelog.d/24943_aggregate_transform_reduce_memory.fix.md new file mode 100644 index 0000000000000..4f3be38504bcd --- /dev/null +++ b/changelog.d/24943_aggregate_transform_reduce_memory.fix.md @@ -0,0 +1,4 @@ +Reduced the memory usage in the `aggregate` transform where previous values were being held +even if `mode` was not set to `Diff`. + +authors: thomasqueirozb diff --git a/src/transforms/aggregate.rs b/src/transforms/aggregate.rs index 3c8622ec2dba5..c00315ad4871a 100644 --- a/src/transforms/aggregate.rs +++ b/src/transforms/aggregate.rs @@ -74,6 +74,65 @@ pub enum AggregationMode { Stdev, } +#[derive(Clone, Debug, Default, PartialEq)] +enum InnerMode { + /// Default mode. Sums incremental metrics and uses the latest value for absolute metrics. + #[default] + Auto, + + /// Sums incremental metrics, ignores absolute + Sum, + + /// Returns the latest value for absolute metrics, ignores incremental + Latest, + + /// Counts metrics for incremental and absolute metrics + Count, + + /// Returns difference between latest value for absolute, ignores incremental + Diff { + prev_map: HashMap, + }, + + /// Max value of absolute metric, ignores incremental + Max, + + /// Min value of absolute metric, ignores incremental + Min, + + /// Mean value of absolute metric, ignores incremental + Mean { + multi_map: HashMap>, + }, + + /// Stdev value of absolute metric, ignores incremental + Stdev { + multi_map: HashMap>, + }, +} + +impl From for InnerMode { + fn from(value: AggregationMode) -> Self { + match value { + AggregationMode::Auto => InnerMode::Auto, + AggregationMode::Sum => InnerMode::Sum, + AggregationMode::Latest => InnerMode::Latest, + AggregationMode::Count => InnerMode::Count, + AggregationMode::Diff => InnerMode::Diff { + prev_map: HashMap::default(), + }, + AggregationMode::Max => InnerMode::Max, + AggregationMode::Min => InnerMode::Min, + AggregationMode::Mean => InnerMode::Mean { + multi_map: HashMap::default(), + }, + AggregationMode::Stdev => InnerMode::Stdev { + multi_map: HashMap::default(), + }, + } + } +} + const fn default_mode() -> AggregationMode { AggregationMode::Auto } @@ -110,9 +169,7 @@ type MetricEntry = (MetricData, EventMetadata); pub struct Aggregate { interval: Duration, map: HashMap, - prev_map: HashMap, - multi_map: HashMap>, - mode: AggregationMode, + mode: InnerMode, } impl Aggregate { @@ -120,38 +177,34 @@ impl Aggregate { Ok(Self { interval: Duration::from_millis(config.interval_ms), map: Default::default(), - prev_map: Default::default(), - multi_map: Default::default(), - mode: config.mode, + mode: config.mode.into(), }) } fn record(&mut self, event: Event) { let (series, data, metadata) = event.into_metric().into_parts(); - match self.mode { - AggregationMode::Auto => match data.kind { + match &mut self.mode { + InnerMode::Auto => match data.kind { MetricKind::Incremental => self.record_sum(series, data, metadata), MetricKind::Absolute => { self.map.insert(series, (data, metadata)); } }, - AggregationMode::Sum => self.record_sum(series, data, metadata), - AggregationMode::Latest | AggregationMode::Diff => match data.kind { + InnerMode::Sum => self.record_sum(series, data, metadata), + InnerMode::Latest | InnerMode::Diff { .. } => match data.kind { MetricKind::Incremental => (), MetricKind::Absolute => { self.map.insert(series, (data, metadata)); } }, - AggregationMode::Count => self.record_count(series, data, metadata), - AggregationMode::Max | AggregationMode::Min => { - self.record_comparison(series, data, metadata) - } - AggregationMode::Mean | AggregationMode::Stdev => match data.kind { + InnerMode::Count => self.record_count(series, data, metadata), + InnerMode::Max | InnerMode::Min => self.record_comparison(series, data, metadata), + InnerMode::Mean { multi_map } | InnerMode::Stdev { multi_map } => match data.kind { MetricKind::Incremental => (), MetricKind::Absolute => { if matches!(data.value, MetricValue::Gauge { value: _ }) { - match self.multi_map.entry(series) { + match multi_map.entry(series) { Entry::Occupied(mut entry) => { let existing = entry.get_mut(); existing.push((data, metadata)); @@ -227,8 +280,8 @@ impl Aggregate { && let MetricValue::Gauge { value: new_value } = data.value() { let should_update = match self.mode { - AggregationMode::Max => new_value > existing_value, - AggregationMode::Min => new_value < existing_value, + InnerMode::Max => new_value > existing_value, + InnerMode::Min => new_value < existing_value, _ => false, }; if should_update { @@ -251,8 +304,8 @@ impl Aggregate { let map = std::mem::take(&mut self.map); for (series, entry) in map.clone().into_iter() { let mut metric = Metric::from_parts(series, entry.0, entry.1); - if matches!(self.mode, AggregationMode::Diff) - && let Some(prev_entry) = self.prev_map.get(metric.series()) + if let InnerMode::Diff { prev_map } = &self.mode + && let Some(prev_entry) = prev_map.get(metric.series()) && metric.data().kind == prev_entry.0.kind && !metric.subtract(&prev_entry.0) { @@ -261,7 +314,13 @@ impl Aggregate { output.push(Event::Metric(metric)); } - let multi_map = std::mem::take(&mut self.multi_map); + let multi_map = match &mut self.mode { + InnerMode::Mean { multi_map } | InnerMode::Stdev { multi_map } => { + std::mem::take(multi_map) + } + _ => HashMap::default(), + }; + 'outer: for (series, entries) in multi_map.into_iter() { if entries.is_empty() { continue; @@ -287,11 +346,11 @@ impl Aggregate { let final_mean = final_sum.clone(); match self.mode { - AggregationMode::Mean => { + InnerMode::Mean { .. } => { let metric = Metric::from_parts(series, final_mean, final_metadata); output.push(Event::Metric(metric)); } - AggregationMode::Stdev => { + InnerMode::Stdev { .. } => { let variance = entries .iter() .filter_map(|(data, _)| { @@ -315,7 +374,9 @@ impl Aggregate { } } - self.prev_map = map; + if let InnerMode::Diff { prev_map } = &mut self.mode { + *prev_map = map; + } emit!(AggregateFlushed); } } From 42b844e76882fa5cbf1fd5cfafcf653101dc9ad0 Mon Sep 17 00:00:00 2001 From: Thomas Date: Tue, 17 Mar 2026 11:13:06 -0400 Subject: [PATCH 014/326] chore(website deps): remove unused fs package (#24945) --- website/package.json | 1 - website/yarn.lock | 5 ----- 2 files changed, 6 deletions(-) diff --git a/website/package.json b/website/package.json index fec13b066895d..f3deb2d5ac37c 100644 --- a/website/package.json +++ b/website/package.json @@ -40,7 +40,6 @@ "domhandler": "^4.2.0", "dotenv-defaults": "^2.0.2", "downshift": "^6.1.3", - "fs": "^0.0.1-security", "glob-promise": "^4.2.0", "lodash.chunk": "^4.2.0", "path": "^0.12.7", diff --git a/website/yarn.lock b/website/yarn.lock index 6e031634c03ef..57cd6d50741c0 100644 --- a/website/yarn.lock +++ b/website/yarn.lock @@ -2432,11 +2432,6 @@ fs.realpath@^1.0.0: resolved "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz" integrity sha1-FQStJSMVjKpA20onh8sBQRmU6k8= -fs@^0.0.1-security: - version "0.0.1-security" - resolved "https://registry.npmjs.org/fs/-/fs-0.0.1-security.tgz" - integrity sha1-invTcYa23d84E/I4WLV+yq9eQdQ= - fsevents@~2.3.2: version "2.3.2" resolved "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz" From 218aa4d29b7b0922f604b7f6d8a3ebd0cc8e0931 Mon Sep 17 00:00:00 2001 From: Thomas Date: Tue, 17 Mar 2026 12:01:18 -0400 Subject: [PATCH 015/326] chore(website): Add osv-scanner.toml and ignore MAL-2025-3174 (#24946) Add osv-scanner.toml --- website/osv-scanner.toml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 website/osv-scanner.toml diff --git a/website/osv-scanner.toml b/website/osv-scanner.toml new file mode 100644 index 0000000000000..65d943205867c --- /dev/null +++ b/website/osv-scanner.toml @@ -0,0 +1,3 @@ +[[IgnoredVulns]] +id = "MAL-2025-3174" +reason = "typesense-sync: package pulled in from s3 and not from npm" From 83bcf4cb297f85525f88a3cff1870ab8fbddd47b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 17 Mar 2026 12:23:49 -0400 Subject: [PATCH 016/326] chore(deps): bump quinn-proto from 0.11.9 to 0.11.14 (#24926) * chore(deps): bump quinn-proto from 0.11.9 to 0.11.14 Bumps [quinn-proto](https://github.com/quinn-rs/quinn) from 0.11.9 to 0.11.14. - [Release notes](https://github.com/quinn-rs/quinn/releases) - [Commits](https://github.com/quinn-rs/quinn/compare/quinn-proto-0.11.9...quinn-proto-0.11.14) --- updated-dependencies: - dependency-name: quinn-proto dependency-version: 0.11.14 dependency-type: indirect ... Signed-off-by: dependabot[bot] * Update licenses --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Thomas --- Cargo.lock | 15 +++++++++++---- LICENSE-3rdparty.csv | 1 + 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 610fc0628a731..08becf0e3552c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6424,6 +6424,12 @@ dependencies = [ "linked-hash-map", ] +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + [[package]] name = "lua-src" version = "547.0.0" @@ -8660,13 +8666,14 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.9" +version = "0.11.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2fe5ef3495d7d2e377ff17b1a8ce2ee2ec2a18cde8b6ad6619d65d0701c135d" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" dependencies = [ "bytes 1.11.1", - "getrandom 0.2.15", - "rand 0.8.5", + "getrandom 0.3.4", + "lru-slab", + "rand 0.9.2", "ring", "rustc-hash", "rustls 0.23.23", diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 523769af4238a..09e39573c8126 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -450,6 +450,7 @@ lockfree-object-pool,https://github.com/EVaillant/lockfree-object-pool,BSL-1.0,E log,https://github.com/rust-lang/log,MIT OR Apache-2.0,The Rust Project Developers lru,https://github.com/jeromefroe/lru-rs,MIT,Jerome Froelich lru-cache,https://github.com/contain-rs/lru-cache,MIT OR Apache-2.0,Stepan Koltsov +lru-slab,https://github.com/Ralith/lru-slab,MIT OR Apache-2.0 OR Zlib,Benjamin Saunders lz4,https://github.com/10xGenomics/lz4-rs,MIT,"Jens Heyens , Artem V. Navrotskiy , Patrick Marks " lz4-sys,https://github.com/10xGenomics/lz4-rs,MIT,"Jens Heyens , Artem V. Navrotskiy , Patrick Marks " lz4_flex,https://github.com/pseitz/lz4_flex,MIT,"Pascal Seitz , Arthur Silva , ticki " From dc9d8de0c8f2ff7a04a7373820f1ef652ed30520 Mon Sep 17 00:00:00 2001 From: Vladimir Zhuk Date: Tue, 17 Mar 2026 19:29:20 +0100 Subject: [PATCH 017/326] chore(datadog_metrics sink): switch to v2 endpoint (#24842) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(datadog_metrics sink): switch to Datadog Metrics V2 endpoint * chore(regression): Add statsd_to_datadog_metrics performance test Add regression test to validate datadog_metrics sink v2 endpoint performance under realistic high-throughput DogStatsD load. Test Configuration: - Load: Default lading dogstatsd settings (realistic ~2KB messages) - Throughput: 500 Mb/s → ~250k events/sec - Batch: Default settings (100k max_events, 2s timeout) - Validates batch splitting when payloads exceed v2 size limits This test ensures v2 endpoint correctly handles batch splitting with realistic high-cardinality DogStatsD metrics under load. Co-Authored-By: Claude Sonnet 4.5 * fix(datadog_metrics sink): apply per-endpoint batch size limit Different series endpoints have different uncompressed payload limits (v2 is 12x smaller than v1). This ensures each batch fits in a single HTTP request without splitting, reducing memory overhead. Co-Authored-By: Claude Sonnet 4.6 * fix(datadog_metrics sink): fix test compilation errors in encoder and config - Remove references to DatadogMetricsCompression and request_compression from encoder.rs tests (those symbols don't exist in current codebase; they belong to an unmerged compression-options branch) - Fix batcher_user_max_bytes_is_preserved test to avoid struct update syntax with private PhantomData fields in BatchConfig Co-Authored-By: Claude Sonnet 4.6 * chore(datadog_metrics sink): warn on deprecated VECTOR_TEMP_USE_DD_METRICS_SERIES_V2_API env var The old opt-in env var is now a no-op since v2 is the default. Emit a one-time warning so existing users know they can safely remove it. Co-Authored-By: Claude Sonnet 4.6 * trigger build * feat(datadog_metrics sink): use separate batch size limits for Series and Sketches Series v2 has a 5 MiB uncompressed payload limit while Sketches allows 60 MiB. Previously both used the same (Series-derived) cap, which over-fragmented sketch-heavy workloads into many small requests. To support per-partition batch configuration, `PartitionedBatcher` now passes the partition key to the batch config factory closure (`Fn(&Key) -> C` instead of `Fn() -> C`). The explicit `timeout: Duration` parameter replaces the previous extraction via `settings().timeout()`. All existing callers are updated mechanically. The `datadog_metrics` sink uses the new capability to select the appropriate byte size limit per endpoint partition, keeping Series at 5 MiB and Sketches at 60 MiB. Co-Authored-By: Claude Sonnet 4.6 * fix(datadog_metrics sink): fix CI failures after partitioned batcher refactor - Add `Batch = B` constraint to `with_timer` test constructor so the compiler can infer `B` from the batch config type - Replace `Box::new(move |_| ...)` with unboxed `move |_: &u8| ...` in tests to avoid HRTB inference issues with `Fn(&Key) -> C` - Run `make fmt` to fix formatting in several sink files Co-Authored-By: Claude Sonnet 4.6 * feat(datadog_metrics sink): add series_api_version config field, remove env var support - Add `series_api_version` config option (`v1` | `v2`, default `v2`) to `DatadogMetricsConfig` - Make `SeriesApiVersion` a proper `#[configurable_component]` enum with serde support - Remove `VECTOR_TEMP_USE_DD_METRICS_SERIES_V1_API` and `VECTOR_TEMP_USE_DD_METRICS_SERIES_V2_API` env vars - Remove `get_api_version_backwards_compatible()` and `DatadogMetricsEndpoint::series()` helpers - Thread `series_api_version` through config, sink, partitioner, and request builder - Add `v1_batch_config_uses_v1_size_limit` test; update existing tests to pass explicit version Rationale: Replace the hidden, process-global env var mechanism with a standard config field so users can control the series API version per-sink without a full Vector restart, and so the option is properly documented alongside other sink settings. * update docs * mark v1 as deprecated * fix(datadog_metrics sink): fix gauge interval assertion after normalization strips interval_ms `into_absolute()` always sets `interval_ms: None`, so the encoded interval for gauges is always 0. The test expectation of 10 was dead code until v2 became the default endpoint. Co-Authored-By: Claude Sonnet 4.6 * test(datadog_metrics sink): validate both v1 and v2 series endpoints in e2e tests - Add series_api_version matrix dimension ['v1', 'v2'] to test.yaml, expanding coverage from 3 to 6 environments - Pass CONFIG_SERIES_API_VERSION through docker compose environment to the Vector container - Parameterize series_api_version in vector.toml from the env var - Refactor series::validate() to dispatch to v1 or v2 fetch function based on CONFIG_SERIES_API_VERSION - Extract assertions into compare_intakes() helper (no new assertion logic) - Replace blocking std::thread::sleep with async tokio::time::sleep in the test entry point Rationale: Vector now supports configurable series_api_version (v1/v2) in the datadog_metrics sink. The e2e test previously hardcoded v1 for the vector pipeline. This change runs the same assertions against both API versions via the CI matrix, ensuring correctness is validated for each version independently without duplicating test logic. --------- Co-authored-by: Claude Sonnet 4.5 --- ...g_metrics_series_v2_default.enhancement.md | 5 + lib/vector-stream/src/partitioned_batcher.rs | 21 ++- .../statsd_to_datadog_metrics/experiment.yaml | 10 ++ .../lading/lading.yaml | 16 +++ .../vector/vector.yaml | 25 ++++ src/sinks/aws_cloudwatch_logs/sink.rs | 2 +- src/sinks/azure_common/sink.rs | 4 +- src/sinks/clickhouse/sink.rs | 3 +- src/sinks/datadog/logs/sink.rs | 2 +- src/sinks/datadog/metrics/config.rs | 120 +++++++++++++----- src/sinks/datadog/metrics/encoder.rs | 71 +++++++++++ .../datadog/metrics/integration_tests.rs | 4 +- src/sinks/datadog/metrics/request_builder.rs | 5 +- src/sinks/datadog/metrics/sink.rs | 61 ++++++--- src/sinks/datadog/traces/sink.rs | 4 +- src/sinks/doris/sink.rs | 4 +- src/sinks/gcp_chronicle/sink.rs | 4 +- src/sinks/gcs_common/sink.rs | 4 +- src/sinks/greptimedb/logs/sink.rs | 3 +- src/sinks/http/sink.rs | 8 +- src/sinks/loki/sink.rs | 4 +- src/sinks/opendal_common.rs | 4 +- src/sinks/prometheus/remote_write/sink.rs | 12 +- src/sinks/s3_common/sink.rs | 4 +- src/sinks/splunk_hec/logs/sink.rs | 3 +- src/sinks/splunk_hec/metrics/sink.rs | 4 +- src/sinks/util/builder.rs | 8 +- tests/e2e/datadog-metrics/config/compose.yaml | 1 + tests/e2e/datadog-metrics/config/test.yaml | 2 + tests/e2e/datadog-metrics/data/vector.toml | 1 + tests/e2e/datadog/metrics/series.rs | 37 ++++-- .../sinks/generated/datadog_metrics.cue | 24 ++++ 32 files changed, 382 insertions(+), 98 deletions(-) create mode 100644 changelog.d/datadog_metrics_series_v2_default.enhancement.md create mode 100644 regression/cases/statsd_to_datadog_metrics/experiment.yaml create mode 100644 regression/cases/statsd_to_datadog_metrics/lading/lading.yaml create mode 100644 regression/cases/statsd_to_datadog_metrics/vector/vector.yaml diff --git a/changelog.d/datadog_metrics_series_v2_default.enhancement.md b/changelog.d/datadog_metrics_series_v2_default.enhancement.md new file mode 100644 index 0000000000000..39e857b73f99d --- /dev/null +++ b/changelog.d/datadog_metrics_series_v2_default.enhancement.md @@ -0,0 +1,5 @@ +The `datadog_metrics` sink now defaults to the Datadog series v2 endpoint (`/api/v2/series`) and +exposes a new `series_api_version` configuration option (`v1` or `v2`) to control which endpoint is +used. Set `series_api_version: v1` to fall back to the legacy v1 endpoint if needed. + +authors: vladimir-dd diff --git a/lib/vector-stream/src/partitioned_batcher.rs b/lib/vector-stream/src/partitioned_batcher.rs index 30cf039ec3840..0cd0413548595 100644 --- a/lib/vector-stream/src/partitioned_batcher.rs +++ b/lib/vector-stream/src/partitioned_batcher.rs @@ -211,10 +211,9 @@ where Prt::Key: Eq + Hash + Clone, Prt::Item: ByteSizeOf, C: BatchConfig, - F: Fn() -> C + Send, + F: Fn(&Prt::Key) -> C + Send, { - pub fn new(stream: St, partitioner: Prt, settings: F) -> Self { - let timeout = settings().timeout(); + pub fn new(stream: St, partitioner: Prt, timeout: Duration, settings: F) -> Self { Self { state: settings, batches: HashMap::default(), @@ -233,8 +232,8 @@ where Prt: Partitioner + Unpin, Prt::Key: Eq + Hash + Clone, Prt::Item: ByteSizeOf, - C: BatchConfig, - F: Fn() -> C + Send, + C: BatchConfig, + F: Fn(&Prt::Key) -> C + Send, { pub fn with_timer(stream: St, partitioner: Prt, timer: KT, settings: F) -> Self { Self { @@ -256,7 +255,7 @@ where Prt::Item: ByteSizeOf, KT: KeyedTimer, C: BatchConfig, - F: Fn() -> C + Send, + F: Fn(&Prt::Key) -> C + Send, { type Item = (Prt::Key, B); @@ -307,7 +306,7 @@ where let batch = if let Some(batch) = this.batches.get_mut(&item_key) { batch } else { - let batch = (this.state)(); + let batch = (this.state)(&item_key); this.batches.insert(item_key.clone(), batch); this.timer.insert(item_key.clone()); this.batches @@ -479,7 +478,7 @@ mod test { let batch_settings = BatcherSettings::new(Duration::from_secs(1), allocation_limit, item_limit); let batcher = PartitionedBatcher::with_timer(&mut stream, partitioner, timer, - Box::new(move || batch_settings.as_byte_size_config())); + move |_: &u8| batch_settings.as_byte_size_config::()); let batcher_size_hint = batcher.size_hint(); assert_eq!(stream_size_hint, batcher_size_hint); @@ -503,7 +502,7 @@ mod test { let allocation_limit = NonZeroUsize::new(allocation_limit as usize).unwrap(); let batch_settings = BatcherSettings::new(Duration::from_secs(1), allocation_limit, item_limit); let mut batcher = PartitionedBatcher::with_timer(&mut stream, partitioner, - timer, Box::new(move || batch_settings.as_byte_size_config())); + timer, move |_: &u8| batch_settings.as_byte_size_config::()); let mut batcher = Pin::new(&mut batcher); loop { @@ -574,7 +573,7 @@ mod test { let allocation_limit = NonZeroUsize::new(allocation_limit as usize).unwrap(); let batch_settings = BatcherSettings::new(Duration::from_secs(1), allocation_limit, item_limit); let mut batcher = PartitionedBatcher::with_timer(&mut stream, partitioner, - timer, Box::new(move || batch_settings.as_byte_size_config())); + timer, move |_: &u8| batch_settings.as_byte_size_config::()); let mut batcher = Pin::new(&mut batcher); loop { @@ -618,7 +617,7 @@ mod test { let allocation_limit = NonZeroUsize::new(allocation_limit as usize).unwrap(); let batch_settings = BatcherSettings::new(Duration::from_secs(1), allocation_limit, item_limit); let mut batcher = PartitionedBatcher::with_timer(&mut stream, partitioner, - timer, Box::new(move || batch_settings.as_byte_size_config())); + timer, move |_: &u8| batch_settings.as_byte_size_config::()); let mut batcher = Pin::new(&mut batcher); let mut observed_items = 0; diff --git a/regression/cases/statsd_to_datadog_metrics/experiment.yaml b/regression/cases/statsd_to_datadog_metrics/experiment.yaml new file mode 100644 index 0000000000000..7c0e069a9eddc --- /dev/null +++ b/regression/cases/statsd_to_datadog_metrics/experiment.yaml @@ -0,0 +1,10 @@ +optimization_goal: egress_throughput + +target: + name: vector + command: /usr/bin/vector + cpu_allotment: 6 + memory_allotment: 8GiB + + environment: + VECTOR_THREADS: 4 diff --git a/regression/cases/statsd_to_datadog_metrics/lading/lading.yaml b/regression/cases/statsd_to_datadog_metrics/lading/lading.yaml new file mode 100644 index 0000000000000..3494ed826fd6f --- /dev/null +++ b/regression/cases/statsd_to_datadog_metrics/lading/lading.yaml @@ -0,0 +1,16 @@ +generator: + - tcp: + seed: [2, 3, 5, 7, 11, 13, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131, 137] + addr: "0.0.0.0:8125" + variant: + dogstatsd: {} + bytes_per_second: "500 Mb" + maximum_prebuild_cache_size_bytes: "256 Mb" + +blackhole: + - http: + binding_addr: "0.0.0.0:8080" + +target_metrics: + - prometheus: + uri: "http://127.0.0.1:9090/metrics" diff --git a/regression/cases/statsd_to_datadog_metrics/vector/vector.yaml b/regression/cases/statsd_to_datadog_metrics/vector/vector.yaml new file mode 100644 index 0000000000000..990d4d00c349f --- /dev/null +++ b/regression/cases/statsd_to_datadog_metrics/vector/vector.yaml @@ -0,0 +1,25 @@ +data_dir: "/var/lib/vector" + +sources: + internal_metrics: + type: "internal_metrics" + + statsd: + type: "statsd" + address: "0.0.0.0:8125" + mode: "tcp" + +sinks: + prometheus: + type: "prometheus_exporter" + inputs: ["internal_metrics"] + address: "0.0.0.0:9090" + + datadog_metrics: + type: "datadog_metrics" + inputs: ["statsd"] + endpoint: "http://0.0.0.0:8080" + default_api_key: "DEADBEEF" + default_namespace: "vector" + healthcheck: + enabled: false diff --git a/src/sinks/aws_cloudwatch_logs/sink.rs b/src/sinks/aws_cloudwatch_logs/sink.rs index da272b2fafd28..7b47a5b34ff8f 100644 --- a/src/sinks/aws_cloudwatch_logs/sink.rs +++ b/src/sinks/aws_cloudwatch_logs/sink.rs @@ -49,7 +49,7 @@ where let age_range = start..end; future::ready(age_range.contains(&req.timestamp)) }) - .batched_partitioned(CloudwatchPartitioner, || { + .batched_partitioned(CloudwatchPartitioner, batcher_settings.timeout, |_| { batcher_settings.as_byte_size_config() }) .map(|(key, events)| { diff --git a/src/sinks/azure_common/sink.rs b/src/sinks/azure_common/sink.rs index 1309ae3ba0d66..de567272cf8cd 100644 --- a/src/sinks/azure_common/sink.rs +++ b/src/sinks/azure_common/sink.rs @@ -47,7 +47,9 @@ where let request_builder = self.request_builder; input - .batched_partitioned(partitioner, || settings.as_byte_size_config()) + .batched_partitioned(partitioner, settings.timeout, |_| { + settings.as_byte_size_config() + }) .filter_map(|(key, batch)| async move { // We don't need to emit an error here if the event is dropped since this will occur if the template // couldn't be rendered during the partitioning. A `TemplateRenderingError` is already emitted when diff --git a/src/sinks/clickhouse/sink.rs b/src/sinks/clickhouse/sink.rs index e86f484d89024..ea2f6d434097e 100644 --- a/src/sinks/clickhouse/sink.rs +++ b/src/sinks/clickhouse/sink.rs @@ -43,7 +43,8 @@ where input .batched_partitioned( KeyPartitioner::new(self.database, self.table, self.format), - || batch_settings.as_byte_size_config(), + batch_settings.timeout, + |_| batch_settings.as_byte_size_config(), ) .filter_map(|(key, batch)| async move { key.map(move |k| (k, batch)) }) .request_builder( diff --git a/src/sinks/datadog/logs/sink.rs b/src/sinks/datadog/logs/sink.rs index 862d6973aea5f..0abf4bed1ade5 100644 --- a/src/sinks/datadog/logs/sink.rs +++ b/src/sinks/datadog/logs/sink.rs @@ -386,7 +386,7 @@ where conforms_as_agent: self.conforms_as_agent, }); - let input = input.batched_partitioned(partitioner, || { + let input = input.batched_partitioned(partitioner, batch_settings.timeout, |_| { batch_settings.as_item_size_config(HttpJsonBatchSizer) }); input diff --git a/src/sinks/datadog/metrics/config.rs b/src/sinks/datadog/metrics/config.rs index 6d004e694e0ea..e296bd823d60c 100644 --- a/src/sinks/datadog/metrics/config.rs +++ b/src/sinks/datadog/metrics/config.rs @@ -1,9 +1,9 @@ -use std::sync::OnceLock; - use http::Uri; use snafu::ResultExt; use tower::ServiceBuilder; -use vector_lib::{config::proxy::ProxyConfig, configurable::configurable_component}; +use vector_lib::{ + config::proxy::ProxyConfig, configurable::configurable_component, stream::BatcherSettings, +}; use super::{ request_builder::DatadogMetricsRequestBuilder, @@ -21,18 +21,13 @@ use crate::{ }, tls::{MaybeTlsSettings, TlsEnableableConfig}, }; - #[derive(Clone, Copy, Debug, Default)] pub struct DatadogMetricsDefaultBatchSettings; -// This default is centered around "series" data, which should be the lion's share of what we -// process. Given that a single series, when encoded, is in the 150-300 byte range, we can fit a -// lot of these into a single request, something like 150-200K series. Simply to be a little more -// conservative, though, we use 100K here. This will also get a little more tricky when it comes to -// distributions and sketches, but we're going to have to implement incremental encoding to handle -// "we've exceeded our maximum payload size, split this batch" scenarios anyways. impl SinkBatchSettings for DatadogMetricsDefaultBatchSettings { const MAX_EVENTS: Option = Some(100_000); + // No default byte cap here; the appropriate limit (v1: 60 MiB, v2: 5 MiB) is applied at + // sink build time based on the active series API version. const MAX_BYTES: Option = None; const TIMEOUT_SECS: f64 = 2.0; } @@ -41,9 +36,21 @@ pub(super) const SERIES_V1_PATH: &str = "/api/v1/series"; pub(super) const SERIES_V2_PATH: &str = "/api/v2/series"; pub(super) const SKETCHES_PATH: &str = "/api/beta/sketches"; -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +/// The API version to use when submitting series metrics to Datadog. +#[configurable_component] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +#[serde(rename_all = "snake_case")] pub enum SeriesApiVersion { + /// Use the v1 series endpoint (`/api/v1/series`). + /// + /// This is a legacy endpoint. Prefer `v2` unless you have a specific reason to use v1. + #[configurable(deprecated)] V1, + + /// Use the v2 series endpoint (`/api/v2/series`). + /// + /// This is the recommended and default endpoint. + #[default] V2, } @@ -54,15 +61,6 @@ impl SeriesApiVersion { Self::V2 => SERIES_V2_PATH, } } - fn get_api_version() -> Self { - static API_VERSION: OnceLock = OnceLock::new(); - *API_VERSION.get_or_init(|| { - match std::env::var("VECTOR_TEMP_USE_DD_METRICS_SERIES_V2_API") { - Ok(_) => Self::V2, - Err(_) => Self::V1, - } - }) - } } /// Various metric type-specific API types. @@ -94,14 +92,8 @@ impl DatadogMetricsEndpoint { matches!(self, Self::Series { .. }) } - // Creates an instance of the `Series` variant with the default API version. - pub fn series() -> Self { - Self::Series(SeriesApiVersion::get_api_version()) - } - pub(super) const fn payload_limits(self) -> DatadogMetricsPayloadLimits { // from https://docs.datadoghq.com/api/latest/metrics/#submit-metrics - let (uncompressed, compressed) = match self { // Sketches use the same payload size limits as v1 series DatadogMetricsEndpoint::Series(SeriesApiVersion::V1) @@ -162,6 +154,13 @@ pub struct DatadogMetricsConfig { #[serde(default)] pub default_namespace: Option, + /// Controls which Datadog series API endpoint is used to submit metrics. + /// + /// Defaults to `v2` (`/api/v2/series`). Set to `v1` (`/api/v1/series`) only if you need to + /// fall back to the legacy endpoint. + #[serde(default)] + pub series_api_version: SeriesApiVersion, + #[configurable(derived)] #[serde(default)] pub batch: BatchConfig, @@ -222,7 +221,7 @@ impl DatadogMetricsConfig { ) -> crate::Result { let base_uri = self.get_base_agent_endpoint(dd_common); - let series_endpoint = build_uri(&base_uri, SeriesApiVersion::get_api_version().get_path())?; + let series_endpoint = build_uri(&base_uri, self.series_api_version.get_path())?; let sketches_endpoint = build_uri(&base_uri, SKETCHES_PATH)?; Ok(DatadogMetricsEndpointConfiguration::new( @@ -253,7 +252,8 @@ impl DatadogMetricsConfig { dd_common: &DatadogCommonConfig, client: HttpClient, ) -> crate::Result { - let batcher_settings = self.batch.into_batcher_settings()?; + let (batcher_settings, sketches_batcher_settings) = + resolve_endpoint_batch_settings(self.batch, self.series_api_version)?; // TODO: revisit our concurrency and batching defaults let request_limits = self.request.into_settings(); @@ -269,10 +269,18 @@ impl DatadogMetricsConfig { let request_builder = DatadogMetricsRequestBuilder::new( endpoint_configuration, self.default_namespace.clone(), + self.series_api_version, )?; let protocol = self.get_protocol(dd_common); - let sink = DatadogMetricsSink::new(service, request_builder, batcher_settings, protocol); + let sink = DatadogMetricsSink::new( + service, + request_builder, + batcher_settings, + sketches_batcher_settings, + protocol, + self.series_api_version, + ); Ok(VectorSink::from_event_streamsink(sink)) } @@ -287,6 +295,28 @@ impl DatadogMetricsConfig { } } +/// Returns `(series_settings, sketches_settings)`. +/// +/// When the user has not set an explicit `max_bytes`, each endpoint is capped to its own +/// uncompressed payload limit (5 MiB for Series v2, 60 MiB for Sketches). When an explicit +/// limit is configured, both endpoints share it. +fn resolve_endpoint_batch_settings( + batch: BatchConfig, + series_version: SeriesApiVersion, +) -> crate::Result<(BatcherSettings, BatcherSettings)> { + let mut series = batch.into_batcher_settings()?; + let mut sketches = series; + if series.size_limit == usize::MAX { + series.size_limit = DatadogMetricsEndpoint::Series(series_version) + .payload_limits() + .uncompressed; + sketches.size_limit = DatadogMetricsEndpoint::Sketches + .payload_limits() + .uncompressed; + } + Ok((series, sketches)) +} + fn build_uri(host: &str, endpoint: &str) -> crate::Result { let result = format!("{host}{endpoint}") .parse::() @@ -302,4 +332,36 @@ mod tests { fn generate_config() { crate::test_util::test_generate_config::(); } + + // When max_bytes is unset, each endpoint gets its own API payload limit. + #[test] + fn default_batch_config_uses_endpoint_specific_size_limits() { + let (series, sketches) = + resolve_endpoint_batch_settings(BatchConfig::default(), SeriesApiVersion::V2).unwrap(); + + assert_eq!(series.size_limit, 5_242_880); // 5 MiB — Series v2 limit + assert_eq!(sketches.size_limit, 62_914_560); // 60 MiB — Sketches limit + } + + #[test] + fn v1_batch_config_uses_v1_size_limit() { + let (series, sketches) = + resolve_endpoint_batch_settings(BatchConfig::default(), SeriesApiVersion::V1).unwrap(); + + assert_eq!(series.size_limit, 62_914_560); // 60 MiB — Series v1 limit + assert_eq!(sketches.size_limit, 62_914_560); // 60 MiB — Sketches limit + } + + // When the user sets max_bytes, both endpoints share that limit unchanged. + #[test] + fn explicit_max_bytes_applies_to_both_endpoints() { + let mut config = BatchConfig::::default(); + config.max_bytes = Some(1_000_000); + + let (series, sketches) = + resolve_endpoint_batch_settings(config, SeriesApiVersion::V2).unwrap(); + + assert_eq!(series.size_limit, 1_000_000); + assert_eq!(sketches.size_limit, 1_000_000); + } } diff --git a/src/sinks/datadog/metrics/encoder.rs b/src/sinks/datadog/metrics/encoder.rs index 617e69c1e5e6b..9f0b71c87ba98 100644 --- a/src/sinks/datadog/metrics/encoder.rs +++ b/src/sinks/datadog/metrics/encoder.rs @@ -1582,6 +1582,77 @@ mod tests { assert_eq!(result, Some((usize::MAX, compression_overhead_len + 1))); } + #[test] + fn default_payload_limits_are_endpoint_aware() { + let v1 = DatadogMetricsEndpoint::Series(SeriesApiVersion::V1).payload_limits(); + assert_eq!(v1.uncompressed, 62_914_560); + assert_eq!(v1.compressed, 3_200_000); + + let v2 = DatadogMetricsEndpoint::Series(SeriesApiVersion::V2).payload_limits(); + assert_eq!(v2.uncompressed, 5_242_880); + assert_eq!(v2.compressed, 512_000); + + let sketches = DatadogMetricsEndpoint::Sketches.payload_limits(); + assert_eq!(sketches.uncompressed, 62_914_560); + assert_eq!(sketches.compressed, 3_200_000); + } + + #[test] + fn v2_series_default_limits_split_large_batches() { + // Simulate a large send and validate that default V2 limits split payloads into multiple + // requests, while still making forward progress each pass. + let mut pending = vec![get_simple_counter(); 120_000]; + let mut encoded_batches = 0; + let mut encoded_metrics = 0; + + while !pending.is_empty() { + let mut encoder = DatadogMetricsEncoder::new( + DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), + None, + ) + .expect("default payload size limits should be valid"); + + let mut next_pending = Vec::new(); + let mut hit_limit = false; + for metric in pending.drain(..) { + match encoder.try_encode(metric.clone()) { + Ok(None) => {} + Ok(Some(returned_metric)) => { + hit_limit = true; + next_pending.push(returned_metric); + } + Err(error) => panic!("unexpected encoding error: {error}"), + } + } + + let finish_result = encoder.finish(); + assert!(finish_result.is_ok()); + let (_payload, processed) = finish_result.unwrap(); + assert!( + !processed.is_empty(), + "encoder should always make progress for a non-empty batch" + ); + + encoded_metrics += processed.len(); + encoded_batches += 1; + + if hit_limit { + assert!( + !next_pending.is_empty(), + "hitting limits should leave metrics to process in the next batch" + ); + } + + pending = next_pending; + } + + assert_eq!(encoded_metrics, 120_000); + assert!( + encoded_batches > 1, + "expected multiple batches for V2 default limits" + ); + } + #[test] fn encode_series_breaks_out_when_limit_reached_uncompressed() { // We manually create the encoder with an arbitrarily low "uncompressed" limit but high diff --git a/src/sinks/datadog/metrics/integration_tests.rs b/src/sinks/datadog/metrics/integration_tests.rs index e20ef106f7ee2..e63d4c35e73c9 100644 --- a/src/sinks/datadog/metrics/integration_tests.rs +++ b/src/sinks/datadog/metrics/integration_tests.rs @@ -321,7 +321,9 @@ fn validate_protobuf_set_gauge_rate(request: &(Parts, Bytes)) { ddmetric_proto::metric_payload::MetricType::Gauge ); assert_eq!(gauge.points[0].value, 5678.0); - assert_eq!(gauge.interval, 10); + // interval_ms is dropped during normalization (incremental → absolute conversion + // via into_absolute() sets interval_ms: None), so interval is always 0 for gauges. + assert_eq!(gauge.interval, 0); } // validate counter w interval = rate diff --git a/src/sinks/datadog/metrics/request_builder.rs b/src/sinks/datadog/metrics/request_builder.rs index 32f3a7e2e5ae5..3b1c2f4607d39 100644 --- a/src/sinks/datadog/metrics/request_builder.rs +++ b/src/sinks/datadog/metrics/request_builder.rs @@ -8,7 +8,7 @@ use vector_lib::{ }; use super::{ - config::{DatadogMetricsEndpoint, DatadogMetricsEndpointConfiguration}, + config::{DatadogMetricsEndpoint, DatadogMetricsEndpointConfiguration, SeriesApiVersion}, encoder::{CreateError, DatadogMetricsEncoder, EncoderError, FinishError}, service::DatadogMetricsRequest, }; @@ -80,11 +80,12 @@ impl DatadogMetricsRequestBuilder { pub fn new( endpoint_configuration: DatadogMetricsEndpointConfiguration, default_namespace: Option, + series_api_version: SeriesApiVersion, ) -> Result { Ok(Self { endpoint_configuration, series_encoder: DatadogMetricsEncoder::new( - DatadogMetricsEndpoint::series(), + DatadogMetricsEndpoint::Series(series_api_version), default_namespace.clone(), )?, sketches_encoder: DatadogMetricsEncoder::new( diff --git a/src/sinks/datadog/metrics/sink.rs b/src/sinks/datadog/metrics/sink.rs index 231aca75485ee..1cd075a831d9c 100644 --- a/src/sinks/datadog/metrics/sink.rs +++ b/src/sinks/datadog/metrics/sink.rs @@ -16,8 +16,10 @@ use vector_lib::{ }; use super::{ - config::DatadogMetricsEndpoint, normalizer::DatadogMetricsNormalizer, - request_builder::DatadogMetricsRequestBuilder, service::DatadogMetricsRequest, + config::{DatadogMetricsEndpoint, SeriesApiVersion}, + normalizer::DatadogMetricsNormalizer, + request_builder::DatadogMetricsRequestBuilder, + service::DatadogMetricsRequest, }; use crate::{ internal_events::DatadogMetricsEncodingError, @@ -33,21 +35,24 @@ use crate::{ /// Generally speaking, all "basic" metrics -- counter, gauge, set, aggregated summary-- are sent to /// the Series API, while distributions, aggregated histograms, and sketches (hehe) are sent to the /// Sketches API. -struct DatadogMetricsTypePartitioner; +struct DatadogMetricsTypePartitioner { + series_api_version: SeriesApiVersion, +} impl Partitioner for DatadogMetricsTypePartitioner { type Item = Metric; type Key = (Option>, DatadogMetricsEndpoint); fn partition(&self, item: &Self::Item) -> Self::Key { + let series = DatadogMetricsEndpoint::Series(self.series_api_version); let endpoint = match item.data().value() { - MetricValue::Counter { .. } => DatadogMetricsEndpoint::series(), - MetricValue::Gauge { .. } => DatadogMetricsEndpoint::series(), - MetricValue::Set { .. } => DatadogMetricsEndpoint::series(), + MetricValue::Counter { .. } => series, + MetricValue::Gauge { .. } => series, + MetricValue::Set { .. } => series, MetricValue::Distribution { .. } => DatadogMetricsEndpoint::Sketches, MetricValue::AggregatedHistogram { .. } => DatadogMetricsEndpoint::Sketches, // NOTE: AggregatedSummary will be split into counters and gauges during normalization - MetricValue::AggregatedSummary { .. } => DatadogMetricsEndpoint::series(), + MetricValue::AggregatedSummary { .. } => series, MetricValue::Sketch { .. } => DatadogMetricsEndpoint::Sketches, }; (item.metadata().datadog_api_key(), endpoint) @@ -57,8 +62,10 @@ impl Partitioner for DatadogMetricsTypePartitioner { pub(crate) struct DatadogMetricsSink { service: S, request_builder: DatadogMetricsRequestBuilder, - batch_settings: BatcherSettings, + series_batch_settings: BatcherSettings, + sketches_batch_settings: BatcherSettings, protocol: String, + series_api_version: SeriesApiVersion, } impl DatadogMetricsSink @@ -72,20 +79,28 @@ where pub const fn new( service: S, request_builder: DatadogMetricsRequestBuilder, - batch_settings: BatcherSettings, + series_batch_settings: BatcherSettings, + sketches_batch_settings: BatcherSettings, protocol: String, + series_api_version: SeriesApiVersion, ) -> Self { DatadogMetricsSink { service, request_builder, - batch_settings, + series_batch_settings, + sketches_batch_settings, protocol, + series_api_version, } } async fn run_inner(self: Box, input: BoxStream<'_, Event>) -> Result<(), ()> { let mut splitter: MetricSplitter = MetricSplitter::default(); - let batch_settings = self.batch_settings; + let series_batch_settings = self.series_batch_settings; + let sketches_batch_settings = self.sketches_batch_settings; + let partitioner = DatadogMetricsTypePartitioner { + series_api_version: self.series_api_version, + }; input // Convert `Event` to `Metric` so we don't have to deal with constant conversions. @@ -99,10 +114,20 @@ where // what quantiles to generate, etc. .normalized_with_default::() // We batch metrics by their endpoint: series endpoint for counters, gauge, and sets vs sketch endpoint for - // distributions, aggregated histograms, and sketches. - .batched_partitioned(DatadogMetricsTypePartitioner, || { - batch_settings.as_byte_size_config() - }) + // distributions, aggregated histograms, and sketches. Each endpoint uses its own byte size + // limit: 5 MiB for Series v2 and 60 MiB for Sketches. + .batched_partitioned( + partitioner, + series_batch_settings.timeout, + |(_api_key, endpoint)| match endpoint { + DatadogMetricsEndpoint::Series(_) => { + series_batch_settings.as_byte_size_config() + } + DatadogMetricsEndpoint::Sketches => { + sketches_batch_settings.as_byte_size_config() + } + }, + ) // Aggregate counters with identical timestamps, otherwise identical counters (same // series and same timestamp, when rounded to whole seconds) will be dropped in a // last-write-wins situation when they hit the DD metrics intake. @@ -233,10 +258,8 @@ mod tests { use chrono::{DateTime, Utc}; use proptest::prelude::*; - use vector_lib::{ - event::{Metric, MetricKind, MetricValue}, - metric_tags, - }; + use vector_lib::event::{Metric, MetricKind, MetricValue}; + use vector_lib::metric_tags; use super::sort_and_collapse_counters_by_series_and_timestamp; diff --git a/src/sinks/datadog/traces/sink.rs b/src/sinks/datadog/traces/sink.rs index 8f48ab0acc63b..77f931fb931ab 100644 --- a/src/sinks/datadog/traces/sink.rs +++ b/src/sinks/datadog/traces/sink.rs @@ -108,7 +108,9 @@ where let batch_settings = self.batch_settings; input - .batched_partitioned(EventPartitioner, || batch_settings.as_byte_size_config()) + .batched_partitioned(EventPartitioner, batch_settings.timeout, |_| { + batch_settings.as_byte_size_config() + }) .incremental_request_builder(self.request_builder) .flat_map(stream::iter) .filter_map(|request| async move { diff --git a/src/sinks/doris/sink.rs b/src/sinks/doris/sink.rs index ac9a77f09742e..c87e76557c653 100644 --- a/src/sinks/doris/sink.rs +++ b/src/sinks/doris/sink.rs @@ -34,7 +34,9 @@ where let batch_settings = self.batch_settings; let key_partitioner = DorisKeyPartitioner::new(self.database, self.table); input - .batched_partitioned(key_partitioner, || batch_settings.as_byte_size_config()) + .batched_partitioned(key_partitioner, batch_settings.timeout, |_| { + batch_settings.as_byte_size_config() + }) .filter_map(|(key, batch)| async move { key.map(move |k| (k, batch)) }) .request_builder( default_request_builder_concurrency_limit(), diff --git a/src/sinks/gcp_chronicle/sink.rs b/src/sinks/gcp_chronicle/sink.rs index 8f37872ee4f72..bd98487b2ed96 100644 --- a/src/sinks/gcp_chronicle/sink.rs +++ b/src/sinks/gcp_chronicle/sink.rs @@ -48,7 +48,9 @@ where let request_builder = self.request_builder; input - .batched_partitioned(partitioner, || settings.as_byte_size_config()) + .batched_partitioned(partitioner, settings.timeout, |_| { + settings.as_byte_size_config() + }) .filter_map(|(key, batch)| async move { // A `TemplateRenderingError` will have been emitted by `KeyPartitioner` if the key here is `None`, // thus no further `EventsDropped` event needs emitting at this stage. diff --git a/src/sinks/gcs_common/sink.rs b/src/sinks/gcs_common/sink.rs index c545aaa6b196b..b2e4010ba301a 100644 --- a/src/sinks/gcs_common/sink.rs +++ b/src/sinks/gcs_common/sink.rs @@ -50,7 +50,9 @@ where let request_builder = self.request_builder; input - .batched_partitioned(partitioner, || settings.as_byte_size_config()) + .batched_partitioned(partitioner, settings.timeout, |_| { + settings.as_byte_size_config() + }) .filter_map(|(key, batch)| async move { // A `TemplateRenderingError` will have been emitted by `KeyPartitioner` if the key here is `None`, // thus no further `EventsDropped` event needs emitting at this stage. diff --git a/src/sinks/greptimedb/logs/sink.rs b/src/sinks/greptimedb/logs/sink.rs index c61ce3d76dfc3..a2b5b68acdb81 100644 --- a/src/sinks/greptimedb/logs/sink.rs +++ b/src/sinks/greptimedb/logs/sink.rs @@ -52,7 +52,8 @@ where self.logs_sink_setting.pipeline_name, self.logs_sink_setting.pipeline_version, ), - || batcher_settings.as_byte_size_config(), + batcher_settings.timeout, + |_| batcher_settings.as_byte_size_config(), ) .filter_map(|(key, batch)| async move { key.map(move |k| (k, batch)) }) .request_builder( diff --git a/src/sinks/http/sink.rs b/src/sinks/http/sink.rs index 982a6515aac93..0c9a14d269daf 100644 --- a/src/sinks/http/sink.rs +++ b/src/sinks/http/sink.rs @@ -43,9 +43,11 @@ where }; input // Batch the input stream with size calculation based on the configured codec - .batched_partitioned(KeyPartitioner::new(self.uri, self.headers), || { - self.batch_settings.as_item_size_config(batch_sizer.clone()) - }) + .batched_partitioned( + KeyPartitioner::new(self.uri, self.headers), + self.batch_settings.timeout, + |_| self.batch_settings.as_item_size_config(batch_sizer.clone()), + ) .filter_map(|(key, batch)| async move { key.map(move |k| (k, batch)) }) // Build requests with default concurrency limit. .request_builder( diff --git a/src/sinks/loki/sink.rs b/src/sinks/loki/sink.rs index 55a74dcbcfc0f..1bff5845f3618 100644 --- a/src/sinks/loki/sink.rs +++ b/src/sinks/loki/sink.rs @@ -513,7 +513,9 @@ impl LokiSink { .map(|event| encoder.encode_event(event)) .filter_map(|event| async { event }) .map(|record| filter.filter_record(record)) - .batched_partitioned(RecordPartitioner, || batch_settings.as_byte_size_config()) + .batched_partitioned(RecordPartitioner, batch_settings.timeout, |_| { + batch_settings.as_byte_size_config() + }) .filter_map(|(partition, batch)| async { if let Some(partition) = partition { let mut count: usize = 0; diff --git a/src/sinks/opendal_common.rs b/src/sinks/opendal_common.rs index 4396002ae64b3..53e23e33ae8cc 100644 --- a/src/sinks/opendal_common.rs +++ b/src/sinks/opendal_common.rs @@ -78,7 +78,9 @@ where let request_builder = self.request_builder; input - .batched_partitioned(partitioner, || settings.as_byte_size_config()) + .batched_partitioned(partitioner, settings.timeout, |_| { + settings.as_byte_size_config() + }) .filter_map(|(key, batch)| async move { // We don't need to emit an error here if the event is dropped since this will occur if the template // couldn't be rendered during the partitioning. A `TemplateRenderingError` is already emitted when diff --git a/src/sinks/prometheus/remote_write/sink.rs b/src/sinks/prometheus/remote_write/sink.rs index 1b5f82ad76916..858f535a0d98e 100644 --- a/src/sinks/prometheus/remote_write/sink.rs +++ b/src/sinks/prometheus/remote_write/sink.rs @@ -182,10 +182,14 @@ where .filter_map(move |event| { future::ready(make_remote_write_event(tenant_id.as_ref(), event)) }) - .batched_partitioned(PrometheusTenantIdPartitioner, || { - batch_settings - .as_reducer_config(ByteSizeOfItemSize, EventCollection::new(self.aggregate)) - }) + .batched_partitioned( + PrometheusTenantIdPartitioner, + batch_settings.timeout, + |_| { + batch_settings + .as_reducer_config(ByteSizeOfItemSize, EventCollection::new(self.aggregate)) + }, + ) .request_builder(default_request_builder_concurrency_limit(), request_builder) .filter_map(|request| async move { match request { diff --git a/src/sinks/s3_common/sink.rs b/src/sinks/s3_common/sink.rs index 729a67153f641..5cad7ba89b6ee 100644 --- a/src/sinks/s3_common/sink.rs +++ b/src/sinks/s3_common/sink.rs @@ -47,7 +47,9 @@ where let request_builder = self.request_builder; input - .batched_partitioned(partitioner, || settings.as_byte_size_config()) + .batched_partitioned(partitioner, settings.timeout, |_| { + settings.as_byte_size_config() + }) .filter_map(|(key, batch)| async move { key.map(move |k| (k, batch)) }) .request_builder(default_request_builder_concurrency_limit(), request_builder) .filter_map(|request| async move { diff --git a/src/sinks/splunk_hec/logs/sink.rs b/src/sinks/splunk_hec/logs/sink.rs index fb6742586130e..d80c44838c915 100644 --- a/src/sinks/splunk_hec/logs/sink.rs +++ b/src/sinks/splunk_hec/logs/sink.rs @@ -86,7 +86,8 @@ where } else { EventPartitioner::new(None, None, None, None) }, - || batch_settings.as_byte_size_config(), + batch_settings.timeout, + |_| batch_settings.as_byte_size_config(), ) .request_builder( default_request_builder_concurrency_limit(), diff --git a/src/sinks/splunk_hec/metrics/sink.rs b/src/sinks/splunk_hec/metrics/sink.rs index bb61a4ef1ecfb..57e05cc3953a4 100644 --- a/src/sinks/splunk_hec/metrics/sink.rs +++ b/src/sinks/splunk_hec/metrics/sink.rs @@ -53,7 +53,9 @@ where default_namespace, )) }) - .batched_partitioned(EventPartitioner, || batch_settings.as_byte_size_config()) + .batched_partitioned(EventPartitioner, batch_settings.timeout, |_| { + batch_settings.as_byte_size_config() + }) .request_builder( default_request_builder_concurrency_limit(), self.request_builder, diff --git a/src/sinks/util/builder.rs b/src/sinks/util/builder.rs index 6d2b2a68753f5..725e20d974782 100644 --- a/src/sinks/util/builder.rs +++ b/src/sinks/util/builder.rs @@ -46,9 +46,13 @@ pub trait SinkBuilderExt: Stream { /// The stream will yield batches of events, with their partition key, when either a batch fills /// up or times out. [`Partitioner`] operates on a per-event basis, and has access to the event /// itself, and so can access any and all fields of an event. + /// + /// The `settings` closure receives the partition key for each new partition, allowing callers + /// to vary the batch configuration (e.g. byte size limit) per partition. fn batched_partitioned( self, partitioner: P, + timeout: Duration, settings: F, ) -> PartitionedBatcher, C, F, B> where @@ -57,9 +61,9 @@ pub trait SinkBuilderExt: Stream { P::Key: Eq + Hash + Clone, P::Item: ByteSizeOf, C: BatchConfig, - F: Fn() -> C + Send, + F: Fn(&P::Key) -> C + Send, { - PartitionedBatcher::new(self, partitioner, settings) + PartitionedBatcher::new(self, partitioner, timeout, settings) } /// Batches the stream based on the given batch settings and item size calculator. diff --git a/tests/e2e/datadog-metrics/config/compose.yaml b/tests/e2e/datadog-metrics/config/compose.yaml index d1beb57030c42..84c0f89278863 100644 --- a/tests/e2e/datadog-metrics/config/compose.yaml +++ b/tests/e2e/datadog-metrics/config/compose.yaml @@ -52,6 +52,7 @@ services: image: ${CONFIG_VECTOR_IMAGE} environment: - FEATURES=e2e-tests-datadog + - CONFIG_SERIES_API_VERSION working_dir: /home/vector command: - "/usr/bin/vector" diff --git a/tests/e2e/datadog-metrics/config/test.yaml b/tests/e2e/datadog-metrics/config/test.yaml index 0199618fe7f42..2b551be47c147 100644 --- a/tests/e2e/datadog-metrics/config/test.yaml +++ b/tests/e2e/datadog-metrics/config/test.yaml @@ -14,6 +14,8 @@ runner: matrix: # validate against the Agent latest nightly and also stable v6 and v7 agent_version: ['latest', '6', '7'] + # validate both series API versions + series_api_version: ['v1', 'v2'] # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch diff --git a/tests/e2e/datadog-metrics/data/vector.toml b/tests/e2e/datadog-metrics/data/vector.toml index c99e2f60ab9ba..c689e5c5ff2ed 100644 --- a/tests/e2e/datadog-metrics/data/vector.toml +++ b/tests/e2e/datadog-metrics/data/vector.toml @@ -14,3 +14,4 @@ type = "datadog_metrics" default_api_key = "unused" endpoint = "http://fakeintake-vector:80" batch.timeout_secs = 1 +series_api_version = "${CONFIG_SERIES_API_VERSION}" diff --git a/tests/e2e/datadog/metrics/series.rs b/tests/e2e/datadog/metrics/series.rs index 6b53c9282b01d..2c4e20a7ab269 100644 --- a/tests/e2e/datadog/metrics/series.rs +++ b/tests/e2e/datadog/metrics/series.rs @@ -218,19 +218,7 @@ async fn get_v2_series_from_pipeline(address: String) -> SeriesIntake { intake } -pub(super) async fn validate() { - info!("==== getting series data from agent-only pipeline ==== "); - let agent_intake = get_v2_series_from_pipeline(fake_intake_agent_address()).await; - - info!("==== getting series data from agent-vector pipeline ===="); - let vector_intake = get_v1_series_from_pipeline(fake_intake_vector_address()).await; - - assert_eq!( - agent_intake.len(), - vector_intake.len(), - "different number of unique Series contexts received" - ); - +fn compare_intakes(agent_intake: &SeriesIntake, vector_intake: &SeriesIntake) { // The assertions we make below can be summarized as follows: // - For each metric type, we have a different set of assertions which are relevant to // the expectations for that metric type's behavior. For example, gauges are a last @@ -249,6 +237,12 @@ pub(super) async fn validate() { // used in the test case is not introducing inconsistencies in the data flowing between the // Agent and the Datadog backend. + assert_eq!( + agent_intake.len(), + vector_intake.len(), + "different number of unique Series contexts received" + ); + agent_intake .iter() .zip(vector_intake.iter()) @@ -302,3 +296,20 @@ pub(super) async fn validate() { } }); } + +pub(super) async fn validate() { + let api_version = std::env::var("CONFIG_SERIES_API_VERSION") + .expect("CONFIG_SERIES_API_VERSION must be set (e.g. 'v1' or 'v2')"); + + info!("==== getting series data from agent-only pipeline ==== "); + let agent_intake = get_v2_series_from_pipeline(fake_intake_agent_address()).await; + + info!("==== getting series data from agent-vector pipeline ({api_version}) ===="); + let vector_intake = match api_version.as_str() { + "v1" => get_v1_series_from_pipeline(fake_intake_vector_address()).await, + "v2" => get_v2_series_from_pipeline(fake_intake_vector_address()).await, + v => panic!("Unknown CONFIG_SERIES_API_VERSION: {v}"), + }; + + compare_intakes(&agent_intake, &vector_intake); +} diff --git a/website/cue/reference/components/sinks/generated/datadog_metrics.cue b/website/cue/reference/components/sinks/generated/datadog_metrics.cue index 564c8c2422e3b..6d7ebc4378dd6 100644 --- a/website/cue/reference/components/sinks/generated/datadog_metrics.cue +++ b/website/cue/reference/components/sinks/generated/datadog_metrics.cue @@ -284,6 +284,30 @@ generated: components: sinks: datadog_metrics: configuration: { } } } + series_api_version: { + description: """ + Controls which Datadog series API endpoint is used to submit metrics. + + Defaults to `v2` (`/api/v2/series`). Set to `v1` (`/api/v1/series`) only if you need to + fall back to the legacy endpoint. + """ + required: false + type: string: { + default: "v2" + enum: { + v1: """ + Use the v1 series endpoint (`/api/v1/series`). + + This is a legacy endpoint. Prefer `v2` unless you have a specific reason to use v1. + """ + v2: """ + Use the v2 series endpoint (`/api/v2/series`). + + This is the recommended and default endpoint. + """ + } + } + } site: { description: """ The Datadog [site][dd_site] to send observability data to. From 1444b2bac009aa51343820dba7a36ce119dce56f Mon Sep 17 00:00:00 2001 From: zapdos26 Date: Tue, 17 Mar 2026 14:44:09 -0400 Subject: [PATCH 018/326] chore(deps): Bump rdkafka from 0.38.0 to 0.39.0 (#24602) * Update rdkafka to 0.39.0 * Update 3rd party license CSV * Update licenses * Add libsasl2 MacOS intructions * Fix spellcheck * Fix markdown * Improve docs --------- Co-authored-by: Thomas Co-authored-by: Pavlos Rontidis --- .github/actions/spelling/allow.txt | 4 ++++ Cargo.lock | 8 ++++---- Cargo.toml | 2 +- docs/DEVELOPING.md | 13 +++++++++++++ 4 files changed, 22 insertions(+), 5 deletions(-) diff --git a/.github/actions/spelling/allow.txt b/.github/actions/spelling/allow.txt index 2672ed4c357b7..2e594047a89a2 100644 --- a/.github/actions/spelling/allow.txt +++ b/.github/actions/spelling/allow.txt @@ -562,6 +562,10 @@ zstandard ZTE Zync sighup +CPPFLAGS +LDFLAGS +libsasl +pkgconfig CLAUDE linting lexers diff --git a/Cargo.lock b/Cargo.lock index 08becf0e3552c..3db4f718a1aaf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8930,9 +8930,9 @@ dependencies = [ [[package]] name = "rdkafka" -version = "0.38.0" +version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f1856d72dbbbea0d2a5b2eaf6af7fb3847ef2746e883b11781446a51dbc85c0" +checksum = "d7956f9ac12b5712e50372d9749a3102f4810a8d42481c5eae3748d36d585bcf" dependencies = [ "futures-channel", "futures-util", @@ -8948,9 +8948,9 @@ dependencies = [ [[package]] name = "rdkafka-sys" -version = "4.9.0+2.10.0" +version = "4.10.0+2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5230dca48bc354d718269f3e4353280e188b610f7af7e2fcf54b7a79d5802872" +checksum = "e234cf318915c1059d4921ef7f75616b5219b10b46e9f3a511a15eb4b56a3f77" dependencies = [ "cmake", "curl-sys", diff --git a/Cargo.toml b/Cargo.toml index 86f06951af994..b4565050babe5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -419,7 +419,7 @@ pulsar = { version = "6.7.0", default-features = false, features = ["tokio-runti quick-junit = { version = "0.5.1" } rand.workspace = true rand_distr.workspace = true -rdkafka = { version = "0.38.0", default-features = false, features = ["curl-static", "tokio", "libz", "ssl", "zstd"], optional = true } +rdkafka = { version = "0.39.0", default-features = false, features = ["curl-static", "tokio", "libz", "ssl", "zstd"], optional = true } redis = { version = "0.32.4", default-features = false, features = ["connection-manager", "sentinel", "tokio-comp", "tokio-native-tls-comp"], optional = true } regex.workspace = true roaring = { version = "0.11.2", default-features = false, features = ["std"], optional = true } diff --git a/docs/DEVELOPING.md b/docs/DEVELOPING.md index d286c9feca343..63633d2bdac7b 100644 --- a/docs/DEVELOPING.md +++ b/docs/DEVELOPING.md @@ -123,6 +123,19 @@ To build Vector on your own host will require a fairly complete development envi Loosely, you'll need the following: - **To build Vector:** Have working Rustup, Protobuf tools, C++/C build tools (LLVM, GCC, or MSVC), Python, and Perl, `make` (the GNU one preferably), `bash`, `cmake`, `GNU coreutils`, and `autotools`. + - You may also need to install `libsasl2` if you are compiling with default features or with any `kafka`-related features. + - Installing libsasl2 on Ubuntu + - Run `sudo apt-get install -y libsasl2-dev` + - Installing libsasl2 on MacOS + - Run `brew install cyrus-sasl` + - You will need to set the following environment variables: + + ```bash + export LDFLAGS="-L$(brew --prefix cyrus-sasl)/lib $LDFLAGS" + export CPPFLAGS="-I$(brew --prefix cyrus-sasl)/include $CPPFLAGS" + export PKG_CONFIG_PATH="$(brew --prefix cyrus-sasl)/lib/pkgconfig:$PKG_CONFIG_PATH" + ``` + - **To run `make test`:** Install [`cargo-nextest`](https://nexte.st/) - **To run integration tests:** Have `docker` available, or a real live version of that service. (Use `AUTOSPAWN=false`) - **To run `make check-component-features`:** Have `remarshal` installed. From 089db6775500eb6cbdfa5f4b7e6069185a8d61c5 Mon Sep 17 00:00:00 2001 From: Naoki Aoyama <9667078+naa0yama@users.noreply.github.com> Date: Wed, 18 Mar 2026 03:45:28 +0900 Subject: [PATCH 019/326] feat(enrichment tables): add `network` CIDR field to lookup results (#24576) * feat(geoip enrichment table): add network CIDR field to lookup results (#24411) * chore: add changelog fragment for #24411 * docs(geoip enrichment table): add network and Anonymous-IP fields to ENRICHMENT_TABLE_EXPLAINER (#24411) --------- Co-authored-by: Pavlos Rontidis --- .../24411_geoip_network_field.enhancement.md | 3 ++ .../find_enrichment_table_records.json | 2 +- .../get_enrichment_table_record.json | 2 +- lib/vector-vrl/enrichment/src/lib.rs | 12 ++++++++ src/enrichment_tables/geoip.rs | 29 +++++++++++++++---- 5 files changed, 40 insertions(+), 8 deletions(-) create mode 100644 changelog.d/24411_geoip_network_field.enhancement.md diff --git a/changelog.d/24411_geoip_network_field.enhancement.md b/changelog.d/24411_geoip_network_field.enhancement.md new file mode 100644 index 0000000000000..65d33f50f6b1b --- /dev/null +++ b/changelog.d/24411_geoip_network_field.enhancement.md @@ -0,0 +1,3 @@ +The `geoip` enrichment table now includes a `network` field containing the CIDR network associated with the lookup result, available for all database types (City, ISP/ASN, Connection-Type, Anonymous-IP). + +authors: naa0yama diff --git a/docs/generated/find_enrichment_table_records.json b/docs/generated/find_enrichment_table_records.json index 105c16d7ca3bd..52840357ad746 100644 --- a/docs/generated/find_enrichment_table_records.json +++ b/docs/generated/find_enrichment_table_records.json @@ -2,7 +2,7 @@ "anchor": "find_enrichment_table_records", "name": "find_enrichment_table_records", "category": "Enrichment", - "description": "Searches an [enrichment table](/docs/reference/glossary/#enrichment-tables) for rows that match the provided condition.\n\nFor `file` enrichment tables, this condition needs to be a VRL object in which\nthe key-value pairs indicate a field to search mapped to a value to search in that field.\nThis function returns the rows that match the provided condition(s). _All_ fields need to\nmatch for rows to be returned; if any fields do not match, then no rows are returned.\n\nThere are three forms of search criteria:\n\n1. **Exact match search**. The given field must match the value exactly. Case sensitivity\n can be specified using the `case_sensitive` argument. An exact match search can use an\n index directly into the dataset, which should make this search fairly \"cheap\" from a\n performance perspective.\n\n2. **Wildcard match search**. The given fields specified by the exact match search may also\n be matched exactly to the value provided to the `wildcard` parameter.\n A wildcard match search can also use an index directly into the dataset.\n\n3. **Date range search**. The given field must be greater than or equal to the `from` date\n and/or less than or equal to the `to` date. A date range search involves\n sequentially scanning through the rows that have been located using any exact match\n criteria. This can be an expensive operation if there are many rows returned by any exact\n match criteria. Therefore, use date ranges as the _only_ criteria when the enrichment\n data set is very small.\n\nFor `geoip` and `mmdb` enrichment tables, this condition needs to be a VRL object with a single key-value pair\nwhose value needs to be a valid IP address. Example: `{\"ip\": .ip }`. If a return field is expected\nand without a value, `null` is used. This table can return the following fields:\n\n* ISP databases:\n * `autonomous_system_number`\n * `autonomous_system_organization`\n * `isp`\n * `organization`\n\n* City databases:\n * `city_name`\n * `continent_code`\n * `country_code`\n * `country_name`\n * `region_code`\n * `region_name`\n * `metro_code`\n * `latitude`\n * `longitude`\n * `postal_code`\n * `timezone`\n\n* Connection-Type databases:\n * `connection_type`\n\nTo use this function, you need to update your configuration to\ninclude an\n[`enrichment_tables`](/docs/reference/configuration/global-options/#enrichment_tables)\nparameter.", + "description": "Searches an [enrichment table](/docs/reference/glossary/#enrichment-tables) for rows that match the provided condition.\n\nFor `file` enrichment tables, this condition needs to be a VRL object in which\nthe key-value pairs indicate a field to search mapped to a value to search in that field.\nThis function returns the rows that match the provided condition(s). _All_ fields need to\nmatch for rows to be returned; if any fields do not match, then no rows are returned.\n\nThere are three forms of search criteria:\n\n1. **Exact match search**. The given field must match the value exactly. Case sensitivity\n can be specified using the `case_sensitive` argument. An exact match search can use an\n index directly into the dataset, which should make this search fairly \"cheap\" from a\n performance perspective.\n\n2. **Wildcard match search**. The given fields specified by the exact match search may also\n be matched exactly to the value provided to the `wildcard` parameter.\n A wildcard match search can also use an index directly into the dataset.\n\n3. **Date range search**. The given field must be greater than or equal to the `from` date\n and/or less than or equal to the `to` date. A date range search involves\n sequentially scanning through the rows that have been located using any exact match\n criteria. This can be an expensive operation if there are many rows returned by any exact\n match criteria. Therefore, use date ranges as the _only_ criteria when the enrichment\n data set is very small.\n\nFor `geoip` and `mmdb` enrichment tables, this condition needs to be a VRL object with a single key-value pair\nwhose value needs to be a valid IP address. Example: `{\"ip\": .ip }`. If a return field is expected\nand without a value, `null` is used. This table can return the following fields:\n\n* ISP databases:\n * `autonomous_system_number`\n * `autonomous_system_organization`\n * `isp`\n * `organization`\n * `network`\n\n* City databases:\n * `city_name`\n * `continent_code`\n * `country_code`\n * `country_name`\n * `region_code`\n * `region_name`\n * `metro_code`\n * `latitude`\n * `longitude`\n * `postal_code`\n * `timezone`\n * `network`\n\n* Connection-Type databases:\n * `connection_type`\n * `network`\n\n* Anonymous-IP databases:\n * `is_anonymous`\n * `is_anonymous_vpn`\n * `is_hosting_provider`\n * `is_public_proxy`\n * `is_residential_proxy`\n * `is_tor_exit_node`\n * `network`\n\nTo use this function, you need to update your configuration to\ninclude an\n[`enrichment_tables`](/docs/reference/configuration/global-options/#enrichment_tables)\nparameter.", "arguments": [ { "name": "table", diff --git a/docs/generated/get_enrichment_table_record.json b/docs/generated/get_enrichment_table_record.json index 947e8b3e65374..45a49c7606110 100644 --- a/docs/generated/get_enrichment_table_record.json +++ b/docs/generated/get_enrichment_table_record.json @@ -2,7 +2,7 @@ "anchor": "get_enrichment_table_record", "name": "get_enrichment_table_record", "category": "Enrichment", - "description": "Searches an [enrichment table](/docs/reference/glossary/#enrichment-tables) for a row that matches the provided condition. A single row must be matched. If no rows are found or more than one row is found, an error is returned.\n\nFor `file` enrichment tables, this condition needs to be a VRL object in which\nthe key-value pairs indicate a field to search mapped to a value to search in that field.\nThis function returns the rows that match the provided condition(s). _All_ fields need to\nmatch for rows to be returned; if any fields do not match, then no rows are returned.\n\nThere are three forms of search criteria:\n\n1. **Exact match search**. The given field must match the value exactly. Case sensitivity\n can be specified using the `case_sensitive` argument. An exact match search can use an\n index directly into the dataset, which should make this search fairly \"cheap\" from a\n performance perspective.\n\n2. **Wildcard match search**. The given fields specified by the exact match search may also\n be matched exactly to the value provided to the `wildcard` parameter.\n A wildcard match search can also use an index directly into the dataset.\n\n3. **Date range search**. The given field must be greater than or equal to the `from` date\n and/or less than or equal to the `to` date. A date range search involves\n sequentially scanning through the rows that have been located using any exact match\n criteria. This can be an expensive operation if there are many rows returned by any exact\n match criteria. Therefore, use date ranges as the _only_ criteria when the enrichment\n data set is very small.\n\nFor `geoip` and `mmdb` enrichment tables, this condition needs to be a VRL object with a single key-value pair\nwhose value needs to be a valid IP address. Example: `{\"ip\": .ip }`. If a return field is expected\nand without a value, `null` is used. This table can return the following fields:\n\n* ISP databases:\n * `autonomous_system_number`\n * `autonomous_system_organization`\n * `isp`\n * `organization`\n\n* City databases:\n * `city_name`\n * `continent_code`\n * `country_code`\n * `country_name`\n * `region_code`\n * `region_name`\n * `metro_code`\n * `latitude`\n * `longitude`\n * `postal_code`\n * `timezone`\n\n* Connection-Type databases:\n * `connection_type`\n\nTo use this function, you need to update your configuration to\ninclude an\n[`enrichment_tables`](/docs/reference/configuration/global-options/#enrichment_tables)\nparameter.", + "description": "Searches an [enrichment table](/docs/reference/glossary/#enrichment-tables) for a row that matches the provided condition. A single row must be matched. If no rows are found or more than one row is found, an error is returned.\n\nFor `file` enrichment tables, this condition needs to be a VRL object in which\nthe key-value pairs indicate a field to search mapped to a value to search in that field.\nThis function returns the rows that match the provided condition(s). _All_ fields need to\nmatch for rows to be returned; if any fields do not match, then no rows are returned.\n\nThere are three forms of search criteria:\n\n1. **Exact match search**. The given field must match the value exactly. Case sensitivity\n can be specified using the `case_sensitive` argument. An exact match search can use an\n index directly into the dataset, which should make this search fairly \"cheap\" from a\n performance perspective.\n\n2. **Wildcard match search**. The given fields specified by the exact match search may also\n be matched exactly to the value provided to the `wildcard` parameter.\n A wildcard match search can also use an index directly into the dataset.\n\n3. **Date range search**. The given field must be greater than or equal to the `from` date\n and/or less than or equal to the `to` date. A date range search involves\n sequentially scanning through the rows that have been located using any exact match\n criteria. This can be an expensive operation if there are many rows returned by any exact\n match criteria. Therefore, use date ranges as the _only_ criteria when the enrichment\n data set is very small.\n\nFor `geoip` and `mmdb` enrichment tables, this condition needs to be a VRL object with a single key-value pair\nwhose value needs to be a valid IP address. Example: `{\"ip\": .ip }`. If a return field is expected\nand without a value, `null` is used. This table can return the following fields:\n\n* ISP databases:\n * `autonomous_system_number`\n * `autonomous_system_organization`\n * `isp`\n * `organization`\n * `network`\n\n* City databases:\n * `city_name`\n * `continent_code`\n * `country_code`\n * `country_name`\n * `region_code`\n * `region_name`\n * `metro_code`\n * `latitude`\n * `longitude`\n * `postal_code`\n * `timezone`\n * `network`\n\n* Connection-Type databases:\n * `connection_type`\n * `network`\n\n* Anonymous-IP databases:\n * `is_anonymous`\n * `is_anonymous_vpn`\n * `is_hosting_provider`\n * `is_public_proxy`\n * `is_residential_proxy`\n * `is_tor_exit_node`\n * `network`\n\nTo use this function, you need to update your configuration to\ninclude an\n[`enrichment_tables`](/docs/reference/configuration/global-options/#enrichment_tables)\nparameter.", "arguments": [ { "name": "table", diff --git a/lib/vector-vrl/enrichment/src/lib.rs b/lib/vector-vrl/enrichment/src/lib.rs index df4575e91f08e..293a5db343d61 100644 --- a/lib/vector-vrl/enrichment/src/lib.rs +++ b/lib/vector-vrl/enrichment/src/lib.rs @@ -185,6 +185,7 @@ pub(crate) const ENRICHMENT_TABLE_EXPLAINER: &str = indoc! {r#" * `autonomous_system_organization` * `isp` * `organization` + * `network` * City databases: * `city_name` @@ -198,9 +199,20 @@ pub(crate) const ENRICHMENT_TABLE_EXPLAINER: &str = indoc! {r#" * `longitude` * `postal_code` * `timezone` + * `network` * Connection-Type databases: * `connection_type` + * `network` + + * Anonymous-IP databases: + * `is_anonymous` + * `is_anonymous_vpn` + * `is_hosting_provider` + * `is_public_proxy` + * `is_residential_proxy` + * `is_tor_exit_node` + * `network` To use this function, you need to update your configuration to include an diff --git a/src/enrichment_tables/geoip.rs b/src/enrichment_tables/geoip.rs index 782dbe8179afa..6716b4eaf2216 100644 --- a/src/enrichment_tables/geoip.rs +++ b/src/enrichment_tables/geoip.rs @@ -118,9 +118,15 @@ pub struct Geoip { fn lookup_value<'de, A: Deserialize<'de>>( dbreader: &'de Reader>, address: IpAddr, -) -> crate::Result> { +) -> crate::Result> { let result = dbreader.lookup(address)?; - Ok(result.decode::()?) + match result.decode::()? { + Some(data) => { + let network = result.network()?.to_string(); + Ok(Some((data, network))) + } + None => Ok(None), + } } impl Geoip { @@ -174,7 +180,7 @@ impl Geoip { match self.dbkind { DatabaseKind::Asn | DatabaseKind::Isp => { - let data = lookup_value::(&self.dbreader, ip).ok()??; + let (data, network) = lookup_value::(&self.dbreader, ip).ok()??; add_field!("autonomous_system_number", data.autonomous_system_number); add_field!( @@ -183,9 +189,11 @@ impl Geoip { ); add_field!("isp", data.isp); add_field!("organization", data.organization); + add_field!("network", Some(network)); } DatabaseKind::City => { - let data: City = lookup_value::(&self.dbreader, ip).ok()??; + let (data, network): (City, String) = + lookup_value::(&self.dbreader, ip).ok()??; add_field!("city_name", self.take_translation(&data.city.names)); @@ -223,14 +231,16 @@ impl Geoip { subdivision.and_then(|subdivision| subdivision.iso_code) ); add_field!("postal_code", data.postal.code); + add_field!("network", Some(network)); } DatabaseKind::ConnectionType => { - let data = lookup_value::(&self.dbreader, ip).ok()??; + let (data, network) = lookup_value::(&self.dbreader, ip).ok()??; add_field!("connection_type", data.connection_type); + add_field!("network", Some(network)); } DatabaseKind::AnonymousIp => { - let data = lookup_value::(&self.dbreader, ip).ok()??; + let (data, network) = lookup_value::(&self.dbreader, ip).ok()??; add_field!("is_anonymous", data.is_anonymous); add_field!("is_anonymous_vpn", data.is_anonymous_vpn); @@ -238,6 +248,7 @@ impl Geoip { add_field!("is_public_proxy", data.is_public_proxy); add_field!("is_residential_proxy", data.is_residential_proxy); add_field!("is_tor_exit_node", data.is_tor_exit_node); + add_field!("network", Some(network)); } } @@ -367,6 +378,7 @@ mod tests { expected.insert("longitude".into(), Value::from(-1.25)); expected.insert("postal_code".into(), "OX1".into()); expected.insert("metro_code".into(), Value::Null); + expected.insert("network".into(), "2.125.160.216/29".into()); assert_eq!(values, expected); } @@ -403,6 +415,7 @@ mod tests { expected.insert("longitude".into(), Value::from(90.5)); expected.insert("postal_code".into(), Value::Null); expected.insert("metro_code".into(), Value::Null); + expected.insert("network".into(), "67.43.156.0/24".into()); assert_eq!(values, expected); } @@ -426,6 +439,7 @@ mod tests { ); expected.insert("isp".into(), "Verizon Business".into()); expected.insert("organization".into(), "Verizon Business".into()); + expected.insert("network".into(), "208.192.0.0/10".into()); assert_eq!(values, expected); } @@ -442,6 +456,7 @@ mod tests { ); expected.insert("isp".into(), Value::Null); expected.insert("organization".into(), Value::Null); + expected.insert("network".into(), "2600:7000::/24".into()); assert_eq!(values, expected); } @@ -463,6 +478,7 @@ mod tests { let mut expected = ObjectMap::new(); expected.insert("connection_type".into(), "Corporate".into()); + expected.insert("network".into(), "201.243.200.0/24".into()); assert_eq!(values, expected); } @@ -494,6 +510,7 @@ mod tests { expected.insert("is_tor_exit_node".into(), true.into()); expected.insert("is_public_proxy".into(), Value::Null); expected.insert("is_residential_proxy".into(), Value::Null); + expected.insert("network".into(), "101.99.92.179/32".into()); assert_eq!(values, expected); } From 5011c5dfd20a0200c37a42a6c3001ae3df94ef4d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 17 Mar 2026 15:16:34 -0400 Subject: [PATCH 020/326] chore(ci): bump docker/build-push-action from 6.18.0 to 6.19.2 (#24734) Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 6.18.0 to 6.19.2. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/263435318d21b8e681c14492fe198d362a7d2c83...10e90e3645eae34f1e60eeb005ba3a3d33f178e8) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-version: 6.19.2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/environment.yml | 4 ++-- .github/workflows/regression.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/environment.yml b/.github/workflows/environment.yml index 129e4c6612d9b..fce6b25c3c181 100644 --- a/.github/workflows/environment.yml +++ b/.github/workflows/environment.yml @@ -69,7 +69,7 @@ jobs: run: sudo -E bash scripts/ci-free-disk-space.sh - name: Build image (no push) - uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 with: context: . file: ./scripts/environment/Dockerfile @@ -89,7 +89,7 @@ jobs: - name: Push image if: env.SHOULD_PUBLISH == 'true' - uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 with: context: . file: ./scripts/environment/Dockerfile diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 2621118a9df11..89a28f5121360 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -212,7 +212,7 @@ jobs: uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - name: Build 'vector' target image - uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 with: context: baseline-vector/ cache-from: type=gha @@ -255,7 +255,7 @@ jobs: uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - name: Build 'vector' target image - uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 with: context: comparison-vector/ cache-from: type=gha From 6527346294203cec7fb0d8b86c7fc98b26c14ff0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 17 Mar 2026 19:17:07 +0000 Subject: [PATCH 021/326] chore(ci): bump aws-actions/configure-aws-credentials from 5.1.1 to 6.0.0 (#24733) chore(ci): bump aws-actions/configure-aws-credentials Bumps [aws-actions/configure-aws-credentials](https://github.com/aws-actions/configure-aws-credentials) from 5.1.1 to 6.0.0. - [Release notes](https://github.com/aws-actions/configure-aws-credentials/releases) - [Changelog](https://github.com/aws-actions/configure-aws-credentials/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws-actions/configure-aws-credentials/compare/61815dcd50bd041e203e49132bacad1fd04d2708...8df5847569e6427dd6c4fb1cf565c83acfa8afa7) --- updated-dependencies: - dependency-name: aws-actions/configure-aws-credentials dependency-version: 6.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/regression.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 89a28f5121360..a8c6981e53a9d 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -281,7 +281,7 @@ jobs: - resolve-inputs steps: - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708 # v5.1.1 + uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0 with: aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} @@ -315,7 +315,7 @@ jobs: docker load --input baseline-image.tar - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708 # v5.1.1 + uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0 with: aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} @@ -355,7 +355,7 @@ jobs: docker load --input comparison-image.tar - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708 # v5.1.1 + uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0 with: aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} @@ -394,7 +394,7 @@ jobs: ref: ${{ needs.resolve-inputs.outputs.comparison-sha }} - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708 # v5.1.1 + uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0 with: aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} @@ -467,7 +467,7 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708 # v5.1.1 + uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0 with: aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} @@ -510,7 +510,7 @@ jobs: ref: ${{ needs.resolve-inputs.outputs.comparison-sha }} - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708 # v5.1.1 + uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0 with: aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} From b1a43a4ba24b0e76e57736953a0f6f9d0ee58328 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 17 Mar 2026 20:52:26 +0000 Subject: [PATCH 022/326] chore(deps): bump sysinfo from 0.37.2 to 0.38.2 (#24816) * chore(deps): bump sysinfo from 0.37.2 to 0.38.2 Bumps [sysinfo](https://github.com/GuillaumeGomez/sysinfo) from 0.37.2 to 0.38.2. - [Changelog](https://github.com/GuillaumeGomez/sysinfo/blob/main/CHANGELOG.md) - [Commits](https://github.com/GuillaumeGomez/sysinfo/compare/v0.37.2...v0.38.2) --- updated-dependencies: - dependency-name: sysinfo dependency-version: 0.38.2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * update licenses --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Pavlos Rontidis --- Cargo.lock | 97 ++++++++++++++++++++++++++------------------ LICENSE-3rdparty.csv | 3 ++ 2 files changed, 61 insertions(+), 39 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3db4f718a1aaf..f19fbc7c50f2c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7397,18 +7397,18 @@ dependencies = [ [[package]] name = "objc2-core-foundation" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c10c2894a6fed806ade6027bcd50662746363a9589d3ec9d9bef30a4e4bc166" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ "bitflags 2.10.0", ] [[package]] name = "objc2-io-kit" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71c1c64d6120e51cd86033f67176b1cb66780c2efe34dec55176f77befd93c0a" +checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15" dependencies = [ "libc", "objc2-core-foundation", @@ -10897,7 +10897,7 @@ dependencies = [ "ntapi 0.4.1", "objc2-core-foundation", "objc2-io-kit", - "windows 0.60.0", + "windows 0.61.3", ] [[package]] @@ -13463,24 +13463,24 @@ dependencies = [ [[package]] name = "windows" -version = "0.60.0" +version = "0.61.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddf874e74c7a99773e62b1c671427abf01a425e77c3d3fb9fb1e4883ea934529" +checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" dependencies = [ "windows-collections", - "windows-core 0.60.1", + "windows-core 0.61.2", "windows-future", - "windows-link 0.1.0", + "windows-link 0.1.3", "windows-numerics", ] [[package]] name = "windows-collections" -version = "0.1.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5467f79cc1ba3f52ebb2ed41dbb459b8e7db636cc3429458d9a852e15bc24dec" +checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8" dependencies = [ - "windows-core 0.60.1", + "windows-core 0.61.2", ] [[package]] @@ -13507,25 +13507,26 @@ dependencies = [ [[package]] name = "windows-core" -version = "0.60.1" +version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca21a92a9cae9bf4ccae5cf8368dce0837100ddf6e6d57936749e85f152f6247" +checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" dependencies = [ - "windows-implement 0.59.0", - "windows-interface 0.59.1", - "windows-link 0.1.0", - "windows-result 0.3.1", - "windows-strings 0.3.1", + "windows-implement 0.60.2", + "windows-interface 0.59.3", + "windows-link 0.1.3", + "windows-result 0.3.4", + "windows-strings 0.4.2", ] [[package]] name = "windows-future" -version = "0.1.1" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a787db4595e7eb80239b74ce8babfb1363d8e343ab072f2ffe901400c03349f0" +checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" dependencies = [ - "windows-core 0.60.1", - "windows-link 0.1.0", + "windows-core 0.61.2", + "windows-link 0.1.3", + "windows-threading", ] [[package]] @@ -13541,9 +13542,9 @@ dependencies = [ [[package]] name = "windows-implement" -version = "0.59.0" +version = "0.60.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83577b051e2f49a058c308f17f273b570a6a758386fc291b5f6a934dd84e48c1" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2 1.0.106", "quote 1.0.44", @@ -13563,9 +13564,9 @@ dependencies = [ [[package]] name = "windows-interface" -version = "0.59.1" +version = "0.59.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2 1.0.106", "quote 1.0.44", @@ -13574,9 +13575,9 @@ dependencies = [ [[package]] name = "windows-link" -version = "0.1.0" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dccfd733ce2b1753b03b6d3c65edf020262ea35e20ccdf3e288043e6dd620e3" +checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" [[package]] name = "windows-link" @@ -13586,12 +13587,12 @@ checksum = "45e46c0661abb7180e7b9c281db115305d49ca1709ab8242adf09666d2173c65" [[package]] name = "windows-numerics" -version = "0.1.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "005dea54e2f6499f2cee279b8f703b3cf3b5734a2d8d21867c8f44003182eeed" +checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1" dependencies = [ - "windows-core 0.60.1", - "windows-link 0.1.0", + "windows-core 0.61.2", + "windows-link 0.1.3", ] [[package]] @@ -13600,8 +13601,8 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c44a98275e31bfd112bb06ba96c8ab13c03383a3753fdddd715406a1824c7e0" dependencies = [ - "windows-link 0.1.0", - "windows-result 0.3.1", + "windows-link 0.1.3", + "windows-result 0.3.4", "windows-strings 0.3.1", ] @@ -13616,11 +13617,11 @@ dependencies = [ [[package]] name = "windows-result" -version = "0.3.1" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06374efe858fab7e4f881500e6e86ec8bc28f9462c47e5a9941a0142ad86b189" +checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" dependencies = [ - "windows-link 0.1.0", + "windows-link 0.1.3", ] [[package]] @@ -13650,7 +13651,16 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "87fa48cc5d406560701792be122a10132491cff9d0aeb23583cc2dcafc847319" dependencies = [ - "windows-link 0.1.0", + "windows-link 0.1.3", +] + +[[package]] +name = "windows-strings" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" +dependencies = [ + "windows-link 0.1.3", ] [[package]] @@ -13769,6 +13779,15 @@ dependencies = [ "windows_x86_64_msvc 0.53.0", ] +[[package]] +name = "windows-threading" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b66463ad2e0ea3bbf808b7f1d371311c80e115c0b71d60efc142cafbcfb057a6" +dependencies = [ + "windows-link 0.1.3", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.42.2" diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 09e39573c8126..2079e5d6819ba 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -888,7 +888,9 @@ windows-collections,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Th windows-core,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft windows-future,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows-future Authors windows-implement,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows-implement,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows-implement Authors windows-interface,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows-interface,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows-interface Authors windows-link,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft windows-link,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows-link Authors windows-numerics,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows-numerics Authors @@ -899,6 +901,7 @@ windows-strings,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Micros windows-sys,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft windows-sys,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows-sys Authors windows-targets,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft +windows-threading,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft windows_aarch64_gnullvm,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft windows_aarch64_msvc,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft windows_i686_gnu,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Microsoft From cfee4fb9d1853a5ae7db85e0bd5647ca7a7b57b0 Mon Sep 17 00:00:00 2001 From: Thomas Date: Tue, 17 Mar 2026 16:54:19 -0400 Subject: [PATCH 023/326] chore(deps): upgrade mold linker from 1.2.1/2.0.0 to 2.40.4 (#24947) --- .github/actions/setup/action.yml | 2 +- scripts/environment/bootstrap-ubuntu-24.04.sh | 2 +- tilt/Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 8dbb5416de92f..3d70368bac027 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -182,7 +182,7 @@ runs: # We explicitly put `mold-wrapper.so` right beside `mold` itself because it's hard-coded to look in the same directory # first when trying to load the shared object, so we can dodge having to care about the "right" lib folder to put it in. TEMP=$(mktemp -d) - MOLD_VERSION=1.2.1 + MOLD_VERSION=2.40.4 MOLD_TARGET=mold-${MOLD_VERSION}-$(uname -m)-linux curl -fsSL "https://github.com/rui314/mold/releases/download/v${MOLD_VERSION}/${MOLD_TARGET}.tar.gz" \ --output "$TEMP/${MOLD_TARGET}.tar.gz" diff --git a/scripts/environment/bootstrap-ubuntu-24.04.sh b/scripts/environment/bootstrap-ubuntu-24.04.sh index 5e59d383dbc1a..9d135f779b933 100755 --- a/scripts/environment/bootstrap-ubuntu-24.04.sh +++ b/scripts/environment/bootstrap-ubuntu-24.04.sh @@ -145,7 +145,7 @@ if [ -z "${DISABLE_MOLD:-""}" ] ; then # We explicitly put `mold-wrapper.so` right beside `mold` itself because it's hard-coded to look in the same directory # first when trying to load the shared object, so we can dodge having to care about the "right" lib folder to put it in. TEMP=$(mktemp -d) - MOLD_VERSION=1.2.1 + MOLD_VERSION=2.40.4 MOLD_TARGET=mold-${MOLD_VERSION}-$(uname -m)-linux curl -fsSL "https://github.com/rui314/mold/releases/download/v${MOLD_VERSION}/${MOLD_TARGET}.tar.gz" \ --output "$TEMP/${MOLD_TARGET}.tar.gz" diff --git a/tilt/Dockerfile b/tilt/Dockerfile index cbf78acf36bf4..f8c1b1df1cabd 100644 --- a/tilt/Dockerfile +++ b/tilt/Dockerfile @@ -11,7 +11,7 @@ RUN apt-get update && apt-get -y --no-install-recommends install build-essential RUN git clone https://github.com/rui314/mold.git \ && mkdir mold/build \ && cd mold/build \ - && git checkout v2.0.0 \ + && git checkout v2.40.4 \ && ../install-build-deps.sh \ && cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=c++ .. \ && cmake --build . -j $(nproc) \ From fbb1e4b501d6081d72f864da891a98464e60d097 Mon Sep 17 00:00:00 2001 From: Bruce Guenter Date: Tue, 17 Mar 2026 18:05:08 -0600 Subject: [PATCH 024/326] fix(buffers): Improve buffer utilization metric tracking (#24911) The buffer usage metrics, in particular the value of the current utilization levels, were tracked using an atomic `u64` which was updated using a `fetch_update` mechanism in order to protect against underflowing. This same mechanism was extended to all of the atomics as well for consistency. The problem with that is that `fetch_udpate` internally uses a loop around a `compare-and-exchange` operation which is very expensive, particularly when contended. In comparison, the base `fetch_add` is typically a single locked instruction which completes in many fewer cycles. This change returns these atomics to only ever use `fetch_add` and then calculate the current level by subtracting the count of increments from the count of decrements. This improves correctness by avoiding problems around underflow as well as optimizing performace (very slightly). --- ...-buffer-utilization-metric-tracking.fix.md | 3 + lib/vector-buffers/src/buffer_usage_data.rs | 158 +++++++----------- 2 files changed, 66 insertions(+), 95 deletions(-) create mode 100644 changelog.d/24911-buffer-utilization-metric-tracking.fix.md diff --git a/changelog.d/24911-buffer-utilization-metric-tracking.fix.md b/changelog.d/24911-buffer-utilization-metric-tracking.fix.md new file mode 100644 index 0000000000000..fd1c23e5414b6 --- /dev/null +++ b/changelog.d/24911-buffer-utilization-metric-tracking.fix.md @@ -0,0 +1,3 @@ +Fixed regression in buffer utilization metric tracking around underflow. + +authors: bruceg diff --git a/lib/vector-buffers/src/buffer_usage_data.rs b/lib/vector-buffers/src/buffer_usage_data.rs index 61bda12f91160..ce7a67648db3d 100644 --- a/lib/vector-buffers/src/buffer_usage_data.rs +++ b/lib/vector-buffers/src/buffer_usage_data.rs @@ -19,34 +19,6 @@ use crate::{ /// always used a "relaxed" ordering when updating them. const ORDERING: Ordering = Ordering::Relaxed; -fn increment_counter(counter: &AtomicU64, delta: u64) { - counter - .fetch_update(ORDERING, ORDERING, |current| { - Some(current.checked_add(delta).unwrap_or_else(|| { - warn!( - current, - delta, "Buffer counter overflowed. Clamping value to `u64::MAX`." - ); - u64::MAX - })) - }) - .ok(); -} - -fn decrement_counter(counter: &AtomicU64, delta: u64) { - counter - .fetch_update(ORDERING, ORDERING, |current| { - Some(current.checked_sub(delta).unwrap_or_else(|| { - warn!( - current, - delta, "Buffer counter underflowed. Clamping value to `0`." - ); - 0 - })) - }) - .ok(); -} - /// Snapshot of category metrics. struct CategorySnapshot { event_count: u64, @@ -62,10 +34,15 @@ impl CategorySnapshot { /// Per-category metrics. /// -/// This tracks the number of events, and their size in the buffer, that a given category has interacted with. A -/// category in this case could be something like the receive or send categories i.e. being written into the buffer, and -/// then read out of the buffer. Overall, it's a simple grouping mechanism because we often want to track the change in -/// both number of events, and their size as measured by the buffer. +/// This tracks the number of events, and their size in the buffer, that a given category has +/// interacted with. A category in this case could be something like the receive or send categories +/// i.e. being written into the buffer, and then read out of the buffer. Overall, it's a simple +/// grouping mechanism because we often want to track the change in both number of events, and their +/// size as measured by the buffer. +/// +/// At a sustained 1 GiB/sec, which is still faster than Vector can currently achieve, a `u64` byte +/// counter would take over 500 years to overflow. As such, we don't handle failures due to +/// overflow as it is effectively impossible. #[derive(Debug, Default)] struct CategoryMetrics { event_count: AtomicU64, @@ -75,14 +52,8 @@ struct CategoryMetrics { impl CategoryMetrics { /// Increments the event count and byte size by the given amounts. fn increment(&self, event_count: u64, event_byte_size: u64) { - increment_counter(&self.event_count, event_count); - increment_counter(&self.event_byte_size, event_byte_size); - } - - /// Decrements the event count and byte size by the given amounts. - fn decrement(&self, event_count: u64, event_byte_size: u64) { - decrement_counter(&self.event_count, event_count); - decrement_counter(&self.event_byte_size, event_byte_size); + self.event_count.fetch_add(event_count, ORDERING); + self.event_byte_size.fetch_add(event_byte_size, ORDERING); } /// Sets the event count and event byte size to the given amount. @@ -115,6 +86,41 @@ impl CategoryMetrics { } } +/// `CurrentMetrics` is a wrapper around a pair of `CategoryMetrics` that are used to track a +/// "current" value that may increment or decrement. The challenge this solves is that the +/// increments and decrements may race and result in underflows that are hard to handle using only +/// efficient atomic operations. By tracking the increments and decrements separately, we can ensure +/// that the current value is always accurate even if the increments and decrements race. +#[derive(Debug, Default)] +struct CurrentMetrics { + increments: CategoryMetrics, + decrements: CategoryMetrics, +} + +impl CurrentMetrics { + fn increment(&self, event_count: u64, event_byte_size: u64) { + self.increments.increment(event_count, event_byte_size); + } + + fn decrement(&self, event_count: u64, event_byte_size: u64) { + self.decrements.increment(event_count, event_byte_size); + } + + fn get(&self) -> CategorySnapshot { + let entered_total = self.increments.get(); + let left_total = self.decrements.get(); + + CategorySnapshot { + event_count: entered_total + .event_count + .saturating_sub(left_total.event_count), + event_byte_size: entered_total + .event_byte_size + .saturating_sub(left_total.event_byte_size), + } + } +} + /// Handle to buffer usage metrics for a specific buffer stage. #[derive(Clone, Debug)] pub struct BufferUsageHandle { @@ -195,7 +201,7 @@ struct BufferUsageData { dropped: CategoryMetrics, dropped_intentional: CategoryMetrics, max_size: CategoryMetrics, - current: CategoryMetrics, + current: CurrentMetrics, } impl BufferUsageData { @@ -372,68 +378,30 @@ impl BufferUsage { #[cfg(test)] mod tests { - use std::thread; - use super::*; #[test] - fn test_multithreaded_updates_are_correct() { - const NUM_THREADS: u64 = 16; - const INCREMENTS_PER_THREAD: u64 = 10_000; - - let counter = Arc::new(AtomicU64::new(0)); - - let mut handles = vec![]; - - for _ in 0..NUM_THREADS { - let counter = Arc::clone(&counter); - let handle = thread::spawn(move || { - for _ in 0..INCREMENTS_PER_THREAD { - increment_counter(&counter, 1); - decrement_counter(&counter, 1); - } - }); - handles.push(handle); - } - - for handle in handles { - handle.join().unwrap(); - } - - assert_eq!(counter.load(ORDERING), 0); - } - - #[test] - fn test_decrement_counter_prevents_negatives() { - let counter = AtomicU64::new(100); - - decrement_counter(&counter, 50); - assert_eq!(counter.load(ORDERING), 50); - - decrement_counter(&counter, 100); - assert_eq!(counter.load(ORDERING), 0); + fn current_usage_is_derived_from_entered_and_left_totals() { + let handle = BufferUsageHandle { + state: Arc::new(BufferUsageData::new(0)), + }; - decrement_counter(&counter, 50); - assert_eq!(counter.load(ORDERING), 0); + handle.increment_received_event_count_and_byte_size(10, 1000); + handle.increment_sent_event_count_and_byte_size(3, 300); + handle.increment_dropped_event_count_and_byte_size(2, 200, false); - decrement_counter(&counter, u64::MAX); - assert_eq!(counter.load(ORDERING), 0); + let current = handle.state.current.get(); + assert_eq!(current.event_count, 5); + assert_eq!(current.event_byte_size, 500); } #[test] - fn test_increment_counter_prevents_overflow() { - let counter = AtomicU64::new(u64::MAX - 2); - - increment_counter(&counter, 1); - assert_eq!(counter.load(ORDERING), u64::MAX - 1); - - increment_counter(&counter, 1); - assert_eq!(counter.load(ORDERING), u64::MAX); - - increment_counter(&counter, 1); - assert_eq!(counter.load(ORDERING), u64::MAX); + fn current_usage_saturates_at_zero() { + let data = BufferUsageData::new(0); + data.current.decrement(10, 1000); - increment_counter(&counter, u64::MAX); - assert_eq!(counter.load(ORDERING), u64::MAX); + let current = data.current.get(); + assert_eq!(current.event_count, 0); + assert_eq!(current.event_byte_size, 0); } } From 95f439420ae1dea0df7fbd295e7f354b144360c8 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Wed, 18 Mar 2026 10:25:48 -0400 Subject: [PATCH 025/326] fix(api): place per-output sent events in Events Out column (#24951) * fix(top): place per-output sent events in Events Out column Output sub-rows for components with multiple output ports (e.g. route transforms with `_unmatched`, `success` outputs) were placing the per-output sent events metric in column index 5 (Bytes In) instead of column index 6 (Events Out). Co-Authored-By: Claude Sonnet 4.6 * add scopes * changelog: add entry for PR #24951 Co-Authored-By: Claude Sonnet 4.6 * fix name --------- Co-authored-by: Claude Sonnet 4.6 --- .github/workflows/semantic.yml | 6 ++++-- changelog.d/24951_top_events_out_column.fix.md | 3 +++ lib/vector-top/src/dashboard.rs | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) create mode 100644 changelog.d/24951_top_events_out_column.fix.md diff --git a/.github/workflows/semantic.yml b/.github/workflows/semantic.yml index f873f32108c5c..79a36a94ec7cf 100644 --- a/.github/workflows/semantic.yml +++ b/.github/workflows/semantic.yml @@ -15,8 +15,8 @@ permissions: jobs: main: permissions: - pull-requests: read # for amannn/action-semantic-pull-request to analyze PRs - statuses: write # for amannn/action-semantic-pull-request to mark status of analyzed PR + pull-requests: read # for amannn/action-semantic-pull-request to analyze PRs + statuses: write # for amannn/action-semantic-pull-request to mark status of analyzed PR name: Check Semantic PR runs-on: ubuntu-24.04 steps: @@ -36,6 +36,8 @@ jobs: scopes: | administration api + api top + api tap ARC architecture auth diff --git a/changelog.d/24951_top_events_out_column.fix.md b/changelog.d/24951_top_events_out_column.fix.md new file mode 100644 index 0000000000000..e4191569cc205 --- /dev/null +++ b/changelog.d/24951_top_events_out_column.fix.md @@ -0,0 +1,3 @@ +Fixed `vector top` displaying per-output sent events in the wrong column (Bytes In instead of Events Out) for components with multiple output ports. + +authors: pront diff --git a/lib/vector-top/src/dashboard.rs b/lib/vector-top/src/dashboard.rs index c4078ebde6afb..1b73d5b89e9c8 100644 --- a/lib/vector-top/src/dashboard.rs +++ b/lib/vector-top/src/dashboard.rs @@ -373,7 +373,7 @@ impl<'a> Widgets<'a> { .map(Cell::from) .collect::>(); data[1] = Cell::from(id.as_str()); - data[5] = Cell::from(sent_events_metric); + data[6] = Cell::from(sent_events_metric); items.push(Row::new(data).style(Style::default())); } } From 30936678f92a03c827f9fd8506fa02693332f59b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ensar=20Saraj=C4=8Di=C4=87?= Date: Wed, 18 Mar 2026 19:34:43 +0100 Subject: [PATCH 026/326] feat(cli): add support for edge_attributes in graph configuration (#24593) * feat(cli): add support for edge_attributes in graph configuration Closes: #24591 * Add changelog entry * feat(cli): include memory enrichment tables in graph output * feat(cli): add memory enrichment tables in graph mermaid output * Generate docs * Inline id * Use whole `OutputId` as key in edge_attributes * Fallback to just component name if port is not present in edge_attributes * Generate component docs --- .../24593_graph_edge_attributes.feature.md | 3 + src/config/dot_graph.rs | 41 ++- src/graph.rs | 139 ++++++++-- .../reference/components/generated/sinks.cue | 69 +++-- .../components/generated/sources.cue | 67 +++-- .../components/generated/transforms.cue | 67 +++-- .../cue/reference/generated/configuration.cue | 244 ++++++++++++++---- 7 files changed, 505 insertions(+), 125 deletions(-) create mode 100644 changelog.d/24593_graph_edge_attributes.feature.md diff --git a/changelog.d/24593_graph_edge_attributes.feature.md b/changelog.d/24593_graph_edge_attributes.feature.md new file mode 100644 index 0000000000000..719cbc6b4635a --- /dev/null +++ b/changelog.d/24593_graph_edge_attributes.feature.md @@ -0,0 +1,3 @@ +`graph.edge_attributes` can now be added to transforms and sinks to add attributes to edges in graphs generated using `vector graph`. Memory enrichment tables are also considered for graphs, because they can have inputs and outputs. + +authors: esensar Quad9DNS diff --git a/src/config/dot_graph.rs b/src/config/dot_graph.rs index 0d3d7ca1e4900..a006837cf672f 100644 --- a/src/config/dot_graph.rs +++ b/src/config/dot_graph.rs @@ -15,15 +15,52 @@ pub struct GraphConfig { /// They are added to the node as provided #[configurable(metadata( docs::additional_props_description = "A single graph node attribute in graphviz DOT language.", - docs::examples = "example_graph_options()" + docs::examples = "example_node_options()" ))] + #[serde(default)] pub node_attributes: HashMap, + + /// Edge attributes to add to the edges linked to this component's node in resulting graph + /// + /// They are added to the edge as provided + #[configurable(metadata( + docs::additional_props_description = "A collection of graph edge attributes in graphviz DOT language, related to a single input component.", + docs::examples = "example_edges_options()" + ))] + #[serde(default)] + pub edge_attributes: HashMap, } -fn example_graph_options() -> HashMap { +#[configurable_component] +#[configurable(metadata(docs::advanced))] +#[derive(Clone, Debug, Default, Eq, PartialEq)] +#[serde(deny_unknown_fields)] +/// A collection of graph edge attributes in graphviz DOT language, related to a single input +/// component. +pub struct EdgeAttributes( + #[configurable(metadata( + docs::additional_props_description = "A single graph edge attribute in graphviz DOT language.", + docs::examples = "example_edge_options()" + ))] + pub HashMap, +); + +fn example_node_options() -> HashMap { HashMap::<_, _>::from_iter([ ("name".to_string(), "Example Node".to_string()), ("color".to_string(), "red".to_string()), ("width".to_string(), "5.0".to_string()), ]) } + +fn example_edges_options() -> HashMap> { + HashMap::<_, _>::from_iter([("example_input".to_string(), example_edge_options())]) +} + +fn example_edge_options() -> HashMap { + HashMap::<_, _>::from_iter([ + ("label".to_string(), "Example Edge".to_string()), + ("color".to_string(), "red".to_string()), + ("width".to_string(), "5.0".to_string()), + ]) +} diff --git a/src/graph.rs b/src/graph.rs index cfdcfffb3c6f5..241833b32f59c 100644 --- a/src/graph.rs +++ b/src/graph.rs @@ -1,9 +1,17 @@ -use std::{collections::HashMap, fmt::Write as _, path::PathBuf}; +use std::{ + collections::{HashMap, HashSet}, + fmt::Write as _, + path::PathBuf, +}; use clap::Parser; use itertools::Itertools; +use vector_lib::{config::OutputId, id::ComponentKey}; -use crate::config; +use crate::config::{ + self, + dot_graph::{EdgeAttributes, GraphConfig}, +}; #[derive(Parser, Debug)] #[command(rename_all = "kebab-case")] @@ -94,6 +102,16 @@ fn node_attributes_to_string(attributes: &HashMap, default_shape attrs.iter().map(|(k, v)| format!("{k}=\"{v}\"")).join(" ") } +fn edge_attributes_to_string(attributes: &EdgeAttributes, default_label: Option<&str>) -> String { + let mut attrs = attributes.0.clone(); + if let Some(default_label) = default_label + && !attrs.contains_key("label") + { + attrs.insert("label".to_string(), default_label.to_string()); + } + attrs.iter().map(|(k, v)| format!("{k}=\"{v}\"")).join(" ") +} + pub(crate) fn cmd(opts: &Opts) -> exitcode::ExitCode { let paths = opts.paths_with_formats(); let paths = match config::process_paths(&paths) { @@ -122,6 +140,43 @@ pub(crate) fn cmd(opts: &Opts) -> exitcode::ExitCode { fn render_dot(config: config::Config) -> exitcode::ExitCode { let mut dot = String::from("digraph {\n"); + let mut written_tables = HashSet::::new(); + + for (id, table) in config + .enrichment_tables + .iter() + .filter_map(|(key, table)| table.as_source(key)) + { + writeln!( + dot, + " \"{}\" [{}]", + id, + node_attributes_to_string(&table.graph.node_attributes, "cylinder") + ) + .expect("write to String never fails"); + written_tables.insert(id); + } + + for (id, table) in config + .enrichment_tables + .iter() + .filter_map(|(key, table)| table.as_sink(key)) + { + if !written_tables.contains(&id) { + writeln!( + dot, + " \"{}\" [{}]", + id, + node_attributes_to_string(&table.graph.node_attributes, "cylinder") + ) + .expect("write to String never fails"); + } + + for input in table.inputs.iter() { + render_dot_edge(&mut dot, &id, input, &table.graph); + } + } + for (id, source) in config.sources() { writeln!( dot, @@ -142,16 +197,7 @@ fn render_dot(config: config::Config) -> exitcode::ExitCode { .expect("write to String never fails"); for input in transform.inputs.iter() { - if let Some(port) = &input.port { - writeln!( - dot, - " \"{}\" -> \"{}\" [label=\"{}\"]", - input.component, id, port - ) - .expect("write to String never fails"); - } else { - writeln!(dot, " \"{input}\" -> \"{id}\"").expect("write to String never fails"); - } + render_dot_edge(&mut dot, id, input, &transform.graph); } } @@ -165,16 +211,7 @@ fn render_dot(config: config::Config) -> exitcode::ExitCode { .expect("write to String never fails"); for input in &sink.inputs { - if let Some(port) = &input.port { - writeln!( - dot, - " \"{}\" -> \"{}\" [label=\"{}\"]", - input.component, id, port - ) - .expect("write to String never fails"); - } else { - writeln!(dot, " \"{input}\" -> \"{id}\"").expect("write to String never fails"); - } + render_dot_edge(&mut dot, id, input, &sink.graph); } } @@ -188,9 +225,67 @@ fn render_dot(config: config::Config) -> exitcode::ExitCode { exitcode::OK } +fn render_dot_edge(into: &mut String, id: &ComponentKey, input: &OutputId, graph: &GraphConfig) { + let edge_attributes = graph + .edge_attributes + .get(&input.to_string()) + .or_else(|| graph.edge_attributes.get(&input.component.to_string())); + if let Some(port) = &input.port { + writeln!( + into, + " \"{}\" -> \"{id}\" [{}]", + input.component, + edge_attributes_to_string( + edge_attributes.unwrap_or(&EdgeAttributes::default()), + Some(port) + ) + ) + .expect("write to String never fails"); + } else if let Some(edge_attributes) = edge_attributes { + writeln!( + into, + " \"{input}\" -> \"{id}\" [{}]", + edge_attributes_to_string(edge_attributes, None) + ) + .expect("write to String never fails"); + } else { + writeln!(into, " \"{input}\" -> \"{id}\"").expect("write to String never fails"); + } +} + fn render_mermaid(config: config::Config) -> exitcode::ExitCode { let mut mermaid = String::from("flowchart TD;\n"); + writeln!(mermaid, "\n %% Enrichment tables").unwrap(); + let mut written_tables = HashSet::::new(); + + for (id, _) in config + .enrichment_tables + .iter() + .filter_map(|(key, table)| table.as_source(key)) + { + writeln!(mermaid, " {id}[({id})]").unwrap(); + written_tables.insert(id); + } + + for (id, table) in config + .enrichment_tables + .iter() + .filter_map(|(key, table)| table.as_sink(key)) + { + if !written_tables.contains(&id) { + writeln!(mermaid, " {id}[({id})]").unwrap(); + } + + for input in table.inputs.iter() { + if let Some(port) = &input.port { + writeln!(mermaid, " {0} -->|{port}| {id}", input.component).unwrap(); + } else { + writeln!(mermaid, " {0} --> {id}", input.component).unwrap(); + } + } + } + writeln!(mermaid, "\n %% Sources").unwrap(); for (id, _) in config.sources() { writeln!(mermaid, " {id}[/{id}/]").unwrap(); diff --git a/website/cue/reference/components/generated/sinks.cue b/website/cue/reference/components/generated/sinks.cue index 9bb1bc943749e..6268666fc49c5 100644 --- a/website/cue/reference/components/generated/sinks.cue +++ b/website/cue/reference/components/generated/sinks.cue @@ -82,23 +82,58 @@ generated: components: sinks: configuration: { Configure output for component when generated with graph command """ required: false - type: object: options: node_attributes: { - description: """ - Node attributes to add to this component's node in resulting graph - - They are added to the node as provided - """ - required: false - type: object: { - examples: [{ - color: "red" - name: "Example Node" - width: "5.0" - }] - options: "*": { - description: "A single graph node attribute in graphviz DOT language." - required: true - type: string: {} + type: object: options: { + edge_attributes: { + description: """ + Edge attributes to add to the edges linked to this component's node in resulting graph + + They are added to the edge as provided + """ + required: false + type: object: { + examples: [{ + example_input: { + color: "red" + label: "Example Edge" + width: "5.0" + } + }] + options: "*": { + description: "A collection of graph edge attributes in graphviz DOT language, related to a single input component." + required: true + type: object: { + examples: [{ + color: "red" + label: "Example Edge" + width: "5.0" + }] + options: "*": { + description: "A single graph edge attribute in graphviz DOT language." + required: true + type: string: {} + } + } + } + } + } + node_attributes: { + description: """ + Node attributes to add to this component's node in resulting graph + + They are added to the node as provided + """ + required: false + type: object: { + examples: [{ + color: "red" + name: "Example Node" + width: "5.0" + }] + options: "*": { + description: "A single graph node attribute in graphviz DOT language." + required: true + type: string: {} + } } } } diff --git a/website/cue/reference/components/generated/sources.cue b/website/cue/reference/components/generated/sources.cue index 03b0c109f2be9..04ce33d83212f 100644 --- a/website/cue/reference/components/generated/sources.cue +++ b/website/cue/reference/components/generated/sources.cue @@ -8,23 +8,58 @@ generated: components: sources: configuration: { Configure output for component when generated with graph command """ required: false - type: object: options: node_attributes: { - description: """ - Node attributes to add to this component's node in resulting graph + type: object: options: { + edge_attributes: { + description: """ + Edge attributes to add to the edges linked to this component's node in resulting graph + + They are added to the edge as provided + """ + required: false + type: object: { + examples: [{ + example_input: { + color: "red" + label: "Example Edge" + width: "5.0" + } + }] + options: "*": { + description: "A collection of graph edge attributes in graphviz DOT language, related to a single input component." + required: true + type: object: { + examples: [{ + color: "red" + label: "Example Edge" + width: "5.0" + }] + options: "*": { + description: "A single graph edge attribute in graphviz DOT language." + required: true + type: string: {} + } + } + } + } + } + node_attributes: { + description: """ + Node attributes to add to this component's node in resulting graph - They are added to the node as provided - """ - required: false - type: object: { - examples: [{ - color: "red" - name: "Example Node" - width: "5.0" - }] - options: "*": { - description: "A single graph node attribute in graphviz DOT language." - required: true - type: string: {} + They are added to the node as provided + """ + required: false + type: object: { + examples: [{ + color: "red" + name: "Example Node" + width: "5.0" + }] + options: "*": { + description: "A single graph node attribute in graphviz DOT language." + required: true + type: string: {} + } } } } diff --git a/website/cue/reference/components/generated/transforms.cue b/website/cue/reference/components/generated/transforms.cue index a7bee15d2958e..14181061b6fdd 100644 --- a/website/cue/reference/components/generated/transforms.cue +++ b/website/cue/reference/components/generated/transforms.cue @@ -8,23 +8,58 @@ generated: components: transforms: configuration: { Configure output for component when generated with graph command """ required: false - type: object: options: node_attributes: { - description: """ - Node attributes to add to this component's node in resulting graph + type: object: options: { + edge_attributes: { + description: """ + Edge attributes to add to the edges linked to this component's node in resulting graph - They are added to the node as provided - """ - required: false - type: object: { - examples: [{ - color: "red" - name: "Example Node" - width: "5.0" - }] - options: "*": { - description: "A single graph node attribute in graphviz DOT language." - required: true - type: string: {} + They are added to the edge as provided + """ + required: false + type: object: { + examples: [{ + example_input: { + color: "red" + label: "Example Edge" + width: "5.0" + } + }] + options: "*": { + description: "A collection of graph edge attributes in graphviz DOT language, related to a single input component." + required: true + type: object: { + examples: [{ + color: "red" + label: "Example Edge" + width: "5.0" + }] + options: "*": { + description: "A single graph edge attribute in graphviz DOT language." + required: true + type: string: {} + } + } + } + } + } + node_attributes: { + description: """ + Node attributes to add to this component's node in resulting graph + + They are added to the node as provided + """ + required: false + type: object: { + examples: [{ + color: "red" + name: "Example Node" + width: "5.0" + }] + options: "*": { + description: "A single graph node attribute in graphviz DOT language." + required: true + type: string: {} + } } } } diff --git a/website/cue/reference/generated/configuration.cue b/website/cue/reference/generated/configuration.cue index 0dbca1b677111..5b98f0c6fbe35 100644 --- a/website/cue/reference/generated/configuration.cue +++ b/website/cue/reference/generated/configuration.cue @@ -53,25 +53,60 @@ generated: configuration: { type: object: options: "*": { type: object: options: { graph: { - type: object: options: node_attributes: { - type: object: { - options: "*": { - type: string: {} - required: true - description: "A single graph node attribute in graphviz DOT language." + type: object: options: { + edge_attributes: { + type: object: { + options: "*": { + type: object: { + options: "*": { + type: string: {} + required: true + description: "A single graph edge attribute in graphviz DOT language." + } + examples: [{ + color: "red" + label: "Example Edge" + width: "5.0" + }] + } + description: "A collection of graph edge attributes in graphviz DOT language, related to a single input component." + required: true + } + examples: [{ + example_input: { + color: "red" + label: "Example Edge" + width: "5.0" + } + }] } - examples: [{ - color: "red" - name: "Example Node" - width: "5.0" - }] + description: """ + Edge attributes to add to the edges linked to this component's node in resulting graph + + They are added to the edge as provided + """ + required: false } - description: """ + node_attributes: { + type: object: { + options: "*": { + type: string: {} + required: true + description: "A single graph node attribute in graphviz DOT language." + } + examples: [{ + color: "red" + name: "Example Node" + width: "5.0" + }] + } + description: """ Node attributes to add to this component's node in resulting graph They are added to the node as provided """ - required: false + required: false + } } description: """ Extra graph configuration @@ -855,25 +890,60 @@ generated: configuration: { required: false } graph: { - type: object: options: node_attributes: { - type: object: { - options: "*": { - type: string: {} - required: true - description: "A single graph node attribute in graphviz DOT language." + type: object: options: { + edge_attributes: { + type: object: { + options: "*": { + type: object: { + options: "*": { + type: string: {} + required: true + description: "A single graph edge attribute in graphviz DOT language." + } + examples: [{ + color: "red" + label: "Example Edge" + width: "5.0" + }] + } + description: "A collection of graph edge attributes in graphviz DOT language, related to a single input component." + required: true + } + examples: [{ + example_input: { + color: "red" + label: "Example Edge" + width: "5.0" + } + }] } - examples: [{ - color: "red" - name: "Example Node" - width: "5.0" - }] + description: """ + Edge attributes to add to the edges linked to this component's node in resulting graph + + They are added to the edge as provided + """ + required: false } - description: """ + node_attributes: { + type: object: { + options: "*": { + type: string: {} + required: true + description: "A single graph node attribute in graphviz DOT language." + } + examples: [{ + color: "red" + name: "Example Node" + width: "5.0" + }] + } + description: """ Node attributes to add to this component's node in resulting graph They are added to the node as provided """ - required: false + required: false + } } description: """ Extra graph configuration @@ -996,25 +1066,60 @@ generated: configuration: { type: object: options: "*": { type: object: options: { graph: { - type: object: options: node_attributes: { - type: object: { - options: "*": { - type: string: {} - required: true - description: "A single graph node attribute in graphviz DOT language." + type: object: options: { + edge_attributes: { + type: object: { + options: "*": { + type: object: { + options: "*": { + type: string: {} + required: true + description: "A single graph edge attribute in graphviz DOT language." + } + examples: [{ + color: "red" + label: "Example Edge" + width: "5.0" + }] + } + description: "A collection of graph edge attributes in graphviz DOT language, related to a single input component." + required: true + } + examples: [{ + example_input: { + color: "red" + label: "Example Edge" + width: "5.0" + } + }] } - examples: [{ - color: "red" - name: "Example Node" - width: "5.0" - }] + description: """ + Edge attributes to add to the edges linked to this component's node in resulting graph + + They are added to the edge as provided + """ + required: false } - description: """ + node_attributes: { + type: object: { + options: "*": { + type: string: {} + required: true + description: "A single graph node attribute in graphviz DOT language." + } + examples: [{ + color: "red" + name: "Example Node" + width: "5.0" + }] + } + description: """ Node attributes to add to this component's node in resulting graph They are added to the node as provided """ - required: false + required: false + } } description: """ Extra graph configuration @@ -1093,25 +1198,60 @@ generated: configuration: { type: object: options: "*": { type: object: options: { graph: { - type: object: options: node_attributes: { - type: object: { - options: "*": { - type: string: {} - required: true - description: "A single graph node attribute in graphviz DOT language." + type: object: options: { + edge_attributes: { + type: object: { + options: "*": { + type: object: { + options: "*": { + type: string: {} + required: true + description: "A single graph edge attribute in graphviz DOT language." + } + examples: [{ + color: "red" + label: "Example Edge" + width: "5.0" + }] + } + description: "A collection of graph edge attributes in graphviz DOT language, related to a single input component." + required: true + } + examples: [{ + example_input: { + color: "red" + label: "Example Edge" + width: "5.0" + } + }] } - examples: [{ - color: "red" - name: "Example Node" - width: "5.0" - }] + description: """ + Edge attributes to add to the edges linked to this component's node in resulting graph + + They are added to the edge as provided + """ + required: false } - description: """ + node_attributes: { + type: object: { + options: "*": { + type: string: {} + required: true + description: "A single graph node attribute in graphviz DOT language." + } + examples: [{ + color: "red" + name: "Example Node" + width: "5.0" + }] + } + description: """ Node attributes to add to this component's node in resulting graph They are added to the node as provided """ - required: false + required: false + } } description: """ Extra graph configuration From a8f3a9b45e6d2f6a897995bce85e3ad9a9a13442 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ensar=20Saraj=C4=8Di=C4=87?= Date: Wed, 18 Mar 2026 19:39:25 +0100 Subject: [PATCH 027/326] enhancement(dnstap source): expose message size when parsing dnstap data (#24552) * enhancement(dnstap source): expose raw message size when parsing dnstap data Closes: #24545 * Add changelog entry * Change message size from frame size to DNS message size * Split `messageSize` into `requestMessageSize` and `responseMessageSize` --- ..._dnstap_parser_message_size.enhancement.md | 3 ++ lib/vector-vrl/dnstap-parser/src/parser.rs | 28 +++++++++++++++++++ lib/vector-vrl/dnstap-parser/src/schema.rs | 14 ++++++++++ .../src/vrl_functions/parse_dnstap.rs | 6 ++++ 4 files changed, 51 insertions(+) create mode 100644 changelog.d/24552_dnstap_parser_message_size.enhancement.md diff --git a/changelog.d/24552_dnstap_parser_message_size.enhancement.md b/changelog.d/24552_dnstap_parser_message_size.enhancement.md new file mode 100644 index 0000000000000..9472bde7f3346 --- /dev/null +++ b/changelog.d/24552_dnstap_parser_message_size.enhancement.md @@ -0,0 +1,3 @@ +Adds new fields to parsed dnstap data: `requestMessageSize` and `responseMessageSize`. It represents the size of the DNS message. + +authors: esensar Quad9DNS diff --git a/lib/vector-vrl/dnstap-parser/src/parser.rs b/lib/vector-vrl/dnstap-parser/src/parser.rs index 3876ff363bec6..5e14303c4a5a2 100644 --- a/lib/vector-vrl/dnstap-parser/src/parser.rs +++ b/lib/vector-vrl/dnstap-parser/src/parser.rs @@ -260,6 +260,13 @@ impl DnstapParser { match dnstap_message_type_id { 1..=12 => { if let Some(query_message) = dnstap_message.query_message { + DnstapParser::insert( + event, + prefix.clone(), + &DNSTAP_VALUE_PATHS.request_message_size, + query_message.len(), + ); + let mut query_message_parser = DnsMessageParser::with_options(query_message, parsing_options.clone()); if let Err(error) = DnstapParser::parse_dns_query_message( @@ -278,6 +285,12 @@ impl DnstapParser { } if let Some(response_message) = dnstap_message.response_message { + DnstapParser::insert( + event, + prefix.clone(), + &DNSTAP_VALUE_PATHS.response_message_size, + response_message.len(), + ); let mut response_message_parser = DnsMessageParser::with_options(response_message, parsing_options); if let Err(error) = DnstapParser::parse_dns_query_message( @@ -297,6 +310,12 @@ impl DnstapParser { } 13 | 14 => { if let Some(update_request_message) = dnstap_message.query_message { + DnstapParser::insert( + event, + prefix.clone(), + &DNSTAP_VALUE_PATHS.request_message_size, + update_request_message.len(), + ); let mut update_request_message_parser = DnsMessageParser::with_options( update_request_message, parsing_options.clone(), @@ -317,6 +336,12 @@ impl DnstapParser { } if let Some(update_response_message) = dnstap_message.response_message { + DnstapParser::insert( + event, + prefix.clone(), + &DNSTAP_VALUE_PATHS.response_message_size, + update_response_message.len(), + ); let mut update_response_message_parser = DnsMessageParser::with_options(update_response_message, parsing_options); if let Err(error) = DnstapParser::parse_dns_update_message( @@ -1060,6 +1085,7 @@ mod tests { ("dataTypeId", Value::Integer(1)), ("messageType", Value::Bytes(Bytes::from("ResolverQuery"))), ("messageTypeId", Value::Integer(3)), + ("requestMessageSize", Value::Integer(54)), ("queryZone", Value::Bytes(Bytes::from("com."))), ("requestData.fullRcode", Value::Integer(0)), ("requestData.header.aa", Value::Boolean(false)), @@ -1256,6 +1282,8 @@ mod tests { ("dataTypeId", Value::Integer(1)), ("messageType", Value::Bytes(Bytes::from("UpdateResponse"))), ("messageTypeId", Value::Integer(14)), + ("requestMessageSize", Value::Integer(29)), + ("responseMessageSize", Value::Integer(29)), ("requestData.fullRcode", Value::Integer(0)), ("requestData.header.adCount", Value::Integer(0)), ("requestData.header.id", Value::Integer(28811)), diff --git a/lib/vector-vrl/dnstap-parser/src/schema.rs b/lib/vector-vrl/dnstap-parser/src/schema.rs index 29a1b861f6e69..c2fea507d60e1 100644 --- a/lib/vector-vrl/dnstap-parser/src/schema.rs +++ b/lib/vector-vrl/dnstap-parser/src/schema.rs @@ -174,6 +174,16 @@ impl DnstapEventSchema { Kind::object(self.request_message_schema_definition()), None, ) + .optional_field( + &DNSTAP_VALUE_PATHS.request_message_size, + Kind::integer(), + None, + ) + .optional_field( + &DNSTAP_VALUE_PATHS.response_message_size, + Kind::integer(), + None, + ) } /// The schema definition for a dns tap message. @@ -211,6 +221,8 @@ pub struct DnstapPaths { pub message_type_id: OwnedValuePath, pub request_message: OwnedValuePath, pub response_message: OwnedValuePath, + pub request_message_size: OwnedValuePath, + pub response_message_size: OwnedValuePath, // DnsQueryMessageSchema pub response_code: OwnedValuePath, @@ -311,6 +323,8 @@ pub static DNSTAP_VALUE_PATHS: LazyLock = LazyLock::new(|| DnstapPa message_type_id: owned_value_path!("messageTypeId"), request_message: owned_value_path!("requestData"), response_message: owned_value_path!("responseData"), + request_message_size: owned_value_path!("requestMessageSize"), + response_message_size: owned_value_path!("responseMessageSize"), response_code: owned_value_path!("fullRcode"), response: owned_value_path!("rcodeName"), header: owned_value_path!("header"), diff --git a/lib/vector-vrl/dnstap-parser/src/vrl_functions/parse_dnstap.rs b/lib/vector-vrl/dnstap-parser/src/vrl_functions/parse_dnstap.rs index 8c0a5d07f35e7..b534226b68cb7 100644 --- a/lib/vector-vrl/dnstap-parser/src/vrl_functions/parse_dnstap.rs +++ b/lib/vector-vrl/dnstap-parser/src/vrl_functions/parse_dnstap.rs @@ -67,6 +67,8 @@ impl Function for ParseDnstap { "extraInfo": "", "messageType": "ResolverQuery", "messageTypeId": 3, + "requestMessageSize": 54, + "responseMessageSize": 100, "queryZone": "com.", "requestData": { "fullRcode": 0, @@ -244,6 +246,8 @@ mod tests { extraInfo: "", messageType: "ResolverQuery", messageTypeId: 3, + requestMessageSize: 54, + responseMessageSize: 100, queryZone: "com.", requestData: { fullRcode: 0, @@ -358,6 +362,8 @@ mod tests { dataTypeId: 1, messageType: "UpdateResponse", messageTypeId: 14, + requestMessageSize: 29, + responseMessageSize: 29, requestData: { fullRcode: 0, header: { From f34e8d32140ab25024ff51863ed9b5038defdc80 Mon Sep 17 00:00:00 2001 From: Thomas Date: Thu, 19 Mar 2026 09:13:57 -0400 Subject: [PATCH 028/326] docs(dnstap source): regenerate VRL docs with message size fields (#24960) --- docs/generated/parse_dnstap.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/generated/parse_dnstap.json b/docs/generated/parse_dnstap.json index 6bc7356fe5201..c30eb26893e02 100644 --- a/docs/generated/parse_dnstap.json +++ b/docs/generated/parse_dnstap.json @@ -41,6 +41,8 @@ "extraInfo": "", "messageType": "ResolverQuery", "messageTypeId": 3, + "requestMessageSize": 54, + "responseMessageSize": 100, "queryZone": "com.", "requestData": { "fullRcode": 0, From 483db702e3745f4b823edac33e1d76bf7060f964 Mon Sep 17 00:00:00 2001 From: Vladimir Zhuk Date: Thu, 19 Mar 2026 14:16:35 +0100 Subject: [PATCH 029/326] fix(releasing): add libsasl2-2 runtime dep to Debian-based images (#24962) fix(docker): add libsasl2-2 runtime dep to Debian-based images After fd766a073 switched rdkafka from gssapi-vendored (static) to gssapi (dynamic linking), the vector binary now requires libsasl2.so.2 at runtime. The Debian final stages in both the distribution and regression Dockerfiles were missing this package, causing the smoke-test RUN to fail with: /usr/bin/vector: error while loading shared libraries: libsasl2.so.2 Co-authored-by: Claude Sonnet 4.6 --- distribution/docker/debian/Dockerfile | 2 +- regression/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/distribution/docker/debian/Dockerfile b/distribution/docker/debian/Dockerfile index 095989978231a..a726f8a72004c 100644 --- a/distribution/docker/debian/Dockerfile +++ b/distribution/docker/debian/Dockerfile @@ -16,7 +16,7 @@ LABEL org.opencontainers.image.documentation="https://vector.dev/docs" # we want the latest versions of these # hadolint ignore=DL3008 -RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates tzdata systemd && rm -rf /var/lib/apt/lists/* +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates tzdata systemd libsasl2-2 && rm -rf /var/lib/apt/lists/* COPY --from=builder /usr/bin/vector /usr/bin/vector COPY --from=builder /usr/share/vector /usr/share/vector diff --git a/regression/Dockerfile b/regression/Dockerfile index dc1a037e47636..98b03547af9f7 100644 --- a/regression/Dockerfile +++ b/regression/Dockerfile @@ -14,7 +14,7 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \ # TARGET # FROM docker.io/debian:trixie-slim@sha256:c85a2732e97694ea77237c61304b3bb410e0e961dd6ee945997a06c788c545bb -RUN apt-get update && apt-get dist-upgrade -y && apt-get -y --no-install-recommends install zlib1g ca-certificates && rm -rf /var/lib/apt/lists/* +RUN apt-get update && apt-get dist-upgrade -y && apt-get -y --no-install-recommends install zlib1g ca-certificates libsasl2-2 && rm -rf /var/lib/apt/lists/* COPY --from=builder /vector/vector /usr/bin/vector RUN mkdir --parents --mode=0777 /var/lib/vector From 5f0f03e181a031ee51a71e2bd9fcf0698f073a5b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 19 Mar 2026 15:13:14 +0000 Subject: [PATCH 030/326] chore(deps): bump fakedata_generator from 0.5.0 to 0.7.1 (#24800) * chore(deps): bump fakedata_generator from 0.5.0 to 0.7.1 Bumps [fakedata_generator](https://github.com/KevinGimbel/fakedata_generator) from 0.5.0 to 0.7.1. - [Changelog](https://github.com/KevinGimbel/fakedata_generator/blob/main/CHANGELOG.md) - [Commits](https://github.com/KevinGimbel/fakedata_generator/commits/v0.7.1) --- updated-dependencies: - dependency-name: fakedata_generator dependency-version: 0.7.1 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * update licenses * Update generated VRL docs --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Pavlos Rontidis Co-authored-by: github-actions[bot] --- Cargo.lock | 6 +++--- LICENSE-3rdparty.csv | 2 +- lib/fakedata/Cargo.toml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f19fbc7c50f2c..925ecad0423d2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3998,12 +3998,12 @@ dependencies = [ [[package]] name = "fakedata_generator" -version = "0.5.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57b82fba4b485b819fde74012109688a9d2bd4ce7b22583ac12c9fa239f74a02" +checksum = "b42ec18312c068f0d6008ad66ad2fa289ab40a16d07a06406c200baa09eb1e7c" dependencies = [ "passt", - "rand 0.8.5", + "rand 0.9.2", "serde", "serde_json", ] diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 2079e5d6819ba..bb490b1376fbe 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -276,7 +276,7 @@ event-listener-strategy,https://github.com/smol-rs/event-listener-strategy,Apach evmap,https://github.com/jonhoo/rust-evmap,MIT OR Apache-2.0,Jon Gjengset evmap-derive,https://github.com/jonhoo/rust-evmap,MIT OR Apache-2.0,Jon Gjengset exitcode,https://github.com/benwilber/exitcode,Apache-2.0,Ben Wilber -fakedata_generator,https://github.com/kevingimbel/fakedata_generator,MIT,Kevin Gimbel +fakedata_generator,https://github.com/KevinGimbel/fakedata_generator,MIT,Kevin Gimbel fallible-iterator,https://github.com/sfackler/rust-fallible-iterator,MIT OR Apache-2.0,Steven Fackler fancy-regex,https://github.com/fancy-regex/fancy-regex,MIT,"Raph Levien , Robin Stocker , Keith Hall " fastrand,https://github.com/smol-rs/fastrand,Apache-2.0 OR MIT,Stjepan Glavina diff --git a/lib/fakedata/Cargo.toml b/lib/fakedata/Cargo.toml index 708086e24002c..576ad22e96cad 100644 --- a/lib/fakedata/Cargo.toml +++ b/lib/fakedata/Cargo.toml @@ -8,5 +8,5 @@ license = "MPL-2.0" [dependencies] chrono.workspace = true -fakedata_generator = "0.5.0" +fakedata_generator = "0.7.1" rand.workspace = true From 3c69a8b8a668c6d20ea57d6883a3771803893648 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Thu, 19 Mar 2026 11:43:24 -0400 Subject: [PATCH 031/326] chore(external docs): instruct agents to use PR template when creating PRs (#24965) * docs(agents): instruct agents to use PR template when creating PRs Co-Authored-By: Claude Sonnet 4.6 * docs(agents): simplify PR template instruction to avoid duplication Co-Authored-By: Claude Sonnet 4.6 * docs(agents): improve PR template instruction wording Co-Authored-By: Claude Sonnet 4.6 --------- Co-authored-by: Claude Sonnet 4.6 --- AGENTS.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index 24fd23e790e76..63460acdae3e9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -254,6 +254,10 @@ cargo install dd-rust-license-tool --locked make build-licenses ``` +## Creating Pull Requests + +Before opening a PR, read [`.github/PULL_REQUEST_TEMPLATE.md`](.github/PULL_REQUEST_TEMPLATE.md) and use it as the reference for the PR body structure and title. + ## Reference Documentation These documents provide context that AI agents and developers need when working on Vector code. From f02424389cfa4abf8e1168b5b63fb227499c9bd4 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Thu, 19 Mar 2026 11:54:48 -0400 Subject: [PATCH 032/326] feat(ci): add Datadog static analysis workflow (#24966) Co-authored-by: Claude Sonnet 4.6 --- .github/workflows/static-analysis.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 .github/workflows/static-analysis.yml diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml new file mode 100644 index 0000000000000..d3330f07b7280 --- /dev/null +++ b/.github/workflows/static-analysis.yml @@ -0,0 +1,26 @@ +name: Static Analysis + +on: + push: + branches: + - master + - "**" # Run on all branches (includes PR branches) + workflow_dispatch: # Allow manual trigger from GitHub UI + +permissions: + contents: read + +jobs: + static-analysis: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Datadog Static Analyzer + if: ${{ secrets.DD_API_KEY != '' }} + uses: DataDog/datadog-static-analyzer-github-action@8340f18875fcefca86844b5f947ce2431387e552 # v3.0.0 + with: + dd_api_key: ${{ secrets.DD_API_KEY }} + dd_app_key: ${{ secrets.DD_APP_KEY }} + dd_site: datadoghq.com + secrets_enabled: true From a7fae1f7ad17b51e3960850300b45955ac0952e0 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Thu, 19 Mar 2026 12:27:12 -0400 Subject: [PATCH 033/326] feat(ci): add code coverage workflow with Datadog upload (#24964) * feat(ci): add code coverage workflow with Datadog upload - Add coverage.yml generating lcov coverage with cargo-llvm-cov and uploading to Datadog via datadog-ci (runs on master pushes and weekly) - Add cargo-llvm-cov (v0.8.4) to prepare.sh and the setup composite action; llvm-tools-preview is installed automatically inside maybe_install_cargo_tool when cargo-llvm-cov is requested Co-Authored-By: Claude Sonnet 4.6 * chore(ci): remove weekly schedule from coverage workflow Runs on every master push so the schedule is redundant. Co-Authored-By: Claude Sonnet 4.6 --------- Co-authored-by: Claude Sonnet 4.6 --- .github/actions/setup/action.yml | 7 +++++++ .github/workflows/coverage.yml | 36 ++++++++++++++++++++++++++++++++ scripts/environment/prepare.sh | 13 ++++++++++-- 3 files changed, 54 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/coverage.yml diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 3d70368bac027..379afd232341c 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -58,6 +58,10 @@ inputs: required: false default: false description: "Install cargo-hack for feature compilation checks." + cargo-llvm-cov: + required: false + default: false + description: "Install cargo-llvm-cov for code coverage." dd-rust-license-tool: required: false default: false @@ -111,6 +115,7 @@ runs: "${{ inputs.cargo-deny }}" \ "${{ inputs.cargo-msrv }}" \ "${{ inputs.cargo-hack }}" \ + "${{ inputs.cargo-llvm-cov }}" \ "${{ inputs.dd-rust-license-tool }}" \ "${{ inputs.wasm-pack }}"; do if [[ "$val" == "true" ]]; then @@ -259,6 +264,7 @@ runs: ~/.cargo/bin/cargo-deny ~/.cargo/bin/cargo-msrv ~/.cargo/bin/cargo-hack + ~/.cargo/bin/cargo-llvm-cov ~/.cargo/bin/dd-rust-license-tool ~/.cargo/bin/wasm-pack /usr/local/bin/markdownlint @@ -278,6 +284,7 @@ runs: [[ "${{ inputs.cargo-deny }}" == "true" ]] && mods+=("cargo-deny") [[ "${{ inputs.cargo-msrv }}" == "true" ]] && mods+=("cargo-msrv") [[ "${{ inputs.cargo-hack }}" == "true" ]] && mods+=("cargo-hack") + [[ "${{ inputs.cargo-llvm-cov }}" == "true" ]] && mods+=("cargo-llvm-cov") [[ "${{ inputs.dd-rust-license-tool }}" == "true" ]] && mods+=("dd-rust-license-tool") [[ "${{ inputs.wasm-pack }}" == "true" ]] && mods+=("wasm-pack") [[ "${{ inputs.markdownlint }}" == "true" ]] && mods+=("markdownlint") diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml new file mode 100644 index 0000000000000..7afeaa70f1c35 --- /dev/null +++ b/.github/workflows/coverage.yml @@ -0,0 +1,36 @@ +name: Code Coverage + +on: + push: + branches: + - master + workflow_dispatch: # Allow manual trigger from GitHub UI + +permissions: + contents: read + +env: + CI: true + CARGO_INCREMENTAL: '0' # Disable incremental compilation for coverage + +jobs: + coverage: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - uses: ./.github/actions/setup + with: + rust: true + cargo-llvm-cov: true + datadog-ci: true + + - name: "Generate code coverage" + run: cargo llvm-cov --all-features --workspace --lcov --output-path lcov.info + + - name: "Upload coverage to Datadog" + env: + DD_API_KEY: ${{ secrets.DD_API_KEY }} + DD_SITE: datadoghq.com + DD_ENV: ci + run: datadog-ci coverage upload lcov.info diff --git a/scripts/environment/prepare.sh b/scripts/environment/prepare.sh index a06f50844c353..5bd5a29a79ac2 100755 --- a/scripts/environment/prepare.sh +++ b/scripts/environment/prepare.sh @@ -35,6 +35,7 @@ CARGO_DENY_VERSION="0.19.0" CARGO_MSRV_VERSION="0.18.4" CARGO_HACK_VERSION="0.6.43" DD_RUST_LICENSE_TOOL_VERSION="1.0.6" +CARGO_LLVM_COV_VERSION="0.8.4" WASM_PACK_VERSION="0.13.1" MARKDOWNLINT_VERSION="0.45.0" DATADOG_CI_VERSION="5.9.0" @@ -48,6 +49,7 @@ ALL_MODULES=( cargo-deny cargo-msrv cargo-hack + cargo-llvm-cov dd-rust-license-tool wasm-pack markdownlint @@ -87,6 +89,7 @@ Modules: cargo-deny cargo-msrv cargo-hack + cargo-llvm-cov dd-rust-license-tool wasm-pack markdownlint @@ -138,6 +141,11 @@ maybe_install_cargo_tool() { if ! $version_cmd --version 2>/dev/null | grep -q "^${version_pattern}"; then cargo "${install[@]}" "$tool" --version "$version" --force --locked fi + + # cargo-llvm-cov requires the llvm-tools-preview rustup component + if [[ "$tool" == "cargo-llvm-cov" ]]; then + rustup component add llvm-tools-preview + fi } # Helper for NPM packages @@ -161,8 +169,8 @@ maybe_install_npm_package() { # Always ensure git safe.directory is set git config --global --add safe.directory "$(pwd)" -REQUIRES_RUSTUP=(dd-rust-license-tool cargo-deb cross cargo-nextest cargo-deny cargo-msrv cargo-hack wasm-pack vdev) -REQUIRES_BINSTALL=(cargo-deb cross cargo-nextest cargo-deny cargo-msrv cargo-hack wasm-pack vdev) +REQUIRES_RUSTUP=(dd-rust-license-tool cargo-deb cross cargo-nextest cargo-deny cargo-msrv cargo-hack cargo-llvm-cov wasm-pack vdev) +REQUIRES_BINSTALL=(cargo-deb cross cargo-nextest cargo-deny cargo-msrv cargo-hack cargo-llvm-cov wasm-pack vdev) require_binstall=false for tool in "${REQUIRES_BINSTALL[@]}"; do @@ -205,6 +213,7 @@ maybe_install_cargo_tool cargo-nextest "${CARGO_NEXTEST_VERSION}" maybe_install_cargo_tool cargo-deny "${CARGO_DENY_VERSION}" maybe_install_cargo_tool cargo-msrv "${CARGO_MSRV_VERSION}" maybe_install_cargo_tool cargo-hack "${CARGO_HACK_VERSION}" +maybe_install_cargo_tool cargo-llvm-cov "${CARGO_LLVM_COV_VERSION}" maybe_install_cargo_tool dd-rust-license-tool "${DD_RUST_LICENSE_TOOL_VERSION}" maybe_install_cargo_tool wasm-pack "${WASM_PACK_VERSION}" maybe_install_cargo_tool vdev "${VDEV_VERSION}" From 37a48ae53eb0e5930f4f4506903eb5b10c613789 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 19 Mar 2026 21:44:04 +0000 Subject: [PATCH 034/326] chore(deps): bump cargo-lock from 10.1.0 to 11.0.1 (#24748) Bumps [cargo-lock](https://github.com/rustsec/rustsec) from 10.1.0 to 11.0.1. - [Release notes](https://github.com/rustsec/rustsec/releases) - [Commits](https://github.com/rustsec/rustsec/commits/cargo-lock/v11.0.1) --- updated-dependencies: - dependency-name: cargo-lock dependency-version: 11.0.1 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 46 ++++++------------------ lib/vector-vrl/web-playground/Cargo.toml | 2 +- 2 files changed, 11 insertions(+), 37 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 925ecad0423d2..5efa8b3da18ea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2227,13 +2227,13 @@ checksum = "6bd91ee7b2422bcb158d90ef4d14f75ef67f340943fc4149891dcce8f8b972a3" [[package]] name = "cargo-lock" -version = "10.1.0" +version = "11.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06acb4f71407ba205a07cb453211e0e6a67b21904e47f6ba1f9589e38f2e454" +checksum = "63585cdf8572aa7adf0e30a253f988f2b77233bfac1973d52efb6dd53a75920e" dependencies = [ "semver", "serde", - "toml 0.8.23", + "toml", "url", ] @@ -2550,7 +2550,7 @@ dependencies = [ "syslog_loose 0.23.0", "tokio", "tokio-util", - "toml 0.9.8", + "toml", "tracing 0.1.44", "tracing-test", "uuid", @@ -9899,7 +9899,7 @@ version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fc44799282f511a5d403d72a4ff028dc2c87f7fe6830abe3c33bb2fa6dfccec" dependencies = [ - "toml 0.9.8", + "toml", ] [[package]] @@ -10006,15 +10006,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "serde_spanned" -version = "0.6.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" -dependencies = [ - "serde", -] - [[package]] name = "serde_spanned" version = "1.0.3" @@ -11454,18 +11445,6 @@ dependencies = [ "webpki-roots 0.26.1", ] -[[package]] -name = "toml" -version = "0.8.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" -dependencies = [ - "serde", - "serde_spanned 0.6.9", - "toml_datetime 0.6.11", - "toml_edit 0.22.27", -] - [[package]] name = "toml" version = "0.9.8" @@ -11474,7 +11453,7 @@ checksum = "f0dc8b1fb61449e27716ec0e1bdf0f6b8f3e8f6b05391e8497b8b6d7804ea6d8" dependencies = [ "indexmap 2.12.0", "serde_core", - "serde_spanned 1.0.3", + "serde_spanned", "toml_datetime 0.7.3", "toml_parser", "toml_writer", @@ -11486,9 +11465,6 @@ name = "toml_datetime" version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" -dependencies = [ - "serde", -] [[package]] name = "toml_datetime" @@ -11517,8 +11493,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ "indexmap 2.12.0", - "serde", - "serde_spanned 0.6.9", "toml_datetime 0.6.11", "toml_write", "winnow 0.7.13", @@ -12392,7 +12366,7 @@ dependencies = [ "sha2", "strum 0.28.0", "tempfile", - "toml 0.9.8", + "toml", "toml_edit 0.23.9", "vector-vrl-functions", "vrl", @@ -12579,7 +12553,7 @@ dependencies = [ "tokio-test", "tokio-tungstenite 0.20.1", "tokio-util", - "toml 0.9.8", + "toml", "tonic 0.11.0", "tonic-build 0.11.0", "tonic-health", @@ -12725,7 +12699,7 @@ dependencies = [ "serde_json", "serde_with", "snafu 0.8.9", - "toml 0.9.8", + "toml", "tracing 0.1.44", "url", "vector-config-common", @@ -12825,7 +12799,7 @@ dependencies = [ "tokio-stream", "tokio-test", "tokio-util", - "toml 0.9.8", + "toml", "tonic 0.11.0", "tracing 0.1.44", "tracing-subscriber", diff --git a/lib/vector-vrl/web-playground/Cargo.toml b/lib/vector-vrl/web-playground/Cargo.toml index d84b2c8d69ad4..1c7fd0f360d64 100644 --- a/lib/vector-vrl/web-playground/Cargo.toml +++ b/lib/vector-vrl/web-playground/Cargo.toml @@ -23,4 +23,4 @@ enrichment = { path = "../enrichment" } getrandom = { version = "0.2.15", features = ["js"] } [build-dependencies] -cargo-lock = "10.1.0" +cargo-lock = "11.0.1" From 97dfb6c6a870a162d07827706e85f5ee2fe3b3a9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 20 Mar 2026 09:53:12 -0400 Subject: [PATCH 035/326] chore(deps): bump const-str from 1.0.0 to 1.1.0 (#24815) Bumps [const-str](https://github.com/Nugine/const-str) from 1.0.0 to 1.1.0. - [Release notes](https://github.com/Nugine/const-str/releases) - [Commits](https://github.com/Nugine/const-str/compare/v1.0.0...v1.1.0) --- updated-dependencies: - dependency-name: const-str dependency-version: 1.1.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5efa8b3da18ea..7f5b22746bac2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2766,9 +2766,9 @@ dependencies = [ [[package]] name = "const-str" -version = "1.0.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93e19f68b180ebff43d6d42005c4b5f046c65fcac28369ba8b3beaad633f9ec0" +checksum = "18f12cc9948ed9604230cdddc7c86e270f9401ccbe3c2e98a4378c5e7632212f" [[package]] name = "convert_case" diff --git a/Cargo.toml b/Cargo.toml index b4565050babe5..fc2ba6790f508 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -155,7 +155,7 @@ chrono-tz = { version = "0.10.4", default-features = false, features = ["serde"] clap = { version = "4.5.60", default-features = false, features = ["derive", "error-context", "env", "help", "std", "string", "usage", "wrap_help"] } clap_complete = "4.5.66" colored = { version = "3.1.1", default-features = false } -const-str = { version = "1.0.0", default-features = false } +const-str = { version = "1.1.0", default-features = false } crossbeam-utils = { version = "0.8.21", default-features = false } darling = { version = "0.20.11", default-features = false, features = ["suggestions"] } dashmap = { version = "6.1.0", default-features = false } From 18898e0149f43924a07554303cce3d0a4595eb5b Mon Sep 17 00:00:00 2001 From: Thomas Date: Fri, 20 Mar 2026 13:22:10 -0400 Subject: [PATCH 036/326] chore(deps): bump quickcheck to 1.1.0 and use workspace dep (#24970) * chore(deps): Bump quickcheck from 1.0.3 to 1.1.0 * Update generated VRL docs * Update licenses * chore(deps): enable wasm_js feature for getrandom 0.4 in web-playground --------- Co-authored-by: github-actions[bot] --- Cargo.lock | 222 ++++++++++++++++++++--- Cargo.toml | 3 +- LICENSE-3rdparty.csv | 15 +- lib/file-source-common/Cargo.toml | 2 +- lib/file-source/Cargo.toml | 2 +- lib/vector-buffers/Cargo.toml | 2 +- lib/vector-core/Cargo.toml | 2 +- lib/vector-vrl/web-playground/Cargo.toml | 1 + 8 files changed, 218 insertions(+), 31 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7f5b22746bac2..35f095a19e231 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2042,7 +2042,7 @@ checksum = "77e9d642a7e3a318e37c2c9427b5a6a48aa1ad55dcd986f3034ab2239045a645" dependencies = [ "darling 0.21.3", "ident_case", - "prettyplease 0.2.15", + "prettyplease 0.2.37", "proc-macro2 1.0.106", "quote 1.0.44", "rustversion", @@ -3858,16 +3858,6 @@ dependencies = [ "regex", ] -[[package]] -name = "env_logger" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a19187fea3ac7e84da7dacf48de0c45d63c6a76f9490dae389aead16c243fce3" -dependencies = [ - "log", - "regex", -] - [[package]] name = "env_logger" version = "0.11.6" @@ -4401,8 +4391,24 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "r-efi", + "r-efi 5.3.0", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 6.0.0", + "rand_core 0.10.0", "wasip2", + "wasip3", "wasm-bindgen", ] @@ -5530,6 +5536,12 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + [[package]] name = "ident_case" version = "1.0.1" @@ -5919,7 +5931,7 @@ dependencies = [ name = "k8s-e2e-tests" version = "0.1.0" dependencies = [ - "env_logger 0.11.6", + "env_logger", "futures 0.3.31", "indoc", "k8s-openapi", @@ -6153,6 +6165,12 @@ dependencies = [ "spin 0.5.2", ] +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + [[package]] name = "lexical-core" version = "1.0.6" @@ -8131,9 +8149,9 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.15" +version = "0.2.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2 1.0.106", "syn 2.0.117", @@ -8361,7 +8379,7 @@ dependencies = [ "multimap", "once_cell", "petgraph", - "prettyplease 0.2.15", + "prettyplease 0.2.37", "prost 0.12.6", "prost-types 0.12.6", "regex", @@ -8381,7 +8399,7 @@ dependencies = [ "multimap", "once_cell", "petgraph", - "prettyplease 0.2.15", + "prettyplease 0.2.37", "prost 0.13.5", "prost-types 0.13.5", "regex", @@ -8626,20 +8644,20 @@ dependencies = [ [[package]] name = "quickcheck" -version = "1.0.3" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "588f6378e4dd99458b60ec275b4477add41ce4fa9f64dcba6f15adccb19b50d6" +checksum = "95c589f335db0f6aaa168a7cd27b1fc6920f5e1470c804f814d9cd6e62a0f70b" dependencies = [ - "env_logger 0.8.4", + "env_logger", "log", - "rand 0.8.5", + "rand 0.10.0", ] [[package]] name = "quickcheck_macros" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f71ee38b42f8459a88d3362be6f9b841ad2d5421844f61eb1c59c11bff3ac14a" +checksum = "a9a28b8493dd664c8b171dd944da82d933f7d456b829bfb236738e1fe06c5ba4" dependencies = [ "proc-macro2 1.0.106", "quote 1.0.44", @@ -8729,6 +8747,12 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + [[package]] name = "radium" version = "0.7.0" @@ -8766,6 +8790,16 @@ dependencies = [ "rand_core 0.9.0", ] +[[package]] +name = "rand" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc266eb313df6c5c09c1c7b1fbe2510961e5bcd3add930c1e31f7ed9da0feff8" +dependencies = [ + "getrandom 0.4.2", + "rand_core 0.10.0", +] + [[package]] name = "rand_chacha" version = "0.3.1" @@ -8805,6 +8839,12 @@ dependencies = [ "zerocopy 0.8.16", ] +[[package]] +name = "rand_core" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c8d0fd677905edcbeedbf2edb6494d676f0e98d54d5cf9bda0b061cb8fb8aba" + [[package]] name = "rand_distr" version = "0.5.1" @@ -11614,7 +11654,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be4ef6dd70a610078cb4e338a0f79d06bc759ff1b22d2120c2ff02ae264ba9c2" dependencies = [ - "prettyplease 0.2.15", + "prettyplease 0.2.37", "proc-macro2 1.0.106", "prost-build 0.12.6", "quote 1.0.44", @@ -12980,6 +13020,7 @@ dependencies = [ "cargo-lock", "enrichment", "getrandom 0.2.15", + "getrandom 0.4.2", "gloo-utils", "serde", "vector-vrl-functions", @@ -13198,7 +13239,16 @@ version = "1.0.1+wasi-0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" dependencies = [ - "wit-bindgen", + "wit-bindgen 0.46.0", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen 0.51.0", ] [[package]] @@ -13264,6 +13314,28 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap 2.12.0", + "wasm-encoder", + "wasmparser", +] + [[package]] name = "wasm-streams" version = "0.4.0" @@ -13277,6 +13349,18 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags 2.10.0", + "hashbrown 0.15.2", + "indexmap 2.12.0", + "semver", +] + [[package]] name = "wasmtimer" version = "0.4.3" @@ -13999,6 +14083,94 @@ version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck 0.5.0", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck 0.5.0", + "indexmap 2.12.0", + "prettyplease 0.2.37", + "syn 2.0.117", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease 0.2.37", + "proc-macro2 1.0.106", + "quote 1.0.44", + "syn 2.0.117", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags 2.10.0", + "indexmap 2.12.0", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap 2.12.0", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid 0.2.4", + "wasmparser", +] + [[package]] name = "woothee" version = "0.13.0" diff --git a/Cargo.toml b/Cargo.toml index fc2ba6790f508..1cfdf70ae5530 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -185,6 +185,7 @@ prost = { version = "0.12", default-features = false, features = ["std"] } prost-build = { version = "0.12", default-features = false } prost-reflect = { version = "0.14", features = ["serde"], default-features = false } prost-types = { version = "0.12", default-features = false } +quickcheck = { version = "1.1.0" } rand = { version = "0.9.2", default-features = false, features = ["small_rng", "thread_rng"] } rand_distr = { version = "0.5.1", default-features = false } regex = { version = "1.12.3", default-features = false, features = ["std", "perf"] } @@ -483,7 +484,7 @@ itertools.workspace = true libc.workspace = true mock_instant.workspace = true proptest.workspace = true -quickcheck = "1.0.3" +quickcheck.workspace = true reqwest.workspace = true rstest = { version = "0.26.1" } serial_test.workspace = true diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index bb490b1376fbe..72c0f17e00d3d 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -263,7 +263,8 @@ enum-ordinalize-derive,https://github.com/magiclen/enum-ordinalize,MIT,The enum- enum_dispatch,https://gitlab.com/antonok/enum_dispatch,MIT OR Apache-2.0,Anton Lazarev enumflags2,https://github.com/meithecatte/enumflags2,MIT OR Apache-2.0,"maik klein , Maja Kądziołka " enumflags2_derive,https://github.com/meithecatte/enumflags2,MIT OR Apache-2.0,"maik klein , Maja Kądziołka " -env_logger,https://github.com/env-logger-rs/env_logger,MIT OR Apache-2.0,The Rust Project Developers +env_filter,https://github.com/rust-cli/env_logger,MIT OR Apache-2.0,The env_filter Authors +env_logger,https://github.com/rust-cli/env_logger,MIT OR Apache-2.0,The env_logger Authors equivalent,https://github.com/cuviper/equivalent,Apache-2.0 OR MIT,The equivalent Authors erased-serde,https://github.com/dtolnay/erased-serde,MIT OR Apache-2.0,David Tolnay errno,https://github.com/lambda-fairy/rust-errno,MIT OR Apache-2.0,Chris Wong @@ -382,6 +383,7 @@ icu_properties,https://github.com/unicode-org/icu4x,Unicode-3.0,The ICU4X Projec icu_properties_data,https://github.com/unicode-org/icu4x,Unicode-3.0,The ICU4X Project Developers icu_provider,https://github.com/unicode-org/icu4x,Unicode-3.0,The ICU4X Project Developers icu_provider_macros,https://github.com/unicode-org/icu4x,Unicode-3.0,The ICU4X Project Developers +id-arena,https://github.com/fitzgen/id-arena,MIT OR Apache-2.0,"Nick Fitzgerald , Aleksey Kladov " ident_case,https://github.com/TedDriggs/ident_case,MIT OR Apache-2.0,Ted Driggs idna,https://github.com/servo/rust-url,MIT OR Apache-2.0,The rust-url developers idna_adapter,https://github.com/hsivonen/idna_adapter,Apache-2.0 OR MIT,The rust-url developers @@ -426,6 +428,7 @@ kube-runtime,https://github.com/kube-rs/kube,Apache-2.0,"clux lapin,https://github.com/amqp-rs/lapin,MIT,"Geoffroy Couprie , Marc-Antoine Perennou " lazy_static,https://github.com/rust-lang-nursery/lazy-static.rs,MIT OR Apache-2.0,Marvin Löbel +leb128fmt,https://github.com/bluk/leb128fmt,MIT OR Apache-2.0,Bryant Luk lexical-core,https://github.com/Alexhuszagh/rust-lexical,MIT OR Apache-2.0,Alex Huszagh lexical-parse-float,https://github.com/Alexhuszagh/rust-lexical,MIT OR Apache-2.0,Alex Huszagh lexical-parse-integer,https://github.com/Alexhuszagh/rust-lexical,MIT OR Apache-2.0,Alex Huszagh @@ -618,6 +621,7 @@ radix_trie,https://github.com/michaelsproul/rust_radix_trie,MIT,Michael Sproul < rand,https://github.com/rust-random/rand,MIT OR Apache-2.0,"The Rand Project Developers, The Rust Project Developers" rand_chacha,https://github.com/rust-random/rand,MIT OR Apache-2.0,"The Rand Project Developers, The Rust Project Developers, The CryptoCorrosion Contributors" rand_core,https://github.com/rust-random/rand,MIT OR Apache-2.0,"The Rand Project Developers, The Rust Project Developers" +rand_core,https://github.com/rust-random/rand_core,MIT OR Apache-2.0,The Rand Project Developers rand_distr,https://github.com/rust-random/rand_distr,MIT OR Apache-2.0,The Rand Project Developers rand_xorshift,https://github.com/rust-random/rngs,MIT OR Apache-2.0,"The Rand Project Developers, The Rust Project Developers" ratatui,https://github.com/ratatui/ratatui,MIT,"Florian Dehau , The Ratatui Developers" @@ -863,13 +867,17 @@ want,https://github.com/seanmonstar/want,MIT,Sean McArthur wasi,https://github.com/bytecodealliance/wasi,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,The Cranelift Project Developers wasip2,https://github.com/bytecodealliance/wasi-rs,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,The wasip2 Authors +wasip3,https://github.com/bytecodealliance/wasi-rs,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,The wasip3 Authors wasite,https://github.com/ardaku/wasite,Apache-2.0 OR BSL-1.0 OR MIT,The wasite Authors wasm-bindgen,https://github.com/wasm-bindgen/wasm-bindgen,MIT OR Apache-2.0,The wasm-bindgen Developers wasm-bindgen-futures,https://github.com/rustwasm/wasm-bindgen/tree/master/crates/futures,MIT OR Apache-2.0,The wasm-bindgen Developers wasm-bindgen-macro,https://github.com/wasm-bindgen/wasm-bindgen/tree/master/crates/macro,MIT OR Apache-2.0,The wasm-bindgen Developers wasm-bindgen-macro-support,https://github.com/wasm-bindgen/wasm-bindgen/tree/master/crates/macro-support,MIT OR Apache-2.0,The wasm-bindgen Developers wasm-bindgen-shared,https://github.com/wasm-bindgen/wasm-bindgen/tree/master/crates/shared,MIT OR Apache-2.0,The wasm-bindgen Developers +wasm-encoder,https://github.com/bytecodealliance/wasm-tools/tree/main/crates/wasm-encoder,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,Nick Fitzgerald +wasm-metadata,https://github.com/bytecodealliance/wasm-tools/tree/main/crates/wasm-metadata,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,The wasm-metadata Authors wasm-streams,https://github.com/MattiasBuelens/wasm-streams,MIT OR Apache-2.0,Mattias Buelens +wasmparser,https://github.com/bytecodealliance/wasm-tools/tree/main/crates/wasmparser,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,Yury Delendik wasmtimer,https://github.com/whizsid/wasmtimer-rs,MIT,"WhizSid , Pierre Krieger " web-sys,https://github.com/wasm-bindgen/wasm-bindgen/tree/master/crates/web-sys,MIT OR Apache-2.0,The wasm-bindgen Developers web-time,https://github.com/daxpedda/web-time,MIT OR Apache-2.0,The web-time Authors @@ -913,6 +921,11 @@ windows_x86_64_msvc,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Mi winnow,https://github.com/winnow-rs/winnow,MIT,The winnow Authors winreg,https://github.com/gentoo90/winreg-rs,MIT,Igor Shaula wit-bindgen,https://github.com/bytecodealliance/wit-bindgen,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,Alex Crichton +wit-bindgen-core,https://github.com/bytecodealliance/wit-bindgen,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,Alex Crichton +wit-bindgen-rust,https://github.com/bytecodealliance/wit-bindgen,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,Alex Crichton +wit-bindgen-rust-macro,https://github.com/bytecodealliance/wit-bindgen,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,Alex Crichton +wit-component,https://github.com/bytecodealliance/wasm-tools/tree/main/crates/wit-component,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,Peter Huene +wit-parser,https://github.com/bytecodealliance/wasm-tools/tree/main/crates/wit-parser,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,Alex Crichton woothee,https://github.com/woothee/woothee-rust,Apache-2.0,hhatto write16,https://github.com/hsivonen/write16,Apache-2.0 OR MIT,The write16 Authors writeable,https://github.com/unicode-org/icu4x,Unicode-3.0,The ICU4X Project Developers diff --git a/lib/file-source-common/Cargo.toml b/lib/file-source-common/Cargo.toml index a53c52a27f6b8..e115f3a74bf52 100644 --- a/lib/file-source-common/Cargo.toml +++ b/lib/file-source-common/Cargo.toml @@ -26,6 +26,6 @@ vector-config = { path = "../vector-config", default-features = false } tokio = { workspace = true, features = ["full"] } [dev-dependencies] -quickcheck = "1" +quickcheck.workspace = true tempfile.workspace = true similar-asserts = "1.7.0" diff --git a/lib/file-source/Cargo.toml b/lib/file-source/Cargo.toml index 4b48b35b4f20e..7e8e6b6cd1153 100644 --- a/lib/file-source/Cargo.toml +++ b/lib/file-source/Cargo.toml @@ -25,5 +25,5 @@ async-compression = { version = "0.4.41", features = ["tokio", "gzip"] } [dev-dependencies] tokio = { workspace = true, features = ["full"] } -quickcheck = "1" +quickcheck.workspace = true tempfile.workspace = true diff --git a/lib/vector-buffers/Cargo.toml b/lib/vector-buffers/Cargo.toml index 02b43c29e382f..9f712b573af9b 100644 --- a/lib/vector-buffers/Cargo.toml +++ b/lib/vector-buffers/Cargo.toml @@ -43,7 +43,7 @@ hdrhistogram = "7.5.4" metrics-tracing-context.workspace = true metrics-util = { workspace = true, features = ["debugging"] } proptest = "1.10" -quickcheck = "1.0" +quickcheck.workspace = true rand.workspace = true serde_yaml.workspace = true temp-dir = "0.1.16" diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml index 1147ce8b63e1a..f36a5944c3ea5 100644 --- a/lib/vector-core/Cargo.toml +++ b/lib/vector-core/Cargo.toml @@ -80,7 +80,7 @@ base64 = "0.22.1" chrono-tz.workspace = true criterion = { version = "0.7.0", features = ["html_reports"] } env-test-util = "1.0.1" -quickcheck = "1" +quickcheck.workspace = true quickcheck_macros = "1" proptest = "1.10" similar-asserts = "1.7.0" diff --git a/lib/vector-vrl/web-playground/Cargo.toml b/lib/vector-vrl/web-playground/Cargo.toml index 1c7fd0f360d64..9b3fbd6392576 100644 --- a/lib/vector-vrl/web-playground/Cargo.toml +++ b/lib/vector-vrl/web-playground/Cargo.toml @@ -21,6 +21,7 @@ vector-vrl-functions.workspace = true enrichment = { path = "../enrichment" } # Required per https://docs.rs/getrandom/latest/getrandom/#webassembly-support getrandom = { version = "0.2.15", features = ["js"] } +getrandom_v04 = { package = "getrandom", version = "0.4", features = ["wasm_js"] } [build-dependencies] cargo-lock = "11.0.1" From 2097fc35d7dc8d3caee9607d563165374e312e21 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 20 Mar 2026 19:12:17 +0000 Subject: [PATCH 037/326] chore(deps): bump criterion from 0.7.0 to 0.8.2 (#24803) * chore(deps): bump criterion from 0.7.0 to 0.8.2 Bumps [criterion](https://github.com/criterion-rs/criterion.rs) from 0.7.0 to 0.8.2. - [Release notes](https://github.com/criterion-rs/criterion.rs/releases) - [Changelog](https://github.com/criterion-rs/criterion.rs/blob/master/CHANGELOG.md) - [Commits](https://github.com/criterion-rs/criterion.rs/compare/criterion-plot-v0.7.0...criterion-v0.8.2) --- updated-dependencies: - dependency-name: criterion dependency-version: 0.8.2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * chore(deps): make criterion a workspace dependency --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Thomas --- Cargo.lock | 29 +++++++++++++++++++++++++---- Cargo.toml | 3 ++- lib/dnsmsg-parser/Cargo.toml | 2 +- lib/tracing-limit/Cargo.toml | 2 +- lib/vector-buffers/Cargo.toml | 2 +- lib/vector-core/Cargo.toml | 2 +- 6 files changed, 31 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 35f095a19e231..75e89d42c3280 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -111,6 +111,15 @@ dependencies = [ "alloc-no-stdlib", ] +[[package]] +name = "alloca" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7d05ea6aea7e9e64d25b9156ba2fee3fdd659e34e41063cd2fc7cd020d7f4" +dependencies = [ + "cc", +] + [[package]] name = "allocator-api2" version = "0.2.21" @@ -2920,10 +2929,11 @@ dependencies = [ [[package]] name = "criterion" -version = "0.7.0" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1c047a62b0cc3e145fa84415a3191f628e980b194c2755aa12300a4e6cbd928" +checksum = "950046b2aa2492f9a536f5f4f9a3de7b9e2476e575e05bd6c333371add4d98f3" dependencies = [ + "alloca", "anes", "cast", "ciborium", @@ -2932,6 +2942,7 @@ dependencies = [ "itertools 0.13.0", "num-traits", "oorandom", + "page_size", "plotters", "rayon", "regex", @@ -2944,9 +2955,9 @@ dependencies = [ [[package]] name = "criterion-plot" -version = "0.6.0" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b1bcc0dc7dfae599d84ad0b1a55f80cde8af3725da8313b528da95ef783e338" +checksum = "d8d80a2f4f5b554395e47b5d8305bc3d27813bacb73493eb1001e8f76dae29ea" dependencies = [ "cast", "itertools 0.13.0", @@ -7708,6 +7719,16 @@ dependencies = [ "unicode-width 0.1.13", ] +[[package]] +name = "page_size" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "parking" version = "2.2.0" diff --git a/Cargo.toml b/Cargo.toml index 1cfdf70ae5530..ed8631a6d1e94 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -156,6 +156,7 @@ clap = { version = "4.5.60", default-features = false, features = ["derive", "er clap_complete = "4.5.66" colored = { version = "3.1.1", default-features = false } const-str = { version = "1.1.0", default-features = false } +criterion = "0.8" crossbeam-utils = { version = "0.8.21", default-features = false } darling = { version = "0.20.11", default-features = false, features = ["suggestions"] } dashmap = { version = "6.1.0", default-features = false } @@ -479,7 +480,7 @@ approx = "0.5.1" assert_cmd = { version = "2.0.17", default-features = false } aws-smithy-runtime = { version = "1.8.3", default-features = false, features = ["tls-rustls"] } base64 = "0.22.1" -criterion = { version = "0.7.0", features = ["html_reports", "async_tokio"] } +criterion = { workspace = true, features = ["html_reports", "async_tokio"] } itertools.workspace = true libc.workspace = true mock_instant.workspace = true diff --git a/lib/dnsmsg-parser/Cargo.toml b/lib/dnsmsg-parser/Cargo.toml index 9861bd93718a8..b392579014ef1 100644 --- a/lib/dnsmsg-parser/Cargo.toml +++ b/lib/dnsmsg-parser/Cargo.toml @@ -15,7 +15,7 @@ hickory-proto.workspace = true snafu.workspace = true [dev-dependencies] -criterion = "0.7" +criterion.workspace = true [lib] bench = false diff --git a/lib/tracing-limit/Cargo.toml b/lib/tracing-limit/Cargo.toml index 2308c8ce81084..ba52c12fdea99 100644 --- a/lib/tracing-limit/Cargo.toml +++ b/lib/tracing-limit/Cargo.toml @@ -15,7 +15,7 @@ tracing-subscriber = { workspace = true, features = ["registry", "std"] } dashmap = { version = "6.1.0", default-features = false } [dev-dependencies] -criterion = "0.7" +criterion.workspace = true mock_instant.workspace = true serial_test.workspace = true tracing = "0.1.44" diff --git a/lib/vector-buffers/Cargo.toml b/lib/vector-buffers/Cargo.toml index 9f712b573af9b..04ad96436baf1 100644 --- a/lib/vector-buffers/Cargo.toml +++ b/lib/vector-buffers/Cargo.toml @@ -37,7 +37,7 @@ ordered-float.workspace = true [dev-dependencies] clap.workspace = true -criterion = { version = "0.7", features = ["html_reports", "async_tokio"] } +criterion = { workspace = true, features = ["html_reports", "async_tokio"] } crossbeam-queue = "0.3.12" hdrhistogram = "7.5.4" metrics-tracing-context.workspace = true diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml index f36a5944c3ea5..a7e33377bc09c 100644 --- a/lib/vector-core/Cargo.toml +++ b/lib/vector-core/Cargo.toml @@ -78,7 +78,7 @@ prost-build.workspace = true [dev-dependencies] base64 = "0.22.1" chrono-tz.workspace = true -criterion = { version = "0.7.0", features = ["html_reports"] } +criterion = { workspace = true, features = ["html_reports"] } env-test-util = "1.0.1" quickcheck.workspace = true quickcheck_macros = "1" From 2c334d223578ea17d7ca5bbbf305f21c9b68a7e2 Mon Sep 17 00:00:00 2001 From: Piotr Stalmach <61430212+Azyrys@users.noreply.github.com> Date: Fri, 20 Mar 2026 21:07:16 +0100 Subject: [PATCH 038/326] enhancement(kafka sink): Add support for traces in kafka sink (#24639) * add support for traces in kafka sink * add changelog * add suggestion Made-with: Cursor * fix fmt Made-with: Cursor --- ...9_kafka_sink_add_traces_support.feature.md | 3 + src/sinks/kafka/config.rs | 3 +- src/sinks/kafka/request_builder.rs | 37 ++++- src/sinks/kafka/tests.rs | 129 +++++++++++++++++- 4 files changed, 160 insertions(+), 12 deletions(-) create mode 100644 changelog.d/24639_kafka_sink_add_traces_support.feature.md diff --git a/changelog.d/24639_kafka_sink_add_traces_support.feature.md b/changelog.d/24639_kafka_sink_add_traces_support.feature.md new file mode 100644 index 0000000000000..b89dff7437ff3 --- /dev/null +++ b/changelog.d/24639_kafka_sink_add_traces_support.feature.md @@ -0,0 +1,3 @@ +The `kafka` sink now supports trace events. + +authors: pstalmach diff --git a/src/sinks/kafka/config.rs b/src/sinks/kafka/config.rs index 67af212957d5b..b1e00a3e0c32c 100644 --- a/src/sinks/kafka/config.rs +++ b/src/sinks/kafka/config.rs @@ -288,8 +288,7 @@ impl SinkConfig for KafkaSinkConfig { fn input(&self) -> Input { let requirements = Requirement::empty().optional_meaning("timestamp", Kind::timestamp()); - Input::new(self.encoding.config().input_type() & (DataType::Log | DataType::Metric)) - .with_schema_requirement(requirements) + Input::new(self.encoding.config().input_type()).with_schema_requirement(requirements) } fn acknowledgements(&self) -> &AcknowledgementsConfig { diff --git a/src/sinks/kafka/request_builder.rs b/src/sinks/kafka/request_builder.rs index 23c67a1f40bb0..feb280bbcd864 100644 --- a/src/sinks/kafka/request_builder.rs +++ b/src/sinks/kafka/request_builder.rs @@ -67,28 +67,36 @@ impl RequestBuilder<(String, Event)> for KafkaRequestBuilder { fn get_key(event: &Event, key_field: Option<&OwnedTargetPath>) -> Option { key_field.and_then(|key_field| match event { Event::Log(log) => log.get(key_field).map(|value| value.coerce_to_bytes()), + Event::Trace(trace) => trace.get(key_field).map(|value| value.coerce_to_bytes()), Event::Metric(metric) => metric .tags() .and_then(|tags| tags.get(key_field.to_string().as_str())) .map(|value| value.to_owned().into()), - _ => None, }) } fn get_timestamp_millis(event: &Event) -> Option { match &event { Event::Log(log) => log.get_timestamp().and_then(|v| v.as_timestamp()).copied(), + Event::Trace(trace) => trace + .as_ref() + .get_timestamp() + .and_then(|v| v.as_timestamp()) + .copied(), Event::Metric(metric) => metric.timestamp(), - _ => None, } .map(|ts| ts.timestamp_millis()) } fn get_headers(event: &Event, headers_key: Option<&OwnedTargetPath>) -> Option { headers_key.and_then(|headers_key| { - if let Event::Log(log) = event - && let Some(headers) = log.get(headers_key) - { + let headers = match event { + Event::Log(log) => log.get(headers_key), + Event::Trace(trace) => trace.get(headers_key), + Event::Metric(_) => None, + }; + + if let Some(headers) = headers { match headers { Value::Object(headers_map) => { let mut owned_headers = OwnedHeaders::new_with_capacity(headers_map.len()); @@ -123,7 +131,7 @@ mod tests { use rdkafka::message::Headers; use super::*; - use crate::event::{LogEvent, ObjectMap}; + use crate::event::{LogEvent, ObjectMap, TraceEvent}; #[test] fn kafka_get_headers() { @@ -141,4 +149,21 @@ mod tests { assert_eq!(headers.get(1).key, "b-key"); assert_eq!(headers.get(1).value.unwrap(), "b-value".as_bytes()); } + + #[test] + fn kafka_get_headers_trace() { + let headers_key = OwnedTargetPath::try_from("headers".to_string()).unwrap(); + let mut header_values = ObjectMap::new(); + header_values.insert("a-key".into(), Value::Bytes(Bytes::from("a-value"))); + header_values.insert("b-key".into(), Value::Bytes(Bytes::from("b-value"))); + + let mut event = Event::Trace(TraceEvent::from(LogEvent::from("hello"))); + event.as_mut_trace().insert(&headers_key, header_values); + + let headers = get_headers(&event, Some(&headers_key)).unwrap(); + assert_eq!(headers.get(0).key, "a-key"); + assert_eq!(headers.get(0).value.unwrap(), "a-value".as_bytes()); + assert_eq!(headers.get(1).key, "b-key"); + assert_eq!(headers.get(1).value.unwrap(), "b-value".as_bytes()); + } } diff --git a/src/sinks/kafka/tests.rs b/src/sinks/kafka/tests.rs index c3286ee71ce47..845b17e44061b 100644 --- a/src/sinks/kafka/tests.rs +++ b/src/sinks/kafka/tests.rs @@ -6,14 +6,14 @@ mod integration_test { use std::{collections::HashMap, future::ready, thread, time::Duration}; use bytes::Bytes; - use futures::StreamExt; + use futures::{StreamExt, stream}; use rdkafka::{ Message, Offset, TopicPartitionList, consumer::{BaseConsumer, Consumer}, message::Headers, }; use vector_lib::{ - codecs::TextSerializerConfig, + codecs::{JsonSerializerConfig, TextSerializerConfig}, config::{Tags, Telemetry, init_telemetry}, event::{BatchNotifier, BatchStatus}, lookup::lookup_v2::ConfigTargetPath, @@ -21,7 +21,7 @@ mod integration_test { use super::super::{config::KafkaSinkConfig, sink::KafkaSink, *}; use crate::{ - event::{ObjectMap, Value}, + event::{ObjectMap, TraceEvent, Value}, kafka::{KafkaAuthConfig, KafkaCompression, KafkaSaslConfig}, sinks::prelude::*, test_util::{ @@ -29,7 +29,7 @@ mod integration_test { DATA_VOLUME_SINK_TAGS, SINK_TAGS, assert_data_volume_sink_compliance, assert_sink_compliance, }, - random_lines_with_stream, random_string, wait_for, + map_event_batch_stream, random_lines_with_stream, random_string, wait_for, }, tls::{TEST_PEM_INTERMEDIATE_CA_PATH, TlsConfig, TlsEnableableConfig}, }; @@ -311,6 +311,127 @@ mod integration_test { .await; } + #[tokio::test] + async fn kafka_happy_path_trace_events() { + crate::test_util::trace_init(); + + assert_sink_compliance(&SINK_TAGS, async move { + let topic_prefix = format!("test-trace-{}", random_string(10)); + let topic = format!("{}-{}", topic_prefix, chrono::Utc::now().format("%Y%m%d")); + let key_field = ConfigTargetPath::try_from("trace_key".to_string()).unwrap(); + let headers_key = ConfigTargetPath::try_from("trace_headers".to_string()).unwrap(); + let trace_key = "trace-partition-key"; + let header_key = "trace-header-key"; + let header_value = "trace-header-value"; + + let config = KafkaSinkConfig { + bootstrap_servers: kafka_address(9091), + topic: Template::try_from(format!("{topic_prefix}-%Y%m%d")).unwrap(), + healthcheck_topic: None, + key_field: Some(key_field.clone()), + encoding: JsonSerializerConfig::default().into(), + batch: BatchConfig::default(), + compression: KafkaCompression::None, + auth: KafkaAuthConfig::default(), + socket_timeout_ms: Duration::from_millis(60000), + message_timeout_ms: Duration::from_millis(300000), + rate_limit_duration_secs: 1, + rate_limit_num: i64::MAX as u64, + librdkafka_options: HashMap::new(), + headers_key: Some(headers_key.clone()), + acknowledgements: Default::default(), + }; + + let num_events = 100; + let mut expected_messages = Vec::with_capacity(num_events); + + let (batch, receiver) = BatchNotifier::new_with_receiver(); + let mut events = Vec::with_capacity(num_events); + for i in 0..num_events { + let message = format!("trace-message-{i}"); + expected_messages.push(message.clone()); + + let mut trace = TraceEvent::default(); + trace.insert("message", message); + trace.insert("trace_key", trace_key); + trace.insert("timestamp", chrono::Utc::now()); + + let mut trace_headers = ObjectMap::new(); + trace_headers.insert(header_key.into(), Value::Bytes(Bytes::from(header_value))); + trace.insert(&headers_key, trace_headers); + + events.push(Event::Trace(trace.with_batch_notifier(&batch))); + } + + let sink = KafkaSink::new(config).unwrap(); + let sink = VectorSink::from_event_streamsink(sink); + let stream = map_event_batch_stream(stream::iter(events), Some(batch)); + sink.run(stream).await.unwrap(); + assert_eq!(receiver.await, BatchStatus::Delivered); + + // Read back everything from the beginning. + let mut client_config = rdkafka::ClientConfig::new(); + client_config.set("bootstrap.servers", kafka_address(9091).as_str()); + client_config.set("group.id", random_string(10)); + client_config.set("enable.partition.eof", "true"); + let mut tpl = TopicPartitionList::new(); + tpl.add_partition(&topic, 0) + .set_offset(Offset::Beginning) + .unwrap(); + let consumer: BaseConsumer = client_config.create().unwrap(); + consumer.assign(&tpl).unwrap(); + + wait_for( + || match consumer.fetch_watermarks(&topic, 0, Duration::from_secs(3)) { + Ok((_low, high)) => ready(high >= num_events as i64), + Err(err) => { + println!("retrying due to error fetching watermarks: {err}"); + ready(false) + } + }, + ) + .await; + + let (low, high) = consumer + .fetch_watermarks(&topic, 0, Duration::from_secs(3)) + .unwrap(); + assert_eq!((0, num_events as i64), (low, high)); + + let mut failures = 0; + let mut observed_messages = Vec::new(); + while failures < 100 { + match consumer.poll(Duration::from_secs(3)) { + Some(Ok(msg)) => { + let payload: &str = msg.payload_view().unwrap().unwrap(); + let payload_json: serde_json::Value = + serde_json::from_str(payload).unwrap(); + observed_messages + .push(payload_json["message"].as_str().unwrap().to_owned()); + + let key = msg.key().unwrap(); + assert_eq!(key, trace_key.as_bytes()); + + let timestamp = msg.timestamp().to_millis(); + assert!(timestamp.is_some()); + + let header = msg.headers().unwrap().get(0); + assert_eq!(header.key, header_key); + assert_eq!(header.value.unwrap(), header_value.as_bytes()); + } + None if observed_messages.len() >= num_events => break, + _ => { + failures += 1; + thread::sleep(Duration::from_millis(50)); + } + } + } + + assert_eq!(observed_messages.len(), num_events); + assert_eq!(observed_messages, expected_messages); + }) + .await; + } + async fn kafka_happy_path( server: String, sasl: Option, From c748d6f9e19632cb64f666058794aa2d9004926f Mon Sep 17 00:00:00 2001 From: Thomas Date: Fri, 20 Mar 2026 18:28:58 -0400 Subject: [PATCH 039/326] fix(ci): use cargo nextest in coverage workflow to prevent test pollution (#24973) * fix(ci): use cargo nextest in coverage workflow to prevent test pollution * Add libsasl2 and protoc * fix(ci): drop --all-features from coverage to avoid integration tests --- .github/workflows/coverage.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 7afeaa70f1c35..a4a4c39d4f760 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -22,11 +22,14 @@ jobs: - uses: ./.github/actions/setup with: rust: true + libsasl2: true + protoc: true cargo-llvm-cov: true + cargo-nextest: true datadog-ci: true - name: "Generate code coverage" - run: cargo llvm-cov --all-features --workspace --lcov --output-path lcov.info + run: cargo llvm-cov nextest --workspace --lcov --output-path lcov.info - name: "Upload coverage to Datadog" env: From 4d8ca64dbbbc2434bd4a69a3d46705c3fad6a56c Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 23 Mar 2026 13:53:30 -0400 Subject: [PATCH 040/326] chore(dev): pin localstack image to SHA256 digest (#24988) chore(aws integration): pin localstack image to SHA256 digest --- tests/integration/aws/config/compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/aws/config/compose.yaml b/tests/integration/aws/config/compose.yaml index 202fdfb94e4cd..4fbbd62f6e120 100644 --- a/tests/integration/aws/config/compose.yaml +++ b/tests/integration/aws/config/compose.yaml @@ -4,7 +4,7 @@ services: mock-ec2-metadata: image: public.ecr.aws/aws-ec2/amazon-ec2-metadata-mock:v1.13.0 mock-localstack: - image: docker.io/localstack/localstack:stable + image: docker.io/localstack/localstack@sha256:46302bcb91a7e8008e6394be8afafdbfa40fb77a54d4046a38be35992042d5de environment: - SERVICES=kinesis,s3,cloudwatch,es,firehose,kms,sqs,sns,logs mock-ecs: From 6ef3752d12be1e8de38449df30f953f75301f64f Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 23 Mar 2026 12:52:03 -0400 Subject: [PATCH 041/326] chore(deps): remove futures features and resolve RUSTSEC-2026-0058 (#24975) chore(deps): remove futures compat and io-compat features (RUSTSEC-2026-0058) --- Cargo.lock | 284 +++++++++++++++++++------------------------ Cargo.toml | 2 +- LICENSE-3rdparty.csv | 4 - 3 files changed, 124 insertions(+), 166 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 75e89d42c3280..feb1b17a72aaa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -403,7 +403,7 @@ version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e003216336f70446457e280807a73899dd822feaf02087d31febca1363e2fccc" dependencies = [ - "bytes 1.11.1", + "bytes", "half", "num", ] @@ -680,7 +680,7 @@ dependencies = [ "async-stream", "async-trait", "base64 0.22.1", - "bytes 1.11.1", + "bytes", "chrono", "fnv", "futures-timer", @@ -734,7 +734,7 @@ version = "7.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34ecdaff7c9cffa3614a9f9999bf9ee4c3078fe3ce4d6a6e161736b56febf2de" dependencies = [ - "bytes 1.11.1", + "bytes", "indexmap 2.12.0", "serde", "serde_json", @@ -810,8 +810,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08f6da6d49a956424ca4e28fe93656f790d748b469eaccbc7488fec545315180" dependencies = [ "base64 0.22.1", - "bytes 1.11.1", - "futures 0.3.31", + "bytes", + "futures", "memchr", "nkeys", "nuid", @@ -992,7 +992,7 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", - "bytes 1.11.1", + "bytes", "fastrand", "hex", "http 1.3.1", @@ -1031,7 +1031,7 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", - "bytes 1.11.1", + "bytes", "bytes-utils", "fastrand", "http 0.2.12", @@ -1089,7 +1089,7 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", - "bytes 1.11.1", + "bytes", "fastrand", "http 0.2.12", "http 1.3.1", @@ -1113,7 +1113,7 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", - "bytes 1.11.1", + "bytes", "fastrand", "http 0.2.12", "http 1.3.1", @@ -1137,7 +1137,7 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", - "bytes 1.11.1", + "bytes", "fastrand", "http 0.2.12", "http 1.3.1", @@ -1162,7 +1162,7 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", - "bytes 1.11.1", + "bytes", "fastrand", "http 0.2.12", "http 1.3.1", @@ -1186,7 +1186,7 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", - "bytes 1.11.1", + "bytes", "fastrand", "http 0.2.12", "http 1.3.1", @@ -1214,7 +1214,7 @@ dependencies = [ "aws-smithy-types", "aws-smithy-xml", "aws-types", - "bytes 1.11.1", + "bytes", "fastrand", "hex", "hmac", @@ -1245,7 +1245,7 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", - "bytes 1.11.1", + "bytes", "fastrand", "http 0.2.12", "http 1.3.1", @@ -1294,7 +1294,7 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", - "bytes 1.11.1", + "bytes", "fastrand", "http 0.2.12", "http 1.3.1", @@ -1318,7 +1318,7 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", - "bytes 1.11.1", + "bytes", "fastrand", "http 0.2.12", "http 1.3.1", @@ -1342,7 +1342,7 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", - "bytes 1.11.1", + "bytes", "fastrand", "http 0.2.12", "http 1.3.1", @@ -1386,7 +1386,7 @@ dependencies = [ "aws-smithy-http", "aws-smithy-runtime-api", "aws-smithy-types", - "bytes 1.11.1", + "bytes", "form_urlencoded", "hex", "hmac", @@ -1427,7 +1427,7 @@ checksum = "6750f3dd509b0694a4377f0293ed2f9630d710b1cebe281fa8bac8f099f88bc6" dependencies = [ "aws-smithy-http", "aws-smithy-types", - "bytes 1.11.1", + "bytes", "crc-fast", "hex", "http 1.3.1", @@ -1448,7 +1448,7 @@ checksum = "f77bf78b73a78930daa91cac3974be3ad935fcdf10cc1413a108efb7c79a645a" dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", - "bytes 1.11.1", + "bytes", "flate2", "futures-util", "http 1.3.1", @@ -1465,7 +1465,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf09d74e5e32f76b8762da505a3cd59303e367a664ca67295387baa8c1d7548" dependencies = [ "aws-smithy-types", - "bytes 1.11.1", + "bytes", "crc32fast", ] @@ -1478,7 +1478,7 @@ dependencies = [ "aws-smithy-eventstream", "aws-smithy-runtime-api", "aws-smithy-types", - "bytes 1.11.1", + "bytes", "bytes-utils", "futures-core", "futures-util", @@ -1553,7 +1553,7 @@ dependencies = [ "aws-smithy-observability", "aws-smithy-runtime-api", "aws-smithy-types", - "bytes 1.11.1", + "bytes", "fastrand", "http 0.2.12", "http 1.3.1", @@ -1574,7 +1574,7 @@ checksum = "876ab3c9c29791ba4ba02b780a3049e21ec63dabda09268b175272c3733a79e6" dependencies = [ "aws-smithy-async", "aws-smithy-types", - "bytes 1.11.1", + "bytes", "http 0.2.12", "http 1.3.1", "pin-project-lite", @@ -1590,7 +1590,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2b1117b3b2bbe166d11199b540ceed0d0f7676e36e7b962b5a437a9971eac75" dependencies = [ "base64-simd", - "bytes 1.11.1", + "bytes", "bytes-utils", "futures-core", "http 0.2.12", @@ -1641,7 +1641,7 @@ dependencies = [ "async-trait", "axum-core 0.3.4", "bitflags 1.3.2", - "bytes 1.11.1", + "bytes", "futures-util", "http 0.2.12", "http-body 0.4.6", @@ -1669,7 +1669,7 @@ checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf" dependencies = [ "async-trait", "axum-core 0.4.5", - "bytes 1.11.1", + "bytes", "futures-util", "http 1.3.1", "http-body 1.0.1", @@ -1695,7 +1695,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" dependencies = [ "async-trait", - "bytes 1.11.1", + "bytes", "futures-util", "http 0.2.12", "http-body 0.4.6", @@ -1712,7 +1712,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" dependencies = [ "async-trait", - "bytes 1.11.1", + "bytes", "futures-util", "http 1.3.1", "http-body 1.0.1", @@ -1734,8 +1734,8 @@ dependencies = [ "async-lock 3.4.0", "async-trait", "azure_core_macros", - "bytes 1.11.1", - "futures 0.3.31", + "bytes", + "futures", "openssl", "pin-project", "rustc_version", @@ -1767,7 +1767,7 @@ dependencies = [ "async-lock 3.4.0", "async-trait", "azure_core", - "futures 0.3.31", + "futures", "pin-project", "serde", "time", @@ -1988,7 +1988,7 @@ checksum = "8796b390a5b4c86f9f2e8173a68c2791f4fa6b038b84e96dbc01c016d1e6722c" dependencies = [ "base64 0.22.1", "bollard-stubs", - "bytes 1.11.1", + "bytes", "chrono", "futures-core", "futures-util", @@ -2199,16 +2199,6 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" -[[package]] -name = "bytes" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c" -dependencies = [ - "byteorder", - "iovec", -] - [[package]] name = "bytes" version = "1.11.1" @@ -2224,7 +2214,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e47d3a8076e283f3acd27400535992edb3ba4b5bb72f8891ad8fbe7932a7d4b9" dependencies = [ - "bytes 1.11.1", + "bytes", "either", ] @@ -2527,14 +2517,14 @@ dependencies = [ "apache-avro 0.20.0", "arrow", "async-trait", - "bytes 1.11.1", + "bytes", "chrono", "csv-core", "derivative", "derive_more 2.1.1", "dyn-clone", "flate2", - "futures 0.3.31", + "futures", "indoc", "influxdb-line-protocol", "memchr", @@ -2618,7 +2608,7 @@ version = "4.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35ed6e9d84f0b51a7f52daf1c7d71dd136fd7a3f41a8462b8cdb8c78d920fad4" dependencies = [ - "bytes 1.11.1", + "bytes", "futures-core", "memchr", "pin-project-lite", @@ -3547,7 +3537,7 @@ version = "0.1.0" dependencies = [ "anyhow", "base64 0.22.1", - "bytes 1.11.1", + "bytes", "chrono", "chrono-tz", "dnsmsg-parser", @@ -3599,7 +3589,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a11dd7f04a6a6d2aea0153c6e31f5ea7af8b2efdf52cdaeea7a9a592c7fefef9" dependencies = [ "bumpalo", - "bytes 1.11.1", + "bytes", "domain-macros", "futures-util", "hashbrown 0.14.5", @@ -4053,10 +4043,10 @@ name = "file-source" version = "0.1.0" dependencies = [ "async-compression", - "bytes 1.11.1", + "bytes", "chrono", "file-source-common", - "futures 0.3.31", + "futures", "futures-util", "glob", "indexmap 2.12.0", @@ -4075,7 +4065,7 @@ version = "0.1.0" dependencies = [ "async-compression", "bstr 1.12.1", - "bytes 1.11.1", + "bytes", "chrono", "crc", "dashmap", @@ -4242,12 +4232,6 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" -[[package]] -name = "futures" -version = "0.1.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678" - [[package]] name = "futures" version = "0.3.31" @@ -4355,7 +4339,6 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ - "futures 0.1.31", "futures-channel", "futures-core", "futures-io", @@ -4366,7 +4349,6 @@ dependencies = [ "pin-project-lite", "pin-utils", "slab", - "tokio-io", ] [[package]] @@ -4476,7 +4458,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b1f1228623a5a37d4834f984573a01086708b109bbf0f7c2ee8d70b0c90d7a5" dependencies = [ "arc-swap", - "futures 0.3.31", + "futures", "log", "reqwest 0.12.28", "serde", @@ -4590,7 +4572,7 @@ dependencies = [ "dashmap", "derive_builder", "enum_dispatch", - "futures 0.3.31", + "futures", "futures-util", "greptime-proto", "parking_lot", @@ -4631,7 +4613,7 @@ version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" dependencies = [ - "bytes 1.11.1", + "bytes", "fnv", "futures-core", "futures-sink", @@ -4651,7 +4633,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" dependencies = [ "atomic-waker", - "bytes 1.11.1", + "bytes", "fnv", "futures-core", "futures-sink", @@ -4757,7 +4739,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06683b93020a07e3dbcf5f8c0f6d40080d725bea7936fc01ad345c01b97dc270" dependencies = [ "base64 0.21.7", - "bytes 1.11.1", + "bytes", "headers-core", "http 0.2.12", "httpdate", @@ -4824,7 +4806,7 @@ version = "0.1.0-rc.1" source = "git+https://github.com/vectordotdev/heim.git?branch=update-deps#b62e0420231552ef079e86ecff1e6b98def6a553" dependencies = [ "cfg-if", - "futures 0.3.31", + "futures", "glob", "heim-common", "heim-runtime", @@ -4904,7 +4886,7 @@ name = "heim-runtime" version = "0.1.0-rc.1" source = "git+https://github.com/vectordotdev/heim.git?branch=update-deps#b62e0420231552ef079e86ecff1e6b98def6a553" dependencies = [ - "futures 0.3.31", + "futures", "futures-timer", "once_cell", "smol", @@ -5077,7 +5059,7 @@ version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" dependencies = [ - "bytes 1.11.1", + "bytes", "fnv", "itoa", ] @@ -5088,7 +5070,7 @@ version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" dependencies = [ - "bytes 1.11.1", + "bytes", "fnv", "itoa", ] @@ -5099,7 +5081,7 @@ version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ - "bytes 1.11.1", + "bytes", "http 0.2.12", "pin-project-lite", ] @@ -5110,7 +5092,7 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ - "bytes 1.11.1", + "bytes", "http 1.3.1", ] @@ -5120,7 +5102,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ - "bytes 1.11.1", + "bytes", "futures-core", "http 1.3.1", "http-body 1.0.1", @@ -5167,7 +5149,7 @@ version = "0.14.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" dependencies = [ - "bytes 1.11.1", + "bytes", "futures-channel", "futures-core", "futures-util", @@ -5192,7 +5174,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb3aa54a13a0dfe7fbe3a59e0c76093041720fdc77b110cc0fc260fafb4dc51e" dependencies = [ "atomic-waker", - "bytes 1.11.1", + "bytes", "futures-channel", "futures-core", "h2 0.4.13", @@ -5266,8 +5248,8 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca815a891b24fdfb243fa3239c86154392b0953ee584aa1a2a1f66d20cbe75cc" dependencies = [ - "bytes 1.11.1", - "futures 0.3.31", + "bytes", + "futures", "headers", "http 0.2.12", "hyper 0.14.32", @@ -5342,7 +5324,7 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ - "bytes 1.11.1", + "bytes", "hyper 0.14.32", "native-tls", "tokio", @@ -5355,7 +5337,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ - "bytes 1.11.1", + "bytes", "http-body-util", "hyper 1.7.0", "hyper-util", @@ -5372,7 +5354,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c6995591a8f1380fcb4ba966a252a4b29188d51d2b89e3a252f5305be65aea8" dependencies = [ "base64 0.22.1", - "bytes 1.11.1", + "bytes", "futures-channel", "futures-core", "futures-util", @@ -5632,7 +5614,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22fa7ee6be451ea0b1912b962c91c8380835e97cf1584a77e18264e908448dcb" dependencies = [ - "bytes 1.11.1", + "bytes", "log", "nom 7.1.3", "smallvec", @@ -5688,15 +5670,6 @@ dependencies = [ "rustversion", ] -[[package]] -name = "iovec" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" -dependencies = [ - "libc", -] - [[package]] name = "ipconfig" version = "0.3.2" @@ -5943,7 +5916,7 @@ name = "k8s-e2e-tests" version = "0.1.0" dependencies = [ "env_logger", - "futures 0.3.31", + "futures", "indoc", "k8s-openapi", "k8s-test-framework", @@ -6046,9 +6019,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af97b8b696eb737e5694f087c498ca725b172c2a5bc3a6916328d160225537ee" dependencies = [ "base64 0.22.1", - "bytes 1.11.1", + "bytes", "either", - "futures 0.3.31", + "futures", "http 1.3.1", "http-body 1.0.1", "http-body-util", @@ -6103,7 +6076,7 @@ dependencies = [ "async-stream", "backon", "educe", - "futures 0.3.31", + "futures", "hashbrown 0.16.0", "hostname 0.4.2", "json-patch", @@ -6937,7 +6910,7 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a15d522be0a9c3e46fd2632e272d178f56387bdb5c9fbb3a36c649062e9b5219" dependencies = [ - "bytes 1.11.1", + "bytes", "encoding_rs", "futures-util", "http 1.3.1", @@ -7449,7 +7422,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "126c3ca37c9c44cec575247f43a3e4374d8927684f129d2beeb0d2cef262fe12" dependencies = [ - "bytes 1.11.1", + "bytes", "serde", "smallvec", ] @@ -7516,9 +7489,9 @@ dependencies = [ "anyhow", "backon", "base64 0.22.1", - "bytes 1.11.1", + "bytes", "chrono", - "futures 0.3.31", + "futures", "getrandom 0.2.15", "http 1.3.1", "http-body 1.0.1", @@ -7622,7 +7595,7 @@ dependencies = [ name = "opentelemetry-proto" version = "0.1.0" dependencies = [ - "bytes 1.11.1", + "bytes", "chrono", "glob", "hex", @@ -8080,7 +8053,7 @@ checksum = "3ee9dd5fe15055d2b6806f4736aa0c9637217074e224bbec46d4041b91bb9491" dependencies = [ "base64 0.22.1", "byteorder", - "bytes 1.11.1", + "bytes", "fallible-iterator", "hmac", "md-5", @@ -8096,7 +8069,7 @@ version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54b858f82211e84682fecd373f68e1ceae642d8d751a1ebd13f33de6257b3e20" dependencies = [ - "bytes 1.11.1", + "bytes", "chrono", "fallible-iterator", "postgres-protocol", @@ -8341,7 +8314,7 @@ version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" dependencies = [ - "bytes 1.11.1", + "bytes", "prost-derive 0.11.9", ] @@ -8351,7 +8324,7 @@ version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29" dependencies = [ - "bytes 1.11.1", + "bytes", "prost-derive 0.12.6", ] @@ -8361,7 +8334,7 @@ version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" dependencies = [ - "bytes 1.11.1", + "bytes", "prost-derive 0.13.5", ] @@ -8371,7 +8344,7 @@ version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" dependencies = [ - "bytes 1.11.1", + "bytes", "heck 0.4.1", "itertools 0.10.5", "lazy_static", @@ -8393,7 +8366,7 @@ version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4" dependencies = [ - "bytes 1.11.1", + "bytes", "heck 0.5.0", "itertools 0.12.1", "log", @@ -8561,12 +8534,12 @@ checksum = "24c00856d6c64af2078a74c2de029842cdaece2f35b1223f5166809ea49cf5cd" dependencies = [ "async-channel", "async-trait", - "bytes 1.11.1", + "bytes", "chrono", "crc", "data-url", "flate2", - "futures 0.3.31", + "futures", "log", "lz4", "murmur3", @@ -8691,7 +8664,7 @@ version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62e96808277ec6f97351a2380e6c25114bc9e67037775464979f3037c92d05ef" dependencies = [ - "bytes 1.11.1", + "bytes", "pin-project-lite", "quinn-proto", "quinn-udp", @@ -8709,7 +8682,7 @@ version = "0.11.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" dependencies = [ - "bytes 1.11.1", + "bytes", "getrandom 0.3.4", "lru-slab", "rand 0.9.2", @@ -9032,7 +9005,7 @@ checksum = "7cd3650deebc68526b304898b192fa4102a4ef0b9ada24da096559cb60e0eef8" dependencies = [ "arc-swap", "backon", - "bytes 1.11.1", + "bytes", "cfg-if", "combine 4.6.6", "futures-channel", @@ -9221,7 +9194,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78bf93c4af7a8bb7d879d51cebe797356ff10ae8516ace542b5182d9dcac10b2" dependencies = [ "base64 0.21.7", - "bytes 1.11.1", + "bytes", "encoding_rs", "futures-core", "futures-util", @@ -9261,7 +9234,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ "base64 0.22.1", - "bytes 1.11.1", + "bytes", "cookie", "cookie_store", "encoding_rs", @@ -9329,7 +9302,7 @@ checksum = "105747e3a037fe5bf17458d794de91149e575b6183fc72c85623a44abb9683f5" dependencies = [ "anyhow", "async-trait", - "futures 0.3.31", + "futures", "getrandom 0.2.15", "http 1.3.1", "hyper 1.7.0", @@ -9388,7 +9361,7 @@ checksum = "2297bf9c81a3f0dc96bc9521370b88f054168c29826a75e89c55ff196e7ed6a1" dependencies = [ "bitvec", "bytecheck", - "bytes 1.11.1", + "bytes", "hashbrown 0.12.3", "ptr_meta", "rend", @@ -9519,7 +9492,7 @@ version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1568e15fab2d546f940ed3a21f48bbbd1c494c90c99c4481339364a497f94a9" dependencies = [ - "bytes 1.11.1", + "bytes", "flume 0.11.0", "futures-util", "log", @@ -9539,7 +9512,7 @@ checksum = "35affe401787a9bd846712274d97654355d21b2a2c092a3139aabe31e9022282" dependencies = [ "arrayvec", "borsh", - "bytes 1.11.1", + "bytes", "num-traits", "rand 0.8.5", "rkyv", @@ -10152,7 +10125,7 @@ version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b258109f244e1d6891bf1053a55d63a5cd4f8f4c30cf9a1280989f80e7a1fa9" dependencies = [ - "futures 0.3.31", + "futures", "log", "once_cell", "parking_lot", @@ -10529,7 +10502,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6" dependencies = [ "base64 0.22.1", - "bytes 1.11.1", + "bytes", "chrono", "crc", "crossbeam-queue", @@ -10607,7 +10580,7 @@ dependencies = [ "base64 0.22.1", "bitflags 2.10.0", "byteorder", - "bytes 1.11.1", + "bytes", "chrono", "crc", "digest", @@ -11289,7 +11262,7 @@ version = "1.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" dependencies = [ - "bytes 1.11.1", + "bytes", "libc", "mio", "parking_lot", @@ -11301,17 +11274,6 @@ dependencies = [ "windows-sys 0.61.0", ] -[[package]] -name = "tokio-io" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57fc868aae093479e3131e3d165c93b1c7474109d13c90ec0dda2a1bbfff0674" -dependencies = [ - "bytes 0.4.12", - "futures 0.1.31", - "log", -] - [[package]] name = "tokio-io-timeout" version = "1.2.0" @@ -11362,7 +11324,7 @@ checksum = "2b40d66d9b2cfe04b628173409368e58247e8eddbbd3b0e6c6ba1d09f20f6c9e" dependencies = [ "async-trait", "byteorder", - "bytes 1.11.1", + "bytes", "fallible-iterator", "futures-channel", "futures-util", @@ -11476,7 +11438,7 @@ version = "0.7.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" dependencies = [ - "bytes 1.11.1", + "bytes", "futures-core", "futures-io", "futures-sink", @@ -11492,7 +11454,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f591660438b3038dd04d16c938271c79e7e06260ad2ea2885a4861bfb238605d" dependencies = [ "base64 0.22.1", - "bytes 1.11.1", + "bytes", "futures-core", "futures-sink", "http 1.3.1", @@ -11603,7 +11565,7 @@ dependencies = [ "async-trait", "axum 0.6.20", "base64 0.21.7", - "bytes 1.11.1", + "bytes", "flate2", "h2 0.3.26", "http 0.2.12", @@ -11636,7 +11598,7 @@ dependencies = [ "async-trait", "axum 0.7.5", "base64 0.22.1", - "bytes 1.11.1", + "bytes", "h2 0.4.13", "http 1.3.1", "http-body 1.0.1", @@ -11742,7 +11704,7 @@ checksum = "61c5bb1d698276a2443e5ecfabc1008bf15a36c12e6a7176e7bf089ea9131140" dependencies = [ "async-compression", "bitflags 2.10.0", - "bytes 1.11.1", + "bytes", "futures-core", "futures-util", "http 0.2.12", @@ -11765,7 +11727,7 @@ dependencies = [ "async-compression", "base64 0.22.1", "bitflags 2.10.0", - "bytes 1.11.1", + "bytes", "futures-core", "futures-util", "http 1.3.1", @@ -11876,7 +11838,7 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" dependencies = [ - "futures 0.3.31", + "futures", "futures-task", "pin-project", "tracing 0.1.44", @@ -11972,7 +11934,7 @@ name = "tracing-tower" version = "0.1.0" source = "git+https://github.com/tokio-rs/tracing?rev=e0642d949891546a3bb7e47080365ee7274f05cd#e0642d949891546a3bb7e47080365ee7274f05cd" dependencies = [ - "futures 0.3.31", + "futures", "tower-service", "tracing 0.2.0", "tracing-futures 0.3.0", @@ -12007,7 +11969,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e3dac10fd62eaf6617d3a904ae222845979aec67c615d1c842b4002c7666fb9" dependencies = [ "byteorder", - "bytes 1.11.1", + "bytes", "data-encoding", "http 0.2.12", "httparse", @@ -12026,7 +11988,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ef1a641ea34f399a848dea702823bbecfb4c486f911735368f1f137cb8257e1" dependencies = [ "byteorder", - "bytes 1.11.1", + "bytes", "data-encoding", "http 1.3.1", "httparse", @@ -12097,8 +12059,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44f91ea93fdd5fd4985fcc0a197ed8e8da18705912bef63c9b9b3148d6f35510" dependencies = [ "base64 0.22.1", - "bytes 1.11.1", - "futures 0.3.31", + "bytes", + "futures", "quick-xml 0.38.4", "serde", "serde_json", @@ -12114,7 +12076,7 @@ dependencies = [ "async-trait", "base64 0.22.1", "dyn-clone", - "futures 0.3.31", + "futures", "getrandom 0.3.4", "pin-project", "rand 0.9.2", @@ -12480,7 +12442,7 @@ dependencies = [ "bloomy", "bollard", "byteorder", - "bytes 1.11.1", + "bytes", "bytesize", "cfg-if", "chrono", @@ -12505,7 +12467,7 @@ dependencies = [ "exitcode", "fakedata", "flate2", - "futures 0.3.31", + "futures", "futures-util", "glob", "goauth", @@ -12652,7 +12614,7 @@ dependencies = [ "anyhow", "chrono", "clap", - "futures 0.3.31", + "futures", "graphql_client", "reqwest 0.11.26", "serde", @@ -12672,7 +12634,7 @@ dependencies = [ "async-stream", "async-trait", "bytecheck", - "bytes 1.11.1", + "bytes", "clap", "crc32fast", "criterion", @@ -12681,7 +12643,7 @@ dependencies = [ "dashmap", "derivative", "fslock", - "futures 0.3.31", + "futures", "hdrhistogram", "memmap2", "metrics", @@ -12713,11 +12675,11 @@ name = "vector-common" version = "0.1.0" dependencies = [ "async-stream", - "bytes 1.11.1", + "bytes", "chrono", "crossbeam-utils", "derivative", - "futures 0.3.31", + "futures", "indexmap 2.12.0", "itertools 0.14.0", "metrics", @@ -12803,7 +12765,7 @@ dependencies = [ "async-trait", "base64 0.22.1", "bitmask-enum", - "bytes 1.11.1", + "bytes", "cfg-if", "chrono", "chrono-tz", @@ -12814,7 +12776,7 @@ dependencies = [ "enumflags2", "env-test-util", "float_eq", - "futures 0.3.31", + "futures", "futures-util", "headers", "http 0.2.12", @@ -12914,7 +12876,7 @@ name = "vector-stream" version = "0.1.0" dependencies = [ "async-stream", - "futures 0.3.31", + "futures", "futures-util", "pin-project", "proptest", @@ -12935,7 +12897,7 @@ version = "0.1.0" dependencies = [ "async-graphql", "colored", - "futures 0.3.31", + "futures", "glob", "serde_yaml", "tokio", @@ -12958,7 +12920,7 @@ dependencies = [ "clap", "crossterm", "exitcode", - "futures 0.3.31", + "futures", "futures-util", "glob", "humantime", @@ -13073,7 +13035,7 @@ dependencies = [ "base16", "base62", "base64-simd", - "bytes 1.11.1", + "bytes", "cbc", "cfb-mode", "cfg-if", @@ -13226,7 +13188,7 @@ version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4378d202ff965b011c64817db11d5829506d3404edeadb61f190d111da3f231c" dependencies = [ - "bytes 1.11.1", + "bytes", "futures-channel", "futures-util", "headers", @@ -13388,7 +13350,7 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c598d6b99ea013e35844697fc4670d08339d5cda15588f193c6beedd12f644b" dependencies = [ - "futures 0.3.31", + "futures", "js-sys", "parking_lot", "pin-utils", @@ -14084,7 +14046,7 @@ dependencies = [ "assert-json-diff", "base64 0.22.1", "deadpool", - "futures 0.3.31", + "futures", "http 1.3.1", "http-body-util", "hyper 1.7.0", diff --git a/Cargo.toml b/Cargo.toml index ed8631a6d1e94..d7405ba9f8a09 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -163,7 +163,7 @@ dashmap = { version = "6.1.0", default-features = false } derivative = { version = "2.2.0", default-features = false } exitcode = { version = "1.1.2", default-features = false } flate2 = { version = "1.1.2", default-features = false, features = ["zlib-rs"] } -futures = { version = "0.3.31", default-features = false, features = ["compat", "io-compat", "std"], package = "futures" } +futures = { version = "0.3.31", default-features = false, features = ["std"] } futures-util = { version = "0.3.29", default-features = false } glob = { version = "0.3.3", default-features = false } hickory-proto = { version = "0.25.2", default-features = false, features = ["dnssec-ring"] } diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 72c0f17e00d3d..c023f25e685b5 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -142,7 +142,6 @@ bytecheck_derive,https://github.com/djkoloski/bytecheck,MIT,David Koloski , Joshua Landau " bytemuck,https://github.com/Lokathor/bytemuck,Zlib OR Apache-2.0 OR MIT,Lokathor byteorder,https://github.com/BurntSushi/byteorder,Unlicense OR MIT,Andrew Gallant -bytes,https://github.com/carllerche/bytes,MIT,Carl Lerche bytes,https://github.com/tokio-rs/bytes,MIT,"Carl Lerche , Sean McArthur " bytes-utils,https://github.com/vorner/bytes-utils,Apache-2.0 OR MIT,Michal 'vorner' Vaner bytesize,https://github.com/bytesize-rs/bytesize,Apache-2.0,"Hyunsik Choi , MrCroxx , Rob Ede " @@ -298,7 +297,6 @@ fraction,https://github.com/dnsl48/fraction,MIT OR Apache-2.0,dnsl48 fslock,https://github.com/brunoczim/fslock,MIT,The fslock Authors funty,https://github.com/myrrlyn/funty,MIT,myrrlyn -futures,https://github.com/rust-lang-nursery/futures-rs,MIT OR Apache-2.0,Alex Crichton futures,https://github.com/rust-lang/futures-rs,MIT OR Apache-2.0,The futures Authors futures-channel,https://github.com/rust-lang/futures-rs,MIT OR Apache-2.0,The futures-channel Authors futures-core,https://github.com/rust-lang/futures-rs,MIT OR Apache-2.0,The futures-core Authors @@ -396,7 +394,6 @@ inotify-sys,https://github.com/hannobraun/inotify-sys,ISC,Hanno Braun , Joshka" inventory,https://github.com/dtolnay/inventory,MIT OR Apache-2.0,David Tolnay -iovec,https://github.com/carllerche/iovec,MIT OR Apache-2.0,Carl Lerche ipconfig,https://github.com/liranringel/ipconfig,MIT OR Apache-2.0,Liran Ringel ipcrypt-rs,https://github.com/jedisct1/rust-ipcrypt2,ISC,Frank Denis ipnet,https://github.com/krisprice/ipnet,MIT OR Apache-2.0,Kris Price @@ -783,7 +780,6 @@ tinystr,https://github.com/unicode-org/icu4x,Unicode-3.0,The ICU4X Project Devel tinyvec,https://github.com/Lokathor/tinyvec,Zlib OR Apache-2.0 OR MIT,Lokathor tinyvec_macros,https://github.com/Soveu/tinyvec_macros,MIT OR Apache-2.0 OR Zlib,Soveu tokio,https://github.com/tokio-rs/tokio,MIT,Tokio Contributors -tokio-io,https://github.com/tokio-rs/tokio,MIT,Carl Lerche tokio-io-timeout,https://github.com/sfackler/tokio-io-timeout,MIT OR Apache-2.0,Steven Fackler tokio-macros,https://github.com/tokio-rs/tokio,MIT,Tokio Contributors tokio-native-tls,https://github.com/tokio-rs/tls,MIT,Tokio Contributors From 415059f727929adee4d88801ca760be2ac0b967a Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 23 Mar 2026 15:19:24 -0400 Subject: [PATCH 042/326] chore(deps): bump async-nats from 0.42.0 to 0.46.0 (#24974) --- Cargo.lock | 19 +++++-------------- Cargo.toml | 2 +- LICENSE-3rdparty.csv | 1 - 3 files changed, 6 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index feb1b17a72aaa..308f1e508f129 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -805,16 +805,15 @@ dependencies = [ [[package]] name = "async-nats" -version = "0.42.0" +version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08f6da6d49a956424ca4e28fe93656f790d748b469eaccbc7488fec545315180" +checksum = "df5af9ebfb0a14481d3eaf6101e6391261e4f30d25b26a7635ade8a39482ded0" dependencies = [ "base64 0.22.1", "bytes", - "futures", + "futures-util", "memchr", "nkeys", - "nuid", "once_cell", "pin-project", "portable-atomic", @@ -822,7 +821,7 @@ dependencies = [ "regex", "ring", "rustls-native-certs 0.7.0", - "rustls-pemfile 2.1.0", + "rustls-pki-types", "rustls-webpki 0.102.8", "serde", "serde_json", @@ -832,6 +831,7 @@ dependencies = [ "time", "tokio", "tokio-rustls 0.26.2", + "tokio-stream", "tokio-util", "tokio-websockets", "tracing 0.1.44", @@ -7173,15 +7173,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "nuid" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc895af95856f929163a0aa20c26a78d26bfdc839f51b9d5aa7a5b79e52b7e83" -dependencies = [ - "rand 0.8.5", -] - [[package]] name = "num" version = "0.4.1" diff --git a/Cargo.toml b/Cargo.toml index d7405ba9f8a09..e0fe08655bdef 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -408,7 +408,7 @@ lru = { version = "0.16.3", default-features = false } maxminddb = { version = "0.27.0", default-features = false, optional = true, features = ["simdutf8"] } md-5 = { version = "0.10", default-features = false, optional = true } mongodb = { version = "3.3.0", default-features = false, optional = true, features = ["compat-3-0-0", "dns-resolver", "rustls-tls"] } -async-nats = { version = "0.42.0", default-features = false, optional = true, features = ["ring", "websockets"] } +async-nats = { version = "0.46.0", default-features = false, optional = true, features = ["ring", "websockets", "jetstream", "nkeys"] } nkeys = { version = "0.4.5", default-features = false, optional = true } nom = { workspace = true, optional = true } notify = { version = "8.1.0", default-features = false, features = ["macos_fsevent"] } diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index c023f25e685b5..cf10bc525c9d5 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -505,7 +505,6 @@ notify,https://github.com/notify-rs/notify,CC0-1.0,"Félix Saparelli ntapi,https://github.com/MSxDOS/ntapi,Apache-2.0 OR MIT,MSxDOS nu-ansi-term,https://github.com/nushell/nu-ansi-term,MIT,"ogham@bsago.me, Ryan Scheel (Havvy) , Josh Triplett , The Nushell Project Developers" -nuid,https://github.com/casualjim/rs-nuid,Apache-2.0,Ivan Porto Carrero num,https://github.com/rust-num/num,MIT OR Apache-2.0,The Rust Project Developers num-bigint,https://github.com/rust-num/num-bigint,MIT OR Apache-2.0,The Rust Project Developers num-bigint-dig,https://github.com/dignifiedquire/num-bigint,MIT OR Apache-2.0,"dignifiedquire , The Rust Project Developers" From 05f7b4338b06ef7c8731cd8f391ca4e62f875b43 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Mon, 23 Mar 2026 12:45:40 -0700 Subject: [PATCH 043/326] chore(ci): add missing Windows source files to integration_windows filter (#24992) Co-authored-by: Claude Sonnet 4.6 --- .github/workflows/integration_windows.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/integration_windows.yml b/.github/workflows/integration_windows.yml index ca60a57c1f122..5e89827f52e3a 100644 --- a/.github/workflows/integration_windows.yml +++ b/.github/workflows/integration_windows.yml @@ -23,6 +23,9 @@ jobs: windows: - "src/sources/windows_event_log/**" - "src/internal_events/windows_event_log.rs" + - "src/internal_events/windows.rs" + - "src/vector_windows.rs" + - "src/service.rs" - "tests/integration/windows-event-log/**" - ".github/workflows/integration_windows.yml" From f62a9ffa8729fb1e7e6d074d4d115c96f7f2740a Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Mon, 23 Mar 2026 13:25:16 -0700 Subject: [PATCH 044/326] chore(deps): ignore RUSTSEC-2026-0049 (rustls-webpki) until rustls can be upgraded (#24986) * chore: ignore new RUSTSEC advisories that cannot be immediately fixed Co-Authored-By: Claude Sonnet 4.6 * chore: remove RUSTSEC-2026-0058 ignore, addressed by #24975 Co-Authored-By: Claude Sonnet 4.6 --------- Co-authored-by: Claude Sonnet 4.6 --- deny.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/deny.toml b/deny.toml index 17ad08e1a8a5d..5a3f7dee13c4d 100644 --- a/deny.toml +++ b/deny.toml @@ -49,4 +49,7 @@ ignore = [ { id = "RUSTSEC-2025-0012", reason = "backoff is unmaintained" }, # rustls-pemfile is unmaintained. Blocked by both async-nats and http 1.0.0 upgrade. { id = "RUSTSEC-2025-0134", reason = "rustls-pemfile is unmaintained" }, + # rustls-webpki 0.101.7 vulnerability. Fix requires upgrading rustls from 0.21 to 0.23+, + # which is a significant chain upgrade through aws-smithy-http-client, hyper-rustls, tokio-rustls, etc. + { id = "RUSTSEC-2026-0049", reason = "Fix requires major rustls upgrade (0.21 -> 0.23+); tracked for future upgrade" }, ] From aedd2a66cb27b26621fff2c6d7dbd53a199fe0fa Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Mon, 23 Mar 2026 13:26:02 -0700 Subject: [PATCH 045/326] docs(file sink): add security warning to path template field (#24983) The rendered path can resolve to any filesystem location. Add a docs::warnings annotation so users are made aware when fields from untrusted sources are used in the path template. Co-authored-by: Claude Sonnet 4.6 --- src/sinks/file/mod.rs | 3 +++ website/cue/reference/components/sinks/generated/file.cue | 1 + 2 files changed, 4 insertions(+) diff --git a/src/sinks/file/mod.rs b/src/sinks/file/mod.rs index e3af256f6501b..f6c0934fa2d9e 100644 --- a/src/sinks/file/mod.rs +++ b/src/sinks/file/mod.rs @@ -57,6 +57,9 @@ pub struct FileSinkConfig { docs::examples = "/tmp/application-{{ application_id }}-%Y-%m-%d.log" ))] #[configurable(metadata(docs::examples = "/tmp/vector-%Y-%m-%d.log.zst"))] + #[configurable(metadata( + docs::warnings = "The rendered path can resolve to any location on the filesystem. Vector will write to it if the process has permission." + ))] pub path: Template, /// The amount of time that a file can be idle and stay open. diff --git a/website/cue/reference/components/sinks/generated/file.cue b/website/cue/reference/components/sinks/generated/file.cue index 112bb8b4d90aa..606ccaf957f03 100644 --- a/website/cue/reference/components/sinks/generated/file.cue +++ b/website/cue/reference/components/sinks/generated/file.cue @@ -590,6 +590,7 @@ generated: components: sinks: file: configuration: { examples: ["/tmp/vector-%Y-%m-%d.log", "/tmp/application-{{ application_id }}-%Y-%m-%d.log", "/tmp/vector-%Y-%m-%d.log.zst"] syntax: "template" } + warnings: ["The rendered path can resolve to any location on the filesystem. Vector will write to it if the process has permission."] } timezone: { description: """ From 93a9771b1824c848c8efdf7166638226f01f42d2 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 24 Mar 2026 06:22:00 -0700 Subject: [PATCH 046/326] chore(api)!: replace GraphQL observability API with gRPC, remove async-graphql dependencies (#24364) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(api): grpc server (remove graphql) * reuse event.proto * docs(api): clarify metadata preservation in proto conversion The comment now explicitly mentions that Log.metadata_full preserves event metadata during conversion. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude * test(api): migrate vector_api tests from GraphQL to gRPC Migrate the vector_api integration tests from the deprecated GraphQL API to the new gRPC API. Changes: - Updated test infrastructure to use gRPC client instead of GraphQL - Migrated component queries from GraphQL to gRPC GetComponents RPC - Replaced GraphQL subscriptions with gRPC streaming for tap functionality - Added metrics population in gRPC GetComponents handler to support metrics tests - Fixed config reload tests to work around Vector behavior where components with unchanged names but modified connections don't appear during the reload transition - Increased startup timeout from 2s to 10s to handle slower test environments - Added 500ms delay before polling after SIGHUP to let Vector process the signal Test fixes: - Updated all assertions to use gRPC proto types (ComponentsResponse, ComponentType, etc.) - Changed tap tests to collect events inline instead of storing streams to avoid lifetime issues - Simplified reload tests to completely replace components instead of keeping old names - All 7 tests now pass: tap (4 tests), top (3 tests) 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude * chore(dev): cargo fmt * docs: add changelog for GraphQL to gRPC API migration Add breaking change changelog documenting the removal of the GraphQL Playground and replacement with gRPC API. Includes migration guide with grpcurl examples for common operations. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude * allow grpcurl word * fix(api): address gRPC migration issues P0: - Bind TCP listener before spawning to fail fast on port conflicts P1: - Restore fail-fast behavior on API startup failures - Fix tap output format regression (JSON/YAML/logfmt) - Remove metric batching to support <10 components - Fix double throughput normalization P2: - Validate parameters before casting to unsigned types - Report Vector uptime instead of subscription lifetime 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude * wip chore(dev): cargo fmt * support JSON/YAML/logfmt formats * chore(dev): cargo fmt * feat(api): add gRPC reflection support Enables gRPC server reflection for the observability API, allowing tools like grpcurl and grpcui to introspect the API without needing the .proto files. Changes: - Add tonic-reflection dependency with server feature - Register reflection service in gRPC server - Expose FILE_DESCRIPTOR_SET from generated protobuf code This allows clients to: - List available services: grpcurl -plaintext localhost:8686 list - Describe methods: grpcurl -plaintext localhost:8686 describe - Call methods without proto files 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude * fix(api-client): include third-party proto path for event.proto Fixes CI build failure where protoc couldn't find google/protobuf/timestamp.proto when compiling event.proto in vector-api-client build script. The first tonic_build::configure() call was missing the third-party include path, causing the build to fail when looking for Google well-known types. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude * chore(dev): cargo fmt * chore(dev): cargo vdev build licenses * add note to old blog post * refactor(grpc): apply review fixes to gRPC migration - Fix 1: make Client::new() sync and infallible (URL validated lazily on connect); update all call sites, improve test for connection failure to use port 65535 - Fix 2: avoid throwaway health-check client in tap cmd; extract run_tap_with_client() in TapRunner and thread the pre-connected client through tap_internal() so cmd() reuses it instead of creating a second connection - Fix 3: cap component poll interval at 1s minimum in poll_components to avoid flooding GetComponents at short metrics intervals (e.g. 100ms) - Fix 4: populate on_type with the real plugin type name (e.g. "demo_logs", "kafka") sourced from TapOutput.component_type; update component_to_row to use ComponentType enum for kind and on_type for component_type - Fix 5: document the intentional proto round-trip in serialize_event explaining the vector-api-client / vector-core type separation Co-Authored-By: Claude Sonnet 4.6 * make build-licenses * docs(grpc): clarify std::sync::Mutex usage in async streaming handlers The lock is never held across an .await point — it only guards synchronous HashMap updates inside IntervalStream closures — so std::sync::Mutex is the correct, cheaper choice over tokio::sync::Mutex. Add a comment to prevent future readers from reflexively replacing it. Co-Authored-By: Claude Sonnet 4.6 * fix(grpc): enforce limit >= 1 for StreamOutputEvents at all layers The limit field controls both the reservoir size and the internal channel capacity in stream_output_events, so 0 is not a valid value (unlike GetComponents where 0 means "no limit"). - Proto: document the >= 1 requirement and explain why 0 is not valid - Server: improve the error message to reference the affected internals - CLI: add value_parser range(1..) so `--limit 0` is rejected at parse time with a clear clap error, before the request reaches the server Co-Authored-By: Claude Sonnet 4.6 * fix(grpc): correct reservoir sampling bias in stream_output_events Algorithm R requires drawing a random index from 0..=i (inclusive) for the i-th event seen. The previous code used 0..batch (exclusive), which made the first event after the reservoir fills guaranteed to replace an existing entry (100% probability instead of limit/(limit+1)). The fix is a one-character change: 0..batch -> 0..=batch. The same bug existed in the old GraphQL implementation; this is not a regression but a pre-existing correctness issue now fixed. Co-Authored-By: Claude Sonnet 4.6 * chore: improve --url help text and expand changelog for gRPC migration - Add `(e.g. http://localhost:8686)` to --url help in tap and top - Expand changelog to clarify GraphQL API is fully removed, add --url format migration guide, and note required fields in grpcurl example Co-Authored-By: Claude Sonnet 4.6 * fix(api): remove deprecated graphql and playground config fields The `api.graphql` and `api.playground` fields were silently ignored after the GraphQL→gRPC migration. Remove them so configs containing those fields are rejected with a clear error rather than accepted and ignored. Update the changelog accordingly. Co-Authored-By: Claude Sonnet 4.6 * docs: update website and internal docs for GraphQL→gRPC migration - Rewrite website/docs/reference/api.md for gRPC (remove GraphQL/Playground sections) - Update configuration/api.md description to reference gRPC endpoint - Update api.cue to describe gRPC service and proto file - Update cli.cue: --url flag descriptions now say "gRPC" not "GraphQL" - Update internal_metrics.cue: remove "GraphQL" from api_started_total description - Regenerate generated/configuration.cue (removes graphql/playground fields) - Update AGENTS.md: api/ directory is now gRPC, not GraphQL Co-Authored-By: Claude Sonnet 4.6 * typo fix * chore: apply prettier formatting to modified markdown files Co-Authored-By: Claude Sonnet 4.6 * docs: add deprecation warning to GraphQL-related highlight posts The GraphQL API has been replaced by gRPC. Add a warning banner to the three highlight posts that reference the GraphQL API so users landing on them are aware of the migration. Co-Authored-By: Claude Sonnet 4.6 * docs: set deprecation version to 0.55.0 in GraphQL warning banners Co-Authored-By: Claude Sonnet 4.6 * refactor(api-client): rename GrpcClient to Client, remove deny attributes - Rename GrpcClient → Client; the "Grpc" prefix is redundant given the crate name - Remove #![deny(warnings)] and #![deny(missing_debug_implementations)] Co-Authored-By: Claude Sonnet 4.6 * fix(api): resolve component type for sinks using metric tags TapResource.outputs only covers sources and transforms (components with outputs). Sinks always fell back to "unknown" in GetComponents. Extract the component_type metric tag (which all components emit) as a fallback so sinks report their actual plugin type (e.g. "console"). Co-Authored-By: Claude Sonnet 4.6 * fix(api): propagate component type names via TapResource for all component kinds Sinks were showing as 'unknown' in `vector top` because TapResource.outputs only covers sources and transforms (components with fanout outputs). Sinks have no outputs so were never included. Fix by adding a `type_names` map to TapResource populated directly from RunningTopology for all component types (sources, transforms, and sinks). The gRPC GetComponents handler now reads from this map rather than falling back to metric tags. Co-Authored-By: Claude Sonnet 4.6 * perf(top): share single gRPC connection across all metric streams Previously subscribe() called connect() 11 times sequentially, creating a separate TCP+HTTP/2 handshake per metric stream. HTTP/2 natively multiplexes concurrent streams over one connection, and tonic's Channel is Arc-backed and cheap to clone, so a single connect() is sufficient. This restores the behaviour of the old GraphQL implementation which shared one WebSocket connection across all subscriptions via Arc. Co-Authored-By: Claude Sonnet 4.6 * fix(tap): break outer tap worker loop on notification send failure The break on notification send failure was inside a `for event in events` loop nested within the outer `loop { select! { ... } }`, so it only exited the inner iterator loop. The tap worker would keep running with a live TapController after the client stream had gone away, accumulating orphaned topology taps across reconnects. Fix with a labeled break to exit the outer loop. Co-Authored-By: Claude Sonnet 4.6 * refactor(tap): extract Reservoir struct to eliminate nested loop complexity The tap worker loop had a for-loop inside a select! arm that required a labeled break to exit the outer loop on send failure. Extract the sampling and flushing logic into a Reservoir struct with handle_payload() and flush() methods returning Result<(), ()>. The select! arms become single-line checks, and the labeled break is gone. Co-Authored-By: Claude Sonnet 4.6 * fix(api): sum metric series per component instead of overwriting filter_and_group_metrics used insert() which overwrites earlier values for the same component_id. Metrics like component_sent_events_total emit one series per output label, so multi-output components (e.g. route transform) would report only whichever series was seen last. Switch to entry/or_insert += to sum all series, fixing incorrect totals in GetComponents and the streaming RPCs that feed vector top. Co-Authored-By: Claude Sonnet 4.6 * make build-licenses * fix(top): seed poll_components from init_components snapshot poll_components started with an empty known_components set, so any component removed between init_components() and the first poll (up to 1s later) would never trigger a ComponentRemoved event — the stale row persisted in vector top until reconnect. Fix by passing the initial component IDs from the init_components snapshot through subscribe() into poll_components, so both the UI and the poller start from the same truth. Co-Authored-By: Claude Sonnet 4.6 * make build-licenses * fix(api): cap stream_output_events limit to prevent unbounded allocation A client could send limit=i32::MAX, causing Vec::with_capacity to attempt a multi-GB allocation in Reservoir::new and crash the process. Add a 100_000 server-side cap with a clear error response. Co-Authored-By: Claude Sonnet 4.6 * fix(api): enforce minimum 100ms interval on all metric streams Clients could set interval_ms=1 and hammer capture_metrics() at 1kHz across concurrent subscriptions. Consolidate the per-endpoint interval validation into a single validate_interval_ms helper that rejects anything below MIN_INTERVAL_MS (100ms). Co-Authored-By: Claude Sonnet 4.6 * refactor(api): extract metric stream helpers to eliminate boilerplate - Add BoxStream alias to simplify tonic associated type declarations - Add get_controller() and metric_totals_stream()/metric_throughput_stream() helpers to deduplicate stream setup across handlers - Add validate_interval_ms() to centralize interval floor enforcement - Add output_ports_by_component() on TapResource and ports_to_proto_outputs() helper to expose named output ports in get_components Co-Authored-By: Claude Sonnet 4.6 * fix(tap): stop retrying non-recoverable gRPC errors Permanent errors (INVALID_ARGUMENT, OUT_OF_RANGE, NOT_FOUND, PERMISSION_DENIED, UNAUTHENTICATED, UNIMPLEMENTED, invalid URL) now cause an immediate exit instead of looping forever in the reconnect cycle. - Add Error::is_fatal() to vector-api-client, inspecting tonic::Code before the status is serialised to a string - Add TapExecutorError::Fatal variant and route fatal client errors into it via the From impl - In tap cmd, break on is_fatal() before checking --no-reconnect Co-Authored-By: Claude Sonnet 4.6 * fix(top): exclude poll_components from stream-failure reconnection gate poll_components was added to the same handles vec as the metric stream tasks, so join_all(handles) in cmd.rs would never complete when metric streams failed early (e.g. server-side INVALID_ARGUMENT) while get_components kept succeeding. This left vector top stuck in a connected state with frozen metrics and no reconnect attempt. Split subscribe() return into SubscribeHandles { metric_handles, poll_handle }. cmd.rs now joins only metric_handles and aborts poll_handle afterwards, so any metric stream failure immediately unblocks the reconnect loop. Co-Authored-By: Claude Sonnet 4.6 * fix(top): populate per-output sent events and terminate idle tap workers Two fixes addressing open review threads: 1. Per-output sent events (metrics.rs): The streaming endpoints only emit component-level totals; per-output data is available from GetComponents. poll_components now emits SentEventsTotals with per-output data from the snapshot, restoring per-output rows in vector top for multi-output components (e.g. route transforms with _unmatched/success ports). 2. Idle tap worker leak (service.rs): The interval arm of the tap worker loop called flush() which returns Ok(()) for empty batches, so a client disconnect with no matching events left the worker and its TapController alive indefinitely. Now checks event_tx.is_closed() on each tick. Co-Authored-By: Claude Sonnet 4.6 * fix(top): populate per-output Events Out and unify timing constants Three changes: 1. Fix per-output Events Out showing N/A for multi-output transforms (e.g. route): add filter_and_group_metrics_by_output() on the server to group component_sent_events_total by (component_id, output), and populate Output.sent_events_total in GetComponents responses. poll_components already reads these values and emits SentEventsTotals to state. 2. Remove MIN_POLL_INTERVAL_MS: poll_components now uses the user's --interval directly (already validated >= 100ms server-side), making all metrics -- including per-output data -- update at a single user-controlled frequency. Previously per-output data was capped at 1s even when --interval was shorter. 3. Unify RECONNECT_DELAY: the constant was duplicated in src/top/cmd.rs and src/tap/cmd.rs. Now defined once as RECONNECT_DELAY_MS in vector-api-client and imported by both. Co-Authored-By: Claude Sonnet 4.6 * fix(api,top): address code review issues in gRPC observability API - Add per-output breakdown to streaming sent_events proto messages (output_totals/output_throughputs map fields on ComponentTotalsResponse and ComponentThroughputResponse); introduce dedicated server-side stream builders that populate these fields and wire them to the sent_events RPCs - Remove redundant per-output poll_components emission now that the stream carries the data directly - Validate --interval >= 100 at the CLI boundary in `vector top`, matching the server-side minimum and the pattern already used by `vector tap` - Sort components alphabetically before applying limit in GetComponents for stable, deterministic results across repeated calls - Fix tap_survives_config_reload test to hold the stream open across the reload rather than opening a fresh one; add use<> bound to stream_output_events to prevent lifetime overcapture in Rust 2024 Co-Authored-By: Claude Sonnet 4.6 * fix(api): restore shutdown health semantics in gRPC service The GraphQL health handler returned HTTP 503 during shutdown by checking the topology running AtomicBool. The gRPC migration dropped that flag, making the health RPC always return healthy:true — including while Vector was shutting down. Pass Arc (topology.running) through GrpcServer::start and into ObservabilityService. The health handler now returns Status::UNAVAILABLE when the flag is false, matching the old GraphQL behaviour and honouring the existing shutdown logic in RunningTopology::stop(). Co-Authored-By: Claude Sonnet 4.6 * fix(tap,api): reject logfmt for non-log events, validate tap interval, add multi-output test - Return an error when --format logfmt is used with metric or trace events instead of silently falling back to JSON - Add value_parser range(100..) to vector tap --interval, matching top's validation - Add multi_output_transform_reports_per_output_sent_events integration test asserting GetComponents.outputs and StreamComponentSentEventsTotal.output_totals for a route transform with named outputs Co-Authored-By: Claude Sonnet 4.6 * cargo fmt * fix(api): restore api_started_total metric and startup log for gRPC server The ApiStarted internal event was deleted along with GraphQL. Re-add it to grpc.rs and emit it from setup_api once the gRPC server successfully binds, restoring the api_started_total counter and "API server running" log line. Co-Authored-By: Claude Sonnet 4.6 * chore(api): remove remaining GraphQL references - Delete .graphqlconfig (referenced old GraphQL endpoint) - Remove async-graphql dependabot group - Remove async-graphql dep and api feature from vector-tap Cargo.toml - Strip async-graphql imports and derive attributes from notification.rs - Remove vector-tap/api feature activation from vector-lib - Update stale comments in vector-core config and config/vector.yaml Co-Authored-By: Claude Sonnet 4.6 * chore: cargo fmt and remove async-graphql from Cargo.lock and license inventory Co-Authored-By: Claude Sonnet 4.6 * chore(api): remove stale GraphQL migration comments Co-Authored-By: Claude Sonnet 4.6 * docs: add deprecation warnings to GraphQL docs and fix AGENTS.md API reference - Add warning banner to blog post and highlight pages noting migration to gRPC in 0.55.0 - Update AGENTS.md api/ directory description from GraphQL to gRPC Co-Authored-By: Claude Sonnet 4.6 * docs: address urseberry review feedback - Fix changelog wording: "will now reject" -> "now rejects" - Update warning banners: "we migrated" -> "Datadog migrated" Co-Authored-By: Claude Sonnet 4.6 * docs: fix warning banner wording - remove Datadog reference Co-Authored-By: Claude Sonnet 4.6 * docs: standardize warning banners to multi-line Hugo shortcode form Co-Authored-By: Claude Sonnet 4.6 * docs: extract grpc migration warning into reusable Hugo shortcode Create grpc-migration-warning.html shortcode and replace all four inline warning blocks with a single {{< grpc-migration-warning >}} call. Co-Authored-By: Claude Sonnet 4.6 * chore(api): remove unused tokio-tungstenite and reqwest deps from vector-api-client Leftovers from the GraphQL era (WebSocket subscriptions and HTTP requests). Co-Authored-By: Claude Sonnet 4.6 * docs: address thomasqueirozb review feedback - Add compatibility note to changelog: 0.55+ clients require 0.55+ server - Add grpc.io link to api.md - Link proto file directly in api.md Co-Authored-By: Claude Sonnet 4.6 * fix(api): gate ApiStarted and its imports behind the api feature flag Fixes unused import/dead_code warnings when building without the api feature (e.g. component-validation-tests). Co-Authored-By: Claude Sonnet 4.6 * refactor(api): replace 9 streaming RPCs with unified StreamComponentMetrics Consolidates StreamComponentReceivedEventsTotal, StreamComponentSentEventsTotal, StreamComponentReceivedBytesTotal, StreamComponentSentBytesTotal, StreamComponentErrorsTotal, StreamComponentReceivedEventsThroughput, StreamComponentSentEventsThroughput, StreamComponentReceivedBytesThroughput, and StreamComponentSentBytesThroughput into a single StreamComponentMetrics(ComponentMetricStreamRequest) RPC. The unified response type ComponentMetricResponse uses oneof value { TotalMetric total; ThroughputMetric throughput; } to carry per-output breakdowns alongside the aggregated value, keeping the schema honest about which fields are populated. StreamComponentAllocatedBytes remains a separate RPC since it is semantically distinct and has no per-output breakdown. Also moves ApiStarted internal event from grpc.rs (gated behind sources-vector/opentelemetry) into a new api.rs module gated behind the api feature flag. Co-Authored-By: Claude Sonnet 4.6 * chore(api): apply buf lint fixes to observability proto and cascade Rust updates - Add METRIC_NAME_ prefix to MetricName enum values (ENUM_VALUE_PREFIX) - Rename proto package to vector.observability.v1 (PACKAGE_VERSION_SUFFIX) - Rename service Observability -> ObservabilityService (SERVICE_SUFFIX) - Rename all request/response messages to match their RPC names per RPC_REQUEST_RESPONSE_UNIQUE convention (e.g. MetaRequest -> GetMetaRequest, HeartbeatRequest -> StreamHeartbeatRequest, etc.) - Add COMPONENT_TYPE_UNSPECIFIED zero value to ComponentType enum (ENUM_ZERO_VALUE_SUFFIX) - Cascade all renames through Rust: service.rs, client.rs, metrics.rs, vector-tap, and integration tests - Update grpcurl examples in changelog Co-Authored-By: Claude Sonnet 4.6 * refactor(api): use Uri in Client::new, drop url dep, clean up tests, fix api.md title - Client::new now takes http::Uri instead of String; Endpoint is built eagerly so connect() can no longer fail with InvalidUrl - Remove unused `url` dep from vector-api-client, add `http` - Remove test_invalid_url and test_connection_failure (assert nothing useful) - Update all callers to parse their string URL into Uri - Fix api.md title to "The Vector Observability API" and update stale grpcurl examples to use vector.observability.v1.ObservabilityService Co-Authored-By: Claude Sonnet 4.6 * refactor(tap): gate gRPC client code behind `api` feature Move `TapRunner`, `EventFormatter`, `OutputChannel`, and `TapExecutorError` into a new `runner` module gated by the `api` feature, and make the associated heavy deps (`vector-api-client`, `prost`, `bytes`, `serde_*`, etc.) optional. Also removes the unused `tokio-tungstenite` dep. Restores `vector-lib/api = ["vector-tap/api"]` so the feature activation chain is preserved. Co-Authored-By: Claude Sonnet 4.6 * chore(api): move fn-level allow attributes and apply rustfmt Move `#[allow(clippy::print_stderr)]` to the `fn cmd` level in both `src/top/cmd.rs` and `src/tap/cmd.rs` instead of per-statement blocks, and inline `{url}` format captures where applicable. Also applies rustfmt line-wrapping to service.rs and top.rs. Co-Authored-By: Claude Sonnet 4.6 * refactor(top): remove interval from EventType throughput variants The gRPC server already normalizes throughput values to per-second by dividing by `interval_secs` before streaming them. The `interval` field in the four `*Throughputs` EventType variants was therefore unused (`_interval`) and carried over from the old GraphQL client, which had to do the `v * (1000.0 / interval_ms)` normalization itself. Co-Authored-By: Claude Sonnet 4.6 * style: collapse pub use statement in vector-tap lib.rs Co-Authored-By: Claude Sonnet 4.6 * refactor(api): address thomasqueirozb review feedback - Use OS entropy (thread RNG) instead of epoch-based seed for reservoir sampler RNG in service.rs; removes SystemTime/UNIX_EPOCH boilerplate - Return exitcode::USAGE (bad input) instead of exitcode::UNAVAILABLE when URL fails to parse in tap/top cmd - Rename tap_internal parameter initial_client -> mut client_opt, eliminating the intermediate local binding - Thread http::Uri through top::cmd -> subscription -> metrics::subscribe and metrics::init_components, removing the String -> Uri round-trip; re-export http::Uri from vector-api-client for callers Co-Authored-By: Claude Sonnet 4.6 * fix: use http::Uri directly instead of re-exporting through vector-api-client Add http as a direct dep to vector-top and import http::Uri directly in metrics.rs and top/cmd.rs, rather than leaking it through a re-export in vector-api-client. Co-Authored-By: Claude Sonnet 4.6 * docs: add link to proto file in changelog entry Co-Authored-By: Claude Sonnet 4.6 * cargo fmt * fix(tests): add transforms-route to vector-api-tests feature set The multi_output_transform_reports_per_output_sent_events test uses a route transform, but the feature was not included in vector-api-tests, causing Vector to exit with CONFIG error (78) when starting. Co-Authored-By: Claude Sonnet 4.6 --------- Co-authored-by: Claude --- .github/actions/spelling/allow.txt | 1 + .github/dependabot.yml | 3 - .graphqlconfig | 15 - AGENTS.md | 2 +- Cargo.lock | 298 +- Cargo.toml | 21 +- LICENSE-3rdparty.csv | 17 +- build.rs | 5 +- changelog.d/graphql_to_grpc_api.breaking.md | 42 + config/vector.yaml | 5 +- lib/vector-api-client/Cargo.toml | 32 +- lib/vector-api-client/build.rs | 34 + .../graphql/queries/components.graphql | 62 - .../graphql/queries/health.graphql | 3 - .../queries/meta_version_string.graphql | 5 - lib/vector-api-client/graphql/schema.json | 7206 ----------------- .../subscriptions/component_added.graphql | 7 - .../component_allocated_bytes.graphql | 8 - .../component_errors_totals.graphql | 8 - ...mponent_received_bytes_throughputs.graphql | 6 - .../component_received_bytes_totals.graphql | 8 - ...ponent_received_events_throughputs.graphql | 6 - .../component_received_events_totals.graphql | 8 - .../subscriptions/component_removed.graphql | 6 - .../component_sent_bytes_throughputs.graphql | 6 - .../component_sent_bytes_totals.graphql | 8 - .../component_sent_events_throughputs.graphql | 10 - .../component_sent_events_totals.graphql | 14 - .../graphql/subscriptions/heartbeat.graphql | 5 - ...ut_events_by_component_id_patterns.graphql | 30 - .../processed_bytes_throughput.graphql | 3 - .../graphql/subscriptions/uptime.graphql | 5 - lib/vector-api-client/src/client.rs | 221 +- lib/vector-api-client/src/error.rs | 77 + lib/vector-api-client/src/gql/components.rs | 224 - lib/vector-api-client/src/gql/health.rs | 64 - lib/vector-api-client/src/gql/meta.rs | 26 - lib/vector-api-client/src/gql/metrics.rs | 341 - lib/vector-api-client/src/gql/mod.rs | 14 - lib/vector-api-client/src/gql/tap.rs | 90 - lib/vector-api-client/src/lib.rs | 54 +- lib/vector-api-client/src/subscription.rs | 199 - lib/vector-api-client/src/test/mod.rs | 160 - .../component_by_component_key.graphql | 6 - .../tests/queries/component_links.graphql | 45 - .../queries/components_connection.graphql | 18 - .../tests/queries/file_source_metrics.graphql | 25 - .../tests/subscriptions/errors_total.graphql | 5 - lib/vector-core/src/config/mod.rs | 2 +- lib/vector-tap/Cargo.toml | 37 +- lib/vector-tap/src/controller.rs | 1 + lib/vector-tap/src/lib.rs | 256 +- lib/vector-tap/src/notification.rs | 10 - lib/vector-tap/src/runner.rs | 344 + lib/vector-tap/src/topology.rs | 18 + lib/vector-top/Cargo.toml | 1 + lib/vector-top/src/metrics.rs | 664 +- lib/vector-top/src/state.rs | 41 +- proto/vector/observability.proto | 203 + src/api/grpc/mod.rs | 3 + src/api/grpc/service.rs | 863 ++ src/api/grpc_server.rs | 96 + src/api/handler.rs | 23 - src/api/mod.rs | 10 +- src/api/schema/components/mod.rs | 577 -- src/api/schema/components/sink.rs | 202 - src/api/schema/components/source.rs | 403 - src/api/schema/components/state.rs | 89 - src/api/schema/components/transform.rs | 223 - src/api/schema/events/encoding.rs | 9 - src/api/schema/events/log.rs | 77 - src/api/schema/events/metric.rs | 139 - src/api/schema/events/mod.rs | 157 - src/api/schema/events/output.rs | 70 - src/api/schema/events/trace.rs | 56 - src/api/schema/filter.rs | 219 - src/api/schema/gen.rs | 116 - src/api/schema/health.rs | 43 - src/api/schema/meta.rs | 27 - src/api/schema/metrics/allocated_bytes.rs | 71 - src/api/schema/metrics/errors.rs | 71 - src/api/schema/metrics/filter.rs | 397 - src/api/schema/metrics/host.rs | 373 - src/api/schema/metrics/mod.rs | 284 - src/api/schema/metrics/output.rs | 88 - src/api/schema/metrics/received_bytes.rs | 108 - src/api/schema/metrics/received_events.rs | 107 - src/api/schema/metrics/sent_bytes.rs | 108 - src/api/schema/metrics/sent_events.rs | 133 - src/api/schema/metrics/sink/generic.rs | 33 - src/api/schema/metrics/sink/mod.rs | 26 - src/api/schema/metrics/source/file.rs | 351 - src/api/schema/metrics/source/generic.rs | 33 - src/api/schema/metrics/source/mod.rs | 31 - src/api/schema/metrics/transform/generic.rs | 28 - src/api/schema/metrics/transform/mod.rs | 25 - src/api/schema/metrics/uptime.rs | 34 - src/api/schema/mod.rs | 31 - src/api/schema/relay.rs | 168 - src/api/schema/sort.rs | 61 - src/api/server.rs | 204 - src/api/tests.rs | 589 -- src/app.rs | 41 +- src/config/api.rs | 49 +- src/internal_events/api.rs | 11 +- src/proto/mod.rs | 6 +- src/proto/observability.rs | 13 + src/tap/cmd.rs | 81 +- src/tap/mod.rs | 30 +- src/top/cmd.rs | 86 +- src/top/mod.rs | 24 +- src/topology/controller.rs | 56 +- src/topology/running.rs | 25 +- tests/vector_api/harness.rs | 95 +- tests/vector_api/lib.rs | 4 +- tests/vector_api/tap.rs | 359 +- tests/vector_api/top.rs | 281 +- website/content/en/blog/graphql-api.md | 2 + website/content/en/docs/reference/api.md | 25 +- .../en/docs/reference/configuration/api.md | 3 +- .../en/highlights/2020-12-23-graphql-api.md | 2 + .../en/highlights/2020-12-23-vector-top.md | 2 + .../en/highlights/2021-04-21-vector-tap.md | 2 + website/cue/reference/api.cue | 44 +- website/cue/reference/cli.cue | 4 +- .../components/sources/internal_metrics.cue | 2 +- .../cue/reference/generated/configuration.cue | 23 +- .../shortcodes/grpc-migration-warning.html | 1 + 128 files changed, 3020 insertions(+), 16042 deletions(-) delete mode 100644 .graphqlconfig create mode 100644 changelog.d/graphql_to_grpc_api.breaking.md create mode 100644 lib/vector-api-client/build.rs delete mode 100644 lib/vector-api-client/graphql/queries/components.graphql delete mode 100644 lib/vector-api-client/graphql/queries/health.graphql delete mode 100644 lib/vector-api-client/graphql/queries/meta_version_string.graphql delete mode 100644 lib/vector-api-client/graphql/schema.json delete mode 100644 lib/vector-api-client/graphql/subscriptions/component_added.graphql delete mode 100644 lib/vector-api-client/graphql/subscriptions/component_allocated_bytes.graphql delete mode 100644 lib/vector-api-client/graphql/subscriptions/component_errors_totals.graphql delete mode 100644 lib/vector-api-client/graphql/subscriptions/component_received_bytes_throughputs.graphql delete mode 100644 lib/vector-api-client/graphql/subscriptions/component_received_bytes_totals.graphql delete mode 100644 lib/vector-api-client/graphql/subscriptions/component_received_events_throughputs.graphql delete mode 100644 lib/vector-api-client/graphql/subscriptions/component_received_events_totals.graphql delete mode 100644 lib/vector-api-client/graphql/subscriptions/component_removed.graphql delete mode 100644 lib/vector-api-client/graphql/subscriptions/component_sent_bytes_throughputs.graphql delete mode 100644 lib/vector-api-client/graphql/subscriptions/component_sent_bytes_totals.graphql delete mode 100644 lib/vector-api-client/graphql/subscriptions/component_sent_events_throughputs.graphql delete mode 100644 lib/vector-api-client/graphql/subscriptions/component_sent_events_totals.graphql delete mode 100644 lib/vector-api-client/graphql/subscriptions/heartbeat.graphql delete mode 100644 lib/vector-api-client/graphql/subscriptions/output_events_by_component_id_patterns.graphql delete mode 100644 lib/vector-api-client/graphql/subscriptions/processed_bytes_throughput.graphql delete mode 100644 lib/vector-api-client/graphql/subscriptions/uptime.graphql create mode 100644 lib/vector-api-client/src/error.rs delete mode 100644 lib/vector-api-client/src/gql/components.rs delete mode 100644 lib/vector-api-client/src/gql/health.rs delete mode 100644 lib/vector-api-client/src/gql/meta.rs delete mode 100644 lib/vector-api-client/src/gql/metrics.rs delete mode 100644 lib/vector-api-client/src/gql/mod.rs delete mode 100644 lib/vector-api-client/src/gql/tap.rs delete mode 100644 lib/vector-api-client/src/subscription.rs delete mode 100644 lib/vector-api-client/src/test/mod.rs delete mode 100644 lib/vector-api-client/tests/queries/component_by_component_key.graphql delete mode 100644 lib/vector-api-client/tests/queries/component_links.graphql delete mode 100644 lib/vector-api-client/tests/queries/components_connection.graphql delete mode 100644 lib/vector-api-client/tests/queries/file_source_metrics.graphql delete mode 100644 lib/vector-api-client/tests/subscriptions/errors_total.graphql create mode 100644 lib/vector-tap/src/runner.rs create mode 100644 proto/vector/observability.proto create mode 100644 src/api/grpc/mod.rs create mode 100644 src/api/grpc/service.rs create mode 100644 src/api/grpc_server.rs delete mode 100644 src/api/handler.rs delete mode 100644 src/api/schema/components/mod.rs delete mode 100644 src/api/schema/components/sink.rs delete mode 100644 src/api/schema/components/source.rs delete mode 100644 src/api/schema/components/state.rs delete mode 100644 src/api/schema/components/transform.rs delete mode 100644 src/api/schema/events/encoding.rs delete mode 100644 src/api/schema/events/log.rs delete mode 100644 src/api/schema/events/metric.rs delete mode 100644 src/api/schema/events/mod.rs delete mode 100644 src/api/schema/events/output.rs delete mode 100644 src/api/schema/events/trace.rs delete mode 100644 src/api/schema/filter.rs delete mode 100644 src/api/schema/gen.rs delete mode 100644 src/api/schema/health.rs delete mode 100644 src/api/schema/meta.rs delete mode 100644 src/api/schema/metrics/allocated_bytes.rs delete mode 100644 src/api/schema/metrics/errors.rs delete mode 100644 src/api/schema/metrics/filter.rs delete mode 100644 src/api/schema/metrics/host.rs delete mode 100644 src/api/schema/metrics/mod.rs delete mode 100644 src/api/schema/metrics/output.rs delete mode 100644 src/api/schema/metrics/received_bytes.rs delete mode 100644 src/api/schema/metrics/received_events.rs delete mode 100644 src/api/schema/metrics/sent_bytes.rs delete mode 100644 src/api/schema/metrics/sent_events.rs delete mode 100644 src/api/schema/metrics/sink/generic.rs delete mode 100644 src/api/schema/metrics/sink/mod.rs delete mode 100644 src/api/schema/metrics/source/file.rs delete mode 100644 src/api/schema/metrics/source/generic.rs delete mode 100644 src/api/schema/metrics/source/mod.rs delete mode 100644 src/api/schema/metrics/transform/generic.rs delete mode 100644 src/api/schema/metrics/transform/mod.rs delete mode 100644 src/api/schema/metrics/uptime.rs delete mode 100644 src/api/schema/mod.rs delete mode 100644 src/api/schema/relay.rs delete mode 100644 src/api/schema/sort.rs delete mode 100644 src/api/server.rs delete mode 100644 src/api/tests.rs create mode 100644 src/proto/observability.rs create mode 100644 website/layouts/shortcodes/grpc-migration-warning.html diff --git a/.github/actions/spelling/allow.txt b/.github/actions/spelling/allow.txt index 2e594047a89a2..a3f2dae3ceea2 100644 --- a/.github/actions/spelling/allow.txt +++ b/.github/actions/spelling/allow.txt @@ -567,5 +567,6 @@ LDFLAGS libsasl pkgconfig CLAUDE +grpcurl linting lexers diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 6ed4fe363146b..fd32d14deed15 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -45,9 +45,6 @@ updates: patterns: - "futures" - "futures-util" - graphql: - patterns: - - "async-graphql*" metrics: patterns: - "metrics" diff --git a/.graphqlconfig b/.graphqlconfig deleted file mode 100644 index ca7f234a758f1..0000000000000 --- a/.graphqlconfig +++ /dev/null @@ -1,15 +0,0 @@ -{ - "name": "Vector API", - "schemaPath": "lib/vector-api-client/graphql/schema.json", - "extensions": { - "endpoints": { - "Default GraphQL Endpoint": { - "url": "http://localhost:8686/graphql", - "headers": { - "user-agent": "Headless GraphQL" - }, - "introspect": false - } - } - } -} diff --git a/AGENTS.md b/AGENTS.md index 63460acdae3e9..580018abf0283 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -21,7 +21,7 @@ reduction and improved data quality for observability infrastructure. - `sinks/` - Data output destinations - `config/` - Configuration system and validation - `topology/` - Component graph management - - `api/` - GraphQL API for management and monitoring + - `api/` - gRPC API for management and monitoring - `cli.rs` - Command-line interface - `/lib/` - Modular library crates diff --git a/Cargo.lock b/Cargo.lock index 308f1e508f129..d23731697fa21 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,16 +2,6 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "Inflector" -version = "0.11.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" -dependencies = [ - "lazy_static", - "regex", -] - [[package]] name = "adler2" version = "2.0.0" @@ -539,12 +529,6 @@ dependencies = [ "regex-syntax", ] -[[package]] -name = "ascii" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eab1c04a571841102f5345a8fc0f6bb3d31c315dec879b5c6e42e40ce7ffa34e" - [[package]] name = "ascii-canvas" version = "4.0.0" @@ -668,90 +652,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "async-graphql" -version = "7.0.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "036618f842229ba0b89652ffe425f96c7c16a49f7e3cb23b56fca7f61fd74980" -dependencies = [ - "async-graphql-derive", - "async-graphql-parser", - "async-graphql-value", - "async-stream", - "async-trait", - "base64 0.22.1", - "bytes", - "chrono", - "fnv", - "futures-timer", - "futures-util", - "http 1.3.1", - "indexmap 2.12.0", - "mime", - "multer", - "num-traits", - "pin-project-lite", - "regex", - "serde", - "serde_json", - "serde_urlencoded", - "static_assertions_next", - "thiserror 1.0.68", -] - -[[package]] -name = "async-graphql-derive" -version = "7.0.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd45deb3dbe5da5cdb8d6a670a7736d735ba65b455328440f236dfb113727a3d" -dependencies = [ - "Inflector", - "async-graphql-parser", - "darling 0.20.11", - "proc-macro-crate 3.2.0", - "proc-macro2 1.0.106", - "quote 1.0.44", - "strum 0.26.3", - "syn 2.0.117", - "thiserror 1.0.68", -] - -[[package]] -name = "async-graphql-parser" -version = "7.0.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60b7607e59424a35dadbc085b0d513aa54ec28160ee640cf79ec3b634eba66d3" -dependencies = [ - "async-graphql-value", - "pest", - "serde", - "serde_json", -] - -[[package]] -name = "async-graphql-value" -version = "7.0.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34ecdaff7c9cffa3614a9f9999bf9ee4c3078fe3ce4d6a6e161736b56febf2de" -dependencies = [ - "bytes", - "indexmap 2.12.0", - "serde", - "serde_json", -] - -[[package]] -name = "async-graphql-warp" -version = "7.0.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff6008a33c32d5a048aa72437821eb864dd56a80c0d80c8df48f11f12154db6c" -dependencies = [ - "async-graphql", - "futures-util", - "serde_json", - "warp", -] - [[package]] name = "async-io" version = "2.4.0" @@ -2589,19 +2489,6 @@ dependencies = [ "windows-sys 0.61.0", ] -[[package]] -name = "combine" -version = "3.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da3da6baa321ec19e1cc41d31bf599f00c783d0517095cdaf0332e3fe8d20680" -dependencies = [ - "ascii", - "byteorder", - "either", - "memchr", - "unreachable", -] - [[package]] name = "combine" version = "4.6.6" @@ -4492,64 +4379,6 @@ dependencies = [ "web-time", ] -[[package]] -name = "graphql-introspection-query" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f2a4732cf5140bd6c082434494f785a19cfb566ab07d1382c3671f5812fed6d" -dependencies = [ - "serde", -] - -[[package]] -name = "graphql-parser" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2ebc8013b4426d5b81a4364c419a95ed0b404af2b82e2457de52d9348f0e474" -dependencies = [ - "combine 3.8.1", - "thiserror 1.0.68", -] - -[[package]] -name = "graphql_client" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a50cfdc7f34b7f01909d55c2dcb71d4c13cbcbb4a1605d6c8bd760d654c1144b" -dependencies = [ - "graphql_query_derive", - "serde", - "serde_json", -] - -[[package]] -name = "graphql_client_codegen" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e27ed0c2cf0c0cc52c6bcf3b45c907f433015e580879d14005386251842fb0a" -dependencies = [ - "graphql-introspection-query", - "graphql-parser", - "heck 0.4.1", - "lazy_static", - "proc-macro2 1.0.106", - "quote 1.0.44", - "serde", - "serde_json", - "syn 1.0.109", -] - -[[package]] -name = "graphql_query_derive" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83febfa838f898cfa73dfaa7a8eb69ff3409021ac06ee94cfb3d622f6eeb1a97" -dependencies = [ - "graphql_client_codegen", - "proc-macro2 1.0.106", - "syn 1.0.109", -] - [[package]] name = "greptime-proto" version = "0.1.0" @@ -5816,7 +5645,7 @@ checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" dependencies = [ "cesu8", "cfg-if", - "combine 4.6.6", + "combine", "jni-sys", "log", "thiserror 1.0.68", @@ -6904,24 +6733,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "multer" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a15d522be0a9c3e46fd2632e272d178f56387bdb5c9fbb3a36c649062e9b5219" -dependencies = [ - "bytes", - "encoding_rs", - "futures-util", - "http 1.3.1", - "httparse", - "log", - "memchr", - "mime", - "spin 0.9.8", - "version_check", -] - [[package]] name = "multimap" version = "0.8.3" @@ -8998,7 +8809,7 @@ dependencies = [ "backon", "bytes", "cfg-if", - "combine 4.6.6", + "combine", "futures-channel", "futures-util", "itoa", @@ -10679,12 +10490,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" -[[package]] -name = "static_assertions_next" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7beae5182595e9a8b683fa98c4317f956c9a2dec3b9716990d20023cc60c766" - [[package]] name = "stream-cancel" version = "0.8.2" @@ -10746,9 +10551,6 @@ name = "strum" version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" -dependencies = [ - "strum_macros 0.26.4", -] [[package]] name = "strum" @@ -11406,21 +11208,8 @@ checksum = "212d5dcb2a1ce06d81107c3d0ffa3121fe974b73f068c8282cb1c32328113b6c" dependencies = [ "futures-util", "log", - "rustls 0.21.12", "tokio", - "tungstenite 0.20.1", -] - -[[package]] -name = "tokio-tungstenite" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c83b561d025642014097b66e6c1bb422783339e0909e4429cde4749d1990bc38" -dependencies = [ - "futures-util", - "log", - "tokio", - "tungstenite 0.21.0", + "tungstenite", ] [[package]] @@ -11648,6 +11437,19 @@ dependencies = [ "tonic 0.11.0", ] +[[package]] +name = "tonic-reflection" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "548c227bd5c0fae5925812c4ec6c66ffcfced23ea370cb823f4d18f0fc1cb6a7" +dependencies = [ + "prost 0.12.6", + "prost-types 0.12.6", + "tokio", + "tokio-stream", + "tonic 0.11.0", +] + [[package]] name = "tower" version = "0.4.13" @@ -11972,25 +11774,6 @@ dependencies = [ "utf-8", ] -[[package]] -name = "tungstenite" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ef1a641ea34f399a848dea702823bbecfb4c486f911735368f1f137cb8257e1" -dependencies = [ - "byteorder", - "bytes", - "data-encoding", - "http 1.3.1", - "httparse", - "log", - "rand 0.8.5", - "sha1", - "thiserror 1.0.68", - "url", - "utf-8", -] - [[package]] name = "twox-hash" version = "2.1.2" @@ -12235,15 +12018,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "unreachable" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56" -dependencies = [ - "void", -] - [[package]] name = "unsafe-libyaml" version = "0.2.11" @@ -12398,8 +12172,6 @@ dependencies = [ "arrow-schema", "assert_cmd", "async-compression", - "async-graphql", - "async-graphql-warp", "async-nats", "async-rs", "async-stream", @@ -12565,12 +12337,13 @@ dependencies = [ "tokio-postgres", "tokio-stream", "tokio-test", - "tokio-tungstenite 0.20.1", + "tokio-tungstenite", "tokio-util", "toml", "tonic 0.11.0", "tonic-build 0.11.0", "tonic-health", + "tonic-reflection", "tower 0.5.3", "tower-http 0.4.4", "tower-test", @@ -12600,21 +12373,18 @@ dependencies = [ [[package]] name = "vector-api-client" -version = "0.1.2" +version = "0.2.0" dependencies = [ - "anyhow", - "chrono", - "clap", "futures", - "graphql_client", - "reqwest 0.11.26", - "serde", - "serde_json", + "http 0.2.12", + "prost 0.12.6", + "prost-build 0.12.6", + "prost-types 0.12.6", + "snafu 0.8.9", "tokio", "tokio-stream", - "tokio-tungstenite 0.20.1", - "url", - "uuid", + "tonic 0.11.0", + "tonic-build 0.11.0", ] [[package]] @@ -12886,14 +12656,18 @@ dependencies = [ name = "vector-tap" version = "0.1.0" dependencies = [ - "async-graphql", + "bytes", + "clap", + "codecs", "colored", "futures", "glob", + "prost 0.12.6", + "serde_json", "serde_yaml", "tokio", "tokio-stream", - "tokio-tungstenite 0.20.1", + "tokio-util", "tracing 0.1.44", "url", "uuid", @@ -12914,6 +12688,7 @@ dependencies = [ "futures", "futures-util", "glob", + "http 0.2.12", "humantime", "indoc", "num-format", @@ -13009,12 +12784,6 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" -[[package]] -name = "void" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" - [[package]] name = "vrl" version = "0.31.0" @@ -13195,7 +12964,6 @@ dependencies = [ "serde_json", "serde_urlencoded", "tokio", - "tokio-tungstenite 0.21.0", "tokio-util", "tower-service", "tracing 0.1.44", diff --git a/Cargo.toml b/Cargo.toml index e0fe08655bdef..0a577cde861d9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,13 +19,6 @@ name = "vector" test = false bench = false -[[bin]] -name = "graphql-schema" -path = "src/api/schema/gen.rs" -test = false -bench = false -required-features = ["default-no-api-client"] - [[bin]] name = "secret-backend-example" path = "src/config/loading/secret_backend_example.rs" @@ -207,6 +200,7 @@ toml = { version = "0.9.8", default-features = false, features = ["serde", "disp tonic = { version = "0.11", default-features = false, features = ["transport", "codegen", "prost", "tls", "tls-roots", "gzip"] } tonic-build = { version = "0.11", default-features = false, features = ["transport", "prost"] } tonic-health = { version = "0.11", default-features = false } +tonic-reflection = { version = "0.11", default-features = false, features = ["server"] } tracing = { version = "0.1.44", default-features = false } tracing-subscriber = { version = "0.3.22", default-features = false, features = ["fmt"] } url = { version = "2.5.4", default-features = false, features = ["serde"] } @@ -344,10 +338,6 @@ lapin = { version = "4.3.0", default-features = false, features = ["tokio", "nat async-rs = { version = "0.8", default-features = false, features = ["tokio"], optional = true } deadpool = { version = "0.12.2", default-features = false, features = ["managed", "rt_tokio_1"], optional = true } -# API -async-graphql = { version = "7.0.17", default-features = false, optional = true, features = ["chrono", "playground"] } -async-graphql-warp = { version = "7.0.17", default-features = false, optional = true } - # Opentelemetry hex = { version = "0.4.3", default-features = false, optional = true } @@ -442,6 +432,7 @@ toml.workspace = true hickory-proto = { workspace = true, optional = true } tonic = { workspace = true, optional = true } tonic-health = { workspace = true, optional = true } +tonic-reflection = { workspace = true, optional = true } thread_local = { version = "1.1.9", default-features = false, optional = true } typetag = { version = "0.2.20", default-features = false } url.workspace = true @@ -565,9 +556,12 @@ docker = ["dep:bollard", "dep:dirs-next"] # API api = [ - "dep:async-graphql", - "dep:async-graphql-warp", "dep:base64", + "dep:tonic", + "dep:tonic-reflection", + "dep:prost", + "dep:prost-types", + "protobuf-build", "vector-lib/api", ] @@ -1077,6 +1071,7 @@ vector-api-tests = [ "sources-demo_logs", "transforms-log_to_metric", "transforms-remap", + "transforms-route", "sinks-blackhole" ] vector-unit-test-tests = [ diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index cf10bc525c9d5..8a5759685500e 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -1,5 +1,4 @@ Component,Origin,License,Copyright -Inflector,https://github.com/whatisinternet/inflector,BSD-2-Clause,Josh Teeter adler2,https://github.com/oyvindln/adler2,0BSD OR MIT OR Apache-2.0,"Jonas Schievink , oyvindln " adler32,https://github.com/remram44/adler32-rs,Zlib,Remi Rampin aead,https://github.com/RustCrypto/traits,MIT OR Apache-2.0,RustCrypto Developers @@ -41,7 +40,6 @@ arrow-row,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow arrow-select,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow arrow-string,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow -ascii,https://github.com/tomprogrammer/rust-ascii,Apache-2.0 OR MIT,"Thomas Bahn , Torbjørn Birch Moltu , Simon Sapin " async-broadcast,https://github.com/smol-rs/async-broadcast,MIT OR Apache-2.0,"Stjepan Glavina , Yoshua Wuyts , Zeeshan Ali Khan " async-channel,https://github.com/smol-rs/async-channel,Apache-2.0 OR MIT,Stjepan Glavina async-compat,https://github.com/smol-rs/async-compat,Apache-2.0 OR MIT,Stjepan Glavina @@ -49,11 +47,6 @@ async-compression,https://github.com/Nullus157/async-compression,MIT OR Apache-2 async-executor,https://github.com/smol-rs/async-executor,Apache-2.0 OR MIT,"Stjepan Glavina , John Nunley " async-fs,https://github.com/smol-rs/async-fs,Apache-2.0 OR MIT,Stjepan Glavina async-global-executor,https://github.com/Keruspe/async-global-executor,Apache-2.0 OR MIT,Marc-Antoine Perennou -async-graphql,https://github.com/async-graphql/async-graphql,MIT OR Apache-2.0,"sunli , Koxiaet" -async-graphql-derive,https://github.com/async-graphql/async-graphql,MIT OR Apache-2.0,"sunli , Koxiaet" -async-graphql-parser,https://github.com/async-graphql/async-graphql,MIT OR Apache-2.0,"sunli , Koxiaet" -async-graphql-value,https://github.com/async-graphql/async-graphql,MIT OR Apache-2.0,"sunli , Koxiaet" -async-graphql-warp,https://github.com/async-graphql/async-graphql,MIT OR Apache-2.0,"sunli , Koxiaet" async-io,https://github.com/smol-rs/async-io,Apache-2.0 OR MIT,Stjepan Glavina async-lock,https://github.com/smol-rs/async-lock,Apache-2.0 OR MIT,Stjepan Glavina async-native-tls,https://github.com/async-email/async-native-tls,MIT OR Apache-2.0,dignifiedquire @@ -315,11 +308,6 @@ glob,https://github.com/rust-lang/glob,MIT OR Apache-2.0,The Rust Project Develo gloo-timers,https://github.com/rustwasm/gloo/tree/master/crates/timers,MIT OR Apache-2.0,Rust and WebAssembly Working Group goauth,https://github.com/durch/rust-goauth,MIT,Drazen Urch governor,https://github.com/boinkor-net/governor,MIT,Andreas Fuchs -graphql-introspection-query,https://github.com/graphql-rust/graphql-client,Apache-2.0 OR MIT,Tom Houlé -graphql-parser,https://github.com/graphql-rust/graphql-parser,MIT OR Apache-2.0,Paul Colomiets -graphql_client,https://github.com/graphql-rust/graphql-client,Apache-2.0 OR MIT,Tom Houlé -graphql_client_codegen,https://github.com/graphql-rust/graphql-client,Apache-2.0 OR MIT,Tom Houlé -graphql_query_derive,https://github.com/graphql-rust/graphql-client,Apache-2.0 OR MIT,Tom Houlé greptime-proto,https://github.com/GreptimeTeam/greptime-proto,Apache-2.0,The greptime-proto Authors greptimedb-ingester,https://github.com/GreptimeTeam/greptimedb-ingester-rust,Apache-2.0,The greptimedb-ingester Authors grok,https://github.com/mmastrac/grok,Apache-2.0,"Matt Mastracci , Michael Nitschinger " @@ -487,7 +475,6 @@ mongocrypt,https://github.com/mongodb/libmongocrypt-rust,Apache-2.0,"Abraham Egn mongocrypt-sys,https://github.com/mongodb/libmongocrypt-rust,Apache-2.0,"Abraham Egnor , Isabel Atkinson " mongodb,https://github.com/mongodb/mongo-rust-driver,Apache-2.0,"Saghm Rossi , Patrick Freed , Isabel Atkinson , Abraham Egnor , Kaitlin Mahar , Patrick Meredith " mongodb-internal-macros,https://github.com/mongodb/mongo-rust-driver,Apache-2.0,The mongodb-internal-macros Authors -multer,https://github.com/rousan/multer-rs,MIT,Rousan Ali murmur3,https://github.com/stusmall/murmur3,MIT OR Apache-2.0,Stu Small native-tls,https://github.com/sfackler/rust-native-tls,MIT OR Apache-2.0,Steven Fackler ndk-context,https://github.com/rust-windowing/android-ndk-rs,MIT OR Apache-2.0,The Rust Windowing contributors @@ -741,7 +728,6 @@ sqlx-postgres,https://github.com/launchbadge/sqlx,MIT OR Apache-2.0,"Ryan Leckey sqlx-sqlite,https://github.com/launchbadge/sqlx,MIT OR Apache-2.0,"Ryan Leckey , Austin Bonander , Chloe Ross , Daniel Akhterov " stable_deref_trait,https://github.com/storyyeller/stable_deref_trait,MIT OR Apache-2.0,Robert Grosse static_assertions,https://github.com/nvzqz/static-assertions-rs,MIT OR Apache-2.0,Nikolai Vazquez -static_assertions_next,https://github.com/scuffletv/static-assertions,MIT OR Apache-2.0,Nikolai Vazquez stream-cancel,https://github.com/jonhoo/stream-cancel,MIT OR Apache-2.0,Jon Gjengset stringprep,https://github.com/sfackler/rust-stringprep,MIT OR Apache-2.0,Steven Fackler strip-ansi-escapes,https://github.com/luser/strip-ansi-escapes,Apache-2.0 OR MIT,Ted Mielczarek @@ -799,6 +785,7 @@ toml_write,https://github.com/toml-rs/toml,MIT OR Apache-2.0,The toml_write Auth toml_writer,https://github.com/toml-rs/toml,MIT OR Apache-2.0,The toml_writer Authors tonic,https://github.com/hyperium/tonic,MIT,Lucio Franco tonic-health,https://github.com/hyperium/tonic,MIT,James Nugent +tonic-reflection,https://github.com/hyperium/tonic,MIT,"James Nugent , Samani G. Gikandi " tower,https://github.com/tower-rs/tower,MIT,Tower Maintainers tower-http,https://github.com/tower-rs/tower-http,MIT,Tower Maintainers tower-layer,https://github.com/tower-rs/tower,MIT,Tower Maintainers @@ -838,7 +825,6 @@ unicode-width,https://github.com/unicode-rs/unicode-width,MIT OR Apache-2.0,"kwa unicode-xid,https://github.com/unicode-rs/unicode-xid,MIT OR Apache-2.0,"erick.tryzelaar , kwantam , Manish Goregaokar " unit-prefix,https://codeberg.org/commons-rs/unit-prefix,MIT,"Fabio Valentini , Benjamin Sago " universal-hash,https://github.com/RustCrypto/traits,MIT OR Apache-2.0,RustCrypto Developers -unreachable,https://github.com/reem/rust-unreachable,MIT OR Apache-2.0,Jonathan Reem unsafe-libyaml,https://github.com/dtolnay/unsafe-libyaml,MIT,David Tolnay untrusted,https://github.com/briansmith/untrusted,ISC,Brian Smith uom,https://github.com/iliekturtles/uom,Apache-2.0 OR MIT,Mike Boutin @@ -852,7 +838,6 @@ utf8parse,https://github.com/alacritty/vte,Apache-2.0 OR MIT,"Joe Wilm , Dylan DPC, Hunar Roop Kahlon" uuid-simd,https://github.com/Nugine/simd,MIT,The uuid-simd Authors valuable,https://github.com/tokio-rs/valuable,MIT,The valuable Authors -void,https://github.com/reem/rust-void,MIT,Jonathan Reem vrl,https://github.com/vectordotdev/vrl,MPL-2.0,Vector Contributors vsimd,https://github.com/Nugine/simd,MIT,The vsimd Authors vte,https://github.com/alacritty/vte,Apache-2.0 OR MIT,"Joe Wilm , Christian Duerr " diff --git a/build.rs b/build.rs index cf35846cbe61c..5d6d533e86423 100644 --- a/build.rs +++ b/build.rs @@ -120,6 +120,7 @@ fn main() { println!("cargo:rerun-if-changed=proto/vector/dd_trace.proto"); println!("cargo:rerun-if-changed=proto/vector/ddsketch_full.proto"); println!("cargo:rerun-if-changed=proto/vector/vector.proto"); + println!("cargo:rerun-if-changed=proto/vector/observability.proto"); // Create and store the "file descriptor set" from the compiled Protocol Buffers packages. // @@ -133,7 +134,8 @@ fn main() { let mut prost_build = prost_build::Config::new(); prost_build .btree_map(["."]) - .file_descriptor_set_path(protobuf_fds_path); + .file_descriptor_set_path(protobuf_fds_path) + .extern_path(".event", "crate::event::proto"); // Use existing event types from vector-core tonic_build::configure() .protoc_arg("--experimental_allow_proto3_optional") @@ -147,6 +149,7 @@ fn main() { "proto/third-party/google/pubsub/v1/pubsub.proto", "proto/third-party/google/rpc/status.proto", "proto/vector/vector.proto", + "proto/vector/observability.proto", ], &[ "proto/third-party", diff --git a/changelog.d/graphql_to_grpc_api.breaking.md b/changelog.d/graphql_to_grpc_api.breaking.md new file mode 100644 index 0000000000000..20ab9d67b35e3 --- /dev/null +++ b/changelog.d/graphql_to_grpc_api.breaking.md @@ -0,0 +1,42 @@ +The Vector observability API has been migrated from GraphQL to gRPC for improved +performance, efficiency and maintainability. The `vector top` and `vector tap` +commands continue to work as before, as they have been updated to use the new +gRPC API internally. The gRPC service definition is available in +[`proto/vector/observability.proto`](https://github.com/vectordotdev/vector/blob/master/proto/vector/observability.proto). + +Note: `vector top` and `vector tap` from version 0.55.0 or later are not +compatible with Vector instances running earlier versions. + +- Remove the `api.graphql` and `api.playground` fields from your config. Vector + now rejects configs that contain them. + +- If you use `vector top` or `vector tap` with an explicit `--url`, remove the + `/graphql` path suffix: + +```bash +# Old +vector top --url http://localhost:8686/graphql + +# New (the gRPC API listens at the root) +vector top --url http://localhost:8686 +``` + +- The GraphQL API (HTTP endpoint `/graphql`, WebSocket subscriptions, and the + GraphQL Playground at `/playground`) has been removed. You can interact with + the new gRPC API using tools like + [grpcurl](https://github.com/fullstorydev/grpcurl): + +```bash +# Check health +grpcurl -plaintext localhost:8686 vector.observability.v1.ObservabilityService/Health + +# List components +grpcurl -plaintext localhost:8686 vector.observability.v1.ObservabilityService/GetComponents + +# Stream events (tap) — limit and interval_ms are required and must be >= 1 +grpcurl -plaintext \ + -d '{"outputs_patterns": ["*"], "limit": 100, "interval_ms": 500}' \ + localhost:8686 vector.observability.v1.ObservabilityService/StreamOutputEvents +``` + +authors: pront diff --git a/config/vector.yaml b/config/vector.yaml index 590c2aa9ba152..c48be7abd7a31 100644 --- a/config/vector.yaml +++ b/config/vector.yaml @@ -41,9 +41,8 @@ sinks: json: pretty: true -# Vector's GraphQL API (disabled by default) -# Uncomment to try it out with the `vector top` command or -# in your browser at http://localhost:8686 +# Vector's API (disabled by default) +# Uncomment to try it out with the `vector top` command # api: # enabled: true # address: "127.0.0.1:8686" diff --git a/lib/vector-api-client/Cargo.toml b/lib/vector-api-client/Cargo.toml index 3fbc200af231b..00243fb9d0ecd 100644 --- a/lib/vector-api-client/Cargo.toml +++ b/lib/vector-api-client/Cargo.toml @@ -1,34 +1,28 @@ [package] name = "vector-api-client" -version = "0.1.2" +version = "0.2.0" authors = ["Vector Contributors "] edition = "2024" publish = false license = "MPL-2.0" [dependencies] +# gRPC +tonic.workspace = true +prost.workspace = true +prost-types.workspace = true -# Serde -serde.workspace = true -serde_json.workspace = true - -# Error handling -anyhow.workspace = true - -# Tokio / Futures +# Async futures.workspace = true tokio = { workspace = true, features = ["macros", "rt", "sync"] } tokio-stream = { workspace = true, features = ["sync"] } -# GraphQL -graphql_client = { version = "0.14.0", default-features = false, features = ["graphql_query_derive"] } +# Error handling +snafu.workspace = true -# HTTP / WebSockets -reqwest = { version = "0.11.26", default-features = false, features = ["json"] } -tokio-tungstenite = { workspace = true, features = ["connect", "rustls"] } +# Utils +http = { version = "0.2", default-features = false } -# External libs -chrono.workspace = true -clap.workspace = true -url = { version = "2.5.4", default-features = false } -uuid.workspace = true +[build-dependencies] +tonic-build.workspace = true +prost-build.workspace = true diff --git a/lib/vector-api-client/build.rs b/lib/vector-api-client/build.rs new file mode 100644 index 0000000000000..a85641a4b2476 --- /dev/null +++ b/lib/vector-api-client/build.rs @@ -0,0 +1,34 @@ +fn main() -> Result<(), Box> { + println!("cargo:rerun-if-changed=../../proto/vector/observability.proto"); + println!("cargo:rerun-if-changed=../../proto"); + println!("cargo:rerun-if-changed=../../lib/vector-core/proto/event.proto"); + + // First, generate event.proto types + tonic_build::configure().build_server(false).compile( + &["../../lib/vector-core/proto/event.proto"], + &["../../lib/vector-core/proto", "../../proto/third-party"], + )?; + + // Then, generate observability.proto using extern_path to reference the event types + let mut prost_config = prost_build::Config::new(); + prost_config.extern_path(".event", "crate::proto::event"); + // Allow clippy warning for large enum variant in generated code + // TappedEvent contains a large EventWrapper while EventNotification is small + prost_config.type_attribute( + ".vector.observability.v1.StreamOutputEventsResponse", + "#[allow(clippy::large_enum_variant)]", + ); + + tonic_build::configure() + .build_server(false) + .compile_with_config( + prost_config, + &["../../proto/vector/observability.proto"], + &[ + "../../proto", + "../../proto/third-party", + "../../lib/vector-core/proto", + ], + )?; + Ok(()) +} diff --git a/lib/vector-api-client/graphql/queries/components.graphql b/lib/vector-api-client/graphql/queries/components.graphql deleted file mode 100644 index 3a6f78fc27989..0000000000000 --- a/lib/vector-api-client/graphql/queries/components.graphql +++ /dev/null @@ -1,62 +0,0 @@ -query ComponentsQuery($first: Int!) { - components(first: $first) { - edges { - node { - __typename - componentId - componentType - ... on Source { - outputs { - outputId - sentEventsTotal { - sentEventsTotal - } - } - metrics { - __typename - receivedBytesTotal { - receivedBytesTotal - } - receivedEventsTotal { - receivedEventsTotal - } - sentEventsTotal { - sentEventsTotal - } - } - } - ... on Transform { - outputs { - outputId - sentEventsTotal { - sentEventsTotal - } - } - metrics { - __typename - receivedEventsTotal { - receivedEventsTotal - } - sentEventsTotal { - sentEventsTotal - } - } - } - ... on Sink { - metrics { - __typename - receivedEventsTotal { - receivedEventsTotal - } - sentBytesTotal { - sentBytesTotal - } - sentEventsTotal { - sentEventsTotal - } - } - } - } - } - } -} diff --git a/lib/vector-api-client/graphql/queries/health.graphql b/lib/vector-api-client/graphql/queries/health.graphql deleted file mode 100644 index 6d4aa89849307..0000000000000 --- a/lib/vector-api-client/graphql/queries/health.graphql +++ /dev/null @@ -1,3 +0,0 @@ -query HealthQuery { - health -} diff --git a/lib/vector-api-client/graphql/queries/meta_version_string.graphql b/lib/vector-api-client/graphql/queries/meta_version_string.graphql deleted file mode 100644 index 82f41372efa07..0000000000000 --- a/lib/vector-api-client/graphql/queries/meta_version_string.graphql +++ /dev/null @@ -1,5 +0,0 @@ -query MetaVersionStringQuery { - meta { - versionString - } -} diff --git a/lib/vector-api-client/graphql/schema.json b/lib/vector-api-client/graphql/schema.json deleted file mode 100644 index 034780a80d718..0000000000000 --- a/lib/vector-api-client/graphql/schema.json +++ /dev/null @@ -1,7206 +0,0 @@ -{ - "data": { - "__schema": { - "queryType": { - "name": "Query" - }, - "mutationType": null, - "subscriptionType": { - "name": "Subscription" - }, - "types": [ - { - "kind": "OBJECT", - "name": "AllocatedBytes", - "description": null, - "fields": [ - { - "name": "timestamp", - "description": "Metric timestamp", - "args": [], - "type": { - "kind": "SCALAR", - "name": "DateTime", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "allocatedBytes", - "description": "Allocated bytes", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "SCALAR", - "name": "Boolean", - "description": "The `Boolean` scalar type represents `true` or `false`.", - "fields": null, - "inputFields": null, - "interfaces": null, - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "INTERFACE", - "name": "Component", - "description": null, - "fields": [ - { - "name": "componentId", - "description": null, - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "componentType", - "description": null, - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": null, - "enumValues": null, - "possibleTypes": [ - { - "kind": "OBJECT", - "name": "Source", - "ofType": null - }, - { - "kind": "OBJECT", - "name": "Transform", - "ofType": null - }, - { - "kind": "OBJECT", - "name": "Sink", - "ofType": null - } - ] - }, - { - "kind": "OBJECT", - "name": "ComponentAllocatedBytes", - "description": null, - "fields": [ - { - "name": "componentId", - "description": "Component id", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "metric", - "description": "Allocated bytes metric", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "AllocatedBytes", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "ComponentConnection", - "description": null, - "fields": [ - { - "name": "pageInfo", - "description": "Information to aid in pagination.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "PageInfo", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "edges", - "description": "A list of edges.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "ComponentEdge", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "nodes", - "description": "A list of nodes.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INTERFACE", - "name": "Component", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "totalCount", - "description": "Total result set count", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "ComponentEdge", - "description": "An edge in a connection.", - "fields": [ - { - "name": "cursor", - "description": "A cursor for use in pagination", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "node", - "description": "The item at the end of the edge", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INTERFACE", - "name": "Component", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "ComponentErrorsTotal", - "description": null, - "fields": [ - { - "name": "componentId", - "description": "Component id", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "metric", - "description": "Errors processed metric", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "ErrorsTotal", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "ENUM", - "name": "ComponentKind", - "description": null, - "fields": null, - "inputFields": null, - "interfaces": null, - "enumValues": [ - { - "name": "SOURCE", - "description": null, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "TRANSFORM", - "description": null, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "SINK", - "description": null, - "isDeprecated": false, - "deprecationReason": null - } - ], - "possibleTypes": null - }, - { - "kind": "INPUT_OBJECT", - "name": "ComponentKindFilter", - "description": null, - "fields": null, - "inputFields": [ - { - "name": "equals", - "description": null, - "type": { - "kind": "ENUM", - "name": "ComponentKind", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "notEquals", - "description": null, - "type": { - "kind": "ENUM", - "name": "ComponentKind", - "ofType": null - }, - "defaultValue": null - } - ], - "interfaces": null, - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "ComponentReceivedBytesThroughput", - "description": null, - "fields": [ - { - "name": "componentId", - "description": "Component ID.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "throughput", - "description": "Throughput of bytes sent.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "ComponentReceivedBytesTotal", - "description": null, - "fields": [ - { - "name": "componentId", - "description": "Component ID.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "metric", - "description": "Metric for total bytes received.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "ReceivedBytesTotal", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "ComponentReceivedEventsThroughput", - "description": null, - "fields": [ - { - "name": "componentId", - "description": "Component id", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "throughput", - "description": "Received events throughput", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "ComponentReceivedEventsTotal", - "description": null, - "fields": [ - { - "name": "componentId", - "description": "Component id", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "metric", - "description": "Total received events metric", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "ReceivedEventsTotal", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "ComponentSentBytesThroughput", - "description": null, - "fields": [ - { - "name": "componentId", - "description": "Component ID.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "throughput", - "description": "Throughput of bytes sent.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "ComponentSentBytesTotal", - "description": null, - "fields": [ - { - "name": "componentId", - "description": "Component ID.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "metric", - "description": "Metric for total bytes sent.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "SentBytesTotal", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "ComponentSentEventsThroughput", - "description": null, - "fields": [ - { - "name": "componentId", - "description": "Component id", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "throughput", - "description": "Total events processed throughput", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "outputs", - "description": "Output streams with throughputs", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "OutputThroughput", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "ComponentSentEventsTotal", - "description": null, - "fields": [ - { - "name": "componentId", - "description": "Component id", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "metric", - "description": "Total outgoing events metric", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "SentEventsTotal", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "outputs", - "description": "Output streams with outgoing events metrics", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "Output", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "INPUT_OBJECT", - "name": "ComponentsFilter", - "description": null, - "fields": null, - "inputFields": [ - { - "name": "componentId", - "description": null, - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INPUT_OBJECT", - "name": "StringFilter", - "ofType": null - } - } - }, - "defaultValue": null - }, - { - "name": "componentKind", - "description": null, - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INPUT_OBJECT", - "name": "ComponentKindFilter", - "ofType": null - } - } - }, - "defaultValue": null - }, - { - "name": "or", - "description": null, - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INPUT_OBJECT", - "name": "ComponentsFilter", - "ofType": null - } - } - }, - "defaultValue": null - } - ], - "interfaces": null, - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "INPUT_OBJECT", - "name": "ComponentsSortField", - "description": null, - "fields": null, - "inputFields": [ - { - "name": "field", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "ENUM", - "name": "ComponentsSortFieldName", - "ofType": null - } - }, - "defaultValue": null - }, - { - "name": "direction", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "ENUM", - "name": "Direction", - "ofType": null - } - }, - "defaultValue": "ASC" - } - ], - "interfaces": null, - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "ENUM", - "name": "ComponentsSortFieldName", - "description": null, - "fields": null, - "inputFields": null, - "interfaces": null, - "enumValues": [ - { - "name": "COMPONENT_KEY", - "description": null, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "COMPONENT_KIND", - "description": null, - "isDeprecated": false, - "deprecationReason": null - } - ], - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "CpuMetrics", - "description": null, - "fields": [ - { - "name": "cpuSecondsTotal", - "description": "CPU seconds total", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "SCALAR", - "name": "DateTime", - "description": "Implement the DateTime scalar\n\nThe input/output is a string in RFC3339 format.", - "fields": null, - "inputFields": null, - "interfaces": null, - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "ENUM", - "name": "Direction", - "description": null, - "fields": null, - "inputFields": null, - "interfaces": null, - "enumValues": [ - { - "name": "ASC", - "description": null, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "DESC", - "description": null, - "isDeprecated": false, - "deprecationReason": null - } - ], - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "DiskMetrics", - "description": null, - "fields": [ - { - "name": "readBytesTotal", - "description": "Total bytes read", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "readsCompletedTotal", - "description": "Total reads completed", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "writtenBytesTotal", - "description": "Total bytes written", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "writesCompletedTotal", - "description": "Total writes completed", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "ErrorsTotal", - "description": null, - "fields": [ - { - "name": "timestamp", - "description": "Metric timestamp", - "args": [], - "type": { - "kind": "SCALAR", - "name": "DateTime", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "errorsTotal", - "description": "Total error count", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "ENUM", - "name": "EventEncodingType", - "description": "Encoding format for the event", - "fields": null, - "inputFields": null, - "interfaces": null, - "enumValues": [ - { - "name": "JSON", - "description": null, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "YAML", - "description": null, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "LOGFMT", - "description": null, - "isDeprecated": false, - "deprecationReason": null - } - ], - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "EventNotification", - "description": "A notification regarding events observation", - "fields": [ - { - "name": "notification", - "description": "Notification details", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "UNION", - "name": "Notification", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "message", - "description": "The human-readable message associated with the notification", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "FileSourceMetricFile", - "description": null, - "fields": [ - { - "name": "name", - "description": "File name", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "receivedBytesTotal", - "description": "Metric indicating bytes received for the current file", - "args": [], - "type": { - "kind": "OBJECT", - "name": "ReceivedBytesTotal", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "receivedEventsTotal", - "description": "Metric indicating received events for the current file", - "args": [], - "type": { - "kind": "OBJECT", - "name": "ReceivedEventsTotal", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "sentEventsTotal", - "description": "Metric indicating outgoing events for the current file", - "args": [], - "type": { - "kind": "OBJECT", - "name": "SentEventsTotal", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "FileSourceMetricFileConnection", - "description": null, - "fields": [ - { - "name": "pageInfo", - "description": "Information to aid in pagination.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "PageInfo", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "edges", - "description": "A list of edges.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "FileSourceMetricFileEdge", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "nodes", - "description": "A list of nodes.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "FileSourceMetricFile", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "totalCount", - "description": "Total result set count", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "FileSourceMetricFileEdge", - "description": "An edge in a connection.", - "fields": [ - { - "name": "cursor", - "description": "A cursor for use in pagination", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "node", - "description": "The item at the end of the edge", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "FileSourceMetricFile", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "INPUT_OBJECT", - "name": "FileSourceMetricFilesSortField", - "description": null, - "fields": null, - "inputFields": [ - { - "name": "field", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "ENUM", - "name": "FileSourceMetricFilesSortFieldName", - "ofType": null - } - }, - "defaultValue": null - }, - { - "name": "direction", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "ENUM", - "name": "Direction", - "ofType": null - } - }, - "defaultValue": "ASC" - } - ], - "interfaces": null, - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "ENUM", - "name": "FileSourceMetricFilesSortFieldName", - "description": null, - "fields": null, - "inputFields": null, - "interfaces": null, - "enumValues": [ - { - "name": "NAME", - "description": null, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "RECEIVED_BYTES_TOTAL", - "description": null, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "RECEIVED_EVENTS_TOTAL", - "description": null, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "SENT_EVENTS_TOTAL", - "description": null, - "isDeprecated": false, - "deprecationReason": null - } - ], - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "FileSourceMetrics", - "description": null, - "fields": [ - { - "name": "files", - "description": "File metrics", - "args": [ - { - "name": "after", - "description": null, - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "before", - "description": null, - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "first", - "description": null, - "type": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "last", - "description": null, - "type": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "filter", - "description": null, - "type": { - "kind": "INPUT_OBJECT", - "name": "FileSourceMetricsFilesFilter", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "sort", - "description": null, - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INPUT_OBJECT", - "name": "FileSourceMetricFilesSortField", - "ofType": null - } - } - }, - "defaultValue": null - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "FileSourceMetricFileConnection", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "receivedBytesTotal", - "description": "Total received bytes for the current file source", - "args": [], - "type": { - "kind": "OBJECT", - "name": "ReceivedBytesTotal", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "receivedEventsTotal", - "description": "Total received events for the current file source", - "args": [], - "type": { - "kind": "OBJECT", - "name": "ReceivedEventsTotal", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "sentEventsTotal", - "description": "Total sent events for the current file source", - "args": [], - "type": { - "kind": "OBJECT", - "name": "SentEventsTotal", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [ - { - "kind": "INTERFACE", - "name": "SourceMetrics", - "ofType": null - } - ], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "INPUT_OBJECT", - "name": "FileSourceMetricsFilesFilter", - "description": null, - "fields": null, - "inputFields": [ - { - "name": "name", - "description": null, - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INPUT_OBJECT", - "name": "StringFilter", - "ofType": null - } - } - }, - "defaultValue": null - }, - { - "name": "or", - "description": null, - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INPUT_OBJECT", - "name": "FileSourceMetricsFilesFilter", - "ofType": null - } - } - }, - "defaultValue": null - } - ], - "interfaces": null, - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "FileSystemMetrics", - "description": null, - "fields": [ - { - "name": "freeBytes", - "description": "Free bytes", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "totalBytes", - "description": "Total bytes", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "usedBytes", - "description": "Used bytes", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "SCALAR", - "name": "Float", - "description": "The `Float` scalar type represents signed double-precision fractional values as specified by [IEEE 754](https://en.wikipedia.org/wiki/IEEE_floating_point).", - "fields": null, - "inputFields": null, - "interfaces": null, - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "GenericSinkMetrics", - "description": null, - "fields": [ - { - "name": "receivedEventsTotal", - "description": "Total received events for the current sink", - "args": [], - "type": { - "kind": "OBJECT", - "name": "ReceivedEventsTotal", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "sentBytesTotal", - "description": "Total sent bytes for the current sink", - "args": [], - "type": { - "kind": "OBJECT", - "name": "SentBytesTotal", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "sentEventsTotal", - "description": "Total sent events for the current sink", - "args": [], - "type": { - "kind": "OBJECT", - "name": "SentEventsTotal", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [ - { - "kind": "INTERFACE", - "name": "SinkMetrics", - "ofType": null - } - ], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "GenericSourceMetrics", - "description": null, - "fields": [ - { - "name": "receivedBytesTotal", - "description": "Total received bytes for the current source", - "args": [], - "type": { - "kind": "OBJECT", - "name": "ReceivedBytesTotal", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "receivedEventsTotal", - "description": "Total received events for the current source", - "args": [], - "type": { - "kind": "OBJECT", - "name": "ReceivedEventsTotal", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "sentEventsTotal", - "description": "Total sent events for the current source", - "args": [], - "type": { - "kind": "OBJECT", - "name": "SentEventsTotal", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [ - { - "kind": "INTERFACE", - "name": "SourceMetrics", - "ofType": null - } - ], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "GenericTransformMetrics", - "description": null, - "fields": [ - { - "name": "receivedEventsTotal", - "description": "Total received events for the current transform", - "args": [], - "type": { - "kind": "OBJECT", - "name": "ReceivedEventsTotal", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "sentEventsTotal", - "description": "Total sent events for the current transform", - "args": [], - "type": { - "kind": "OBJECT", - "name": "SentEventsTotal", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [ - { - "kind": "INTERFACE", - "name": "TransformMetrics", - "ofType": null - } - ], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "Heartbeat", - "description": null, - "fields": [ - { - "name": "utc", - "description": null, - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "DateTime", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "HostMetrics", - "description": "Vector host metrics", - "fields": [ - { - "name": "memory", - "description": "Memory metrics", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "MemoryMetrics", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "swap", - "description": "Swap metrics", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "SwapMetrics", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "cpu", - "description": "CPU metrics", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "CpuMetrics", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "loadAverage", - "description": "Load average metrics (*nix only)", - "args": [], - "type": { - "kind": "OBJECT", - "name": "LoadAverageMetrics", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "network", - "description": "Network metrics", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "NetworkMetrics", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "filesystem", - "description": "Filesystem metrics", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "FileSystemMetrics", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "disk", - "description": "Disk metrics", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "DiskMetrics", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "SCALAR", - "name": "ID", - "description": null, - "fields": null, - "inputFields": null, - "interfaces": null, - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "SCALAR", - "name": "Int", - "description": "The `Int` scalar type represents non-fractional whole numeric values.", - "fields": null, - "inputFields": null, - "interfaces": null, - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "InvalidMatch", - "description": "The pattern matched source(s) which cannot be tapped for inputs or sink(s)\nwhich cannot be tapped for outputs", - "fields": [ - { - "name": "pattern", - "description": "Pattern that raised the notification", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "invalidMatches", - "description": "Any invalid matches for the pattern", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "LoadAverageMetrics", - "description": null, - "fields": [ - { - "name": "load1", - "description": "Load 1 average", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "load5", - "description": "Load 5 average", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "load15", - "description": "Load 15 average", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "Log", - "description": "Log event with fields for querying log data", - "fields": [ - { - "name": "componentId", - "description": "Id of the component associated with the log event", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "componentType", - "description": "Type of component associated with the log event", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "componentKind", - "description": "Kind of component associated with the log event", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "message", - "description": "Log message", - "args": [], - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "timestamp", - "description": "Log timestamp", - "args": [], - "type": { - "kind": "SCALAR", - "name": "DateTime", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "string", - "description": "Log event as an encoded string format", - "args": [ - { - "name": "encoding", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "ENUM", - "name": "EventEncodingType", - "ofType": null - } - }, - "defaultValue": null - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "json", - "description": "Get JSON field data on the log event, by field name", - "args": [ - { - "name": "field", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "defaultValue": null - } - ], - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "Matched", - "description": "A component was found that matched the provided pattern", - "fields": [ - { - "name": "pattern", - "description": "Pattern that raised the notification", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "MemoryMetrics", - "description": "Host memory metrics", - "fields": [ - { - "name": "totalBytes", - "description": "Total bytes", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "freeBytes", - "description": "Free bytes", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "availableBytes", - "description": "Available bytes", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "activeBytes", - "description": "Active bytes (Linux/macOS only)", - "args": [], - "type": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "buffersBytes", - "description": "Buffers bytes (Linux only)", - "args": [], - "type": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "cachedBytes", - "description": "Cached bytes (Linux only)", - "args": [], - "type": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "sharedBytes", - "description": "Shared bytes (Linux only)", - "args": [], - "type": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "usedBytes", - "description": "Used bytes (Linux only)", - "args": [], - "type": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "inactiveBytes", - "description": "Inactive bytes (macOS only)", - "args": [], - "type": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "wiredBytes", - "description": "Wired bytes (macOS only)", - "args": [], - "type": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "Meta", - "description": null, - "fields": [ - { - "name": "versionString", - "description": "Vector version", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "hostname", - "description": "Hostname", - "args": [], - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "Metric", - "description": "Metric event with fields for querying metric data", - "fields": [ - { - "name": "componentId", - "description": "Id of the component associated with the metric event", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "componentType", - "description": "Type of component associated with the metric event", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "componentKind", - "description": "Kind of component associated with the metric event", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "timestamp", - "description": "Metric timestamp", - "args": [], - "type": { - "kind": "SCALAR", - "name": "DateTime", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "name", - "description": "Metric name", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "namespace", - "description": "Metric namespace", - "args": [], - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "kind", - "description": "Metric kind", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "ENUM", - "name": "MetricKind", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "valueType", - "description": "Metric type", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "value", - "description": "Metric value in human readable form", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "tags", - "description": "Metric tags", - "args": [], - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "MetricTag", - "ofType": null - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "string", - "description": "Metric event as an encoded string format", - "args": [ - { - "name": "encoding", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "ENUM", - "name": "EventEncodingType", - "ofType": null - } - }, - "defaultValue": null - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "ENUM", - "name": "MetricKind", - "description": null, - "fields": null, - "inputFields": null, - "interfaces": null, - "enumValues": [ - { - "name": "INCREMENTAL", - "description": "Incremental metrics update previous values", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "ABSOLUTE", - "description": "Absolute metrics set the reference value for future updates", - "isDeprecated": false, - "deprecationReason": null - } - ], - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "MetricTag", - "description": null, - "fields": [ - { - "name": "key", - "description": "Metric tag key", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "value", - "description": "Metric tag value", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "INTERFACE", - "name": "MetricType", - "description": null, - "fields": [ - { - "name": "timestamp", - "description": null, - "args": [], - "type": { - "kind": "SCALAR", - "name": "DateTime", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": null, - "enumValues": null, - "possibleTypes": [ - { - "kind": "OBJECT", - "name": "Uptime", - "ofType": null - } - ] - }, - { - "kind": "OBJECT", - "name": "NetworkMetrics", - "description": null, - "fields": [ - { - "name": "receiveBytesTotal", - "description": "Total bytes received", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "receiveErrsTotal", - "description": "Total errors received", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "receivePacketsTotal", - "description": "Total packets received", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "transmitBytesTotal", - "description": "Total bytes transmitted", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "transmitErrsTotal", - "description": "Total errors transmitted", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "transmitPacketsDropTotal", - "description": "Total transmission packets dropped (Linux/Windows only)", - "args": [], - "type": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "transmitPacketsTotal", - "description": "Total transmission packets (Linux/Windows only)", - "args": [], - "type": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "NotMatched", - "description": "There isn't currently a component that matches this pattern", - "fields": [ - { - "name": "pattern", - "description": "Pattern that raised the notification", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "UNION", - "name": "Notification", - "description": "A specific kind of notification with additional details", - "fields": null, - "inputFields": null, - "interfaces": null, - "enumValues": null, - "possibleTypes": [ - { - "kind": "OBJECT", - "name": "Matched", - "ofType": null - }, - { - "kind": "OBJECT", - "name": "NotMatched", - "ofType": null - }, - { - "kind": "OBJECT", - "name": "InvalidMatch", - "ofType": null - } - ] - }, - { - "kind": "OBJECT", - "name": "Output", - "description": null, - "fields": [ - { - "name": "outputId", - "description": "Id of the output stream", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "sentEventsTotal", - "description": "Total sent events for the current output stream", - "args": [], - "type": { - "kind": "OBJECT", - "name": "SentEventsTotal", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "UNION", - "name": "OutputEventsPayload", - "description": "An event or a notification", - "fields": null, - "inputFields": null, - "interfaces": null, - "enumValues": null, - "possibleTypes": [ - { - "kind": "OBJECT", - "name": "Log", - "ofType": null - }, - { - "kind": "OBJECT", - "name": "Metric", - "ofType": null - }, - { - "kind": "OBJECT", - "name": "EventNotification", - "ofType": null - }, - { - "kind": "OBJECT", - "name": "Trace", - "ofType": null - } - ] - }, - { - "kind": "OBJECT", - "name": "OutputThroughput", - "description": null, - "fields": [ - { - "name": "outputId", - "description": "Id of the output stream", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "throughput", - "description": "Throughput for the output stream", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "PageInfo", - "description": "Information about pagination in a connection", - "fields": [ - { - "name": "hasPreviousPage", - "description": "When paginating backwards, are there more items?", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Boolean", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "hasNextPage", - "description": "When paginating forwards, are there more items?", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Boolean", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "startCursor", - "description": "When paginating backwards, the cursor to continue.", - "args": [], - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "endCursor", - "description": "When paginating forwards, the cursor to continue.", - "args": [], - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "Query", - "description": null, - "fields": [ - { - "name": "health", - "description": "Returns `true` to denote the GraphQL server is reachable", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Boolean", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "components", - "description": "Configured components (sources/transforms/sinks)", - "args": [ - { - "name": "after", - "description": null, - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "before", - "description": null, - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "first", - "description": null, - "type": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "last", - "description": null, - "type": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "filter", - "description": null, - "type": { - "kind": "INPUT_OBJECT", - "name": "ComponentsFilter", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "sort", - "description": null, - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INPUT_OBJECT", - "name": "ComponentsSortField", - "ofType": null - } - } - }, - "defaultValue": null - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "ComponentConnection", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "sources", - "description": "Configured sources", - "args": [ - { - "name": "after", - "description": null, - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "before", - "description": null, - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "first", - "description": null, - "type": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "last", - "description": null, - "type": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "filter", - "description": null, - "type": { - "kind": "INPUT_OBJECT", - "name": "SourcesFilter", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "sort", - "description": null, - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INPUT_OBJECT", - "name": "SourcesSortField", - "ofType": null - } - } - }, - "defaultValue": null - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "SourceConnection", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "transforms", - "description": "Configured transforms", - "args": [ - { - "name": "after", - "description": null, - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "before", - "description": null, - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "first", - "description": null, - "type": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "last", - "description": null, - "type": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "filter", - "description": null, - "type": { - "kind": "INPUT_OBJECT", - "name": "TransformsFilter", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "sort", - "description": null, - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INPUT_OBJECT", - "name": "TransformsSortField", - "ofType": null - } - } - }, - "defaultValue": null - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "TransformConnection", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "sinks", - "description": "Configured sinks", - "args": [ - { - "name": "after", - "description": null, - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "before", - "description": null, - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "first", - "description": null, - "type": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "last", - "description": null, - "type": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "filter", - "description": null, - "type": { - "kind": "INPUT_OBJECT", - "name": "SinksFilter", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "sort", - "description": null, - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INPUT_OBJECT", - "name": "SinksSortField", - "ofType": null - } - } - }, - "defaultValue": null - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "SinkConnection", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "componentByComponentKey", - "description": "Gets a configured component by component_key", - "args": [ - { - "name": "componentId", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "defaultValue": null - } - ], - "type": { - "kind": "INTERFACE", - "name": "Component", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "hostMetrics", - "description": "Vector host metrics", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "HostMetrics", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "meta", - "description": null, - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "Meta", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "ReceivedBytesTotal", - "description": null, - "fields": [ - { - "name": "timestamp", - "description": "Metric timestamp.", - "args": [], - "type": { - "kind": "SCALAR", - "name": "DateTime", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "receivedBytesTotal", - "description": "Total number of bytes received.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "ReceivedEventsTotal", - "description": null, - "fields": [ - { - "name": "timestamp", - "description": "Metric timestamp", - "args": [], - "type": { - "kind": "SCALAR", - "name": "DateTime", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "receivedEventsTotal", - "description": "Total incoming events", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "SentBytesTotal", - "description": null, - "fields": [ - { - "name": "timestamp", - "description": "Metric timestamp.", - "args": [], - "type": { - "kind": "SCALAR", - "name": "DateTime", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "sentBytesTotal", - "description": "Total number of bytes sent.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "SentEventsTotal", - "description": null, - "fields": [ - { - "name": "timestamp", - "description": "Metric timestamp", - "args": [], - "type": { - "kind": "SCALAR", - "name": "DateTime", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "sentEventsTotal", - "description": "Total sent events", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "Sink", - "description": null, - "fields": [ - { - "name": "componentId", - "description": "Sink component_id", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "componentType", - "description": "Sink type", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "sources", - "description": "Source inputs", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "Source", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "transforms", - "description": "Transform inputs", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "Transform", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "metrics", - "description": "Sink metrics", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INTERFACE", - "name": "SinkMetrics", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [ - { - "kind": "INTERFACE", - "name": "Component", - "ofType": null - } - ], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "SinkConnection", - "description": null, - "fields": [ - { - "name": "pageInfo", - "description": "Information to aid in pagination.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "PageInfo", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "edges", - "description": "A list of edges.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "SinkEdge", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "nodes", - "description": "A list of nodes.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "Sink", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "totalCount", - "description": "Total result set count", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "SinkEdge", - "description": "An edge in a connection.", - "fields": [ - { - "name": "cursor", - "description": "A cursor for use in pagination", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "node", - "description": "The item at the end of the edge", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "Sink", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "INTERFACE", - "name": "SinkMetrics", - "description": null, - "fields": [ - { - "name": "receivedEventsTotal", - "description": null, - "args": [], - "type": { - "kind": "OBJECT", - "name": "ReceivedEventsTotal", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "sentBytesTotal", - "description": null, - "args": [], - "type": { - "kind": "OBJECT", - "name": "SentBytesTotal", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "sentEventsTotal", - "description": null, - "args": [], - "type": { - "kind": "OBJECT", - "name": "SentEventsTotal", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": null, - "enumValues": null, - "possibleTypes": [ - { - "kind": "OBJECT", - "name": "GenericSinkMetrics", - "ofType": null - } - ] - }, - { - "kind": "INPUT_OBJECT", - "name": "SinksFilter", - "description": null, - "fields": null, - "inputFields": [ - { - "name": "componentId", - "description": null, - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INPUT_OBJECT", - "name": "StringFilter", - "ofType": null - } - } - }, - "defaultValue": null - }, - { - "name": "componentType", - "description": null, - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INPUT_OBJECT", - "name": "StringFilter", - "ofType": null - } - } - }, - "defaultValue": null - }, - { - "name": "or", - "description": null, - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INPUT_OBJECT", - "name": "SinksFilter", - "ofType": null - } - } - }, - "defaultValue": null - } - ], - "interfaces": null, - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "INPUT_OBJECT", - "name": "SinksSortField", - "description": null, - "fields": null, - "inputFields": [ - { - "name": "field", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "ENUM", - "name": "SinksSortFieldName", - "ofType": null - } - }, - "defaultValue": null - }, - { - "name": "direction", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "ENUM", - "name": "Direction", - "ofType": null - } - }, - "defaultValue": "ASC" - } - ], - "interfaces": null, - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "ENUM", - "name": "SinksSortFieldName", - "description": null, - "fields": null, - "inputFields": null, - "interfaces": null, - "enumValues": [ - { - "name": "COMPONENT_KEY", - "description": null, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "COMPONENT_TYPE", - "description": null, - "isDeprecated": false, - "deprecationReason": null - } - ], - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "Source", - "description": null, - "fields": [ - { - "name": "componentId", - "description": "Source component_id", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "componentType", - "description": "Source type", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "outputTypes", - "description": "Source output type", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "ENUM", - "name": "SourceOutputType", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "outputs", - "description": "Source output streams", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "Output", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "transforms", - "description": "Transform outputs", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "Transform", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "sinks", - "description": "Sink outputs", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "Sink", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "metrics", - "description": "Source metrics", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INTERFACE", - "name": "SourceMetrics", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [ - { - "kind": "INTERFACE", - "name": "Component", - "ofType": null - } - ], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "SourceConnection", - "description": null, - "fields": [ - { - "name": "pageInfo", - "description": "Information to aid in pagination.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "PageInfo", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "edges", - "description": "A list of edges.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "SourceEdge", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "nodes", - "description": "A list of nodes.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "Source", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "totalCount", - "description": "Total result set count", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "SourceEdge", - "description": "An edge in a connection.", - "fields": [ - { - "name": "cursor", - "description": "A cursor for use in pagination", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "node", - "description": "The item at the end of the edge", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "Source", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "INTERFACE", - "name": "SourceMetrics", - "description": null, - "fields": [ - { - "name": "receivedBytesTotal", - "description": null, - "args": [], - "type": { - "kind": "OBJECT", - "name": "ReceivedBytesTotal", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "receivedEventsTotal", - "description": null, - "args": [], - "type": { - "kind": "OBJECT", - "name": "ReceivedEventsTotal", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "sentEventsTotal", - "description": null, - "args": [], - "type": { - "kind": "OBJECT", - "name": "SentEventsTotal", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": null, - "enumValues": null, - "possibleTypes": [ - { - "kind": "OBJECT", - "name": "GenericSourceMetrics", - "ofType": null - }, - { - "kind": "OBJECT", - "name": "FileSourceMetrics", - "ofType": null - } - ] - }, - { - "kind": "ENUM", - "name": "SourceOutputType", - "description": null, - "fields": null, - "inputFields": null, - "interfaces": null, - "enumValues": [ - { - "name": "LOG", - "description": null, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "METRIC", - "description": null, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "TRACE", - "description": null, - "isDeprecated": false, - "deprecationReason": null - } - ], - "possibleTypes": null - }, - { - "kind": "INPUT_OBJECT", - "name": "SourceOutputTypeFilter", - "description": null, - "fields": null, - "inputFields": [ - { - "name": "equals", - "description": null, - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "ENUM", - "name": "SourceOutputType", - "ofType": null - } - } - }, - "defaultValue": null - }, - { - "name": "notEquals", - "description": null, - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "ENUM", - "name": "SourceOutputType", - "ofType": null - } - } - }, - "defaultValue": null - }, - { - "name": "contains", - "description": null, - "type": { - "kind": "ENUM", - "name": "SourceOutputType", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "notContains", - "description": null, - "type": { - "kind": "ENUM", - "name": "SourceOutputType", - "ofType": null - }, - "defaultValue": null - } - ], - "interfaces": null, - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "INPUT_OBJECT", - "name": "SourcesFilter", - "description": null, - "fields": null, - "inputFields": [ - { - "name": "componentId", - "description": null, - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INPUT_OBJECT", - "name": "StringFilter", - "ofType": null - } - } - }, - "defaultValue": null - }, - { - "name": "componentType", - "description": null, - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INPUT_OBJECT", - "name": "StringFilter", - "ofType": null - } - } - }, - "defaultValue": null - }, - { - "name": "outputType", - "description": null, - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INPUT_OBJECT", - "name": "SourceOutputTypeFilter", - "ofType": null - } - } - }, - "defaultValue": null - }, - { - "name": "or", - "description": null, - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INPUT_OBJECT", - "name": "SourcesFilter", - "ofType": null - } - } - }, - "defaultValue": null - } - ], - "interfaces": null, - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "INPUT_OBJECT", - "name": "SourcesSortField", - "description": null, - "fields": null, - "inputFields": [ - { - "name": "field", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "ENUM", - "name": "SourcesSortFieldName", - "ofType": null - } - }, - "defaultValue": null - }, - { - "name": "direction", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "ENUM", - "name": "Direction", - "ofType": null - } - }, - "defaultValue": "ASC" - } - ], - "interfaces": null, - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "ENUM", - "name": "SourcesSortFieldName", - "description": null, - "fields": null, - "inputFields": null, - "interfaces": null, - "enumValues": [ - { - "name": "COMPONENT_KEY", - "description": null, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "COMPONENT_TYPE", - "description": null, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "OUTPUT_TYPE", - "description": null, - "isDeprecated": false, - "deprecationReason": null - } - ], - "possibleTypes": null - }, - { - "kind": "SCALAR", - "name": "String", - "description": "The `String` scalar type represents textual data, represented as UTF-8\ncharacter sequences. The String type is most often used by GraphQL to\nrepresent free-form human-readable text.", - "fields": null, - "inputFields": null, - "interfaces": null, - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "INPUT_OBJECT", - "name": "StringFilter", - "description": "Filter for String values", - "fields": null, - "inputFields": [ - { - "name": "equals", - "description": null, - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "notEquals", - "description": null, - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "contains", - "description": null, - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "notContains", - "description": null, - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "startsWith", - "description": null, - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "endsWith", - "description": null, - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "defaultValue": null - } - ], - "interfaces": null, - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "Subscription", - "description": null, - "fields": [ - { - "name": "heartbeat", - "description": "Heartbeat, containing the UTC timestamp of the last server-sent payload", - "args": [ - { - "name": "interval", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "defaultValue": "1000" - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "Heartbeat", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "uptime", - "description": "Metrics for how long the Vector instance has been running", - "args": [ - { - "name": "interval", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "defaultValue": "1000" - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "Uptime", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "receivedEventsTotal", - "description": "Total received events metrics", - "args": [ - { - "name": "interval", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "defaultValue": "1000" - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "ReceivedEventsTotal", - "ofType": null - } - }, - "isDeprecated": true, - "deprecationReason": "Use component_received_events_totals instead" - }, - { - "name": "receivedEventsThroughput", - "description": "Total received events throughput sampled over the provided millisecond `interval`", - "args": [ - { - "name": "interval", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "defaultValue": "1000" - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "isDeprecated": true, - "deprecationReason": "Use component_received_events_throughputs instead" - }, - { - "name": "componentReceivedEventsThroughputs", - "description": "Total incoming component events throughput metrics over `interval`", - "args": [ - { - "name": "interval", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "defaultValue": "1000" - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "ComponentReceivedEventsThroughput", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "componentReceivedEventsTotals", - "description": "Total received component event metrics over `interval`", - "args": [ - { - "name": "interval", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "defaultValue": "1000" - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "ComponentReceivedEventsTotal", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "sentEventsTotal", - "description": "Total sent events metrics", - "args": [ - { - "name": "interval", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "defaultValue": "1000" - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "SentEventsTotal", - "ofType": null - } - }, - "isDeprecated": true, - "deprecationReason": "Use component_sent_events_totals instead" - }, - { - "name": "sentEventsThroughput", - "description": "Total outgoing events throughput sampled over the provided millisecond `interval`", - "args": [ - { - "name": "interval", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "defaultValue": "1000" - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "isDeprecated": true, - "deprecationReason": "Use component_sent_events_throughputs instead" - }, - { - "name": "componentSentEventsThroughputs", - "description": "Total outgoing component event throughput metrics over `interval`", - "args": [ - { - "name": "interval", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "defaultValue": "1000" - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "ComponentSentEventsThroughput", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "componentSentEventsTotals", - "description": "Total outgoing component event metrics over `interval`", - "args": [ - { - "name": "interval", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "defaultValue": "1000" - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "ComponentSentEventsTotal", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "componentReceivedBytesTotals", - "description": "Component bytes received metrics over `interval`.", - "args": [ - { - "name": "interval", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "defaultValue": "1000" - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "ComponentReceivedBytesTotal", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "componentReceivedBytesThroughputs", - "description": "Component bytes received throughput over `interval`", - "args": [ - { - "name": "interval", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "defaultValue": "1000" - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "ComponentReceivedBytesThroughput", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "componentSentBytesTotals", - "description": "Component bytes sent metrics over `interval`.", - "args": [ - { - "name": "interval", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "defaultValue": "1000" - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "ComponentSentBytesTotal", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "componentSentBytesThroughputs", - "description": "Component bytes sent throughput over `interval`", - "args": [ - { - "name": "interval", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "defaultValue": "1000" - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "ComponentSentBytesThroughput", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "errorsTotal", - "description": "Total error metrics.", - "args": [ - { - "name": "interval", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "defaultValue": "1000" - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "ErrorsTotal", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "allocatedBytes", - "description": "Allocated bytes metrics.", - "args": [ - { - "name": "interval", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "defaultValue": "1000" - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "AllocatedBytes", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "componentAllocatedBytes", - "description": "Component allocation metrics", - "args": [ - { - "name": "interval", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "defaultValue": "1000" - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "ComponentAllocatedBytes", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "componentErrorsTotals", - "description": "Component error metrics over `interval`.", - "args": [ - { - "name": "interval", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "defaultValue": "1000" - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "ComponentErrorsTotal", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "metrics", - "description": "All metrics.", - "args": [ - { - "name": "interval", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "defaultValue": "1000" - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INTERFACE", - "name": "MetricType", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "componentAdded", - "description": "Subscribes to all newly added components", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INTERFACE", - "name": "Component", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "componentRemoved", - "description": "Subscribes to all removed components", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INTERFACE", - "name": "Component", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "outputEventsByComponentIdPatterns", - "description": "A stream of events emitted from matched component ID patterns", - "args": [ - { - "name": "outputsPatterns", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - } - } - }, - "defaultValue": null - }, - { - "name": "inputsPatterns", - "description": null, - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - } - }, - "defaultValue": null - }, - { - "name": "interval", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "defaultValue": "500" - }, - { - "name": "limit", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "defaultValue": "100" - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "UNION", - "name": "OutputEventsPayload", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "SwapMetrics", - "description": null, - "fields": [ - { - "name": "freeBytes", - "description": "Swap free bytes", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "totalBytes", - "description": "Swap total bytes", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "usedBytes", - "description": "Swap used bytes", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "swappedInBytesTotal", - "description": "Swapped in bytes total (not available on Windows)", - "args": [], - "type": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "swappedOutBytesTotal", - "description": "Swapped out bytes total (not available on Windows)", - "args": [], - "type": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "Trace", - "description": "Trace event with fields for querying trace data", - "fields": [ - { - "name": "componentId", - "description": "Id of the component associated with the trace event", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "componentType", - "description": "Type of component associated with the trace event", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "componentKind", - "description": "Kind of component associated with the trace event", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "string", - "description": "Trace event as an encoded string format", - "args": [ - { - "name": "encoding", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "ENUM", - "name": "EventEncodingType", - "ofType": null - } - }, - "defaultValue": null - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "json", - "description": "Get JSON field data on the trace event, by field name", - "args": [ - { - "name": "field", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "defaultValue": null - } - ], - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "Transform", - "description": null, - "fields": [ - { - "name": "componentId", - "description": "Transform component_id", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "componentType", - "description": "Transform type", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "outputs", - "description": "Transform output streams", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "Output", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "sources", - "description": "Source inputs", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "Source", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "transforms", - "description": "Transform outputs", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "Transform", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "sinks", - "description": "Sink outputs", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "Sink", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "metrics", - "description": "Transform metrics", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INTERFACE", - "name": "TransformMetrics", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [ - { - "kind": "INTERFACE", - "name": "Component", - "ofType": null - } - ], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "TransformConnection", - "description": null, - "fields": [ - { - "name": "pageInfo", - "description": "Information to aid in pagination.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "PageInfo", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "edges", - "description": "A list of edges.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "TransformEdge", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "nodes", - "description": "A list of nodes.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "Transform", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "totalCount", - "description": "Total result set count", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "TransformEdge", - "description": "An edge in a connection.", - "fields": [ - { - "name": "cursor", - "description": "A cursor for use in pagination", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "node", - "description": "The item at the end of the edge", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "Transform", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "INTERFACE", - "name": "TransformMetrics", - "description": null, - "fields": [ - { - "name": "receivedEventsTotal", - "description": null, - "args": [], - "type": { - "kind": "OBJECT", - "name": "ReceivedEventsTotal", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "sentEventsTotal", - "description": null, - "args": [], - "type": { - "kind": "OBJECT", - "name": "SentEventsTotal", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": null, - "enumValues": null, - "possibleTypes": [ - { - "kind": "OBJECT", - "name": "GenericTransformMetrics", - "ofType": null - } - ] - }, - { - "kind": "INPUT_OBJECT", - "name": "TransformsFilter", - "description": null, - "fields": null, - "inputFields": [ - { - "name": "componentId", - "description": null, - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INPUT_OBJECT", - "name": "StringFilter", - "ofType": null - } - } - }, - "defaultValue": null - }, - { - "name": "componentType", - "description": null, - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INPUT_OBJECT", - "name": "StringFilter", - "ofType": null - } - } - }, - "defaultValue": null - }, - { - "name": "or", - "description": null, - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INPUT_OBJECT", - "name": "TransformsFilter", - "ofType": null - } - } - }, - "defaultValue": null - } - ], - "interfaces": null, - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "INPUT_OBJECT", - "name": "TransformsSortField", - "description": null, - "fields": null, - "inputFields": [ - { - "name": "field", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "ENUM", - "name": "TransformsSortFieldName", - "ofType": null - } - }, - "defaultValue": null - }, - { - "name": "direction", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "ENUM", - "name": "Direction", - "ofType": null - } - }, - "defaultValue": "ASC" - } - ], - "interfaces": null, - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "ENUM", - "name": "TransformsSortFieldName", - "description": null, - "fields": null, - "inputFields": null, - "interfaces": null, - "enumValues": [ - { - "name": "COMPONENT_KEY", - "description": null, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "COMPONENT_TYPE", - "description": null, - "isDeprecated": false, - "deprecationReason": null - } - ], - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "Uptime", - "description": null, - "fields": [ - { - "name": "timestamp", - "description": "Metric timestamp", - "args": [], - "type": { - "kind": "SCALAR", - "name": "DateTime", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "seconds", - "description": "Number of seconds the Vector instance has been alive", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [ - { - "kind": "INTERFACE", - "name": "MetricType", - "ofType": null - } - ], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "__Directive", - "description": "A Directive provides a way to describe alternate runtime execution and type\nvalidation behavior in a GraphQL document.\n\nIn some cases, you need to provide options to alter GraphQL's execution\nbehavior in ways field arguments will not suffice, such as conditionally\nincluding or skipping a field. Directives provide this by describing\nadditional information to the executor.", - "fields": [ - { - "name": "name", - "description": null, - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "description", - "description": null, - "args": [], - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "locations", - "description": null, - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "ENUM", - "name": "__DirectiveLocation", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "args", - "description": null, - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "__InputValue", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "isRepeatable", - "description": null, - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Boolean", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "ENUM", - "name": "__DirectiveLocation", - "description": "A Directive can be adjacent to many parts of the GraphQL language, a\n__DirectiveLocation describes one such possible adjacencies.", - "fields": null, - "inputFields": null, - "interfaces": null, - "enumValues": [ - { - "name": "QUERY", - "description": "Location adjacent to a query operation.", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "MUTATION", - "description": "Location adjacent to a mutation operation.", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "SUBSCRIPTION", - "description": "Location adjacent to a subscription operation.", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "FIELD", - "description": "Location adjacent to a field.", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "FRAGMENT_DEFINITION", - "description": "Location adjacent to a fragment definition.", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "FRAGMENT_SPREAD", - "description": "Location adjacent to a fragment spread.", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "INLINE_FRAGMENT", - "description": "Location adjacent to an inline fragment.", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "VARIABLE_DEFINITION", - "description": "Location adjacent to a variable definition.", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "SCHEMA", - "description": "Location adjacent to a schema definition.", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "SCALAR", - "description": "Location adjacent to a scalar definition.", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "OBJECT", - "description": "Location adjacent to an object type definition.", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "FIELD_DEFINITION", - "description": "Location adjacent to a field definition.", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "ARGUMENT_DEFINITION", - "description": "Location adjacent to an argument definition.", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "INTERFACE", - "description": "Location adjacent to an interface definition.", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "UNION", - "description": "Location adjacent to a union definition.", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "ENUM", - "description": "Location adjacent to an enum definition.", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "ENUM_VALUE", - "description": "Location adjacent to an enum value definition.", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "INPUT_OBJECT", - "description": "Location adjacent to an input object type definition.", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "INPUT_FIELD_DEFINITION", - "description": "Location adjacent to an input object field definition.", - "isDeprecated": false, - "deprecationReason": null - } - ], - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "__EnumValue", - "description": "One possible value for a given Enum. Enum values are unique values, not a\nplaceholder for a string or numeric value. However an Enum value is returned\nin a JSON response as a string.", - "fields": [ - { - "name": "name", - "description": null, - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "description", - "description": null, - "args": [], - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "isDeprecated", - "description": null, - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Boolean", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "deprecationReason", - "description": null, - "args": [], - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "__Field", - "description": "Object and Interface types are described by a list of Fields, each of which\nhas a name, potentially a list of arguments, and a return type.", - "fields": [ - { - "name": "name", - "description": null, - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "description", - "description": null, - "args": [], - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "args", - "description": null, - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "__InputValue", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "type", - "description": null, - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "__Type", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "isDeprecated", - "description": null, - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Boolean", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "deprecationReason", - "description": null, - "args": [], - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "__InputValue", - "description": "Arguments provided to Fields or Directives and the input fields of an\nInputObject are represented as Input Values which describe their type and\noptionally a default value.", - "fields": [ - { - "name": "name", - "description": null, - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "description", - "description": null, - "args": [], - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "type", - "description": null, - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "__Type", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "defaultValue", - "description": null, - "args": [], - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "__Schema", - "description": "A GraphQL Schema defines the capabilities of a GraphQL server. It exposes\nall available types and directives on the server, as well as the entry\npoints for query, mutation, and subscription operations.", - "fields": [ - { - "name": "types", - "description": "A list of all types supported by this server.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "__Type", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "queryType", - "description": "The type that query operations will be rooted at.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "__Type", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "mutationType", - "description": "If this server supports mutation, the type that mutation operations will\nbe rooted at.", - "args": [], - "type": { - "kind": "OBJECT", - "name": "__Type", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "subscriptionType", - "description": "If this server support subscription, the type that subscription\noperations will be rooted at.", - "args": [], - "type": { - "kind": "OBJECT", - "name": "__Type", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "directives", - "description": "A list of all directives supported by this server.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "__Directive", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "__Type", - "description": "The fundamental unit of any GraphQL Schema is the type. There are many kinds\nof types in GraphQL as represented by the `__TypeKind` enum.\n\nDepending on the kind of a type, certain fields describe information about\nthat type. Scalar types provide no information beyond a name and\ndescription, while Enum types provide their values. Object and Interface\ntypes provide the fields they describe. Abstract types, Union and Interface,\nprovide the Object types possible at runtime. List and NonNull types compose\nother types.", - "fields": [ - { - "name": "kind", - "description": null, - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "ENUM", - "name": "__TypeKind", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "name", - "description": null, - "args": [], - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "description", - "description": null, - "args": [], - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "fields", - "description": null, - "args": [ - { - "name": "includeDeprecated", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Boolean", - "ofType": null - } - }, - "defaultValue": "false" - } - ], - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "__Field", - "ofType": null - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "interfaces", - "description": null, - "args": [], - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "__Type", - "ofType": null - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "possibleTypes", - "description": null, - "args": [], - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "__Type", - "ofType": null - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "enumValues", - "description": null, - "args": [ - { - "name": "includeDeprecated", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Boolean", - "ofType": null - } - }, - "defaultValue": "false" - } - ], - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "__EnumValue", - "ofType": null - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "inputFields", - "description": null, - "args": [], - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "__InputValue", - "ofType": null - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "ofType", - "description": null, - "args": [], - "type": { - "kind": "OBJECT", - "name": "__Type", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "specifiedByURL", - "description": null, - "args": [], - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "isOneOf", - "description": null, - "args": [], - "type": { - "kind": "SCALAR", - "name": "Boolean", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "ENUM", - "name": "__TypeKind", - "description": "An enum describing what kind of type a given `__Type` is.", - "fields": null, - "inputFields": null, - "interfaces": null, - "enumValues": [ - { - "name": "SCALAR", - "description": "Indicates this type is a scalar.", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "OBJECT", - "description": "Indicates this type is an object. `fields` and `interfaces` are valid\nfields.", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "INTERFACE", - "description": "Indicates this type is an interface. `fields` and `possibleTypes` are\nvalid fields.", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "UNION", - "description": "Indicates this type is a union. `possibleTypes` is a valid field.", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "ENUM", - "description": "Indicates this type is an enum. `enumValues` is a valid field.", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "INPUT_OBJECT", - "description": "Indicates this type is an input object. `inputFields` is a valid field.", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "LIST", - "description": "Indicates this type is a list. `ofType` is a valid field.", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "NON_NULL", - "description": "Indicates this type is a non-null. `ofType` is a valid field.", - "isDeprecated": false, - "deprecationReason": null - } - ], - "possibleTypes": null - } - ], - "directives": [ - { - "name": "include", - "description": "Directs the executor to include this field or fragment only when the `if` argument is true.", - "locations": [ - "FIELD", - "FRAGMENT_SPREAD", - "INLINE_FRAGMENT" - ], - "args": [ - { - "name": "if", - "description": "Included when true.", - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Boolean", - "ofType": null - } - }, - "defaultValue": null - } - ] - }, - { - "name": "skip", - "description": "Directs the executor to skip this field or fragment when the `if` argument is true.", - "locations": [ - "FIELD", - "FRAGMENT_SPREAD", - "INLINE_FRAGMENT" - ], - "args": [ - { - "name": "if", - "description": "Skipped when true.", - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Boolean", - "ofType": null - } - }, - "defaultValue": null - } - ] - } - ] - } - } -} diff --git a/lib/vector-api-client/graphql/subscriptions/component_added.graphql b/lib/vector-api-client/graphql/subscriptions/component_added.graphql deleted file mode 100644 index 9197209bc053b..0000000000000 --- a/lib/vector-api-client/graphql/subscriptions/component_added.graphql +++ /dev/null @@ -1,7 +0,0 @@ -subscription ComponentAddedSubscription { - componentAdded { - __typename - componentId - componentType - } -} diff --git a/lib/vector-api-client/graphql/subscriptions/component_allocated_bytes.graphql b/lib/vector-api-client/graphql/subscriptions/component_allocated_bytes.graphql deleted file mode 100644 index 2624d3dc8dc75..0000000000000 --- a/lib/vector-api-client/graphql/subscriptions/component_allocated_bytes.graphql +++ /dev/null @@ -1,8 +0,0 @@ -subscription ComponentAllocatedBytesSubscription ($interval: Int!) { - componentAllocatedBytes(interval: $interval) { - componentId - metric { - allocatedBytes - } - } -} diff --git a/lib/vector-api-client/graphql/subscriptions/component_errors_totals.graphql b/lib/vector-api-client/graphql/subscriptions/component_errors_totals.graphql deleted file mode 100644 index d8e0b2554e74f..0000000000000 --- a/lib/vector-api-client/graphql/subscriptions/component_errors_totals.graphql +++ /dev/null @@ -1,8 +0,0 @@ -subscription ComponentErrorsTotalsSubscription ($interval: Int!) { - componentErrorsTotals(interval: $interval) { - componentId - metric { - errorsTotal - } - } -} diff --git a/lib/vector-api-client/graphql/subscriptions/component_received_bytes_throughputs.graphql b/lib/vector-api-client/graphql/subscriptions/component_received_bytes_throughputs.graphql deleted file mode 100644 index ec1b018d27107..0000000000000 --- a/lib/vector-api-client/graphql/subscriptions/component_received_bytes_throughputs.graphql +++ /dev/null @@ -1,6 +0,0 @@ -subscription ComponentReceivedBytesThroughputsSubscription($interval: Int!) { - componentReceivedBytesThroughputs(interval: $interval) { - componentId - throughput - } -} diff --git a/lib/vector-api-client/graphql/subscriptions/component_received_bytes_totals.graphql b/lib/vector-api-client/graphql/subscriptions/component_received_bytes_totals.graphql deleted file mode 100644 index 748b8930fb087..0000000000000 --- a/lib/vector-api-client/graphql/subscriptions/component_received_bytes_totals.graphql +++ /dev/null @@ -1,8 +0,0 @@ -subscription ComponentReceivedBytesTotalsSubscription($interval: Int!) { - componentReceivedBytesTotals(interval: $interval) { - componentId - metric { - receivedBytesTotal - } - } -} diff --git a/lib/vector-api-client/graphql/subscriptions/component_received_events_throughputs.graphql b/lib/vector-api-client/graphql/subscriptions/component_received_events_throughputs.graphql deleted file mode 100644 index 6ddea6400db92..0000000000000 --- a/lib/vector-api-client/graphql/subscriptions/component_received_events_throughputs.graphql +++ /dev/null @@ -1,6 +0,0 @@ -subscription ComponentReceivedEventsThroughputsSubscription($interval: Int!) { - componentReceivedEventsThroughputs(interval: $interval) { - componentId - throughput - } -} diff --git a/lib/vector-api-client/graphql/subscriptions/component_received_events_totals.graphql b/lib/vector-api-client/graphql/subscriptions/component_received_events_totals.graphql deleted file mode 100644 index 59bb58a614067..0000000000000 --- a/lib/vector-api-client/graphql/subscriptions/component_received_events_totals.graphql +++ /dev/null @@ -1,8 +0,0 @@ -subscription ComponentReceivedEventsTotalsSubscription($interval: Int!) { - componentReceivedEventsTotals(interval: $interval) { - componentId - metric { - receivedEventsTotal - } - } -} diff --git a/lib/vector-api-client/graphql/subscriptions/component_removed.graphql b/lib/vector-api-client/graphql/subscriptions/component_removed.graphql deleted file mode 100644 index 7232522a20498..0000000000000 --- a/lib/vector-api-client/graphql/subscriptions/component_removed.graphql +++ /dev/null @@ -1,6 +0,0 @@ -subscription ComponentRemovedSubscription { - componentRemoved { - __typename - componentId - } -} diff --git a/lib/vector-api-client/graphql/subscriptions/component_sent_bytes_throughputs.graphql b/lib/vector-api-client/graphql/subscriptions/component_sent_bytes_throughputs.graphql deleted file mode 100644 index a1b4f54c1a300..0000000000000 --- a/lib/vector-api-client/graphql/subscriptions/component_sent_bytes_throughputs.graphql +++ /dev/null @@ -1,6 +0,0 @@ -subscription ComponentSentBytesThroughputsSubscription($interval: Int!) { - componentSentBytesThroughputs(interval: $interval) { - componentId - throughput - } -} diff --git a/lib/vector-api-client/graphql/subscriptions/component_sent_bytes_totals.graphql b/lib/vector-api-client/graphql/subscriptions/component_sent_bytes_totals.graphql deleted file mode 100644 index aa2c2f6dd0832..0000000000000 --- a/lib/vector-api-client/graphql/subscriptions/component_sent_bytes_totals.graphql +++ /dev/null @@ -1,8 +0,0 @@ -subscription ComponentSentBytesTotalsSubscription($interval: Int!) { - componentSentBytesTotals(interval: $interval) { - componentId - metric { - sentBytesTotal - } - } -} diff --git a/lib/vector-api-client/graphql/subscriptions/component_sent_events_throughputs.graphql b/lib/vector-api-client/graphql/subscriptions/component_sent_events_throughputs.graphql deleted file mode 100644 index fe6c86298b0af..0000000000000 --- a/lib/vector-api-client/graphql/subscriptions/component_sent_events_throughputs.graphql +++ /dev/null @@ -1,10 +0,0 @@ -subscription ComponentSentEventsThroughputsSubscription($interval: Int!) { - componentSentEventsThroughputs(interval: $interval) { - componentId - outputs { - outputId - throughput - } - throughput - } -} diff --git a/lib/vector-api-client/graphql/subscriptions/component_sent_events_totals.graphql b/lib/vector-api-client/graphql/subscriptions/component_sent_events_totals.graphql deleted file mode 100644 index 9951c53e3ab16..0000000000000 --- a/lib/vector-api-client/graphql/subscriptions/component_sent_events_totals.graphql +++ /dev/null @@ -1,14 +0,0 @@ -subscription ComponentSentEventsTotalsSubscription($interval: Int!) { - componentSentEventsTotals(interval: $interval) { - componentId - outputs { - outputId - sentEventsTotal { - sentEventsTotal - } - } - metric { - sentEventsTotal - } - } -} diff --git a/lib/vector-api-client/graphql/subscriptions/heartbeat.graphql b/lib/vector-api-client/graphql/subscriptions/heartbeat.graphql deleted file mode 100644 index f5ae0b40f6d6b..0000000000000 --- a/lib/vector-api-client/graphql/subscriptions/heartbeat.graphql +++ /dev/null @@ -1,5 +0,0 @@ -subscription HeartbeatSubscription($interval: Int!) { - heartbeat(interval: $interval) { - utc - } -} diff --git a/lib/vector-api-client/graphql/subscriptions/output_events_by_component_id_patterns.graphql b/lib/vector-api-client/graphql/subscriptions/output_events_by_component_id_patterns.graphql deleted file mode 100644 index 6c17e197b8e7e..0000000000000 --- a/lib/vector-api-client/graphql/subscriptions/output_events_by_component_id_patterns.graphql +++ /dev/null @@ -1,30 +0,0 @@ -subscription OutputEventsByComponentIdPatternsSubscription( - $outputsPatterns: [String!]!, $inputsPatterns: [String!], $limit: Int!, $interval: Int!, $encoding: EventEncodingType!){ - outputEventsByComponentIdPatterns(outputsPatterns: $outputsPatterns, inputsPatterns: $inputsPatterns, limit: $limit, interval: $interval) { - __typename - ... on Log { - componentId - componentType - componentKind - message - timestamp - string(encoding: $encoding) - } - ... on Metric { - componentId - componentType - componentKind - timestamp - string(encoding: $encoding) - } - ... on Trace { - componentId - componentType - componentKind - string(encoding: $encoding) - } - ... on EventNotification { - message - } - } -} diff --git a/lib/vector-api-client/graphql/subscriptions/processed_bytes_throughput.graphql b/lib/vector-api-client/graphql/subscriptions/processed_bytes_throughput.graphql deleted file mode 100644 index d8dabdecc5ed4..0000000000000 --- a/lib/vector-api-client/graphql/subscriptions/processed_bytes_throughput.graphql +++ /dev/null @@ -1,3 +0,0 @@ -subscription ProcessedBytesThroughputSubscription($interval: Int!) { - processedBytesThroughput(interval: $interval) -} diff --git a/lib/vector-api-client/graphql/subscriptions/uptime.graphql b/lib/vector-api-client/graphql/subscriptions/uptime.graphql deleted file mode 100644 index 187d1e861877a..0000000000000 --- a/lib/vector-api-client/graphql/subscriptions/uptime.graphql +++ /dev/null @@ -1,5 +0,0 @@ -subscription UptimeSubscription { - uptime { - seconds - } -} diff --git a/lib/vector-api-client/src/client.rs b/lib/vector-api-client/src/client.rs index 941ffa7963f27..d4991efef0894 100644 --- a/lib/vector-api-client/src/client.rs +++ b/lib/vector-api-client/src/client.rs @@ -1,57 +1,182 @@ -use anyhow::Context; -use graphql_client::GraphQLQuery; -use url::Url; +use http::Uri; +use tokio_stream::{Stream, StreamExt}; +use tonic::transport::{Channel, Endpoint}; -use crate::gql::HealthQueryExt; +use crate::{ + error::{Error, Result}, + proto::{ + GetComponentsRequest, GetComponentsResponse, GetMetaRequest, GetMetaResponse, + HealthRequest, HealthResponse, MetricName, StreamComponentAllocatedBytesRequest, + StreamComponentAllocatedBytesResponse, StreamComponentMetricsRequest, + StreamComponentMetricsResponse, StreamHeartbeatRequest, StreamHeartbeatResponse, + StreamOutputEventsRequest, StreamOutputEventsResponse, StreamUptimeRequest, + StreamUptimeResponse, observability_service_client::ObservabilityServiceClient, + }, +}; -/// Wrapped `Result` type, that returns deserialized GraphQL response data. -pub type QueryResult = - anyhow::Result::ResponseData>>; - -/// GraphQL query client over HTTP. -#[derive(Debug)] +/// gRPC client for the Vector observability API +#[derive(Debug, Clone)] pub struct Client { - url: Url, + endpoint: Endpoint, + client: Option>, } impl Client { - /// Returns a new GraphQL query client, bound to the provided URL. - pub fn new(url: Url) -> Self { - Self { url } - } - - /// Send a health query - pub async fn healthcheck(&self) -> Result<(), ()> { - self.health_query().await.map(|_| ()).map_err(|_| ()) - } - - /// Issue a GraphQL query using Reqwest, serializing the response to the associated - /// GraphQL type for the given `request_body`. - pub async fn query( - &self, - request_body: &graphql_client::QueryBody, - ) -> QueryResult { - let client = reqwest::Client::new(); - - client - .post(self.url.clone()) - .json(request_body) - .send() - .await - .with_context(|| { - format!( - "Couldn't send '{}' query to {}", - request_body.operation_name, - &self.url.as_str() - ) - })? - .json() - .await - .with_context(|| { - format!( - "Couldn't serialize the response for '{}' query: {:?}", - request_body.operation_name, request_body.query - ) + /// Create a new gRPC client + /// + /// The client is not connected until `connect()` is called. + /// + /// # Arguments + /// + /// * `uri` - The gRPC server URI (e.g., `"http://localhost:9999".parse().unwrap()`) + pub fn new(uri: Uri) -> Self { + Self { + endpoint: Endpoint::from(uri), + client: None, + } + } + + /// Connect to the gRPC server + pub async fn connect(&mut self) -> Result<()> { + let channel = self.endpoint.connect().await?; + self.client = Some(ObservabilityServiceClient::new(channel)); + Ok(()) + } + + /// Ensure the client is connected + fn ensure_connected(&mut self) -> Result<&mut ObservabilityServiceClient> { + self.client.as_mut().ok_or(Error::NotConnected) + } + + // ========== Unary RPCs ========== + + /// Check if the API server is healthy + pub async fn health(&mut self) -> Result { + let client = self.ensure_connected()?; + let response = client.health(HealthRequest {}).await?; + Ok(response.into_inner()) + } + + /// Get metadata about the Vector instance + pub async fn get_meta(&mut self) -> Result { + let client = self.ensure_connected()?; + let response = client.get_meta(GetMetaRequest {}).await?; + Ok(response.into_inner()) + } + + /// Get information about configured components + /// + /// # Arguments + /// + /// * `limit` - Maximum number of components to return (0 = no limit) + pub async fn get_components(&mut self, limit: i32) -> Result { + let client = self.ensure_connected()?; + let response = client + .get_components(GetComponentsRequest { limit }) + .await?; + Ok(response.into_inner()) + } + + // ========== Streaming RPCs ========== + + /// Stream periodic heartbeat timestamps + /// + /// # Arguments + /// + /// * `interval_ms` - Update interval in milliseconds + pub async fn stream_heartbeat( + &mut self, + interval_ms: i32, + ) -> Result>> { + let client = self.ensure_connected()?; + let response = client + .stream_heartbeat(StreamHeartbeatRequest { interval_ms }) + .await?; + Ok(response.into_inner().map(|r| r.map_err(Error::from))) + } + + /// Stream uptime in seconds + /// + /// # Arguments + /// + /// * `interval_ms` - Update interval in milliseconds + pub async fn stream_uptime( + &mut self, + interval_ms: i32, + ) -> Result>> { + let client = self.ensure_connected()?; + let response = client + .stream_uptime(StreamUptimeRequest { interval_ms }) + .await?; + Ok(response.into_inner().map(|r| r.map_err(Error::from))) + } + + /// Stream memory allocated per component + /// + /// # Arguments + /// + /// * `interval_ms` - Update interval in milliseconds + pub async fn stream_component_allocated_bytes( + &mut self, + interval_ms: i32, + ) -> Result>> { + let client = self.ensure_connected()?; + let response = client + .stream_component_allocated_bytes(StreamComponentAllocatedBytesRequest { interval_ms }) + .await?; + Ok(response.into_inner().map(|r| r.map_err(Error::from))) + } + + /// Stream per-component metrics for a chosen metric name. + /// + /// # Arguments + /// + /// * `metric` - Which metric to stream + /// * `interval_ms` - Update interval in milliseconds + pub async fn stream_component_metrics( + &mut self, + metric: MetricName, + interval_ms: i32, + ) -> Result>> { + let client = self.ensure_connected()?; + let response = client + .stream_component_metrics(StreamComponentMetricsRequest { + interval_ms, + metric: metric as i32, }) + .await?; + Ok(response.into_inner().map(|r| r.map_err(Error::from))) + } + + /// Stream events from components matching patterns + /// + /// This is used by `vector tap` to capture events. + pub async fn stream_output_events( + &mut self, + request: StreamOutputEventsRequest, + ) -> Result> + use<>> { + let client = self.ensure_connected()?; + let response = client.stream_output_events(request).await?; + Ok(response.into_inner().map(|r| r.map_err(Error::from))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_not_connected_error() { + let mut client = Client::new("http://localhost:9999".parse().unwrap()); + let result = client.health().await; + assert!(matches!(result, Err(Error::NotConnected))); + } + + #[test] + fn test_ensure_connected() { + let mut client = Client::new("http://localhost:9999".parse().unwrap()); + let result = client.ensure_connected(); + assert!(result.is_err()); + assert!(matches!(result.unwrap_err(), Error::NotConnected)); } } diff --git a/lib/vector-api-client/src/error.rs b/lib/vector-api-client/src/error.rs new file mode 100644 index 0000000000000..65fcfd80b511f --- /dev/null +++ b/lib/vector-api-client/src/error.rs @@ -0,0 +1,77 @@ +use snafu::Snafu; + +/// Error types for the gRPC client +#[derive(Debug, Snafu)] +#[snafu(visibility(pub))] +pub enum Error { + #[snafu(display("Connection error: {}", source))] + Connection { source: tonic::transport::Error }, + + #[snafu(display("gRPC error: {}", source))] + Grpc { source: tonic::Status }, + + #[snafu(display("Invalid URL: {}", message))] + InvalidUrl { message: String }, + + #[snafu(display("Not connected to gRPC server"))] + NotConnected, + + #[snafu(display("Stream error: {}", message))] + Stream { message: String }, +} + +impl Error { + /// Returns true if this error is permanent and should not be retried. + pub fn is_fatal(&self) -> bool { + match self { + Error::InvalidUrl { .. } => true, + Error::Grpc { source } => matches!( + source.code(), + tonic::Code::InvalidArgument + | tonic::Code::OutOfRange + | tonic::Code::NotFound + | tonic::Code::PermissionDenied + | tonic::Code::Unauthenticated + | tonic::Code::Unimplemented + ), + _ => false, + } + } +} + +impl From for Error { + fn from(source: tonic::Status) -> Self { + Error::Grpc { source } + } +} + +impl From for Error { + fn from(source: tonic::transport::Error) -> Self { + Error::Connection { source } + } +} + +pub type Result = std::result::Result; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_error_display() { + let err = Error::NotConnected; + assert_eq!(err.to_string(), "Not connected to gRPC server"); + + let err = Error::InvalidUrl { + message: "invalid format".to_string(), + }; + assert_eq!(err.to_string(), "Invalid URL: invalid format"); + } + + #[test] + fn test_error_from_tonic_status() { + let status = tonic::Status::unavailable("service down"); + let err: Error = status.into(); + assert!(matches!(err, Error::Grpc { .. })); + } +} diff --git a/lib/vector-api-client/src/gql/components.rs b/lib/vector-api-client/src/gql/components.rs deleted file mode 100644 index d278b30c317c8..0000000000000 --- a/lib/vector-api-client/src/gql/components.rs +++ /dev/null @@ -1,224 +0,0 @@ -use std::fmt; - -use graphql_client::GraphQLQuery; - -use crate::{BoxedSubscription, QueryResult}; - -/// Components query for returning sources, transforms, and sinks -#[derive(GraphQLQuery, Debug, Copy, Clone)] -#[graphql( - schema_path = "graphql/schema.json", - query_path = "graphql/queries/components.graphql", - response_derives = "Debug" -)] -pub struct ComponentsQuery; - -/// Components subscription for notification when a component has been added -#[derive(GraphQLQuery, Debug, Copy, Clone)] -#[graphql( - schema_path = "graphql/schema.json", - query_path = "graphql/subscriptions/component_added.graphql", - response_derives = "Debug" -)] -pub struct ComponentAddedSubscription; - -/// Components subscription for notification when a component has been removed -#[derive(GraphQLQuery, Debug, Copy, Clone)] -#[graphql( - schema_path = "graphql/schema.json", - query_path = "graphql/subscriptions/component_removed.graphql", - response_derives = "Debug" -)] -pub struct ComponentRemovedSubscription; - -pub trait ComponentsQueryExt { - async fn components_query(&self, first: i64) -> crate::QueryResult; -} - -impl ComponentsQueryExt for crate::Client { - async fn components_query(&self, first: i64) -> QueryResult { - let request_body = ComponentsQuery::build_query(components_query::Variables { first }); - self.query::(&request_body).await - } -} - -pub trait ComponentsSubscriptionExt { - fn component_added(&self) -> crate::BoxedSubscription; - fn component_removed(&self) -> crate::BoxedSubscription; -} - -impl ComponentsSubscriptionExt for crate::SubscriptionClient { - /// Subscription for when a component has been added - fn component_added(&self) -> BoxedSubscription { - let request_body = - ComponentAddedSubscription::build_query(component_added_subscription::Variables); - - self.start::(&request_body) - } - - /// Subscription for when a component has been removed - fn component_removed(&self) -> BoxedSubscription { - let request_body = - ComponentRemovedSubscription::build_query(component_removed_subscription::Variables); - - self.start::(&request_body) - } -} - -impl components_query::ComponentsQueryComponentsEdgesNodeOn { - pub fn received_bytes_total(&self) -> i64 { - // This is network bytes received, and only sources can receive events. - match self { - components_query::ComponentsQueryComponentsEdgesNodeOn::Source(s) => s - .metrics - .received_bytes_total - .as_ref() - .map(|p| p.received_bytes_total as i64) - .unwrap_or(0), - components_query::ComponentsQueryComponentsEdgesNodeOn::Transform(_) => 0, - components_query::ComponentsQueryComponentsEdgesNodeOn::Sink(_) => 0, - } - } - - pub fn received_events_total(&self) -> i64 { - match self { - components_query::ComponentsQueryComponentsEdgesNodeOn::Source(s) => s - .metrics - .received_events_total - .as_ref() - .map(|p| p.received_events_total as i64) - .unwrap_or(0), - components_query::ComponentsQueryComponentsEdgesNodeOn::Transform(t) => t - .metrics - .received_events_total - .as_ref() - .map(|p| p.received_events_total as i64) - .unwrap_or(0), - components_query::ComponentsQueryComponentsEdgesNodeOn::Sink(s) => s - .metrics - .received_events_total - .as_ref() - .map(|p| p.received_events_total as i64) - .unwrap_or(0), - } - } - - pub fn sent_bytes_total(&self) -> i64 { - // This is network bytes sent, and only sinks can send out events. - match self { - components_query::ComponentsQueryComponentsEdgesNodeOn::Source(_) => 0, - components_query::ComponentsQueryComponentsEdgesNodeOn::Transform(_) => 0, - components_query::ComponentsQueryComponentsEdgesNodeOn::Sink(s) => s - .metrics - .sent_bytes_total - .as_ref() - .map(|p| p.sent_bytes_total as i64) - .unwrap_or(0), - } - } - - pub fn sent_events_total(&self) -> i64 { - match self { - components_query::ComponentsQueryComponentsEdgesNodeOn::Source(s) => s - .metrics - .sent_events_total - .as_ref() - .map(|p| p.sent_events_total as i64) - .unwrap_or(0), - components_query::ComponentsQueryComponentsEdgesNodeOn::Transform(t) => t - .metrics - .sent_events_total - .as_ref() - .map(|p| p.sent_events_total as i64) - .unwrap_or(0), - components_query::ComponentsQueryComponentsEdgesNodeOn::Sink(s) => s - .metrics - .sent_events_total - .as_ref() - .map(|p| p.sent_events_total as i64) - .unwrap_or(0), - } - } - - pub fn outputs(&self) -> Vec<(String, i64)> { - match self { - components_query::ComponentsQueryComponentsEdgesNodeOn::Source(s) => s - .outputs - .iter() - .map(|o| { - ( - o.output_id.clone(), - o.sent_events_total - .as_ref() - .map(|p| p.sent_events_total as i64) - .unwrap_or(0), - ) - }) - .collect(), - components_query::ComponentsQueryComponentsEdgesNodeOn::Transform(t) => t - .outputs - .iter() - .map(|o| { - ( - o.output_id.clone(), - o.sent_events_total - .as_ref() - .map(|p| p.sent_events_total as i64) - .unwrap_or(0), - ) - }) - .collect(), - components_query::ComponentsQueryComponentsEdgesNodeOn::Sink(_) => vec![], - } - } -} - -impl fmt::Display for components_query::ComponentsQueryComponentsEdgesNodeOn { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let res = match self { - components_query::ComponentsQueryComponentsEdgesNodeOn::Source(_) => "source", - components_query::ComponentsQueryComponentsEdgesNodeOn::Transform(_) => "transform", - components_query::ComponentsQueryComponentsEdgesNodeOn::Sink(_) => "sink", - }; - - write!(f, "{res}") - } -} - -impl fmt::Display for component_added_subscription::ComponentAddedSubscriptionComponentAddedOn { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let res = match self { - component_added_subscription::ComponentAddedSubscriptionComponentAddedOn::Source => { - "source" - } - component_added_subscription::ComponentAddedSubscriptionComponentAddedOn::Transform => { - "transform" - } - component_added_subscription::ComponentAddedSubscriptionComponentAddedOn::Sink => { - "sink" - } - }; - - write!(f, "{res}") - } -} - -impl fmt::Display - for component_removed_subscription::ComponentRemovedSubscriptionComponentRemovedOn -{ - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let res = match self { - component_removed_subscription::ComponentRemovedSubscriptionComponentRemovedOn::Source => { - "source" - } - component_removed_subscription::ComponentRemovedSubscriptionComponentRemovedOn::Transform => { - "transform" - } - component_removed_subscription::ComponentRemovedSubscriptionComponentRemovedOn::Sink => { - "sink" - } - }; - - write!(f, "{res}") - } -} diff --git a/lib/vector-api-client/src/gql/health.rs b/lib/vector-api-client/src/gql/health.rs deleted file mode 100644 index f3bd4968c8b30..0000000000000 --- a/lib/vector-api-client/src/gql/health.rs +++ /dev/null @@ -1,64 +0,0 @@ -//! Health queries/subscriptions, for asserting a GraphQL API server is alive. - -use graphql_client::GraphQLQuery; - -/// Shorthand for a Chrono datetime, set to UTC. -type DateTime = chrono::DateTime; - -/// HealthQuery is generally used to assert that the GraphQL API server is alive. -/// The `health` field returns true. -#[derive(GraphQLQuery, Debug, Copy, Clone)] -#[graphql( - schema_path = "graphql/schema.json", - query_path = "graphql/queries/health.graphql", - response_derives = "Debug" -)] -pub struct HealthQuery; - -/// HeartbeatSubscription is a subscription that returns a 'heartbeat' in the form -/// of a UTC timestamp. The use-case is allowing a client to assert that the server is -/// sending regular payloads, by using the timestamp to determine when the last healthcheck -/// was successful. -#[derive(GraphQLQuery, Debug, Copy, Clone)] -#[graphql( - schema_path = "graphql/schema.json", - query_path = "graphql/subscriptions/heartbeat.graphql", - response_derives = "Debug" -)] -pub struct HeartbeatSubscription; - -/// Extension methods for health queries. -pub trait HealthQueryExt { - /// Executes a health query. - async fn health_query(&self) -> crate::QueryResult; -} - -impl HealthQueryExt for crate::Client { - /// Executes a health query. - async fn health_query(&self) -> crate::QueryResult { - self.query::(&HealthQuery::build_query(health_query::Variables)) - .await - } -} - -/// Extension methods for health subscriptions -pub trait HealthSubscriptionExt { - /// Executes a heartbeat subscription, on a millisecond `interval`. - fn heartbeat_subscription( - &self, - interval: i64, - ) -> crate::BoxedSubscription; -} - -impl HealthSubscriptionExt for crate::SubscriptionClient { - /// Executes a heartbeat subscription, on a millisecond `interval`. - fn heartbeat_subscription( - &self, - interval: i64, - ) -> crate::BoxedSubscription { - let request_body = - HeartbeatSubscription::build_query(heartbeat_subscription::Variables { interval }); - - self.start::(&request_body) - } -} diff --git a/lib/vector-api-client/src/gql/meta.rs b/lib/vector-api-client/src/gql/meta.rs deleted file mode 100644 index d8d9a734342f3..0000000000000 --- a/lib/vector-api-client/src/gql/meta.rs +++ /dev/null @@ -1,26 +0,0 @@ -use graphql_client::GraphQLQuery; - -/// MetaVersionStringQuery returns the version string of the queried Vector instance. -#[derive(GraphQLQuery, Debug, Copy, Clone)] -#[graphql( - schema_path = "graphql/schema.json", - query_path = "graphql/queries/meta_version_string.graphql", - response_derives = "Debug" -)] -pub struct MetaVersionStringQuery; - -/// Extension methods for meta queries. -pub trait MetaQueryExt { - /// Executes a meta version string query. - async fn meta_version_string(&self) -> crate::QueryResult; -} - -impl MetaQueryExt for crate::Client { - /// Executes a meta version string query. - async fn meta_version_string(&self) -> crate::QueryResult { - self.query::(&MetaVersionStringQuery::build_query( - meta_version_string_query::Variables, - )) - .await - } -} diff --git a/lib/vector-api-client/src/gql/metrics.rs b/lib/vector-api-client/src/gql/metrics.rs deleted file mode 100644 index a9df90f7c0bd7..0000000000000 --- a/lib/vector-api-client/src/gql/metrics.rs +++ /dev/null @@ -1,341 +0,0 @@ -//! Metrics queries/subscriptions. - -use graphql_client::GraphQLQuery; - -use crate::BoxedSubscription; - -/// UptimeSubscription returns uptime metrics to determine how long the Vector -/// instance has been running. -#[derive(GraphQLQuery, Debug, Copy, Clone)] -#[graphql( - schema_path = "graphql/schema.json", - query_path = "graphql/subscriptions/uptime.graphql", - response_derives = "Debug" -)] -pub struct UptimeSubscription; - -/// ComponentAllocatedBytesSubscription contains metrics on the number of allocated bytes -/// that have been processed by a Vector instance, against specific components. -#[derive(GraphQLQuery, Debug, Copy, Clone)] -#[graphql( - schema_path = "graphql/schema.json", - query_path = "graphql/subscriptions/component_allocated_bytes.graphql", - response_derives = "Debug" -)] -pub struct ComponentAllocatedBytesSubscription; - -/// ComponentReceivedBytesThroughputsSubscription contains metrics on the number of bytes -/// that have been received between `interval` samples, against specific components. -#[derive(GraphQLQuery, Debug, Copy, Clone)] -#[graphql( - schema_path = "graphql/schema.json", - query_path = "graphql/subscriptions/component_received_bytes_throughputs.graphql", - response_derives = "Debug" -)] -pub struct ComponentReceivedBytesThroughputsSubscription; - -/// ComponentReceivedBytesTotalsSubscription contains metrics on the number of bytes -/// that have been received by a Vector instance, against a specific component. -#[derive(GraphQLQuery, Debug, Copy, Clone)] -#[graphql( - schema_path = "graphql/schema.json", - query_path = "graphql/subscriptions/component_received_bytes_totals.graphql", - response_derives = "Debug" -)] -pub struct ComponentReceivedBytesTotalsSubscription; - -/// ComponentReceivedEventsThroughputsSubscription contains metrics on the number of events -/// that have been accepted for processing between `interval` samples, against specific components. -#[derive(GraphQLQuery, Debug, Copy, Clone)] -#[graphql( - schema_path = "graphql/schema.json", - query_path = "graphql/subscriptions/component_received_events_throughputs.graphql", - response_derives = "Debug" -)] -pub struct ComponentReceivedEventsThroughputsSubscription; - -/// ComponentReceivedEventsTotalsSubscription contains metrics on the number of events -/// that have been accepted for processing by a Vector instance, against specific components. -#[derive(GraphQLQuery, Debug, Copy, Clone)] -#[graphql( - schema_path = "graphql/schema.json", - query_path = "graphql/subscriptions/component_received_events_totals.graphql", - response_derives = "Debug" -)] -pub struct ComponentReceivedEventsTotalsSubscription; - -/// ComponentSentBytesThroughputsSubscription contains metrics on the number of bytes -/// that have been received between `interval` samples, against specific components. -#[derive(GraphQLQuery, Debug, Copy, Clone)] -#[graphql( - schema_path = "graphql/schema.json", - query_path = "graphql/subscriptions/component_sent_bytes_throughputs.graphql", - response_derives = "Debug" -)] -pub struct ComponentSentBytesThroughputsSubscription; - -/// ComponentSentBytesTotalsSubscription contains metrics on the number of bytes -/// that have been received by a Vector instance, against a specific component. -#[derive(GraphQLQuery, Debug, Copy, Clone)] -#[graphql( - schema_path = "graphql/schema.json", - query_path = "graphql/subscriptions/component_sent_bytes_totals.graphql", - response_derives = "Debug" -)] -pub struct ComponentSentBytesTotalsSubscription; - -/// ComponentSentEventsThroughputsSubscription contains metrics on the number of events -/// that have been emitted between `interval` samples, against specific components. -#[derive(GraphQLQuery, Debug, Copy, Clone)] -#[graphql( - schema_path = "graphql/schema.json", - query_path = "graphql/subscriptions/component_sent_events_throughputs.graphql", - response_derives = "Debug" -)] -pub struct ComponentSentEventsThroughputsSubscription; - -/// ComponentSentEventsTotalsSubscription contains metrics on the number of events -/// that have been emitted by a Vector instance, against specific components. -#[derive(GraphQLQuery, Debug, Copy, Clone)] -#[graphql( - schema_path = "graphql/schema.json", - query_path = "graphql/subscriptions/component_sent_events_totals.graphql", - response_derives = "Debug" -)] -pub struct ComponentSentEventsTotalsSubscription; - -impl component_sent_events_totals_subscription::ComponentSentEventsTotalsSubscriptionComponentSentEventsTotals { - pub fn outputs(&self) -> Vec<(String, i64)> { - self.outputs - .iter() - .map(|output| { - ( - output.output_id.clone(), - output - .sent_events_total - .as_ref() - .map(|p| p.sent_events_total as i64) - .unwrap_or(0), - ) - }) - .collect() - } -} - -impl component_sent_events_throughputs_subscription::ComponentSentEventsThroughputsSubscriptionComponentSentEventsThroughputs { - pub fn outputs(&self) -> Vec<(String, i64)> { - self.outputs - .iter() - .map(|output| { - ( - output.output_id.clone(), - output.throughput, - ) - }) - .collect() - } - -} - -/// ComponentErrorsTotalsSubscription contains metrics on the number of errors -/// (metrics ending in `_errors_total`), against specific components. -#[derive(GraphQLQuery, Debug, Copy, Clone)] -#[graphql( - schema_path = "graphql/schema.json", - query_path = "graphql/subscriptions/component_errors_totals.graphql", - response_derives = "Debug" -)] -pub struct ComponentErrorsTotalsSubscription; - -/// Extension methods for metrics subscriptions -pub trait MetricsSubscriptionExt { - /// Executes an uptime metrics subscription. - fn uptime_subscription(&self) -> crate::BoxedSubscription; - - /// Executes an all component allocated bytes subscription. - fn component_allocated_bytes_subscription( - &self, - interval: i64, - ) -> BoxedSubscription; - - /// Executes a component bytes received totals subscription. - fn component_received_bytes_totals_subscription( - &self, - interval: i64, - ) -> crate::BoxedSubscription; - - /// Executes a component bytes received throughput subscription. - fn component_received_bytes_throughputs_subscription( - &self, - interval: i64, - ) -> crate::BoxedSubscription; - - /// Executes a component received events totals subscription. - fn component_received_events_totals_subscription( - &self, - interval: i64, - ) -> crate::BoxedSubscription; - - /// Executes an component events in throughputs subscription. - fn component_received_events_throughputs_subscription( - &self, - interval: i64, - ) -> crate::BoxedSubscription; - - /// Executes a component bytes sent totals subscription. - fn component_sent_bytes_totals_subscription( - &self, - interval: i64, - ) -> crate::BoxedSubscription; - - /// Executes a component bytes sent throughput subscription. - fn component_sent_bytes_throughputs_subscription( - &self, - interval: i64, - ) -> crate::BoxedSubscription; - - /// Executes a component events totals subscription. - fn component_sent_events_totals_subscription( - &self, - interval: i64, - ) -> crate::BoxedSubscription; - - /// Executes a component sent events throughputs subscription. - fn component_sent_events_throughputs_subscription( - &self, - interval: i64, - ) -> crate::BoxedSubscription; - - fn component_errors_totals_subscription( - &self, - interval: i64, - ) -> crate::BoxedSubscription; -} - -impl MetricsSubscriptionExt for crate::SubscriptionClient { - /// Executes an uptime metrics subscription. - fn uptime_subscription(&self) -> BoxedSubscription { - let request_body = UptimeSubscription::build_query(uptime_subscription::Variables); - - self.start::(&request_body) - } - - /// Executes an all component allocated bytes subscription. - fn component_allocated_bytes_subscription( - &self, - interval: i64, - ) -> BoxedSubscription { - let request_body = ComponentAllocatedBytesSubscription::build_query( - component_allocated_bytes_subscription::Variables { interval }, - ); - - self.start::(&request_body) - } - - /// Executes an all component bytes received totals subscription. - fn component_received_bytes_totals_subscription( - &self, - interval: i64, - ) -> BoxedSubscription { - let request_body = ComponentReceivedBytesTotalsSubscription::build_query( - component_received_bytes_totals_subscription::Variables { interval }, - ); - - self.start::(&request_body) - } - - /// Executes a component bytes received throughput subscription. - fn component_received_bytes_throughputs_subscription( - &self, - interval: i64, - ) -> BoxedSubscription { - let request_body = ComponentReceivedBytesThroughputsSubscription::build_query( - component_received_bytes_throughputs_subscription::Variables { interval }, - ); - - self.start::(&request_body) - } - - /// Executes an all component received events totals subscription. - fn component_received_events_totals_subscription( - &self, - interval: i64, - ) -> BoxedSubscription { - let request_body = ComponentReceivedEventsTotalsSubscription::build_query( - component_received_events_totals_subscription::Variables { interval }, - ); - - self.start::(&request_body) - } - - /// Executes an all component received events throughputs subscription. - fn component_received_events_throughputs_subscription( - &self, - interval: i64, - ) -> BoxedSubscription { - let request_body = ComponentReceivedEventsThroughputsSubscription::build_query( - component_received_events_throughputs_subscription::Variables { interval }, - ); - - self.start::(&request_body) - } - - /// Executes an all component bytes sent totals subscription. - fn component_sent_bytes_totals_subscription( - &self, - interval: i64, - ) -> BoxedSubscription { - let request_body = ComponentSentBytesTotalsSubscription::build_query( - component_sent_bytes_totals_subscription::Variables { interval }, - ); - - self.start::(&request_body) - } - - /// Executes a component bytes sent throughput subscription. - fn component_sent_bytes_throughputs_subscription( - &self, - interval: i64, - ) -> BoxedSubscription { - let request_body = ComponentSentBytesThroughputsSubscription::build_query( - component_sent_bytes_throughputs_subscription::Variables { interval }, - ); - - self.start::(&request_body) - } - - /// Executes a component sent events totals subscription. - fn component_sent_events_totals_subscription( - &self, - interval: i64, - ) -> crate::BoxedSubscription { - let request_body = ComponentSentEventsTotalsSubscription::build_query( - component_sent_events_totals_subscription::Variables { interval }, - ); - - self.start::(&request_body) - } - - /// Executes a component sent events throughputs subscription. - fn component_sent_events_throughputs_subscription( - &self, - interval: i64, - ) -> crate::BoxedSubscription { - let request_body = ComponentSentEventsThroughputsSubscription::build_query( - component_sent_events_throughputs_subscription::Variables { interval }, - ); - - self.start::(&request_body) - } - - fn component_errors_totals_subscription( - &self, - interval: i64, - ) -> BoxedSubscription { - let request_body = ComponentErrorsTotalsSubscription::build_query( - component_errors_totals_subscription::Variables { interval }, - ); - - self.start::(&request_body) - } -} diff --git a/lib/vector-api-client/src/gql/mod.rs b/lib/vector-api-client/src/gql/mod.rs deleted file mode 100644 index 6a690768f5ee9..0000000000000 --- a/lib/vector-api-client/src/gql/mod.rs +++ /dev/null @@ -1,14 +0,0 @@ -//! Queries, subscriptions, and extension methods for executing them - -mod components; -mod health; -mod meta; -mod metrics; -mod tap; - -pub use components::*; -pub use health::*; -pub use metrics::*; -pub use tap::*; - -pub use self::meta::*; diff --git a/lib/vector-api-client/src/gql/tap.rs b/lib/vector-api-client/src/gql/tap.rs deleted file mode 100644 index 1aef36838175c..0000000000000 --- a/lib/vector-api-client/src/gql/tap.rs +++ /dev/null @@ -1,90 +0,0 @@ -#![allow(clippy::upper_case_acronyms)] - -use graphql_client::GraphQLQuery; - -use crate::BoxedSubscription; - -/// Shorthand for a Chrono datetime, set to UTC. -type DateTime = chrono::DateTime; - -/// OutputEventsByComponentIdPatternsSubscription allows observability into the events that are -/// generated from component(s). -#[derive(GraphQLQuery, Debug, Copy, Clone)] -#[graphql( - schema_path = "graphql/schema.json", - query_path = "graphql/subscriptions/output_events_by_component_id_patterns.graphql", - response_derives = "Debug" -)] -pub struct OutputEventsByComponentIdPatternsSubscription; - -/// Tap encoding format type that is more convenient to use for public clients than the -/// generated `output_events_by_component_id_patterns_subscription::EventEncodingType`. -#[derive(clap::ValueEnum, Debug, Clone, Copy)] -pub enum TapEncodingFormat { - Json, - Yaml, - Logfmt, -} - -/// String -> TapEncodingFormat, typically for parsing user input. -impl std::str::FromStr for TapEncodingFormat { - type Err = String; - - fn from_str(s: &str) -> Result { - match s { - "json" => Ok(Self::Json), - "yaml" => Ok(Self::Yaml), - "logfmt" => Ok(Self::Logfmt), - _ => Err("Invalid encoding format".to_string()), - } - } -} - -/// Map the public-facing `TapEncodingFormat` to the internal `EventEncodingType`. -impl From - for output_events_by_component_id_patterns_subscription::EventEncodingType -{ - fn from(encoding: TapEncodingFormat) -> Self { - match encoding { - TapEncodingFormat::Json => Self::JSON, - TapEncodingFormat::Yaml => Self::YAML, - TapEncodingFormat::Logfmt => Self::LOGFMT, - } - } -} - -pub trait TapSubscriptionExt { - /// Executes an output events subscription. - fn output_events_by_component_id_patterns_subscription( - &self, - outputs_patterns: Vec, - inputs_patterns: Vec, - encoding: TapEncodingFormat, - limit: i64, - interval: i64, - ) -> crate::BoxedSubscription; -} - -impl TapSubscriptionExt for crate::SubscriptionClient { - /// Executes an output events subscription. - fn output_events_by_component_id_patterns_subscription( - &self, - outputs_patterns: Vec, - inputs_patterns: Vec, - encoding: TapEncodingFormat, - limit: i64, - interval: i64, - ) -> BoxedSubscription { - let request_body = OutputEventsByComponentIdPatternsSubscription::build_query( - output_events_by_component_id_patterns_subscription::Variables { - outputs_patterns, - inputs_patterns: Some(inputs_patterns), - limit, - interval, - encoding: encoding.into(), - }, - ); - - self.start::(&request_body) - } -} diff --git a/lib/vector-api-client/src/lib.rs b/lib/vector-api-client/src/lib.rs index e9850b80c235c..a05002ebb3e61 100644 --- a/lib/vector-api-client/src/lib.rs +++ b/lib/vector-api-client/src/lib.rs @@ -1,23 +1,43 @@ -//! Vector GraphQL client library, for the Vector GraphQL API server. +//! Vector gRPC API client library //! -//! Contains: +//! This library provides a Rust client for the Vector gRPC observability API. //! -//! 1. A GraphQL query client, for queries/mutations over HTTP(s) -//! 2. A GraphQL subscription client, for long-lived, multiplexed subscriptions over WebSockets -//! 3. GraphQL queries/mutations/subscriptions, defined in `graphql/**/*.graphql` files -//! 4. Extension methods for each client, for executing queries/subscriptions, and returning -//! deserialized JSON responses +//! # Example //! - -#![deny(warnings)] -#![deny(missing_debug_implementations, missing_copy_implementations)] -#![allow(async_fn_in_trait)] +//! ```no_run +//! use vector_api_client::Client; +//! +//! # async fn example() -> Result<(), Box> { +//! let mut client = Client::new("http://localhost:9999".parse().unwrap()); +//! client.connect().await?; +//! +//! // Check health +//! let health = client.health().await?; +//! println!("Healthy: {}", health.healthy); +//! +//! // Get components +//! let components = client.get_components(0).await?; +//! for component in components.components { +//! println!("Component: {}", component.component_id); +//! } +//! # Ok(()) +//! # } +//! ``` mod client; -/// GraphQL queries -pub mod gql; -mod subscription; -pub mod test; +mod error; + +pub use client::Client; +pub use error::{Error, Result}; + +/// How long (ms) to wait before attempting to reconnect to the Vector API after a disconnect. +pub const RECONNECT_DELAY_MS: u64 = 5000; + +/// Re-export generated protobuf types +pub mod proto { + pub mod event { + tonic::include_proto!("event"); + } -pub use client::*; -pub use subscription::*; + tonic::include_proto!("vector.observability.v1"); +} diff --git a/lib/vector-api-client/src/subscription.rs b/lib/vector-api-client/src/subscription.rs deleted file mode 100644 index 0a4308c6ed7ae..0000000000000 --- a/lib/vector-api-client/src/subscription.rs +++ /dev/null @@ -1,199 +0,0 @@ -use std::{ - collections::HashMap, - pin::Pin, - sync::{Arc, Mutex}, -}; - -use futures::SinkExt; -use graphql_client::GraphQLQuery; -use serde::{Deserialize, Serialize}; -use serde_json::json; -use tokio::sync::{ - broadcast::{self, Sender}, - mpsc, oneshot, -}; -use tokio_stream::{Stream, StreamExt, wrappers::BroadcastStream}; -use tokio_tungstenite::{connect_async, tungstenite::Message}; -use url::Url; -use uuid::Uuid; - -/// Subscription GraphQL response, returned from an active stream. -pub type BoxedSubscription = Pin< - Box< - dyn Stream::ResponseData>>> - + Send - + Sync, - >, ->; - -/// Payload contains the raw data received back from a GraphQL subscription. At the point -/// of receiving data, the only known fields are { id, type }; what's contained inside the -/// `payload` field is unknown until we attempt to deserialize it against a generated -/// GraphQLQuery::ResponseData later. -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct Payload { - id: Uuid, - #[serde(rename = "type")] - payload_type: String, - payload: serde_json::Value, -} - -impl Payload { - /// Returns a "start" payload necessary for starting a new subscription. - pub fn start( - id: Uuid, - payload: &graphql_client::QueryBody, - ) -> Self { - Self { - id, - payload_type: "start".to_owned(), - payload: json!(payload), - } - } - - /// Returns a "stop" payload for terminating the subscription in the GraphQL server. - fn stop(id: Uuid) -> Self { - Self { - id, - payload_type: "stop".to_owned(), - payload: serde_json::Value::Null, - } - } - - /// Attempts to return a definitive ResponseData on the `payload` field, matched against - /// a generated `GraphQLQuery`. - fn response( - &self, - ) -> Option> { - serde_json::from_value::>(self.payload.clone()) - .ok() - } -} - -/// A single `SubscriptionClient` enables subscription multiplexing. -#[derive(Debug)] -pub struct SubscriptionClient { - tx: mpsc::UnboundedSender, - subscriptions: Arc>>>, - _shutdown_tx: oneshot::Sender<()>, -} - -impl SubscriptionClient { - /// Create a new subscription client. `tx` is a channel for sending `Payload`s to the - /// GraphQL server; `rx` is a channel for `Payload` back. - fn new(tx: mpsc::UnboundedSender, mut rx: mpsc::UnboundedReceiver) -> Self { - // Oneshot channel for cancelling the listener if SubscriptionClient is dropped - let (_shutdown_tx, mut shutdown_rx) = oneshot::channel::<()>(); - - let subscriptions = Arc::new(Mutex::new(HashMap::new())); - let subscriptions_clone = Arc::clone(&subscriptions); - - // Spawn a handler for shutdown, and relaying received `Payload`s back to the relevant - // subscription. - let tx_clone = tx.clone(); - tokio::spawn(async move { - loop { - tokio::select! { - // Break the loop if shutdown is triggered. This happens implicitly once - // the client goes out of scope - _ = &mut shutdown_rx => { - let subscriptions = subscriptions_clone.lock().unwrap(); - for id in subscriptions.keys() { - _ = tx_clone.send(Payload::stop(*id)); - } - break - }, - - // Handle receiving payloads back _from_ the server - message = rx.recv() => { - match message { - Some(p) => { - let subscriptions = subscriptions_clone.lock().unwrap(); - let s: Option<&Sender> = subscriptions.get::(&p.id); - if let Some(s) = s { - _ = s.send(p); - } - } - None => { - subscriptions_clone.lock().unwrap().clear(); - break; - }, - } - } - } - } - }); - - Self { - tx, - subscriptions, - _shutdown_tx, - } - } - - /// Start a new subscription request. - pub fn start( - &self, - request_body: &graphql_client::QueryBody, - ) -> BoxedSubscription - where - T: GraphQLQuery + Send + Sync, - ::ResponseData: Unpin + Send + Sync + 'static, - { - // Generate a unique ID for the subscription. Subscriptions can be multiplexed - // over a single connection, so we'll keep a copy of this against the client to - // handling routing responses back to the relevant subscriber. - let id = Uuid::new_v4(); - - let (tx, rx) = broadcast::channel::(100); - - self.subscriptions.lock().unwrap().insert(id, tx); - - // Send start subscription command with the relevant control messages. - _ = self.tx.send(Payload::start::(id, request_body)); - - Box::pin( - BroadcastStream::new(rx) - .filter(Result::is_ok) - .map(|p| p.unwrap().response::()), - ) - } -} - -/// Connect to a new WebSocket GraphQL server endpoint, and return a `SubscriptionClient`. -/// This method will a) connect to a ws(s):// endpoint, and perform the initial handshake, and b) -/// set up channel forwarding to expose just the returned `Payload`s to the client. -pub async fn connect_subscription_client( - url: Url, -) -> Result { - let (ws, _) = connect_async(url).await?; - let (mut ws_tx, mut ws_rx) = futures::StreamExt::split(ws); - - let (send_tx, mut send_rx) = mpsc::unbounded_channel::(); - let (recv_tx, recv_rx) = mpsc::unbounded_channel::(); - - // Initialize the connection - _ = ws_tx - .send(Message::Text(r#"{"type":"connection_init"}"#.to_string())) - .await; - - // Forwarded received messages back upstream to the GraphQL server - tokio::spawn(async move { - while let Some(p) = send_rx.recv().await { - _ = ws_tx - .send(Message::Text(serde_json::to_string(&p).unwrap())) - .await; - } - }); - - // Forward received messages to the receiver channel. - tokio::spawn(async move { - while let Some(Ok(Message::Text(m))) = ws_rx.next().await { - if let Ok(p) = serde_json::from_str::(&m) { - _ = recv_tx.send(p); - } - } - }); - - Ok(SubscriptionClient::new(send_tx, recv_rx)) -} diff --git a/lib/vector-api-client/src/test/mod.rs b/lib/vector-api-client/src/test/mod.rs deleted file mode 100644 index 1f7716d96214f..0000000000000 --- a/lib/vector-api-client/src/test/mod.rs +++ /dev/null @@ -1,160 +0,0 @@ -use graphql_client::GraphQLQuery; - -use crate::{BoxedSubscription, QueryResult}; - -/// Component links query for returning linked components for sources, transforms, and sinks -#[derive(GraphQLQuery, Debug, Copy, Clone)] -#[graphql( - schema_path = "graphql/schema.json", - query_path = "tests/queries/component_links.graphql", - response_derives = "Debug" -)] -pub struct ComponentLinksQuery; - -/// Errors total subscription -#[derive(GraphQLQuery, Debug, Copy, Clone)] -#[graphql( - schema_path = "graphql/schema.json", - query_path = "tests/subscriptions/errors_total.graphql", - response_derives = "Debug" -)] -pub struct ErrorsTotalSubscription; - -/// File source metrics query -#[derive(GraphQLQuery, Debug, Copy, Clone)] -#[graphql( - schema_path = "graphql/schema.json", - query_path = "tests/queries/file_source_metrics.graphql", - response_derives = "Debug" -)] -pub struct FileSourceMetricsQuery; - -/// Component by id query -#[derive(GraphQLQuery, Debug, Copy, Clone)] -#[graphql( - schema_path = "graphql/schema.json", - query_path = "tests/queries/component_by_component_key.graphql", - response_derives = "Debug" -)] -pub struct ComponentByComponentKeyQuery; - -/// Component by id query -#[derive(GraphQLQuery, Debug, Copy, Clone)] -#[graphql( - schema_path = "graphql/schema.json", - query_path = "tests/queries/components_connection.graphql", - response_derives = "Debug" -)] -pub struct ComponentsConnectionQuery; - -pub trait TestQueryExt { - async fn component_links_query( - &self, - after: Option, - before: Option, - first: Option, - last: Option, - ) -> crate::QueryResult; - async fn file_source_metrics_query( - &self, - after: Option, - before: Option, - first: Option, - last: Option, - ) -> crate::QueryResult; - async fn component_by_component_key_query( - &self, - component_id: &str, - ) -> crate::QueryResult; - async fn components_connection_query( - &self, - after: Option, - before: Option, - first: Option, - last: Option, - ) -> crate::QueryResult; -} - -impl TestQueryExt for crate::Client { - async fn component_links_query( - &self, - after: Option, - before: Option, - first: Option, - last: Option, - ) -> QueryResult { - let request_body = ComponentLinksQuery::build_query(component_links_query::Variables { - after, - before, - first, - last, - }); - self.query::(&request_body).await - } - - async fn file_source_metrics_query( - &self, - after: Option, - before: Option, - first: Option, - last: Option, - ) -> QueryResult { - let request_body = - FileSourceMetricsQuery::build_query(file_source_metrics_query::Variables { - after, - before, - first, - last, - }); - self.query::(&request_body).await - } - - async fn component_by_component_key_query( - &self, - component_id: &str, - ) -> QueryResult { - let request_body = ComponentByComponentKeyQuery::build_query( - component_by_component_key_query::Variables { - component_id: component_id.to_string(), - }, - ); - self.query::(&request_body) - .await - } - - async fn components_connection_query( - &self, - after: Option, - before: Option, - first: Option, - last: Option, - ) -> QueryResult { - let request_body = - ComponentsConnectionQuery::build_query(components_connection_query::Variables { - after, - before, - first, - last, - }); - self.query::(&request_body).await - } -} - -pub trait TestSubscriptionExt { - fn errors_total_subscription( - &self, - interval: i64, - ) -> crate::BoxedSubscription; -} - -impl TestSubscriptionExt for crate::SubscriptionClient { - fn errors_total_subscription( - &self, - interval: i64, - ) -> BoxedSubscription { - let request_body = - ErrorsTotalSubscription::build_query(errors_total_subscription::Variables { interval }); - - self.start::(&request_body) - } -} diff --git a/lib/vector-api-client/tests/queries/component_by_component_key.graphql b/lib/vector-api-client/tests/queries/component_by_component_key.graphql deleted file mode 100644 index e29946f9148c3..0000000000000 --- a/lib/vector-api-client/tests/queries/component_by_component_key.graphql +++ /dev/null @@ -1,6 +0,0 @@ -query ComponentByComponentKeyQuery($componentId: String!) { - componentByComponentKey(componentId: $componentId) { - __typename - componentId - } -} diff --git a/lib/vector-api-client/tests/queries/component_links.graphql b/lib/vector-api-client/tests/queries/component_links.graphql deleted file mode 100644 index 3259b6c9b6661..0000000000000 --- a/lib/vector-api-client/tests/queries/component_links.graphql +++ /dev/null @@ -1,45 +0,0 @@ -query ComponentLinksQuery($after: String, $before: String, $first: Int, $last: Int) { - sources(after: $after, before: $before, first: $first, last: $last) { - edges { - node { - componentId - transforms { - componentId - } - sinks { - componentId - } - } - } - } - transforms(after: $after, before: $before, first: $first, last: $last) { - edges { - node { - componentId - sources { - componentId - } - transforms { - componentId - } - sinks { - componentId - } - } - } - } - sinks(after: $after, before: $before, first: $first, last: $last) { - edges { - node { - componentId - sources { - componentId - } - transforms { - componentId - } - } - } - } -} - diff --git a/lib/vector-api-client/tests/queries/components_connection.graphql b/lib/vector-api-client/tests/queries/components_connection.graphql deleted file mode 100644 index 835908282634d..0000000000000 --- a/lib/vector-api-client/tests/queries/components_connection.graphql +++ /dev/null @@ -1,18 +0,0 @@ -query ComponentsConnectionQuery($after: String, $before: String, $first: Int, $last: Int) { - components(after: $after, before: $before, first: $first, last: $last) { - edges { - cursor - node { - __typename - componentId - } - } - pageInfo { - hasNextPage - hasPreviousPage - startCursor - endCursor - } - totalCount - } -} diff --git a/lib/vector-api-client/tests/queries/file_source_metrics.graphql b/lib/vector-api-client/tests/queries/file_source_metrics.graphql deleted file mode 100644 index 7b9d2afb0f10e..0000000000000 --- a/lib/vector-api-client/tests/queries/file_source_metrics.graphql +++ /dev/null @@ -1,25 +0,0 @@ -query FileSourceMetricsQuery($after: String, $before: String, $first: Int, $last: Int) { - sources { - __typename - edges { - node { - metrics { - __typename - ...on FileSourceMetrics { - __typename - files(after: $after, before: $before, first: $first, last: $last) { - edges { - node { - name - receivedEventsTotal { - receivedEventsTotal - } - } - } - } - } - } - } - } - } -} diff --git a/lib/vector-api-client/tests/subscriptions/errors_total.graphql b/lib/vector-api-client/tests/subscriptions/errors_total.graphql deleted file mode 100644 index b0e18d07f76b0..0000000000000 --- a/lib/vector-api-client/tests/subscriptions/errors_total.graphql +++ /dev/null @@ -1,5 +0,0 @@ -subscription ErrorsTotalSubscription($interval: Int!) { - errorsTotal(interval: $interval) { - errorsTotal - } -} diff --git a/lib/vector-core/src/config/mod.rs b/lib/vector-core/src/config/mod.rs index c86848d7b0be5..4e822bd84f59c 100644 --- a/lib/vector-core/src/config/mod.rs +++ b/lib/vector-core/src/config/mod.rs @@ -27,7 +27,7 @@ pub const MEMORY_BUFFER_DEFAULT_MAX_EVENTS: NonZeroUsize = vector_buffers::config::memory_buffer_default_max_events(); // This enum should be kept alphabetically sorted as the bitmask value is used when -// sorting sources by data type in the GraphQL API. +// sorting sources by data type in the API. #[bitmask(u8)] #[bitmask_config(flags_iter)] pub enum DataType { diff --git a/lib/vector-tap/Cargo.toml b/lib/vector-tap/Cargo.toml index 976233f446915..127c2e1ea32bd 100644 --- a/lib/vector-tap/Cargo.toml +++ b/lib/vector-tap/Cargo.toml @@ -7,24 +7,43 @@ publish = false license = "MPL-2.0" [features] -api = ["dep:async-graphql"] +api = [ + "dep:bytes", + "dep:clap", + "dep:codecs", + "dep:colored", + "dep:prost", + "dep:serde_json", + "dep:serde_yaml", + "dep:tokio-stream", + "dep:tokio-util", + "dep:url", + "dep:vector-api-client", +] [dependencies] -async-graphql = { version = "7.0.17", default-features = false, features = ["playground"], optional = true } -colored.workspace = true +# Server-side (always compiled) futures.workspace = true glob.workspace = true -serde_yaml.workspace = true tokio = { workspace = true, features = ["time"] } -tokio-stream = { workspace = true, features = ["sync"] } -tokio-tungstenite.workspace = true tracing.workspace = true -url = { version = "2.5.4", default-features = false } uuid.workspace = true -vector-api-client = { path = "../vector-api-client" } +vector-buffers = { path = "../vector-buffers" } vector-common = { path = "../vector-common" } vector-core = { path = "../vector-core" } -vector-buffers = { path = "../vector-buffers" } + +# Client-side (gated behind the `api` feature) +bytes = { workspace = true, optional = true } +clap = { workspace = true, optional = true } +codecs = { path = "../codecs", optional = true } +colored = { workspace = true, optional = true } +prost = { workspace = true, optional = true } +serde_json = { workspace = true, optional = true } +serde_yaml = { workspace = true, optional = true } +tokio-stream = { workspace = true, features = ["sync"], optional = true } +tokio-util = { version = "0.7", default-features = false, features = ["codec"], optional = true } +url = { version = "2.5.4", default-features = false, optional = true } +vector-api-client = { path = "../vector-api-client", optional = true } [dev-dependencies] tokio = { workspace = true, features = ["test-util"] } diff --git a/lib/vector-tap/src/controller.rs b/lib/vector-tap/src/controller.rs index dba9a7ec7b885..2a8208179f3e8 100644 --- a/lib/vector-tap/src/controller.rs +++ b/lib/vector-tap/src/controller.rs @@ -306,6 +306,7 @@ async fn tap_handler( source_keys, sink_keys, removals, + .. } = watch_rx.borrow().clone(); // Remove tap sinks from components that have gone away/can no longer match. diff --git a/lib/vector-tap/src/lib.rs b/lib/vector-tap/src/lib.rs index 30ec0eca2aa3a..46291301a079d 100644 --- a/lib/vector-tap/src/lib.rs +++ b/lib/vector-tap/src/lib.rs @@ -7,255 +7,7 @@ pub mod controller; pub mod notification; pub mod topology; -use std::{borrow::Cow, collections::BTreeMap}; - -use colored::{ColoredString, Colorize}; -use tokio::{ - sync::mpsc as tokio_mpsc, - time::{Duration, Instant, timeout}, -}; -use tokio_stream::StreamExt; -use url::Url; -use vector_api_client::{ - connect_subscription_client, - gql::{ - TapEncodingFormat, TapSubscriptionExt, - output_events_by_component_id_patterns_subscription::OutputEventsByComponentIdPatternsSubscriptionOutputEventsByComponentIdPatterns as GraphQLTapOutputEvent, - }, -}; - -#[derive(Clone, Debug)] -pub struct EventFormatter { - meta: bool, - format: TapEncodingFormat, - component_id_label: ColoredString, - component_kind_label: ColoredString, - component_type_label: ColoredString, -} - -impl EventFormatter { - pub fn new(meta: bool, format: TapEncodingFormat) -> Self { - Self { - meta, - format, - component_id_label: "component_id".green(), - component_kind_label: "component_kind".green(), - component_type_label: "component_type".green(), - } - } - - pub fn format<'a>( - &self, - component_id: &str, - component_kind: &str, - component_type: &str, - event: &'a str, - ) -> Cow<'a, str> { - if self.meta { - match self.format { - TapEncodingFormat::Json => format!( - r#"{{"{}":"{}","{}":"{}","{}":"{}","event":{}}}"#, - self.component_id_label, - component_id.green(), - self.component_kind_label, - component_kind.green(), - self.component_type_label, - component_type.green(), - event - ) - .into(), - TapEncodingFormat::Yaml => { - let mut value: BTreeMap = BTreeMap::new(); - value.insert("event".to_string(), serde_yaml::from_str(event).unwrap()); - // We interpolate to include component_id rather than - // include it in the map to correctly preserve color - // formatting - format!( - "{}{}: {}\n{}: {}\n{}: {}\n", - serde_yaml::to_string(&value).unwrap(), - self.component_id_label, - component_id.green(), - self.component_kind_label, - component_kind.green(), - self.component_type_label, - component_type.green() - ) - .into() - } - TapEncodingFormat::Logfmt => format!( - "{}={} {}={} {}={} {}", - self.component_id_label, - component_id.green(), - self.component_kind_label, - component_kind.green(), - self.component_type_label, - component_type.green(), - event - ) - .into(), - } - } else { - event.into() - } - } -} - -#[derive(Clone, Debug)] -pub enum OutputChannel { - Stdout(EventFormatter), - AsyncChannel(tokio_mpsc::Sender>), -} - -/// Error type for DNS message parsing -#[derive(Debug)] -pub enum TapExecutorError { - ConnectionFailure(tokio_tungstenite::tungstenite::Error), - GraphQLError, -} - -#[derive(Debug)] -pub struct TapRunner<'a> { - url: &'a Url, - input_patterns: Vec, - output_patterns: Vec, - output_channel: &'a OutputChannel, - format: TapEncodingFormat, -} - -impl<'a> TapRunner<'a> { - pub fn new( - url: &'a Url, - input_patterns: Vec, - output_patterns: Vec, - output_channel: &'a OutputChannel, - format: TapEncodingFormat, - ) -> Self { - TapRunner { - url, - input_patterns, - output_patterns, - output_channel, - format, - } - } - - pub async fn run_tap( - &self, - interval: i64, - limit: i64, - duration_ms: Option, - quiet: bool, - ) -> Result<(), TapExecutorError> { - let subscription_client = connect_subscription_client((*self.url).clone()) - .await - .map_err(TapExecutorError::ConnectionFailure)?; - - tokio::pin! { - let stream = subscription_client.output_events_by_component_id_patterns_subscription( - self.output_patterns.clone(), - self.input_patterns.clone(), - self.format, - limit, - interval, - ); - } - - let start_time = Instant::now(); - let stream_duration = duration_ms - .map(Duration::from_millis) - .unwrap_or(Duration::MAX); - - // Loop over the returned results, printing out tap events. - loop { - let time_elapsed = start_time.elapsed(); - if time_elapsed >= stream_duration { - return Ok(()); - } - - let message = timeout(stream_duration - time_elapsed, stream.next()).await; - match message { - Ok(Some(Some(res))) => { - if let Some(d) = res.data { - let output_events: Vec = d - .output_events_by_component_id_patterns - .into_iter() - .filter(|event| { - !matches!( - (quiet, event), - (true, GraphQLTapOutputEvent::EventNotification(_)) - ) - }) - .collect(); - - match &self.output_channel { - OutputChannel::Stdout(formatter) => { - self.output_event_stdout(&output_events, formatter); - } - OutputChannel::AsyncChannel(sender_tx) => { - if let Err(error) = sender_tx.send(output_events).await { - error!("Could not send tap events: {error}"); - } - } - } - } - } - Err(_) => - // If the stream times out, that indicates the duration specified by the user - // has elapsed. We should exit gracefully. - { - return Ok(()); - } - Ok(_) => return Err(TapExecutorError::GraphQLError), - } - } - } - - #[allow(clippy::print_stdout)] - fn output_event_stdout( - &self, - output_events: &[GraphQLTapOutputEvent], - formatter: &EventFormatter, - ) { - for tap_event in output_events.iter() { - match tap_event { - GraphQLTapOutputEvent::Log(ev) => { - println!( - "{}", - formatter.format( - ev.component_id.as_ref(), - ev.component_kind.as_ref(), - ev.component_type.as_ref(), - ev.string.as_ref() - ) - ); - } - GraphQLTapOutputEvent::Metric(ev) => { - println!( - "{}", - formatter.format( - ev.component_id.as_ref(), - ev.component_kind.as_ref(), - ev.component_type.as_ref(), - ev.string.as_ref() - ) - ); - } - GraphQLTapOutputEvent::Trace(ev) => { - println!( - "{}", - formatter.format( - ev.component_id.as_ref(), - ev.component_kind.as_ref(), - ev.component_type.as_ref(), - ev.string.as_ref() - ) - ); - } - #[allow(clippy::print_stderr)] - GraphQLTapOutputEvent::EventNotification(ev) => { - eprintln!("{}", ev.message); - } - } - } - } -} +#[cfg(feature = "api")] +mod runner; +#[cfg(feature = "api")] +pub use runner::{EventFormatter, OutputChannel, TapEncodingFormat, TapExecutorError, TapRunner}; diff --git a/lib/vector-tap/src/notification.rs b/lib/vector-tap/src/notification.rs index 04ba715718993..785e780147ebc 100644 --- a/lib/vector-tap/src/notification.rs +++ b/lib/vector-tap/src/notification.rs @@ -1,11 +1,6 @@ -#[cfg(feature = "api")] -use async_graphql::{SimpleObject, Union}; - #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(feature = "api", derive(SimpleObject))] /// A component was found that matched the provided pattern pub struct Matched { - #[cfg_attr(feature = "api", graphql(skip))] message: String, /// Pattern that raised the notification pub pattern: String, @@ -21,10 +16,8 @@ impl Matched { } #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(feature = "api", derive(SimpleObject))] /// There isn't currently a component that matches this pattern pub struct NotMatched { - #[cfg_attr(feature = "api", graphql(skip))] message: String, /// Pattern that raised the notification pub pattern: String, @@ -42,11 +35,9 @@ impl NotMatched { } #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(feature = "api", derive(SimpleObject))] /// The pattern matched source(s) which cannot be tapped for inputs or sink(s) /// which cannot be tapped for outputs pub struct InvalidMatch { - #[cfg_attr(feature = "api", graphql(skip))] message: String, /// Pattern that raised the notification pattern: String, @@ -65,7 +56,6 @@ impl InvalidMatch { } #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(feature = "api", derive(Union))] /// A specific kind of notification with additional details pub enum Notification { Matched(Matched), diff --git a/lib/vector-tap/src/runner.rs b/lib/vector-tap/src/runner.rs new file mode 100644 index 0000000000000..8c6ab815a8bf8 --- /dev/null +++ b/lib/vector-tap/src/runner.rs @@ -0,0 +1,344 @@ +use std::{borrow::Cow, collections::BTreeMap}; + +use bytes::Bytes; +use colored::{ColoredString, Colorize}; +use prost::Message; +use tokio::{ + sync::mpsc as tokio_mpsc, + time::{Duration, Instant, timeout}, +}; +use tokio_stream::StreamExt; +use tokio_util::codec::Encoder; +use url::Url; +use vector_api_client::{ + Client, + proto::{StreamOutputEventsRequest, StreamOutputEventsResponse}, +}; +use vector_core::event::Event; + +#[derive(Clone, Copy, Debug, clap::ValueEnum)] +pub enum TapEncodingFormat { + Json, + Yaml, + Logfmt, +} + +// Note: TapEncodingFormat is kept for CLI compatibility but not used in the gRPC API +// The server now sends proto events directly, which clients serialize as needed + +#[derive(Clone, Debug)] +pub struct EventFormatter { + meta: bool, + format: TapEncodingFormat, + component_id_label: ColoredString, + component_kind_label: ColoredString, + component_type_label: ColoredString, +} + +impl EventFormatter { + pub fn new(meta: bool, format: TapEncodingFormat) -> Self { + Self { + meta, + format, + component_id_label: "component_id".green(), + component_kind_label: "component_kind".green(), + component_type_label: "component_type".green(), + } + } + + pub fn format<'a>( + &self, + component_id: &str, + component_kind: &str, + component_type: &str, + event: &'a str, + ) -> Cow<'a, str> { + if self.meta { + match self.format { + TapEncodingFormat::Json => format!( + r#"{{"{}":"{}","{}":"{}","{}":"{}","event":{}}}"#, + self.component_id_label, + component_id.green(), + self.component_kind_label, + component_kind.green(), + self.component_type_label, + component_type.green(), + event + ) + .into(), + TapEncodingFormat::Yaml => { + let mut value: BTreeMap = BTreeMap::new(); + value.insert("event".to_string(), serde_yaml::from_str(event).unwrap()); + // We interpolate to include component_id rather than + // include it in the map to correctly preserve color + // formatting + format!( + "{}{}: {}\n{}: {}\n{}: {}\n", + serde_yaml::to_string(&value).unwrap(), + self.component_id_label, + component_id.green(), + self.component_kind_label, + component_kind.green(), + self.component_type_label, + component_type.green() + ) + .into() + } + TapEncodingFormat::Logfmt => format!( + "{}={} {}={} {}={} {}", + self.component_id_label, + component_id.green(), + self.component_kind_label, + component_kind.green(), + self.component_type_label, + component_type.green(), + event + ) + .into(), + } + } else { + event.into() + } + } +} + +#[derive(Clone, Debug)] +pub enum OutputChannel { + Stdout(EventFormatter), + AsyncChannel(tokio_mpsc::Sender>), +} + +/// Error type for tap execution +#[derive(Debug)] +pub enum TapExecutorError { + ConnectionFailure(String), + GrpcError(String), + /// Permanent error that should not trigger a reconnect (e.g. invalid arguments). + Fatal(String), +} + +impl TapExecutorError { + pub fn is_fatal(&self) -> bool { + matches!(self, TapExecutorError::Fatal(_)) + } +} + +impl From for TapExecutorError { + fn from(err: vector_api_client::Error) -> Self { + if err.is_fatal() { + TapExecutorError::Fatal(format!("{}", err)) + } else { + TapExecutorError::GrpcError(format!("{}", err)) + } + } +} + +#[derive(Debug)] +pub struct TapRunner<'a> { + url: &'a Url, + input_patterns: Vec, + output_patterns: Vec, + output_channel: &'a OutputChannel, +} + +impl<'a> TapRunner<'a> { + pub fn new( + url: &'a Url, + input_patterns: Vec, + output_patterns: Vec, + output_channel: &'a OutputChannel, + ) -> Self { + TapRunner { + url, + input_patterns, + output_patterns, + output_channel, + } + } + + pub async fn run_tap( + &self, + interval: i64, + limit: i64, + duration_ms: Option, + quiet: bool, + ) -> Result<(), TapExecutorError> { + let uri = self + .url + .as_str() + .parse() + .map_err(|e| TapExecutorError::Fatal(format!("Invalid URL: {e}")))?; + let mut client = Client::new(uri); + client.connect().await?; + self.run_tap_with_client(client, interval, limit, duration_ms, quiet) + .await + } + + /// Run tap using a pre-connected client (avoids an extra connection round-trip when the + /// caller has already connected and health-checked the client). + pub async fn run_tap_with_client( + &self, + mut client: Client, + interval: i64, + limit: i64, + duration_ms: Option, + quiet: bool, + ) -> Result<(), TapExecutorError> { + let request = StreamOutputEventsRequest { + outputs_patterns: self.output_patterns.clone(), + inputs_patterns: self.input_patterns.clone(), + limit: limit as i32, + interval_ms: interval as i32, + }; + + let mut stream = client.stream_output_events(request).await?; + + let start_time = Instant::now(); + let stream_duration = duration_ms + .map(Duration::from_millis) + .unwrap_or(Duration::MAX); + + // Loop over the returned results, processing tap events + loop { + let time_elapsed = start_time.elapsed(); + if time_elapsed >= stream_duration { + return Ok(()); + } + + let message = timeout(stream_duration - time_elapsed, stream.next()).await; + match message { + Ok(Some(Ok(output_event))) => { + // Filter out notifications if quiet mode is enabled + if quiet + && matches!( + output_event.event, + Some( + vector_api_client::proto::stream_output_events_response::Event::Notification( + _ + ) + ) + ) + { + continue; + } + + match &self.output_channel { + OutputChannel::Stdout(formatter) => { + self.output_event_stdout(&output_event, formatter); + } + OutputChannel::AsyncChannel(sender_tx) => { + if let Err(error) = sender_tx.send(vec![output_event]).await { + error!("Could not send tap events: {error}"); + } + } + } + } + Err(_) => + // If the stream times out, that indicates the duration specified by the user + // has elapsed. We should exit gracefully. + { + return Ok(()); + } + Ok(None) => { + return Err(TapExecutorError::GrpcError( + "Stream ended unexpectedly".to_string(), + )); + } + Ok(Some(Err(err))) => return Err(TapExecutorError::from(err)), + } + } + } + + /// Convert and serialize a protobuf EventWrapper to the requested format + fn serialize_event( + event_wrapper: &vector_api_client::proto::event::EventWrapper, + format: TapEncodingFormat, + ) -> Result { + // INTENTIONAL round-trip through protobuf bytes: `vector_api_client` compiles + // `event.proto` independently to avoid taking a dependency on `vector_core` (a large + // crate with many features). Both types share the same proto schema, so encoding one + // and decoding the other is always safe. If the schemas ever diverge this will surface + // as a runtime decode error. + let bytes = event_wrapper.encode_to_vec(); + + let core_event_wrapper = + vector_core::event::proto::EventWrapper::decode(Bytes::from(bytes)) + .map_err(|e| format!("Failed to decode event: {}", e))?; + + // Convert to vector-core Event (which has Serialize) + let event: Event = core_event_wrapper.into(); + + // Serialize based on format + match format { + TapEncodingFormat::Json => serde_json::to_string(&event) + .map_err(|e| format!("JSON serialization failed: {}", e)), + TapEncodingFormat::Yaml => serde_yaml::to_string(&event) + .map_err(|e| format!("YAML serialization failed: {}", e)), + TapEncodingFormat::Logfmt => { + // For logfmt, we need to extract the log event and serialize it + match event { + Event::Log(log_event) => { + let mut serializer = + codecs::encoding::format::LogfmtSerializerConfig.build(); + let mut bytes = bytes::BytesMut::new(); + // Wrap the LogEvent back into Event for the serializer + serializer + .encode(Event::Log(log_event), &mut bytes) + .map_err(|e| format!("Logfmt serialization failed: {}", e))?; + String::from_utf8(bytes.to_vec()) + .map_err(|e| format!("UTF-8 conversion failed: {}", e)) + } + Event::Metric(_) => { + Err("logfmt format is only supported for log events".to_string()) + } + Event::Trace(_) => { + Err("logfmt format is only supported for log events".to_string()) + } + } + } + } + } + + #[allow(clippy::print_stdout)] + fn output_event_stdout( + &self, + output_event: &StreamOutputEventsResponse, + formatter: &EventFormatter, + ) { + use vector_api_client::proto::stream_output_events_response::Event as OutputEventType; + + match &output_event.event { + Some(OutputEventType::TappedEvent(ev)) => { + // Format the proto event for display + let encoded_string = if let Some(ref event_wrapper) = ev.event { + match Self::serialize_event(event_wrapper, formatter.format) { + Ok(s) => s, + Err(e) => { + error!(message = "Failed to serialize event.", error = %e); + format!("{:?}", event_wrapper) + } + } + } else { + "No event data".to_string() + }; + + println!( + "{}", + formatter.format( + &ev.component_id, + &ev.component_kind, + &ev.component_type, + &encoded_string + ) + ); + } + #[allow(clippy::print_stderr)] + Some(OutputEventType::Notification(ev)) => { + eprintln!("{}", ev.message); + } + None => { + error!("Received StreamOutputEventsResponse with no event data"); + } + } + } +} diff --git a/lib/vector-tap/src/topology.rs b/lib/vector-tap/src/topology.rs index 7b281f7c47151..eb13e93e2a9fe 100644 --- a/lib/vector-tap/src/topology.rs +++ b/lib/vector-tap/src/topology.rs @@ -26,6 +26,24 @@ pub struct TapResource { pub sink_keys: Vec, // Components removed on a reload (used to drop TapSinks) pub removals: HashSet, + // Plugin type name for every component (e.g. "demo_logs", "remap", "console") + pub type_names: HashMap, +} + +impl TapResource { + /// Returns each component's output port names, derived from the full `outputs` snapshot. + /// + /// Keys are all components that have at least one output (sources and transforms). + /// The port value is `None` for the default output and `Some(name)` for named ports. + pub fn output_ports_by_component(&self) -> HashMap<&ComponentKey, Vec>> { + let mut map: HashMap<&ComponentKey, Vec>> = HashMap::new(); + for tap_output in self.outputs.keys() { + map.entry(&tap_output.output_id.component) + .or_default() + .push(tap_output.output_id.port.as_deref()); + } + map + } } // Watcher types for topology changes. diff --git a/lib/vector-top/Cargo.toml b/lib/vector-top/Cargo.toml index c0da764b4de51..869fec71484f1 100644 --- a/lib/vector-top/Cargo.toml +++ b/lib/vector-top/Cargo.toml @@ -22,6 +22,7 @@ unit-prefix = { version = "0.5.2", default-features = false, features = ["std"] num-format = { version = "0.4.4", default-features = false, features = ["with-num-bigint"] } ratatui = { version = "0.30.0", default-features = false, features = ["crossterm", "layout-cache"] } regex.workspace = true +http = { version = "0.2", default-features = false } vector-common = { path = "../vector-common" } vector-api-client = { path = "../vector-api-client" } diff --git a/lib/vector-top/src/metrics.rs b/lib/vector-top/src/metrics.rs index 89012f80030ff..52cf46beff244 100644 --- a/lib/vector-top/src/metrics.rs +++ b/lib/vector-top/src/metrics.rs @@ -1,14 +1,16 @@ use std::{ - collections::{BTreeMap, HashMap}, + collections::{BTreeMap, HashSet}, sync::Arc, + time::Duration, }; use glob::Pattern; +use http::Uri; use tokio::task::JoinHandle; use tokio_stream::StreamExt; use vector_api_client::{ - Client, SubscriptionClient, - gql::{ComponentsQueryExt, ComponentsSubscriptionExt, MetricsSubscriptionExt}, + Client, + proto::{Component, ComponentType, MetricName, stream_component_metrics_response::Value}, }; use crate::state::{self, OutputMetrics, SentEventsMetric}; @@ -24,526 +26,546 @@ fn component_matches_patterns(component_id: &str, components_patterns: &[Pattern .any(|pattern| pattern.matches(component_id)) } -/// Components that have been added -async fn component_added( - client: Arc, +/// Component polling task +/// +/// Polls for component changes every interval and diffs to detect adds/removes. +async fn poll_components( + mut client: Client, tx: state::EventTx, + interval_ms: i64, components_patterns: Arc>, + initial_components: HashSet, ) { - tokio::pin! { - let stream = client.component_added(); - }; + let mut known_components = initial_components; + let poll_interval = Duration::from_millis(interval_ms as u64); + let mut consecutive_errors = 0; + const MAX_CONSECUTIVE_ERRORS: u32 = 3; + + loop { + tokio::time::sleep(poll_interval).await; + + // Fetch current components + let Ok(response) = client.get_components(0).await else { + consecutive_errors += 1; + if consecutive_errors >= MAX_CONSECUTIVE_ERRORS { + // Exit to trigger reconnection after sustained failures + return; + } + continue; + }; + consecutive_errors = 0; + + let current_components: std::collections::HashSet = response + .components + .iter() + .map(|c| c.component_id.clone()) + .collect(); + + // Detect added components + for component in &response.components { + let component_id = &component.component_id; + if !known_components.contains(component_id) + && component_matches_patterns(component_id, &components_patterns) + { + let row = component_to_row(component); + _ = tx.send(state::EventType::ComponentAdded(row)).await; + } + } - while let Some(Some(res)) = stream.next().await { - if let Some(d) = res.data { - let c = d.component_added; - let component_id = c.component_id; - if !component_matches_patterns(&component_id, &components_patterns) { - continue; + // Detect removed components + for old_id in &known_components { + if !current_components.contains(old_id) + && component_matches_patterns(old_id, &components_patterns) + { + let key = ComponentKey::from(old_id.as_str()); + _ = tx.send(state::EventType::ComponentRemoved(key)).await; } - let key = ComponentKey::from(component_id); - _ = tx - .send(state::EventType::ComponentAdded(state::ComponentRow { - key, - kind: c.on.to_string(), - component_type: c.component_type, - outputs: HashMap::new(), - received_bytes_total: 0, - received_bytes_throughput_sec: 0, - received_events_total: 0, - received_events_throughput_sec: 0, - sent_bytes_total: 0, - sent_bytes_throughput_sec: 0, - sent_events_total: 0, - sent_events_throughput_sec: 0, - #[cfg(feature = "allocation-tracing")] - allocated_bytes: 0, - errors: 0, - })) - .await; } + + known_components = current_components; + } +} + +fn component_to_row(component: &Component) -> state::ComponentRow { + let key = ComponentKey::from(component.component_id.as_str()); + let metrics = component.metrics.as_ref(); + + state::ComponentRow { + key: key.clone(), + kind: match component.component_type() { + ComponentType::Unspecified => "unknown", + ComponentType::Source => "source", + ComponentType::Transform => "transform", + ComponentType::Sink => "sink", + } + .to_string(), + component_type: component.on_type.clone(), // actual plugin type e.g. "demo_logs", "kafka" + outputs: component + .outputs + .iter() + .map(|o| { + ( + o.output_id.clone(), + OutputMetrics::from(o.sent_events_total), + ) + }) + .collect(), + received_bytes_total: metrics.and_then(|m| m.received_bytes_total).unwrap_or(0), + received_bytes_throughput_sec: 0, + received_events_total: metrics.and_then(|m| m.received_events_total).unwrap_or(0), + received_events_throughput_sec: 0, + sent_bytes_total: metrics.and_then(|m| m.sent_bytes_total).unwrap_or(0), + sent_bytes_throughput_sec: 0, + sent_events_total: metrics.and_then(|m| m.sent_events_total).unwrap_or(0), + sent_events_throughput_sec: 0, + #[cfg(feature = "allocation-tracing")] + allocated_bytes: 0, + errors: 0, } } /// Allocated bytes per component #[cfg(feature = "allocation-tracing")] async fn allocated_bytes( - client: Arc, + mut client: Client, tx: state::EventTx, interval: i64, components_patterns: Arc>, ) { - tokio::pin! { - let stream = client.component_allocated_bytes_subscription(interval); + let Ok(mut stream) = client + .stream_component_allocated_bytes(interval as i32) + .await + else { + // Failed to establish stream, will retry on reconnection + return; }; - while let Some(Some(res)) = stream.next().await { - if let Some(d) = res.data { - let c = d.component_allocated_bytes; - _ = tx - .send(state::EventType::AllocatedBytes( - c.into_iter() - .filter(|c| { - component_matches_patterns(&c.component_id, &components_patterns) - }) - .map(|c| { - ( - ComponentKey::from(c.component_id), - c.metric.allocated_bytes as i64, - ) - }) - .collect(), - )) - .await; + while let Some(Ok(response)) = stream.next().await { + let component_id = &response.component_id; + if !component_matches_patterns(component_id, &components_patterns) { + continue; } - } -} -/// Components that have been removed -async fn component_removed( - client: Arc, - tx: state::EventTx, - components_patterns: Arc>, -) { - tokio::pin! { - let stream = client.component_removed(); - }; - while let Some(Some(res)) = stream.next().await { - if let Some(d) = res.data { - let c = d.component_removed; - let component_id = c.component_id; - if !component_matches_patterns(&component_id, &components_patterns) { - continue; - } - let id = ComponentKey::from(component_id); - _ = tx.send(state::EventType::ComponentRemoved(id)).await; - } + _ = tx + .send(state::EventType::AllocatedBytes(vec![( + ComponentKey::from(component_id.as_str()), + response.allocated_bytes, + )])) + .await; } } async fn received_bytes_totals( - client: Arc, + mut client: Client, tx: state::EventTx, interval: i64, components_patterns: Arc>, ) { - tokio::pin! { - let stream = client.component_received_bytes_totals_subscription(interval); + let Ok(mut stream) = client + .stream_component_metrics(MetricName::ReceivedBytesTotal, interval as i32) + .await + else { + return; }; - while let Some(Some(res)) = stream.next().await { - if let Some(d) = res.data { - let c = d.component_received_bytes_totals; - + while let Some(Ok(response)) = stream.next().await { + let component_id = &response.component_id; + if !component_matches_patterns(component_id, &components_patterns) { + continue; + } + if let Some(Value::Total(total)) = response.value { _ = tx - .send(state::EventType::ReceivedBytesTotals( - c.into_iter() - .filter(|c| { - component_matches_patterns(&c.component_id, &components_patterns) - }) - .map(|c| { - ( - ComponentKey::from(c.component_id), - c.metric.received_bytes_total as i64, - ) - }) - .collect(), - )) + .send(state::EventType::ReceivedBytesTotals(vec![( + ComponentKey::from(component_id.as_str()), + total.value, + )])) .await; } } } async fn received_bytes_throughputs( - client: Arc, + mut client: Client, tx: state::EventTx, interval: i64, components_patterns: Arc>, ) { - tokio::pin! { - let stream = client.component_received_bytes_throughputs_subscription(interval); + let Ok(mut stream) = client + .stream_component_metrics(MetricName::ReceivedBytesThroughput, interval as i32) + .await + else { + return; }; - while let Some(Some(res)) = stream.next().await { - if let Some(d) = res.data { - let c = d.component_received_bytes_throughputs; + while let Some(Ok(response)) = stream.next().await { + let component_id = &response.component_id; + if !component_matches_patterns(component_id, &components_patterns) { + continue; + } + if let Some(Value::Throughput(tp)) = response.value { _ = tx - .send(state::EventType::ReceivedBytesThroughputs( - interval, - c.into_iter() - .filter(|c| { - component_matches_patterns(&c.component_id, &components_patterns) - }) - .map(|c| (ComponentKey::from(c.component_id), c.throughput)) - .collect(), - )) + .send(state::EventType::ReceivedBytesThroughputs(vec![( + ComponentKey::from(component_id.as_str()), + tp.value as i64, + )])) .await; } } } async fn received_events_totals( - client: Arc, + mut client: Client, tx: state::EventTx, interval: i64, components_patterns: Arc>, ) { - tokio::pin! { - let stream = client.component_received_events_totals_subscription(interval); + let Ok(mut stream) = client + .stream_component_metrics(MetricName::ReceivedEventsTotal, interval as i32) + .await + else { + return; }; - while let Some(Some(res)) = stream.next().await { - if let Some(d) = res.data { - let c = d.component_received_events_totals; + while let Some(Ok(response)) = stream.next().await { + let component_id = &response.component_id; + if !component_matches_patterns(component_id, &components_patterns) { + continue; + } + if let Some(Value::Total(total)) = response.value { _ = tx - .send(state::EventType::ReceivedEventsTotals( - c.into_iter() - .filter(|c| { - component_matches_patterns(&c.component_id, &components_patterns) - }) - .map(|c| { - ( - ComponentKey::from(c.component_id), - c.metric.received_events_total as i64, - ) - }) - .collect(), - )) + .send(state::EventType::ReceivedEventsTotals(vec![( + ComponentKey::from(component_id.as_str()), + total.value, + )])) .await; } } } async fn received_events_throughputs( - client: Arc, + mut client: Client, tx: state::EventTx, interval: i64, components_patterns: Arc>, ) { - tokio::pin! { - let stream = client.component_received_events_throughputs_subscription(interval); + let Ok(mut stream) = client + .stream_component_metrics(MetricName::ReceivedEventsThroughput, interval as i32) + .await + else { + return; }; - while let Some(Some(res)) = stream.next().await { - if let Some(d) = res.data { - let c = d.component_received_events_throughputs; + while let Some(Ok(response)) = stream.next().await { + let component_id = &response.component_id; + if !component_matches_patterns(component_id, &components_patterns) { + continue; + } + if let Some(Value::Throughput(tp)) = response.value { _ = tx - .send(state::EventType::ReceivedEventsThroughputs( - interval, - c.into_iter() - .filter(|c| { - component_matches_patterns(&c.component_id, &components_patterns) - }) - .map(|c| (ComponentKey::from(c.component_id), c.throughput)) - .collect(), - )) + .send(state::EventType::ReceivedEventsThroughputs(vec![( + ComponentKey::from(component_id.as_str()), + tp.value as i64, + )])) .await; } } } async fn sent_bytes_totals( - client: Arc, + mut client: Client, tx: state::EventTx, interval: i64, components_patterns: Arc>, ) { - tokio::pin! { - let stream = client.component_sent_bytes_totals_subscription(interval); + let Ok(mut stream) = client + .stream_component_metrics(MetricName::SentBytesTotal, interval as i32) + .await + else { + return; }; - while let Some(Some(res)) = stream.next().await { - if let Some(d) = res.data { - let c = d.component_sent_bytes_totals; + while let Some(Ok(response)) = stream.next().await { + let component_id = &response.component_id; + if !component_matches_patterns(component_id, &components_patterns) { + continue; + } + if let Some(Value::Total(total)) = response.value { _ = tx - .send(state::EventType::SentBytesTotals( - c.into_iter() - .filter(|c| { - component_matches_patterns(&c.component_id, &components_patterns) - }) - .map(|c| { - ( - ComponentKey::from(c.component_id), - c.metric.sent_bytes_total as i64, - ) - }) - .collect(), - )) + .send(state::EventType::SentBytesTotals(vec![( + ComponentKey::from(component_id.as_str()), + total.value, + )])) .await; } } } async fn sent_bytes_throughputs( - client: Arc, + mut client: Client, tx: state::EventTx, interval: i64, components_patterns: Arc>, ) { - tokio::pin! { - let stream = client.component_sent_bytes_throughputs_subscription(interval); + let Ok(mut stream) = client + .stream_component_metrics(MetricName::SentBytesThroughput, interval as i32) + .await + else { + return; }; - while let Some(Some(res)) = stream.next().await { - if let Some(d) = res.data { - let c = d.component_sent_bytes_throughputs; + while let Some(Ok(response)) = stream.next().await { + let component_id = &response.component_id; + if !component_matches_patterns(component_id, &components_patterns) { + continue; + } + if let Some(Value::Throughput(tp)) = response.value { _ = tx - .send(state::EventType::SentBytesThroughputs( - interval, - c.into_iter() - .filter(|c| { - component_matches_patterns(&c.component_id, &components_patterns) - }) - .map(|c| (ComponentKey::from(c.component_id), c.throughput)) - .collect(), - )) + .send(state::EventType::SentBytesThroughputs(vec![( + ComponentKey::from(component_id.as_str()), + tp.value as i64, + )])) .await; } } } async fn sent_events_totals( - client: Arc, + mut client: Client, tx: state::EventTx, interval: i64, components_patterns: Arc>, ) { - tokio::pin! { - let stream = client.component_sent_events_totals_subscription(interval); + let Ok(mut stream) = client + .stream_component_metrics(MetricName::SentEventsTotal, interval as i32) + .await + else { + return; }; - while let Some(Some(res)) = stream.next().await { - if let Some(d) = res.data { - let c = d.component_sent_events_totals; + while let Some(Ok(response)) = stream.next().await { + let component_id = &response.component_id; + if !component_matches_patterns(component_id, &components_patterns) { + continue; + } + if let Some(Value::Total(total)) = response.value { _ = tx - .send(state::EventType::SentEventsTotals( - c.into_iter() - .filter(|c| { - component_matches_patterns(&c.component_id, &components_patterns) - }) - .map(|c| SentEventsMetric { - key: ComponentKey::from(c.component_id.as_str()), - total: c.metric.sent_events_total as i64, - outputs: c.outputs().into_iter().collect(), - }) - .collect(), - )) + .send(state::EventType::SentEventsTotals(vec![SentEventsMetric { + key: ComponentKey::from(component_id.as_str()), + total: total.value, + outputs: total.output_totals.into_iter().collect(), + }])) .await; } } } async fn sent_events_throughputs( - client: Arc, + mut client: Client, tx: state::EventTx, interval: i64, components_patterns: Arc>, ) { - tokio::pin! { - let stream = client.component_sent_events_throughputs_subscription(interval); + let Ok(mut stream) = client + .stream_component_metrics(MetricName::SentEventsThroughput, interval as i32) + .await + else { + return; }; - while let Some(Some(res)) = stream.next().await { - if let Some(d) = res.data { - let c = d.component_sent_events_throughputs; + while let Some(Ok(response)) = stream.next().await { + let component_id = &response.component_id; + if !component_matches_patterns(component_id, &components_patterns) { + continue; + } + if let Some(Value::Throughput(tp)) = response.value { _ = tx - .send(state::EventType::SentEventsThroughputs( - interval, - c.into_iter() - .filter(|c| { - component_matches_patterns(&c.component_id, &components_patterns) - }) - .map(|c| SentEventsMetric { - key: ComponentKey::from(c.component_id.as_str()), - total: c.throughput, - outputs: c.outputs().into_iter().collect(), - }) - .collect(), - )) + .send(state::EventType::SentEventsThroughputs(vec![ + SentEventsMetric { + key: ComponentKey::from(component_id.as_str()), + total: tp.value as i64, + outputs: tp + .output_throughputs + .into_iter() + .map(|(k, v)| (k, v as i64)) + .collect(), + }, + ])) .await; } } } async fn errors_totals( - client: Arc, + mut client: Client, tx: state::EventTx, interval: i64, components_patterns: Arc>, ) { - tokio::pin! { - let stream = client.component_errors_totals_subscription(interval); + let Ok(mut stream) = client + .stream_component_metrics(MetricName::ErrorsTotal, interval as i32) + .await + else { + return; }; - while let Some(Some(res)) = stream.next().await { - if let Some(d) = res.data { - let c = d.component_errors_totals; + while let Some(Ok(response)) = stream.next().await { + let component_id = &response.component_id; + if !component_matches_patterns(component_id, &components_patterns) { + continue; + } + if let Some(Value::Total(total)) = response.value { _ = tx - .send(state::EventType::ErrorsTotals( - c.into_iter() - .filter(|c| { - component_matches_patterns(&c.component_id, &components_patterns) - }) - .map(|c| { - ( - ComponentKey::from(c.component_id.as_str()), - c.metric.errors_total as i64, - ) - }) - .collect(), - )) + .send(state::EventType::ErrorsTotals(vec![( + ComponentKey::from(component_id.as_str()), + total.value, + )])) .await; } } } -async fn uptime_changed(client: Arc, tx: state::EventTx) { - tokio::pin! { - let stream = client.uptime_subscription(); +async fn uptime_changed(mut client: Client, tx: state::EventTx, interval: i64) { + let Ok(mut stream) = client.stream_uptime(interval as i32).await else { + return; }; - while let Some(Some(res)) = stream.next().await { - if let Some(d) = res.data { - _ = tx - .send(state::EventType::UptimeChanged(d.uptime.seconds)) - .await; - } + while let Some(Ok(response)) = stream.next().await { + _ = tx + .send(state::EventType::UptimeChanged( + response.uptime_seconds as f64, + )) + .await; } } -/// Subscribe to each metrics channel through a separate client. This is a temporary workaround -/// until client multiplexing is fixed. In future, we should be able to use a single client -pub fn subscribe( - client: SubscriptionClient, +/// Handles returned by [`subscribe`], split by lifecycle. +pub struct SubscribeHandles { + /// Metric stream tasks — exit when a gRPC stream closes or errors. + /// `cmd.rs` joins these to detect connection loss and trigger reconnection. + pub metric_handles: Vec>, + /// Polls `get_components` for topology changes. Designed to run indefinitely + /// while the server is healthy, so it must be aborted separately rather than + /// joined alongside the metric handles. + pub poll_handle: JoinHandle<()>, +} + +/// Subscribe to each metrics stream, all sharing a single underlying gRPC connection. +/// HTTP/2 multiplexes the concurrent streams — cloning a connected `Client` is cheap +/// (the tonic `Channel` is Arc-backed) and avoids redundant TCP/HTTP2 handshakes. +pub async fn subscribe( + uri: Uri, tx: state::EventTx, interval: i64, components_patterns: Vec, -) -> Vec> { - let client = Arc::new(client); + initial_components: HashSet, +) -> Result { let components_patterns = Arc::new(components_patterns); - vec![ - tokio::spawn(component_added( - Arc::clone(&client), - tx.clone(), - Arc::clone(&components_patterns), - )), - tokio::spawn(component_removed( - Arc::clone(&client), - tx.clone(), - Arc::clone(&components_patterns), - )), + + let mut client = Client::new(uri); + client.connect().await?; + + let poll_handle = tokio::spawn(poll_components( + client.clone(), + tx.clone(), + interval, + Arc::clone(&components_patterns), + initial_components, + )); + + #[cfg_attr(not(feature = "allocation-tracing"), allow(unused_mut))] + let mut metric_handles = vec![ tokio::spawn(received_bytes_totals( - Arc::clone(&client), + client.clone(), tx.clone(), interval, Arc::clone(&components_patterns), )), tokio::spawn(received_bytes_throughputs( - Arc::clone(&client), + client.clone(), tx.clone(), interval, Arc::clone(&components_patterns), )), tokio::spawn(received_events_totals( - Arc::clone(&client), + client.clone(), tx.clone(), interval, Arc::clone(&components_patterns), )), tokio::spawn(received_events_throughputs( - Arc::clone(&client), + client.clone(), tx.clone(), interval, Arc::clone(&components_patterns), )), tokio::spawn(sent_bytes_totals( - Arc::clone(&client), + client.clone(), tx.clone(), interval, Arc::clone(&components_patterns), )), tokio::spawn(sent_bytes_throughputs( - Arc::clone(&client), + client.clone(), tx.clone(), interval, Arc::clone(&components_patterns), )), tokio::spawn(sent_events_totals( - Arc::clone(&client), + client.clone(), tx.clone(), interval, Arc::clone(&components_patterns), )), tokio::spawn(sent_events_throughputs( - Arc::clone(&client), - tx.clone(), - interval, - Arc::clone(&components_patterns), - )), - #[cfg(feature = "allocation-tracing")] - tokio::spawn(allocated_bytes( - Arc::clone(&client), + client.clone(), tx.clone(), interval, Arc::clone(&components_patterns), )), tokio::spawn(errors_totals( - Arc::clone(&client), + client.clone(), tx.clone(), interval, Arc::clone(&components_patterns), )), - tokio::spawn(uptime_changed(Arc::clone(&client), tx)), - ] + tokio::spawn(uptime_changed(client.clone(), tx.clone(), interval)), + ]; + + #[cfg(feature = "allocation-tracing")] + metric_handles.push(tokio::spawn(allocated_bytes( + client, + tx, + interval, + Arc::clone(&components_patterns), + ))); + + Ok(SubscribeHandles { + metric_handles, + poll_handle, + }) } /// Retrieve the initial components/metrics for first paint. Further updating the metrics /// will be handled by subscriptions. pub async fn init_components( - client: &Client, + uri: Uri, components_patterns: &[Pattern], -) -> Result { - // Execute a query to get the latest components, and aggregate metrics for each resource. - // Since we don't know currently have a mechanism for scrolling/paging through results, - // we're using an artificially high page size to capture all likely component configurations. - let rows = client - .components_query(i16::MAX as i64) - .await - .map_err(|_| ())? - .data - .ok_or(())? +) -> Result { + let mut client = Client::new(uri); + client.connect().await?; + + // Get all components + let response = client.get_components(0).await?; + + let rows = response .components - .edges .into_iter() - .filter(|edge| component_matches_patterns(&edge.node.component_id, components_patterns)) - .map(|edge| { - let d = edge.node; - let key = ComponentKey::from(d.component_id); - ( - key.clone(), - state::ComponentRow { - key, - kind: d.on.to_string(), - component_type: d.component_type, - outputs: d - .on - .outputs() - .into_iter() - .map(|(id, sent_events_total)| (id, OutputMetrics::from(sent_events_total))) - .collect(), - received_bytes_total: d.on.received_bytes_total(), - received_bytes_throughput_sec: 0, - received_events_total: d.on.received_events_total(), - received_events_throughput_sec: 0, - sent_bytes_total: d.on.sent_bytes_total(), - sent_bytes_throughput_sec: 0, - sent_events_total: d.on.sent_events_total(), - sent_events_throughput_sec: 0, - #[cfg(feature = "allocation-tracing")] - allocated_bytes: 0, - errors: 0, - }, - ) + .filter(|component| { + component_matches_patterns(&component.component_id, components_patterns) + }) + .map(|component| { + let row = component_to_row(&component); + (row.key.clone(), row) }) .collect::>(); diff --git a/lib/vector-top/src/state.rs b/lib/vector-top/src/state.rs index e4c535f7851f0..a954c804ce467 100644 --- a/lib/vector-top/src/state.rs +++ b/lib/vector-top/src/state.rs @@ -21,7 +21,7 @@ use crate::dashboard::columns; type IdentifiedMetric = (ComponentKey, i64); -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct SentEventsMetric { pub key: ComponentKey, pub total: i64, @@ -33,18 +33,18 @@ pub enum EventType { InitializeState(State), UptimeChanged(f64), ReceivedBytesTotals(Vec), - /// Interval + identified metric - ReceivedBytesThroughputs(i64, Vec), + /// Throughput values already normalized to per-second by the server + ReceivedBytesThroughputs(Vec), ReceivedEventsTotals(Vec), - /// Interval in ms + identified metric - ReceivedEventsThroughputs(i64, Vec), + /// Throughput values already normalized to per-second by the server + ReceivedEventsThroughputs(Vec), SentBytesTotals(Vec), - /// Interval + identified metric - SentBytesThroughputs(i64, Vec), + /// Throughput values already normalized to per-second by the server + SentBytesThroughputs(Vec), // Identified overall metric + output-specific metrics SentEventsTotals(Vec), - /// Interval in ms + identified overall metric + output-specific metrics - SentEventsThroughputs(i64, Vec), + /// Throughput values already normalized to per-second by the server + SentEventsThroughputs(Vec), ErrorsTotals(Vec), #[cfg(feature = "allocation-tracing")] AllocatedBytes(Vec), @@ -437,11 +437,10 @@ pub async fn updater(mut event_rx: EventRx, mut state: State) -> StateRx { } } } - EventType::ReceivedBytesThroughputs(interval, rows) => { + EventType::ReceivedBytesThroughputs(rows) => { for (key, v) in rows { if let Some(r) = state.components.get_mut(&key) { - r.received_bytes_throughput_sec = - (v as f64 * (1000.0 / interval as f64)) as i64; + r.received_bytes_throughput_sec = v; } } } @@ -452,11 +451,10 @@ pub async fn updater(mut event_rx: EventRx, mut state: State) -> StateRx { } } } - EventType::ReceivedEventsThroughputs(interval, rows) => { + EventType::ReceivedEventsThroughputs(rows) => { for (key, v) in rows { if let Some(r) = state.components.get_mut(&key) { - r.received_events_throughput_sec = - (v as f64 * (1000.0 / interval as f64)) as i64; + r.received_events_throughput_sec = v; } } } @@ -467,11 +465,10 @@ pub async fn updater(mut event_rx: EventRx, mut state: State) -> StateRx { } } } - EventType::SentBytesThroughputs(interval, rows) => { + EventType::SentBytesThroughputs(rows) => { for (key, v) in rows { if let Some(r) = state.components.get_mut(&key) { - r.sent_bytes_throughput_sec = - (v as f64 * (1000.0 / interval as f64)) as i64; + r.sent_bytes_throughput_sec = v; } } } @@ -488,17 +485,15 @@ pub async fn updater(mut event_rx: EventRx, mut state: State) -> StateRx { } } } - EventType::SentEventsThroughputs(interval, rows) => { + EventType::SentEventsThroughputs(rows) => { for m in rows { if let Some(r) = state.components.get_mut(&m.key) { - r.sent_events_throughput_sec = - (m.total as f64 * (1000.0 / interval as f64)) as i64; + r.sent_events_throughput_sec = m.total; for (id, v) in m.outputs { - let throughput = (v as f64 * (1000.0 / interval as f64)) as i64; r.outputs .entry(id) .or_insert_with(OutputMetrics::default) - .sent_events_throughput_sec = throughput; + .sent_events_throughput_sec = v; } } } diff --git a/proto/vector/observability.proto b/proto/vector/observability.proto new file mode 100644 index 0000000000000..1c1d508e5d26c --- /dev/null +++ b/proto/vector/observability.proto @@ -0,0 +1,203 @@ +syntax = "proto3"; +package vector.observability.v1; + +import "google/protobuf/timestamp.proto"; +import "event.proto"; + +// Observability service for monitoring Vector instances. +service ObservabilityService { + // ========== Simple Queries ========== + + // Check if the API server is healthy and responding + rpc Health(HealthRequest) returns (HealthResponse); + + // Get metadata about the Vector instance (version, hostname) + rpc GetMeta(GetMetaRequest) returns (GetMetaResponse); + + // Get information about configured components (sources, transforms, sinks) + rpc GetComponents(GetComponentsRequest) returns (GetComponentsResponse); + + // ========== Real-time Metric Streams ========== + // All streaming RPCs send periodic updates at the specified interval + + // Stream periodic heartbeat timestamps + rpc StreamHeartbeat(StreamHeartbeatRequest) returns (stream StreamHeartbeatResponse); + + // Stream uptime in seconds + rpc StreamUptime(StreamUptimeRequest) returns (stream StreamUptimeResponse); + + // Stream memory allocated per component + rpc StreamComponentAllocatedBytes(StreamComponentAllocatedBytesRequest) returns (stream StreamComponentAllocatedBytesResponse); + + // Stream per-component metrics (throughput or totals) for a chosen metric. + // Replaces the former 9 per-metric streaming RPCs with a single unified endpoint. + rpc StreamComponentMetrics(StreamComponentMetricsRequest) returns (stream StreamComponentMetricsResponse); + + // ========== Event Tapping ========== + + // Stream events from components matching the specified patterns (replaces vector tap) + rpc StreamOutputEvents(StreamOutputEventsRequest) returns (stream StreamOutputEventsResponse); +} + +// ========== Health & Meta Messages ========== + +message HealthRequest {} + +message HealthResponse { + bool healthy = 1; +} + +message GetMetaRequest {} + +message GetMetaResponse { + string version = 1; + string hostname = 2; +} + +// ========== Component Messages ========== + +message GetComponentsRequest { + // Maximum number of components to return (0 = no limit) + int32 limit = 1; +} + +message GetComponentsResponse { + repeated Component components = 1; +} + +message Component { + string component_id = 1; + ComponentType component_type = 2; + string on_type = 3; // The specific component type name (e.g., "file", "http") + repeated Output outputs = 4; + ComponentMetrics metrics = 5; +} + +enum ComponentType { + COMPONENT_TYPE_UNSPECIFIED = 0; + COMPONENT_TYPE_SOURCE = 1; + COMPONENT_TYPE_TRANSFORM = 2; + COMPONENT_TYPE_SINK = 3; +} + +message Output { + string output_id = 1; + int64 sent_events_total = 2; +} + +message ComponentMetrics { + optional int64 received_bytes_total = 1; + optional int64 received_events_total = 2; + optional int64 sent_bytes_total = 3; + optional int64 sent_events_total = 4; +} + +// ========== Streaming Metrics Messages ========== + +message StreamComponentAllocatedBytesRequest { + // Update interval in milliseconds + int32 interval_ms = 1; +} + +// Identifies which per-component metric to stream via StreamComponentMetrics. +enum MetricName { + METRIC_NAME_UNSPECIFIED = 0; + METRIC_NAME_RECEIVED_EVENTS_THROUGHPUT = 1; + METRIC_NAME_SENT_EVENTS_THROUGHPUT = 2; + METRIC_NAME_RECEIVED_BYTES_THROUGHPUT = 3; + METRIC_NAME_SENT_BYTES_THROUGHPUT = 4; + METRIC_NAME_RECEIVED_EVENTS_TOTAL = 5; + METRIC_NAME_SENT_EVENTS_TOTAL = 6; + METRIC_NAME_RECEIVED_BYTES_TOTAL = 7; + METRIC_NAME_SENT_BYTES_TOTAL = 8; + METRIC_NAME_ERRORS_TOTAL = 9; +} + +message StreamComponentMetricsRequest { + // Update interval in milliseconds + int32 interval_ms = 1; + MetricName metric = 2; +} + +// Cumulative total value for a component metric. +// output_totals is only populated for METRIC_NAME_SENT_EVENTS_TOTAL. +message TotalMetric { + int64 value = 1; + map output_totals = 2; +} + +// Throughput value (per-second rate) for a component metric. +// output_throughputs is only populated for METRIC_NAME_SENT_EVENTS_THROUGHPUT. +message ThroughputMetric { + double value = 1; + map output_throughputs = 2; +} + +message StreamComponentMetricsResponse { + string component_id = 1; + oneof value { + TotalMetric total = 2; + ThroughputMetric throughput = 3; + } +} + +message StreamHeartbeatRequest { + // Update interval in milliseconds + int32 interval_ms = 1; +} + +message StreamHeartbeatResponse { + google.protobuf.Timestamp utc = 1; +} + +message StreamUptimeRequest { + // Update interval in milliseconds + int32 interval_ms = 1; +} + +message StreamUptimeResponse { + int64 uptime_seconds = 1; +} + +message StreamComponentAllocatedBytesResponse { + string component_id = 1; + int64 allocated_bytes = 2; +} + +// ========== Event Tapping Messages ========== + +message StreamOutputEventsRequest { + // Glob patterns for output component IDs to match + repeated string outputs_patterns = 1; + + // Glob patterns for input component IDs to match (optional) + repeated string inputs_patterns = 2; + + // Maximum number of events to sample per interval. Must be >= 1. + // Unlike GetComponents, 0 is not valid here: limit controls both the + // reservoir size and the internal channel capacity, which require a + // positive value. + int32 limit = 3; + + // Sampling interval in milliseconds + int32 interval_ms = 4; +} + +message StreamOutputEventsResponse { + oneof event { + TappedEvent tapped_event = 1; + EventNotification notification = 2; + } +} + +// Event captured from a component with metadata about where it came from +message TappedEvent { + string component_id = 1; + string component_type = 2; + string component_kind = 3; + event.EventWrapper event = 4; // The actual event data (uses canonical proto definition) +} + +message EventNotification { + string message = 1; +} diff --git a/src/api/grpc/mod.rs b/src/api/grpc/mod.rs new file mode 100644 index 0000000000000..f3df61dc08da2 --- /dev/null +++ b/src/api/grpc/mod.rs @@ -0,0 +1,3 @@ +mod service; + +pub use service::ObservabilityService; diff --git a/src/api/grpc/service.rs b/src/api/grpc/service.rs new file mode 100644 index 0000000000000..171f65853cc91 --- /dev/null +++ b/src/api/grpc/service.rs @@ -0,0 +1,863 @@ +use std::collections::{HashMap, HashSet}; +use std::pin::Pin; +// std::sync::Mutex is intentional: the lock is never held across an .await point +// (only used in synchronous map updates inside IntervalStream closures), so the +// cheaper std mutex is correct here. tokio::sync::Mutex is only needed when the +// critical section itself contains .await. +use std::sync::{ + Arc, Mutex, + atomic::{AtomicBool, Ordering}, +}; +use std::time::Duration; + +use futures::{StreamExt as FuturesStreamExt, stream}; +use rand::{Rng, SeedableRng as _, rngs::SmallRng}; +use tokio::select; +use tokio::sync::mpsc; +use tokio::time::{self, interval}; +use tokio_stream::{ + Stream, + wrappers::{IntervalStream, ReceiverStream}, +}; +use tonic::{Request, Response, Status}; +use vector_lib::tap::{ + controller::{TapController, TapPatterns, TapPayload}, + topology::WatchRx, +}; + +use crate::event::{Metric, MetricValue}; +use crate::metrics::Controller; +use crate::proto::observability::{ + self, Component as ProtoComponent, ComponentType, EventNotification, TappedEvent, *, +}; + +type BoxStream = Pin> + Send>>; + +/// Helper function to extract metric value as f64 +fn get_metric_value(metric: &Metric) -> Option { + match metric.value() { + MetricValue::Counter { value } => Some(*value), + MetricValue::Gauge { value } => Some(*value), + _ => None, + } +} + +/// Helper function to filter metrics by name and group by component_id tag +fn filter_and_group_metrics(metrics: &[Metric], metric_name: &str) -> HashMap { + let mut result = HashMap::new(); + + for metric in metrics.iter().filter(|m| m.name() == metric_name) { + if let Some(tags) = metric.tags() + && let Some(component_id) = tags.get("component_id") + && let Some(value) = get_metric_value(metric) + { + *result.entry(component_id.to_string()).or_insert(0.0) += value; + } + } + + result +} + +/// Filter metrics by name and group by (component_id, output) tag pair. +/// Used to populate per-output metrics in GetComponents responses. +fn filter_and_group_metrics_by_output( + metrics: &[Metric], + metric_name: &str, +) -> HashMap<(String, String), f64> { + let mut result = HashMap::new(); + + for metric in metrics.iter().filter(|m| m.name() == metric_name) { + if let Some(tags) = metric.tags() + && let Some(component_id) = tags.get("component_id") + && let Some(value) = get_metric_value(metric) + { + let output = tags.get("output").unwrap_or("_default").to_string(); + *result + .entry((component_id.to_string(), output)) + .or_insert(0.0) += value; + } + } + + result +} + +/// Extract all component metrics and group by component_id +fn extract_component_metrics(metrics: &[Metric]) -> HashMap { + let received_bytes = filter_and_group_metrics(metrics, "component_received_bytes_total"); + let received_events = filter_and_group_metrics(metrics, "component_received_events_total"); + let sent_bytes = filter_and_group_metrics(metrics, "component_sent_bytes_total"); + let sent_events = filter_and_group_metrics(metrics, "component_sent_events_total"); + + let mut all_component_ids = HashSet::new(); + all_component_ids.extend(received_bytes.keys().cloned()); + all_component_ids.extend(received_events.keys().cloned()); + all_component_ids.extend(sent_bytes.keys().cloned()); + all_component_ids.extend(sent_events.keys().cloned()); + + let mut result = HashMap::new(); + for component_id in all_component_ids { + result.insert( + component_id.clone(), + ComponentMetrics { + received_bytes_total: received_bytes.get(&component_id).map(|v| *v as i64), + received_events_total: received_events.get(&component_id).map(|v| *v as i64), + sent_bytes_total: sent_bytes.get(&component_id).map(|v| *v as i64), + sent_events_total: sent_events.get(&component_id).map(|v| *v as i64), + }, + ); + } + + result +} + +/// Helper function to calculate throughput by comparing current and previous values +fn calculate_throughput( + current: &HashMap, + previous: &HashMap, + interval_secs: f64, +) -> HashMap { + let mut result = HashMap::new(); + + for (component_id, current_value) in current { + if let Some(previous_value) = previous.get(component_id) { + let delta = current_value - previous_value; + let throughput = if interval_secs > 0.0 { + delta / interval_secs + } else { + 0.0 + }; + result.insert(component_id.clone(), throughput.max(0.0)); + } + } + + result +} + +/// Helper function to calculate throughput for per-output metrics by comparing +/// current and previous values keyed by `(component_id, output)`. +fn calculate_throughput_by_output( + current: &HashMap<(String, String), f64>, + previous: &HashMap<(String, String), f64>, + interval_secs: f64, +) -> HashMap<(String, String), f64> { + let mut result = HashMap::new(); + + for (key, current_value) in current { + if let Some(previous_value) = previous.get(key) { + let delta = current_value - previous_value; + let throughput = if interval_secs > 0.0 { + delta / interval_secs + } else { + 0.0 + }; + result.insert(key.clone(), throughput.max(0.0)); + } + } + + result +} + +/// Minimum allowed polling interval for metric streams (100 ms). +/// +/// Prevents clients from setting `interval_ms=1` and hammering `capture_metrics()` +/// at 1 kHz, which would cause high CPU load across concurrent subscriptions. +const MIN_INTERVAL_MS: i32 = 100; + +/// Validates `interval_ms` from a streaming request, returning the value as `u64` +/// or a gRPC error if it is out of range. +fn validate_interval_ms(interval_ms: i32) -> Result { + if interval_ms < MIN_INTERVAL_MS { + return Err(Status::invalid_argument(format!( + "interval_ms must be >= {MIN_INTERVAL_MS}" + ))); + } + Ok(interval_ms as u64) +} + +fn get_controller() -> Result<&'static Controller, Status> { + Controller::get().map_err(|_| Status::internal("Metrics system not initialized")) +} + +/// Builds a stream that emits per-component totals for `metric_name` every `duration`. +fn metric_totals_stream( + duration: Duration, + metric_name: &'static str, +) -> Result, Status> { + let controller = get_controller()?; + Ok(Box::pin( + tokio_stream::StreamExt::map(IntervalStream::new(interval(duration)), move |_| { + let metrics = controller.capture_metrics(); + let component_metrics = filter_and_group_metrics(&metrics, metric_name); + tokio_stream::iter( + component_metrics + .into_iter() + .map(|(component_id, total)| { + Ok(StreamComponentMetricsResponse { + component_id, + value: Some(stream_component_metrics_response::Value::Total( + TotalMetric { + value: total as i64, + output_totals: Default::default(), + }, + )), + }) + }) + .collect::>(), + ) + }) + .flatten(), + )) +} + +/// Builds a stream that emits per-component throughputs for `metric_name` every `duration`. +fn metric_throughput_stream( + duration: Duration, + metric_name: &'static str, +) -> Result, Status> { + let controller = get_controller()?; + let interval_secs = duration.as_secs_f64(); + let previous_values = Arc::new(Mutex::new(HashMap::new())); + Ok(Box::pin( + tokio_stream::StreamExt::map(IntervalStream::new(interval(duration)), move |_| { + let metrics = controller.capture_metrics(); + let current_values = filter_and_group_metrics(&metrics, metric_name); + let mut prev = match previous_values.lock() { + Ok(guard) => guard, + Err(poisoned) => { + error!("Mutex poisoned for metric throughput, recovering."); + poisoned.into_inner() + } + }; + let throughputs = calculate_throughput(¤t_values, &prev, interval_secs); + *prev = current_values; + tokio_stream::iter( + throughputs + .into_iter() + .map(|(component_id, throughput)| { + Ok(StreamComponentMetricsResponse { + component_id, + value: Some(stream_component_metrics_response::Value::Throughput( + ThroughputMetric { + value: throughput, + output_throughputs: Default::default(), + }, + )), + }) + }) + .collect::>(), + ) + }) + .flatten(), + )) +} + +/// Builds a stream that emits per-component sent_events totals with per-output breakdown. +fn sent_events_totals_stream( + duration: Duration, +) -> Result, Status> { + let controller = get_controller()?; + Ok(Box::pin( + tokio_stream::StreamExt::map(IntervalStream::new(interval(duration)), move |_| { + let metrics = controller.capture_metrics(); + let component_totals = + filter_and_group_metrics(&metrics, "component_sent_events_total"); + let by_output = + filter_and_group_metrics_by_output(&metrics, "component_sent_events_total"); + + // Group per-output values by component_id + let mut output_by_component: HashMap> = HashMap::new(); + for ((component_id, output), value) in by_output { + output_by_component + .entry(component_id) + .or_default() + .insert(output, value as i64); + } + + tokio_stream::iter( + component_totals + .into_iter() + .map(|(component_id, total)| { + let output_totals = output_by_component + .remove(&component_id) + .unwrap_or_default() + .into_iter() + .collect(); + Ok(StreamComponentMetricsResponse { + component_id, + value: Some(stream_component_metrics_response::Value::Total( + TotalMetric { + value: total as i64, + output_totals, + }, + )), + }) + }) + .collect::>(), + ) + }) + .flatten(), + )) +} + +/// Builds a stream that emits per-component sent_events throughputs with per-output breakdown. +fn sent_events_throughput_stream( + duration: Duration, +) -> Result, Status> { + let controller = get_controller()?; + let interval_secs = duration.as_secs_f64(); + let previous_totals = Arc::new(Mutex::new(HashMap::::new())); + let previous_outputs = Arc::new(Mutex::new(HashMap::<(String, String), f64>::new())); + Ok(Box::pin( + tokio_stream::StreamExt::map(IntervalStream::new(interval(duration)), move |_| { + let metrics = controller.capture_metrics(); + let current_totals = filter_and_group_metrics(&metrics, "component_sent_events_total"); + let current_outputs = + filter_and_group_metrics_by_output(&metrics, "component_sent_events_total"); + + let mut prev_t = match previous_totals.lock() { + Ok(guard) => guard, + Err(poisoned) => { + error!("Mutex poisoned for sent_events throughput totals, recovering."); + poisoned.into_inner() + } + }; + let mut prev_o = match previous_outputs.lock() { + Ok(guard) => guard, + Err(poisoned) => { + error!("Mutex poisoned for sent_events throughput outputs, recovering."); + poisoned.into_inner() + } + }; + + let throughputs = calculate_throughput(¤t_totals, &prev_t, interval_secs); + let output_throughputs_flat = + calculate_throughput_by_output(¤t_outputs, &prev_o, interval_secs); + + *prev_t = current_totals; + *prev_o = current_outputs; + + // Group per-output throughputs by component_id + let mut output_by_component: HashMap> = HashMap::new(); + for ((component_id, output), tp) in output_throughputs_flat { + output_by_component + .entry(component_id) + .or_default() + .insert(output, tp); + } + + tokio_stream::iter( + throughputs + .into_iter() + .map(|(component_id, throughput)| { + let output_throughputs = output_by_component + .remove(&component_id) + .unwrap_or_default() + .into_iter() + .collect(); + Ok(StreamComponentMetricsResponse { + component_id, + value: Some(stream_component_metrics_response::Value::Throughput( + ThroughputMetric { + value: throughput, + output_throughputs, + }, + )), + }) + }) + .collect::>(), + ) + }) + .flatten(), + )) +} + +/// Converts a component's output port names into proto `Output` messages, +/// populating `sent_events_total` from the per-output metric snapshot. +/// +/// `None` port means the default output (represented as `"_default"`). +fn ports_to_proto_outputs( + ports: &[Option<&str>], + component_id: &str, + sent_events_by_output: &HashMap<(String, String), f64>, +) -> Vec { + ports + .iter() + .map(|port| { + let output_id = port.unwrap_or("_default").to_string(); + let sent_events_total = sent_events_by_output + .get(&(component_id.to_string(), output_id.clone())) + .copied() + .unwrap_or(0.0) as i64; + Output { + output_id, + sent_events_total, + } + }) + .collect() +} + +/// gRPC observability service implementation. +pub struct ObservabilityService { + watch_rx: WatchRx, + running: Arc, +} + +impl ObservabilityService { + pub const fn new(watch_rx: WatchRx, running: Arc) -> Self { + Self { watch_rx, running } + } +} + +#[tonic::async_trait] +impl observability::Service for ObservabilityService { + // ========== Simple Queries ========== + + async fn health( + &self, + _request: Request, + ) -> Result, Status> { + if self.running.load(Ordering::Relaxed) { + Ok(Response::new(HealthResponse { healthy: true })) + } else { + Err(Status::unavailable("Vector is shutting down")) + } + } + + async fn get_meta( + &self, + _request: Request, + ) -> Result, Status> { + let version = crate::get_version().to_string(); + let hostname = hostname::get() + .ok() + .and_then(|h| h.into_string().ok()) + .unwrap_or_else(|| "unknown".to_string()); + + Ok(Response::new(GetMetaResponse { version, hostname })) + } + + async fn get_components( + &self, + request: Request, + ) -> Result, Status> { + let limit = request.into_inner().limit; + + // Get the current topology snapshot + let tap_resource = self.watch_rx.borrow().clone(); + + // Get metrics for all components + let controller = + Controller::get().map_err(|_| Status::internal("Metrics system not initialized"))?; + let metrics = controller.capture_metrics(); + let component_metrics_map = extract_component_metrics(&metrics); + let sent_events_by_output = + filter_and_group_metrics_by_output(&metrics, "component_sent_events_total"); + + let type_map = &tap_resource.type_names; + + // `tap_resource.outputs` and `tap_resource.inputs` are both full topology snapshots, + // maintained incrementally on RunningTopology. `source_keys`/`sink_keys` are diff-only + // (changed/added in the last reload) and must not be used for enumeration here. + // + // Component kind is determined by set membership: + // source = has outputs, no inputs + // transform = has outputs AND inputs + // sink = has inputs, no outputs + let output_ports = tap_resource.output_ports_by_component(); + + let mut components = Vec::new(); + + // Sources: present in `output_ports` but not in `inputs`. + // `output_ports` is keyed by component, so no duplicates. + for (key, ports) in &output_ports { + if tap_resource.inputs.contains_key(*key) { + continue; // transform or sink, handled below + } + let key_str = key.to_string(); + let on_type = type_map + .get(&key_str) + .cloned() + .unwrap_or_else(|| "unknown".to_string()); + components.push(ProtoComponent { + component_id: key_str.clone(), + component_type: ComponentType::Source as i32, + on_type, + outputs: ports_to_proto_outputs(ports, &key_str, &sent_events_by_output), + metrics: component_metrics_map.get(&key_str).cloned(), + }); + } + + // Transforms and sinks: present in `inputs` (full snapshot) + for component_key in tap_resource.inputs.keys() { + let key_str = component_key.to_string(); + let (component_type, outputs) = if let Some(ports) = output_ports.get(component_key) { + ( + ComponentType::Transform, + ports_to_proto_outputs(ports, &key_str, &sent_events_by_output), + ) + } else { + (ComponentType::Sink, vec![]) + }; + let on_type = type_map + .get(&key_str) + .cloned() + .unwrap_or_else(|| "unknown".to_string()); + components.push(ProtoComponent { + component_id: key_str.clone(), + component_type: component_type as i32, + on_type, + outputs, + metrics: component_metrics_map.get(&key_str).cloned(), + }); + } + + // Sort alphabetically for stable, deterministic ordering before applying limit + components.sort_unstable_by(|a, b| a.component_id.cmp(&b.component_id)); + if limit > 0 { + components.truncate(limit as usize); + } + + Ok(Response::new(GetComponentsResponse { components })) + } + + // ========== Streaming Metrics ========== + + type StreamHeartbeatStream = BoxStream; + + async fn stream_heartbeat( + &self, + request: Request, + ) -> Result, Status> { + let duration = + Duration::from_millis(validate_interval_ms(request.into_inner().interval_ms)?); + let stream = tokio_stream::StreamExt::map(IntervalStream::new(interval(duration)), |_| { + let utc = Some(prost_types::Timestamp { + seconds: chrono::Utc::now().timestamp(), + nanos: 0, + }); + Ok(StreamHeartbeatResponse { utc }) + }); + + Ok(Response::new(Box::pin(stream))) + } + + type StreamUptimeStream = BoxStream; + + async fn stream_uptime( + &self, + request: Request, + ) -> Result, Status> { + let duration = + Duration::from_millis(validate_interval_ms(request.into_inner().interval_ms)?); + let controller = get_controller()?; + let stream = + tokio_stream::StreamExt::map(IntervalStream::new(interval(duration)), move |_| { + let metrics = controller.capture_metrics(); + let uptime_seconds = metrics + .iter() + .find(|m| m.name() == "uptime_seconds") + .and_then(get_metric_value) + .unwrap_or(0.0) as i64; + + Ok(StreamUptimeResponse { uptime_seconds }) + }); + + Ok(Response::new(Box::pin(stream))) + } + + type StreamComponentAllocatedBytesStream = BoxStream; + + async fn stream_component_allocated_bytes( + &self, + request: Request, + ) -> Result, Status> { + let duration = + Duration::from_millis(validate_interval_ms(request.into_inner().interval_ms)?); + let controller = get_controller()?; + let stream = + tokio_stream::StreamExt::map(IntervalStream::new(interval(duration)), move |_| { + let metrics = controller.capture_metrics(); + let component_metrics = + filter_and_group_metrics(&metrics, "component_allocated_bytes"); + + tokio_stream::iter( + component_metrics + .into_iter() + .map(|(component_id, allocated_bytes)| { + Ok(StreamComponentAllocatedBytesResponse { + component_id, + allocated_bytes: allocated_bytes as i64, + }) + }) + .collect::>(), + ) + }) + .flatten(); + + Ok(Response::new(Box::pin(stream))) + } + + type StreamComponentMetricsStream = BoxStream; + + async fn stream_component_metrics( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + let duration = Duration::from_millis(validate_interval_ms(req.interval_ms)?); + let metric = MetricName::try_from(req.metric) + .map_err(|_| Status::invalid_argument("Unknown metric value"))?; + + let stream: BoxStream = match metric { + MetricName::Unspecified => { + return Err(Status::invalid_argument("metric must be specified")); + } + MetricName::ReceivedEventsTotal => { + metric_totals_stream(duration, "component_received_events_total")? + } + MetricName::SentEventsTotal => sent_events_totals_stream(duration)?, + MetricName::ReceivedBytesTotal => { + metric_totals_stream(duration, "component_received_bytes_total")? + } + MetricName::SentBytesTotal => { + metric_totals_stream(duration, "component_sent_bytes_total")? + } + MetricName::ErrorsTotal => metric_totals_stream(duration, "component_errors_total")?, + MetricName::ReceivedEventsThroughput => { + metric_throughput_stream(duration, "component_received_events_total")? + } + MetricName::SentEventsThroughput => sent_events_throughput_stream(duration)?, + MetricName::ReceivedBytesThroughput => { + metric_throughput_stream(duration, "component_received_bytes_total")? + } + MetricName::SentBytesThroughput => { + metric_throughput_stream(duration, "component_sent_bytes_total")? + } + }; + + Ok(Response::new(stream)) + } + + // ========== Event Tapping ========== + + type StreamOutputEventsStream = BoxStream; + + async fn stream_output_events( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + + // Validate before casting to prevent negative values from becoming large positive values + if req.limit <= 0 { + return Err(Status::invalid_argument( + "limit must be >= 1 (controls reservoir size and channel capacity)", + )); + } + + const MAX_TAP_LIMIT: i32 = 100_000; + if req.limit > MAX_TAP_LIMIT { + return Err(Status::invalid_argument(format!( + "limit must be <= {MAX_TAP_LIMIT}" + ))); + } + + let interval_ms = validate_interval_ms(req.interval_ms)?; + let limit = req.limit as usize; + + let patterns = TapPatterns { + for_outputs: req.outputs_patterns.into_iter().collect(), + for_inputs: req.inputs_patterns.into_iter().collect(), + }; + + // Channel for receiving tap payloads + let (tap_tx, tap_rx) = mpsc::channel(limit); + + // Channel for sending events to the client + let (event_tx, event_rx) = mpsc::channel::>(10); + + let watch_rx = self.watch_rx.clone(); + + tokio::spawn(async move { + let _tap_controller = TapController::new(watch_rx, tap_tx, patterns); + let mut tap_rx = ReceiverStream::new(tap_rx); + let mut interval = time::interval(time::Duration::from_millis(interval_ms)); + let mut reservoir = Reservoir::new(limit); + + loop { + select! { + Some(tap_payload) = tokio_stream::StreamExt::next(&mut tap_rx) => { + if reservoir.handle_payload(tap_payload, &event_tx).await.is_err() { + break; + } + } + _ = interval.tick() => { + if event_tx.is_closed() || reservoir.flush(&event_tx).await.is_err() { + break; + } + } + } + } + }); + + let stream = FuturesStreamExt::flat_map(ReceiverStream::new(event_rx), |events| { + stream::iter(events.into_iter().map(Ok)) + }); + + Ok(Response::new(Box::pin(stream))) + } +} + +/// Reservoir sampler for tap events, batched and flushed on an interval. +struct Reservoir { + events: Vec<(usize, StreamOutputEventsResponse)>, + rng: SmallRng, + batch: usize, + limit: usize, +} + +impl Reservoir { + fn new(limit: usize) -> Self { + Self { + events: Vec::with_capacity(limit), + rng: SmallRng::from_rng(&mut rand::rng()), + batch: 0, + limit, + } + } + + /// Process a tap payload: notifications are forwarded immediately; data events + /// are reservoir-sampled (Algorithm R) for the next flush. + async fn handle_payload( + &mut self, + payload: TapPayload, + tx: &mpsc::Sender>, + ) -> Result<(), ()> { + for event in tap_payload_to_output_events(payload) { + if matches!( + event.event, + Some(stream_output_events_response::Event::Notification(_)) + ) { + tx.send(vec![event]).await.map_err(|err| { + debug!(message = "Couldn't send notification.", error = ?err); + })?; + } else { + // Reservoir sampling (Algorithm R). + // Draw from 0..=batch (inclusive) so that event i has + // exactly limit/(i+1) probability of entering the reservoir. + // Using 0..batch (exclusive) would guarantee replacement on + // the first post-fill event (100% instead of limit/(limit+1)). + if self.limit > self.events.len() { + self.events.push((self.batch, event)); + } else { + let idx = self.rng.random_range(0..=self.batch); + if idx < self.events.len() { + self.events[idx] = (self.batch, event); + } + } + self.batch += 1; + } + } + Ok(()) + } + + /// Flush sampled events to the client, sorted by arrival order. + async fn flush( + &mut self, + tx: &mpsc::Sender>, + ) -> Result<(), ()> { + if self.events.is_empty() { + return Ok(()); + } + self.batch = 0; + self.events.sort_by_key(|(batch, _)| *batch); + let events = self.events.drain(..).map(|(_, e)| e).collect(); + tx.send(events).await.map_err(|err| { + debug!(message = "Couldn't send events.", error = ?err); + }) + } +} + +/// Convert TapPayload to gRPC StreamOutputEventsResponse(s) +fn tap_payload_to_output_events(payload: TapPayload) -> Vec { + use crate::event::proto::{Event, EventWrapper}; + + match payload { + TapPayload::Log(output, log_array) => log_array + .into_iter() + .map(|log_event| { + // Convert Vector's internal LogEvent to proto Log (metadata is preserved in Log.metadata_full) + let proto_log: crate::event::proto::Log = log_event.into(); + let event_wrapper = Some(EventWrapper { + event: Some(Event::Log(proto_log)), + }); + + StreamOutputEventsResponse { + event: Some(stream_output_events_response::Event::TappedEvent( + TappedEvent { + component_id: output.output_id.component.id().to_string(), + component_type: output.component_type.to_string(), + component_kind: output.component_kind.to_string(), + event: event_wrapper, + }, + )), + } + }) + .collect(), + TapPayload::Metric(output, metric_array) => metric_array + .into_iter() + .map(|metric_event| { + // Convert Vector's internal Metric to proto Metric + let proto_metric: crate::event::proto::Metric = metric_event.into(); + let event_wrapper = Some(EventWrapper { + event: Some(Event::Metric(proto_metric)), + }); + + StreamOutputEventsResponse { + event: Some(stream_output_events_response::Event::TappedEvent( + TappedEvent { + component_id: output.output_id.component.id().to_string(), + component_type: output.component_type.to_string(), + component_kind: output.component_kind.to_string(), + event: event_wrapper, + }, + )), + } + }) + .collect(), + TapPayload::Trace(output, trace_array) => trace_array + .into_iter() + .map(|trace_event| { + // Convert Vector's internal TraceEvent to proto Trace + let proto_trace: crate::event::proto::Trace = trace_event.into(); + let event_wrapper = Some(EventWrapper { + event: Some(Event::Trace(proto_trace)), + }); + + StreamOutputEventsResponse { + event: Some(stream_output_events_response::Event::TappedEvent( + TappedEvent { + component_id: output.output_id.component.id().to_string(), + component_type: output.component_type.to_string(), + component_kind: output.component_kind.to_string(), + event: event_wrapper, + }, + )), + } + }) + .collect(), + TapPayload::Notification(notification) => { + vec![create_notification_event(notification.as_str())] + } + } +} + +fn create_notification_event(message: &str) -> StreamOutputEventsResponse { + StreamOutputEventsResponse { + event: Some(stream_output_events_response::Event::Notification( + EventNotification { + message: message.to_string(), + }, + )), + } +} diff --git a/src/api/grpc_server.rs b/src/api/grpc_server.rs new file mode 100644 index 0000000000000..714459cd39b79 --- /dev/null +++ b/src/api/grpc_server.rs @@ -0,0 +1,96 @@ +use std::{ + error::Error as StdError, + net::SocketAddr, + sync::{Arc, atomic::AtomicBool}, +}; +use tokio::sync::oneshot; +use tokio_stream::wrappers::TcpListenerStream; +use tonic::transport::Server as TonicServer; +use vector_lib::tap::topology::WatchRx; + +use super::grpc::ObservabilityService; +use crate::{config::Config, proto::observability::Server as ObservabilityServer}; + +/// gRPC API server for Vector observability. +pub struct GrpcServer { + _shutdown: oneshot::Sender<()>, + addr: SocketAddr, +} + +impl GrpcServer { + /// Start the gRPC API server. + /// + /// This creates a new gRPC server listening on the configured address and spawns + /// it in the background. The server will shut down gracefully when this struct + /// is dropped. + /// + /// Returns an error if the server fails to bind to the configured address. + pub async fn start( + config: &Config, + watch_rx: WatchRx, + running: Arc, + ) -> crate::Result { + let addr = config.api.address.ok_or_else(|| { + crate::Error::from("API address not configured in config.api.address") + })?; + + // Bind the TCP listener first to ensure the port is available + // This will fail fast if the address is already in use + let listener = tokio::net::TcpListener::bind(addr).await.map_err(|e| { + crate::Error::from(format!("Failed to bind gRPC API server to {}: {}", addr, e)) + })?; + + let actual_addr = listener + .local_addr() + .map_err(|e| crate::Error::from(format!("Failed to get local address: {}", e)))?; + + info!("GRPC API server bound to {}.", actual_addr); + + let service = ObservabilityService::new(watch_rx, running); + + let (_shutdown, rx) = oneshot::channel(); + + // Spawn the server with the already-bound listener + tokio::spawn(async move { + let incoming = TcpListenerStream::new(listener); + + // Build reflection service for tools like grpcurl + let reflection_service = tonic_reflection::server::Builder::configure() + .register_encoded_file_descriptor_set( + crate::proto::observability::FILE_DESCRIPTOR_SET, + ) + .build() + .expect("Failed to build reflection service"); + + let result = TonicServer::builder() + .add_service(ObservabilityServer::new(service)) + .add_service(reflection_service) + .serve_with_incoming_shutdown(incoming, async { + rx.await.ok(); + info!("GRPC API server shutting down."); + }) + .await; + + if let Err(e) = result { + error!( + message = "GRPC server encountered an error.", + error = %e, + error_source = ?e.source(), + bind_addr = %actual_addr, + ); + } + }); + + info!("GRPC API server started on {}.", actual_addr); + + Ok(Self { + _shutdown, + addr: actual_addr, + }) + } + + /// Get the address the server is listening on + pub const fn addr(&self) -> SocketAddr { + self.addr + } +} diff --git a/src/api/handler.rs b/src/api/handler.rs deleted file mode 100644 index 72ecfce87c1dc..0000000000000 --- a/src/api/handler.rs +++ /dev/null @@ -1,23 +0,0 @@ -use std::sync::{ - Arc, - atomic::{self, AtomicBool}, -}; - -use serde_json::json; -use warp::{Rejection, Reply, reply::json}; - -// Health handler, responds with '{ ok: true }' when running and '{ ok: false}' -// when shutting down -pub(super) async fn health(running: Arc) -> Result { - if running.load(atomic::Ordering::Relaxed) { - Ok(warp::reply::with_status( - json(&json!({"ok": true})), - warp::http::StatusCode::OK, - )) - } else { - Ok(warp::reply::with_status( - json(&json!({"ok": false})), - warp::http::StatusCode::SERVICE_UNAVAILABLE, - )) - } -} diff --git a/src/api/mod.rs b/src/api/mod.rs index 947321277e728..f1992193e685a 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -1,9 +1,5 @@ #![allow(missing_docs)] -mod handler; -mod schema; -mod server; -#[cfg(all(test, feature = "vector-api-tests"))] -mod tests; +mod grpc; +mod grpc_server; -pub use schema::build_schema; -pub use server::Server; +pub use grpc_server::GrpcServer; diff --git a/src/api/schema/components/mod.rs b/src/api/schema/components/mod.rs deleted file mode 100644 index 69d509e449a50..0000000000000 --- a/src/api/schema/components/mod.rs +++ /dev/null @@ -1,577 +0,0 @@ -pub mod sink; -pub mod source; -pub mod state; -pub mod transform; - -use std::{ - cmp, - collections::{HashMap, HashSet}, - sync::LazyLock, -}; - -use async_graphql::{Enum, InputObject, Interface, Object, Subscription}; -use tokio_stream::{Stream, StreamExt, wrappers::BroadcastStream}; -use vector_lib::internal_event::DEFAULT_OUTPUT; - -use crate::{ - api::schema::{ - components::state::component_by_component_key, - filter::{self, filter_items}, - relay, sort, - }, - config::{ComponentKey, Config, get_transform_output_ids}, - filter_check, -}; - -#[allow(clippy::duplicated_attributes)] // False positive caused by `ty = "String"` -#[derive(Debug, Clone, Interface)] -#[graphql( - field(name = "component_id", ty = "String"), - field(name = "component_type", ty = "String") -)] -pub enum Component { - Source(source::Source), - Transform(transform::Transform), - Sink(sink::Sink), -} - -#[derive(Enum, Copy, Clone, Eq, PartialEq, Ord, PartialOrd)] -pub enum ComponentKind { - Source, - Transform, - Sink, -} - -impl Component { - const fn get_component_key(&self) -> &ComponentKey { - match self { - Component::Source(c) => &c.0.component_key, - Component::Transform(c) => &c.0.component_key, - Component::Sink(c) => &c.0.component_key, - } - } - - const fn get_component_kind(&self) -> ComponentKind { - match self { - Component::Source(_) => ComponentKind::Source, - Component::Transform(_) => ComponentKind::Transform, - Component::Sink(_) => ComponentKind::Sink, - } - } -} - -#[derive(Default, InputObject)] -pub struct ComponentsFilter { - component_id: Option>, - component_kind: Option>>, - or: Option>, -} - -impl filter::CustomFilter for ComponentsFilter { - fn matches(&self, component: &Component) -> bool { - filter_check!( - self.component_id.as_ref().map(|f| f - .iter() - .all(|f| f.filter_value(&component.get_component_key().to_string()))), - self.component_kind.as_ref().map(|f| f - .iter() - .all(|f| f.filter_value(component.get_component_kind()))) - ); - true - } - - fn or(&self) -> Option<&Vec> { - self.or.as_ref() - } -} - -#[derive(Enum, Copy, Clone, Eq, PartialEq)] -pub enum ComponentsSortFieldName { - ComponentKey, - ComponentKind, -} - -impl sort::SortableByField for Component { - fn sort(&self, rhs: &Self, field: &ComponentsSortFieldName) -> cmp::Ordering { - match field { - ComponentsSortFieldName::ComponentKey => { - Ord::cmp(&self.get_component_key(), &rhs.get_component_key()) - } - ComponentsSortFieldName::ComponentKind => { - Ord::cmp(&self.get_component_kind(), &rhs.get_component_kind()) - } - } - } -} - -#[derive(Default)] -pub struct ComponentsQuery; - -#[allow(clippy::too_many_arguments)] -#[Object] -impl ComponentsQuery { - /// Configured components (sources/transforms/sinks) - async fn components( - &self, - after: Option, - before: Option, - first: Option, - last: Option, - filter: Option, - sort: Option>>, - ) -> relay::ConnectionResult { - let filter = filter.unwrap_or_default(); - let mut components = filter_items(state::get_components().into_iter(), &filter); - - if let Some(sort_fields) = sort { - sort::by_fields(&mut components, &sort_fields); - } - - relay::query( - components.into_iter(), - relay::Params::new(after, before, first, last), - 10, - ) - .await - } - - /// Configured sources - async fn sources( - &self, - after: Option, - before: Option, - first: Option, - last: Option, - filter: Option, - sort: Option>>, - ) -> relay::ConnectionResult { - let filter = filter.unwrap_or_default(); - let mut sources = filter_items(state::get_sources().into_iter(), &filter); - - if let Some(sort_fields) = sort { - sort::by_fields(&mut sources, &sort_fields); - } - - relay::query( - sources.into_iter(), - relay::Params::new(after, before, first, last), - 10, - ) - .await - } - - /// Configured transforms - async fn transforms( - &self, - after: Option, - before: Option, - first: Option, - last: Option, - filter: Option, - sort: Option>>, - ) -> relay::ConnectionResult { - let filter = filter.unwrap_or_default(); - let mut transforms = filter_items(state::get_transforms().into_iter(), &filter); - - if let Some(sort_fields) = sort { - sort::by_fields(&mut transforms, &sort_fields); - } - - relay::query( - transforms.into_iter(), - relay::Params::new(after, before, first, last), - 10, - ) - .await - } - - /// Configured sinks - async fn sinks( - &self, - after: Option, - before: Option, - first: Option, - last: Option, - filter: Option, - sort: Option>>, - ) -> relay::ConnectionResult { - let filter = filter.unwrap_or_default(); - let mut sinks = filter_items(state::get_sinks().into_iter(), &filter); - - if let Some(sort_fields) = sort { - sort::by_fields(&mut sinks, &sort_fields); - } - - relay::query( - sinks.into_iter(), - relay::Params::new(after, before, first, last), - 10, - ) - .await - } - - /// Gets a configured component by component_key - async fn component_by_component_key(&self, component_id: String) -> Option { - let key = ComponentKey::from(component_id); - component_by_component_key(&key) - } -} - -#[derive(Clone, Debug)] -enum ComponentChanged { - Added(Component), - Removed(Component), -} - -static COMPONENT_CHANGED: LazyLock> = - LazyLock::new(|| { - let (tx, _) = tokio::sync::broadcast::channel(10); - tx - }); - -#[derive(Debug, Default)] -pub struct ComponentsSubscription; - -#[Subscription] -impl ComponentsSubscription { - /// Subscribes to all newly added components - async fn component_added(&self) -> impl Stream + use<> { - BroadcastStream::new(COMPONENT_CHANGED.subscribe()).filter_map(|c| match c { - Ok(ComponentChanged::Added(c)) => Some(c), - _ => None, - }) - } - - /// Subscribes to all removed components - async fn component_removed(&self) -> impl Stream + use<> { - BroadcastStream::new(COMPONENT_CHANGED.subscribe()).filter_map(|c| match c { - Ok(ComponentChanged::Removed(c)) => Some(c), - _ => None, - }) - } -} - -/// Update the 'global' configuration that will be consumed by component queries -pub fn update_config(config: &Config) { - let mut new_components = HashMap::new(); - - // Sources - let table_sources = config - .enrichment_tables() - .filter_map(|(k, e)| e.as_source(k)) - .collect::>(); - for (component_key, source) in config - .sources() - .chain(table_sources.iter().map(|(k, s)| (k, s))) - { - new_components.insert( - component_key.clone(), - Component::Source(source::Source(source::Data { - component_key: component_key.clone(), - component_type: source.inner.get_component_name().to_string(), - // TODO(#10745): This is obviously wrong, but there are a lot of assumptions in the - // API modules about `output_type` as it's a sortable field, etc. This is a stopgap - // until we decide how we want to change the rest of the usages. - output_type: source - .inner - .outputs(config.schema.log_namespace()) - .pop() - .unwrap() - .ty, - outputs: source - .inner - .outputs(config.schema.log_namespace()) - .into_iter() - .map(|output| output.port.unwrap_or_else(|| DEFAULT_OUTPUT.to_string())) - .collect(), - })), - ); - } - - // Transforms - for (component_key, transform) in config.transforms() { - new_components.insert( - component_key.clone(), - Component::Transform(transform::Transform(transform::Data { - component_key: component_key.clone(), - component_type: transform.inner.get_component_name().to_string(), - inputs: transform.inputs.clone(), - outputs: get_transform_output_ids( - transform.inner.as_ref(), - "".into(), - config.schema.log_namespace(), - ) - .map(|output| output.port.unwrap_or_else(|| DEFAULT_OUTPUT.to_string())) - .collect(), - })), - ); - } - - // Sinks - let table_sinks = config - .enrichment_tables() - .filter_map(|(k, e)| e.as_sink(k)) - .collect::>(); - for (component_key, sink) in config - .sinks() - .chain(table_sinks.iter().map(|(k, s)| (k, s))) - { - new_components.insert( - component_key.clone(), - Component::Sink(sink::Sink(sink::Data { - component_key: component_key.clone(), - component_type: sink.inner.get_component_name().to_string(), - inputs: sink.inputs.clone(), - })), - ); - } - - // Get the component_ids of existing components - let existing_component_keys = state::get_component_keys(); - let new_component_keys = new_components - .keys() - .cloned() - .collect::>(); - - // Publish all components that have been removed - existing_component_keys - .difference(&new_component_keys) - .for_each(|component_key| { - _ = COMPONENT_CHANGED.send(ComponentChanged::Removed( - state::component_by_component_key(component_key) - .expect("Couldn't get component by key"), - )); - }); - - // Publish all components that have been added - new_component_keys - .difference(&existing_component_keys) - .for_each(|component_key| { - _ = COMPONENT_CHANGED.send(ComponentChanged::Added( - new_components.get(component_key).unwrap().clone(), - )); - }); - - // Override the old component state - state::update(new_components); -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::{ - api::schema::sort, - config::{ComponentKey, DataType, OutputId}, - }; - - /// Generate component fixes for use with tests - fn component_fixtures() -> Vec { - vec![ - Component::Source(source::Source(source::Data { - component_key: ComponentKey::from("gen1"), - component_type: "demo_logs".to_string(), - output_type: DataType::Metric, - outputs: vec![], - })), - Component::Source(source::Source(source::Data { - component_key: ComponentKey::from("gen2"), - component_type: "demo_logs".to_string(), - output_type: DataType::Metric, - outputs: vec![], - })), - Component::Source(source::Source(source::Data { - component_key: ComponentKey::from("gen3"), - component_type: "demo_logs".to_string(), - output_type: DataType::Metric, - outputs: vec![], - })), - Component::Transform(transform::Transform(transform::Data { - component_key: ComponentKey::from("parse_json"), - component_type: "json".to_string(), - inputs: vec![OutputId::from("gen1"), OutputId::from("gen2")].into(), - outputs: vec![], - })), - Component::Sink(sink::Sink(sink::Data { - component_key: ComponentKey::from("devnull"), - component_type: "blackhole".to_string(), - inputs: vec![OutputId::from("gen3"), OutputId::from("parse_json")].into(), - })), - ] - } - - #[test] - fn components_filter_contains() { - let filter = ComponentsFilter { - component_id: Some(vec![filter::StringFilter { - contains: Some("gen".to_string()), - ..Default::default() - }]), - ..Default::default() - }; - - let components = filter_items(component_fixtures().into_iter(), &filter); - - assert_eq!(components.len(), 3); - } - - #[test] - fn components_filter_equals_or() { - let filter = ComponentsFilter { - component_id: Some(vec![filter::StringFilter { - equals: Some("gen1".to_string()), - ..Default::default() - }]), - or: Some(vec![ComponentsFilter { - component_id: Some(vec![filter::StringFilter { - equals: Some("devnull".to_string()), - ..Default::default() - }]), - ..Default::default() - }]), - ..Default::default() - }; - - let components = filter_items(component_fixtures().into_iter(), &filter); - - assert_eq!(components.len(), 2); - } - - #[test] - fn components_filter_and() { - let filter = ComponentsFilter { - component_id: Some(vec![filter::StringFilter { - equals: Some("gen1".to_string()), - ..Default::default() - }]), - component_kind: Some(vec![filter::EqualityFilter { - equals: Some(ComponentKind::Source), - not_equals: None, - }]), - ..Default::default() - }; - - let components = filter_items(component_fixtures().into_iter(), &filter); - - assert_eq!(components.len(), 1); - } - - #[test] - fn components_filter_and_or() { - let filter = ComponentsFilter { - component_id: Some(vec![filter::StringFilter { - equals: Some("gen1".to_string()), - ..Default::default() - }]), - component_kind: Some(vec![filter::EqualityFilter { - equals: Some(ComponentKind::Source), - not_equals: None, - }]), - or: Some(vec![ComponentsFilter { - component_kind: Some(vec![filter::EqualityFilter { - equals: Some(ComponentKind::Sink), - not_equals: None, - }]), - ..Default::default() - }]), - }; - - let components = filter_items(component_fixtures().into_iter(), &filter); - - assert_eq!(components.len(), 2); - } - - #[test] - fn components_sort_asc() { - let mut components = component_fixtures(); - let fields = vec![sort::SortField:: { - field: ComponentsSortFieldName::ComponentKey, - direction: sort::Direction::Asc, - }]; - sort::by_fields(&mut components, &fields); - - let expectations = ["devnull", "gen1", "gen2", "gen3", "parse_json"]; - - for (i, component_id) in expectations.iter().enumerate() { - assert_eq!(components[i].get_component_key().id(), *component_id); - } - } - - #[test] - fn components_sort_desc() { - let mut components = component_fixtures(); - let fields = vec![sort::SortField:: { - field: ComponentsSortFieldName::ComponentKey, - direction: sort::Direction::Desc, - }]; - sort::by_fields(&mut components, &fields); - - let expectations = ["parse_json", "gen3", "gen2", "gen1", "devnull"]; - - for (i, component_id) in expectations.iter().enumerate() { - assert_eq!(components[i].get_component_key().id(), *component_id); - } - } - - #[test] - fn components_sort_multi() { - let mut components = vec![ - Component::Sink(sink::Sink(sink::Data { - component_key: ComponentKey::from("a"), - component_type: "blackhole".to_string(), - inputs: vec![OutputId::from("gen3"), OutputId::from("parse_json")].into(), - })), - Component::Sink(sink::Sink(sink::Data { - component_key: ComponentKey::from("b"), - component_type: "blackhole".to_string(), - inputs: vec![OutputId::from("gen3"), OutputId::from("parse_json")].into(), - })), - Component::Transform(transform::Transform(transform::Data { - component_key: ComponentKey::from("c"), - component_type: "json".to_string(), - inputs: vec![OutputId::from("gen1"), OutputId::from("gen2")].into(), - outputs: vec![], - })), - Component::Source(source::Source(source::Data { - component_key: ComponentKey::from("e"), - component_type: "demo_logs".to_string(), - output_type: DataType::Metric, - outputs: vec![], - })), - Component::Source(source::Source(source::Data { - component_key: ComponentKey::from("d"), - component_type: "demo_logs".to_string(), - output_type: DataType::Metric, - outputs: vec![], - })), - Component::Source(source::Source(source::Data { - component_key: ComponentKey::from("g"), - component_type: "demo_logs".to_string(), - output_type: DataType::Metric, - outputs: vec![], - })), - Component::Source(source::Source(source::Data { - component_key: ComponentKey::from("f"), - component_type: "demo_logs".to_string(), - output_type: DataType::Metric, - outputs: vec![], - })), - ]; - - let fields = vec![ - sort::SortField:: { - field: ComponentsSortFieldName::ComponentKind, - direction: sort::Direction::Asc, - }, - sort::SortField:: { - field: ComponentsSortFieldName::ComponentKey, - direction: sort::Direction::Asc, - }, - ]; - sort::by_fields(&mut components, &fields); - - let expectations = ["d", "e", "f", "g", "c", "a", "b"]; - for (i, component_id) in expectations.iter().enumerate() { - assert_eq!(components[i].get_component_key().id(), *component_id); - } - } -} diff --git a/src/api/schema/components/sink.rs b/src/api/schema/components/sink.rs deleted file mode 100644 index 99bcca38cc634..0000000000000 --- a/src/api/schema/components/sink.rs +++ /dev/null @@ -1,202 +0,0 @@ -use std::cmp; - -use async_graphql::{Enum, InputObject, Object}; - -use super::{Component, source, state, transform}; -use crate::{ - api::schema::{ - filter, - metrics::{self, IntoSinkMetrics}, - sort, - }, - config::{ComponentKey, Inputs, OutputId}, - filter_check, -}; - -#[derive(Debug, Clone)] -pub struct Data { - pub component_key: ComponentKey, - pub component_type: String, - pub inputs: Inputs, -} - -#[derive(Debug, Clone)] -pub struct Sink(pub Data); - -impl Sink { - pub const fn get_component_key(&self) -> &ComponentKey { - &self.0.component_key - } - - pub const fn get_component_type(&self) -> &str { - self.0.component_type.as_str() - } -} - -#[derive(Default, InputObject)] -pub struct SinksFilter { - component_id: Option>, - component_type: Option>, - or: Option>, -} - -impl filter::CustomFilter for SinksFilter { - fn matches(&self, sink: &Sink) -> bool { - filter_check!( - self.component_id.as_ref().map(|f| f - .iter() - .all(|f| f.filter_value(&sink.get_component_key().to_string()))), - self.component_type - .as_ref() - .map(|f| f.iter().all(|f| f.filter_value(sink.get_component_type()))) - ); - true - } - - fn or(&self) -> Option<&Vec> { - self.or.as_ref() - } -} - -#[derive(Enum, Copy, Clone, Eq, PartialEq)] -pub enum SinksSortFieldName { - ComponentKey, - ComponentType, -} - -impl sort::SortableByField for Sink { - fn sort(&self, rhs: &Self, field: &SinksSortFieldName) -> cmp::Ordering { - match field { - SinksSortFieldName::ComponentKey => { - Ord::cmp(self.get_component_key(), rhs.get_component_key()) - } - SinksSortFieldName::ComponentType => { - Ord::cmp(self.get_component_type(), rhs.get_component_type()) - } - } - } -} - -#[Object] -impl Sink { - /// Sink component_id - pub async fn component_id(&self) -> &str { - self.get_component_key().id() - } - - /// Sink type - pub async fn component_type(&self) -> &str { - self.get_component_type() - } - - /// Source inputs - pub async fn sources(&self) -> Vec { - self.0 - .inputs - .iter() - .filter_map(|output_id| match state::component_by_output_id(output_id) { - Some(Component::Source(s)) => Some(s), - _ => None, - }) - .collect() - } - - /// Transform inputs - pub async fn transforms(&self) -> Vec { - self.0 - .inputs - .iter() - .filter_map(|output_id| match state::component_by_output_id(output_id) { - Some(Component::Transform(t)) => Some(t), - _ => None, - }) - .collect() - } - - /// Sink metrics - pub async fn metrics(&self) -> metrics::SinkMetrics { - metrics::by_component_key(self.get_component_key()) - .into_sink_metrics(self.get_component_type()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn sink_fixtures() -> Vec { - vec![ - Sink(Data { - component_key: ComponentKey::from("webserver"), - component_type: "http".to_string(), - inputs: Inputs::default(), - }), - Sink(Data { - component_key: ComponentKey::from("db"), - component_type: "clickhouse".to_string(), - inputs: Inputs::default(), - }), - Sink(Data { - component_key: ComponentKey::from("zip_drive"), - component_type: "file".to_string(), - inputs: Inputs::default(), - }), - ] - } - - #[test] - fn sort_component_id_asc() { - let mut sinks = sink_fixtures(); - let fields = vec![sort::SortField:: { - field: SinksSortFieldName::ComponentKey, - direction: sort::Direction::Asc, - }]; - sort::by_fields(&mut sinks, &fields); - - for (i, component_id) in ["db", "webserver", "zip_drive"].iter().enumerate() { - assert_eq!(sinks[i].get_component_key().to_string(), *component_id); - } - } - - #[test] - fn sort_component_id_desc() { - let mut sinks = sink_fixtures(); - let fields = vec![sort::SortField:: { - field: SinksSortFieldName::ComponentKey, - direction: sort::Direction::Desc, - }]; - sort::by_fields(&mut sinks, &fields); - - for (i, component_id) in ["zip_drive", "webserver", "db"].iter().enumerate() { - assert_eq!(sinks[i].get_component_key().to_string(), *component_id); - } - } - - #[test] - fn sort_component_type_asc() { - let mut sinks = sink_fixtures(); - let fields = vec![sort::SortField:: { - field: SinksSortFieldName::ComponentType, - direction: sort::Direction::Asc, - }]; - sort::by_fields(&mut sinks, &fields); - - for (i, component_id) in ["db", "zip_drive", "webserver"].iter().enumerate() { - assert_eq!(sinks[i].get_component_key().to_string(), *component_id); - } - } - - #[test] - fn sort_component_type_desc() { - let mut sinks = sink_fixtures(); - let fields = vec![sort::SortField:: { - field: SinksSortFieldName::ComponentType, - direction: sort::Direction::Desc, - }]; - sort::by_fields(&mut sinks, &fields); - - for (i, component_id) in ["webserver", "zip_drive", "db"].iter().enumerate() { - assert_eq!(sinks[i].get_component_key().to_string(), *component_id); - } - } -} diff --git a/src/api/schema/components/source.rs b/src/api/schema/components/source.rs deleted file mode 100644 index c2289c4b6a15b..0000000000000 --- a/src/api/schema/components/source.rs +++ /dev/null @@ -1,403 +0,0 @@ -use std::cmp; - -use async_graphql::{Enum, InputObject, Object}; - -use super::{Component, sink, state, transform}; -use crate::{ - api::schema::{ - filter, - metrics::{self, IntoSourceMetrics, Output, outputs_by_component_key}, - sort, - }, - config::{ComponentKey, DataType, OutputId}, - filter_check, -}; - -#[derive(Debug, Enum, Eq, PartialEq, Copy, Clone, Ord, PartialOrd)] -pub enum SourceOutputType { - Log, - Metric, - Trace, -} - -#[derive(Debug, Clone)] -pub struct Data { - pub component_key: ComponentKey, - pub component_type: String, - pub output_type: DataType, - pub outputs: Vec, -} - -#[derive(Enum, Copy, Clone, Eq, PartialEq)] -pub enum SourcesSortFieldName { - ComponentKey, - ComponentType, - OutputType, -} - -#[derive(Debug, Clone)] -pub struct Source(pub Data); - -impl Source { - #[allow(clippy::missing_const_for_fn)] // const cannot run destructor - pub fn get_component_key(&self) -> &ComponentKey { - &self.0.component_key - } - pub const fn get_component_type(&self) -> &str { - self.0.component_type.as_str() - } - pub fn get_output_types(&self) -> Vec { - [ - SourceOutputType::Log, - SourceOutputType::Metric, - SourceOutputType::Trace, - ] - .iter() - .copied() - .filter(|s| self.0.output_type.contains(s.into())) - .collect() - } - - pub fn get_outputs(&self) -> &[String] { - self.0.outputs.as_ref() - } -} - -impl From<&SourceOutputType> for DataType { - fn from(s: &SourceOutputType) -> Self { - match s { - SourceOutputType::Log => DataType::Log, - SourceOutputType::Metric => DataType::Metric, - SourceOutputType::Trace => DataType::Trace, - } - } -} - -impl sort::SortableByField for Source { - fn sort(&self, rhs: &Self, field: &SourcesSortFieldName) -> cmp::Ordering { - match field { - SourcesSortFieldName::ComponentKey => { - Ord::cmp(self.get_component_key(), rhs.get_component_key()) - } - SourcesSortFieldName::ComponentType => { - Ord::cmp(self.get_component_type(), rhs.get_component_type()) - } - SourcesSortFieldName::OutputType => { - Ord::cmp(&u8::from(self.0.output_type), &u8::from(rhs.0.output_type)) - } - } - } -} - -#[Object] -impl Source { - /// Source component_id - pub async fn component_id(&self) -> &str { - self.0.component_key.id() - } - - /// Source type - pub async fn component_type(&self) -> &str { - self.get_component_type() - } - - /// Source output type - pub async fn output_types(&self) -> Vec { - self.get_output_types() - } - - /// Source output streams - pub async fn outputs(&self) -> Vec { - outputs_by_component_key(self.get_component_key(), self.get_outputs()) - } - - /// Transform outputs - pub async fn transforms(&self) -> Vec { - state::filter_components(|(_component_key, components)| match components { - Component::Transform(t) - if t.0.inputs.contains(&OutputId::from(&self.0.component_key)) => - { - Some(t.clone()) - } - _ => None, - }) - } - - /// Sink outputs - pub async fn sinks(&self) -> Vec { - state::filter_components(|(_component_key, components)| match components { - Component::Sink(s) if s.0.inputs.contains(&OutputId::from(&self.0.component_key)) => { - Some(s.clone()) - } - _ => None, - }) - } - - /// Source metrics - pub async fn metrics(&self) -> metrics::SourceMetrics { - metrics::by_component_key(&self.0.component_key) - .into_source_metrics(self.get_component_type()) - } -} - -#[derive(Default, InputObject)] -pub(super) struct SourcesFilter { - component_id: Option>, - component_type: Option>, - output_type: Option>>, - or: Option>, -} - -impl filter::CustomFilter for SourcesFilter { - fn matches(&self, source: &Source) -> bool { - filter_check!( - self.component_id.as_ref().map(|f| f - .iter() - .all(|f| f.filter_value(&source.get_component_key().to_string()))), - self.component_type.as_ref().map(|f| f - .iter() - .all(|f| f.filter_value(source.get_component_type()))), - self.output_type - .as_ref() - .map(|f| f.iter().all(|f| f.filter_value(source.get_output_types()))) - ); - true - } - - fn or(&self) -> Option<&Vec> { - self.or.as_ref() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::config::DataType; - - /// Generate component fixes for use with tests - fn source_fixtures() -> Vec { - vec![ - Source(Data { - component_key: ComponentKey::from("gen1"), - component_type: "demo_logs".to_string(), - output_type: DataType::Log | DataType::Metric, - outputs: vec![], - }), - Source(Data { - component_key: ComponentKey::from("gen2"), - component_type: "demo_logs".to_string(), - output_type: DataType::Log, - outputs: vec![], - }), - Source(Data { - component_key: ComponentKey::from("gen3"), - component_type: "demo_logs".to_string(), - output_type: DataType::Metric, - outputs: vec![], - }), - ] - } - - #[test] - fn filter_output_type() { - struct Test { - component_id: &'static str, - output_types: Vec, - } - - let tests = vec![ - Test { - component_id: "gen1", - output_types: vec![SourceOutputType::Log, SourceOutputType::Metric], - }, - Test { - component_id: "gen2", - output_types: vec![SourceOutputType::Log], - }, - Test { - component_id: "gen3", - output_types: vec![SourceOutputType::Metric], - }, - ]; - - for t in tests { - let filter = SourcesFilter { - component_id: Some(vec![filter::StringFilter { - equals: Some(t.component_id.to_string()), - ..Default::default() - }]), - output_type: Some(vec![filter::ListFilter:: { - equals: Some(t.output_types), - not_equals: None, - contains: None, - not_contains: None, - }]), - ..Default::default() - }; - - let sources = filter::filter_items(source_fixtures().into_iter(), &filter); - assert_eq!(sources.len(), 1); - } - } - - #[test] - fn sort_component_id_desc() { - let mut sources = source_fixtures(); - let fields = vec![sort::SortField:: { - field: SourcesSortFieldName::ComponentKey, - direction: sort::Direction::Desc, - }]; - sort::by_fields(&mut sources, &fields); - - for (i, component_id) in ["gen3", "gen2", "gen1"].iter().enumerate() { - assert_eq!(sources[i].get_component_key().to_string(), *component_id); - } - } - - #[test] - fn sort_component_type_asc() { - let mut sources = vec![ - Source(Data { - component_key: ComponentKey::from("gen2"), - component_type: "file".to_string(), - output_type: DataType::Log | DataType::Metric, - outputs: vec![], - }), - Source(Data { - component_key: ComponentKey::from("gen3"), - component_type: "demo_logs".to_string(), - output_type: DataType::Log, - outputs: vec![], - }), - Source(Data { - component_key: ComponentKey::from("gen1"), - component_type: "docker_logs".to_string(), - output_type: DataType::Metric, - outputs: vec![], - }), - ]; - - let fields = vec![sort::SortField:: { - field: SourcesSortFieldName::ComponentType, - direction: sort::Direction::Asc, - }]; - sort::by_fields(&mut sources, &fields); - - for (i, component_id) in ["gen3", "gen1", "gen2"].iter().enumerate() { - assert_eq!(sources[i].get_component_key().to_string(), *component_id); - } - } - - #[test] - fn sort_component_type_desc() { - let mut sources = vec![ - Source(Data { - component_key: ComponentKey::from("gen3"), - component_type: "file".to_string(), - output_type: DataType::Log | DataType::Metric, - outputs: vec![], - }), - Source(Data { - component_key: ComponentKey::from("gen2"), - component_type: "demo_logs".to_string(), - output_type: DataType::Log, - outputs: vec![], - }), - Source(Data { - component_key: ComponentKey::from("gen1"), - component_type: "docker_logs".to_string(), - output_type: DataType::Metric, - outputs: vec![], - }), - ]; - - let fields = vec![sort::SortField:: { - field: SourcesSortFieldName::ComponentType, - direction: sort::Direction::Desc, - }]; - sort::by_fields(&mut sources, &fields); - - for (i, component_id) in ["gen3", "gen1", "gen2"].iter().enumerate() { - assert_eq!(sources[i].get_component_key().to_string(), *component_id); - } - } - - #[test] - fn sort_output_type_asc() { - let mut sources = vec![ - Source(Data { - component_key: ComponentKey::from("gen4"), - component_type: "demo_trace".to_string(), - output_type: DataType::Trace, - outputs: vec![], - }), - Source(Data { - component_key: ComponentKey::from("gen1"), - component_type: "demo_logs".to_string(), - output_type: DataType::Metric, - outputs: vec![], - }), - Source(Data { - component_key: ComponentKey::from("gen2"), - component_type: "file".to_string(), - output_type: DataType::Log, - outputs: vec![], - }), - Source(Data { - component_key: ComponentKey::from("gen3"), - component_type: "multiple_type".to_string(), - output_type: DataType::Log | DataType::Metric | DataType::Trace, - outputs: vec![], - }), - ]; - - let fields = vec![sort::SortField:: { - field: SourcesSortFieldName::OutputType, - direction: sort::Direction::Asc, - }]; - sort::by_fields(&mut sources, &fields); - - for (i, component_id) in ["gen2", "gen1", "gen4", "gen3"].iter().enumerate() { - assert_eq!(sources[i].get_component_key().to_string(), *component_id); - } - } - - #[test] - fn sort_output_type_desc() { - let mut sources = vec![ - Source(Data { - component_key: ComponentKey::from("gen4"), - component_type: "demo_trace".to_string(), - output_type: DataType::Trace, - outputs: vec![], - }), - Source(Data { - component_key: ComponentKey::from("gen1"), - component_type: "demo_logs".to_string(), - output_type: DataType::Metric, - outputs: vec![], - }), - Source(Data { - component_key: ComponentKey::from("gen2"), - component_type: "file".to_string(), - output_type: DataType::Log, - outputs: vec![], - }), - Source(Data { - component_key: ComponentKey::from("gen3"), - component_type: "multiple_type".to_string(), - output_type: DataType::Log | DataType::Metric | DataType::Trace, - outputs: vec![], - }), - ]; - - let fields = vec![sort::SortField:: { - field: SourcesSortFieldName::OutputType, - direction: sort::Direction::Desc, - }]; - sort::by_fields(&mut sources, &fields); - for (i, component_id) in ["gen3", "gen4", "gen1", "gen2"].iter().enumerate() { - assert_eq!(sources[i].get_component_key().to_string(), *component_id); - } - } -} diff --git a/src/api/schema/components/state.rs b/src/api/schema/components/state.rs deleted file mode 100644 index 1bb04894d81a9..0000000000000 --- a/src/api/schema/components/state.rs +++ /dev/null @@ -1,89 +0,0 @@ -use std::{ - collections::{HashMap, HashSet}, - sync::{Arc, LazyLock, RwLock}, -}; - -use super::{Component, sink, source, transform}; -use crate::config::{ComponentKey, OutputId}; - -pub const INVARIANT: &str = "Couldn't acquire lock on Vector components. Please report this."; - -pub static COMPONENTS: LazyLock>>> = - LazyLock::new(|| Arc::new(RwLock::new(HashMap::new()))); - -/// Filter components with the provided `map_func` -pub fn filter_components(map_func: impl Fn((&ComponentKey, &Component)) -> Option) -> Vec { - COMPONENTS - .read() - .expect(INVARIANT) - .iter() - .filter_map(map_func) - .collect() -} - -/// Returns all components -pub fn get_components() -> Vec { - filter_components(|(_component_key, components)| Some(components.clone())) -} - -/// Filters components, and returns a clone of sources -pub fn get_sources() -> Vec { - filter_components(|(_, components)| match components { - Component::Source(s) => Some(s.clone()), - _ => None, - }) -} - -/// Filters components, and returns a clone of transforms -pub fn get_transforms() -> Vec { - filter_components(|(_, components)| match components { - Component::Transform(t) => Some(t.clone()), - _ => None, - }) -} - -/// Filters components, and returns a clone of sinks -pub fn get_sinks() -> Vec { - filter_components(|(_, components)| match components { - Component::Sink(s) => Some(s.clone()), - _ => None, - }) -} - -/// Returns the current component component_keys as a HashSet -pub fn get_component_keys() -> HashSet { - COMPONENTS - .read() - .expect(INVARIANT) - .keys() - .cloned() - .collect::>() -} - -/// Gets a component by component_key -pub fn component_by_component_key(component_key: &ComponentKey) -> Option { - Some( - COMPONENTS - .read() - .expect(INVARIANT) - .get(component_key)? - .clone(), - ) -} - -/// Gets a component by output_id -pub fn component_by_output_id(output_id: &OutputId) -> Option { - filter_components(|(key, component)| { - if key == &output_id.component { - Some(component.clone()) - } else { - None - } - }) - .pop() -} - -/// Overwrites component state with new components. -pub fn update(new_components: HashMap) { - *COMPONENTS.write().expect(INVARIANT) = new_components -} diff --git a/src/api/schema/components/transform.rs b/src/api/schema/components/transform.rs deleted file mode 100644 index 7fe95238b7dda..0000000000000 --- a/src/api/schema/components/transform.rs +++ /dev/null @@ -1,223 +0,0 @@ -use std::cmp; - -use async_graphql::{Enum, InputObject, Object}; - -use super::{Component, sink, source, state}; -use crate::{ - api::schema::{ - filter, - metrics::{self, IntoTransformMetrics, Output, outputs_by_component_key}, - sort, - }, - config::{ComponentKey, Inputs, OutputId}, - filter_check, -}; - -#[derive(Debug, Clone)] -pub struct Data { - pub component_key: ComponentKey, - pub component_type: String, - pub inputs: Inputs, - pub outputs: Vec, -} - -#[derive(Debug, Clone)] -pub struct Transform(pub Data); - -impl Transform { - pub const fn get_component_key(&self) -> &ComponentKey { - &self.0.component_key - } - pub const fn get_component_type(&self) -> &str { - self.0.component_type.as_str() - } - pub fn get_outputs(&self) -> &[String] { - self.0.outputs.as_ref() - } -} - -#[derive(Enum, Copy, Clone, Eq, PartialEq)] -pub enum TransformsSortFieldName { - ComponentKey, - ComponentType, -} - -impl sort::SortableByField for Transform { - fn sort(&self, rhs: &Self, field: &TransformsSortFieldName) -> cmp::Ordering { - match field { - TransformsSortFieldName::ComponentKey => { - Ord::cmp(self.get_component_key(), rhs.get_component_key()) - } - TransformsSortFieldName::ComponentType => { - Ord::cmp(self.get_component_type(), rhs.get_component_type()) - } - } - } -} - -#[Object] -impl Transform { - /// Transform component_id - pub async fn component_id(&self) -> &str { - self.0.component_key.id() - } - - /// Transform type - pub async fn component_type(&self) -> &str { - self.get_component_type() - } - - /// Transform output streams - pub async fn outputs(&self) -> Vec { - outputs_by_component_key(self.get_component_key(), self.get_outputs()) - } - - /// Source inputs - pub async fn sources(&self) -> Vec { - self.0 - .inputs - .iter() - .filter_map(|output_id| match state::component_by_output_id(output_id) { - Some(Component::Source(s)) => Some(s), - _ => None, - }) - .collect() - } - - /// Transform outputs - pub async fn transforms(&self) -> Vec { - state::filter_components(|(_component_key, components)| match components { - Component::Transform(t) - if t.0.inputs.contains(&OutputId::from(&self.0.component_key)) => - { - Some(t.clone()) - } - _ => None, - }) - } - - /// Sink outputs - pub async fn sinks(&self) -> Vec { - state::filter_components(|(_component_key, components)| match components { - Component::Sink(s) if s.0.inputs.contains(&OutputId::from(&self.0.component_key)) => { - Some(s.clone()) - } - _ => None, - }) - } - - /// Transform metrics - pub async fn metrics(&self) -> metrics::TransformMetrics { - metrics::by_component_key(&self.0.component_key) - .into_transform_metrics(self.get_component_type()) - } -} - -#[derive(Default, InputObject)] -pub struct TransformsFilter { - component_id: Option>, - component_type: Option>, - or: Option>, -} - -impl filter::CustomFilter for TransformsFilter { - fn matches(&self, transform: &Transform) -> bool { - filter_check!( - self.component_id.as_ref().map(|f| f - .iter() - .all(|f| f.filter_value(&transform.get_component_key().to_string()))), - self.component_type.as_ref().map(|f| f - .iter() - .all(|f| f.filter_value(transform.get_component_type()))) - ); - true - } - - fn or(&self) -> Option<&Vec> { - self.or.as_ref() - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn transform_fixtures() -> Vec { - vec![ - Transform(Data { - component_key: ComponentKey::from("parse_json"), - component_type: "json".to_string(), - inputs: Inputs::default(), - outputs: vec![], - }), - Transform(Data { - component_key: ComponentKey::from("field_adder"), - component_type: "add_fields".to_string(), - inputs: Inputs::default(), - outputs: vec![], - }), - Transform(Data { - component_key: ComponentKey::from("append"), - component_type: "concat".to_string(), - inputs: Inputs::default(), - outputs: vec![], - }), - ] - } - - #[test] - fn sort_component_id_asc() { - let mut transforms = transform_fixtures(); - let fields = vec![sort::SortField:: { - field: TransformsSortFieldName::ComponentKey, - direction: sort::Direction::Asc, - }]; - sort::by_fields(&mut transforms, &fields); - - for (i, component_id) in ["append", "field_adder", "parse_json"].iter().enumerate() { - assert_eq!(transforms[i].get_component_key().to_string(), *component_id); - } - } - - #[test] - fn sort_component_id_desc() { - let mut transforms = transform_fixtures(); - let fields = vec![sort::SortField:: { - field: TransformsSortFieldName::ComponentKey, - direction: sort::Direction::Desc, - }]; - sort::by_fields(&mut transforms, &fields); - - for (i, component_id) in ["parse_json", "field_adder", "append"].iter().enumerate() { - assert_eq!(transforms[i].get_component_key().to_string(), *component_id); - } - } - - #[test] - fn sort_component_type_asc() { - let mut transforms = transform_fixtures(); - let fields = vec![sort::SortField:: { - field: TransformsSortFieldName::ComponentType, - direction: sort::Direction::Asc, - }]; - sort::by_fields(&mut transforms, &fields); - - for (i, component_id) in ["field_adder", "append", "parse_json"].iter().enumerate() { - assert_eq!(transforms[i].get_component_key().to_string(), *component_id); - } - } - - #[test] - fn sort_component_type_desc() { - let mut transforms = transform_fixtures(); - let fields = vec![sort::SortField:: { - field: TransformsSortFieldName::ComponentType, - direction: sort::Direction::Desc, - }]; - sort::by_fields(&mut transforms, &fields); - - for (i, component_id) in ["parse_json", "append", "field_adder"].iter().enumerate() { - assert_eq!(transforms[i].get_component_key().to_string(), *component_id); - } - } -} diff --git a/src/api/schema/events/encoding.rs b/src/api/schema/events/encoding.rs deleted file mode 100644 index 4b455b5e2a2fc..0000000000000 --- a/src/api/schema/events/encoding.rs +++ /dev/null @@ -1,9 +0,0 @@ -use async_graphql::Enum; - -#[derive(Enum, Copy, Clone, PartialEq, Eq)] -/// Encoding format for the event -pub enum EventEncodingType { - Json, - Yaml, - Logfmt, -} diff --git a/src/api/schema/events/log.rs b/src/api/schema/events/log.rs deleted file mode 100644 index a06a3ff5f5eab..0000000000000 --- a/src/api/schema/events/log.rs +++ /dev/null @@ -1,77 +0,0 @@ -use std::borrow::Cow; - -use async_graphql::Object; -use chrono::{DateTime, Utc}; -use vector_lib::{encode_logfmt, event, tap::topology::TapOutput}; -use vrl::event_path; - -use super::EventEncodingType; - -#[derive(Debug, Clone)] -pub struct Log { - output: TapOutput, - event: event::LogEvent, -} - -impl Log { - pub const fn new(output: TapOutput, event: event::LogEvent) -> Self { - Self { output, event } - } - - pub fn get_message(&self) -> Option> { - Some(self.event.get(event_path!("message"))?.to_string_lossy()) - } - - pub fn get_timestamp(&self) -> Option<&DateTime> { - self.event.get(event_path!("timestamp"))?.as_timestamp() - } -} - -#[Object] -/// Log event with fields for querying log data -impl Log { - /// Id of the component associated with the log event - async fn component_id(&self) -> &str { - self.output.output_id.component.id() - } - - /// Type of component associated with the log event - async fn component_type(&self) -> &str { - self.output.component_type.as_ref() - } - - /// Kind of component associated with the log event - async fn component_kind(&self) -> &str { - self.output.component_kind - } - - /// Log message - async fn message(&self) -> Option { - self.get_message().map(Into::into) - } - - /// Log timestamp - async fn timestamp(&self) -> Option<&DateTime> { - self.get_timestamp() - } - - /// Log event as an encoded string format - async fn string(&self, encoding: EventEncodingType) -> String { - match encoding { - EventEncodingType::Json => serde_json::to_string(&self.event) - .expect("JSON serialization of log event failed. Please report."), - EventEncodingType::Yaml => serde_yaml::to_string(&self.event) - .expect("YAML serialization of log event failed. Please report."), - EventEncodingType::Logfmt => encode_logfmt::encode_value(self.event.value()) - .expect("logfmt serialization of log event failed. Please report."), - } - } - - /// Get JSON field data on the log event, by field name - async fn json(&self, field: String) -> Option { - self.event.get(event_path!(field.as_str())).map(|field| { - serde_json::to_string(field) - .expect("JSON serialization of trace event field failed. Please report.") - }) - } -} diff --git a/src/api/schema/events/metric.rs b/src/api/schema/events/metric.rs deleted file mode 100644 index b9ae2a31a3d27..0000000000000 --- a/src/api/schema/events/metric.rs +++ /dev/null @@ -1,139 +0,0 @@ -use async_graphql::{Enum, Object}; -use chrono::{DateTime, Utc}; -use serde_json::Value; -use vector_lib::{encode_logfmt, event, tap::topology::TapOutput}; - -use super::EventEncodingType; - -#[derive(Debug, Clone)] -pub struct Metric { - output: TapOutput, - event: event::Metric, -} - -impl Metric { - pub const fn new(output: TapOutput, event: event::Metric) -> Self { - Self { output, event } - } -} - -#[derive(Copy, Clone, Debug, PartialEq, Eq, Enum)] -enum MetricKind { - /// Incremental metrics update previous values - Incremental, - /// Absolute metrics set the reference value for future updates - Absolute, -} - -impl From for MetricKind { - fn from(kind: event::MetricKind) -> Self { - match kind { - event::MetricKind::Incremental => Self::Incremental, - event::MetricKind::Absolute => Self::Absolute, - } - } -} - -struct MetricTag { - key: String, - value: String, -} - -#[Object] -impl MetricTag { - /// Metric tag key - async fn key(&self) -> &str { - self.key.as_ref() - } - - /// Metric tag value - async fn value(&self) -> &str { - self.value.as_ref() - } -} - -#[Object] -/// Metric event with fields for querying metric data -impl Metric { - /// Id of the component associated with the metric event - async fn component_id(&self) -> &str { - self.output.output_id.component.id() - } - - /// Type of component associated with the metric event - async fn component_type(&self) -> &str { - self.output.component_type.as_ref() - } - - /// Kind of component associated with the metric event - async fn component_kind(&self) -> &str { - self.output.component_kind - } - - /// Metric timestamp - async fn timestamp(&self) -> Option<&DateTime> { - self.event.data().timestamp() - } - - /// Metric name - async fn name(&self) -> &str { - self.event.name() - } - - /// Metric namespace - async fn namespace(&self) -> Option<&str> { - self.event.namespace() - } - - /// Metric kind - async fn kind(&self) -> MetricKind { - self.event.kind().into() - } - - /// Metric type - async fn value_type(&self) -> &str { - self.event.value().as_name() - } - - /// Metric value in human readable form - async fn value(&self) -> String { - self.event.value().to_string() - } - - /// Metric tags - async fn tags(&self) -> Option> { - self.event.tags().map(|tags| { - tags.iter_single() - .map(|(key, value)| MetricTag { - key: key.to_owned(), - value: value.to_owned(), - }) - .collect() - }) - } - - /// Metric event as an encoded string format - async fn string(&self, encoding: EventEncodingType) -> String { - match encoding { - EventEncodingType::Json => serde_json::to_string(&self.event) - .expect("JSON serialization of metric event failed. Please report."), - EventEncodingType::Yaml => serde_yaml::to_string(&self.event) - .expect("YAML serialization of metric event failed. Please report."), - EventEncodingType::Logfmt => { - let json = serde_json::to_value(&self.event) - .expect("logfmt serialization of metric event failed: conversion to serde Value failed. Please report."); - match json { - Value::Object(map) => encode_logfmt::encode_map( - &map.into_iter() - .map(|(k, v)| (event::KeyString::from(k), v)) - .collect(), - ) - .expect("logfmt serialization of metric event failed. Please report."), - _ => panic!( - "logfmt serialization of metric event failed: metric converted to unexpected serde Value. Please report." - ), - } - } - } - } -} diff --git a/src/api/schema/events/mod.rs b/src/api/schema/events/mod.rs deleted file mode 100644 index 9bebbdc194351..0000000000000 --- a/src/api/schema/events/mod.rs +++ /dev/null @@ -1,157 +0,0 @@ -pub mod encoding; -pub mod log; -pub mod metric; -pub mod output; -pub mod trace; - -use std::time::{SystemTime, UNIX_EPOCH}; - -use async_graphql::{Context, Subscription}; -use encoding::EventEncodingType; -use futures::{Stream, StreamExt, stream}; -use output::{OutputEventsPayload, from_tap_payload_to_output_events}; -use rand::{Rng, SeedableRng, rngs::SmallRng}; -use tokio::{select, sync::mpsc, time}; -use tokio_stream::wrappers::ReceiverStream; -use vector_lib::tap::{ - controller::{TapController, TapPatterns}, - topology::WatchRx, -}; - -#[derive(Debug, Default)] -pub struct EventsSubscription; - -#[Subscription] -impl EventsSubscription { - /// A stream of events emitted from matched component ID patterns - pub async fn output_events_by_component_id_patterns<'a>( - &'a self, - ctx: &'a Context<'a>, - outputs_patterns: Vec, - inputs_patterns: Option>, - #[graphql(default = 500)] interval: u32, - #[graphql(default = 100, validator(minimum = 1, maximum = 10_000))] limit: u32, - ) -> impl Stream> + 'a { - let watch_rx = ctx.data_unchecked::().clone(); - - let patterns = TapPatterns { - for_outputs: outputs_patterns.into_iter().collect(), - for_inputs: inputs_patterns.unwrap_or_default().into_iter().collect(), - }; - // Client input is confined to `u32` to provide sensible bounds. - create_events_stream(watch_rx, patterns, interval as u64, limit as usize) - } -} - -/// Creates an events stream based on component ids, and a provided interval. Will emit -/// control messages that bubble up the application if the sink goes away. The stream contains -/// all matching events; filtering should be done at the caller level. -pub(crate) fn create_events_stream( - watch_rx: WatchRx, - patterns: TapPatterns, - interval: u64, - limit: usize, -) -> impl Stream> { - // Channel for receiving individual tap payloads. Since we can process at most `limit` per - // interval, this is capped to the same value. - let (tap_tx, tap_rx) = mpsc::channel(limit); - let mut tap_rx = ReceiverStream::new(tap_rx) - .flat_map(|payload| stream::iter(from_tap_payload_to_output_events(payload))); - - // The resulting vector of `Event` sent to the client. Only one result set will be streamed - // back to the client at a time. This value is set higher than `1` to prevent blocking the event - // pipeline on slower client connections, but low enough to apply a modest cap on mem usage. - let (event_tx, event_rx) = mpsc::channel::>(10); - - tokio::spawn(async move { - // Create a tap controller. When this drops out of scope, clean up will be performed on the - // event handlers and topology observation that the tap controller provides. - let _tap_controller = TapController::new(watch_rx, tap_tx, patterns); - - // A tick interval to represent when to 'cut' the results back to the client. - let mut interval = time::interval(time::Duration::from_millis(interval)); - - // Temporary structure to hold sortable values of `Event`. - struct SortableOutputEventsPayload { - batch: usize, - payload: OutputEventsPayload, - } - - // Collect a vector of results, with a capacity of `limit`. As new `Event`s come in, - // they will be sampled and added to results. - let mut results = Vec::::with_capacity(limit); - - // Random number generator to allow for sampling. Speed trumps cryptographic security here. - // The RNG must be Send + Sync to use with the `select!` loop below, hence `SmallRng`. - let seed = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_nanos() as u64; - let mut rng = SmallRng::seed_from_u64(seed); - - // Keep a count of the batch size, which will be used as a seed for random eviction - // per the sampling strategy used below. - let mut batch = 0; - - loop { - select! { - // Process `TapPayload`s. A tap payload could contain log/metric events or a - // notification. Notifications are emitted immediately; events buffer until - // the next `interval`. - Some(payload) = tap_rx.next() => { - // Emit notifications immediately; these don't count as a 'batch'. - if let OutputEventsPayload::Notification(_) = payload { - // If an error occurs when sending, the subscription has likely gone - // away. Break the loop to terminate the thread. - if let Err(err) = event_tx.send(vec![payload]).await { - debug!(message = "Couldn't send notification.", error = ?err); - break; - } - } else { - // Wrap tap in a 'sortable' wrapper, using the batch as a key, to - // re-sort after random eviction. - let payload = SortableOutputEventsPayload { batch, payload }; - - // A simple implementation of "Algorithm R" per - // https://en.wikipedia.org/wiki/Reservoir_sampling. As we're unable to - // pluck the nth result, this is chosen over the more optimal "Algorithm L" - // since discarding results isn't an option. - if limit > results.len() { - results.push(payload); - } else { - let random_number = rng.random_range(0..batch); - if random_number < results.len() { - results[random_number] = payload; - } - } - // Increment the batch count, to be used for the next Algo R loop. - batch += 1; - } - } - _ = interval.tick() => { - // If there are any existing results after the interval tick, emit. - if !results.is_empty() { - // Reset the batch count, to adjust sampling probability for the next round. - batch = 0; - - // Since events will appear out of order per the random sampling - // strategy, drain the existing results and sort by timestamp. - results.sort_by_key(|r| r.batch); - let results = results.drain(..) - .map(|r| r.payload) - .collect(); - - // If we get an error here, it likely means that the subscription has - // gone has away. This is a valid/common situation. - if let Err(err) = event_tx.send(results).await { - debug!(message = "Couldn't send events.", error = ?err); - break; - } - } - } - } - } - }); - - ReceiverStream::new(event_rx) -} diff --git a/src/api/schema/events/output.rs b/src/api/schema/events/output.rs deleted file mode 100644 index de54bfefaad39..0000000000000 --- a/src/api/schema/events/output.rs +++ /dev/null @@ -1,70 +0,0 @@ -use async_graphql::{Object, Union}; -use vector_lib::tap::{controller::TapPayload, notification::Notification}; - -use crate::api::schema::events::{log::Log, metric::Metric, trace::Trace}; - -/// This wrapper struct hoists `message` up from [`Notification`] for a more -/// natural querying experience. While ideally [`Notification`] would be a -/// GraphQL interface with a common `message` field, an interface cannot be -/// directly nested into the union of [`super::OutputEventsPayload`]. -/// -/// The GraphQL specification forbids such a nesting: -/// -#[derive(Debug, Clone)] -pub struct EventNotification { - pub notification: Notification, -} - -#[Object] -/// A notification regarding events observation -impl EventNotification { - /// Notification details - async fn notification(&self) -> &Notification { - &self.notification - } - - /// The human-readable message associated with the notification - async fn message(&self) -> &str { - self.notification.as_str() - } -} - -#[derive(Union, Debug, Clone)] -#[allow(clippy::large_enum_variant)] -/// An event or a notification -pub enum OutputEventsPayload { - /// Log event - Log(Log), - - /// Metric event - Metric(Metric), - - // Notification - Notification(EventNotification), - - /// Trace event - Trace(Trace), -} - -/// Convert an `api::TapPayload` to the equivalent GraphQL type. -pub(crate) fn from_tap_payload_to_output_events(t: TapPayload) -> Vec { - match t { - TapPayload::Log(output, log_array) => log_array - .into_iter() - .map(|log| OutputEventsPayload::Log(Log::new(output.clone(), log))) - .collect(), - TapPayload::Metric(output, metric_array) => metric_array - .into_iter() - .map(|metric| OutputEventsPayload::Metric(Metric::new(output.clone(), metric))) - .collect(), - TapPayload::Notification(notification) => { - vec![OutputEventsPayload::Notification(EventNotification { - notification, - })] - } - TapPayload::Trace(output, trace_array) => trace_array - .into_iter() - .map(|trace| OutputEventsPayload::Trace(Trace::new(output.clone(), trace))) - .collect(), - } -} diff --git a/src/api/schema/events/trace.rs b/src/api/schema/events/trace.rs deleted file mode 100644 index 35b697b351a9e..0000000000000 --- a/src/api/schema/events/trace.rs +++ /dev/null @@ -1,56 +0,0 @@ -use async_graphql::Object; -use vector_lib::{encode_logfmt, event, tap::topology::TapOutput}; -use vrl::event_path; - -use super::EventEncodingType; - -#[derive(Debug, Clone)] -pub struct Trace { - output: TapOutput, - event: event::TraceEvent, -} - -impl Trace { - pub const fn new(output: TapOutput, event: event::TraceEvent) -> Self { - Self { output, event } - } -} - -#[Object] -/// Trace event with fields for querying trace data -impl Trace { - /// Id of the component associated with the trace event - async fn component_id(&self) -> &str { - self.output.output_id.component.id() - } - - /// Type of component associated with the trace event - async fn component_type(&self) -> &str { - self.output.component_type.as_ref() - } - - /// Kind of component associated with the trace event - async fn component_kind(&self) -> &str { - self.output.component_kind - } - - /// Trace event as an encoded string format - async fn string(&self, encoding: EventEncodingType) -> String { - match encoding { - EventEncodingType::Json => serde_json::to_string(&self.event) - .expect("JSON serialization of log event failed. Please report."), - EventEncodingType::Yaml => serde_yaml::to_string(&self.event) - .expect("YAML serialization of log event failed. Please report."), - EventEncodingType::Logfmt => encode_logfmt::encode_map(self.event.as_map()) - .expect("logfmt serialization of log event failed. Please report."), - } - } - - /// Get JSON field data on the trace event, by field name - async fn json(&self, field: String) -> Option { - self.event.get(event_path!(field.as_str())).map(|field| { - serde_json::to_string(field) - .expect("JSON serialization of log event field failed. Please report.") - }) - } -} diff --git a/src/api/schema/filter.rs b/src/api/schema/filter.rs deleted file mode 100644 index ac8f7f551d2ae..0000000000000 --- a/src/api/schema/filter.rs +++ /dev/null @@ -1,219 +0,0 @@ -use std::collections::BTreeSet; - -use async_graphql::{InputObject, InputType}; - -use super::components::{ComponentKind, source}; - -/// Takes an `&Option` and returns early if false -#[macro_export] -macro_rules! filter_check { - ($($match:expr_2021),+) => { - $( - if matches!($match, Some(t) if !t) { - return false; - } - )+ - } -} - -#[derive(Default, InputObject)] -/// Filter for String values -pub struct StringFilter { - pub equals: Option, - pub not_equals: Option, - pub contains: Option, - pub not_contains: Option, - pub starts_with: Option, - pub ends_with: Option, -} - -impl StringFilter { - pub fn filter_value(&self, value: &str) -> bool { - filter_check!( - // Equals - self.equals.as_ref().map(|s| value.eq(s)), - // Not equals - self.not_equals.as_ref().map(|s| !value.eq(s)), - // Contains - self.contains.as_ref().map(|s| value.contains(s)), - // Does not contain - self.not_contains.as_ref().map(|s| !value.contains(s)), - // Starts with - self.starts_with.as_ref().map(|s| value.starts_with(s)), - // Ends with - self.ends_with.as_ref().map(|s| value.ends_with(s)) - ); - true - } -} - -#[derive(InputObject)] -#[graphql(concrete(name = "SourceOutputTypeFilter", params(source::SourceOutputType)))] -// Filter for GraphQL lists -pub struct ListFilter { - pub equals: Option>, - pub not_equals: Option>, - pub contains: Option, - pub not_contains: Option, -} - -impl ListFilter { - pub fn filter_value(&self, value: Vec) -> bool { - let val = BTreeSet::from_iter(value.iter()); - filter_check!( - // Equals - self.equals - .as_ref() - .map(|s| BTreeSet::from_iter(s.iter()).eq(&val)), - // Not Equals - self.not_equals - .as_ref() - .map(|s| !BTreeSet::from_iter(s.iter()).eq(&val)), - // Contains - self.contains.as_ref().map(|s| val.contains(s)), - // Not Contains - self.not_contains.as_ref().map(|s| !val.contains(s)) - ); - true - } -} - -#[derive(InputObject)] -#[graphql(concrete(name = "ComponentKindFilter", params(ComponentKind)))] -pub struct EqualityFilter { - pub equals: Option, - pub not_equals: Option, -} - -impl EqualityFilter { - pub fn filter_value(&self, value: T) -> bool { - filter_check!( - // Equals - self.equals.as_ref().map(|s| value.eq(s)), - // Not equals - self.not_equals.as_ref().map(|s| !value.eq(s)) - ); - true - } -} - -/// CustomFilter trait to determine whether to include/exclude fields based on matches. -pub trait CustomFilter { - fn matches(&self, item: &T) -> bool; - fn or(&self) -> Option<&Vec> - where - Self: Sized; -} - -/// Returns true if a provided `Item` passes all 'AND' or 'OR' filter rules, recursively. -fn filter_item(item: &Item, f: &Filter) -> bool -where - Filter: CustomFilter, -{ - f.matches(item) - || f.or() - .map_or_else(|| false, |f| f.iter().any(|f| filter_item(item, f))) -} - -/// Filters items based on an implementation of `CustomFilter`. -pub fn filter_items(items: Iter, f: &Filter) -> Vec -where - Iter: Iterator, - Filter: CustomFilter, -{ - items.filter(|c| filter_item(c, f)).collect() -} - -#[cfg(test)] -mod test { - use super::StringFilter; - - #[test] - fn string_equals() { - let value = "test"; - - let sf = StringFilter { - equals: value.to_string().into(), - ..Default::default() - }; - - assert!(sf.filter_value(value)); - assert!(!sf.filter_value("not found")); - } - - #[test] - fn string_not_equals() { - let value = "value"; - let diff_value = "different value"; - - let sf = StringFilter { - not_equals: diff_value.to_string().into(), - ..Default::default() - }; - - assert!(sf.filter_value(value)); - assert!(!sf.filter_value(diff_value)); - } - - #[test] - fn string_contains() { - let sf = StringFilter { - contains: "234".to_string().into(), - ..Default::default() - }; - - assert!(sf.filter_value("12345")); - assert!(!sf.filter_value("xxx")); - } - - #[test] - fn string_not_contains() { - let contains = "xyz"; - - let sf = StringFilter { - not_contains: contains.to_string().into(), - ..Default::default() - }; - - assert!(sf.filter_value("abc")); - assert!(!sf.filter_value(contains)); - } - - #[test] - fn string_starts_with() { - let sf = StringFilter { - starts_with: "abc".to_string().into(), - ..Default::default() - }; - - assert!(sf.filter_value("abcdef")); - assert!(!sf.filter_value("xyz")); - } - - #[test] - fn string_ends_with() { - let sf = StringFilter { - ends_with: "456".to_string().into(), - ..Default::default() - }; - - assert!(sf.filter_value("123456")); - assert!(!sf.filter_value("123")); - } - - #[test] - fn string_multiple_all_match() { - let value = "123456"; - let sf = StringFilter { - equals: value.to_string().into(), - not_equals: "xyz".to_string().into(), - contains: "234".to_string().into(), - not_contains: "678".to_string().into(), - starts_with: "123".to_string().into(), - ends_with: "456".to_string().into(), - }; - - assert!(sf.filter_value(value)); - assert!(!sf.filter_value("should fail")); - } -} diff --git a/src/api/schema/gen.rs b/src/api/schema/gen.rs deleted file mode 100644 index c359f0db3b640..0000000000000 --- a/src/api/schema/gen.rs +++ /dev/null @@ -1,116 +0,0 @@ -use std::fs; - -use indoc::indoc; -use vector::api::build_schema; - -static INTROSPECTION_QUERY: &str = indoc! {r#" - query IntrospectionQuery { - __schema { - queryType { - name - } - mutationType { - name - } - subscriptionType { - name - } - types { - ...FullType - } - directives { - name - description - locations - args { - ...InputValue - } - } - } - } - fragment FullType on __Type { - kind - name - description - fields(includeDeprecated: true) { - name - description - args { - ...InputValue - } - type { - ...TypeRef - } - isDeprecated - deprecationReason - } - inputFields { - ...InputValue - } - interfaces { - ...TypeRef - } - enumValues(includeDeprecated: true) { - name - description - isDeprecated - deprecationReason - } - possibleTypes { - ...TypeRef - } - } - fragment InputValue on __InputValue { - name - description - type { - ...TypeRef - } - defaultValue - } - fragment TypeRef on __Type { - kind - name - ofType { - kind - name - ofType { - kind - name - ofType { - kind - name - ofType { - kind - name - ofType { - kind - name - ofType { - kind - name - ofType { - kind - name - } - } - } - } - } - } - } - } -"#}; - -#[tokio::main] -async fn main() { - let schema = build_schema().finish(); - let res = schema.execute(INTROSPECTION_QUERY).await; - let json = serde_json::to_string_pretty(&res).unwrap(); - - fs::write( - "lib/vector-api-client/graphql/schema.json", - format!("{json}\n"), - ) - .expect("Couldn't save schema file"); -} diff --git a/src/api/schema/health.rs b/src/api/schema/health.rs deleted file mode 100644 index 0ee3d1d897cb1..0000000000000 --- a/src/api/schema/health.rs +++ /dev/null @@ -1,43 +0,0 @@ -use async_graphql::{Object, SimpleObject, Subscription}; -use chrono::{DateTime, Utc}; -use tokio::time::Duration; -use tokio_stream::{Stream, StreamExt, wrappers::IntervalStream}; - -#[derive(SimpleObject)] -pub struct Heartbeat { - utc: DateTime, -} - -impl Heartbeat { - fn new() -> Self { - Heartbeat { utc: Utc::now() } - } -} - -#[derive(Default)] -pub(super) struct HealthQuery; - -#[Object] -impl HealthQuery { - /// Returns `true` to denote the GraphQL server is reachable - async fn health(&self) -> bool { - true - } -} - -#[derive(Default)] -pub struct HealthSubscription; - -#[Subscription] -impl HealthSubscription { - /// Heartbeat, containing the UTC timestamp of the last server-sent payload - async fn heartbeat( - &self, - #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream + use<> { - IntervalStream::new(tokio::time::interval(Duration::from_millis( - interval as u64, - ))) - .map(|_| Heartbeat::new()) - } -} diff --git a/src/api/schema/meta.rs b/src/api/schema/meta.rs deleted file mode 100644 index 76cbf2749c225..0000000000000 --- a/src/api/schema/meta.rs +++ /dev/null @@ -1,27 +0,0 @@ -use async_graphql::Object; - -#[derive(Default)] -pub struct Meta; - -#[Object] -impl Meta { - /// Vector version - async fn version_string(&self) -> String { - crate::get_version() - } - - /// Hostname - async fn hostname(&self) -> Option { - crate::get_hostname().ok() - } -} - -#[derive(Default)] -pub struct MetaQuery; - -#[Object] -impl MetaQuery { - async fn meta(&self) -> Meta { - Meta - } -} diff --git a/src/api/schema/metrics/allocated_bytes.rs b/src/api/schema/metrics/allocated_bytes.rs deleted file mode 100644 index 2f2f93ac833ad..0000000000000 --- a/src/api/schema/metrics/allocated_bytes.rs +++ /dev/null @@ -1,71 +0,0 @@ -use async_graphql::Object; -use chrono::{DateTime, Utc}; - -use crate::{ - config::ComponentKey, - event::{Metric, MetricValue}, -}; - -pub struct AllocatedBytes(Metric); - -impl AllocatedBytes { - pub const fn new(m: Metric) -> Self { - Self(m) - } -} - -#[Object] -impl AllocatedBytes { - /// Metric timestamp - pub async fn timestamp(&self) -> Option> { - self.0.timestamp() - } - - /// Allocated bytes - pub async fn allocated_bytes(&self) -> f64 { - match self.0.value() { - MetricValue::Gauge { value } => *value, - _ => 0.00, - } - } -} - -impl From for AllocatedBytes { - fn from(m: Metric) -> Self { - Self(m) - } -} - -pub struct ComponentAllocatedBytes { - component_key: ComponentKey, - metric: Metric, -} - -impl ComponentAllocatedBytes { - /// Returns a new `ComponentAllocatedBytes` struct, which is a GraphQL type. The - /// component id is hoisted for clear field resolution in the resulting payload - pub fn new(metric: Metric) -> Self { - let component_key = metric.tag_value("component_id").expect( - "Returned a metric without a `component_id`, which shouldn't happen. Please report.", - ); - let component_key = ComponentKey::from(component_key); - - Self { - component_key, - metric, - } - } -} - -#[Object] -impl ComponentAllocatedBytes { - /// Component id - async fn component_id(&self) -> &str { - self.component_key.id() - } - - /// Allocated bytes metric - async fn metric(&self) -> AllocatedBytes { - AllocatedBytes::new(self.metric.clone()) - } -} diff --git a/src/api/schema/metrics/errors.rs b/src/api/schema/metrics/errors.rs deleted file mode 100644 index acca7b74bdf26..0000000000000 --- a/src/api/schema/metrics/errors.rs +++ /dev/null @@ -1,71 +0,0 @@ -use async_graphql::Object; -use chrono::{DateTime, Utc}; - -use crate::{ - config::ComponentKey, - event::{Metric, MetricValue}, -}; - -pub struct ErrorsTotal(Metric); - -impl ErrorsTotal { - pub const fn new(m: Metric) -> Self { - Self(m) - } -} - -#[Object] -impl ErrorsTotal { - /// Metric timestamp - pub async fn timestamp(&self) -> Option> { - self.0.timestamp() - } - - /// Total error count - pub async fn errors_total(&self) -> f64 { - match self.0.value() { - MetricValue::Counter { value } => *value, - _ => 0.00, - } - } -} - -impl From for ErrorsTotal { - fn from(m: Metric) -> Self { - Self(m) - } -} - -pub struct ComponentErrorsTotal { - component_key: ComponentKey, - metric: Metric, -} - -impl ComponentErrorsTotal { - /// Returns a new `ComponentErrorsTotal` struct, which is a GraphQL type. The - /// component id is hoisted for clear field resolution in the resulting payload - pub fn new(metric: Metric) -> Self { - let component_key = metric.tag_value("component_id").expect( - "Returned a metric without a `component_id`, which shouldn't happen. Please report.", - ); - let component_key = ComponentKey::from(component_key); - - Self { - component_key, - metric, - } - } -} - -#[Object] -impl ComponentErrorsTotal { - /// Component id - async fn component_id(&self) -> &str { - self.component_key.id() - } - - /// Errors processed metric - async fn metric(&self) -> ErrorsTotal { - ErrorsTotal::new(self.metric.clone()) - } -} diff --git a/src/api/schema/metrics/filter.rs b/src/api/schema/metrics/filter.rs deleted file mode 100644 index c85379edeeff4..0000000000000 --- a/src/api/schema/metrics/filter.rs +++ /dev/null @@ -1,397 +0,0 @@ -use std::collections::{BTreeMap, HashSet}; - -use async_stream::stream; -use tokio::time::Duration; -use tokio_stream::{Stream, StreamExt}; - -use super::{ - OutputThroughput, ReceivedBytesTotal, ReceivedEventsTotal, SentBytesTotal, SentEventsTotal, - filter_output_metric, -}; -use crate::{ - config::ComponentKey, - event::{Metric, MetricValue}, - metrics::Controller, -}; - -fn get_controller() -> &'static Controller { - Controller::get().expect("Metrics system not initialized. Please report.") -} - -/// Sums an iterable of `&Metric`, by folding metric values. Convenience function typically -/// used to get aggregate metrics. -pub fn sum_metrics<'a, I: IntoIterator>(metrics: I) -> Option { - let mut iter = metrics.into_iter(); - let m = iter.next()?; - - Some(iter.fold( - m.clone(), - |mut m1, m2| { - if m1.update(m2) { m1 } else { m2.clone() } - }, - )) -} - -/// Sums an iterable of `Metric`, by folding metric values. Convenience function typically -/// used to get aggregate metrics. -fn sum_metrics_owned>(metrics: I) -> Option { - let mut iter = metrics.into_iter(); - let m = iter.next()?; - - Some(iter.fold(m, |mut m1, m2| if m1.update(&m2) { m1 } else { m2 })) -} - -pub trait MetricsFilter<'a> { - fn received_bytes_total(&self) -> Option; - fn received_events_total(&self) -> Option; - fn sent_bytes_total(&self) -> Option; - fn sent_events_total(&self) -> Option; -} - -impl MetricsFilter<'_> for Vec { - fn received_bytes_total(&self) -> Option { - let sum = sum_metrics( - self.iter() - .filter(|m| m.name() == "component_received_bytes_total"), - )?; - - Some(ReceivedBytesTotal::new(sum)) - } - - fn received_events_total(&self) -> Option { - let sum = sum_metrics( - self.iter() - .filter(|m| m.name() == "component_received_events_total"), - )?; - - Some(ReceivedEventsTotal::new(sum)) - } - - fn sent_bytes_total(&self) -> Option { - let sum = sum_metrics( - self.iter() - .filter(|m| m.name() == "component_sent_bytes_total"), - )?; - - Some(SentBytesTotal::new(sum)) - } - - fn sent_events_total(&self) -> Option { - let sum = sum_metrics( - self.iter() - .filter(|m| m.name() == "component_sent_events_total"), - )?; - - Some(SentEventsTotal::new(sum)) - } -} - -impl<'a> MetricsFilter<'a> for Vec<&'a Metric> { - fn received_bytes_total(&self) -> Option { - let sum = sum_metrics( - self.iter() - .filter(|m| m.name() == "component_received_bytes_total") - .copied(), - )?; - - Some(ReceivedBytesTotal::new(sum)) - } - - fn received_events_total(&self) -> Option { - let sum = sum_metrics( - self.iter() - .filter(|m| m.name() == "component_received_events_total") - .copied(), - )?; - - Some(ReceivedEventsTotal::new(sum)) - } - - fn sent_bytes_total(&self) -> Option { - let sum = sum_metrics( - self.iter() - .filter(|m| m.name() == "component_sent_bytes_total") - .copied(), - )?; - - Some(SentBytesTotal::new(sum)) - } - - fn sent_events_total(&self) -> Option { - let sum = sum_metrics( - self.iter() - .filter(|m| m.name() == "component_sent_events_total") - .copied(), - )?; - - Some(SentEventsTotal::new(sum)) - } -} - -/// Returns a stream of `Metric`s, collected at the provided millisecond interval. -pub fn get_metrics(interval: i32) -> impl Stream { - let controller = get_controller(); - let mut interval = tokio::time::interval(Duration::from_millis(interval as u64)); - - stream! { - loop { - interval.tick().await; - for m in controller.capture_metrics() { - yield m; - } - } - } -} - -pub fn get_all_metrics(interval: i32) -> impl Stream> { - let controller = get_controller(); - let mut interval = tokio::time::interval(Duration::from_millis(interval as u64)); - - stream! { - loop { - interval.tick().await; - yield controller.capture_metrics() - } - } -} - -/// Return [`Vec`] based on a component id tag. -pub fn by_component_key(component_key: &ComponentKey) -> Vec { - get_controller() - .capture_metrics() - .into_iter() - .filter_map(|m| { - m.tag_matches("component_id", component_key.id()) - .then_some(m) - }) - .collect() -} - -type MetricFilterFn = dyn Fn(&Metric) -> bool + Send + Sync; - -/// Returns a stream of `Vec`, where `metric_name` matches the name of the metric -/// (e.g. "component_sent_events_total"), and the value is derived from `MetricValue::Counter`. Uses a -/// local cache to match against the `component_id` of a metric, to return results only when -/// the value of a current iteration is greater than the previous. This is useful for the client -/// to be notified as metrics increase without returning 'empty' or identical results. -pub fn component_counter_metrics( - interval: i32, - filter_fn: &'static MetricFilterFn, -) -> impl Stream> { - let mut cache = BTreeMap::new(); - - component_to_filtered_metrics(interval, filter_fn).map(move |map| { - map.into_iter() - .filter_map(|(id, metrics)| { - let m = sum_metrics_owned(metrics)?; - match m.value() { - MetricValue::Counter { value } - if cache.insert(id, *value).unwrap_or(0.00) < *value => - { - Some(m) - } - _ => None, - } - }) - .collect() - }) -} - -/// Returns a stream of `Vec`, where `metric_name` matches the name of the metric -/// (e.g. "component_sent_events_total"), and the value is derived from `MetricValue::Gauge`. Uses a -/// local cache to match against the `component_id` of a metric, to return results only when -/// the value of a current iteration is greater than the previous. This is useful for the client -/// to be notified as metrics increase without returning 'empty' or identical results. -pub fn component_gauge_metrics( - interval: i32, - filter_fn: &'static MetricFilterFn, -) -> impl Stream> { - let mut cache = BTreeMap::new(); - - component_to_filtered_metrics(interval, filter_fn).map(move |map| { - map.into_iter() - .filter_map(|(id, metrics)| { - let m = sum_metrics_owned(metrics)?; - match m.value() { - MetricValue::Gauge { value } - if cache.insert(id, *value).unwrap_or(0.00) < *value => - { - Some(m) - } - _ => None, - } - }) - .collect() - }) -} - -/// Returns the throughput of a 'counter' metric, sampled over `interval` milliseconds -/// and filtered by the provided `filter_fn`. -pub fn counter_throughput( - interval: i32, - filter_fn: &'static MetricFilterFn, -) -> impl Stream { - let mut last = 0.00; - - get_metrics(interval) - .filter(filter_fn) - .filter_map(move |m| match m.value() { - MetricValue::Counter { value } if *value > last => { - let throughput = value - last; - last = *value; - Some((m, throughput)) - } - _ => None, - }) - // Ignore the first, since we only care about sampling between `interval` - .skip(1) -} - -/// Returns the throughput of a 'counter' metric, sampled over `interval` milliseconds -/// and filtered by the provided `filter_fn`, aggregated against each component. -pub fn component_counter_throughputs( - interval: i32, - filter_fn: &'static MetricFilterFn, -) -> impl Stream> { - let mut cache = BTreeMap::new(); - - component_to_filtered_metrics(interval, filter_fn) - .map(move |map| { - map.into_iter() - .filter_map(|(id, metrics)| { - let m = sum_metrics_owned(metrics)?; - match m.value() { - MetricValue::Counter { value } => { - let last = cache.insert(id, *value).unwrap_or(0.00); - let throughput = value - last; - Some((m, throughput)) - } - _ => None, - } - }) - .collect() - }) - // Ignore the first, since we only care about sampling between `interval` - .skip(1) -} - -/// Returns a stream of `Vec<(Metric, Vec)>`, where `Metric` is the -/// total `component_sent_events_total` metric for a component and `Vec` -/// is the `component_sent_events_total` metric split by output -pub fn component_sent_events_totals_metrics_with_outputs( - interval: i32, -) -> impl Stream)>> { - let mut cache = BTreeMap::new(); - - component_to_filtered_metrics(interval, &|m| m.name() == "component_sent_events_total").map( - move |map| { - map.into_iter() - .filter_map(|(id, metrics)| { - let outputs = metrics - .iter() - .filter_map(|m| m.tag_value("output")) - .collect::>(); - - let metric_by_outputs = outputs - .iter() - .filter_map(|output| { - let m = filter_output_metric(metrics.as_ref(), output.as_ref())?; - match m.value() { - MetricValue::Counter { value } - if cache - .insert(format!("{id}.{output}"), *value) - .unwrap_or(0.00) - < *value => - { - Some(m) - } - _ => None, - } - }) - .collect(); - - let sum = sum_metrics_owned(metrics)?; - match sum.value() { - MetricValue::Counter { value } - if cache.insert(id, *value).unwrap_or(0.00) < *value => - { - Some((sum, metric_by_outputs)) - } - _ => None, - } - }) - .collect() - }, - ) -} - -/// Returns the throughput of the 'component_sent_events_total' metric, sampled over `interval` milliseconds, -/// for each component. Within a particular component, throughput per output stream is also included. -pub fn component_sent_events_total_throughputs_with_outputs( - interval: i32, -) -> impl Stream)>> { - let mut cache = BTreeMap::new(); - - component_to_filtered_metrics(interval, &|m| m.name() == "component_sent_events_total") - .map(move |map| { - map.into_iter() - .filter_map(|(id, metrics)| { - let outputs = metrics - .iter() - .filter_map(|m| m.tag_value("output")) - .collect::>(); - - let throughput_by_outputs = outputs - .iter() - .filter_map(|output| { - let m = filter_output_metric(metrics.as_ref(), output.as_ref())?; - let throughput = throughput(&m, format!("{id}.{output}"), &mut cache)?; - Some(OutputThroughput::new(output.clone(), throughput as i64)) - }) - .collect::>(); - - let sum = sum_metrics_owned(metrics)?; - let total_throughput = throughput(&sum, id.clone(), &mut cache)?; - Some(( - ComponentKey::from(id), - total_throughput as i64, - throughput_by_outputs, - )) - }) - .collect() - }) - // Ignore the first, since we only care about sampling between `interval` - .skip(1) -} - -/// Returns a map of Component ID to list of metrics where metrics have been -/// filtered by `filter_fn` -fn component_to_filtered_metrics( - interval: i32, - filter_fn: &'static MetricFilterFn, -) -> impl Stream>> { - get_all_metrics(interval).map(move |m| { - m.into_iter() - .filter(filter_fn) - .filter_map(|m| m.tag_value("component_id").map(|id| (id, m))) - .fold( - BTreeMap::new(), - |mut map: BTreeMap>, (id, m)| { - map.entry(id).or_default().push(m); - map - }, - ) - }) -} - -/// Returns throughput based on a metric and provided `cache` of previous values -fn throughput(metric: &Metric, id: String, cache: &mut BTreeMap) -> Option { - match metric.value() { - MetricValue::Counter { value } => { - let last = cache.insert(id, *value).unwrap_or(0.00); - let throughput = value - last; - Some(throughput) - } - _ => None, - } -} diff --git a/src/api/schema/metrics/host.rs b/src/api/schema/metrics/host.rs deleted file mode 100644 index 08a8fbcb973be..0000000000000 --- a/src/api/schema/metrics/host.rs +++ /dev/null @@ -1,373 +0,0 @@ -use async_graphql::Object; -use cfg_if::cfg_if; - -use crate::{ - event::{Metric, MetricValue}, - sources::host_metrics, -}; - -pub struct MemoryMetrics(Vec); - -#[Object] -/// Host memory metrics -impl MemoryMetrics { - /// Total bytes - async fn total_bytes(&self) -> f64 { - filter_host_metric(&self.0, "memory_total_bytes") - } - - /// Free bytes - async fn free_bytes(&self) -> f64 { - filter_host_metric(&self.0, "memory_free_bytes") - } - - /// Available bytes - async fn available_bytes(&self) -> f64 { - filter_host_metric(&self.0, "memory_available_bytes") - } - - /// Active bytes (Linux/macOS only) - async fn active_bytes(&self) -> Option { - if cfg!(any(target_os = "linux", target_os = "macos")) { - Some(filter_host_metric(&self.0, "memory_active_bytes")) - } else { - None - } - } - - /// Buffers bytes (Linux only) - async fn buffers_bytes(&self) -> Option { - if cfg!(target_os = "linux") { - Some(filter_host_metric(&self.0, "memory_buffers_bytes")) - } else { - None - } - } - - /// Cached bytes (Linux only) - async fn cached_bytes(&self) -> Option { - if cfg!(target_os = "linux") { - Some(filter_host_metric(&self.0, "memory_cached_bytes")) - } else { - None - } - } - - /// Shared bytes (Linux only) - async fn shared_bytes(&self) -> Option { - if cfg!(target_os = "linux") { - Some(filter_host_metric(&self.0, "memory_shared_bytes")) - } else { - None - } - } - - /// Used bytes (Linux only) - async fn used_bytes(&self) -> Option { - if cfg!(target_os = "linux") { - Some(filter_host_metric(&self.0, "memory_used_bytes")) - } else { - None - } - } - - /// Inactive bytes (macOS only) - async fn inactive_bytes(&self) -> Option { - if cfg!(target_os = "macos") { - Some(filter_host_metric(&self.0, "memory_inactive_bytes")) - } else { - None - } - } - - /// Wired bytes (macOS only) - async fn wired_bytes(&self) -> Option { - if cfg!(target_os = "macos") { - Some(filter_host_metric(&self.0, "memory_wired_bytes")) - } else { - None - } - } -} - -pub struct SwapMetrics(Vec); - -#[Object] -impl SwapMetrics { - /// Swap free bytes - async fn free_bytes(&self) -> f64 { - filter_host_metric(&self.0, "memory_swap_free_bytes") - } - - /// Swap total bytes - async fn total_bytes(&self) -> f64 { - filter_host_metric(&self.0, "memory_swap_total_bytes") - } - - /// Swap used bytes - async fn used_bytes(&self) -> f64 { - filter_host_metric(&self.0, "memory_swap_used_bytes") - } - - /// Swapped in bytes total (not available on Windows) - async fn swapped_in_bytes_total(&self) -> Option { - if cfg!(not(windows)) { - Some(filter_host_metric(&self.0, "memory_swapped_in_bytes_total")) - } else { - None - } - } - - /// Swapped out bytes total (not available on Windows) - async fn swapped_out_bytes_total(&self) -> Option { - if cfg!(not(windows)) { - Some(filter_host_metric( - &self.0, - "memory_swapped_out_bytes_total", - )) - } else { - None - } - } -} - -pub struct CpuMetrics(Vec); - -#[Object] -impl CpuMetrics { - /// CPU seconds total - async fn cpu_seconds_total(&self) -> f64 { - filter_host_metric(&self.0, "cpu_seconds_total") - } -} - -pub struct LoadAverageMetrics(Vec); - -#[Object] -impl LoadAverageMetrics { - /// Load 1 average - async fn load1(&self) -> f64 { - filter_host_metric(&self.0, "load1") - } - - /// Load 5 average - async fn load5(&self) -> f64 { - filter_host_metric(&self.0, "load5") - } - - /// Load 15 average - async fn load15(&self) -> f64 { - filter_host_metric(&self.0, "load15") - } -} - -pub struct NetworkMetrics(Vec); - -#[Object] -impl NetworkMetrics { - /// Total bytes received - async fn receive_bytes_total(&self) -> f64 { - filter_host_metric(&self.0, "network_receive_bytes_total") - } - - /// Total errors received - async fn receive_errs_total(&self) -> f64 { - filter_host_metric(&self.0, "network_receive_errs_total") - } - - /// Total packets received - async fn receive_packets_total(&self) -> f64 { - filter_host_metric(&self.0, "network_receive_packets_total") - } - - /// Total bytes transmitted - async fn transmit_bytes_total(&self) -> f64 { - filter_host_metric(&self.0, "network_transmit_bytes_total") - } - - /// Total errors transmitted - async fn transmit_errs_total(&self) -> f64 { - filter_host_metric(&self.0, "network_transmit_errs_total") - } - - /// Total transmission packets dropped (Linux/Windows only) - async fn transmit_packets_drop_total(&self) -> Option { - if cfg!(any(target_os = "linux", windows)) { - Some(filter_host_metric( - &self.0, - "network_transmit_packets_drop_total", - )) - } else { - None - } - } - - /// Total transmission packets (Linux/Windows only) - async fn transmit_packets_total(&self) -> Option { - if cfg!(any(target_os = "linux", windows)) { - Some(filter_host_metric( - &self.0, - "network_transmit_packets_total", - )) - } else { - None - } - } -} - -pub struct FileSystemMetrics(Vec); - -#[Object] -impl FileSystemMetrics { - /// Free bytes - async fn free_bytes(&self) -> f64 { - filter_host_metric(&self.0, "filesystem_free_bytes") - } - - /// Total bytes - async fn total_bytes(&self) -> f64 { - filter_host_metric(&self.0, "filesystem_total_bytes") - } - - /// Used bytes - async fn used_bytes(&self) -> f64 { - filter_host_metric(&self.0, "filesystem_used_bytes") - } -} - -pub struct DiskMetrics(Vec); - -#[Object] -impl DiskMetrics { - /// Total bytes read - async fn read_bytes_total(&self) -> f64 { - filter_host_metric(&self.0, "disk_read_bytes_total") - } - - /// Total reads completed - async fn reads_completed_total(&self) -> f64 { - filter_host_metric(&self.0, "disk_reads_completed_total") - } - - /// Total bytes written - async fn written_bytes_total(&self) -> f64 { - filter_host_metric(&self.0, "disk_written_bytes_total") - } - - /// Total writes completed - async fn writes_completed_total(&self) -> f64 { - filter_host_metric(&self.0, "disk_writes_completed_total") - } -} - -cfg_if! { - if #[cfg(target_os = "linux")] { - pub struct TCPMetrics(Vec); - - #[Object] - impl TCPMetrics { - /// Total TCP connections - async fn tcp_conns_total(&self) -> f64 { - filter_host_metric(&self.0, "tcp_connections_total") - } - - /// Total bytes in the send queue across all connections. - async fn tcp_tx_queued_bytes_total(&self) -> f64 { - filter_host_metric(&self.0, "tcp_tx_queued_bytes_total") - } - - /// Total bytes in the receive queue across all connections. - async fn tcp_rx_queued_bytes_total(&self) -> f64 { - filter_host_metric(&self.0, "tcp_rx_queued_bytes_total") - } - } - } -} - -pub struct HostMetrics(host_metrics::HostMetrics); - -impl HostMetrics { - /// Primes the host metrics pump by passing through a new `HostMetrics` - pub fn new() -> Self { - let config = host_metrics::HostMetricsConfig::default(); - Self(host_metrics::HostMetrics::new(config)) - } -} - -#[Object] -/// Vector host metrics -impl HostMetrics { - /// Memory metrics - async fn memory(&self) -> MemoryMetrics { - let mut buffer = self.0.buffer(); - self.0.memory_metrics(&mut buffer).await; - MemoryMetrics(buffer.metrics) - } - - /// Swap metrics - async fn swap(&self) -> SwapMetrics { - let mut buffer = self.0.buffer(); - self.0.swap_metrics(&mut buffer).await; - SwapMetrics(buffer.metrics) - } - - /// CPU metrics - async fn cpu(&self) -> CpuMetrics { - let mut buffer = self.0.buffer(); - self.0.cpu_metrics(&mut buffer).await; - CpuMetrics(buffer.metrics) - } - - /// Load average metrics (*nix only) - async fn load_average(&self) -> Option { - if cfg!(unix) { - let mut buffer = self.0.buffer(); - self.0.loadavg_metrics(&mut buffer).await; - Some(LoadAverageMetrics(buffer.metrics)) - } else { - None - } - } - - /// Network metrics - async fn network(&self) -> NetworkMetrics { - let mut buffer = self.0.buffer(); - self.0.network_metrics(&mut buffer).await; - NetworkMetrics(buffer.metrics) - } - - /// Filesystem metrics - async fn filesystem(&self) -> FileSystemMetrics { - let mut buffer = self.0.buffer(); - self.0.filesystem_metrics(&mut buffer).await; - FileSystemMetrics(buffer.metrics) - } - - /// Disk metrics - async fn disk(&self) -> DiskMetrics { - let mut buffer = self.0.buffer(); - self.0.disk_metrics(&mut buffer).await; - DiskMetrics(buffer.metrics) - } - - #[cfg(target_os = "linux")] - /// TCP metrics - async fn tcp(&self) -> TCPMetrics { - let mut buffer = self.0.buffer(); - self.0.tcp_metrics(&mut buffer).await; - TCPMetrics(buffer.metrics) - } -} - -/// Filters a [`Vec`] by name, returning the inner `value` or 0.00 if not found -fn filter_host_metric(metrics: &[Metric], name: &str) -> f64 { - metrics - .iter() - .find(|m| matches!(m.namespace(), Some(n) if n == "host") && m.name() == name) - .map(|m| match m.value() { - MetricValue::Gauge { value } => *value, - MetricValue::Counter { value } => *value, - _ => 0.00, - }) - .unwrap_or_else(|| 0.00) -} diff --git a/src/api/schema/metrics/mod.rs b/src/api/schema/metrics/mod.rs deleted file mode 100644 index 056b3c27a7986..0000000000000 --- a/src/api/schema/metrics/mod.rs +++ /dev/null @@ -1,284 +0,0 @@ -mod allocated_bytes; -mod errors; -pub mod filter; -mod output; -mod received_bytes; -mod received_events; -mod sent_bytes; -mod sent_events; -mod sink; -pub mod source; -mod transform; -mod uptime; - -#[cfg(feature = "sources-host_metrics")] -mod host; - -pub use allocated_bytes::{AllocatedBytes, ComponentAllocatedBytes}; -use async_graphql::{Interface, Subscription}; -use chrono::{DateTime, Utc}; -pub use errors::{ComponentErrorsTotal, ErrorsTotal}; -pub use filter::*; -pub use output::*; -pub use received_bytes::{ - ComponentReceivedBytesThroughput, ComponentReceivedBytesTotal, ReceivedBytesTotal, -}; -pub use received_events::{ - ComponentReceivedEventsThroughput, ComponentReceivedEventsTotal, ReceivedEventsTotal, -}; -pub use sent_bytes::{ComponentSentBytesThroughput, ComponentSentBytesTotal, SentBytesTotal}; -pub use sent_events::{ComponentSentEventsThroughput, ComponentSentEventsTotal, SentEventsTotal}; -pub use sink::{IntoSinkMetrics, SinkMetrics}; -pub use source::{IntoSourceMetrics, SourceMetrics}; -use tokio_stream::{Stream, StreamExt}; -pub use transform::{IntoTransformMetrics, TransformMetrics}; -pub use uptime::Uptime; - -use crate::config::ComponentKey; - -#[derive(Interface)] -#[graphql(field(name = "timestamp", ty = "Option>"))] -pub enum MetricType { - Uptime(Uptime), -} - -#[cfg(feature = "sources-host_metrics")] -#[derive(Default)] -pub struct MetricsQuery; - -#[cfg(feature = "sources-host_metrics")] -#[async_graphql::Object] -impl MetricsQuery { - /// Vector host metrics - async fn host_metrics(&self) -> host::HostMetrics { - host::HostMetrics::new() - } -} - -#[derive(Default)] -pub struct MetricsSubscription; - -#[Subscription] -impl MetricsSubscription { - /// Metrics for how long the Vector instance has been running - async fn uptime( - &self, - #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream + use<> { - get_metrics(interval).filter_map(|m| match m.name() { - "uptime_seconds" => Some(Uptime::new(m)), - _ => None, - }) - } - - /// Total received events metrics - #[graphql(deprecation = "Use component_received_events_totals instead")] - async fn received_events_total( - &self, - #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream + use<> { - get_metrics(interval).filter_map(|m| match m.name() { - "component_received_events_total" => Some(ReceivedEventsTotal::new(m)), - _ => None, - }) - } - - /// Total received events throughput sampled over the provided millisecond `interval` - #[graphql(deprecation = "Use component_received_events_throughputs instead")] - async fn received_events_throughput( - &self, - #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream + use<> { - counter_throughput(interval, &|m| m.name() == "component_received_events_total") - .map(|(_, throughput)| throughput as i64) - } - - /// Total incoming component events throughput metrics over `interval` - async fn component_received_events_throughputs( - &self, - #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream> + use<> { - component_counter_throughputs(interval, &|m| m.name() == "component_received_events_total") - .map(|m| { - m.into_iter() - .map(|(m, throughput)| { - ComponentReceivedEventsThroughput::new( - ComponentKey::from(m.tag_value("component_id").unwrap()), - throughput as i64, - ) - }) - .collect() - }) - } - - /// Total received component event metrics over `interval` - async fn component_received_events_totals( - &self, - #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream> + use<> { - component_counter_metrics(interval, &|m| m.name() == "component_received_events_total").map( - |m| { - m.into_iter() - .map(ComponentReceivedEventsTotal::new) - .collect() - }, - ) - } - - /// Total sent events metrics - #[graphql(deprecation = "Use component_sent_events_totals instead")] - async fn sent_events_total( - &self, - #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream + use<> { - get_metrics(interval).filter_map(|m| match m.name() { - "component_sent_events_total" => Some(SentEventsTotal::new(m)), - _ => None, - }) - } - - /// Total outgoing events throughput sampled over the provided millisecond `interval` - #[graphql(deprecation = "Use component_sent_events_throughputs instead")] - async fn sent_events_throughput( - &self, - #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream + use<> { - counter_throughput(interval, &|m| m.name() == "component_sent_events_total") - .map(|(_, throughput)| throughput as i64) - } - - /// Total outgoing component event throughput metrics over `interval` - async fn component_sent_events_throughputs( - &self, - #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream> + use<> { - component_sent_events_total_throughputs_with_outputs(interval).map(|m| { - m.into_iter() - .map(|(key, total_throughput, outputs)| { - ComponentSentEventsThroughput::new(key, total_throughput, outputs) - }) - .collect() - }) - } - - /// Total outgoing component event metrics over `interval` - async fn component_sent_events_totals( - &self, - #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream> + use<> { - component_sent_events_totals_metrics_with_outputs(interval).map(|ms| { - ms.into_iter() - .map(|(m, m_by_outputs)| ComponentSentEventsTotal::new(m, m_by_outputs)) - .collect() - }) - } - - /// Component bytes received metrics over `interval`. - async fn component_received_bytes_totals( - &self, - #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream> + use<> { - component_counter_metrics(interval, &|m| m.name() == "component_received_bytes_total").map( - |m| { - m.into_iter() - .map(ComponentReceivedBytesTotal::new) - .collect() - }, - ) - } - - /// Component bytes received throughput over `interval` - async fn component_received_bytes_throughputs( - &self, - #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream> + use<> { - component_counter_throughputs(interval, &|m| m.name() == "component_received_bytes_total") - .map(|m| { - m.into_iter() - .map(|(m, throughput)| { - ComponentReceivedBytesThroughput::new( - ComponentKey::from(m.tag_value("component_id").unwrap()), - throughput as i64, - ) - }) - .collect() - }) - } - - /// Component bytes sent metrics over `interval`. - async fn component_sent_bytes_totals( - &self, - #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream> + use<> { - component_counter_metrics(interval, &|m| m.name() == "component_sent_bytes_total") - .map(|m| m.into_iter().map(ComponentSentBytesTotal::new).collect()) - } - - /// Component bytes sent throughput over `interval` - async fn component_sent_bytes_throughputs( - &self, - #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream> + use<> { - component_counter_throughputs(interval, &|m| m.name() == "component_sent_bytes_total").map( - |m| { - m.into_iter() - .map(|(m, throughput)| { - ComponentSentBytesThroughput::new( - ComponentKey::from(m.tag_value("component_id").unwrap()), - throughput as i64, - ) - }) - .collect() - }, - ) - } - - /// Total error metrics. - async fn errors_total( - &self, - #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream + use<> { - get_metrics(interval) - .filter(|m| m.name().ends_with("_errors_total")) - .map(ErrorsTotal::new) - } - - /// Allocated bytes metrics. - async fn allocated_bytes( - &self, - #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream + use<> { - get_metrics(interval) - .filter(|m| m.name() == "component_allocated_bytes") - .map(AllocatedBytes::new) - } - - /// Component allocation metrics - async fn component_allocated_bytes( - &self, - #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream> + use<> { - component_gauge_metrics(interval, &|m| m.name() == "component_allocated_bytes") - .map(|m| m.into_iter().map(ComponentAllocatedBytes::new).collect()) - } - - /// Component error metrics over `interval`. - async fn component_errors_totals( - &self, - #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream> + use<> { - component_counter_metrics(interval, &|m| m.name().ends_with("_errors_total")) - .map(|m| m.into_iter().map(ComponentErrorsTotal::new).collect()) - } - - /// All metrics. - async fn metrics( - &self, - #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream + use<> { - get_metrics(interval).filter_map(|m| match m.name() { - "uptime_seconds" => Some(MetricType::Uptime(m.into())), - _ => None, - }) - } -} diff --git a/src/api/schema/metrics/output.rs b/src/api/schema/metrics/output.rs deleted file mode 100644 index e1fd0976e7b48..0000000000000 --- a/src/api/schema/metrics/output.rs +++ /dev/null @@ -1,88 +0,0 @@ -use async_graphql::Object; -use vector_lib::config::ComponentKey; - -use super::{SentEventsTotal, by_component_key, sum_metrics}; -use crate::event::Metric; - -#[derive(Debug, Clone)] -pub struct Output { - output_id: String, - sent_events_total: Option, -} - -impl Output { - pub const fn new(output_id: String, sent_events_total: Option) -> Self { - Self { - output_id, - sent_events_total, - } - } -} - -#[Object] -impl Output { - /// Id of the output stream - pub async fn output_id(&self) -> &str { - self.output_id.as_ref() - } - - /// Total sent events for the current output stream - pub async fn sent_events_total(&self) -> Option { - self.sent_events_total - .as_ref() - .map(|metric| SentEventsTotal::new(metric.clone())) - } -} - -#[derive(Debug, Clone)] -pub struct OutputThroughput { - output_id: String, - throughput: i64, -} - -impl OutputThroughput { - pub const fn new(output_id: String, throughput: i64) -> Self { - Self { - output_id, - throughput, - } - } -} - -#[Object] -impl OutputThroughput { - /// Id of the output stream - pub async fn output_id(&self) -> &str { - self.output_id.as_ref() - } - - /// Throughput for the output stream - pub async fn throughput(&self) -> i64 { - self.throughput - } -} - -pub fn outputs_by_component_key(component_key: &ComponentKey, outputs: &[String]) -> Vec { - let metrics = by_component_key(component_key) - .into_iter() - .filter(|m| m.name() == "component_sent_events_total") - .collect::>(); - - outputs - .iter() - .map(|output| { - Output::new( - output.clone(), - filter_output_metric(&metrics, output.as_ref()), - ) - }) - .collect::>() -} - -pub fn filter_output_metric(metrics: &[Metric], output_name: &str) -> Option { - sum_metrics( - metrics - .iter() - .filter(|m| m.tag_matches("output", output_name)), - ) -} diff --git a/src/api/schema/metrics/received_bytes.rs b/src/api/schema/metrics/received_bytes.rs deleted file mode 100644 index 516631559cbae..0000000000000 --- a/src/api/schema/metrics/received_bytes.rs +++ /dev/null @@ -1,108 +0,0 @@ -use async_graphql::Object; -use chrono::{DateTime, Utc}; - -use crate::{ - config::ComponentKey, - event::{Metric, MetricValue}, -}; - -pub struct ReceivedBytesTotal(Metric); - -impl ReceivedBytesTotal { - pub const fn new(m: Metric) -> Self { - Self(m) - } - - pub fn get_timestamp(&self) -> Option> { - self.0.timestamp() - } - - pub fn get_received_bytes_total(&self) -> f64 { - match self.0.value() { - MetricValue::Counter { value } => *value, - _ => 0.00, - } - } -} - -#[Object] -impl ReceivedBytesTotal { - /// Metric timestamp. - pub async fn timestamp(&self) -> Option> { - self.get_timestamp() - } - - /// Total number of bytes received. - pub async fn received_bytes_total(&self) -> f64 { - self.get_received_bytes_total() - } -} - -impl From for ReceivedBytesTotal { - fn from(m: Metric) -> Self { - Self(m) - } -} - -pub struct ComponentReceivedBytesTotal { - component_key: ComponentKey, - metric: Metric, -} - -impl ComponentReceivedBytesTotal { - /// Returns a new `ComponentReceivedBytesTotal`. - /// - /// Expects that the metric contains a tag for the component ID the metric is referenced to. - pub fn new(metric: Metric) -> Self { - let component_key = metric.tag_value("component_id").expect( - "Returned a metric without a `component_id`, which shouldn't happen. Please report.", - ); - let component_key = ComponentKey::from(component_key); - - Self { - component_key, - metric, - } - } -} - -#[Object] -impl ComponentReceivedBytesTotal { - /// Component ID. - async fn component_id(&self) -> &str { - self.component_key.id() - } - - /// Metric for total bytes received. - async fn metric(&self) -> ReceivedBytesTotal { - ReceivedBytesTotal::new(self.metric.clone()) - } -} - -pub struct ComponentReceivedBytesThroughput { - component_key: ComponentKey, - throughput: i64, -} - -impl ComponentReceivedBytesThroughput { - /// Returns a new `ComponentReceivedBytesThroughput` for the given component. - pub const fn new(component_key: ComponentKey, throughput: i64) -> Self { - Self { - component_key, - throughput, - } - } -} - -#[Object] -impl ComponentReceivedBytesThroughput { - /// Component ID. - async fn component_id(&self) -> &str { - self.component_key.id() - } - - /// Throughput of bytes sent. - async fn throughput(&self) -> i64 { - self.throughput - } -} diff --git a/src/api/schema/metrics/received_events.rs b/src/api/schema/metrics/received_events.rs deleted file mode 100644 index b313004a8ef72..0000000000000 --- a/src/api/schema/metrics/received_events.rs +++ /dev/null @@ -1,107 +0,0 @@ -use async_graphql::Object; -use chrono::{DateTime, Utc}; - -use crate::{ - config::ComponentKey, - event::{Metric, MetricValue}, -}; - -pub struct ReceivedEventsTotal(Metric); - -impl ReceivedEventsTotal { - pub const fn new(m: Metric) -> Self { - Self(m) - } - - pub fn get_timestamp(&self) -> Option> { - self.0.timestamp() - } - - pub fn get_received_events_total(&self) -> f64 { - match self.0.value() { - MetricValue::Counter { value } => *value, - _ => 0.00, - } - } -} - -#[Object] -impl ReceivedEventsTotal { - /// Metric timestamp - pub async fn timestamp(&self) -> Option> { - self.get_timestamp() - } - - /// Total incoming events - pub async fn received_events_total(&self) -> f64 { - self.get_received_events_total() - } -} - -impl From for ReceivedEventsTotal { - fn from(m: Metric) -> Self { - Self(m) - } -} - -pub struct ComponentReceivedEventsTotal { - component_key: ComponentKey, - metric: Metric, -} - -impl ComponentReceivedEventsTotal { - /// Returns a new `ComponentReceivedEventsTotal` struct, which is a GraphQL type. The - /// component id is hoisted for clear field resolution in the resulting payload. - pub fn new(metric: Metric) -> Self { - let component_key = metric.tag_value("component_id").expect( - "Returned a metric without a `component_id`, which shouldn't happen. Please report.", - ); - let component_key = ComponentKey::from(component_key); - - Self { - component_key, - metric, - } - } -} - -#[Object] -impl ComponentReceivedEventsTotal { - /// Component id - async fn component_id(&self) -> &str { - self.component_key.id() - } - - /// Total received events metric - async fn metric(&self) -> ReceivedEventsTotal { - ReceivedEventsTotal::new(self.metric.clone()) - } -} - -pub struct ComponentReceivedEventsThroughput { - component_key: ComponentKey, - throughput: i64, -} - -impl ComponentReceivedEventsThroughput { - /// Returns a new `ComponentReceivedEventsThroughput`, set to the provided id/throughput values. - pub const fn new(component_key: ComponentKey, throughput: i64) -> Self { - Self { - component_key, - throughput, - } - } -} - -#[Object] -impl ComponentReceivedEventsThroughput { - /// Component id - async fn component_id(&self) -> &str { - self.component_key.id() - } - - /// Received events throughput - async fn throughput(&self) -> i64 { - self.throughput - } -} diff --git a/src/api/schema/metrics/sent_bytes.rs b/src/api/schema/metrics/sent_bytes.rs deleted file mode 100644 index 02fb833a39adc..0000000000000 --- a/src/api/schema/metrics/sent_bytes.rs +++ /dev/null @@ -1,108 +0,0 @@ -use async_graphql::Object; -use chrono::{DateTime, Utc}; - -use crate::{ - config::ComponentKey, - event::{Metric, MetricValue}, -}; - -pub struct SentBytesTotal(Metric); - -impl SentBytesTotal { - pub const fn new(m: Metric) -> Self { - Self(m) - } - - pub fn get_timestamp(&self) -> Option> { - self.0.timestamp() - } - - pub fn get_sent_bytes_total(&self) -> f64 { - match self.0.value() { - MetricValue::Counter { value } => *value, - _ => 0.00, - } - } -} - -#[Object] -impl SentBytesTotal { - /// Metric timestamp. - pub async fn timestamp(&self) -> Option> { - self.get_timestamp() - } - - /// Total number of bytes sent. - pub async fn sent_bytes_total(&self) -> f64 { - self.get_sent_bytes_total() - } -} - -impl From for SentBytesTotal { - fn from(m: Metric) -> Self { - Self(m) - } -} - -pub struct ComponentSentBytesTotal { - component_key: ComponentKey, - metric: Metric, -} - -impl ComponentSentBytesTotal { - /// Returns a new `ComponentSentBytesTotal` for the given metric. - /// - /// Expects that the metric contains a tag for the component ID the metric is referenced to. - pub fn new(metric: Metric) -> Self { - let component_key = metric.tag_value("component_id").expect( - "Returned a metric without a `component_id`, which shouldn't happen. Please report.", - ); - let component_key = ComponentKey::from(component_key); - - Self { - component_key, - metric, - } - } -} - -#[Object] -impl ComponentSentBytesTotal { - /// Component ID. - async fn component_id(&self) -> &str { - self.component_key.id() - } - - /// Metric for total bytes sent. - async fn metric(&self) -> SentBytesTotal { - SentBytesTotal::new(self.metric.clone()) - } -} - -pub struct ComponentSentBytesThroughput { - component_key: ComponentKey, - throughput: i64, -} - -impl ComponentSentBytesThroughput { - /// Returns a new `ComponentSentBytesThroughput` for the given component. - pub const fn new(component_key: ComponentKey, throughput: i64) -> Self { - Self { - component_key, - throughput, - } - } -} - -#[Object] -impl ComponentSentBytesThroughput { - /// Component ID. - async fn component_id(&self) -> &str { - self.component_key.id() - } - - /// Throughput of bytes sent. - async fn throughput(&self) -> i64 { - self.throughput - } -} diff --git a/src/api/schema/metrics/sent_events.rs b/src/api/schema/metrics/sent_events.rs deleted file mode 100644 index 5085cebc05334..0000000000000 --- a/src/api/schema/metrics/sent_events.rs +++ /dev/null @@ -1,133 +0,0 @@ -use async_graphql::Object; -use chrono::{DateTime, Utc}; - -use super::{Output, OutputThroughput}; -use crate::{ - config::ComponentKey, - event::{Metric, MetricValue}, -}; - -pub struct SentEventsTotal(Metric); - -impl SentEventsTotal { - pub const fn new(m: Metric) -> Self { - Self(m) - } - - pub fn get_timestamp(&self) -> Option> { - self.0.timestamp() - } - - pub fn get_sent_events_total(&self) -> f64 { - match self.0.value() { - MetricValue::Counter { value } => *value, - _ => 0.00, - } - } -} - -#[Object] -impl SentEventsTotal { - /// Metric timestamp - pub async fn timestamp(&self) -> Option> { - self.get_timestamp() - } - - /// Total sent events - pub async fn sent_events_total(&self) -> f64 { - self.get_sent_events_total() - } -} - -impl From for SentEventsTotal { - fn from(m: Metric) -> Self { - Self(m) - } -} - -pub struct ComponentSentEventsTotal { - component_key: ComponentKey, - outputs: Vec, - metric: Metric, -} - -impl ComponentSentEventsTotal { - /// Returns a new `ComponentSentEventsTotal` struct, which is a GraphQL type. The - /// component id is hoisted for clear field resolution in the resulting payload. - pub fn new(metric: Metric, metric_by_outputs: Vec) -> Self { - let component_key = metric.tag_value("component_id").expect( - "Returned a metric without a `component_id`, which shouldn't happen. Please report.", - ); - let component_key = ComponentKey::from(component_key); - let outputs = metric_by_outputs - .iter() - .filter_map(|m| { - m.tag_value("output") - .map(|output_name| Output::new(output_name, Some(m.clone()))) - }) - .collect::>(); - - Self { - component_key, - outputs, - metric, - } - } -} - -#[Object] -impl ComponentSentEventsTotal { - /// Component id - async fn component_id(&self) -> &str { - self.component_key.id() - } - - /// Total outgoing events metric - async fn metric(&self) -> SentEventsTotal { - SentEventsTotal::new(self.metric.clone()) - } - - /// Output streams with outgoing events metrics - async fn outputs(&self) -> &Vec { - &self.outputs - } -} - -pub struct ComponentSentEventsThroughput { - component_key: ComponentKey, - throughput: i64, - outputs: Vec, -} - -impl ComponentSentEventsThroughput { - /// Returns a new `ComponentSentEventsThroughput`, set to the provided id/throughput values - pub const fn new( - component_key: ComponentKey, - throughput: i64, - outputs: Vec, - ) -> Self { - Self { - component_key, - throughput, - outputs, - } - } -} - -#[Object] -impl ComponentSentEventsThroughput { - /// Component id - async fn component_id(&self) -> &str { - self.component_key.id() - } - - /// Total events processed throughput - async fn throughput(&self) -> i64 { - self.throughput - } - - /// Output streams with throughputs - async fn outputs(&self) -> &Vec { - &self.outputs - } -} diff --git a/src/api/schema/metrics/sink/generic.rs b/src/api/schema/metrics/sink/generic.rs deleted file mode 100644 index 83c50bc6557b7..0000000000000 --- a/src/api/schema/metrics/sink/generic.rs +++ /dev/null @@ -1,33 +0,0 @@ -use async_graphql::Object; - -use crate::{ - api::schema::metrics::{self, MetricsFilter}, - event::Metric, -}; - -#[derive(Debug, Clone)] -pub struct GenericSinkMetrics(Vec); - -impl GenericSinkMetrics { - pub const fn new(metrics: Vec) -> Self { - Self(metrics) - } -} - -#[Object] -impl GenericSinkMetrics { - /// Total received events for the current sink - pub async fn received_events_total(&self) -> Option { - self.0.received_events_total() - } - - /// Total sent bytes for the current sink - pub async fn sent_bytes_total(&self) -> Option { - self.0.sent_bytes_total() - } - - /// Total sent events for the current sink - pub async fn sent_events_total(&self) -> Option { - self.0.sent_events_total() - } -} diff --git a/src/api/schema/metrics/sink/mod.rs b/src/api/schema/metrics/sink/mod.rs deleted file mode 100644 index c864270834cec..0000000000000 --- a/src/api/schema/metrics/sink/mod.rs +++ /dev/null @@ -1,26 +0,0 @@ -mod generic; - -use async_graphql::Interface; - -use super::{ReceivedEventsTotal, SentBytesTotal, SentEventsTotal}; -use crate::event::Metric; - -#[derive(Debug, Clone, Interface)] -#[graphql( - field(name = "received_events_total", ty = "Option"), - field(name = "sent_bytes_total", ty = "Option"), - field(name = "sent_events_total", ty = "Option") -)] -pub enum SinkMetrics { - GenericSinkMetrics(generic::GenericSinkMetrics), -} - -pub trait IntoSinkMetrics { - fn into_sink_metrics(self, component_type: &str) -> SinkMetrics; -} - -impl IntoSinkMetrics for Vec { - fn into_sink_metrics(self, _component_type: &str) -> SinkMetrics { - SinkMetrics::GenericSinkMetrics(generic::GenericSinkMetrics::new(self)) - } -} diff --git a/src/api/schema/metrics/source/file.rs b/src/api/schema/metrics/source/file.rs deleted file mode 100644 index 43c2a2751ceda..0000000000000 --- a/src/api/schema/metrics/source/file.rs +++ /dev/null @@ -1,351 +0,0 @@ -use std::{cmp::Ordering, collections::BTreeMap}; - -use async_graphql::{Enum, InputObject, Object}; - -use crate::{ - api::schema::{ - filter::{CustomFilter, StringFilter, filter_items}, - metrics::{self, MetricsFilter}, - relay, sort, - }, - event::Metric, - filter_check, -}; - -#[derive(Clone)] -pub struct FileSourceMetricFile<'a> { - name: String, - metrics: Vec<&'a Metric>, -} - -impl<'a> FileSourceMetricFile<'a> { - /// Returns a new FileSourceMetricFile from a (name, Vec<&Metric>) tuple - #[allow(clippy::missing_const_for_fn)] // const cannot run destructor - fn from_tuple((name, metrics): (String, Vec<&'a Metric>)) -> Self { - Self { name, metrics } - } - - pub const fn get_name(&self) -> &str { - self.name.as_str() - } -} - -#[Object] -impl FileSourceMetricFile<'_> { - /// File name - async fn name(&self) -> &str { - &*self.name - } - - /// Metric indicating bytes received for the current file - async fn received_bytes_total(&self) -> Option { - self.metrics.received_bytes_total() - } - - /// Metric indicating received events for the current file - async fn received_events_total(&self) -> Option { - self.metrics.received_events_total() - } - - /// Metric indicating outgoing events for the current file - async fn sent_events_total(&self) -> Option { - self.metrics.sent_events_total() - } -} - -#[derive(Debug, Clone)] -pub struct FileSourceMetrics(Vec); - -impl FileSourceMetrics { - pub const fn new(metrics: Vec) -> Self { - Self(metrics) - } - - pub fn get_files(&self) -> Vec> { - self.0 - .iter() - .filter_map(|m| m.tag_value("file").map(|file| (file, m))) - .fold( - BTreeMap::new(), - |mut map: BTreeMap>, (file, m)| { - map.entry(file).or_default().push(m); - map - }, - ) - .into_iter() - .map(FileSourceMetricFile::from_tuple) - .collect() - } -} - -#[derive(Enum, Copy, Clone, Eq, PartialEq)] -pub enum FileSourceMetricFilesSortFieldName { - Name, - ReceivedBytesTotal, - ReceivedEventsTotal, - SentEventsTotal, -} - -impl sort::SortableByField for FileSourceMetricFile<'_> { - fn sort(&self, rhs: &Self, field: &FileSourceMetricFilesSortFieldName) -> Ordering { - match field { - FileSourceMetricFilesSortFieldName::Name => Ord::cmp(&self.name, &rhs.name), - FileSourceMetricFilesSortFieldName::ReceivedBytesTotal => Ord::cmp( - &self - .metrics - .received_bytes_total() - .map(|m| m.get_received_bytes_total() as i64) - .unwrap_or(0), - &rhs.metrics - .received_bytes_total() - .map(|m| m.get_received_bytes_total() as i64) - .unwrap_or(0), - ), - FileSourceMetricFilesSortFieldName::ReceivedEventsTotal => Ord::cmp( - &self - .metrics - .received_events_total() - .map(|m| m.get_received_events_total() as i64) - .unwrap_or(0), - &rhs.metrics - .received_events_total() - .map(|m| m.get_received_events_total() as i64) - .unwrap_or(0), - ), - FileSourceMetricFilesSortFieldName::SentEventsTotal => Ord::cmp( - &self - .metrics - .sent_events_total() - .map(|m| m.get_sent_events_total() as i64) - .unwrap_or(0), - &rhs.metrics - .sent_events_total() - .map(|m| m.get_sent_events_total() as i64) - .unwrap_or(0), - ), - } - } -} - -#[derive(Default, InputObject)] -pub struct FileSourceMetricsFilesFilter { - name: Option>, - or: Option>, -} - -impl CustomFilter> for FileSourceMetricsFilesFilter { - fn matches(&self, file: &FileSourceMetricFile<'_>) -> bool { - filter_check!( - self.name - .as_ref() - .map(|f| f.iter().all(|f| f.filter_value(file.get_name()))) - ); - true - } - - fn or(&self) -> Option<&Vec> { - self.or.as_ref() - } -} - -#[allow(clippy::too_many_arguments)] -#[Object] -impl FileSourceMetrics { - /// File metrics - pub async fn files( - &self, - after: Option, - before: Option, - first: Option, - last: Option, - filter: Option, - sort: Option>>, - ) -> relay::ConnectionResult> { - let filter = filter.unwrap_or_default(); - let mut files = filter_items(self.get_files().into_iter(), &filter); - - if let Some(sort_fields) = sort { - sort::by_fields(&mut files, &sort_fields); - } - - relay::query( - files.into_iter(), - relay::Params::new(after, before, first, last), - 10, - ) - .await - } - - /// Total received bytes for the current file source - pub async fn received_bytes_total(&self) -> Option { - self.0.received_bytes_total() - } - - /// Total received events for the current file source - pub async fn received_events_total(&self) -> Option { - self.0.received_events_total() - } - - /// Total sent events for the current file source - pub async fn sent_events_total(&self) -> Option { - self.0.sent_events_total() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::{ - api::schema::sort::SortField, - event::{MetricKind, MetricValue}, - }; - - struct FileSourceMetricTest { - name: &'static str, - events_metric: Metric, - bytes_metric: Metric, - } - - impl FileSourceMetricTest { - fn new(name: &'static str, events_processed: f64, bytes_processed: f64) -> Self { - Self { - name, - events_metric: metric("component_sent_events_total", events_processed), - bytes_metric: metric("component_received_bytes_total", bytes_processed), - } - } - - fn get_metric(&self) -> FileSourceMetricFile<'_> { - FileSourceMetricFile::from_tuple(( - self.name.to_string(), - vec![&self.bytes_metric, &self.events_metric], - )) - } - } - - fn metric(name: &str, value: f64) -> Metric { - Metric::new( - name, - MetricKind::Incremental, - MetricValue::Counter { value }, - ) - } - - fn by_name(name: &'static str) -> FileSourceMetricTest { - FileSourceMetricTest::new(name, 0.00, 0.00) - } - - #[test] - fn sort_name_asc() { - let t1 = by_name("/path/to/file/2"); - let t2 = by_name("/path/to/file/3"); - let t3 = by_name("/path/to/file/1"); - - let mut files = vec![t1.get_metric(), t2.get_metric(), t3.get_metric()]; - let fields = vec![SortField:: { - field: FileSourceMetricFilesSortFieldName::Name, - direction: sort::Direction::Asc, - }]; - - sort::by_fields(&mut files, &fields); - - for (i, f) in ["1", "2", "3"].iter().enumerate() { - assert_eq!(files[i].name.as_str(), format!("/path/to/file/{f}")); - } - } - - #[test] - fn sort_name_desc() { - let t1 = by_name("/path/to/file/2"); - let t2 = by_name("/path/to/file/3"); - let t3 = by_name("/path/to/file/1"); - - let mut files = vec![t1.get_metric(), t2.get_metric(), t3.get_metric()]; - let fields = vec![SortField:: { - field: FileSourceMetricFilesSortFieldName::Name, - direction: sort::Direction::Desc, - }]; - - sort::by_fields(&mut files, &fields); - - for (i, f) in ["3", "2", "1"].iter().enumerate() { - assert_eq!(files[i].name.as_str(), format!("/path/to/file/{f}")); - } - } - - #[test] - fn processed_events_asc() { - let t1 = FileSourceMetricTest::new("a", 1000.00, 100.00); - let t2 = FileSourceMetricTest::new("b", 500.00, 300.00); - let t3 = FileSourceMetricTest::new("c", 250.00, 200.00); - - let mut files = vec![t1.get_metric(), t2.get_metric(), t3.get_metric()]; - let fields = vec![SortField:: { - field: FileSourceMetricFilesSortFieldName::SentEventsTotal, - direction: sort::Direction::Asc, - }]; - - sort::by_fields(&mut files, &fields); - - for (i, f) in ["c", "b", "a"].iter().enumerate() { - assert_eq!(&files[i].name, *f); - } - } - - #[test] - fn processed_events_desc() { - let t1 = FileSourceMetricTest::new("a", 1000.00, 100.00); - let t2 = FileSourceMetricTest::new("b", 500.00, 300.00); - let t3 = FileSourceMetricTest::new("c", 250.00, 200.00); - - let mut files = vec![t1.get_metric(), t2.get_metric(), t3.get_metric()]; - let fields = vec![SortField:: { - field: FileSourceMetricFilesSortFieldName::SentEventsTotal, - direction: sort::Direction::Desc, - }]; - - sort::by_fields(&mut files, &fields); - - for (i, f) in ["a", "b", "c"].iter().enumerate() { - assert_eq!(&files[i].name, *f); - } - } - - #[test] - fn received_bytes_asc() { - let t1 = FileSourceMetricTest::new("a", 1000.00, 100.00); - let t2 = FileSourceMetricTest::new("b", 500.00, 300.00); - let t3 = FileSourceMetricTest::new("c", 250.00, 200.00); - - let mut files = vec![t1.get_metric(), t2.get_metric(), t3.get_metric()]; - let fields = vec![SortField:: { - field: FileSourceMetricFilesSortFieldName::ReceivedBytesTotal, - direction: sort::Direction::Asc, - }]; - - sort::by_fields(&mut files, &fields); - - for (i, f) in ["a", "c", "b"].iter().enumerate() { - assert_eq!(&files[i].name, *f); - } - } - - #[test] - fn received_bytes_desc() { - let t1 = FileSourceMetricTest::new("a", 1000.00, 100.00); - let t2 = FileSourceMetricTest::new("b", 500.00, 300.00); - let t3 = FileSourceMetricTest::new("c", 250.00, 200.00); - - let mut files = vec![t1.get_metric(), t2.get_metric(), t3.get_metric()]; - let fields = vec![SortField:: { - field: FileSourceMetricFilesSortFieldName::ReceivedBytesTotal, - direction: sort::Direction::Desc, - }]; - - sort::by_fields(&mut files, &fields); - - for (i, f) in ["b", "c", "a"].iter().enumerate() { - assert_eq!(&files[i].name, *f); - } - } -} diff --git a/src/api/schema/metrics/source/generic.rs b/src/api/schema/metrics/source/generic.rs deleted file mode 100644 index 12435c1b36e90..0000000000000 --- a/src/api/schema/metrics/source/generic.rs +++ /dev/null @@ -1,33 +0,0 @@ -use async_graphql::Object; - -use crate::{ - api::schema::metrics::{self, MetricsFilter}, - event::Metric, -}; - -#[derive(Debug, Clone)] -pub struct GenericSourceMetrics(Vec); - -impl GenericSourceMetrics { - pub const fn new(metrics: Vec) -> Self { - Self(metrics) - } -} - -#[Object] -impl GenericSourceMetrics { - /// Total received bytes for the current source - pub async fn received_bytes_total(&self) -> Option { - self.0.received_bytes_total() - } - - /// Total received events for the current source - pub async fn received_events_total(&self) -> Option { - self.0.received_events_total() - } - - /// Total sent events for the current source - pub async fn sent_events_total(&self) -> Option { - self.0.sent_events_total() - } -} diff --git a/src/api/schema/metrics/source/mod.rs b/src/api/schema/metrics/source/mod.rs deleted file mode 100644 index fe5c40ff6c346..0000000000000 --- a/src/api/schema/metrics/source/mod.rs +++ /dev/null @@ -1,31 +0,0 @@ -pub mod file; -mod generic; - -use async_graphql::Interface; - -use super::{ReceivedBytesTotal, ReceivedEventsTotal, SentEventsTotal}; -use crate::event::Metric; - -#[derive(Debug, Clone, Interface)] -#[graphql( - field(name = "received_bytes_total", ty = "Option"), - field(name = "received_events_total", ty = "Option"), - field(name = "sent_events_total", ty = "Option") -)] -pub enum SourceMetrics { - GenericSourceMetrics(generic::GenericSourceMetrics), - FileSourceMetrics(file::FileSourceMetrics), -} - -pub trait IntoSourceMetrics { - fn into_source_metrics(self, component_type: &str) -> SourceMetrics; -} - -impl IntoSourceMetrics for Vec { - fn into_source_metrics(self, component_type: &str) -> SourceMetrics { - match component_type { - "file" => SourceMetrics::FileSourceMetrics(file::FileSourceMetrics::new(self)), - _ => SourceMetrics::GenericSourceMetrics(generic::GenericSourceMetrics::new(self)), - } - } -} diff --git a/src/api/schema/metrics/transform/generic.rs b/src/api/schema/metrics/transform/generic.rs deleted file mode 100644 index 854a845226cc1..0000000000000 --- a/src/api/schema/metrics/transform/generic.rs +++ /dev/null @@ -1,28 +0,0 @@ -use async_graphql::Object; - -use crate::{ - api::schema::metrics::{self, MetricsFilter}, - event::Metric, -}; - -#[derive(Debug, Clone)] -pub struct GenericTransformMetrics(Vec); - -impl GenericTransformMetrics { - pub const fn new(metrics: Vec) -> Self { - Self(metrics) - } -} - -#[Object] -impl GenericTransformMetrics { - /// Total received events for the current transform - pub async fn received_events_total(&self) -> Option { - self.0.received_events_total() - } - - /// Total sent events for the current transform - pub async fn sent_events_total(&self) -> Option { - self.0.sent_events_total() - } -} diff --git a/src/api/schema/metrics/transform/mod.rs b/src/api/schema/metrics/transform/mod.rs deleted file mode 100644 index 183a97907f592..0000000000000 --- a/src/api/schema/metrics/transform/mod.rs +++ /dev/null @@ -1,25 +0,0 @@ -mod generic; - -use async_graphql::Interface; - -use super::{ReceivedEventsTotal, SentEventsTotal}; -use crate::event::Metric; - -#[derive(Debug, Clone, Interface)] -#[graphql( - field(name = "received_events_total", ty = "Option"), - field(name = "sent_events_total", ty = "Option") -)] -pub enum TransformMetrics { - GenericTransformMetrics(generic::GenericTransformMetrics), -} - -pub trait IntoTransformMetrics { - fn into_transform_metrics(self, component_type: &str) -> TransformMetrics; -} - -impl IntoTransformMetrics for Vec { - fn into_transform_metrics(self, _component_type: &str) -> TransformMetrics { - TransformMetrics::GenericTransformMetrics(generic::GenericTransformMetrics::new(self)) - } -} diff --git a/src/api/schema/metrics/uptime.rs b/src/api/schema/metrics/uptime.rs deleted file mode 100644 index 7c02c223e0859..0000000000000 --- a/src/api/schema/metrics/uptime.rs +++ /dev/null @@ -1,34 +0,0 @@ -use async_graphql::Object; -use chrono::{DateTime, Utc}; - -use crate::event::{Metric, MetricValue}; - -pub struct Uptime(Metric); - -impl Uptime { - pub const fn new(m: Metric) -> Self { - Self(m) - } -} - -#[Object] -impl Uptime { - /// Metric timestamp - pub async fn timestamp(&self) -> Option> { - self.0.timestamp() - } - - /// Number of seconds the Vector instance has been alive - pub async fn seconds(&self) -> f64 { - match self.0.value() { - MetricValue::Gauge { value } => *value, - _ => 0.00, - } - } -} - -impl From for Uptime { - fn from(m: Metric) -> Self { - Self(m) - } -} diff --git a/src/api/schema/mod.rs b/src/api/schema/mod.rs deleted file mode 100644 index 58c22a03b687f..0000000000000 --- a/src/api/schema/mod.rs +++ /dev/null @@ -1,31 +0,0 @@ -pub mod components; -pub mod events; -pub mod filter; -mod health; -mod meta; -mod metrics; -mod relay; -pub mod sort; - -use async_graphql::{EmptyMutation, MergedObject, MergedSubscription, Schema, SchemaBuilder}; - -#[derive(MergedObject, Default)] -pub struct Query( - health::HealthQuery, - components::ComponentsQuery, - #[cfg(feature = "sources-host_metrics")] metrics::MetricsQuery, - meta::MetaQuery, -); - -#[derive(MergedSubscription, Default)] -pub struct Subscription( - health::HealthSubscription, - metrics::MetricsSubscription, - components::ComponentsSubscription, - events::EventsSubscription, -); - -/// Build a new GraphQL schema, comprised of Query, Mutation and Subscription types -pub fn build_schema() -> SchemaBuilder { - Schema::build(Query::default(), EmptyMutation, Subscription::default()) -} diff --git a/src/api/schema/relay.rs b/src/api/schema/relay.rs deleted file mode 100644 index 3124a2272a66e..0000000000000 --- a/src/api/schema/relay.rs +++ /dev/null @@ -1,168 +0,0 @@ -use std::convert::Infallible; - -use async_graphql::{ - Result, SimpleObject, - connection::{self, Connection, CursorType, Edge, EmptyFields}, -}; -use base64::prelude::{BASE64_URL_SAFE_NO_PAD, Engine as _}; - -/// Base64 invalid states, used by `Base64Cursor`. -pub enum Base64CursorError { - /// Invalid cursor. This can happen if the base64 string is valid, but its contents don't - /// conform to the `name:index` pattern. - Invalid, - /// Decoding error. If this happens, the string isn't valid base64. - DecodeError(base64::DecodeError), -} - -impl std::fmt::Display for Base64CursorError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "Invalid cursor") - } -} - -/// Base64 cursor implementation -pub struct Base64Cursor { - name: &'static str, - index: usize, -} - -impl Base64Cursor { - const fn new(index: usize) -> Self { - Self { - name: "Cursor", - index, - } - } - - /// Returns a base64 string representation of the cursor - fn encode(&self) -> String { - BASE64_URL_SAFE_NO_PAD.encode(format!("{}:{}", self.name, self.index)) - } - - /// Decodes a base64 string into a cursor result - fn decode(s: &str) -> Result { - let bytes = BASE64_URL_SAFE_NO_PAD - .decode(s) - .map_err(Base64CursorError::DecodeError)?; - - let cursor = String::from_utf8(bytes).map_err(|_| Base64CursorError::Invalid)?; - let index = cursor - .split(':') - .next_back() - .map(|s| s.parse::()) - .ok_or(Base64CursorError::Invalid)? - .map_err(|_| Base64CursorError::Invalid)?; - - Ok(Self::new(index)) - } - - /// Increment and return the index. Uses saturating_add to avoid overflow - /// issues. - const fn increment(&self) -> usize { - self.index.saturating_add(1) - } -} - -impl From for usize { - fn from(cursor: Base64Cursor) -> Self { - cursor.index - } -} - -/// Makes the `Base64Cursor` compatible with Relay connections -impl CursorType for Base64Cursor { - type Error = Base64CursorError; - - fn decode_cursor(s: &str) -> Result { - Base64Cursor::decode(s) - } - - fn encode_cursor(&self) -> String { - self.encode() - } -} - -/// Additional fields to attach to the connection -#[derive(SimpleObject)] -pub struct ConnectionFields { - /// Total result set count - total_count: usize, -} - -/// Relay connection result -pub type ConnectionResult = Result>; - -/// Relay-compliant connection parameters to page results by cursor/page size -pub struct Params { - after: Option, - before: Option, - first: Option, - last: Option, -} - -impl Params { - pub const fn new( - after: Option, - before: Option, - first: Option, - last: Option, - ) -> Self { - Self { - after, - before, - first, - last, - } - } -} - -/// Creates a new Relay-compliant connection. Iterator must implement `ExactSizeIterator` to -/// determine page position in the total result set. -pub async fn query>( - iter: I, - p: Params, - default_page_size: usize, -) -> ConnectionResult { - connection::query::<_, _, Base64Cursor, _, _, ConnectionFields, _, _, _, Infallible>( - p.after, - p.before, - p.first, - p.last, - |after, before, first, last| async move { - let iter_len = iter.len(); - - let (start, end) = { - let after = after.map(|a| a.increment()).unwrap_or(0); - let before: usize = before.map(|b| b.into()).unwrap_or(iter_len); - - // Calculate start/end based on the provided first/last. Note that async-graphql disallows - // providing both (returning an error), so we can safely assume we have, at most, one of - // first or last. - match (first, last) { - // First - (Some(first), _) => (after, (after.saturating_add(first)).min(before)), - // Last - (_, Some(last)) => ((before.saturating_sub(last)).max(after), before), - // Default page size - _ => (after, default_page_size.min(before)), - } - }; - - let mut connection = Connection::with_additional_fields( - start > 0, - end < iter_len, - ConnectionFields { - total_count: iter_len, - }, - ); - connection.edges.extend( - (start..end) - .zip(iter.skip(start)) - .map(|(cursor, node)| Edge::new(Base64Cursor::new(cursor), node)), - ); - Ok(connection) - }, - ) - .await -} diff --git a/src/api/schema/sort.rs b/src/api/schema/sort.rs deleted file mode 100644 index 8e40750c88a83..0000000000000 --- a/src/api/schema/sort.rs +++ /dev/null @@ -1,61 +0,0 @@ -use std::cmp::Ordering; - -use async_graphql::{Enum, InputObject, InputType}; -use itertools::{ - FoldWhile::{Continue, Done}, - Itertools, -}; - -use crate::api::schema::{ - components::{ - ComponentsSortFieldName, sink::SinksSortFieldName, source::SourcesSortFieldName, - transform::TransformsSortFieldName, - }, - metrics::source::file::FileSourceMetricFilesSortFieldName, -}; - -#[derive(Enum, Copy, Clone, Default, PartialEq, Eq)] -pub enum Direction { - #[default] - Asc, - Desc, -} - -#[derive(InputObject)] -#[graphql(concrete(name = "ComponentsSortField", params(ComponentsSortFieldName)))] -#[graphql(concrete(name = "SourcesSortField", params(SourcesSortFieldName)))] -#[graphql(concrete(name = "TransformsSortField", params(TransformsSortFieldName)))] -#[graphql(concrete(name = "SinksSortField", params(SinksSortFieldName)))] -#[graphql(concrete( - name = "FileSourceMetricFilesSortField", - params(FileSourceMetricFilesSortFieldName) -))] -pub struct SortField { - pub field: T, - #[graphql(default_with = "Direction::default()")] - pub direction: Direction, -} - -/// Defines a type as sortable by a given field -pub trait SortableByField { - fn sort(&self, rhs: &Self, field: &T) -> Ordering; -} - -/// Performs an in-place sort against a slice of `Sortable`, with the provided [`SortField`]s -pub fn by_fields(f: &mut [impl SortableByField], sort_fields: &[SortField]) { - f.sort_by(|a, b| { - sort_fields - .iter() - .fold_while(Ordering::Equal, |cmp, f| match cmp { - Ordering::Equal => { - let cmp = a.sort(b, &f.field); - Continue(match f.direction { - Direction::Desc => cmp.reverse(), - _ => cmp, - }) - } - _ => Done(cmp), - }) - .into_inner() - }); -} diff --git a/src/api/server.rs b/src/api/server.rs deleted file mode 100644 index 9ddbc407c7f22..0000000000000 --- a/src/api/server.rs +++ /dev/null @@ -1,204 +0,0 @@ -use std::{ - convert::Infallible, - net::SocketAddr, - sync::{Arc, atomic::AtomicBool}, -}; - -use async_graphql::{ - Data, Request, Schema, - http::{GraphQLPlaygroundConfig, WebSocketProtocols, playground_source}, -}; -use async_graphql_warp::{GraphQLResponse, GraphQLWebSocket, graphql_protocol}; -use hyper::{Server as HyperServer, server::conn::AddrIncoming, service::make_service_fn}; -use tokio::{runtime::Handle, sync::oneshot}; -use tower::ServiceBuilder; -use tracing::Span; -use vector_lib::tap::topology; -use warp::{Filter, Reply, filters::BoxedFilter, http::Response, ws::Ws}; - -use super::{handler, schema}; -use crate::{ - config::{self, api}, - http::build_http_trace_layer, - internal_events::{SocketBindError, SocketMode}, -}; - -pub struct Server { - _shutdown: oneshot::Sender<()>, - addr: SocketAddr, -} - -impl Server { - /// Start the API server. This creates the routes and spawns a Warp server. The server is - /// gracefully shut down when Self falls out of scope by way of the oneshot sender closing. - pub fn start( - config: &config::Config, - watch_rx: topology::WatchRx, - running: Arc, - handle: &Handle, - ) -> crate::Result { - let routes = make_routes(config.api, watch_rx, running); - - let (_shutdown, rx) = oneshot::channel(); - // warp uses `tokio::spawn` and so needs us to enter the runtime context. - let _guard = handle.enter(); - - let addr = config.api.address.expect("No socket address"); - let incoming = AddrIncoming::bind(&addr).inspect_err(|error| { - emit!(SocketBindError { - mode: SocketMode::Tcp, - error, - }); - })?; - - let span = Span::current(); - let make_svc = make_service_fn(move |_conn| { - let svc = ServiceBuilder::new() - .layer(build_http_trace_layer(span.clone())) - .service(warp::service(routes.clone())); - futures_util::future::ok::<_, Infallible>(svc) - }); - - let server = async move { - HyperServer::builder(incoming) - .serve(make_svc) - .with_graceful_shutdown(async { - rx.await.ok(); - }) - .await - .map_err(|err| { - error!("An error occurred: {:?}.", err); - }) - }; - - // Update component schema with the config before starting the server. - schema::components::update_config(config); - - // Spawn the server in the background. - handle.spawn(server); - - Ok(Self { _shutdown, addr }) - } - - /// Returns a copy of the SocketAddr that the server was started on. - pub const fn addr(&self) -> SocketAddr { - self.addr - } - - /// Update the configuration of a running server. While this instance method doesn't - /// directly involve `self`, it provides a neater API to expose an internal implementation - /// detail than exposing the function of the sub-mod directly. - pub fn update_config(&self, config: &config::Config) { - schema::components::update_config(config) - } -} - -fn make_routes( - api: api::Options, - watch_tx: topology::WatchRx, - running: Arc, -) -> BoxedFilter<(impl Reply,)> { - // Routes... - - // Health. - let health = warp::path("health") - .and(with_shared(running)) - .and_then(handler::health); - - // 404. - let not_found_graphql = warp::any().and_then(|| async { Err(warp::reject::not_found()) }); - let not_found = warp::any().and_then(|| async { Err(warp::reject::not_found()) }); - - // GraphQL subscription handler. Creates a Warp WebSocket handler and for each connection, - // parses the required headers for GraphQL and builds per-connection context based on the - // provided `WatchTx` channel sender. This allows GraphQL resolvers to subscribe to - // topology changes. - let graphql_subscription_handler = - warp::ws() - .and(graphql_protocol()) - .map(move |ws: Ws, protocol: WebSocketProtocols| { - let schema = schema::build_schema().finish(); - let watch_tx = watch_tx.clone(); - - let reply = ws.on_upgrade(move |socket| { - let mut data = Data::default(); - data.insert(watch_tx); - - GraphQLWebSocket::new(socket, schema, protocol) - .with_data(data) - .serve() - }); - - warp::reply::with_header( - reply, - "Sec-WebSocket-Protocol", - protocol.sec_websocket_protocol(), - ) - }); - - // Handle GraphQL queries. Headers will first be parsed to determine whether the query is - // a subscription and if so, an attempt will be made to upgrade the connection to WebSockets. - // All other queries will fall back to the default HTTP handler. - let graphql_handler = if api.graphql { - warp::path("graphql") - .and(graphql_subscription_handler.or( - async_graphql_warp::graphql(schema::build_schema().finish()).and_then( - |(schema, request): (Schema<_, _, _>, Request)| async move { - Ok::<_, Infallible>(GraphQLResponse::from(schema.execute(request).await)) - }, - ), - )) - .boxed() - } else { - not_found_graphql.boxed() - }; - - // Provide a playground for executing GraphQL queries/mutations/subscriptions. - let graphql_playground = if api.playground && api.graphql { - warp::path("playground") - .map(move || { - Response::builder() - .header("content-type", "text/html") - .body(playground_source( - GraphQLPlaygroundConfig::new("/graphql").subscription_endpoint("/graphql"), - )) - }) - .boxed() - } else { - not_found.boxed() - }; - - // Wire up the health + GraphQL endpoints. Provides a permissive CORS policy to allow for - // cross-origin interaction with the Vector API. - health - .or(graphql_handler) - .or(graphql_playground) - .or(not_found) - .with( - warp::cors() - .allow_any_origin() - .allow_headers(vec![ - "User-Agent", - "Sec-Fetch-Mode", - "Referer", - "Origin", - "Access-Control-Request-Method", - "Access-Control-Allow-Origin", - "Access-Control-Request-Headers", - "Content-Type", - "X-Apollo-Tracing", // for Apollo GraphQL clients - "Pragma", - "Host", - "Connection", - "Cache-Control", - ]) - .allow_methods(vec!["POST", "GET"]), - ) - .boxed() -} - -fn with_shared( - shared: Arc, -) -> impl Filter,), Error = Infallible> + Clone { - warp::any().map(move || Arc::::clone(&shared)) -} diff --git a/src/api/tests.rs b/src/api/tests.rs deleted file mode 100644 index 56794376b2a25..0000000000000 --- a/src/api/tests.rs +++ /dev/null @@ -1,589 +0,0 @@ -use std::{ - collections::{HashMap, HashSet}, - time::Duration, -}; - -use futures::StreamExt; -use tokio::sync::{mpsc, watch}; -use vector_lib::{ - config::ComponentKey, - fanout, - tap::{ - controller::{TapController, TapPatterns, TapPayload}, - notification::{InvalidMatch, Matched, NotMatched, Notification}, - topology::{TapOutput, TapResource}, - }, -}; - -use crate::{ - api::schema::events::{create_events_stream, log, metric, output::OutputEventsPayload}, - config::{Config, OutputId}, - event::{LogEvent, Metric, MetricKind, MetricValue}, - sinks::blackhole::BlackholeConfig, - sources::demo_logs::{DemoLogsConfig, OutputFormat}, - test_util::{start_topology, trace_init}, - transforms::{ - log_to_metric::{LogToMetricConfig, MetricConfig, MetricTypeConfig}, - remap::RemapConfig, - }, -}; - -#[tokio::test] -/// A tap sink should match a pattern, receive the correct notifications, -/// and receive events -async fn sink_events() { - let pattern_matched = "tes*"; - let pattern_not_matched = "xyz"; - let id = OutputId::from(&ComponentKey::from("test")); - - let (mut fanout, control_tx) = fanout::Fanout::new(); - let mut outputs = HashMap::new(); - outputs.insert( - TapOutput { - output_id: id.clone(), - component_kind: "source", - component_type: "demo".to_string(), - }, - control_tx, - ); - let tap_resource = TapResource { - outputs, - inputs: HashMap::new(), - source_keys: Vec::new(), - sink_keys: Vec::new(), - removals: HashSet::new(), - }; - - let (watch_tx, watch_rx) = watch::channel(TapResource::default()); - let (sink_tx, mut sink_rx) = mpsc::channel(10); - - let _controller = TapController::new( - watch_rx, - sink_tx, - TapPatterns::new( - HashSet::from([pattern_matched.to_string(), pattern_not_matched.to_string()]), - HashSet::new(), - ), - ); - - // Add the outputs to trigger a change event. - watch_tx.send(tap_resource).unwrap(); - - // First two events should contain a notification that one pattern matched, and - // one that didn't. - #[allow(clippy::mixed_read_write_in_expression)] - let notifications = vec![sink_rx.recv().await, sink_rx.recv().await]; - - for notification in notifications.into_iter() { - match notification { - Some(TapPayload::Notification(Notification::Matched(matched))) - if matched.pattern == pattern_matched => - { - continue; - } - Some(TapPayload::Notification(Notification::NotMatched(not_matched))) - if not_matched.pattern == pattern_not_matched => - { - continue; - } - _ => panic!("unexpected payload"), - } - } - - // Send some events down the wire. Waiting until the first notifications are in - // to ensure the event handler has been initialized. - let log_event = LogEvent::default(); - let metric_event = Metric::new( - id.to_string(), - MetricKind::Incremental, - MetricValue::Counter { value: 1.0 }, - ); - - fanout - .send(vec![metric_event].into(), None) - .await - .expect("should not fail"); - fanout - .send(vec![log_event].into(), None) - .await - .expect("should not fail"); - - // 3rd payload should be the metric event - assert!(matches!( - sink_rx.recv().await, - Some(TapPayload::Metric(output, _)) if output.output_id == id - )); - - // 4th payload should be the log event - assert!(matches!( - sink_rx.recv().await, - Some(TapPayload::Log(output, _)) if output.output_id == id - )); -} - -fn assert_notification(payload: OutputEventsPayload) -> Notification { - if let OutputEventsPayload::Notification(event_notification) = payload { - event_notification.notification - } else { - panic!("Expected payload to be a Notification") - } -} - -fn assert_log(payload: OutputEventsPayload) -> log::Log { - if let OutputEventsPayload::Log(log) = payload { - log - } else { - panic!("Expected payload to be a Log") - } -} - -fn assert_metric(payload: OutputEventsPayload) -> metric::Metric { - if let OutputEventsPayload::Metric(metric) = payload { - metric - } else { - panic!("Expected payload to be a Metric") - } -} - -#[tokio::test] -async fn integration_test_source_log() { - trace_init(); - - let mut config = Config::builder(); - config.add_source( - "in", - DemoLogsConfig { - interval: Duration::from_secs_f64(0.01), - count: 200, - format: OutputFormat::Json, - ..Default::default() - }, - ); - config.add_sink( - "out", - &["in"], - BlackholeConfig { - print_interval_secs: Duration::from_secs(1), - rate: None, - acknowledgements: Default::default(), - }, - ); - - let (topology, _) = start_topology(config.build().unwrap(), false).await; - - let source_tap_stream = create_events_stream( - topology.watch(), - TapPatterns::new(HashSet::from(["in".to_string()]), HashSet::new()), - 500, - 100, - ); - - let source_tap_events: Vec<_> = source_tap_stream.take(2).collect().await; - - assert_eq!( - assert_notification(source_tap_events[0][0].clone()), - Notification::Matched(Matched::new("in".to_string())) - ); - let _log = assert_log(source_tap_events[1][0].clone()); -} - -#[tokio::test] -async fn integration_test_source_metric() { - trace_init(); - - let mut config = Config::builder(); - config.add_source( - "in", - DemoLogsConfig { - interval: Duration::from_secs_f64(0.01), - count: 200, - format: OutputFormat::Shuffle { - sequence: false, - lines: vec!["1".to_string()], - }, - ..Default::default() - }, - ); - config.add_transform( - "to_metric", - &["in"], - LogToMetricConfig { - metrics: Some(vec![MetricConfig { - field: "message".try_into().expect("Fixed template string"), - name: None, - namespace: None, - tags: None, - metric: MetricTypeConfig::Gauge, - }]), - all_metrics: None, - }, - ); - config.add_sink( - "out", - &["to_metric"], - BlackholeConfig { - print_interval_secs: Duration::from_secs(1), - rate: None, - acknowledgements: Default::default(), - }, - ); - - let (topology, _) = start_topology(config.build().unwrap(), false).await; - - let source_tap_stream = create_events_stream( - topology.watch(), - TapPatterns::new(HashSet::from(["to_metric".to_string()]), HashSet::new()), - 500, - 100, - ); - - let source_tap_events: Vec<_> = source_tap_stream.take(2).collect().await; - - assert_eq!( - assert_notification(source_tap_events[0][0].clone()), - Notification::Matched(Matched::new("to_metric".to_string())) - ); - assert_metric(source_tap_events[1][0].clone()); -} - -#[tokio::test] -async fn integration_test_transform() { - trace_init(); - - let mut config = Config::builder(); - config.add_source( - "in", - DemoLogsConfig { - interval: Duration::from_secs_f64(0.01), - count: 200, - format: OutputFormat::Json, - ..Default::default() - }, - ); - config.add_transform( - "transform", - &["in"], - RemapConfig { - source: Some("".to_string()), - ..Default::default() - }, - ); - config.add_sink( - "out", - &["transform"], - BlackholeConfig { - print_interval_secs: Duration::from_secs(1), - rate: None, - acknowledgements: Default::default(), - }, - ); - - let (topology, _) = start_topology(config.build().unwrap(), false).await; - - let transform_tap_stream = create_events_stream( - topology.watch(), - TapPatterns::new(HashSet::from(["transform".to_string()]), HashSet::new()), - 500, - 100, - ); - - let transform_tap_events: Vec<_> = transform_tap_stream.take(2).collect().await; - - assert_eq!( - assert_notification(transform_tap_events[0][0].clone()), - Notification::Matched(Matched::new("transform".to_string())) - ); - let _log = assert_log(transform_tap_events[1][0].clone()); -} - -#[tokio::test] -async fn integration_test_transform_input() { - trace_init(); - - let mut config = Config::builder(); - config.add_source( - "in", - DemoLogsConfig { - interval: Duration::from_secs_f64(0.01), - count: 200, - format: OutputFormat::Shuffle { - sequence: false, - lines: vec!["test".to_string()], - }, - ..Default::default() - }, - ); - config.add_transform( - "transform", - &["in"], - RemapConfig { - source: Some(".message = \"new message\"".to_string()), - ..Default::default() - }, - ); - config.add_sink( - "out", - &["in"], - BlackholeConfig { - print_interval_secs: Duration::from_secs(1), - rate: None, - acknowledgements: Default::default(), - }, - ); - - let (topology, _) = start_topology(config.build().unwrap(), false).await; - - let tap_stream = create_events_stream( - topology.watch(), - TapPatterns::new( - HashSet::new(), - HashSet::from(["transform".to_string(), "in".to_string()]), - ), - 500, - 100, - ); - - let tap_events: Vec<_> = tap_stream.take(4).collect().await; - - let notifications = [ - assert_notification(tap_events[0][0].clone()), - assert_notification(tap_events[1][0].clone()), - assert_notification(tap_events[2][0].clone()), - ]; - assert!( - notifications - .iter() - .any(|n| *n == Notification::Matched(Matched::new("transform".to_string()))) - ); - // "in" is not matched since it corresponds to a source - assert!( - notifications - .iter() - .any(|n| *n == Notification::NotMatched(NotMatched::new("in".to_string()))) - ); - // "in" generates an invalid match notification to warn against an - // attempt to tap the input of a source - assert!(notifications.iter().any(|n| *n - == Notification::InvalidMatch(InvalidMatch::new("[tap] Warning: source inputs cannot be tapped. Input pattern 'in' matches sources [\"in\"]".to_string(), "in".to_string(), vec!["in".to_string()])))); - - assert_eq!( - assert_log(tap_events[3][0].clone()) - .get_message() - .unwrap_or_default(), - "test" - ); -} - -#[tokio::test] -async fn integration_test_sink() { - trace_init(); - - let mut config = Config::builder(); - config.add_source( - "in", - DemoLogsConfig { - interval: Duration::from_secs_f64(0.01), - count: 200, - format: OutputFormat::Shuffle { - sequence: false, - lines: vec!["test".to_string()], - }, - ..Default::default() - }, - ); - config.add_transform( - "transform", - &["in"], - RemapConfig { - source: Some(".message = \"new message\"".to_string()), - ..Default::default() - }, - ); - config.add_sink( - "out", - &["transform"], - BlackholeConfig { - print_interval_secs: Duration::from_secs(1), - rate: None, - acknowledgements: Default::default(), - }, - ); - - let (topology, _) = start_topology(config.build().unwrap(), false).await; - - let tap_stream = create_events_stream( - topology.watch(), - TapPatterns::new(HashSet::new(), HashSet::from(["out".to_string()])), - 500, - 100, - ); - - let tap_events: Vec<_> = tap_stream.take(2).collect().await; - - assert_eq!( - assert_notification(tap_events[0][0].clone()), - Notification::Matched(Matched::new("out".to_string())) - ); - assert_eq!( - assert_log(tap_events[1][0].clone()) - .get_message() - .unwrap_or_default(), - "new message" - ); -} - -#[tokio::test] -async fn integration_test_tap_non_default_output() { - trace_init(); - - let mut config = Config::builder(); - config.add_source( - "in", - DemoLogsConfig { - interval: Duration::from_secs_f64(0.01), - count: 200, - format: OutputFormat::Shuffle { - sequence: false, - lines: vec!["test2".to_string()], - }, - ..Default::default() - }, - ); - config.add_transform( - "transform", - &["in"], - RemapConfig { - source: Some("assert_eq!(.message, \"test1\")".to_string()), - drop_on_error: true, - reroute_dropped: true, - ..Default::default() - }, - ); - config.add_sink( - "out", - &["transform"], - BlackholeConfig { - print_interval_secs: Duration::from_secs(1), - rate: None, - acknowledgements: Default::default(), - }, - ); - - let (topology, _) = start_topology(config.build().unwrap(), false).await; - - let transform_tap_remap_dropped_stream = create_events_stream( - topology.watch(), - TapPatterns::new( - HashSet::from(["transform.dropped".to_string()]), - HashSet::new(), - ), - 500, - 100, - ); - - let transform_tap_events: Vec<_> = transform_tap_remap_dropped_stream.take(2).collect().await; - - assert_eq!( - assert_notification(transform_tap_events[0][0].clone()), - Notification::Matched(Matched::new("transform.dropped".to_string())) - ); - assert_eq!( - assert_log(transform_tap_events[1][0].clone()) - .get_message() - .unwrap_or_default(), - "test2" - ); -} - -#[tokio::test] -async fn integration_test_tap_multiple_outputs() { - trace_init(); - - let mut config = Config::builder(); - config.add_source( - "in-test1", - DemoLogsConfig { - interval: Duration::from_secs_f64(0.01), - count: 1, - format: OutputFormat::Shuffle { - sequence: false, - lines: vec!["test1".to_string()], - }, - ..Default::default() - }, - ); - config.add_source( - "in-test2", - DemoLogsConfig { - interval: Duration::from_secs_f64(0.01), - count: 1, - format: OutputFormat::Shuffle { - sequence: false, - lines: vec!["test2".to_string()], - }, - ..Default::default() - }, - ); - config.add_transform( - "transform", - &["in*"], - RemapConfig { - source: Some("assert_eq!(.message, \"test1\")".to_string()), - drop_on_error: true, - reroute_dropped: true, - ..Default::default() - }, - ); - config.add_sink( - "out", - &["transform"], - BlackholeConfig { - print_interval_secs: Duration::from_secs(1), - rate: None, - acknowledgements: Default::default(), - }, - ); - - let (topology, _) = start_topology(config.build().unwrap(), false).await; - - let mut transform_tap_all_outputs_stream = create_events_stream( - topology.watch(), - TapPatterns::new(HashSet::from(["transform*".to_string()]), HashSet::new()), - 500, - 100, - ); - - let transform_tap_notifications = transform_tap_all_outputs_stream.next().await.unwrap(); - assert_eq!( - assert_notification(transform_tap_notifications[0].clone()), - Notification::Matched(Matched::new("transform*".to_string())) - ); - - let mut default_output_found = false; - let mut dropped_output_found = false; - for _ in 0..2 { - if default_output_found && dropped_output_found { - break; - } - - match transform_tap_all_outputs_stream.next().await { - Some(tap_events) => { - if !default_output_found { - default_output_found = tap_events - .iter() - .map(|payload| assert_log(payload.clone())) - .any(|log| log.get_message().unwrap_or_default() == "test1"); - } - if !dropped_output_found { - dropped_output_found = tap_events - .iter() - .map(|payload| assert_log(payload.clone())) - .any(|log| log.get_message().unwrap_or_default() == "test2"); - } - } - None => break, - } - } - - assert!(default_output_found && dropped_output_found); -} diff --git a/src/app.rs b/src/app.rs index 9e8a415608a8f..6f30f85de1d20 100644 --- a/src/app.rs +++ b/src/app.rs @@ -20,7 +20,9 @@ use tokio::{ use tokio_stream::wrappers::UnboundedReceiverStream; #[cfg(feature = "api")] -use crate::{api, internal_events::ApiStarted}; +use crate::api; +#[cfg(feature = "api")] +use crate::internal_events::ApiStarted; use crate::{ cli::{LogFormat, Opts, RootOpts, WatchConfigMethod, handle_config_errors}, config::{self, ComponentConfig, ComponentType, Config, ConfigPath}, @@ -34,6 +36,8 @@ use crate::{ }, trace, }; +#[cfg(feature = "api")] +use std::sync::Arc; static WORKER_THREADS: AtomicUsize = AtomicUsize::new(0); @@ -129,28 +133,32 @@ impl ApplicationConfig { Ok(()) } - /// Configure the API server, if applicable + /// Configure the gRPC API server, if applicable #[cfg(feature = "api")] - pub fn setup_api(&self, handle: &Handle) -> Option { + pub fn setup_api(&self, handle: &Handle) -> Option { if self.api.enabled { - match api::Server::start( + // Start gRPC server + let api_server = handle.block_on(api::GrpcServer::start( self.topology.config(), self.topology.watch(), - std::sync::Arc::clone(&self.topology.running), - handle, - ) { - Ok(api_server) => { + Arc::clone(&self.topology.running), + )); + match api_server { + Ok(server) => { emit!(ApiStarted { - addr: self.api.address.unwrap(), - playground: self.api.playground, - graphql: self.api.graphql + addr: server.addr() }); - - Some(api_server) + Some(server) } Err(error) => { let error = error.to_string(); - error!(message = "An error occurred that Vector couldn't handle.", %error, internal_log_rate_limit = false); + error!( + message = "An error occurred that Vector couldn't handle.", + %error, + internal_log_rate_limit = false + ); + // Trigger shutdown because the API was explicitly enabled but failed to start + // This ensures users don't run Vector thinking top/tap will work when they won't _ = self .topology .abort_tx @@ -256,9 +264,12 @@ impl Application { signals, } = self; + #[cfg(feature = "api")] + let api_server = config.setup_api(handle); + let topology_controller = SharedTopologyController::new(TopologyController { #[cfg(feature = "api")] - api_server: config.setup_api(handle), + api_server, topology: config.topology, config_paths: config.config_paths.clone(), require_healthy: root_opts.require_healthy, diff --git a/src/config/api.rs b/src/config/api.rs index 5f99618c166f6..a421fe9f2e1b6 100644 --- a/src/config/api.rs +++ b/src/config/api.rs @@ -11,7 +11,7 @@ use vector_lib::configurable::configurable_component; #[derive(Clone, Copy, Debug, Eq, PartialEq)] #[serde(default, deny_unknown_fields)] pub struct Options { - /// Whether the GraphQL API is enabled for this Vector instance. + /// Whether the API is enabled for this Vector instance. #[serde(default = "default_enabled")] #[configurable(metadata(docs::common = true, docs::required = false))] pub enabled: bool, @@ -24,21 +24,6 @@ pub struct Options { #[configurable(metadata(docs::examples = "127.0.0.1:1234"))] #[configurable(metadata(docs::common = true, docs::required = false))] pub address: Option, - - /// Whether the [GraphQL Playground](https://github.com/graphql/graphql-playground) is enabled - /// for the API. The Playground is accessible via the `/playground` endpoint - /// of the address set using the `bind` parameter. Note that the `playground` - /// endpoint will only be enabled if the `graphql` endpoint is also enabled. - #[serde(default = "default_playground")] - #[configurable(metadata(docs::common = false, docs::required = false))] - pub playground: bool, - - /// Whether the endpoint for receiving and processing GraphQL queries is - /// enabled for the API. The endpoint is accessible via the `/graphql` - /// endpoint of the address set using the `bind` parameter. - #[serde(default = "default_graphql", skip_serializing_if = "is_true")] - #[configurable(metadata(docs::common = true, docs::required = false))] - pub graphql: bool, } impl_generate_config_from_default!(Options); @@ -47,19 +32,11 @@ impl Default for Options { fn default() -> Self { Self { enabled: default_enabled(), - playground: default_playground(), address: default_address(), - graphql: default_graphql(), } } } -// serde passes struct fields as reference -#[allow(clippy::trivially_copy_pass_by_ref)] -const fn is_true(value: &bool) -> bool { - *value -} - const fn default_enabled() -> bool { false } @@ -71,21 +48,13 @@ pub fn default_address() -> Option { Some(SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 8686)) } -/// Default GraphQL API address -pub fn default_graphql_url() -> Url { +/// Default gRPC API address for `vector top` and other API clients +pub fn default_grpc_url() -> Url { let addr = default_address().unwrap(); - Url::parse(&format!("http://{addr}/graphql")) + Url::parse(&format!("http://{addr}")) .expect("Couldn't parse default API URL. Please report this.") } -const fn default_playground() -> bool { - true -} - -const fn default_graphql() -> bool { - true -} - impl Options { pub fn merge(&mut self, other: Self) -> Result<(), String> { // Merge options @@ -108,8 +77,6 @@ impl Options { let options = Options { address, enabled: self.enabled | other.enabled, - playground: self.playground & other.playground, - graphql: self.graphql & other.graphql, }; *self = options; @@ -122,8 +89,6 @@ fn bool_merge() { let mut a = Options { enabled: true, address: None, - playground: false, - graphql: false, }; a.merge(Options::default()).unwrap(); @@ -133,8 +98,6 @@ fn bool_merge() { Options { enabled: true, address: default_address(), - playground: false, - graphql: false } ); } @@ -145,8 +108,6 @@ fn bind_merge() { let mut a = Options { enabled: true, address: Some(address), - playground: true, - graphql: true, }; a.merge(Options::default()).unwrap(); @@ -156,8 +117,6 @@ fn bind_merge() { Options { enabled: true, address: Some(address), - playground: true, - graphql: true, } ); } diff --git a/src/internal_events/api.rs b/src/internal_events/api.rs index 4894fc1de33e6..de2cb9588537d 100644 --- a/src/internal_events/api.rs +++ b/src/internal_events/api.rs @@ -7,20 +7,13 @@ use vector_lib::internal_event::InternalEvent; #[derive(Debug, NamedInternalEvent)] pub struct ApiStarted { pub addr: SocketAddr, - pub playground: bool, - pub graphql: bool, } impl InternalEvent for ApiStarted { fn emit(self) { - let playground = &*format!("http://{}:{}/playground", self.addr.ip(), self.addr.port()); - let graphql = &*format!("http://{}:{}/graphql", self.addr.ip(), self.addr.port()); info!( - message="API server running.", - address = ?self.addr, - playground = %if self.playground { playground } else { "off" }, - graphql = %if self.graphql { graphql } else { "off" } - + message = "API server running.", + address = %self.addr, ); counter!("api_started_total").increment(1); } diff --git a/src/proto/mod.rs b/src/proto/mod.rs index bf576c8f7e542..75796939fb368 100644 --- a/src/proto/mod.rs +++ b/src/proto/mod.rs @@ -1,9 +1,9 @@ -#[cfg(any(feature = "sources-vector", feature = "sinks-vector"))] -use crate::event::proto as event; - #[cfg(any(feature = "sources-vector", feature = "sinks-vector"))] pub mod vector; +#[cfg(feature = "api")] +pub mod observability; + #[cfg(feature = "sinks-datadog_metrics")] pub mod fds { use std::sync::OnceLock; diff --git a/src/proto/observability.rs b/src/proto/observability.rs new file mode 100644 index 0000000000000..f62f9adec9f97 --- /dev/null +++ b/src/proto/observability.rs @@ -0,0 +1,13 @@ +#![allow(clippy::clone_on_ref_ptr)] +#![allow(warnings, clippy::pedantic, clippy::nursery)] + +tonic::include_proto!("vector.observability.v1"); + +pub use observability_service_client::ObservabilityServiceClient as Client; +pub use observability_service_server::{ + ObservabilityService as Service, ObservabilityServiceServer as Server, +}; + +/// File descriptor set for gRPC reflection +pub const FILE_DESCRIPTOR_SET: &[u8] = + include_bytes!(concat!(env!("OUT_DIR"), "/protobuf-fds.bin")); diff --git a/src/tap/cmd.rs b/src/tap/cmd.rs index 46fc21f6b5684..fe2b69bbeb8b5 100644 --- a/src/tap/cmd.rs +++ b/src/tap/cmd.rs @@ -1,24 +1,24 @@ use std::time::Duration; use vector_lib::{ - api_client::Client, + api_client::{Client, RECONNECT_DELAY_MS}, tap::{EventFormatter, OutputChannel, TapRunner}, }; use crate::signal::{SignalRx, SignalTo}; -/// Delay (in milliseconds) before attempting to reconnect to the Vector API -const RECONNECT_DELAY: u64 = 5000; - /// CLI command func for issuing 'tap' queries, and communicating with a local/remote /// Vector API server via HTTP/WebSockets. +#[allow(clippy::print_stderr)] pub(crate) async fn cmd(opts: &super::Opts, signal_rx: SignalRx) -> exitcode::ExitCode { let url = opts.url(); - // Return early with instructions for enabling the API if the endpoint isn't reachable - // via a healthcheck. - let client = Client::new(url.clone()); - #[allow(clippy::print_stderr)] - if client.healthcheck().await.is_err() { + let Ok(uri) = url.as_str().parse() else { + eprintln!("Invalid API URL: {url}"); + return exitcode::USAGE; + }; + let mut client = Client::new(uri); + + if client.connect().await.is_err() || client.health().await.is_err() { eprintln!( indoc::indoc! {" Vector API server isn't reachable ({}). @@ -34,47 +34,66 @@ pub(crate) async fn cmd(opts: &super::Opts, signal_rx: SignalRx) -> exitcode::Ex return exitcode::UNAVAILABLE; } - tap(opts, signal_rx).await + tap_internal(opts, signal_rx, Some(client)).await } /// Observe event flow from specified components -pub async fn tap(opts: &super::Opts, mut signal_rx: SignalRx) -> exitcode::ExitCode { - let subscription_url = opts.web_socket_url(); +pub async fn tap(opts: &super::Opts, signal_rx: SignalRx) -> exitcode::ExitCode { + tap_internal(opts, signal_rx, None).await +} + +async fn tap_internal( + opts: &super::Opts, + mut signal_rx: SignalRx, + mut client_opt: Option, +) -> exitcode::ExitCode { + let url = opts.url(); let output_channel = OutputChannel::Stdout(EventFormatter::new(opts.meta, opts.format)); let tap_runner = TapRunner::new( - &subscription_url, + &url, opts.inputs_of.clone(), opts.outputs_patterns().clone(), &output_channel, - opts.format, ); loop { tokio::select! { biased; Ok(SignalTo::Shutdown(_) | SignalTo::Quit) = signal_rx.recv() => break, - exec_result = tap_runner.run_tap( - opts.interval as i64, - opts.limit as i64, - opts.duration_ms, - opts.quiet, - ) => { + exec_result = async { + if let Some(client) = client_opt.take() { + tap_runner.run_tap_with_client( + client, + opts.interval as i64, + opts.limit as i64, + opts.duration_ms, + opts.quiet, + ).await + } else { + tap_runner.run_tap( + opts.interval as i64, + opts.limit as i64, + opts.duration_ms, + opts.quiet, + ).await + } + } => { match exec_result { Ok(_) => { break; } Err(tap_executor_error) => { - if !opts.no_reconnect { - #[allow(clippy::print_stderr)] - { - eprintln!( - "[tap] Connection failed with error {:?}. Reconnecting in {:?} seconds.", - tap_executor_error, - RECONNECT_DELAY / 1000); - } - tokio::time::sleep(Duration::from_millis(RECONNECT_DELAY)).await; - } - else { + #[allow(clippy::print_stderr)] + if tap_executor_error.is_fatal() { + eprintln!("[tap] Error: {tap_executor_error:?}"); + break; + } else if !opts.no_reconnect { + eprintln!( + "[tap] Connection failed with error {:?}. Reconnecting in {:?} seconds.", + tap_executor_error, + RECONNECT_DELAY_MS / 1000); + tokio::time::sleep(Duration::from_millis(RECONNECT_DELAY_MS)).await; + } else { break; } } diff --git a/src/tap/mod.rs b/src/tap/mod.rs index 9ba77f3ce1a65..2e5fa1bbd4d3d 100644 --- a/src/tap/mod.rs +++ b/src/tap/mod.rs @@ -5,24 +5,24 @@ use clap::Parser; pub(crate) use cmd::cmd; pub use cmd::tap; use url::Url; -use vector_lib::api_client::gql::TapEncodingFormat; +use vector_lib::tap::TapEncodingFormat; -use crate::config::api::default_graphql_url; +use crate::config::api::default_grpc_url; /// Tap options #[derive(Parser, Debug, Clone)] #[command(rename_all = "kebab-case")] pub struct Opts { - /// Interval to sample events at, in milliseconds - #[arg(default_value = "500", short = 'i', long)] + /// Interval to sample events at, in milliseconds (must be >= 100) + #[arg(default_value = "500", short = 'i', long, value_parser = clap::value_parser!(u32).range(100..))] interval: u32, - /// GraphQL API server endpoint + /// gRPC API server endpoint (e.g. http://localhost:8686) #[arg(short, long)] url: Option, - /// Maximum number of events to sample each interval - #[arg(default_value = "100", short = 'l', long)] + /// Maximum number of events to sample each interval (must be >= 1) + #[arg(default_value = "100", short = 'l', long, value_parser = clap::value_parser!(u32).range(1..))] limit: u32, /// Encoding format for events printed to screen @@ -77,21 +77,9 @@ impl Opts { } } - /// Use the provided URL as the Vector GraphQL API server, or default to the local port + /// Use the provided URL as the Vector gRPC API server, or default to the local port /// provided by the API config. pub fn url(&self) -> Url { - self.url.clone().unwrap_or_else(default_graphql_url) - } - - /// URL with scheme set to WebSockets - pub fn web_socket_url(&self) -> Url { - let mut url = self.url(); - url.set_scheme(match url.scheme() { - "https" => "wss", - _ => "ws", - }) - .expect("Couldn't build WebSocket URL. Please report."); - - url + self.url.clone().unwrap_or_else(default_grpc_url) } } diff --git a/src/top/cmd.rs b/src/top/cmd.rs index 6410063781549..e411712261920 100644 --- a/src/top/cmd.rs +++ b/src/top/cmd.rs @@ -3,9 +3,10 @@ use std::time::Duration; use chrono::Local; use futures_util::future::join_all; +use http::Uri; use regex::Regex; use tokio::sync::{mpsc, oneshot}; -use vector_lib::api_client::{Client, connect_subscription_client}; +use vector_lib::api_client::{Client, RECONNECT_DELAY_MS}; use vector_lib::top::{ dashboard::{init_dashboard, is_tty}, @@ -13,26 +14,26 @@ use vector_lib::top::{ state::{self, ConnectionStatus, EventType, State}, }; -/// Delay (in milliseconds) before attempting to reconnect to the Vector API -const RECONNECT_DELAY: u64 = 5000; - /// CLI command func for displaying Vector components, and communicating with a local/remote -/// Vector API server via HTTP/WebSockets +/// Vector API server via gRPC +#[allow(clippy::print_stderr)] pub async fn cmd(opts: &super::Opts) -> exitcode::ExitCode { // Exit early if the terminal is not a teletype if !is_tty() { - #[allow(clippy::print_stderr)] - { - eprintln!("Terminal must be a teletype (TTY) to display a Vector dashboard."); - } + eprintln!("Terminal must be a teletype (TTY) to display a Vector dashboard."); return exitcode::IOERR; } let url = opts.url(); + // Create a new API client for connecting to the local/remote Vector instance. - let client = Client::new(url.clone()); - #[allow(clippy::print_stderr)] - if client.healthcheck().await.is_err() { + let Ok(uri) = url.as_str().parse::() else { + eprintln!("Invalid API URL: {url}"); + return exitcode::USAGE; + }; + let mut client = Client::new(uri.clone()); + + if client.connect().await.is_err() || client.health().await.is_err() { eprintln!( indoc::indoc! {" Vector API server isn't reachable ({}). @@ -48,11 +49,11 @@ pub async fn cmd(opts: &super::Opts) -> exitcode::ExitCode { return exitcode::UNAVAILABLE; } - top(opts, client, "Vector").await + top(opts, uri, "Vector").await } /// General monitoring -pub async fn top(opts: &super::Opts, client: Client, dashboard_title: &str) -> exitcode::ExitCode { +pub async fn top(opts: &super::Opts, uri: Uri, dashboard_title: &str) -> exitcode::ExitCode { // Channel for updating state via event messages let (tx, rx) = tokio::sync::mpsc::channel(20); let mut starting_state = State::new(BTreeMap::new()); @@ -68,7 +69,7 @@ pub async fn top(opts: &super::Opts, client: Client, dashboard_title: &str) -> e // Channel for shutdown signal let (shutdown_tx, shutdown_rx) = oneshot::channel::<()>(); - let connection = tokio::spawn(subscription(opts.clone(), client, tx.clone(), shutdown_tx)); + let connection = tokio::spawn(subscription(opts.clone(), uri, tx.clone(), shutdown_tx)); // Initialize the dashboard match init_dashboard( @@ -97,60 +98,67 @@ pub async fn top(opts: &super::Opts, client: Client, dashboard_title: &str) -> e } } -// This task handles reconnecting the subscription client and all -// subscriptions in the case of a web socket disconnect +// This task handles reconnecting the gRPC client and all +// subscriptions in the case of a connection failure async fn subscription( opts: super::Opts, - client: Client, + uri: Uri, tx: mpsc::Sender, shutdown_tx: oneshot::Sender<()>, ) { - let ws_url = opts.web_socket_url(); - loop { // Initialize state. On future reconnects, we re-initialize state in // order to accurately capture added, removed, and edited // components. - let state = match metrics::init_components(&client, &opts.components).await { + let state = match metrics::init_components(uri.clone(), &opts.components).await { Ok(state) => state, Err(_) => { - tokio::time::sleep(Duration::from_millis(RECONNECT_DELAY)).await; + tokio::time::sleep(Duration::from_millis(RECONNECT_DELAY_MS)).await; continue; } }; + let initial_components = state + .components + .keys() + .map(|k| k.id().to_string()) + .collect(); _ = tx.send(EventType::InitializeState(state)).await; - let subscription_client = match connect_subscription_client(ws_url.clone()).await { - Ok(c) => c, + // Subscribe to updated metrics via gRPC streaming + let handles = match metrics::subscribe( + uri.clone(), + tx.clone(), + opts.interval as i64, + opts.components.clone(), + initial_components, + ) + .await + { + Ok(handles) => handles, Err(_) => { - tokio::time::sleep(Duration::from_millis(RECONNECT_DELAY)).await; + tokio::time::sleep(Duration::from_millis(RECONNECT_DELAY_MS)).await; continue; } }; - // Subscribe to updated metrics - let finished = metrics::subscribe( - subscription_client, - tx.clone(), - opts.interval as i64, - opts.components.clone(), - ); - _ = tx .send(EventType::ConnectionUpdated(ConnectionStatus::Connected( Local::now(), ))) .await; - // Tasks spawned in metrics::subscribe finish when the subscription - // streams have completed. Currently, subscription streams only - // complete when the underlying web socket connection to the GraphQL - // server drops. - _ = join_all(finished).await; + + // Wait for metric stream tasks to finish. poll_components is intentionally + // excluded: it runs indefinitely while get_components succeeds, so joining + // it here would prevent reconnection when metric streams fail first. + _ = join_all(handles.metric_handles).await; + handles.poll_handle.abort(); + _ = tx .send(EventType::ConnectionUpdated( - ConnectionStatus::Disconnected(RECONNECT_DELAY), + ConnectionStatus::Disconnected(RECONNECT_DELAY_MS), )) .await; + if opts.no_reconnect { _ = shutdown_tx.send(()); break; diff --git a/src/top/mod.rs b/src/top/mod.rs index 5fc101f7b13c6..c185e3193a64d 100644 --- a/src/top/mod.rs +++ b/src/top/mod.rs @@ -8,17 +8,17 @@ pub use cmd::{cmd, top}; use url::Url; use vector_lib::top::state::{FilterColumn, SortColumn}; -use crate::config::api::default_graphql_url; +use crate::config::api::default_grpc_url; /// Top options #[derive(Parser, Debug, Clone)] #[command(rename_all = "kebab-case")] pub struct Opts { - /// Interval to sample metrics at, in milliseconds - #[arg(default_value = "1000", short = 'i', long)] + /// Interval to sample metrics at, in milliseconds (must be >= 100) + #[arg(default_value = "1000", short = 'i', long, value_parser = clap::value_parser!(u32).range(100..))] interval: u32, - /// GraphQL API server endpoint + /// gRPC API server endpoint (e.g. http://localhost:8686) #[arg(short, long)] url: Option, @@ -56,21 +56,9 @@ pub struct Opts { } impl Opts { - /// Use the provided URL as the Vector GraphQL API server, or default to the local port + /// Use the provided URL as the Vector gRPC API server, or default to the local port /// provided by the API config. pub fn url(&self) -> Url { - self.url.clone().unwrap_or_else(default_graphql_url) - } - - /// URL with scheme set to WebSockets - pub fn web_socket_url(&self) -> Url { - let mut url = self.url(); - url.set_scheme(match url.scheme() { - "https" => "wss", - _ => "ws", - }) - .expect("Couldn't build WebSocket URL. Please report."); - - url + self.url.clone().unwrap_or_else(default_grpc_url) } } diff --git a/src/topology/controller.rs b/src/topology/controller.rs index 53acc243a71f8..b77629828a577 100644 --- a/src/topology/controller.rs +++ b/src/topology/controller.rs @@ -39,7 +39,7 @@ pub struct TopologyController { pub config_paths: Vec, pub require_healthy: Option, #[cfg(feature = "api")] - pub api_server: Option, + pub api_server: Option, pub extra_context: ExtraContext, } @@ -70,37 +70,39 @@ impl TopologyController { #[cfg(feature = "api")] if !new_config.api.enabled { if let Some(server) = self.api_server.take() { - debug!("Dropping api server."); - drop(server) + debug!("Stopping gRPC API server."); + drop(server); } } else if self.api_server.is_none() { - use std::sync::atomic::AtomicBool; + debug!("Starting gRPC API server."); - use tokio::runtime::Handle; - - use crate::internal_events::ApiStarted; - - debug!("Starting api server."); - - self.api_server = match api::Server::start( + match api::GrpcServer::start( self.topology.config(), self.topology.watch(), - Arc::::clone(&self.topology.running), - &Handle::current(), - ) { + Arc::clone(&self.topology.running), + ) + .await + { Ok(api_server) => { - emit!(ApiStarted { - addr: new_config.api.address.unwrap(), - playground: new_config.api.playground, - graphql: new_config.api.graphql, - }); - - Some(api_server) + let addr = api_server.addr(); + info!( + message = "GRPC API server started.", + addr = %addr, + ); + self.api_server = Some(api_server); } Err(error) => { - let error = error.to_string(); - error!("An error occurred that Vector couldn't handle: {}.", error); - return ReloadOutcome::FatalError(ShutdownError::ApiFailed { error }); + let error_string = error.to_string(); + error!( + message = "An error occurred that Vector couldn't handle.", + error = %error_string, + internal_log_rate_limit = false, + ); + // Fail fast when API is explicitly enabled but fails to start + // This ensures users don't run with a broken configuration thinking top/tap will work + return ReloadOutcome::FatalError(ShutdownError::ApiFailed { + error: error_string, + }); } } } @@ -111,12 +113,6 @@ impl TopologyController { .await { Ok(()) => { - #[cfg(feature = "api")] - // Pass the new config to the API server. - if let Some(ref api_server) = self.api_server { - api_server.update_config(self.topology.config()); - } - emit!(VectorReloaded { config_paths: &self.config_paths }); diff --git a/src/topology/running.rs b/src/topology/running.rs index db667e4891110..6839379cd0f1a 100644 --- a/src/topology/running.rs +++ b/src/topology/running.rs @@ -58,6 +58,7 @@ pub struct RunningTopology { inputs_tap_metadata: HashMap>, outputs: HashMap, outputs_tap_metadata: HashMap, + component_type_names: HashMap, source_tasks: HashMap, tasks: HashMap, shutdown_coordinator: SourceShutdownCoordinator, @@ -82,6 +83,7 @@ impl RunningTopology { inputs_tap_metadata: HashMap::new(), outputs: HashMap::new(), outputs_tap_metadata: HashMap::new(), + component_type_names: HashMap::new(), shutdown_coordinator: SourceShutdownCoordinator::default(), detach_triggers: HashMap::new(), source_tasks: HashMap::new(), @@ -713,17 +715,20 @@ impl RunningTopology { for key in &diff.sources.to_remove { // Sources only have outputs self.outputs_tap_metadata.remove(key); + self.component_type_names.remove(key); } for key in &diff.transforms.to_remove { // Transforms can have both inputs and outputs self.outputs_tap_metadata.remove(key); self.inputs_tap_metadata.remove(key); + self.component_type_names.remove(key); } for key in &diff.sinks.to_remove { // Sinks only have inputs self.inputs_tap_metadata.remove(key); + self.component_type_names.remove(key); } let removed_sinks = diff.enrichment_tables.to_remove.iter().filter(|key| { @@ -749,8 +754,10 @@ impl RunningTopology { for key in diff.sources.changed_and_added() { if let Some(task) = new_pieces.tasks.get(key) { + let typetag = task.typetag().to_string(); self.outputs_tap_metadata - .insert(key.clone(), ("source", task.typetag().to_string())); + .insert(key.clone(), ("source", typetag.clone())); + self.component_type_names.insert(key.clone(), typetag); } } @@ -771,8 +778,17 @@ impl RunningTopology { for key in diff.transforms.changed_and_added() { if let Some(task) = new_pieces.tasks.get(key) { + let typetag = task.typetag().to_string(); self.outputs_tap_metadata - .insert(key.clone(), ("transform", task.typetag().to_string())); + .insert(key.clone(), ("transform", typetag.clone())); + self.component_type_names.insert(key.clone(), typetag); + } + } + + for key in diff.sinks.changed_and_added() { + if let Some(task) = new_pieces.tasks.get(key) { + self.component_type_names + .insert(key.clone(), task.typetag().to_string()); } } @@ -888,6 +904,11 @@ impl RunningTopology { // Note, only sources and transforms are relevant. Sinks do // not have outputs to tap. removals, + type_names: self + .component_type_names + .iter() + .map(|(k, v)| (k.to_string(), v.clone())) + .collect(), }) .expect("Couldn't broadcast config changes."); } diff --git a/tests/vector_api/harness.rs b/tests/vector_api/harness.rs index 2ee288275cda0..274592fac5dae 100644 --- a/tests/vector_api/harness.rs +++ b/tests/vector_api/harness.rs @@ -21,7 +21,7 @@ use vector::test_util::{temp_dir, temp_file}; use vector_lib::api_client::Client; // Constants -pub const STARTUP_TIMEOUT: Duration = Duration::from_secs(2); +pub const STARTUP_TIMEOUT: Duration = Duration::from_secs(10); pub const EVENT_PROCESSING_TIMEOUT: Duration = Duration::from_secs(30); /// Test harness for Vector instances with API enabled @@ -112,14 +112,12 @@ impl TestHarness { .spawn() .map_err(|e| format!("Failed to spawn vector: {e}"))?; - let api_client = Client::new( - format!("http://127.0.0.1:{api_port}/graphql") - .parse() - .map_err(|e| format!("Invalid URL: {e}"))?, - ); + let url = format!("http://127.0.0.1:{api_port}"); + let mut api_client = Client::new(url.parse().expect("valid URL")); // Wait for Vector startup with crash detection - wait_for_startup(&mut vector, &api_client, STARTUP_TIMEOUT).await?; + // This will repeatedly try to connect until Vector is ready + wait_for_startup(&mut vector, &mut api_client, STARTUP_TIMEOUT).await?; Ok(Self { vector, @@ -130,14 +128,9 @@ impl TestHarness { }) } - /// Returns reference to the API client - pub fn api_client(&self) -> &Client { - &self.api_client - } - - /// Returns the API port number - pub fn api_port(&self) -> u16 { - self.api_port + /// Returns mutable reference to the API client + pub fn api_client(&mut self) -> &mut Client { + &mut self.api_client } /// Reloads Vector configuration by sending SIGHUP or using watch mode @@ -169,8 +162,16 @@ impl TestHarness { .map_err(|e| format!("Failed to send SIGHUP: {e}"))?; } + // Give Vector a moment to process the reload signal before polling + sleep(Duration::from_millis(500)).await; + // Poll for reload completion with crash detection - wait_for_topology_match(&mut self.vector, &self.api_client, expected_component_ids).await + wait_for_topology_match( + &mut self.vector, + &mut self.api_client, + expected_component_ids, + ) + .await } /// Checks if Vector is still running @@ -245,10 +246,11 @@ pub fn create_data_directory() -> PathBuf { /// Fails fast if Vector crashes, succeeds fast if API becomes ready quickly. pub async fn wait_for_startup( vector: &mut Child, - client: &Client, + client: &mut Client, timeout: Duration, ) -> Result<(), String> { let start = Instant::now(); + let mut connected = false; loop { // Check if Vector crashed @@ -263,8 +265,13 @@ pub async fn wait_for_startup( }; } - // Check if API is ready - if client.healthcheck().await.is_ok() { + // Try to connect if not connected yet + if !connected && client.connect().await.is_ok() { + connected = true; + } + + // Check if API is ready (only if connected) + if connected && client.health().await.is_ok() { return Ok(()); } @@ -284,12 +291,11 @@ pub async fn wait_for_startup( /// Combines topology polling with crash detection for reload scenarios. pub async fn wait_for_topology_match( vector: &mut Child, - client: &Client, + client: &mut Client, expected_component_ids: &[&str], ) -> Result<(), String> { - use vector_lib::api_client::gql::ComponentsQueryExt; - let start = Instant::now(); + let mut last_components: Vec = Vec::new(); loop { // Check if Vector crashed @@ -298,20 +304,25 @@ pub async fn wait_for_topology_match( } // Query components to see if topology matches - if let Ok(result) = client.components_query(100).await - && let Some(data) = result.data - { - let mut current_ids: Vec<&str> = data + if let Ok(response) = client.get_components(100).await { + let mut current_ids: Vec = response .components - .edges .iter() - .map(|e| e.node.component_id.as_str()) + .map(|c| c.component_id.clone()) .collect(); current_ids.sort_unstable(); - let mut expected_sorted = expected_component_ids.to_vec(); + let mut expected_sorted: Vec = expected_component_ids + .iter() + .map(|s| s.to_string()) + .collect(); expected_sorted.sort_unstable(); + // Track last seen components for better error reporting + if current_ids != last_components { + last_components = current_ids.clone(); + } + if current_ids == expected_sorted { return Ok(()); } @@ -320,7 +331,8 @@ pub async fn wait_for_topology_match( // Check timeout if start.elapsed() >= STARTUP_TIMEOUT { return Err(format!( - "Topology did not match expected components within {STARTUP_TIMEOUT:?}" + "Topology did not match expected components within {STARTUP_TIMEOUT:?}. Last seen: {:?}, expected: {:?}", + last_components, expected_component_ids )); } @@ -330,16 +342,14 @@ pub async fn wait_for_topology_match( /// Waits for a component to process the expected number of events /// -/// Polls the GraphQL API until the component's sent_events_total +/// Polls the gRPC API until the component's sent_events_total /// reaches or exceeds the expected count. pub async fn wait_for_component_events( - client: &Client, + client: &mut Client, component_id: &str, expected_events: i64, timeout: Duration, ) -> Result { - use vector_lib::api_client::gql::ComponentsQueryExt; - let start = Instant::now(); let mut last_count = 0; @@ -350,20 +360,21 @@ pub async fn wait_for_component_events( )); } - let result = client - .components_query(100) + let response = client + .get_components(100) .await .map_err(|e| format!("Query failed: {e}"))?; - let data = result.data.ok_or("No data in response")?; - - if let Some(component) = data + if let Some(component) = response .components - .edges .iter() - .find(|e| e.node.component_id == component_id) + .find(|c| c.component_id == component_id) { - let events = component.node.on.sent_events_total(); + let events = component + .metrics + .as_ref() + .and_then(|m| m.sent_events_total) + .unwrap_or(0); if events != last_count { last_count = events; diff --git a/tests/vector_api/lib.rs b/tests/vector_api/lib.rs index baafa332bc991..e82f39d41b42e 100644 --- a/tests/vector_api/lib.rs +++ b/tests/vector_api/lib.rs @@ -1,6 +1,6 @@ -//! Integration tests for Vector's GraphQL API +//! Integration tests for Vector's gRPC API //! -//! This test suite verifies the GraphQL API that powers both `vector top` and `vector tap` commands. +//! This test suite verifies the gRPC API that powers both `vector top` and `vector tap` commands. //! Tests cover component discovery, metrics collection, event streaming, and config reloading. mod common; diff --git a/tests/vector_api/tap.rs b/tests/vector_api/tap.rs index d29a47b9775e5..fdb71d2fa48fa 100644 --- a/tests/vector_api/tap.rs +++ b/tests/vector_api/tap.rs @@ -1,119 +1,59 @@ //! Integration tests for `vector tap` command //! -//! Provides extensions for WebSocket subscriptions and tests for event streaming. +//! Provides extensions for gRPC streaming and tests for event streaming. use super::{common::*, harness::*}; use indoc::indoc; use std::time::{Duration, Instant}; use tokio_stream::StreamExt; -use vector_lib::api_client::{ - connect_subscription_client, - gql::{ - TapEncodingFormat, TapSubscriptionExt, - output_events_by_component_id_patterns_subscription::OutputEventsByComponentIdPatternsSubscriptionOutputEventsByComponentIdPatterns as TapEvent, - }, -}; +use vector_lib::api_client::proto::{StreamOutputEventsRequest, StreamOutputEventsResponse}; pub const TAP_TIMEOUT: Duration = Duration::from_secs(10); impl TestHarness { - /// Returns WebSocket URL for subscriptions - fn websocket_url(&self) -> url::Url { - format!("ws://127.0.0.1:{}/graphql", self.api_port()) - .parse() - .expect("Valid WebSocket URL") - } - - /// Creates a tap subscription for the given patterns + /// Collects tap events from the given patterns /// - /// Uses sensible defaults: format=JSON, limit=1000, interval=100ms - pub async fn tap_subscription( - &self, - outputs_patterns: &[&str], - inputs_patterns: &[&str], - ) -> Result { - const DEFAULT_LIMIT: i64 = 1000; - const DEFAULT_INTERVAL_MS: i64 = 100; - let url = self.websocket_url(); - - let subscription_client = connect_subscription_client(url) - .await - .map_err(|e| format!("Failed to connect to WebSocket: {e}"))?; - - let stream = subscription_client.output_events_by_component_id_patterns_subscription( - outputs_patterns.iter().map(|s| s.to_string()).collect(), - inputs_patterns.iter().map(|s| s.to_string()).collect(), - TapEncodingFormat::Json, - DEFAULT_LIMIT, - DEFAULT_INTERVAL_MS, - ); - - Ok(TapSubscription { - stream, - _client: subscription_client, - }) - } - - /// Creates a tap subscription and collects initial events - /// - /// Returns both the collected events and the subscription handle. - /// If you need to collect more events later, keep the handle; otherwise it will be dropped. - /// For input pattern filtering, use `tap_subscription` directly. + /// This is a simplified version that collects all events inline without storing a stream. pub async fn tap_and_collect( - &self, + &mut self, outputs_patterns: &[&str], count: usize, - ) -> Result<(Vec, TapSubscription), String> { - let mut tap = self.tap_subscription(outputs_patterns, &[]).await?; - let events = tap.take_events(count, TAP_TIMEOUT).await?; - Ok((events, tap)) - } -} - -/// Wrapper around a tap subscription stream with helper methods -pub struct TapSubscription { - stream: vector_lib::api_client::BoxedSubscription< - vector_lib::api_client::gql::OutputEventsByComponentIdPatternsSubscription, - >, - _client: vector_lib::api_client::SubscriptionClient, // Keep client alive! -} + ) -> Result, String> { + const DEFAULT_LIMIT: i32 = 1000; + const DEFAULT_INTERVAL_MS: i32 = 100; + + let request = StreamOutputEventsRequest { + outputs_patterns: outputs_patterns.iter().map(|s| s.to_string()).collect(), + inputs_patterns: vec![], + limit: DEFAULT_LIMIT, + interval_ms: DEFAULT_INTERVAL_MS, + }; + + let mut stream = self + .api_client() + .stream_output_events(request) + .await + .map_err(|e| format!("Failed to create tap stream: {e}"))?; -impl TapSubscription { - /// Collects tap events until count is reached or timeout occurs - pub async fn take_events( - &mut self, - count: usize, - timeout: Duration, - ) -> Result, String> { let start = Instant::now(); let mut events = Vec::new(); while events.len() < count { - if start.elapsed() >= timeout { + if start.elapsed() >= TAP_TIMEOUT { return Err(format!( "Timeout: collected {}/{} events in {:?}", events.len(), count, - timeout + TAP_TIMEOUT )); } - match tokio::time::timeout(timeout - start.elapsed(), self.stream.as_mut().next()).await - { - Ok(Some(Some(response))) => { - if let Some(data) = response.data { - for event in data.output_events_by_component_id_patterns { - events.push(event); - } - } else if let Some(errors) = response.errors - && !errors.is_empty() - { - return Err(format!("GraphQL errors: {:?}", errors)); - } + match tokio::time::timeout(TAP_TIMEOUT - start.elapsed(), stream.next()).await { + Ok(Some(Ok(event))) => { + events.push(event); } - Ok(Some(None)) => { - // No response in this poll, continue - continue; + Ok(Some(Err(e))) => { + return Err(format!("Stream error: {}", e)); } Ok(None) => { return Err(format!( @@ -138,72 +78,70 @@ impl TapSubscription { #[tokio::test] async fn tap_receives_events() { let config = single_source_config("demo", 0.01, Some(100)); - let harness = TestHarness::new(&config) + let mut harness = TestHarness::new(&config) .await .expect("Failed to start Vector"); // Tap the source output with wildcard pattern and collect events - let (events, _tap) = harness + let events = harness .tap_and_collect(&["*"], 10) .await .expect("Should receive events"); assert!(!events.is_empty(), "Should receive at least one event"); - // Verify we got at least one log event (not just notifications) - let log_events: Vec<_> = events + // Verify we got at least one tapped event (not just notifications) + use vector_lib::api_client::proto::stream_output_events_response::Event; + let tapped_events: Vec<_> = events .iter() - .filter_map(|e| { - if let TapEvent::Log(log) = e { - Some(log) - } else { - None - } + .filter_map(|e| match &e.event { + Some(Event::TappedEvent(tapped)) => Some(tapped), + _ => None, }) .collect(); + let notification_count = events + .iter() + .filter(|e| matches!(&e.event, Some(Event::Notification(_)))) + .count(); + assert!( - !log_events.is_empty(), - "Should receive at least one log event, got {} events total ({} notifications)", + !tapped_events.is_empty(), + "Should receive at least one tapped event, got {} events total ({} notifications)", events.len(), - events - .iter() - .filter(|e| matches!(e, TapEvent::EventNotification(_))) - .count() + notification_count ); } #[tokio::test] async fn tap_specific_component() { let config = dual_source_config("demo1", "demo2", 0.01, Some(100)); - let harness = TestHarness::new(&config) + let mut harness = TestHarness::new(&config) .await .expect("Failed to start Vector"); // Tap only demo1, not demo2 - let (events, _tap) = harness + let events = harness .tap_and_collect(&["demo1"], 10) .await .expect("Should receive events"); // Verify we only got events from demo1 - let log_events: Vec<_> = events + use vector_lib::api_client::proto::stream_output_events_response::Event; + let tapped_events: Vec<_> = events .iter() - .filter_map(|e| { - if let TapEvent::Log(log) = e { - Some(log) - } else { - None - } + .filter_map(|e| match &e.event { + Some(Event::TappedEvent(tapped)) => Some(tapped), + _ => None, }) .collect(); - assert!(!log_events.is_empty(), "Should receive log events"); + assert!(!tapped_events.is_empty(), "Should receive tapped events"); - // All log events should be from demo1 - for log in &log_events { + // All tapped events should be from demo1 + for tapped in &tapped_events { assert_eq!( - log.component_id, "demo1", + tapped.component_id, "demo1", "Should only receive events from demo1" ); } @@ -211,88 +149,106 @@ async fn tap_specific_component() { #[tokio::test] async fn tap_survives_config_reload() { - let config = single_source_config("demo", 0.05, Some(100)); + use vector_lib::api_client::proto::stream_output_events_response::Event; + + let config = single_source_config("demo", 0.05, None); // No count limit - runs continuously let mut harness = TestHarness::new(&config) .await .expect("Failed to start Vector"); - // Start tap with wildcard and collect initial events - let (initial_events, mut tap) = harness - .tap_and_collect(&["*"], 5) + // Open the stream ONCE and keep it alive across the reload. + // tonic channels are Arc-based so the returned stream is owned and does not + // borrow `harness`, allowing us to call reload_with_config below. + let request = StreamOutputEventsRequest { + outputs_patterns: vec!["*".to_string()], + inputs_patterns: vec![], + limit: 1000, + interval_ms: 100, + }; + let mut stream = harness + .api_client() + .stream_output_events(request) .await - .expect("Should receive initial events"); - - assert!(!initial_events.is_empty(), "Should receive initial events"); + .expect("Failed to open tap stream"); + + // Collect a few pre-reload events to confirm the stream is working + let mut pre_reload_count = 0; + let start = Instant::now(); + while pre_reload_count < 5 { + assert!( + start.elapsed() < TAP_TIMEOUT, + "Timeout waiting for pre-reload events" + ); + match tokio::time::timeout(TAP_TIMEOUT - start.elapsed(), stream.next()).await { + Ok(Some(Ok(_))) => pre_reload_count += 1, + Ok(Some(Err(e))) => panic!("Stream error before reload: {e}"), + Ok(None) => panic!("Stream ended unexpectedly before reload"), + Err(_) => panic!("Timeout before reload"), + } + } - // Reload config with a new component + // Reload with completely new components harness .reload_with_config( indoc! {" sources: - demo: + tap_demo1: type: demo_logs format: json interval: 0.05 - count: 100 - new_demo: + tap_demo2: type: demo_logs format: json interval: 0.05 - count: 100 sinks: - blackhole: + tap_blackhole: type: blackhole - inputs: ['demo', 'new_demo'] + inputs: ['tap_demo1', 'tap_demo2'] "}, - &["demo", "new_demo", "blackhole"], + &["tap_demo1", "tap_demo2", "tap_blackhole"], ) .await .expect("Failed to reload config"); - // Tap should still work and see events from both sources - let after_reload = tap - .take_events(5, TAP_TIMEOUT) - .await - .expect("Should receive events after reload"); - - assert!( - !after_reload.is_empty(), - "Should receive events after reload" - ); - - let log_events: Vec<_> = after_reload - .iter() - .filter_map(|e| { - if let TapEvent::Log(log) = e { - Some(log) - } else { - None + // Continue reading from the SAME stream. If the tap implementation drops active + // streams on reload, stream.next() will return None and the test will fail. + let mut post_reload_count = 0; + let start = Instant::now(); + while post_reload_count < 5 { + assert!( + start.elapsed() < TAP_TIMEOUT, + "Stream did not survive reload: only received {post_reload_count} tapped events after reload" + ); + match tokio::time::timeout(TAP_TIMEOUT - start.elapsed(), stream.next()).await { + Ok(Some(Ok(event))) => { + if matches!(&event.event, Some(Event::TappedEvent(_))) { + post_reload_count += 1; + } + // Notifications (component added/removed) are expected during reload - skip them } - }) - .collect(); - - assert!( - !log_events.is_empty(), - "Should receive log events after reload" - ); + Ok(Some(Err(e))) => panic!("Stream error after reload: {e}"), + Ok(None) => panic!("Stream was closed on reload - tap subscription was dropped"), + Err(_) => panic!("Timeout waiting for post-reload tapped events"), + } + } } #[tokio::test] async fn multiple_concurrent_subscriptions() { let config = dual_source_config("demo1", "demo2", 0.01, Some(100)); - let harness = TestHarness::new(&config) + let mut harness = TestHarness::new(&config) .await .expect("Failed to start Vector"); - // Create two separate tap subscriptions using the same harness - let (events1, _tap1) = harness + // Create two separate tap requests using the same harness + let events1 = harness .tap_and_collect(&["demo1"], 5) .await .expect("Should receive events from tap1"); - let (events2, _tap2) = harness + let events2 = harness .tap_and_collect(&["demo2"], 5) .await .expect("Should receive events from tap2"); @@ -313,62 +269,57 @@ async fn multiple_concurrent_subscriptions() { ); // Verify tap1 only sees demo1 - let log_events1: Vec<_> = events1 + use vector_lib::api_client::proto::stream_output_events_response::Event; + let tapped_events1: Vec<_> = events1 .iter() - .filter_map(|e| { - if let TapEvent::Log(log) = e { - Some(log) - } else { - None - } + .filter_map(|e| match &e.event { + Some(Event::TappedEvent(tapped)) => Some(tapped), + _ => None, }) .collect(); - for log in &log_events1 { + for tapped in &tapped_events1 { assert_eq!( - log.component_id, "demo1", + tapped.component_id, "demo1", "Tap1 should only see demo1 events" ); } // Verify tap2 only sees demo2 - let log_events2: Vec<_> = events2 + let tapped_events2: Vec<_> = events2 .iter() - .filter_map(|e| { - if let TapEvent::Log(log) = e { - Some(log) - } else { - None - } + .filter_map(|e| match &e.event { + Some(Event::TappedEvent(tapped)) => Some(tapped), + _ => None, }) .collect(); - for log in &log_events2 { + for tapped in &tapped_events2 { assert_eq!( - log.component_id, "demo2", + tapped.component_id, "demo2", "Tap2 should only see demo2 events" ); } // Verify we got different sets of events assert_ne!( - log_events1.len(), + tapped_events1.len(), 0, - "Should have some log events from demo1" + "Should have some tapped events from demo1" ); assert_ne!( - log_events2.len(), + tapped_events2.len(), 0, - "Should have some log events from demo2" + "Should have some tapped events from demo2" ); // Create new taps to the same components to verify repeatability - let (events1_again, _tap1_again) = harness + let events1_again = harness .tap_and_collect(&["demo1"], 5) .await .expect("Should receive events from tap1 again"); - let (events2_again, _tap2_again) = harness + let events2_again = harness .tap_and_collect(&["demo2"], 5) .await .expect("Should receive events from tap2 again"); @@ -384,47 +335,41 @@ async fn multiple_concurrent_subscriptions() { ); // Verify component isolation is maintained - let log_events1_again: Vec<_> = events1_again + let tapped_events1_again: Vec<_> = events1_again .iter() - .filter_map(|e| { - if let TapEvent::Log(log) = e { - Some(log) - } else { - None - } + .filter_map(|e| match &e.event { + Some(Event::TappedEvent(tapped)) => Some(tapped), + _ => None, }) .collect(); - let log_events2_again: Vec<_> = events2_again + let tapped_events2_again: Vec<_> = events2_again .iter() - .filter_map(|e| { - if let TapEvent::Log(log) = e { - Some(log) - } else { - None - } + .filter_map(|e| match &e.event { + Some(Event::TappedEvent(tapped)) => Some(tapped), + _ => None, }) .collect(); assert!( - !log_events1_again.is_empty(), - "Should have log events from demo1 again" + !tapped_events1_again.is_empty(), + "Should have tapped events from demo1 again" ); assert!( - !log_events2_again.is_empty(), - "Should have log events from demo2 again" + !tapped_events2_again.is_empty(), + "Should have tapped events from demo2 again" ); - for log in &log_events1_again { + for tapped in &tapped_events1_again { assert_eq!( - log.component_id, "demo1", + tapped.component_id, "demo1", "Second tap1 should only see demo1 events" ); } - for log in &log_events2_again { + for tapped in &tapped_events2_again { assert_eq!( - log.component_id, "demo2", + tapped.component_id, "demo2", "Second tap2 should only see demo2 events" ); } diff --git a/tests/vector_api/top.rs b/tests/vector_api/top.rs index 0c85e53b7e21f..79e94a094785c 100644 --- a/tests/vector_api/top.rs +++ b/tests/vector_api/top.rs @@ -1,40 +1,37 @@ //! Integration tests for `vector top` command //! -//! Provides extensions for GraphQL queries and tests for component discovery, +//! Provides extensions for gRPC queries and tests for component discovery, //! metrics collection, and config reloading. use super::{common::*, harness::*}; use indoc::indoc; +use tokio_stream::StreamExt; +use vector_lib::api_client::proto::{ + GetComponentsResponse, MetricName, stream_component_metrics_response::Value, +}; impl TestHarness { - /// Queries all components from the GraphQL API - pub async fn query_components( - &self, - ) -> Result { - use vector_lib::api_client::gql::ComponentsQueryExt; - + /// Queries all components from the gRPC API + pub async fn query_components(&mut self) -> Result { self.api_client() - .components_query(100) + .get_components(100) .await - .map_err(|e| format!("Query failed: {e}"))? - .data - .ok_or_else(|| "No data in response".to_string()) + .map_err(|e| format!("Query failed: {e}")) } - /// Queries component IDs from the GraphQL API - pub async fn query_component_ids(&self) -> Result, String> { - let data = self.query_components().await?; - Ok(data + /// Queries component IDs from the gRPC API + pub async fn query_component_ids(&mut self) -> Result, String> { + let response = self.query_components().await?; + Ok(response .components - .edges .iter() - .map(|e| e.node.component_id.clone()) + .map(|c| c.component_id.clone()) .collect()) } /// Waits for a component to process at least the expected number of events pub async fn wait_for_events( - &self, + &mut self, component_id: &str, expected_events: i64, ) -> Result { @@ -56,7 +53,7 @@ impl TestHarness { async fn displays_pipeline_topology_and_metrics() { const EXPECTED_EVENTS: i64 = 100; - let runner = TestHarness::new(indoc! {" + let mut runner = TestHarness::new(indoc! {" sources: demo: type: demo_logs @@ -93,47 +90,57 @@ async fn displays_pipeline_topology_and_metrics() { assert_eq!(component_ids.len(), 3); // Query full component data for types and metrics - let data = runner + let response = runner .query_components() .await .expect("Failed to query components"); // Verify component types are reported correctly - for edge in &data.components.edges { - let component_id = &edge.node.component_id; - let component_type = edge.node.on.to_string(); + use vector_lib::api_client::proto::ComponentType; + for component in &response.components { + let component_id = &component.component_id; + let component_type = + ComponentType::try_from(component.component_type).expect("Valid component type"); match component_id.as_str() { - "demo" => assert_eq!(component_type, "source"), - "blackhole1" | "blackhole2" => assert_eq!(component_type, "sink"), + "demo" => assert_eq!(component_type, ComponentType::Source), + "blackhole1" | "blackhole2" => assert_eq!(component_type, ComponentType::Sink), _ => panic!("Unexpected component: {}", component_id), } } // Verify source sent events - let source = data + let source = response .components - .edges .iter() - .find(|e| e.node.component_id == "demo") + .find(|c| c.component_id == "demo") .expect("Should find source"); - assert!(source.node.on.sent_events_total() >= EXPECTED_EVENTS); + let sent_events = source + .metrics + .as_ref() + .and_then(|m| m.sent_events_total) + .unwrap_or(0); + assert!(sent_events >= EXPECTED_EVENTS); // Verify at least one sink received events - let sink1 = data + let sink1 = response .components - .edges .iter() - .find(|e| e.node.component_id == "blackhole1") + .find(|c| c.component_id == "blackhole1") .expect("Should find sink1"); - assert!(sink1.node.on.received_events_total() >= EXPECTED_EVENTS); + let received_events = sink1 + .metrics + .as_ref() + .and_then(|m| m.received_events_total) + .unwrap_or(0); + assert!(received_events >= EXPECTED_EVENTS); } #[tokio::test] async fn config_reload_updates_components() { - // Initial config: 1 source -> 1 sink + // Initial config: 1 source -> 1 sink (runs continuously) let config = single_source_config("demo1", 0.1, None); let mut runner = TestHarness::new(&config) .await @@ -146,81 +153,83 @@ async fn config_reload_updates_components() { assert!(component_ids.contains(&"blackhole".to_string())); assert_eq!(component_ids.len(), 2); - // RELOAD 1: Add components + // RELOAD 1: Replace with completely new set runner .reload_with_config( indoc! {" sources: - demo1: + new_demo1: type: demo_logs format: json interval: 0.1 - demo2: + new_demo2: type: demo_logs format: json interval: 0.1 sinks: - blackhole1: + new_blackhole1: type: blackhole - inputs: ['demo1'] + inputs: ['new_demo1'] - blackhole2: + new_blackhole2: type: blackhole - inputs: ['demo2'] + inputs: ['new_demo2'] "}, - &["demo1", "demo2", "blackhole1", "blackhole2"], + &["new_demo1", "new_demo2", "new_blackhole1", "new_blackhole2"], ) .await .expect("Failed to reload config"); - // Verify additions + // Verify old components removed and new ones added assert!(runner.check_running(), "Vector should still be running"); let component_ids = runner.query_component_ids().await.expect("Failed to query"); - assert!(component_ids.contains(&"demo1".to_string())); - assert!(component_ids.contains(&"demo2".to_string())); - assert!(component_ids.contains(&"blackhole1".to_string())); - assert!(component_ids.contains(&"blackhole2".to_string())); + assert!(!component_ids.contains(&"demo1".to_string())); + assert!(!component_ids.contains(&"blackhole".to_string())); + assert!(component_ids.contains(&"new_demo1".to_string())); + assert!(component_ids.contains(&"new_demo2".to_string())); + assert!(component_ids.contains(&"new_blackhole1".to_string())); + assert!(component_ids.contains(&"new_blackhole2".to_string())); assert_eq!(component_ids.len(), 4); // Verify new components are processing events runner - .wait_for_events("demo2", 10) + .wait_for_events("new_demo1", 10) .await .expect("New source should process events"); - // RELOAD 2: Replace with completely new components (same count, different names) + // RELOAD 2: Scale back down to single component runner .reload_with_config( indoc! {" sources: - new_demo: + final_demo: type: demo_logs format: json interval: 0.1 sinks: - new_blackhole: + final_blackhole: type: blackhole - inputs: ['new_demo'] + inputs: ['final_demo'] "}, - &["new_demo", "new_blackhole"], + &["final_demo", "final_blackhole"], ) .await .expect("Failed to reload config"); - // Verify old components removed and new ones added + // Verify all previous components removed and final ones added let component_ids = runner.query_component_ids().await.expect("Failed to query"); - assert!(!component_ids.contains(&"demo1".to_string())); - assert!(!component_ids.contains(&"demo2".to_string())); - assert!(!component_ids.contains(&"blackhole1".to_string())); - assert!(!component_ids.contains(&"blackhole2".to_string())); - assert!(component_ids.contains(&"new_demo".to_string())); - assert!(component_ids.contains(&"new_blackhole".to_string())); + assert!(!component_ids.contains(&"new_demo1".to_string())); + assert!(!component_ids.contains(&"new_demo2".to_string())); + assert!(!component_ids.contains(&"new_blackhole1".to_string())); + assert!(!component_ids.contains(&"new_blackhole2".to_string())); + assert!(component_ids.contains(&"final_demo".to_string())); + assert!(component_ids.contains(&"final_blackhole".to_string())); assert_eq!(component_ids.len(), 2); } @@ -233,33 +242,171 @@ async fn watch_mode_auto_reloads() { .expect("Failed to start Vector"); // Verify initial state: 1 source + 1 sink - let data = runner.query_components().await.expect("Failed to query"); - assert_eq!(data.components.edges.len(), 2); + let response = runner.query_components().await.expect("Failed to query"); + assert_eq!(response.components.len(), 2); // Modify config file - Vector should auto-reload (no SIGHUP needed!) + // Use completely new component names to avoid reload issues runner .reload_with_config( indoc! {" sources: - demo: + watch_demo1: type: demo_logs format: json interval: 0.1 - demo2: + watch_demo2: type: demo_logs format: json interval: 0.1 sinks: - blackhole: + watch_blackhole: type: blackhole - inputs: ['demo', 'demo2'] + inputs: ['watch_demo1', 'watch_demo2'] "}, - &["demo", "demo2", "blackhole"], + &["watch_demo1", "watch_demo2", "watch_blackhole"], ) .await .expect("Watch mode reload failed"); + // Verify reload worked + let component_ids = runner.query_component_ids().await.expect("Failed to query"); + assert_eq!(component_ids.len(), 3); + // Cleanup happens automatically via Drop } + +#[tokio::test] +async fn multi_output_transform_reports_per_output_sent_events() { + // Pipeline: demo_logs -> route (two named outputs) -> two blackhole sinks + // The route transform splits events into "all" (everything) and "has_host" (subset). + // We verify that: + // 1. GetComponents reports two Output entries for the route transform + // 2. StreamComponentSentEventsTotal populates output_totals for the route transform + const EXPECTED_EVENTS: i64 = 50; + + let mut runner = TestHarness::new(indoc! {" + sources: + demo: + type: demo_logs + format: json + interval: 0.01 + + transforms: + splitter: + type: route + inputs: ['demo'] + route: + all: 'true' + has_host: 'exists(.host)' + + sinks: + sink_all: + type: blackhole + inputs: ['splitter.all'] + + sink_has_host: + type: blackhole + inputs: ['splitter.has_host'] + "}) + .await + .expect("Failed to start Vector"); + + // Wait for events to reach the sinks + runner + .wait_for_events("demo", EXPECTED_EVENTS) + .await + .expect("Source never sent expected events"); + + // --- Assert 1: GetComponents reports per-output entries for the route transform --- + let response = runner + .query_components() + .await + .expect("Failed to query components"); + + let splitter = response + .components + .iter() + .find(|c| c.component_id == "splitter") + .expect("splitter transform not found"); + + // Route transform must report at least 2 named outputs + assert!( + splitter.outputs.len() >= 2, + "Expected >= 2 outputs for route transform, got {}", + splitter.outputs.len() + ); + + let output_ids: Vec<&str> = splitter + .outputs + .iter() + .map(|o| o.output_id.as_str()) + .collect(); + assert!( + output_ids.contains(&"all"), + "Missing 'all' output, got: {:?}", + output_ids + ); + assert!( + output_ids.contains(&"has_host"), + "Missing 'has_host' output, got: {:?}", + output_ids + ); + + // --- Assert 2: StreamComponentMetrics(SentEventsTotal) populates output_totals --- + let mut stream = runner + .api_client() + .stream_component_metrics(MetricName::SentEventsTotal, 500) + .await + .expect("Failed to open sent_events_total stream"); + + // Drain stream updates until we get output_totals for the splitter, or timeout + let deadline = tokio::time::Instant::now() + EVENT_PROCESSING_TIMEOUT; + let mut splitter_totals_found = false; + + while tokio::time::Instant::now() < deadline { + match tokio::time::timeout(std::time::Duration::from_millis(600), stream.next()).await { + Ok(Some(Ok(msg))) => { + if msg.component_id == "splitter" + && let Some(Value::Total(total)) = msg.value + && !total.output_totals.is_empty() + { + // Both named outputs must be present with positive counts + let all_total = total.output_totals.get("all").copied().unwrap_or(0); + let has_host_total = total.output_totals.get("has_host").copied().unwrap_or(0); + + assert!( + all_total > 0, + "Expected positive total for 'all' output, got {}", + all_total + ); + assert!( + has_host_total > 0, + "Expected positive total for 'has_host' output, got {}", + has_host_total + ); + // "all" must be >= "has_host" since it matches every event + assert!( + all_total >= has_host_total, + "'all' ({}) should be >= 'has_host' ({})", + all_total, + has_host_total + ); + + splitter_totals_found = true; + break; + } + } + Ok(Some(Err(e))) => panic!("Stream error: {e}"), + Ok(None) => panic!("Stream ended unexpectedly"), + Err(_) => continue, // timeout on this tick, keep looping + } + } + + assert!( + splitter_totals_found, + "Never received non-empty output_totals for splitter transform within timeout" + ); +} diff --git a/website/content/en/blog/graphql-api.md b/website/content/en/blog/graphql-api.md index 92692735e90a2..f623c4caa921d 100644 --- a/website/content/en/blog/graphql-api.md +++ b/website/content/en/blog/graphql-api.md @@ -9,6 +9,8 @@ badges: tags: ["graphql", "api", "metrics"] --- +{{< grpc-migration-warning >}} + Although Vector is an observability tool, it's nonetheless important to be able to observe Vector itself, especially in production environments where it serves as critical infrastructure. That's why we're excited to announce the new [Vector diff --git a/website/content/en/docs/reference/api.md b/website/content/en/docs/reference/api.md index a28724feeb0a7..11a3566f6acfa 100644 --- a/website/content/en/docs/reference/api.md +++ b/website/content/en/docs/reference/api.md @@ -1,11 +1,12 @@ --- -title: The Vector API +title: The Vector Observability API short: API weight: 6 -tags: ["api", "graphql"] +tags: ["api", "grpc"] --- -Vector ships with a [GraphQL] API that allows you to interact with a running Vector instance. This page covers how to configure and enable Vector's API. +Vector ships with a [gRPC](https://grpc.io) API that allows you to interact with a running Vector +instance. This page covers how to configure and enable Vector's API. ## Configuration @@ -17,12 +18,20 @@ Vector ships with a [GraphQL] API that allows you to interact with a running Vec ## How it works -### GraphQL +The API exposes a gRPC service defined in [`proto/vector/observability.proto`](https://github.com/vectordotdev/vector/blob/master/proto/vector/observability.proto). +You can interact with it using any standard gRPC tooling. -Vector chose [GraphQL] for its API because GraphQL is self-documenting and type safe. We believe that this offers a superior client experience and makes Vector richly programmable through its API. +### Example using grpcurl -### Playground +```bash +# Check health +grpcurl -plaintext localhost:8686 vector.observability.v1.ObservabilityService/Health -Vector's GraphQL API ships with a built-in playground that allows you to explore the available commands and manually run queries against the API. This can be accessed at the `/playground` path. +# List components +grpcurl -plaintext localhost:8686 vector.observability.v1.ObservabilityService/GetComponents -[graphql]: https://graphql.org +# Stream events (tap) +grpcurl -plaintext \ + -d '{"outputs_patterns": ["*"], "limit": 100, "interval_ms": 500}' \ + localhost:8686 vector.observability.v1.ObservabilityService/StreamOutputEvents +``` diff --git a/website/content/en/docs/reference/configuration/api.md b/website/content/en/docs/reference/configuration/api.md index e3ba6a0372ff7..068968d900e6d 100644 --- a/website/content/en/docs/reference/configuration/api.md +++ b/website/content/en/docs/reference/configuration/api.md @@ -7,7 +7,8 @@ show_toc: true This page documents the configuration for Vector's observability API. -The API enables you to query Vector's topology, metrics, and health information through a GraphQL endpoint, as well as access an interactive GraphQL playground for development. +The API enables you to query Vector's topology, metrics, and health information +through a gRPC endpoint. {{< config-cross-links group="api" >}} diff --git a/website/content/en/highlights/2020-12-23-graphql-api.md b/website/content/en/highlights/2020-12-23-graphql-api.md index 7873c32482560..70c40ebfac28a 100644 --- a/website/content/en/highlights/2020-12-23-graphql-api.md +++ b/website/content/en/highlights/2020-12-23-graphql-api.md @@ -11,6 +11,8 @@ badges: domains: ["api"] --- +{{< grpc-migration-warning >}} + Vector now has a [GraphQL] API that you can use for a variety of purposes: * To view Vector's internal metrics diff --git a/website/content/en/highlights/2020-12-23-vector-top.md b/website/content/en/highlights/2020-12-23-vector-top.md index 1b99b30cc62e5..138c6344aae5a 100644 --- a/website/content/en/highlights/2020-12-23-vector-top.md +++ b/website/content/en/highlights/2020-12-23-vector-top.md @@ -12,6 +12,8 @@ badges: domains: ["cli", "observability"] --- +{{< grpc-migration-warning >}} + [`vector top`][top] is a command for the Vector [CLI] that displays both metrics emitted by your Vector instance as well as information about your Vector [topology] through a beautiful dashboard-style interface reminiscent of tools like [htop]. To use it, run `vector top` and specify the URL of the running Vector instance you want to monitor: diff --git a/website/content/en/highlights/2021-04-21-vector-tap.md b/website/content/en/highlights/2021-04-21-vector-tap.md index 867bd0f445f1c..be234b5ac9d89 100644 --- a/website/content/en/highlights/2021-04-21-vector-tap.md +++ b/website/content/en/highlights/2021-04-21-vector-tap.md @@ -11,6 +11,8 @@ badges: domains: ["graphql", "cli", "observability"] --- +{{< grpc-migration-warning >}} + Vector 0.13 introduces a new [`vector tap`][tap] subcommand that allows for tapping into the events flowing through Vector. This can be used to "live tail" events while troubleshooting incidents events or simply to debug your Vector diff --git a/website/cue/reference/api.cue b/website/cue/reference/api.cue index 7575915c4be26..38feeff331893 100644 --- a/website/cue/reference/api.cue +++ b/website/cue/reference/api.cue @@ -1,7 +1,7 @@ package metadata // These sources produce JSON providing a structured representation of the -// Vector GraphQL API +// Vector gRPC API api: { description: !="" @@ -11,34 +11,16 @@ api: { } api: { - description: """ - The Vector [GraphQL](\(urls.graphql)) API allows you to interact with a + description: """ + The Vector gRPC API allows you to interact with a running Vector instance, enabling introspection and management of - Vector in real-time. + Vector in real-time. The service definition is available in + `proto/vector/observability.proto`. """ - schema_json_url: "https://github.com/vectordotdev/vector/blob/master/lib/vector-api-client/graphql/schema.json" + schema_json_url: "https://github.com/vectordotdev/vector/blob/master/proto/vector/observability.proto" configuration: generated.configuration.configuration.api.type.object.options endpoints: { - "/graphql": { - POST: { - description: """ - Main endpoint for receiving and processing - GraphQL queries. - """ - responses: { - "200": { - description: """ - The query has been processed. GraphQL returns 200 - regardless if the query was successful or not. This - is due to the fact that queries can partially fail. - Please check for the `errors` key to determine if - there were any errors in your query. - """ - } - } - } - } "/health": { GET: { description: """ @@ -52,19 +34,5 @@ api: { } } } - "/playground": { - GET: { - description: """ - A bundled GraphQL playground that enables you - to explore the available queries and manually - run queries. - """ - responses: { - "200": { - description: "Vector is initialized and running." - } - } - } - } } } diff --git a/website/cue/reference/cli.cue b/website/cue/reference/cli.cue index 763188264df78..b570bd823aa77 100644 --- a/website/cue/reference/cli.cue +++ b/website/cue/reference/cli.cue @@ -334,7 +334,7 @@ cli: { } "url": { _short: "u" - description: "Vector GraphQL API server endpoint" + description: "Vector gRPC API server endpoint" type: "string" } "limit": { @@ -414,7 +414,7 @@ cli: { } "url": { _short: "u" - description: "Vector GraphQL API server endpoint" + description: "Vector gRPC API server endpoint" type: "string" } } diff --git a/website/cue/reference/components/sources/internal_metrics.cue b/website/cue/reference/components/sources/internal_metrics.cue index 22ebc5a49a5a6..98ed1ffe025d7 100644 --- a/website/cue/reference/components/sources/internal_metrics.cue +++ b/website/cue/reference/components/sources/internal_metrics.cue @@ -79,7 +79,7 @@ components: sources: internal_metrics: { tags: _component_tags } api_started_total: { - description: "The number of times the Vector GraphQL API has been started." + description: "The number of times the Vector API has been started." type: "counter" default_namespace: "vector" tags: _internal_metrics_tags diff --git a/website/cue/reference/generated/configuration.cue b/website/cue/reference/generated/configuration.cue index 5b98f0c6fbe35..fdb5d0cb15bd0 100644 --- a/website/cue/reference/generated/configuration.cue +++ b/website/cue/reference/generated/configuration.cue @@ -19,31 +19,10 @@ generated: configuration: { } enabled: { type: bool: default: false - description: "Whether the GraphQL API is enabled for this Vector instance." + description: "Whether the API is enabled for this Vector instance." common: true required: false } - graphql: { - type: bool: default: true - description: """ - Whether the endpoint for receiving and processing GraphQL queries is - enabled for the API. The endpoint is accessible via the `/graphql` - endpoint of the address set using the `bind` parameter. - """ - common: true - required: false - } - playground: { - type: bool: default: true - description: """ - Whether the [GraphQL Playground](https://github.com/graphql/graphql-playground) is enabled - for the API. The Playground is accessible via the `/playground` endpoint - of the address set using the `bind` parameter. Note that the `playground` - endpoint will only be enabled if the `graphql` endpoint is also enabled. - """ - common: false - required: false - } } description: "API options." warnings: ["The API currently does not support authentication. Only enable it in isolated environments or for debugging. It must not be exposed to untrusted clients."] diff --git a/website/layouts/shortcodes/grpc-migration-warning.html b/website/layouts/shortcodes/grpc-migration-warning.html new file mode 100644 index 0000000000000..b02140664df1a --- /dev/null +++ b/website/layouts/shortcodes/grpc-migration-warning.html @@ -0,0 +1 @@ +{{ partial "admonition.html" (dict "type" "warning" "icon" "exclamation-triangle" "content" "The Vector observability API migrated from GraphQL to gRPC in version 0.55.0.") }} From f1d8a93402168a2c5daf1eab2f4584aa5762bac4 Mon Sep 17 00:00:00 2001 From: iMithrellas Date: Tue, 24 Mar 2026 16:52:15 +0100 Subject: [PATCH 047/326] fix(windows platform): correct stop/start state checks and add configurable stop timeout (#24772) * fix(windows service): wait for stopped state on stop * enhancement(windows service): make stop wait timeout configurable for stop and uninstall --------- Co-authored-by: Pavlos Rontidis --- .../windows_service_stop_state_wait.fix.md | 3 + src/service.rs | 71 ++++++++++++++++--- src/vector_windows.rs | 58 +++++++++------ 3 files changed, 100 insertions(+), 32 deletions(-) create mode 100644 changelog.d/windows_service_stop_state_wait.fix.md diff --git a/changelog.d/windows_service_stop_state_wait.fix.md b/changelog.d/windows_service_stop_state_wait.fix.md new file mode 100644 index 0000000000000..45b60f01ca5fa --- /dev/null +++ b/changelog.d/windows_service_stop_state_wait.fix.md @@ -0,0 +1,3 @@ +Fixed Windows service state checks in `vector service start`/`stop`, and made `vector service stop` wait until the service reaches `Stopped`. Added `--stop-timeout` to `vector service stop` and `vector service uninstall`. + +authors: iMithrellas diff --git a/src/service.rs b/src/service.rs index 5f384b888490d..974ac1ddeaf70 100644 --- a/src/service.rs +++ b/src/service.rs @@ -101,6 +101,18 @@ impl InstallOpts { } } +#[derive(Parser, Debug)] +#[command(rename_all = "kebab-case")] +struct UninstallOpts { + /// The name of the service. + #[arg(long)] + name: Option, + + /// How long to wait for the service to stop before uninstalling, in seconds. + #[arg(default_value = "10", long)] + stop_timeout: u32, +} + #[derive(Parser, Debug)] #[command(rename_all = "kebab-case")] struct RestartOpts { @@ -113,6 +125,18 @@ struct RestartOpts { stop_timeout: u32, } +#[derive(Parser, Debug)] +#[command(rename_all = "kebab-case")] +struct StopOpts { + /// The name of the service. + #[arg(long)] + name: Option, + + /// How long to wait for the service to stop, in seconds. + #[arg(default_value = "10", long)] + stop_timeout: u32, +} + impl RestartOpts { fn service_info(&self) -> ServiceInfo { let mut default_service = ServiceInfo::default(); @@ -123,6 +147,26 @@ impl RestartOpts { } } +impl UninstallOpts { + fn service_info(&self) -> ServiceInfo { + let mut default_service = ServiceInfo::default(); + let service_name = self.name.as_deref().unwrap_or(DEFAULT_SERVICE_NAME); + + default_service.name = OsString::from(service_name); + default_service + } +} + +impl StopOpts { + fn service_info(&self) -> ServiceInfo { + let mut default_service = ServiceInfo::default(); + let service_name = self.name.as_deref().unwrap_or(DEFAULT_SERVICE_NAME); + + default_service.name = OsString::from(service_name); + default_service + } +} + #[derive(Parser, Debug)] #[command(rename_all = "kebab-case")] struct StandardOpts { @@ -147,11 +191,11 @@ enum SubCommand { /// Install the service. Install(InstallOpts), /// Uninstall the service. - Uninstall(StandardOpts), + Uninstall(UninstallOpts), /// Start the service. Start(StandardOpts), /// Stop the service. - Stop(StandardOpts), + Stop(StopOpts), /// Restart the service. Restart(RestartOpts), } @@ -182,9 +226,9 @@ impl Default for ServiceInfo { #[derive(Debug, Clone, PartialEq)] enum ControlAction { Install, - Uninstall, + Uninstall { stop_timeout: Duration }, Start, - Stop, + Stop { stop_timeout: Duration }, Restart { stop_timeout: Duration }, } @@ -196,10 +240,17 @@ pub fn cmd(opts: &Opts) -> exitcode::ExitCode { control_service(&opts.service_info(), ControlAction::Install) } SubCommand::Uninstall(opts) => { - control_service(&opts.service_info(), ControlAction::Uninstall) + let stop_timeout = Duration::from_secs(opts.stop_timeout as u64); + control_service( + &opts.service_info(), + ControlAction::Uninstall { stop_timeout }, + ) } SubCommand::Start(opts) => control_service(&opts.service_info(), ControlAction::Start), - SubCommand::Stop(opts) => control_service(&opts.service_info(), ControlAction::Stop), + SubCommand::Stop(opts) => { + let stop_timeout = Duration::from_secs(opts.stop_timeout as u64); + control_service(&opts.service_info(), ControlAction::Stop { stop_timeout }) + } SubCommand::Restart(opts) => { let stop_timeout = Duration::from_secs(opts.stop_timeout as u64); control_service( @@ -233,17 +284,17 @@ fn control_service(service: &ServiceInfo, action: ControlAction) -> exitcode::Ex &service_definition, vector_windows::service_control::ControlAction::Install, ), - ControlAction::Uninstall => vector_windows::service_control::control( + ControlAction::Uninstall { stop_timeout } => vector_windows::service_control::control( &service_definition, - vector_windows::service_control::ControlAction::Uninstall, + vector_windows::service_control::ControlAction::Uninstall { stop_timeout }, ), ControlAction::Start => vector_windows::service_control::control( &service_definition, vector_windows::service_control::ControlAction::Start, ), - ControlAction::Stop => vector_windows::service_control::control( + ControlAction::Stop { stop_timeout } => vector_windows::service_control::control( &service_definition, - vector_windows::service_control::ControlAction::Stop, + vector_windows::service_control::ControlAction::Stop { stop_timeout }, ), ControlAction::Restart { stop_timeout } => vector_windows::service_control::control( &service_definition, diff --git a/src/vector_windows.rs b/src/vector_windows.rs index dc29efec8170e..66fdfe22dd990 100644 --- a/src/vector_windows.rs +++ b/src/vector_windows.rs @@ -81,9 +81,9 @@ pub mod service_control { #[derive(Debug, Copy, Clone, PartialEq)] pub enum ControlAction { Install, - Uninstall, + Uninstall { stop_timeout: Duration }, Start, - Stop, + Stop { stop_timeout: Duration }, Restart { stop_timeout: Duration }, } @@ -114,9 +114,13 @@ pub mod service_control { fn from_str(s: &str) -> std::result::Result { match s { "install" => Ok(ControlAction::Install), - "uninstall" => Ok(ControlAction::Uninstall), + "uninstall" => Ok(ControlAction::Uninstall { + stop_timeout: Duration::from_secs(10), + }), "start" => Ok(ControlAction::Start), - "stop" => Ok(ControlAction::Stop), + "stop" => Ok(ControlAction::Stop { + stop_timeout: Duration::from_secs(10), + }), _ => Err(format!("invalid option {} for ControlAction", s)), } } @@ -125,10 +129,12 @@ pub mod service_control { pub fn control(service_def: &ServiceDefinition, action: ControlAction) -> crate::Result<()> { match action { ControlAction::Start => start_service(service_def), - ControlAction::Stop => stop_service(service_def), + ControlAction::Stop { stop_timeout } => stop_service(service_def, stop_timeout), ControlAction::Restart { stop_timeout } => restart_service(service_def, stop_timeout), ControlAction::Install => install_service(service_def), - ControlAction::Uninstall => uninstall_service(service_def), + ControlAction::Uninstall { stop_timeout } => { + uninstall_service(service_def, stop_timeout) + } } } @@ -138,7 +144,7 @@ pub mod service_control { let service_status = service.query_status().context(ServiceSnafu)?; if service_status.current_state != ServiceState::StartPending - || service_status.current_state != ServiceState::Running + && service_status.current_state != ServiceState::Running { service.start(&[] as &[OsString]).context(ServiceSnafu)?; emit!(WindowsServiceStart { @@ -155,26 +161,31 @@ pub mod service_control { Ok(()) } - fn stop_service(service_def: &ServiceDefinition) -> crate::Result<()> { + fn stop_service(service_def: &ServiceDefinition, stop_timeout: Duration) -> crate::Result<()> { let service_access = ServiceAccess::QUERY_STATUS | ServiceAccess::STOP; let service = open_service(service_def, service_access)?; let service_status = service.query_status().context(ServiceSnafu)?; + let already_stopped = service_status.current_state == ServiceState::Stopped; - if service_status.current_state != ServiceState::StopPending - || service_status.current_state != ServiceState::Stopped - { + if service_status.current_state != ServiceState::StopPending && !already_stopped { service.stop().context(ServiceSnafu)?; - emit!(WindowsServiceStop { - name: &*service_def.name.to_string_lossy(), - already_stopped: false, - }); - } else { - emit!(WindowsServiceStop { - name: &*service_def.name.to_string_lossy(), - already_stopped: true, - }); } + if !already_stopped { + let service_status = ensure_state( + &service, + ServiceState::Stopped, + stop_timeout, + Duration::from_secs(1), + )?; + handle_service_exit_code(service_status.exit_code); + } + + emit!(WindowsServiceStop { + name: &*service_def.name.to_string_lossy(), + already_stopped, + }); + Ok(()) } @@ -242,7 +253,10 @@ pub mod service_control { Ok(()) } - fn uninstall_service(service_def: &ServiceDefinition) -> crate::Result<()> { + fn uninstall_service( + service_def: &ServiceDefinition, + stop_timeout: Duration, + ) -> crate::Result<()> { let service_access = ServiceAccess::QUERY_STATUS | ServiceAccess::STOP | ServiceAccess::DELETE; let service = open_service(service_def, service_access)?; @@ -259,7 +273,7 @@ pub mod service_control { let service_status = ensure_state( &service, ServiceState::Stopped, - Duration::from_secs(10), + stop_timeout, Duration::from_secs(1), )?; handle_service_exit_code(service_status.exit_code); From 0122cf6ec8fe7cce20a2bb694db137e3d5a9038a Mon Sep 17 00:00:00 2001 From: Thomas Date: Tue, 24 Mar 2026 12:04:14 -0400 Subject: [PATCH 048/326] chore(sinks)!: remove deprecated `headers` option from http and opentelemetry sinks (#24994) * chore(sinks): remove deprecated `headers` option from http and opentelemetry sinks * Fix tests --- benches/http.rs | 1 - .../remove_deprecated_http_headers_option.breaking.md | 4 ++++ src/sinks/axiom/config.rs | 1 - src/sinks/http/config.rs | 11 +---------- src/sinks/http/tests.rs | 1 - src/sinks/opentelemetry/mod.rs | 1 - src/sinks/util/http.rs | 7 ------- .../cue/reference/components/sinks/generated/http.cue | 11 ----------- .../components/sinks/generated/opentelemetry.cue | 11 ----------- 9 files changed, 5 insertions(+), 43 deletions(-) create mode 100644 changelog.d/remove_deprecated_http_headers_option.breaking.md diff --git a/benches/http.rs b/benches/http.rs index 0bc99dec551cf..30b35b8b257f7 100644 --- a/benches/http.rs +++ b/benches/http.rs @@ -55,7 +55,6 @@ fn benchmark_http(c: &mut Criterion) { compression: *compression, method: Default::default(), auth: Default::default(), - headers: Default::default(), payload_prefix: Default::default(), payload_suffix: Default::default(), batch, diff --git a/changelog.d/remove_deprecated_http_headers_option.breaking.md b/changelog.d/remove_deprecated_http_headers_option.breaking.md new file mode 100644 index 0000000000000..b2b1fd28b869e --- /dev/null +++ b/changelog.d/remove_deprecated_http_headers_option.breaking.md @@ -0,0 +1,4 @@ +The `headers` option has been removed from the `http` and `opentelemetry` sinks. +Use `request.headers` instead. This option has been deprecated since v0.33.0. + +authors: thomasqueirozb diff --git a/src/sinks/axiom/config.rs b/src/sinks/axiom/config.rs index 0349fc02831b8..61421973d5475 100644 --- a/src/sinks/axiom/config.rs +++ b/src/sinks/axiom/config.rs @@ -170,7 +170,6 @@ impl SinkConfig for AxiomConfig { request, acknowledgements: self.acknowledgements, batch: self.batch, - headers: None, encoding: EncodingConfigWithFraming::new( Some(FramingConfig::NewlineDelimited), SerializerConfig::Json(JsonSerializerConfig { diff --git a/src/sinks/http/config.rs b/src/sinks/http/config.rs index 81df73add1efb..a3a105fa5c608 100644 --- a/src/sinks/http/config.rs +++ b/src/sinks/http/config.rs @@ -57,13 +57,6 @@ pub struct HttpSinkConfig { #[configurable(derived)] pub auth: Option, - /// A list of custom headers to add to each request. - #[configurable(deprecated = "This option has been deprecated, use `request.headers` instead.")] - #[configurable(metadata( - docs::additional_props_description = "An HTTP request header and it's value." - ))] - pub headers: Option>, - #[configurable(derived)] #[serde(default)] pub compression: Compression, @@ -242,8 +235,7 @@ impl SinkConfig for HttpSinkConfig { let encoder = self.build_encoder()?; let transformer = self.encoding.transformer(); - let mut request = self.request.clone(); - request.add_old_option(self.headers.clone()); + let request = self.request.clone(); validate_headers(&request.headers, self.auth.is_some())?; let (static_headers, template_headers) = request.split_headers(); @@ -406,7 +398,6 @@ mod tests { Transformer::default(), ), auth: None, - headers: None, compression: Compression::default(), batch: BatchConfig::default(), request: RequestConfig::default(), diff --git a/src/sinks/http/tests.rs b/src/sinks/http/tests.rs index 7182d608a865a..12ff994239f5d 100644 --- a/src/sinks/http/tests.rs +++ b/src/sinks/http/tests.rs @@ -59,7 +59,6 @@ fn default_cfg(encoding: EncodingConfigWithFraming) -> HttpSinkConfig { uri: Default::default(), method: Default::default(), auth: Default::default(), - headers: Default::default(), compression: Default::default(), encoding, payload_prefix: Default::default(), diff --git a/src/sinks/opentelemetry/mod.rs b/src/sinks/opentelemetry/mod.rs index 88963f8603cde..7d73e25c4030d 100644 --- a/src/sinks/opentelemetry/mod.rs +++ b/src/sinks/opentelemetry/mod.rs @@ -49,7 +49,6 @@ impl Default for Protocol { uri: Default::default(), method: HttpMethod::Post, auth: Default::default(), - headers: Default::default(), compression: Default::default(), payload_prefix: Default::default(), payload_suffix: Default::default(), diff --git a/src/sinks/util/http.rs b/src/sinks/util/http.rs index e21ef48e4ed66..e19025e3c6ee3 100644 --- a/src/sinks/util/http.rs +++ b/src/sinks/util/http.rs @@ -684,13 +684,6 @@ fn headers_examples() -> BTreeMap { } impl RequestConfig { - pub fn add_old_option(&mut self, headers: Option>) { - if let Some(headers) = headers { - warn!("Option `headers` has been deprecated. Use `request.headers` instead."); - self.headers.extend(headers); - } - } - pub fn split_headers(&self) -> (BTreeMap, BTreeMap) { let mut static_headers = BTreeMap::new(); let mut template_headers = BTreeMap::new(); diff --git a/website/cue/reference/components/sinks/generated/http.cue b/website/cue/reference/components/sinks/generated/http.cue index 61fc5eab98ff2..0fcc53fab8092 100644 --- a/website/cue/reference/components/sinks/generated/http.cue +++ b/website/cue/reference/components/sinks/generated/http.cue @@ -780,17 +780,6 @@ generated: components: sinks: http: configuration: { } } } - headers: { - deprecated: true - deprecated_message: "This option has been deprecated, use `request.headers` instead." - description: "A list of custom headers to add to each request." - required: false - type: object: options: "*": { - description: "An HTTP request header and it's value." - required: true - type: string: {} - } - } method: { description: "The HTTP method to use when making the request." required: false diff --git a/website/cue/reference/components/sinks/generated/opentelemetry.cue b/website/cue/reference/components/sinks/generated/opentelemetry.cue index f48dfd18a61a4..4505d8a9bf512 100644 --- a/website/cue/reference/components/sinks/generated/opentelemetry.cue +++ b/website/cue/reference/components/sinks/generated/opentelemetry.cue @@ -783,17 +783,6 @@ generated: components: sinks: opentelemetry: configuration: protocol: { } } } - headers: { - deprecated: true - deprecated_message: "This option has been deprecated, use `request.headers` instead." - description: "A list of custom headers to add to each request." - required: false - type: object: options: "*": { - description: "An HTTP request header and it's value." - required: true - type: string: {} - } - } method: { description: "The HTTP method to use when making the request." required: false From 8bac1dbdcd27550444ea3b73e9dd4ef36172b2d4 Mon Sep 17 00:00:00 2001 From: Vladimir Zhuk Date: Tue, 24 Mar 2026 15:00:40 -0400 Subject: [PATCH 049/326] chore(datadog_metrics sink): switch series v2 and sketches to zstd compression (#24956) * chore(datadog_metrics sink): switch series v2 endpoint to zstd compression Co-Authored-By: Claude Sonnet 4.6 * chore(regression): switch statsd_to_datadog_metrics to ingress_throughput The egress_throughput goal measures compressed bytes received by the blackhole. Switching v2 series from zlib to zstd produces smaller compressed payloads (better compression ratio), which registers as a false regression in egress bytes/sec. ingress_throughput measures how fast Vector consumes statsd data from the generator, which is compression-agnostic and reflects actual pipeline performance. Co-Authored-By: Claude Opus 4.6 (1M context) * fix(datadog_metrics sink): prevent division-by-zero in proptest with zero limits Start proptest ranges at 1 instead of 0 for uncompressed_limit and compressed_limit. The old validate_payload_size_limits rejected zero limits, but with_payload_limits is now infallible, so finish() can panic on division-by-zero when computing recommended_splits. Co-Authored-By: Claude Opus 4.6 (1M context) * chore(datadog_metrics sink): switch sketches endpoint to zstd compression Sketches endpoint now uses zstd instead of zlib, matching Series v2. Only Series v1 remains on zlib. Validated against real Datadog API: 36/36 correctness checks passed, all 18 metrics match between v1 and v2. Co-Authored-By: Claude Opus 4.6 (1M context) * fix(datadog_metrics sink): correct changelog to reflect sketches use zstd The changelog incorrectly stated that Sketches continue to use zlib, but the code routes Sketches to zstd compression. Co-Authored-By: Claude Opus 4.6 (1M context) * refactor(datadog_metrics sink): move compression constants to vector-common Move zlib and zstd compression-bound constants from inline locals in DatadogMetricsCompression::max_compressed_size to lib/vector-common/src/constants.rs with descriptive names and doc comments linking to their specifications. Co-Authored-By: Claude Opus 4.6 (1M context) * refactor(datadog_metrics sink): move inline use statements to module top Move use statements from inside test helper function bodies to the top of the test module, as is conventional in Rust. Co-Authored-By: Claude Opus 4.6 (1M context) * cargo fmt * fix(datadog_metrics sink): update integration test decompression for zstd The decompress_payload helper in integration_tests.rs hardcoded zlib decompression, but Series v2 now uses zstd. Auto-detect the compression format via is_zstd so tests work for both zlib and zstd payloads. Co-Authored-By: Claude Opus 4.6 (1M context) --------- Co-authored-by: Claude Sonnet 4.6 Co-authored-by: Pavlos Rontidis --- ...adog_metrics_zstd_series_v2.enhancement.md | 3 + lib/vector-common/src/constants.rs | 16 + .../statsd_to_datadog_metrics/experiment.yaml | 2 +- src/sinks/datadog/metrics/config.rs | 33 +- src/sinks/datadog/metrics/encoder.rs | 811 ++++++++++++------ .../datadog/metrics/integration_tests.rs | 13 +- src/sinks/datadog/metrics/request_builder.rs | 20 +- src/sinks/datadog/metrics/service.rs | 8 +- 8 files changed, 629 insertions(+), 277 deletions(-) create mode 100644 changelog.d/24956_datadog_metrics_zstd_series_v2.enhancement.md diff --git a/changelog.d/24956_datadog_metrics_zstd_series_v2.enhancement.md b/changelog.d/24956_datadog_metrics_zstd_series_v2.enhancement.md new file mode 100644 index 0000000000000..1ecaef6b46b8c --- /dev/null +++ b/changelog.d/24956_datadog_metrics_zstd_series_v2.enhancement.md @@ -0,0 +1,3 @@ +The `datadog_metrics` sink now uses zstd compression when submitting metrics to the Series v2 (`/api/v2/series`) and Sketches endpoints. Series v1 continues to use zlib (deflate). + +authors: vladimir-dd diff --git a/lib/vector-common/src/constants.rs b/lib/vector-common/src/constants.rs index 1eeadca45b5dc..8b336d7937344 100644 --- a/lib/vector-common/src/constants.rs +++ b/lib/vector-common/src/constants.rs @@ -1,3 +1,19 @@ pub const GZIP_MAGIC: &[u8] = &[0x1f, 0x8b]; pub const ZLIB_MAGIC: &[u8] = &[0x78]; pub const ZSTD_MAGIC: &[u8] = &[0x28, 0xB5, 0x2F, 0xFD]; + +/// Maximum size of a zlib stored (uncompressed) block in bytes. +/// See: +pub const ZLIB_STORED_BLOCK_SIZE: usize = 16384; + +/// Per-block overhead for zlib stored blocks: 1-byte header + 2-byte length + 2-byte ~length. +/// See: +pub const ZLIB_STORED_BLOCK_OVERHEAD: usize = 5; + +/// Zlib frame overhead: 2-byte header + 4-byte Adler-32 checksum trailer. +/// See: +pub const ZLIB_FRAME_OVERHEAD: usize = 6; + +/// Threshold below which zstd's `ZSTD_compressBound` adds extra margin (128 KiB). +/// See: (`ZSTD_compressBound`) +pub const ZSTD_SMALL_INPUT_THRESHOLD: usize = 128 << 10; diff --git a/regression/cases/statsd_to_datadog_metrics/experiment.yaml b/regression/cases/statsd_to_datadog_metrics/experiment.yaml index 7c0e069a9eddc..3650e2c1157a2 100644 --- a/regression/cases/statsd_to_datadog_metrics/experiment.yaml +++ b/regression/cases/statsd_to_datadog_metrics/experiment.yaml @@ -1,4 +1,4 @@ -optimization_goal: egress_throughput +optimization_goal: ingress_throughput target: name: vector diff --git a/src/sinks/datadog/metrics/config.rs b/src/sinks/datadog/metrics/config.rs index e296bd823d60c..a5fdccede6a14 100644 --- a/src/sinks/datadog/metrics/config.rs +++ b/src/sinks/datadog/metrics/config.rs @@ -87,11 +87,6 @@ impl DatadogMetricsEndpoint { } } - // Gets whether or not this is a series endpoint. - pub const fn is_series(self) -> bool { - matches!(self, Self::Series { .. }) - } - pub(super) const fn payload_limits(self) -> DatadogMetricsPayloadLimits { // from https://docs.datadoghq.com/api/latest/metrics/#submit-metrics let (uncompressed, compressed) = match self { @@ -112,6 +107,32 @@ impl DatadogMetricsEndpoint { compressed, } } + + /// Returns the compression scheme used for this endpoint. + pub(super) const fn compression(self) -> DatadogMetricsCompression { + match self { + Self::Series(SeriesApiVersion::V1) => DatadogMetricsCompression::Zlib, + _ => DatadogMetricsCompression::Zstd, + } + } +} + +/// Selects the compressor for a given Datadog metrics endpoint. +#[derive(Clone, Copy, Debug)] +pub(super) enum DatadogMetricsCompression { + /// zlib (deflate) — used by Series v1. + Zlib, + /// zstd — used by Series v2 and Sketches. + Zstd, +} + +impl DatadogMetricsCompression { + pub(super) const fn content_encoding(self) -> &'static str { + match self { + Self::Zstd => "zstd", + Self::Zlib => "deflate", + } + } } /// Maps Datadog metric endpoints to their actual URI. @@ -270,7 +291,7 @@ impl DatadogMetricsConfig { endpoint_configuration, self.default_namespace.clone(), self.series_api_version, - )?; + ); let protocol = self.get_protocol(dd_common); let sink = DatadogMetricsSink::new( diff --git a/src/sinks/datadog/metrics/encoder.rs b/src/sinks/datadog/metrics/encoder.rs index 9f0b71c87ba98..a626244123861 100644 --- a/src/sinks/datadog/metrics/encoder.rs +++ b/src/sinks/datadog/metrics/encoder.rs @@ -16,7 +16,12 @@ use vector_lib::{ request_metadata::GroupedCountByteSize, }; -use super::config::{DatadogMetricsEndpoint, SeriesApiVersion}; +use vector_common::constants::{ + ZLIB_FRAME_OVERHEAD, ZLIB_STORED_BLOCK_OVERHEAD, ZLIB_STORED_BLOCK_SIZE, + ZSTD_SMALL_INPUT_THRESHOLD, +}; + +use super::config::{DatadogMetricsCompression, DatadogMetricsEndpoint, SeriesApiVersion}; use crate::{ common::datadog::{ DatadogMetricType, DatadogPoint, DatadogSeriesMetric, DatadogSeriesMetricMetadata, @@ -47,23 +52,6 @@ mod ddmetric_proto { include!(concat!(env!("OUT_DIR"), "/datadog.agentpayload.rs")); } -#[derive(Debug, Snafu)] -pub enum CreateError { - #[snafu(display("Invalid compressed/uncompressed payload size limits were given"))] - InvalidLimits, -} - -impl CreateError { - /// Gets the telemetry-friendly string version of this error. - /// - /// The value will be a short string with only lowercase letters and underscores. - pub const fn as_error_type(&self) -> &'static str { - match self { - Self::InvalidLimits => "invalid_payload_limits", - } - } -} - #[derive(Debug, Snafu)] pub enum EncoderError { #[snafu(display( @@ -133,6 +121,16 @@ impl FinishError { struct EncoderState { writer: Compressor, written: usize, + /// Upper bound on uncompressed bytes sitting in the compressor's internal buffer (written but + /// not yet flushed to `writer.get_ref()`). All compressors may buffer internally: zstd holds + /// up to 128 KB per block, zlib's BufWriter holds up to 4 KB. Since `get_ref().len()` only + /// reflects bytes that have been flushed through all layers, we track this bound to avoid + /// underestimating the compressed payload size. + /// + /// Increases by `n` on each write. Resets to `n` when a new compressed block is detected in + /// `writer.get_ref()` (the triggering write may straddle the block boundary, so `n` is a safe + /// upper bound on what remains buffered after the flush). + buffered_bound: usize, buf: Vec, processed: Vec, byte_size: GroupedCountByteSize, @@ -141,8 +139,9 @@ struct EncoderState { impl Default for EncoderState { fn default() -> Self { Self { - writer: get_compressor(), + writer: Compression::zlib_default().into(), written: 0, + buffered_bound: 0, buf: Vec::with_capacity(1024), processed: Vec::new(), byte_size: telemetry().create_request_count_byte_size(), @@ -164,45 +163,62 @@ pub struct DatadogMetricsEncoder { impl DatadogMetricsEncoder { /// Creates a new `DatadogMetricsEncoder` for the given endpoint. - pub fn new( - endpoint: DatadogMetricsEndpoint, - default_namespace: Option, - ) -> Result { + pub fn new(endpoint: DatadogMetricsEndpoint, default_namespace: Option) -> Self { let payload_limits = endpoint.payload_limits(); - Self::with_payload_limits( + + Self { endpoint, - default_namespace, - payload_limits.uncompressed, - payload_limits.compressed, - ) + default_namespace: default_namespace.map(Arc::from), + uncompressed_limit: payload_limits.uncompressed, + compressed_limit: payload_limits.compressed, + state: EncoderState { + writer: endpoint.compression().compressor(), + ..Default::default() + }, + log_schema: log_schema(), + origin_product_value: *ORIGIN_PRODUCT_VALUE, + } } +} +#[cfg(test)] +impl DatadogMetricsEncoder { /// Creates a new `DatadogMetricsEncoder` for the given endpoint, with specific payload limits. + /// + /// Only available in tests; production code always uses the API-defined limits via `new`. pub fn with_payload_limits( endpoint: DatadogMetricsEndpoint, default_namespace: Option, uncompressed_limit: usize, compressed_limit: usize, - ) -> Result { - let (uncompressed_limit, compressed_limit) = - validate_payload_size_limits(endpoint, uncompressed_limit, compressed_limit) - .ok_or(CreateError::InvalidLimits)?; - - Ok(Self { + ) -> Self { + Self { endpoint, default_namespace: default_namespace.map(Arc::from), uncompressed_limit, compressed_limit, - state: EncoderState::default(), + state: EncoderState { + writer: endpoint.compression().compressor(), + ..Default::default() + }, log_schema: log_schema(), origin_product_value: *ORIGIN_PRODUCT_VALUE, - }) + } + } + + /// Returns the current `buffered_bound` value for white-box testing of zstd block-flush reset. + fn buffered_bound(&self) -> usize { + self.state.buffered_bound } } impl DatadogMetricsEncoder { fn reset_state(&mut self) -> EncoderState { - mem::take(&mut self.state) + let new_state = EncoderState { + writer: self.endpoint.compression().compressor(), + ..Default::default() + }; + mem::replace(&mut self.state, new_state) } fn encode_single_metric(&mut self, metric: Metric) -> Result, EncoderError> { @@ -342,24 +358,36 @@ impl DatadogMetricsEncoder { // compressor might have internal state around checksums, etc, that can't be similarly // rolled back. // - // Our strategy is thus: simply take the encoded-but-decompressed size and see if it would - // fit within the compressed limit. In `get_endpoint_payload_size_limits`, we calculate the - // expected maximum overhead of zlib based on all input data being incompressible, which - // zlib ensures will be the worst case as it can figure out whether a compressed or - // uncompressed block would take up more space _before_ choosing which strategy to go with. + // Strategy: split the estimate into two parts: + // 1. Bytes already flushed to the output buffer (`get_ref().len()`) — exact compressed size. + // 2. Bytes still in the compressor's internal buffer plus this new metric — estimated via + // max_compressed_size(buffered_bound + n) (worst-case upper bound). // - // Thus, simply put, we've already accounted for the uncertainty by making our check here - // assume the worst case while our limits assume the worst case _overhead_. Maybe our - // numbers are technically off in the end, but `finish` catches that for us, too. - let compressed_len = self.state.writer.get_ref().len(); - let max_compressed_metric_len = n + max_compressed_overhead_len(n); - if compressed_len + max_compressed_metric_len > self.compressed_limit { + // All compressors may buffer data internally before flushing to the output: zstd buffers + // up to 128 KB per block, zlib's BufWriter holds up to 4 KB. `get_ref().len()` only + // reflects bytes that have been flushed through all layers. We track `buffered_bound` — + // an upper bound on uncompressed bytes written but not yet visible in `get_ref()` — and + // include it in the estimate for all compressor types. + let compression = self.endpoint.compression(); + let flushed_compressed = self.state.writer.get_ref().len(); + if flushed_compressed + compression.max_compressed_size(self.state.buffered_bound + n) + > self.compressed_limit + { return Ok(false); } // We should be safe to write our holding buffer to the compressor and store the metric. + // + // Update buffered_bound: if a new block appeared in the output (flushed_compressed grew), + // reset to n — the triggering write may straddle the block boundary, so n is a safe upper + // bound on what remains buffered. Otherwise accumulate. self.state.writer.write_all(&self.state.buf)?; self.state.written += n; + if self.state.writer.get_ref().len() > flushed_compressed { + self.state.buffered_bound = n; + } else { + self.state.buffered_bound += n; + } Ok(true) } @@ -383,7 +411,10 @@ impl DatadogMetricsEncoder { // Make sure we've written our header already. if self.state.written == 0 { match write_payload_header(self.endpoint, &mut self.state.writer) { - Ok(n) => self.state.written += n, + Ok(n) => { + self.state.written += n; + self.state.buffered_bound += n; + } Err(_) => return Ok(Some(metric)), } } @@ -874,75 +905,41 @@ fn generate_series_metrics( }]) } -fn get_compressor() -> Compressor { - // We use the "zlib default" compressor because it's all Datadog supports, and adding it - // generically to `Compression` would make things a little weird because of the conversion trait - // implementations that are also only none vs gzip. - Compression::zlib_default().into() -} - -const fn max_uncompressed_header_len() -> usize { - SERIES_PAYLOAD_HEADER.len() + SERIES_PAYLOAD_FOOTER.len() -} - -// Datadog ingest APIs accept zlib, which is what we're accounting for here. By default, zlib -// has a 2 byte header and 4 byte CRC trailer. [1] -// -// [1] https://www.zlib.net/zlib_tech.html -const ZLIB_HEADER_TRAILER: usize = 6; - -const fn max_compression_overhead_len(compressed_limit: usize) -> usize { - // We calculate the overhead as the zlib header/trailer plus the worst case overhead of - // compressing `compressed_limit` bytes, such that we assume all of the data we write may not be - // compressed at all. - ZLIB_HEADER_TRAILER + max_compressed_overhead_len(compressed_limit) -} - -const fn max_compressed_overhead_len(len: usize) -> usize { - // Datadog ingest APIs accept zlib, which is what we're accounting for here. - // - // Deflate, the underlying compression algorithm, has a technique to ensure that input data - // can't be encoded in such a way where it's expanded by a meaningful amount. This technique - // allows storing blocks of uncompressed data with only 5 bytes of overhead per block. - // Technically, the blocks can be up to 65KB in Deflate, but modern zlib implementations use - // block sizes of 16KB. [1][2] - // - // We calculate the overhead of compressing a given `len` bytes as the worst case of that many - // bytes being written to the compressor and being unable to be compressed at all - // - // [1] https://www.zlib.net/zlib_tech.html - // [2] https://www.bolet.org/~pornin/deflate-flush-fr.html - const STORED_BLOCK_SIZE: usize = 16384; - (1 + len.saturating_sub(ZLIB_HEADER_TRAILER) / STORED_BLOCK_SIZE) * 5 -} - -const fn validate_payload_size_limits( - endpoint: DatadogMetricsEndpoint, - uncompressed_limit: usize, - compressed_limit: usize, -) -> Option<(usize, usize)> { - if endpoint.is_series() { - // For series, we need to make sure the uncompressed limit can account for the header/footer - // we would add that wraps the encoded metrics up in the expected JSON object. This does - // imply that adding 1 to this limit would be allowed, and obviously we can't encode a - // series metric in a single byte, but this is just a simple sanity check, not an exhaustive - // search of the absolute bare minimum size. - let header_len = max_uncompressed_header_len(); - if uncompressed_limit <= header_len { - return None; +impl DatadogMetricsCompression { + fn compressor(self) -> Compressor { + match self { + Self::Zstd => Compression::zstd_default().into(), + Self::Zlib => Compression::zlib_default().into(), } } - // Get the maximum possible overhead of the compression container, based on the incoming - // _uncompressed_ limit. We use the uncompressed limit because we're calculating the maximum - // overhead in the case that, theoretically, none of the input data was compressible. This - // possibility is essentially impossible, but serves as a proper worst-case-scenario check. - let max_compression_overhead = max_compression_overhead_len(uncompressed_limit); - if compressed_limit <= max_compression_overhead { - return None; + /// Returns the worst-case compressed size of `n` uncompressed bytes. + /// + /// For zlib (deflate), the worst case occurs when data is entirely incompressible and stored in + /// uncompressed blocks (5 bytes overhead per 16 KB block, as per the DEFLATE spec). + /// + /// For zstd, this uses the same formula as `ZSTD_compressBound` from the zstd C library. + const fn max_compressed_size(self, n: usize) -> usize { + match self { + Self::Zlib => { + // Deflate stores incompressible data in uncompressed blocks, each with fixed + // overhead. We subtract the zlib frame from the block count since those bytes + // are not stored-block data. + n + (1 + n.saturating_sub(ZLIB_FRAME_OVERHEAD) / ZLIB_STORED_BLOCK_SIZE) + * ZLIB_STORED_BLOCK_OVERHEAD + } + Self::Zstd => { + // zstd_safe::compress_bound is not const, so we use the same formula it uses + // internally: srcSize + (srcSize >> 8) + small correction for inputs < 128 KB. + n + (n >> 8) + + if n < ZSTD_SMALL_INPUT_THRESHOLD { + (ZSTD_SMALL_INPUT_THRESHOLD - n) >> 11 + } else { + 0 + } + } + } } - - Some((uncompressed_limit, compressed_limit)) } fn write_payload_header( @@ -983,11 +980,8 @@ fn write_payload_footer( #[cfg(test)] mod tests { - use std::{ - io::{self, copy}, - num::NonZeroU32, - sync::Arc, - }; + use std::io::{self, Write as _}; + use std::{num::NonZeroU32, sync::Arc}; use bytes::{BufMut, Bytes, BytesMut}; use chrono::{DateTime, TimeZone, Timelike, Utc}; @@ -1010,19 +1004,30 @@ mod tests { use super::{ DatadogMetricsEncoder, EncoderError, ddmetric_proto, encode_proto_key_and_message, - encode_tags, encode_timestamp, generate_series_metrics, get_compressor, - get_sketch_payload_sketches_field_number, max_compression_overhead_len, - max_uncompressed_header_len, series_to_proto_message, sketch_to_proto_message, - validate_payload_size_limits, write_payload_footer, write_payload_header, + encode_tags, encode_timestamp, generate_series_metrics, + get_sketch_payload_sketches_field_number, series_to_proto_message, sketch_to_proto_message, + write_payload_footer, write_payload_header, }; use crate::{ common::datadog::DatadogMetricType, - sinks::datadog::metrics::{ - config::{DatadogMetricsEndpoint, SeriesApiVersion}, - encoder::{DEFAULT_DD_ORIGIN_PRODUCT_VALUE, ORIGIN_PRODUCT_VALUE}, + sinks::{ + datadog::metrics::{ + config::{DatadogMetricsCompression, DatadogMetricsEndpoint, SeriesApiVersion}, + encoder::{DEFAULT_DD_ORIGIN_PRODUCT_VALUE, ORIGIN_PRODUCT_VALUE}, + }, + util::{Compression, Compressor}, }, }; + const fn max_uncompressed_header_len(endpoint: DatadogMetricsEndpoint) -> usize { + match endpoint { + DatadogMetricsEndpoint::Series(SeriesApiVersion::V1) => { + super::SERIES_PAYLOAD_HEADER.len() + super::SERIES_PAYLOAD_FOOTER.len() + } + _ => 0, + } + } + fn get_simple_counter() -> Metric { let value = MetricValue::Counter { value: 3.14 }; Metric::new("basic_counter", MetricKind::Incremental, value).with_timestamp(Some(ts())) @@ -1048,8 +1053,8 @@ mod tests { .with_timestamp(Some(ts())) } - fn get_compressed_empty_series_payload() -> Bytes { - let mut compressor = get_compressor(); + fn get_compressed_empty_series_v1_payload() -> Bytes { + let mut compressor = Compressor::from(Compression::zlib_default()); _ = write_payload_header( DatadogMetricsEndpoint::Series(SeriesApiVersion::V1), @@ -1066,14 +1071,104 @@ mod tests { } fn get_compressed_empty_sketches_payload() -> Bytes { - get_compressor().finish().expect("should not fail").freeze() + Compressor::from(Compression::zstd_default()) + .finish() + .expect("should not fail") + .freeze() + } + + fn get_compressed_empty_series_v2_payload() -> Bytes { + Compressor::from(Compression::zstd_default()) + .finish() + .expect("should not fail") + .freeze() } - fn decompress_payload(payload: Bytes) -> io::Result { + fn decompress_zlib_payload(payload: Bytes) -> io::Result { let mut decompressor = ZlibDecoder::new(&payload[..]); let mut decompressed = BytesMut::new().writer(); - let result = copy(&mut decompressor, &mut decompressed); - result.map(|_| decompressed.into_inner().freeze()) + io::copy(&mut decompressor, &mut decompressed)?; + Ok(decompressed.into_inner().freeze()) + } + + fn decompress_zstd_payload(payload: Bytes) -> io::Result { + let decompressed = zstd::decode_all(&payload[..])?; + Ok(Bytes::from(decompressed)) + } + + /// Returns the number of bytes added to the compressor's output buffer after writing `n` + /// bytes of high-entropy data. Measures only the *incremental* bytes, not the frame overhead + /// that `finish()` would append (Adler-32 / empty final block for zlib, end frame for zstd). + /// + /// This mirrors how `try_compress_buffer` uses `max_compressed_size`: it checks how many + /// more compressed bytes would be produced, against the current running output length. + /// Compresses `n` bytes of high-entropy (worst-case for compression) data and returns the + /// total output size after `finish()`. + fn total_compressed_len(compression: DatadogMetricsCompression, n: usize) -> usize { + // Xorshift64 — period 2^64-1, passes BigCrush, produces statistically random bytes + // that neither zlib nor zstd can compress significantly. + let mut state = 0xdeadbeef_cafebabe_u64; + let data: Vec = (0..n) + .map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state as u8 + }) + .collect(); + let mut compressor = compression.compressor(); + compressor.write_all(&data).expect("write should succeed"); + compressor.finish().expect("finish should succeed").len() + } + + /// Validates that `max_compressed_size(n)` is a true upper bound on the compressed bytes + /// attributable to `n` uncompressed bytes, for both zlib and zstd. + /// + /// We measure `total_compressed_len(n) - total_compressed_len(0)` to strip the fixed frame + /// overhead (header + trailer) written regardless of input size, isolating the bytes + /// contributed by the data itself. + #[test] + fn max_compressed_size_is_upper_bound() { + // zlib stored-block boundary: 16 384 bytes; zstd block boundary: 131 072 bytes. + let test_sizes = [ + 0, 1, 100, 1_000, 16_383, 16_384, 16_385, 32_767, 32_768, 131_071, 131_072, 131_073, + 500_000, + ]; + + let zlib_frame = total_compressed_len(DatadogMetricsCompression::Zlib, 0); + let zstd_frame = total_compressed_len(DatadogMetricsCompression::Zstd, 0); + + // The formula must not overestimate by more than 1% of input + 64 bytes (a small + // constant that covers the zstd correction term for very small inputs). + let max_slack = |n: usize| n / 100 + 64; + + for &n in &test_sizes { + let actual_zlib = total_compressed_len(DatadogMetricsCompression::Zlib, n) - zlib_frame; + let max_zlib = DatadogMetricsCompression::Zlib.max_compressed_size(n); + assert!( + actual_zlib <= max_zlib, + "zlib n={n}: formula underestimates: actual={actual_zlib} > max={max_zlib}" + ); + assert!( + max_zlib - actual_zlib <= max_slack(n), + "zlib n={n}: formula overestimates: slack={} > {}", + max_zlib - actual_zlib, + max_slack(n) + ); + + let actual_zstd = total_compressed_len(DatadogMetricsCompression::Zstd, n) - zstd_frame; + let max_zstd = DatadogMetricsCompression::Zstd.max_compressed_size(n); + assert!( + actual_zstd <= max_zstd, + "zstd n={n}: formula underestimates: actual={actual_zstd} > max={max_zstd}" + ); + assert!( + max_zstd - actual_zstd <= max_slack(n), + "zstd n={n}: formula overestimates: slack={} > {}", + max_zstd - actual_zstd, + max_slack(n) + ); + } } fn ts() -> DateTime { @@ -1149,8 +1244,7 @@ mod tests { #[test] fn incorrect_metric_for_endpoint_causes_error() { // Series metrics can't go to the sketches endpoint. - let mut sketch_encoder = DatadogMetricsEncoder::new(DatadogMetricsEndpoint::Sketches, None) - .expect("default payload size limits should be valid"); + let mut sketch_encoder = DatadogMetricsEncoder::new(DatadogMetricsEndpoint::Sketches, None); let series_result = sketch_encoder.try_encode(get_simple_counter()); assert!(matches!( series_result.err(), @@ -1159,8 +1253,7 @@ mod tests { // And sketches can't go to the series endpoint. let mut series_v1_encoder = - DatadogMetricsEncoder::new(DatadogMetricsEndpoint::Series(SeriesApiVersion::V1), None) - .expect("default payload size limits should be valid"); + DatadogMetricsEncoder::new(DatadogMetricsEndpoint::Series(SeriesApiVersion::V1), None); let sketch_result = series_v1_encoder.try_encode(get_simple_sketch()); assert!(matches!( sketch_result.err(), @@ -1168,8 +1261,7 @@ mod tests { )); let mut series_v2_encoder = - DatadogMetricsEncoder::new(DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), None) - .expect("default payload size limits should be valid"); + DatadogMetricsEncoder::new(DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), None); let sketch_result = series_v2_encoder.try_encode(get_simple_sketch()); assert!(matches!( sketch_result.err(), @@ -1380,8 +1472,7 @@ mod tests { // This is a simple test where we ensure that a single metric, with the default limits, can // be encoded without hitting any errors. let mut encoder = - DatadogMetricsEncoder::new(DatadogMetricsEndpoint::Series(SeriesApiVersion::V1), None) - .expect("default payload size limits should be valid"); + DatadogMetricsEncoder::new(DatadogMetricsEndpoint::Series(SeriesApiVersion::V1), None); let counter = get_simple_counter(); let expected = counter.clone(); @@ -1404,8 +1495,7 @@ mod tests { // This is a simple test where we ensure that a single metric, with the default limits, can // be encoded without hitting any errors. let mut encoder = - DatadogMetricsEncoder::new(DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), None) - .expect("default payload size limits should be valid"); + DatadogMetricsEncoder::new(DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), None); let counter = get_simple_counter(); let expected = counter.clone(); @@ -1427,8 +1517,7 @@ mod tests { fn encode_single_sketch_metric_with_default_limits() { // This is a simple test where we ensure that a single metric, with the default limits, can // be encoded without hitting any errors. - let mut encoder = DatadogMetricsEncoder::new(DatadogMetricsEndpoint::Sketches, None) - .expect("default payload size limits should be valid"); + let mut encoder = DatadogMetricsEncoder::new(DatadogMetricsEndpoint::Sketches, None); let sketch = get_simple_sketch(); let expected = sketch.clone(); @@ -1450,8 +1539,7 @@ mod tests { fn encode_empty_sketch() { // This is a simple test where we ensure that a single metric, with the default limits, can // be encoded without hitting any errors. - let mut encoder = DatadogMetricsEncoder::new(DatadogMetricsEndpoint::Sketches, None) - .expect("default payload size limits should be valid"); + let mut encoder = DatadogMetricsEncoder::new(DatadogMetricsEndpoint::Sketches, None); let sketch = Metric::new( "empty", MetricKind::Incremental, @@ -1511,77 +1599,6 @@ mod tests { assert_eq!(normal_buf, incremental_buf); } - #[test] - fn payload_size_limits_series() { - // Get the maximum length of the header/trailer data. - let header_len = max_uncompressed_header_len(); - - // This is too small. - let result = validate_payload_size_limits( - DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), - header_len, - usize::MAX, - ); - assert_eq!(result, None); - - // This is just right. - let result = validate_payload_size_limits( - DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), - header_len + 1, - usize::MAX, - ); - assert_eq!(result, Some((header_len + 1, usize::MAX))); - - // Get the maximum compressed overhead length, based on our input uncompressed size. This - // represents the worst case overhead based on the input data (of length usize::MAX, in this - // case) being entirely incompressible. - let compression_overhead_len = max_compression_overhead_len(usize::MAX); - - // This is too small. - let result = validate_payload_size_limits( - DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), - usize::MAX, - compression_overhead_len, - ); - assert_eq!(result, None); - - // This is just right. - let result = validate_payload_size_limits( - DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), - usize::MAX, - compression_overhead_len + 1, - ); - assert_eq!(result, Some((usize::MAX, compression_overhead_len + 1))); - } - - #[test] - fn payload_size_limits_sketches() { - // There's no lower bound on uncompressed size for the sketches payload. - let result = validate_payload_size_limits(DatadogMetricsEndpoint::Sketches, 0, usize::MAX); - assert_eq!(result, Some((0, usize::MAX))); - - // Get the maximum compressed overhead length, based on our input uncompressed size. This - // represents the worst case overhead based on the input data (of length usize::MAX, in this - // case) being entirely incompressible. - let compression_overhead_len = max_compression_overhead_len(usize::MAX); - - // This is too small. - let result = validate_payload_size_limits( - DatadogMetricsEndpoint::Sketches, - usize::MAX, - compression_overhead_len, - ); - assert_eq!(result, None); - - // This is just right. - let result = validate_payload_size_limits( - DatadogMetricsEndpoint::Sketches, - usize::MAX, - compression_overhead_len + 1, - ); - assert_eq!(result, Some((usize::MAX, compression_overhead_len + 1))); - } - #[test] fn default_payload_limits_are_endpoint_aware() { let v1 = DatadogMetricsEndpoint::Series(SeriesApiVersion::V1).payload_limits(); @@ -1609,8 +1626,7 @@ mod tests { let mut encoder = DatadogMetricsEncoder::new( DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), None, - ) - .expect("default payload size limits should be valid"); + ); let mut next_pending = Vec::new(); let mut hit_limit = false; @@ -1658,14 +1674,14 @@ mod tests { // We manually create the encoder with an arbitrarily low "uncompressed" limit but high // "compressed" limit to exercise the codepath that should avoid encoding a metric when the // uncompressed payload would exceed the limit. - let header_len = max_uncompressed_header_len(); + let header_len = + max_uncompressed_header_len(DatadogMetricsEndpoint::Series(SeriesApiVersion::V1)); let mut encoder = DatadogMetricsEncoder::with_payload_limits( DatadogMetricsEndpoint::Series(SeriesApiVersion::V1), None, header_len + 1, usize::MAX, - ) - .expect("payload size limits should be valid"); + ); // Trying to encode a metric that would cause us to exceed our uncompressed limits will // _not_ return an error from `try_encode`, but instead will simply return back the metric @@ -1684,11 +1700,11 @@ mod tests { let (payload, processed) = result.unwrap(); assert_eq!( payload.uncompressed_byte_size, - max_uncompressed_header_len() + max_uncompressed_header_len(DatadogMetricsEndpoint::Series(SeriesApiVersion::V1)) ); assert_eq!( payload.into_payload(), - get_compressed_empty_series_payload() + get_compressed_empty_series_v1_payload() ); assert_eq!(processed.len(), 0); } @@ -1703,8 +1719,7 @@ mod tests { None, 1, usize::MAX, - ) - .expect("payload size limits should be valid"); + ); // Trying to encode a metric that would cause us to exceed our uncompressed limits will // _not_ return an error from `try_encode`, but instead will simply return back the metric @@ -1740,8 +1755,7 @@ mod tests { None, uncompressed_limit, compressed_limit, - ) - .expect("payload size limits should be valid"); + ); // Trying to encode a metric that would cause us to exceed our compressed limits will // _not_ return an error from `try_encode`, but instead will simply return back the metric @@ -1760,11 +1774,11 @@ mod tests { let (payload, processed) = result.unwrap(); assert_eq!( payload.uncompressed_byte_size, - max_uncompressed_header_len() + max_uncompressed_header_len(DatadogMetricsEndpoint::Series(SeriesApiVersion::V1)) ); assert_eq!( payload.into_payload(), - get_compressed_empty_series_payload() + get_compressed_empty_series_v1_payload() ); assert_eq!(processed.len(), 0); } @@ -1775,14 +1789,13 @@ mod tests { // "uncompressed" limit to exercise the codepath that should avoid encoding a metric when the // compressed payload would exceed the limit. let uncompressed_limit = 128; - let compressed_limit = 16; + let compressed_limit = 32; let mut encoder = DatadogMetricsEncoder::with_payload_limits( DatadogMetricsEndpoint::Sketches, None, uncompressed_limit, compressed_limit, - ) - .expect("payload size limits should be valid"); + ); // Trying to encode a metric that would cause us to exceed our compressed limits will // _not_ return an error from `try_encode`, but instead will simply return back the metric @@ -1807,6 +1820,284 @@ mod tests { assert_eq!(processed.len(), 0); } + #[test] + fn encode_series_v2_breaks_out_when_limit_reached_compressed() { + // We manually create the encoder with an arbitrarily low "compressed" limit but high + // "uncompressed" limit to exercise the codepath that should avoid encoding a metric when the + // compressed payload would exceed the limit. + let uncompressed_limit = 128; + let compressed_limit = 32; + let mut encoder = DatadogMetricsEncoder::with_payload_limits( + DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), + None, + uncompressed_limit, + compressed_limit, + ); + + // Trying to encode a metric that would cause us to exceed our compressed limits will + // _not_ return an error from `try_encode`, but instead will simply return back the metric + // as it could not be added. + let counter = get_simple_counter(); + let result = encoder.try_encode(counter.clone()); + assert!(result.is_ok()); + assert_eq!(result.unwrap(), Some(counter)); + + // And similarly, since we didn't actually encode a metric, we _should_ be able to finish + // this payload, but it will be empty (effectively, the header/footer will exist) and no + // processed metrics should be returned. + let result = encoder.finish(); + assert!(result.is_ok()); + + let (payload, processed) = result.unwrap(); + assert_eq!(payload.uncompressed_byte_size, 0); + assert_eq!( + payload.into_payload(), + get_compressed_empty_series_v2_payload() + ); + assert_eq!(processed.len(), 0); + } + + #[test] + fn zstd_v2_payload_never_exceeds_512kb_with_incompressible_data() { + // End-to-end regression test using the real 512 KB compressed limit. + // + // Metric names are generated with a xorshift64 PRNG producing random printable ASCII + // (6.5 bits of entropy per byte), making them effectively incompressible for zstd. + // This makes the capacity estimate tight, so the test validates both directions: + // + // Safety (upper bound): payload ≤ 512 KB. + // Without the fix, the encoder ignores zstd's internal 128 KB buffer. When the + // encoder finally stops, finish() flushes that hidden buffer on top of the already + // ~511 KB payload → ~639 KB → TooLarge. + // + // Utilization (lower bound): payload > 95% of 512 KB. + // With incompressible data, actual_compressed ≈ max_cs(uncompressed), so the + // estimate is tight. The ~2.5% gap comes from: (1) compressible proto framing + // (field tags, timestamps, metadata) that zstd compresses better than max_cs + // predicts, (2) the max_cs overhead term (~0.4%), and (3) one-metric stopping + // granularity (~1%). + + // xorshift64 PRNG: 5000 random printable ASCII chars per metric name (0x21..0x7E, + // 93 values ≈ 6.5 bits/byte). Long names ensure the random portion dominates the + // compressible proto framing, maximizing utilization. + const PRINTABLE_START: u8 = 0x21; + const PRINTABLE_END: u8 = 0x7E; + const PRINTABLE_LEN: u64 = (PRINTABLE_END - PRINTABLE_START + 1) as u64; // 93 + let mut xor_state = 0xdeadbeef_cafebabe_u64; + let mut next_name = || -> String { + std::iter::once('m') + .chain((0..4999).map(|_| { + xor_state ^= xor_state << 13; + xor_state ^= xor_state >> 7; + xor_state ^= xor_state << 17; + (PRINTABLE_START + (xor_state % PRINTABLE_LEN) as u8) as char + })) + .collect() + }; + + let mut encoder = + DatadogMetricsEncoder::new(DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), None); + + let mut accepted = 0usize; + loop { + let metric = Metric::new( + next_name(), + MetricKind::Incremental, + MetricValue::Counter { + value: (accepted + 1) as f64, + }, + ) + .with_timestamp(Some(ts())); + + match encoder.try_encode(metric) { + Ok(None) => accepted += 1, + Ok(Some(_)) => break, + Err(e) => panic!("unexpected encoding error: {e}"), + } + } + + assert!(accepted > 0, "at least one metric must be accepted"); + + let compressed_limit = DatadogMetricsEndpoint::Series(SeriesApiVersion::V2) + .payload_limits() + .compressed; + + let (payload, _) = encoder.finish().unwrap_or_else(|e| { + panic!( + "finish() returned an error after {accepted} metrics — \ + the capacity estimate failed to prevent overflow: {e:?}" + ) + }); + let payload_len = payload.into_payload().len(); + + // Safety: the hard limit must never be exceeded. + assert!( + payload_len <= compressed_limit, + "payload ({payload_len} bytes) exceeded the {compressed_limit}-byte compressed limit" + ); + + // Utilization: the encoder should use at least 95% of the available capacity. + let min_utilization = compressed_limit * 95 / 100; + assert!( + payload_len > min_utilization, + "payload ({payload_len} bytes) is below 95% of the {compressed_limit}-byte limit \ + ({min_utilization} bytes) — estimate may be over-conservative" + ); + } + + #[test] + fn compressed_limit_is_respected_regardless_of_compressor_internal_buffering() { + // Regression test for zstd's internal buffering hiding the compressed-size check. + // + // zstd buffers up to 128 KB internally before flushing a block to the output Vec. + // The old capacity check used `get_ref().len()` alone as "compressed bytes so far", which + // returns 0 until zstd's first 128 KB block completes. This caused the encoder to accept + // metrics indefinitely — finish() would then return TooLarge, triggering expensive + // re-encoding. + // + // The fix splits the estimate: exact compressed size for flushed blocks, plus + // max_compressed_size for the unflushed portion (bytes still in zstd's internal buffer). + // This is accurate for flushed data and bounded for unflushed data. + // + // At compressed_limit=512, no zstd block will flush (needs 128 KB of input), so + // get_ref().len() stays 0 throughout. The old code would accept all 100 metrics; + // the new code stops after a handful. + let compressed_limit = 512; + let mut encoder = DatadogMetricsEncoder::with_payload_limits( + DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), + None, + 1_000_000, + compressed_limit, + ); + + let mut accepted = 0; + for i in 0..100 { + let metric = Metric::new( + format!("counter_{i:0>20}"), + MetricKind::Incremental, + MetricValue::Counter { + value: (i + 1) as f64, + }, + ) + .with_timestamp(Some(ts())); + match encoder.try_encode(metric) { + Ok(None) => accepted += 1, + Ok(Some(_)) => break, + Err(e) => panic!("unexpected encoding error: {e}"), + } + } + + assert!(accepted > 0, "encoder should accept at least one metric"); + assert!( + accepted < 10, + "encoder accepted too many metrics — compressed limit was likely not enforced (accepted={accepted})" + ); + + let result = encoder.finish(); + assert!( + result.is_ok(), + "finish() must not return TooLarge: {:?}", + result.err() + ); + let (payload, _) = result.unwrap(); + assert!( + payload.into_payload().len() <= compressed_limit, + "payload exceeded compressed_limit" + ); + } + + #[test] + fn zstd_buffered_bound_resets_to_last_metric_size_after_block_flush() { + // White-box test: directly verifies that buffered_bound resets to exactly n (the last + // metric's encoded size) when a zstd block flush occurs, not to 0 or some other value. + // + // buffered_bound is an upper bound on bytes in zstd's internal buffer. On each write it + // accumulates (+= n). When a flush is detected (get_ref().len() grows), it resets to n — + // meaning only the triggering write could straddle the block boundary. + // + // If it reset to 0 instead, subsequent capacity checks would degenerate to + // flushed_compressed + max_cs(n) + // which vastly underestimates for any data written after the flush, re-introducing the + // original blind spot. If it failed to reset at all, the encoder would become + // over-conservative and reject valid metrics after the first flush. + // + // Strategy: + // 1. Measure a single metric's encoded size by inspecting buffered_bound after one write. + // 2. Feed metrics into a second encoder (with unlimited limits) until buffered_bound + // *decreases*, which signals a block flush. Assert the post-flush value equals + // exactly one metric's encoded size. + + let make_metric = |i: usize| { + // Fixed-width name (600-char zero-padded) gives a constant per-metric encoded size. + // Value is (i + 1) to ensure it is never 0.0: proto3 omits default (zero) values, + // which would make the first metric smaller than the rest. + Metric::new( + format!("counter_{i:0>600}"), + MetricKind::Incremental, + MetricValue::Counter { + value: (i + 1) as f64, + }, + ) + .with_timestamp(Some(ts())) + }; + + // Step 1: measure a single metric's encoded size. + let metric_size = { + let mut probe = DatadogMetricsEncoder::with_payload_limits( + DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), + None, + usize::MAX, + usize::MAX, + ); + assert!( + probe.try_encode(make_metric(0)).unwrap().is_none(), + "first metric must be accepted" + ); + probe.buffered_bound() + }; + assert!(metric_size > 0, "encoded metric must be non-empty"); + + // Step 2: encode metrics until buffered_bound decreases (= flush detected). + let mut encoder = DatadogMetricsEncoder::with_payload_limits( + DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), + None, + usize::MAX, + usize::MAX, + ); + + let mut prev_buffered = 0usize; + let mut flush_seen = false; + + for i in 0..1000 { + let metric = make_metric(i); + match encoder.try_encode(metric) { + Ok(None) => {} + Ok(Some(_)) => panic!("unexpected rejection at i={i} with unlimited limits"), + Err(e) => panic!("unexpected error at i={i}: {e}"), + } + + let curr = encoder.buffered_bound(); + + if curr < prev_buffered { + // A block flush just occurred: buffered_bound must reset to exactly n. + assert_eq!( + curr, metric_size, + "after block flush, buffered_bound should reset to one metric's encoded size \ + ({metric_size} bytes) but got {curr}" + ); + flush_seen = true; + break; + } + + prev_buffered = curr; + } + + assert!( + flush_seen, + "no zstd block flush detected after 1000 metrics — increase loop bound or metric size" + ); + } + fn arb_counter_metric() -> impl Strategy { let name = string_regex("[a-zA-Z][a-zA-Z0-9_]{8,96}").expect("regex should not be invalid"); let value = ARB_POSITIVE_F64; @@ -1827,9 +2118,9 @@ mod tests { proptest! { #[test] - fn encoding_check_for_payload_limit_edge_cases( - uncompressed_limit in 0..64_000_000usize, - compressed_limit in 0..10_000_000usize, + fn encoding_check_for_payload_limit_edge_cases_v1( + uncompressed_limit in 1..64_000_000usize, + compressed_limit in 1..10_000_000usize, metric in arb_counter_metric(), ) { // We simply try to encode a single metric into an encoder, and make sure that when we @@ -1838,25 +2129,51 @@ mod tests { // // We check this with targeted unit tests as well but this is some cheap insurance to // show that we're hopefully not missing any particular corner cases. - let result = DatadogMetricsEncoder::with_payload_limits( + let mut encoder = DatadogMetricsEncoder::with_payload_limits( + DatadogMetricsEndpoint::Series(SeriesApiVersion::V1), + None, + uncompressed_limit, + compressed_limit, + ); + _ = encoder.try_encode(metric); + + if let Ok((payload, _processed)) = encoder.finish() { + let payload = payload.into_payload(); + prop_assert!(payload.len() <= compressed_limit); + + // V1 uses zlib/deflate. + let result = decompress_zlib_payload(payload); + prop_assert!(result.is_ok()); + + let decompressed = result.unwrap(); + prop_assert!(decompressed.len() <= uncompressed_limit); + } + } + + #[test] + fn encoding_check_for_payload_limit_edge_cases_v2( + uncompressed_limit in 1..10_000_000usize, + compressed_limit in 1..1_000_000usize, + metric in arb_counter_metric(), + ) { + let mut encoder = DatadogMetricsEncoder::with_payload_limits( DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), None, uncompressed_limit, compressed_limit, ); - if let Ok(mut encoder) = result { - _ = encoder.try_encode(metric); + _ = encoder.try_encode(metric); - if let Ok((payload, _processed)) = encoder.finish() { - let payload = payload.into_payload(); - prop_assert!(payload.len() <= compressed_limit); + if let Ok((payload, _processed)) = encoder.finish() { + let payload = payload.into_payload(); + prop_assert!(payload.len() <= compressed_limit); - let result = decompress_payload(payload); - prop_assert!(result.is_ok()); + // V2 uses zstd. + let result = decompress_zstd_payload(payload); + prop_assert!(result.is_ok()); - let decompressed = result.unwrap(); - prop_assert!(decompressed.len() <= uncompressed_limit); - } + let decompressed = result.unwrap(); + prop_assert!(decompressed.len() <= uncompressed_limit); } } } diff --git a/src/sinks/datadog/metrics/integration_tests.rs b/src/sinks/datadog/metrics/integration_tests.rs index e63d4c35e73c9..176c2153d88e0 100644 --- a/src/sinks/datadog/metrics/integration_tests.rs +++ b/src/sinks/datadog/metrics/integration_tests.rs @@ -29,6 +29,7 @@ use crate::{ DATA_VOLUME_SINK_TAGS, SINK_TAGS, assert_data_volume_sink_compliance, assert_sink_compliance, }, + compression::is_zstd, map_event_batch_stream, }, }; @@ -142,10 +143,14 @@ async fn start_test(events: Vec) -> (Vec, Receiver<(http::request: } fn decompress_payload(payload: Vec) -> std::io::Result> { - let mut decompressor = ZlibDecoder::new(&payload[..]); - let mut decompressed = Vec::new(); - let result = std::io::copy(&mut decompressor, &mut decompressed); - result.map(|_| decompressed) + if is_zstd(&payload) { + zstd::decode_all(&payload[..]) + } else { + let mut decompressor = ZlibDecoder::new(&payload[..]); + let mut decompressed = Vec::new(); + std::io::copy(&mut decompressor, &mut decompressed)?; + Ok(decompressed) + } } #[tokio::test] diff --git a/src/sinks/datadog/metrics/request_builder.rs b/src/sinks/datadog/metrics/request_builder.rs index 3b1c2f4607d39..ba8a7723dbf07 100644 --- a/src/sinks/datadog/metrics/request_builder.rs +++ b/src/sinks/datadog/metrics/request_builder.rs @@ -9,19 +9,13 @@ use vector_lib::{ use super::{ config::{DatadogMetricsEndpoint, DatadogMetricsEndpointConfiguration, SeriesApiVersion}, - encoder::{CreateError, DatadogMetricsEncoder, EncoderError, FinishError}, + encoder::{DatadogMetricsEncoder, EncoderError, FinishError}, service::DatadogMetricsRequest, }; use crate::sinks::util::{IncrementalRequestBuilder, metadata::RequestMetadataBuilder}; #[derive(Debug, Snafu)] pub enum RequestBuilderError { - #[snafu( - context(false), - display("Failed to build the request builder: {source}") - )] - FailedToBuild { source: CreateError }, - #[snafu(context(false), display("Failed to encode metric: {source}"))] FailedToEncode { source: EncoderError }, @@ -40,7 +34,6 @@ impl RequestBuilderError { /// many events were dropped as a result. pub fn into_parts(self) -> (String, &'static str, u64) { match self { - Self::FailedToBuild { source } => (source.to_string(), source.as_error_type(), 0), // Encoding errors always happen at the per-metric level, so we could only ever drop a // single metric/event at a time. Self::FailedToEncode { source } => (source.to_string(), source.as_error_type(), 1), @@ -81,18 +74,18 @@ impl DatadogMetricsRequestBuilder { endpoint_configuration: DatadogMetricsEndpointConfiguration, default_namespace: Option, series_api_version: SeriesApiVersion, - ) -> Result { - Ok(Self { + ) -> Self { + Self { endpoint_configuration, series_encoder: DatadogMetricsEncoder::new( DatadogMetricsEndpoint::Series(series_api_version), default_namespace.clone(), - )?, + ), sketches_encoder: DatadogMetricsEncoder::new( DatadogMetricsEndpoint::Sketches, default_namespace, - )?, - }) + ), + } } const fn get_encoder( @@ -247,6 +240,7 @@ impl IncrementalRequestBuilder<((Option>, DatadogMetricsEndpoint), Vec< payload, uri, content_type: ddmetrics_metadata.endpoint.content_type(), + content_encoding: ddmetrics_metadata.endpoint.compression().content_encoding(), finalizers: ddmetrics_metadata.finalizers, metadata: request_metadata, } diff --git a/src/sinks/datadog/metrics/service.rs b/src/sinks/datadog/metrics/service.rs index e40c7cd91c406..024493ed88fc2 100644 --- a/src/sinks/datadog/metrics/service.rs +++ b/src/sinks/datadog/metrics/service.rs @@ -44,6 +44,7 @@ pub struct DatadogMetricsRequest { pub payload: Bytes, pub uri: Uri, pub content_type: &'static str, + pub content_encoding: &'static str, pub finalizers: EventFinalizers, pub metadata: RequestMetadata, } @@ -63,11 +64,6 @@ impl DatadogMetricsRequest { HeaderValue::from_str(&key).expect("API key should be only valid ASCII characters") }, ); - // Requests to the metrics endpoints can be compressed, and there's almost no reason to - // _not_ compress them given tha t metric data, when encoded, is very repetitive. Thus, - // here and through the sink code, we always compress requests. Datadog also only supports - // zlib (DEFLATE) compression, which is why it's hard-coded here vs being set via the common - // `Compression` value that most sinks utilize. let request = Request::post(self.uri) .header("DD-API-KEY", api_key) // TODO: The Datadog Agent sends this header to indicate the version of the Go library @@ -82,7 +78,7 @@ impl DatadogMetricsRequest { // this header. .header("DD-Agent-Payload", "4.87.0") .header(CONTENT_TYPE, self.content_type) - .header(CONTENT_ENCODING, "deflate"); + .header(CONTENT_ENCODING, self.content_encoding); request.body(Body::from(self.payload)) } From 753cf9a7c03a5cef85a29f3561666d94e9084d8b Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 24 Mar 2026 15:57:50 -0400 Subject: [PATCH 050/326] docs(internal docs): consolidate reference documentation into detailed docs table in AGENTS.md (#25036) * docs: consolidate reference documentation into detailed docs table in AGENTS.md Co-Authored-By: Claude Sonnet 4.6 * docs(internal docs): add PR title semantic convention section to AGENTS.md Co-Authored-By: Claude Sonnet 4.6 * docs(internal docs): fix markdown lint errors in PR title section Co-Authored-By: Claude Sonnet 4.6 --------- Co-authored-by: Claude Sonnet 4.6 --- AGENTS.md | 37 +++++++++++++++---------------------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 580018abf0283..e70a963047d9f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -191,6 +191,12 @@ Then: `chmod +x .git/hooks/pre-push` | Topic | Document | |-------|----------| | Rust style patterns | [docs/RUST_STYLE.md](docs/RUST_STYLE.md) | +| Code style rules (formatting, const strings, organization) | [STYLE.md](STYLE.md) | +| System architecture (sources, transforms, sinks, topology) | [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) | +| Component specification (naming, configuration, health checks) | [docs/specs/component.md](docs/specs/component.md) | +| Instrumentation requirements (event/metric naming) | [docs/specs/instrumentation.md](docs/specs/instrumentation.md) | +| How to document code changes | [docs/DOCUMENTING.md](docs/DOCUMENTING.md) | +| Adding changelog entries | [changelog.d/README.md](changelog.d/README.md) | ## Architecture Notes @@ -258,28 +264,15 @@ make build-licenses Before opening a PR, read [`.github/PULL_REQUEST_TEMPLATE.md`](.github/PULL_REQUEST_TEMPLATE.md) and use it as the reference for the PR body structure and title. -## Reference Documentation +### PR Title Format -These documents provide context that AI agents and developers need when working on Vector code. +PR titles must follow the [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/) spec and are validated by `.github/workflows/semantic.yml`. -### Essential for Code Changes +Examples: -- **[STYLE.md](STYLE.md)** - Code style rules (formatting, const strings, code organization) -- **[docs/ARCHITECTURE.md](docs/ARCHITECTURE.md)** - System architecture (sources, transforms, sinks, topology) -- **[docs/DEVELOPING.md](docs/DEVELOPING.md)** - Development workflow and testing - -### Component Development - -- **[docs/specs/component.md](docs/specs/component.md)** - Component specification (naming, configuration, health checks) -- **[docs/specs/instrumentation.md](docs/specs/instrumentation.md)** - Instrumentation requirements (event/metric naming) -- **[src/internal_events](src/internal_events)** - Internal event examples for telemetry - -### Adding Documentation - -- **[docs/DOCUMENTING.md](docs/DOCUMENTING.md)** - How to document code changes -- **[changelog.d/README.md](changelog.d/README.md)** - Adding changelog entries - -### Full Guides - -- **[CONTRIBUTING.md](CONTRIBUTING.md)** - Complete contributing guide -- **[website/README.md](website/README.md)** - Website development only (separate from Rust code) +```text +feat(kafka source): add consumer group lag metric +fix(loki sink): handle empty label sets correctly +docs(internal docs): update contributing guide +chore(deps): bump tokio to X +``` From 47b5b02b9fdd13b67294df65175e9eca99c4cd8b Mon Sep 17 00:00:00 2001 From: Thomas Date: Tue, 24 Mar 2026 16:36:30 -0400 Subject: [PATCH 051/326] chore(ci): split deny check into optional (all) and required (licenses) (#24990) ci(deny): split deny check into optional (all) and required (licenses) Co-authored-by: Pavlos Rontidis --- .github/workflows/deny.yml | 27 +++++++++++++++++++++++---- Makefile | 4 ++++ vdev/src/commands/check/deny.rs | 13 +++++++++++-- 3 files changed, 38 insertions(+), 6 deletions(-) diff --git a/.github/workflows/deny.yml b/.github/workflows/deny.yml index 2a9696cc3f76d..52b40d1e1d535 100644 --- a/.github/workflows/deny.yml +++ b/.github/workflows/deny.yml @@ -26,12 +26,14 @@ on: type: string pull_request: + merge_group: + types: [checks_requested] schedule: # Same schedule as nightly.yml - cron: "0 5 * * 2-6" # Runs at 5:00 AM UTC, Tuesday through Saturday concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + group: ${{ github.workflow }}-${{ github.event.number || github.sha }} cancel-in-progress: true permissions: @@ -48,8 +50,6 @@ jobs: timeout-minutes: 30 if: ${{ always() && (github.event_name != 'pull_request' || needs.changes.outputs.deny == 'true') }} needs: [changes] - env: - CARGO_INCREMENTAL: 0 steps: - name: Checkout branch uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -61,5 +61,24 @@ jobs: mold: false cargo-deny: true - - name: Check cargo deny advisories/licenses + - name: Check cargo deny (all) run: make check-deny + + test-deny-licenses: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + if: ${{ always() && (github.event_name != 'pull_request' || needs.changes.outputs.deny == 'true') }} + needs: [changes] + steps: + - name: Checkout branch + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ inputs.ref }} + + - uses: ./.github/actions/setup + with: + mold: false + cargo-deny: true + + - name: Check cargo deny licenses + run: make check-deny-licenses diff --git a/Makefile b/Makefile index c6e5d2423820c..71a324a5a7c5b 100644 --- a/Makefile +++ b/Makefile @@ -518,6 +518,10 @@ check-scripts: ## Check that scripts do not have common mistakes check-deny: ## Check advisories licenses and sources for crate dependencies ${MAYBE_ENVIRONMENT_EXEC} $(VDEV) check deny +.PHONY: check-deny-licenses +check-deny-licenses: ## Check licenses for crate dependencies + ${MAYBE_ENVIRONMENT_EXEC} $(VDEV) check deny --licenses-only + .PHONY: check-events check-events: ## Check that events satisfy patterns set in https://github.com/vectordotdev/vector/blob/master/rfcs/2020-03-17-2064-event-driven-observability.md ${MAYBE_ENVIRONMENT_EXEC} $(VDEV) check events diff --git a/vdev/src/commands/check/deny.rs b/vdev/src/commands/check/deny.rs index 11d04c6710fa9..d24b7454b51da 100644 --- a/vdev/src/commands/check/deny.rs +++ b/vdev/src/commands/check/deny.rs @@ -5,10 +5,19 @@ use crate::app; /// Check for advisories, licenses, and sources for crate dependencies #[derive(clap::Args, Debug)] #[command()] -pub struct Cli {} +pub struct Cli { + /// Only check licenses + #[arg(long)] + licenses_only: bool, +} impl Cli { pub fn exec(self) -> Result<()> { + let check = if self.licenses_only { + "licenses" + } else { + "all" + }; app::exec( "cargo", [ @@ -17,7 +26,7 @@ impl Cli { "error", "--all-features", "check", - "all", + check, ], true, ) From ca9b29bc52b7c7ba857c337b25278edbad1c5e66 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 25 Mar 2026 13:17:07 -0400 Subject: [PATCH 052/326] chore(deps): bump serial_test from 3.2.0 to 3.4.0 (#25022) Bumps [serial_test](https://github.com/palfrey/serial_test) from 3.2.0 to 3.4.0. - [Release notes](https://github.com/palfrey/serial_test/releases) - [Commits](https://github.com/palfrey/serial_test/compare/v3.2.0...v3.4.0) --- updated-dependencies: - dependency-name: serial_test dependency-version: 3.4.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 11 ++++++----- Cargo.toml | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d23731697fa21..229f42dd631ed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9923,11 +9923,12 @@ dependencies = [ [[package]] name = "serial_test" -version = "3.2.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b258109f244e1d6891bf1053a55d63a5cd4f8f4c30cf9a1280989f80e7a1fa9" +checksum = "911bd979bf1070a3f3aa7b691a3b3e9968f339ceeec89e08c280a8a22207a32f" dependencies = [ - "futures", + "futures-executor", + "futures-util", "log", "once_cell", "parking_lot", @@ -9937,9 +9938,9 @@ dependencies = [ [[package]] name = "serial_test_derive" -version = "3.2.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d69265a08751de7844521fd15003ae0a888e035773ba05695c5c759a6f89eef" +checksum = "0a7d91949b85b0d2fb687445e448b40d322b6b3e4af6b44a29b21d9a5f33e6d9" dependencies = [ "proc-macro2 1.0.106", "quote 1.0.44", diff --git a/Cargo.toml b/Cargo.toml index 0a577cde861d9..83da1ac8eb616 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -214,7 +214,7 @@ vector-vrl-category = { path = "lib/vector-vrl/category" } vector-vrl-functions = { path = "lib/vector-vrl/functions", default-features = false } vrl = { git = "https://github.com/vectordotdev/vrl.git", branch = "main", default-features = false, features = ["arbitrary", "cli", "test", "test_framework", "stdlib-base"] } mock_instant = { version = "0.6" } -serial_test = { version = "3.2" } +serial_test = { version = "3.4" } strum = { version = "0.28", features = ["derive"] } [dependencies] From a2918e6ac4b874dc7c67b558a1e2bc9fd2b4ae81 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 25 Mar 2026 13:58:38 -0400 Subject: [PATCH 053/326] chore(deps): bump quick-junit from 0.5.2 to 0.6.0 (#25032) * chore(deps): bump quick-junit from 0.5.2 to 0.6.0 Bumps [quick-junit](https://github.com/nextest-rs/quick-junit) from 0.5.2 to 0.6.0. - [Release notes](https://github.com/nextest-rs/quick-junit/releases) - [Changelog](https://github.com/nextest-rs/quick-junit/blob/main/CHANGELOG.md) - [Commits](https://github.com/nextest-rs/quick-junit/compare/quick-junit-0.5.2...quick-junit-0.6.0) --- updated-dependencies: - dependency-name: quick-junit dependency-version: 0.6.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * fix(deps): update quick-junit version requirement to 0.6.0 in Cargo.toml Dependabot only bumped the version in Cargo.lock but left the version requirement in Cargo.toml at "0.5.1", which resolves to >=0.5.1,<0.6.0 and causes cargo to revert the lock file back to 0.5.2 on any build. Co-Authored-By: Claude Sonnet 4.6 (1M context) --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Pavlos Rontidis Co-authored-by: Claude Sonnet 4.6 (1M context) --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 229f42dd631ed..c5efc9802ce97 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8395,9 +8395,9 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quick-junit" -version = "0.5.2" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ee9342d671fae8d66b3ae9fd7a9714dfd089c04d2a8b1ec0436ef77aee15e5f" +checksum = "e3e64c58c4c88fc1045e8fe98a1b7cec3643187e3dd678f9bbcdd8f12a6933d6" dependencies = [ "chrono", "indexmap 2.12.0", diff --git a/Cargo.toml b/Cargo.toml index 83da1ac8eb616..06b83efedb33e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -408,7 +408,7 @@ ordered-float.workspace = true percent-encoding = { version = "2.3.1", default-features = false } postgres-openssl = { version = "0.5.1", default-features = false, features = ["runtime"], optional = true } pulsar = { version = "6.7.0", default-features = false, features = ["tokio-runtime", "auth-oauth2", "flate2", "lz4", "snap", "zstd"], optional = true } -quick-junit = { version = "0.5.1" } +quick-junit = { version = "0.6.0" } rand.workspace = true rand_distr.workspace = true rdkafka = { version = "0.39.0", default-features = false, features = ["curl-static", "tokio", "libz", "ssl", "zstd"], optional = true } From 209b252deed214144fd2a6d4ed6c5ea3a3e08821 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Wed, 25 Mar 2026 15:11:04 -0400 Subject: [PATCH 054/326] chore(dev): enforce and fix markdownlint rules (#25038) * chore(markdown): move markdownlint config to project root, add fix-markdown target - Move .markdownlintrc from scripts/ to project root - Update vdev check markdown to use new path - Add `make fix-markdown` target for auto-fixing issues - Enable default rules and disable rules incompatible with Hugo's rendering pipeline (MD001, MD041, MD051, MD052, MD053) - Enforce emphasis style as asterisk (MD049) Co-Authored-By: Claude Sonnet 4.6 * chore(vdev): remove .github exclusion from markdown lint check Previously .github/ was excluded because it had unresolved violations. Those are now fixed, so all tracked markdown files are linted. Co-Authored-By: Claude Sonnet 4.6 * chore(markdown): enforce and fix markdownlint rules - Enable default rules; disable rules incompatible with Hugo rendering pipeline (MD001, MD024, MD025, MD041, MD051, MD052, MD053) and rules with intentional exceptions (MD012, MD013, MD014, MD033, MD034, MD049) - Add make fix-markdown target for auto-fixing issues - Remove .github exclusion from markdown lint (violations now fixed) - Fix all existing violations: table formatting (MD055/MD060), ordered list prefixes (MD029), non-descriptive link text (MD059), missing code block languages (MD040), missing alt text (MD045), list/heading spacing (MD022/MD032), list style (MD004), list indent (MD007) Co-Authored-By: Claude Sonnet 4.6 * chore(markdown): rename config to .markdownlint.jsonc for editor compatibility Rename .markdownlintrc to .markdownlint.jsonc so that markdownlint-cli2 (used by Zed, VS Code, and most modern editors) auto-detects the config. Add a comment pointing toward migrating CI from markdownlint-cli (legacy) to markdownlint-cli2 to unify tooling. Co-Authored-By: Claude Sonnet 4.6 --------- Co-authored-by: Claude Sonnet 4.6 --- .github/ISSUE_TEMPLATE/minor-release.md | 3 +- .github/PULL_REQUEST_TEMPLATE.md | 5 +- .github/SUPPORT.md | 14 ++-- .github/actions/spelling/README.md | 2 +- .github/actions/spelling/advice.md | 1 + .../.markdownlintrc => .markdownlint.jsonc | 19 ++++-- AGENTS.md | 2 +- CONTRIBUTING.md | 10 +-- Makefile | 4 ++ PRIVACY.md | 7 +- RELEASES.md | 18 ++--- VERSIONING.md | 14 ++-- distribution/docker/README.md | 12 ++-- docs/DEVELOPING.md | 4 +- docs/README.md | 4 +- docs/tutorials/sinks/1_basic_sink.md | 8 +-- ...6-1999-api-extensions-for-lua-transform.md | 16 ++--- .../2020-04-04-2221-kubernetes-integration.md | 54 +++++++-------- ...-05-25-2685-dev-workflow-simplification.md | 8 +-- rfcs/2020-05-25-2692-more-usable-logevents.md | 10 +-- rfcs/2020-07-28-3642-jmx_rfc.md | 4 +- rfcs/2020-10-15-3480-file-source-rework.md | 32 ++++----- rfcs/2021-02-23-6531-performance-testing.md | 4 +- ...1-03-26-6517-end-to-end-acknowledgement.md | 16 ++--- rfcs/2021-07-19-8216-multiple-pipelines.md | 4 +- rfcs/2021-08-13-8025-internal-tracing.md | 8 +-- rfcs/2021-08-29-8381-vrl-iteration-support.md | 25 +++---- ...-accept-metrics-in-datadog-agent-source.md | 20 +++--- ...021-10-29-8621-framing-and-codecs-sinks.md | 68 +++++++++---------- ...m-stats-along-traces-in-dd-agent-source.md | 2 +- rfcs/2021-11-12-9811-vrl-vm.md | 4 +- ...021-12-06-10298-kubernetes_logs-rewrite.md | 4 +- rfcs/2022-05-17-11532-chronicle-sink.md | 20 +++--- rfcs/2022-10-12-14742-flexible-metric-tags.md | 16 ++--- rfcs/2022-10-31-15056-tooling-revamp.md | 2 +- rfcs/2023-05-03-data-volume-metrics.md | 2 +- rfcs/README.md | 18 ++--- vdev/src/commands/check/markdown.rs | 14 ++-- website/README.md | 4 +- .../en/blog/adaptive-request-concurrency.md | 14 ++-- .../en/blog/highlights-february-2025.md | 4 +- website/content/en/blog/how-we-test-vector.md | 4 +- .../content/en/blog/tracking-allocations.md | 2 +- .../content/en/blog/vector-remap-language.md | 22 +++--- .../en/docs/administration/management.md | 10 +-- .../en/docs/administration/monitoring.md | 16 ++--- .../en/docs/administration/validating.md | 8 +-- .../content/en/docs/introduction/concepts.md | 4 +- .../reference/configuration/unit-tests.md | 36 +++++----- .../content/en/docs/reference/vrl/_index.md | 16 ++--- .../content/en/docs/reference/vrl/errors.md | 22 +++--- .../setup/going-to-prod/arch/aggregator.md | 2 +- .../docs/setup/going-to-prod/arch/unified.md | 2 +- .../setup/installation/manual/from-source.md | 2 +- .../installation/package-managers/helm.md | 4 +- .../installation/platforms/kubernetes.md | 4 +- .../guides/developer/config-autocompletion.md | 4 +- .../content/en/guides/developer/debugging.md | 2 +- .../2020-10-27-kubernetes-integration.md | 6 +- ...20-11-19-prometheus-remote-integrations.md | 2 +- .../2020-12-01-0-11-upgrade-guide.md | 12 ++-- .../2021-02-16-0-12-upgrade-guide.md | 8 +-- .../2021-08-24-vector-aggregator.md | 2 +- .../2021-08-25-0-16-upgrade-guide.md | 8 +-- .../2021-10-08-0-17-upgrade-guide.md | 12 ++-- .../2021-11-18-0-18-0-upgrade-guide.md | 8 +-- .../2021-12-28-0-19-0-upgrade-guide.md | 6 +- .../2022-02-08-0-20-0-upgrade-guide.md | 12 ++-- .../2022-03-22-0-21-0-upgrade-guide.md | 10 +-- .../2022-03-31-native-event-codecs.md | 6 +- .../2022-05-03-0-22-0-upgrade-guide.md | 6 +- .../2022-10-04-0-25-0-upgrade-guide.md | 4 +- .../2023-02-28-0-28-0-upgrade-guide.md | 2 +- .../2023-03-26-0-37-0-upgrade-guide.md | 2 +- .../2023-07-05-0-31-0-upgrade-guide.md | 2 +- .../2023-09-27-0-33-0-upgrade-guide.md | 8 +-- .../2023-11-07-0-34-0-upgrade-guide.md | 16 ++--- .../highlights/2023-11-07-new-linux-repos.md | 10 +-- .../2023-12-19-0-35-0-upgrade-guide.md | 12 ++-- .../2024-05-07-0-38-0-upgrade-guide.md | 2 +- .../2024-06-17-0-39-0-upgrade-guide.md | 2 +- .../2024-07-29-0-40-0-upgrade-guide.md | 4 +- 82 files changed, 415 insertions(+), 407 deletions(-) rename scripts/.markdownlintrc => .markdownlint.jsonc (53%) diff --git a/.github/ISSUE_TEMPLATE/minor-release.md b/.github/ISSUE_TEMPLATE/minor-release.md index 105299559632b..7ab406601e4cb 100644 --- a/.github/ISSUE_TEMPLATE/minor-release.md +++ b/.github/ISSUE_TEMPLATE/minor-release.md @@ -37,6 +37,7 @@ cargo vdev release prepare --version "${NEW_VECTOR_VERSION}" --vrl-version "${NE ``` Automated steps include: + - Create a new release branch from master to freeze commits - `git fetch && git checkout origin/master && git checkout -b "${RELEASE_BRANCH}" && git push -u` - Create a new release preparation branch from `master` @@ -79,7 +80,7 @@ Automated steps include: - [ ] Squash the release preparation commits (but not the cherry-picked commits!) to a single commit. This makes it easier to cherry-pick to master after the release. - [ ] Merge release preparation branch into the release branch. - - `git switch "${RELEASE_BRANCH}" && git merge --ff-only "${PREP_BRANCH}"` + - `git switch "${RELEASE_BRANCH}" && git merge --ff-only "${PREP_BRANCH}"` - [ ] Tag new release - [ ] `git tag v"${NEW_VECTOR_VERSION}" -a -m v"${NEW_VECTOR_VERSION}"` diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 2d239269beeda..0aa52163fafd3 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -9,6 +9,7 @@ This should help the reviewers give feedback faster and with higher quality. --> ## Change Type + - [ ] Bug fix - [ ] New feature - [ ] Dependencies @@ -16,6 +17,7 @@ This should help the reviewers give feedback faster and with higher quality. --> - [ ] Performance ## Is this a breaking change? + - [ ] Yes - [ ] No @@ -36,6 +38,7 @@ Changes to CI, website, playground and similar are generally not considered user --> ## Notes + - Please read our [Vector contributor resources](https://github.com/vectordotdev/vector/tree/master/docs#getting-started). - Do not hesitate to use `@vectordotdev/vector` to reach out to us regarding this PR. - Some CI checks run only after we manually approve them. @@ -48,7 +51,7 @@ Changes to CI, website, playground and similar are generally not considered user - Feel free to push as many commits as you want. They will be squashed into one before merging. - For example, you can run `git merge origin master` and `git push`. - If this PR introduces changes Vector dependencies (modifies `Cargo.lock`), please - run `make build-licenses` to regenerate the [license inventory](https://github.com/vectordotdev/vrl/blob/main/LICENSE-3rdparty.csv) and commit the changes (if any). More details [here](https://crates.io/crates/dd-rust-license-tool). + run `make build-licenses` to regenerate the [license inventory](https://github.com/vectordotdev/vrl/blob/main/LICENSE-3rdparty.csv) and commit the changes (if any). More details on the [dd-rust-license-tool](https://crates.io/crates/dd-rust-license-tool). diff --git a/VERSIONING.md b/VERSIONING.md index a84b608f6ad80..9dd49bd92b68e 100644 --- a/VERSIONING.md +++ b/VERSIONING.md @@ -9,16 +9,16 @@ Please see the [FAQ](#faq) section for more info.** 1. [Convention](#convention) -1. [Public API](#public-api) +2. [Public API](#public-api) 1. [Areas that *are* covered](#areas-that-are-covered) 1. [Intended for *public* consumption](#intended-for-public-consumption) - 1. [Intended for *private* consumption](#intended-for-private-consumption) - 1. [Areas that are *NOT* covered](#areas-that-are-not-covered) -1. [FAQ](#faq) + 2. [Intended for *private* consumption](#intended-for-private-consumption) + 2. [Areas that are *NOT* covered](#areas-that-are-not-covered) +3. [FAQ](#faq) 1. [How often is Vector released?](#how-often-is-vector-released) - 1. [How does Vector treat patch and minor versions?](#how-does-vector-treat-patch-and-minor-versions) - 1. [How does Vector treat major versions \(breaking changes\)?](#how-does-vector-treat-major-versions-breaking-changes) - 1. [How does Vector treat pre-1.0 versions?](#how-does-vector-treat-pre-10-versions) + 2. [How does Vector treat patch and minor versions?](#how-does-vector-treat-patch-and-minor-versions) + 3. [How does Vector treat major versions \(breaking changes\)?](#how-does-vector-treat-major-versions-breaking-changes) + 4. [How does Vector treat pre-1.0 versions?](#how-does-vector-treat-pre-10-versions) diff --git a/distribution/docker/README.md b/distribution/docker/README.md index b6e6a66ea65e5..71cef83c6a4e8 100644 --- a/distribution/docker/README.md +++ b/distribution/docker/README.md @@ -105,12 +105,12 @@ Vector maintains special tags that are automatically updated whenever Vector is | Version | URL | |:-----------------|:---------------------------------------------------------| -| Latest major | `timberio/vector:latest-alpine` | -| Latest minor | `timberio/vector:.X-alpine` | -| Latest patch | `timberio/vector:.X-alpine` | -| Specific version | `timberio/vector:-alpine` | -| Latest nightly | `timberio/vector:nightly-alpine` | -| Specific nightly | `timberio/vector:nightly--alpine` | +| Latest major | `timberio/vector:latest-alpine` | +| Latest minor | `timberio/vector:.X-alpine` | +| Latest patch | `timberio/vector:.X-alpine` | +| Specific version | `timberio/vector:-alpine` | +| Latest nightly | `timberio/vector:nightly-alpine` | +| Specific nightly | `timberio/vector:nightly--alpine` | ### Source Files diff --git a/docs/DEVELOPING.md b/docs/DEVELOPING.md index 63633d2bdac7b..9d3d50f7d6243 100644 --- a/docs/DEVELOPING.md +++ b/docs/DEVELOPING.md @@ -467,8 +467,8 @@ times: We use `flog` to build a sample set of log files to test sending logs from a file. This can be done with the following commands on Mac with `homebrew`. -Installation instruction for flog can be found -[here](https://github.com/mingrammer/flog#installation). +Installation instruction for flog can be found in the +[flog README](https://github.com/mingrammer/flog#installation). ```bash flog --bytes $((100 * 1024 * 1024)) > sample.log diff --git a/docs/README.md b/docs/README.md index cb3ed925a1158..443c706e38faa 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,8 +1,8 @@ # Vector Internal Documentation -**_This folder contains internal documentation for Vector contributors. If +_**This folder contains internal documentation for Vector contributors. If you're a Vector user, please visit , which is powered by -the [website](../website) directory._** +the [website](../website) directory.**_ ## Getting started diff --git a/docs/tutorials/sinks/1_basic_sink.md b/docs/tutorials/sinks/1_basic_sink.md index a099a5530602d..c823c6a26530f 100644 --- a/docs/tutorials/sinks/1_basic_sink.md +++ b/docs/tutorials/sinks/1_basic_sink.md @@ -2,7 +2,7 @@ Let's write a basic sink for Vector. Currently, there are two styles of sink in Vector - 'event' and 'event streams'. The 'event' style sinks are deprecated, but currently a significant portion of Vector's sinks are still developed in this style. A tracking issue that covers which sinks have been converted to -'event streams' can be found [here][event_streams_tracking]. +'event streams' can be found in the [event streams tracking issue][event_streams_tracking]. This tutorial covers writing an 'event stream' Sink. @@ -300,8 +300,8 @@ Change the body of `run_inner` to look like the following: } ``` -More details about instrumenting Vector can be found -[here](https://github.com/vectordotdev/vector/blob/master/docs/specs/instrumentation.md). +More details about instrumenting Vector can be found in the +[instrumentation specification](https://github.com/vectordotdev/vector/blob/master/docs/specs/instrumentation.md). # Running our sink @@ -324,7 +324,7 @@ This simply connects a `stdin` source to our `basic` sink. ## vdev Vector provides a build tool `vdev` that simplifies the task of building Vector. Install -`vdev` using the instructions [here][vdev_install]. +`vdev` using the [installation instructions][vdev_install]. With `vdev` installed we can run Vector using: diff --git a/rfcs/2020-03-06-1999-api-extensions-for-lua-transform.md b/rfcs/2020-03-06-1999-api-extensions-for-lua-transform.md index 4b8b75c032d66..abec1d261a80b 100644 --- a/rfcs/2020-03-06-1999-api-extensions-for-lua-transform.md +++ b/rfcs/2020-03-06-1999-api-extensions-for-lua-transform.md @@ -28,7 +28,7 @@ This RFC proposes a new API for the `lua` transform. Currently, the [`lua` transform](https://vector.dev/docs/reference/transforms/lua/) has some limitations in its API. In particular, the following features are missing: -* **Nested Fields** +* **Nested Fields** Currently accessing nested fields is possible using the field path notation: @@ -44,7 +44,7 @@ Currently, the [`lua` transform](https://vector.dev/docs/reference/transforms/lu See [#706](https://github.com/vectordotdev/vector/issues/706) and [#1406](https://github.com/vectordotdev/vector/issues/1406). -* **Setup Code** +* **Setup Code** Some scripts require expensive setup steps, for example, loading of modules or invoking shell commands. These steps should not be part of the main transform code. @@ -79,7 +79,7 @@ Currently, the [`lua` transform](https://vector.dev/docs/reference/transforms/lu See [#1864](https://github.com/vectordotdev/vector/issues/1864). -* **Control Flow** +* **Control Flow** It should be possible to define channels for output events, similarly to how it is done in [`swimlanes`](https://vector.dev/docs/reference/transforms/swimlanes/) transform. @@ -136,7 +136,7 @@ This example is a log to metric transform which produces metric events from inco 1. There is an internal counter which is increased on each incoming log event. 2. The log events are discarded. -2. Each 10 seconds the transform produces a metric event with the count of received log events. +3. Each 10 seconds the transform produces a metric event with the count of received log events. 4. Edge cases are handled in the following way: 1. If there are no incoming invents, the metric event with the counter equal to 0 still has to be produced. 2. On Vector's shutdown the transform has to produce the final metric event with the count of received events since the last flush. @@ -645,10 +645,10 @@ The mapping between Vector data types and Lua data types is the following: | Vector Type | Lua Type | Comment | | :----------- | :-------- | :------- | -| [`String`](https://vector.dev/docs/architecture/data-model/log/#strings) | [`string`](https://www.lua.org/pil/2.4.html) || -| [`Integer`](https://vector.dev/docs/architecture/data-model/log/#ints) | [`integer`](https://docs.rs/mlua/0.6.0/mlua/type.Integer.html) || -| [`Float`](https://vector.dev/docs/architecture/data-model/log/#floats) | [`number`](https://docs.rs/mlua/0.6.0/mlua/type.Number.html) || -| [`Boolean`](https://vector.dev/docs/architecture/data-model/log/#booleans) | [`boolean`](https://www.lua.org/pil/2.2.html) || +| [`String`](https://vector.dev/docs/architecture/data-model/log/#strings) | [`string`](https://www.lua.org/pil/2.4.html) | | +| [`Integer`](https://vector.dev/docs/architecture/data-model/log/#ints) | [`integer`](https://docs.rs/mlua/0.6.0/mlua/type.Integer.html) | | +| [`Float`](https://vector.dev/docs/architecture/data-model/log/#floats) | [`number`](https://docs.rs/mlua/0.6.0/mlua/type.Number.html) | | +| [`Boolean`](https://vector.dev/docs/architecture/data-model/log/#booleans) | [`boolean`](https://www.lua.org/pil/2.2.html) | | | [`Timestamp`](https://vector.dev/docs/architecture/data-model/log/#timestamps) | [`userdata`](https://www.lua.org/pil/28.1.html) | There is no dedicated timestamp type in Lua. However, there is a standard library function [`os.date`](https://www.lua.org/manual/5.1/manual.html#pdf-os.date) which returns a table with fields `year`, `month`, `day`, `hour`, `min`, `sec`, and some others. Other standard library functions, such as [`os.time`](https://www.lua.org/manual/5.1/manual.html#pdf-os.time), support tables with these fields as arguments. Because of that, Vector timestamps passed to the transform are represented as `userdata` with the same set of accessible fields. In order to have one-to-one correspondence between Vector timestamps and Lua timestamps, `os.date` function from the standard library is patched to return not a table, but `userdata` with the same set of fields as it usually would return instead. This approach makes it possible to have both compatibility with the standard library functions and a dedicated data type for timestamps. | | [`Null`](https://vector.dev/docs/architecture/data-model/log/#null-values) | empty string | In Lua setting a table field to `nil` means deletion of this field. Furthermore, setting an array element to `nil` leads to deletion of this element. In order to avoid inconsistencies, already present `Null` values are visible represented as empty strings from Lua code, and it is impossible to create a new `Null` value in the user-defined code. | | [`Map`](https://vector.dev/docs/architecture/data-model/log/#maps) | [`userdata`](https://www.lua.org/pil/28.1.html) or [`table`](https://www.lua.org/pil/2.5.html) | Maps which are parts of events passed to the transform from Vector have `userdata` type. User-created maps have `table` type. Both types are converted to Vector's `Map` type when they are emitted from the transform. | diff --git a/rfcs/2020-04-04-2221-kubernetes-integration.md b/rfcs/2020-04-04-2221-kubernetes-integration.md index e95ee2b4a5dbc..b2a06a68be1ad 100644 --- a/rfcs/2020-04-04-2221-kubernetes-integration.md +++ b/rfcs/2020-04-04-2221-kubernetes-integration.md @@ -135,7 +135,7 @@ it has to be deployed on every [`Node`][k8s_docs_node] in your cluster. The following diagram demonstrates how this works: - +Kubernetes deployment topology ### What We'll Accomplish @@ -149,7 +149,7 @@ The following diagram demonstrates how this works: #### Deploy using `kubectl` -1. Configure Vector: +1. Configure Vector: Before we can deploy Vector we must configure. This is done by creating a Kubernetes `ConfigMap`: @@ -173,7 +173,7 @@ The following diagram demonstrates how this works: kubectl create secret generic vector-config --from-file=vector.toml=vector.toml ``` -2. Deploy Vector! +2. Deploy Vector! Now that you have your custom `ConfigMap` ready it's time to deploy Vector. Create a `Namespace` and apply your `ConfigMap` and our recommended @@ -190,20 +190,20 @@ The following diagram demonstrates how this works: #### Deploy using Helm -1. Install [`helm`][helm_install]. +1. Install [`helm`][helm_install]. -2. Add our Helm Chart repo. +2. Add our Helm Chart repo. ```shell helm repo add vector https://charts.vector.dev helm repo update ``` -3. Configure Vector. +3. Configure Vector. TODO: address this when we decide on the helm chart internals. -4. Deploy Vector! +4. Deploy Vector! ```shell kubectl create namespace vector @@ -227,9 +227,9 @@ The following diagram demonstrates how this works: #### Deploy using Kustomize -1. Install [`kustomize`][kustomize]. +1. Install [`kustomize`][kustomize]. -1. Prepare `kustomization.yaml`. +2. Prepare `kustomization.yaml`. Use the same config as in [`kubectl` guide][anchor_tutorial_kubectl]. @@ -243,7 +243,7 @@ The following diagram demonstrates how this works: - vector-configmap.yaml ``` -1. Deploy Vector! +3. Deploy Vector! ```shell kustomize build . | kubectl apply -f - @@ -327,9 +327,7 @@ logs such that they're accessible from the following locations: - `/var/log/containers` - legacy location, kept for backward compatibility with pre `1.14` clusters. -To make our lives easier, here's a [link][k8s_src_build_container_logs_directory] -to the part of the k8s source that's responsible for building the path to the -log file. If we encounter issues, this would be a good starting point to unwrap +To make our lives easier, here's a reference to the [k8s source code responsible for building the container log path][k8s_src_build_container_logs_directory]. If we encounter issues, this would be a good starting point to unwrap the k8s code. #### Log file format @@ -603,7 +601,7 @@ This worth a separate dedicated RFC though. #### Security considerations on deployment configuration Security considerations on deployment configuration are grouped together with -other security-related measures. See [here](#deployment-hardening). +other security-related measures. See the [deployment hardening](#deployment-hardening) section. #### Other notable [`PodSpec`][k8s_api_pod_spec] properties @@ -1059,7 +1057,7 @@ We have a matrix of concerns, we'd like to ensure Vectors works properly with. - OCI (via [CRI-O](https://cri-o.io/) or [containerd](https://containerd.io/)) - [runc](https://github.com/opencontainers/runc) - [runhcs](https://github.com/Microsoft/hcsshim/tree/master/cmd/runhcs) - - see more [here][windows_in_kubernetes] + see [Windows in Kubernetes][windows_in_kubernetes] - [Kata Containers](https://github.com/kata-containers/runtime) - [gVisor](https://github.com/google/gvisor) - [Firecracker](https://github.com/firecracker-microvm/firecracker-containerd) @@ -1508,16 +1506,16 @@ nightly!) the supported Vector versions. ## Prior Art 1. [Filebeat k8s integration] -1. [Fluentbit k8s integration] -1. [Fluentd k8s integration] -1. [LogDNA k8s integration] -1. [Honeycomb integration] -1. [Bonzai logging operator] - This is approach is likely outside of the scope +2. [Fluentbit k8s integration] +3. [Fluentd k8s integration] +4. [LogDNA k8s integration] +5. [Honeycomb integration] +6. [Bonzai logging operator] - This is approach is likely outside of the scope of Vector's initial Kubernetes integration because it focuses more on deployment strategies and topologies. There are likely some very useful and interesting tactics in their approach though. -1. [Influx Helm charts] -1. [Awesome Operators List] - an "awesome list" of operators. +7. [Influx Helm charts] +8. [Awesome Operators List] - an "awesome list" of operators. ## Sales Pitch @@ -1541,7 +1539,7 @@ See [motivation](#motivation). namespaces? We'd just need to configure Vector to exclude this namespace?~~ See the [Origin filtering][anchor_origin_filtering] section. -1. ~~From what I understand, Vector requires the Kubernetes `watch` verb in order +2. ~~From what I understand, Vector requires the Kubernetes `watch` verb in order to receive updates to k8s cluster changes. This is required for the `kubernetes_pod_metadata` transform. Yet, Fluentbit [requires the `get`, `list`, and `watch` verbs][fluentbit_role]. Why don't we require the same?~~ @@ -1550,7 +1548,7 @@ See [motivation](#motivation). complete the implementation. It's really trivial to determine from a set of API calls used. See the [Deployment Hardening](#deployment-hardening) section. -1. What are some of the details that set Vector's Kubernetes integration apart? +3. What are some of the details that set Vector's Kubernetes integration apart? This is for marketing purposes and also helps us "raise the bar". ### From Mike @@ -1559,12 +1557,12 @@ See [motivation](#motivation). we want to test against? Some clusters use `docker`, some use `CRI-O`, [etc][container_runtimes]. Some even use [gVisor] or [Firecracker]. There might be differences in how different container runtimes handle logs. -1. How do we want to approach Helm Chart Repository management. -1. How do we implement liveness, readiness and startup probes? +2. How do we want to approach Helm Chart Repository management. +3. How do we implement liveness, readiness and startup probes? Readiness probe is a tricky one. See [Container probes](#container-probes). -1. Can we populate file at `terminationMessagePath` with some meaningful +4. Can we populate file at `terminationMessagePath` with some meaningful information when we exit or crash? -1. Can we allow passing arbitrary fields from the `Pod` object to the event? +5. Can we allow passing arbitrary fields from the `Pod` object to the event? Currently we only to pass `pod_id`, pod `annotations` and pod `labels`. ## Plan Of Attack diff --git a/rfcs/2020-05-25-2685-dev-workflow-simplification.md b/rfcs/2020-05-25-2685-dev-workflow-simplification.md index b5dfb5f41eed6..99a186c843f0c 100644 --- a/rfcs/2020-05-25-2685-dev-workflow-simplification.md +++ b/rfcs/2020-05-25-2685-dev-workflow-simplification.md @@ -183,7 +183,7 @@ There are tools like `hab` (from the Habitat project) and `packer` that can be u ## Outstanding Questions - Windows/Mac/FreeBSD builds via `make build` et all will produce native binaries natively, we should be review those docs. -- This RFC does not scope in integration tests beyond letting the `environment` run them. We may find motivation to explore a more **_slick_** solution in the future. +- This RFC does not scope in integration tests beyond letting the `environment` run them. We may find motivation to explore a more _**slick**_ solution in the future. ## Rationale & Alternatives @@ -207,6 +207,6 @@ Alternatives: 2. Get preliminary consensus this is good path forward 3. Add `DOCKER_SOCKET` passing and support integration testing 4. Cross-OS testing -6. Explore handling `make environment-%` commands via wildcard -7. Acceptance testing (Test including new contributor test) -8. Merge preliminary support +5. Explore handling `make environment-%` commands via wildcard +6. Acceptance testing (Test including new contributor test) +7. Merge preliminary support diff --git a/rfcs/2020-05-25-2692-more-usable-logevents.md b/rfcs/2020-05-25-2692-more-usable-logevents.md index 09ab89b35d7e5..601ef610f626a 100644 --- a/rfcs/2020-05-25-2692-more-usable-logevents.md +++ b/rfcs/2020-05-25-2692-more-usable-logevents.md @@ -183,14 +183,14 @@ In the [WASM transform](https://github.com/vectordotdev/vector/pull/2006/files) This RFC ultimately proposes the following steps: 1. Add UX improvements on `LogEvent`, particularly turning JSON into or from `LogEvent`. -1. Refactor the `PathIter` to make `vector::event::Lookup` type. -1. Add UX improvements on `Lookup` , particularly an internal `String` ↔ `Lookup` with an `Into`/`From` that does not do path parsing, as well as a `::from_str(s: String)` that does. (This also enables `"foo.bar".parse::()?`) -1. Refactor all `LogEvent` to accept `Into` values. +2. Refactor the `PathIter` to make `vector::event::Lookup` type. +3. Add UX improvements on `Lookup` , particularly an internal `String` ↔ `Lookup` with an `Into`/`From` that does not do path parsing, as well as a `::from_str(s: String)` that does. (This also enables `"foo.bar".parse::()?`) +4. Refactor all `LogEvent` to accept `Into` values. 1. Remove obsolete functionality like `insert_path` since the new `Lookup` type covers this. 2. Refactor the `keys` function to return an `Iterator` -1. Add an `Entry` style API to `LogEvent`. +5. Add an `Entry` style API to `LogEvent`. 1. Remove functionality rendered obsolete by the Entry API like `try_insert`, moving them to use the new Entry API -1. Provide `iter` and `iter_mut` functions that yield `(Lookup, Value)`. +6. Provide `iter` and `iter_mut` functions that yield `(Lookup, Value)`. 1. Remove the `all_fields` function, moving them to the new iterator. We believe these steps will provide a more ergonomic and consistent API. diff --git a/rfcs/2020-07-28-3642-jmx_rfc.md b/rfcs/2020-07-28-3642-jmx_rfc.md index b872b276574f5..7f0181b65620c 100644 --- a/rfcs/2020-07-28-3642-jmx_rfc.md +++ b/rfcs/2020-07-28-3642-jmx_rfc.md @@ -161,7 +161,7 @@ The query will return these metrics by parsing the query results and converting The type of metric for any metric marked `untyped` is unclear but likely gauges. We'll need to do deeper research to determine: 1. Whether we want to keep them. -1. What type they are if we retain them. +2. What type they are if we retain them. Naming of metrics is determined via: @@ -215,7 +215,7 @@ Additionally, as part of Vector's vision to be the "one tool" for ingesting and 1. Having users run telegraf or Prom node exporter and using Vector's Prometheus source to scrape it. We could not add the source directly to Vector and instead instruct users to run Prometheus' `jmx_exporter` and point Vector at the resulting data. -1. Or we could use something like [jmxtrans](https://github.com/jmxtrans/jmxtrans) and add a Vector OutputWriter. +2. Or we could use something like [jmxtrans](https://github.com/jmxtrans/jmxtrans) and add a Vector OutputWriter. I decided against both of these this as they would be in conflict with one of the listed principles of Vector: diff --git a/rfcs/2020-10-15-3480-file-source-rework.md b/rfcs/2020-10-15-3480-file-source-rework.md index da199d16e5c56..e7becd118c5d1 100644 --- a/rfcs/2020-10-15-3480-file-source-rework.md +++ b/rfcs/2020-10-15-3480-file-source-rework.md @@ -194,10 +194,10 @@ In the pseudocode above, read scheduling is controlled by `sort`, `should_read`, would need to answer the following questions: 1. In what order should I read the available files? -1. Should I finish one file before moving on to the next? -1. Should I back off reads to this file? -1. Should I back off reads to all files (i.e. sleep)? -1. How long should I spend working on a single file? +2. Should I finish one file before moving on to the next? +3. Should I back off reads to this file? +4. Should I back off reads to all files (i.e. sleep)? +5. How long should I spend working on a single file? These questions would then be answered by a combination of configuration, file metadata, and gathered statistics. @@ -226,9 +226,9 @@ out a magic identifier, but also the logic to update our list of watched files based on those identifiers. It needs to answer the following: 1. Given a visible path, does it contain a file I've seen before? -1. If I have seen it before, has it been renamed? -1. If I have seen it before, am I now seeing it in multiple places? -1. If I'm seeing duplicates, how should I choose which to follow? +2. If I have seen it before, has it been renamed? +3. If I have seen it before, am I now seeing it in multiple places? +4. If I'm seeing duplicates, how should I choose which to follow? This logic is mostly in one place in the current implementation, so it should not be terribly difficult to extract it. The use of `Fingerprinter` should @@ -238,9 +238,9 @@ In addition to simply consolidating the logic, we can expand and make the use of `Fingerprinter` more intelligent. We currently have three ways it can work: 1. Checksum (usually reliable but frustrating for small files) -1. Device and inode (simple and works with small files, but doesn't handle edge +2. Device and inode (simple and works with small files, but doesn't handle edge cases well) -1. First line checksum (solid for intended use case but not yet general) +3. First line checksum (solid for intended use case but not yet general) I'd first propose that we drop device and inode fingerprinting and add path-based fingerprinting in its place. This gives users the option to do the @@ -255,8 +255,8 @@ in place, the algorithm can look something like the following: 1. Read up to `max_line_length` bytes from the file starting at `ignored_header_bytes` -1. Return no fingerprint if there is no newline in the returned bytes -1. Otherwise, return the checksum of the bytes up to the first newline +2. Return no fingerprint if there is no newline in the returned bytes +3. Otherwise, return the checksum of the bytes up to the first newline This should give a good balance between usability and flexibility for the default strategy. As we implement it, we should evolve the current @@ -278,11 +278,11 @@ about providing an understandable config UI than designing the right interface. This should be driven by real world use cases. For example: 1. Ignoring existing checkpoints -1. Start at the beginning or end of existing files, optionally taking into +2. Start at the beginning or end of existing files, optionally taking into account factors like mtime -1. Start at the beginning or end of files added while we're watching (this can +3. Start at the beginning or end of files added while we're watching (this can be tricky) -1. Ordering which of the above concerns take precedence +4. Ordering which of the above concerns take precedence I would suggest a config like the following: @@ -327,8 +327,8 @@ reads, but it will require a bit of experimentation before we're able to determine if it's worthwhile. There are a few possible approaches: 1. Dispatch reads to an explicit threadpool -1. Spawn a limited number of blocking tokio tasks -1. Implement something with `iouring` +2. Spawn a limited number of blocking tokio tasks +3. Implement something with `iouring` The first two both introduce the questions of sizing and the ability of the underlying file system to enable concurrent access in a way that actually adds diff --git a/rfcs/2021-02-23-6531-performance-testing.md b/rfcs/2021-02-23-6531-performance-testing.md index e60a21b02f577..fbb5205bcb3ed 100644 --- a/rfcs/2021-02-23-6531-performance-testing.md +++ b/rfcs/2021-02-23-6531-performance-testing.md @@ -47,10 +47,10 @@ processes in our work on Vector: 1. Performance is a first-class testing concern for Vector. We will drive our process to identify regressions or opportunities for optimization as close to introduction as possible. -1. Identifying _that_ a regression has happened is often easier than _why_. We +2. Identifying _that_ a regression has happened is often easier than _why_. We will continuously improve Vector’s diagnosis tooling to reduce the time to debug and repair detected issues. -1. Performance regressions will inevitably, unintentionally make their way +3. Performance regressions will inevitably, unintentionally make their way into a release. When this happens we will treat this just like we would a correctness regression, relying on our diagnostic tools and rolling the experiences of repair back into the tooling. diff --git a/rfcs/2021-03-26-6517-end-to-end-acknowledgement.md b/rfcs/2021-03-26-6517-end-to-end-acknowledgement.md index b669c9ebc7c68..a69bdbc719c2f 100644 --- a/rfcs/2021-03-26-6517-end-to-end-acknowledgement.md +++ b/rfcs/2021-03-26-6517-end-to-end-acknowledgement.md @@ -330,39 +330,39 @@ source handles acknowledgements. The above structure provides for several considerations: -1. This the minimum amount of data that can be added to the metadata to +1. This the minimum amount of data that can be added to the metadata to fully support this feature, amounting to a single shared reference as the `Option` is optimized into the `Arc`. -2. The use of `Arc` reference counting for finalization prevents events +2. The use of `Arc` reference counting for finalization prevents events from "escaping" without providing a status indication. -3. If a source does not need or is not configured to require +3. If a source does not need or is not configured to require finalization, it will not contribute to the list of sources and so has no additional event overhead, and no additional allocations when the event is created. -4. Sending the finalization status to the source does not require any +4. Sending the finalization status to the source does not require any lookups or topology traversal. -5. No additional work is required to handle dropped sources due to +5. No additional work is required to handle dropped sources due to topology reconfiguration, other than the expected checking for a closed channel when sending. ## Drawbacks -1. This adds a base size overhead to each event, even for +1. This adds a base size overhead to each event, even for configurations that do not support or require end-to-end acknowledgement. ## Alternatives -1. The set of sources could be stored in a more customary `Vec`. This +1. The set of sources could be stored in a more customary `Vec`. This provides for merging multiple sources with a minimum of code. However, merged events already have other overhead, and it increases the data required for this array by an additional word. -2. The source could be stored as simply the unique identifier +2. The source could be stored as simply the unique identifier string. This requires that all reporting of finalization status proceed through a dictionary lookup instead of simply sending it through a channel, increasing the run-time overhead. diff --git a/rfcs/2021-07-19-8216-multiple-pipelines.md b/rfcs/2021-07-19-8216-multiple-pipelines.md index a628fe1843e08..6176d99ec2272 100644 --- a/rfcs/2021-07-19-8216-multiple-pipelines.md +++ b/rfcs/2021-07-19-8216-multiple-pipelines.md @@ -36,8 +36,8 @@ Large Vector users often require complex Vector topologies to facilitate the col This change will introduce the concept of pipelines to users. A pipeline is defined as: 1. A collection of transforms defined together, outside of the top-level configuration file -1. Able to draw input from and send output to components defined in the top-level configuration, but are isolated from other pipelines -1. Having each contained component's internal metrics tagged with the `id` of the pipeline +2. Able to draw input from and send output to components defined in the top-level configuration, but are isolated from other pipelines +3. Having each contained component's internal metrics tagged with the `id` of the pipeline Pipelines will be loaded from a `pipelines` sub-directory relative to the Vector configuration directory (e.g., `/etc/vector/pipelines`). Therefore, if a user changes the location of the Vector configuration directory they will also change the `pipelines` directory path. They are coupled. diff --git a/rfcs/2021-08-13-8025-internal-tracing.md b/rfcs/2021-08-13-8025-internal-tracing.md index 37dc48afc418b..d86a9f8ce4035 100644 --- a/rfcs/2021-08-13-8025-internal-tracing.md +++ b/rfcs/2021-08-13-8025-internal-tracing.md @@ -81,10 +81,10 @@ The step after that is a bit fuzzy, however. You could jump straight to profiling, but it has a few weaknesses: 1. It's not something that can easily be run in customer environments -1. The focus on CPU time vs wall clock time means it can miss issues like bad +2. The focus on CPU time vs wall clock time means it can miss issues like bad rate limits, waiting on downstream components, slow IO operations or syscalls, etc -1. The output tends to require interpretation by an experienced engineer, and +3. The output tends to require interpretation by an experienced engineer, and doesn't always indicate clearly where time is being spent from a Vector perspective @@ -251,10 +251,10 @@ as follows: corresponding visualizations means it would take more interpretation and external knowledge to derive the same signal. -1. Their aggregated nature would put a limit on the level of detail (e.g. no +2. Their aggregated nature would put a limit on the level of detail (e.g. no file name field on a `read` span from the file source). -1. Collecting timings directly is likely to require more explicit +3. Collecting timings directly is likely to require more explicit instrumentation code than simply adding spans. ### Do nothing diff --git a/rfcs/2021-08-29-8381-vrl-iteration-support.md b/rfcs/2021-08-29-8381-vrl-iteration-support.md index 39ab28cc9f717..020e9be84c66b 100644 --- a/rfcs/2021-08-29-8381-vrl-iteration-support.md +++ b/rfcs/2021-08-29-8381-vrl-iteration-support.md @@ -338,14 +338,14 @@ individual use-cases, this list shows one available solution per use-case. . = map_keys(., recursive: true) -> |key| { trim_start(key, "_") } ``` -11. [add prefix to all keys](https://discord.com/channels/742820443487993987/764187584452493323/883274684576182302) +1. [add prefix to all keys](https://discord.com/channels/742820443487993987/764187584452493323/883274684576182302) ```coffee . = map_keys(., recursive: true) -> |key| { "my_" + key } ``` -12. [parse message using list of Grok patterns until one matches](https://discord.com/channels/742820443487993987/764187584452493323/870353108692271104) +2. [parse message using list of Grok patterns until one matches](https://discord.com/channels/742820443487993987/764187584452493323/870353108692271104) ```coffee patterns = [] @@ -359,7 +359,7 @@ individual use-cases, this list shows one available solution per use-case. } ``` -13. [find match against list of regular expressions](https://discord.com/channels/742820443487993987/764187584452493323/864496206947942400) +3. [find match against list of regular expressions](https://discord.com/channels/742820443487993987/764187584452493323/864496206947942400) ```coffee matched = false @@ -377,13 +377,13 @@ individual use-cases, this list shows one available solution per use-case. matched = any(patterns) -> |pattern| { match(.message, pattern) } ``` -14. [remove prefix from keys](https://discord.com/channels/742820443487993987/764187584452493323/864496206947942400) +4. [remove prefix from keys](https://discord.com/channels/742820443487993987/764187584452493323/864496206947942400) ```coffee . = map_keys(. ,recursive: true) -> |key| { replace(key, "my_prefix_", "") } ``` -15. [run `encode_json` on all top-level object fields](https://discord.com/channels/742820443487993987/746070591097798688/841787442271879209) +5. [run `encode_json` on all top-level object fields](https://discord.com/channels/742820443487993987/746070591097798688/841787442271879209) ```coffee . = map_values(.) -> |value| { @@ -395,7 +395,7 @@ individual use-cases, this list shows one available solution per use-case. } ``` -16. [map key/value pairs to object with ”key” and ”value” fields](https://discord.com/channels/742820443487993987/746070591097798688/832684085771370587) +6. [map key/value pairs to object with ”key” and ”value” fields](https://discord.com/channels/742820443487993987/746070591097798688/832684085771370587) ```coffee . = { "labels": { "key1": "value1", "key2": "value2" } } @@ -405,7 +405,8 @@ individual use-cases, this list shows one available solution per use-case. } .labels = new_labels - ``` + + ```text **NOTE** this is similar to [Jq’s `to_entries` function](https://stedolan.github.io/jq/manual/#to_entries,from_entries,with_entries), @@ -422,7 +423,7 @@ individual use-cases, this list shows one available solution per use-case. . = to_entries(.) ``` -17. [run `parse_json` on multiple strings in array, and emit as multiple +1. [run `parse_json` on multiple strings in array, and emit as multiple events](https://discord.com/channels/742820443487993987/746070591097798688/832257215506415657) ```coffee @@ -432,7 +433,7 @@ individual use-cases, this list shows one available solution per use-case. ``` -18. [convert object to specific string format](https://discord.com/channels/742820443487993987/764187584452493323/824574475495407639) +2. [convert object to specific string format](https://discord.com/channels/742820443487993987/764187584452493323/824574475495407639) ```coffee . = { "key1": "value1", "key2": "value2" } @@ -451,7 +452,7 @@ individual use-cases, this list shows one available solution per use-case. "{" + join(strings, ",") + "}" ``` -19. [re-introduce previous `only_fields` functionality using iteration](https://github.com/vectordotdev/vector/issues/7347) +3. [re-introduce previous `only_fields` functionality using iteration](https://github.com/vectordotdev/vector/issues/7347) ```coffee @@ -471,7 +472,7 @@ individual use-cases, this list shows one available solution per use-case. . = filter(.) -> |key, _| { includes(only_fields, key) } ``` -20. [map complex dynamic object based on conditionals](https://github.com/vectordotdev/vector/discussions/12387#discussioncomment-2639876) +4. [map complex dynamic object based on conditionals](https://github.com/vectordotdev/vector/discussions/12387#discussioncomment-2639876) ```coffee .input = map_values(.input) -> |input| { @@ -507,7 +508,7 @@ individual use-cases, this list shows one available solution per use-case. } ``` -21. merge array of objects into single object +5. merge array of objects into single object ```coffee result = {} diff --git a/rfcs/2021-09-01-8547-accept-metrics-in-datadog-agent-source.md b/rfcs/2021-09-01-8547-accept-metrics-in-datadog-agent-source.md index ac95ef626762e..c0471ed349985 100644 --- a/rfcs/2021-09-01-8547-accept-metrics-in-datadog-agent-source.md +++ b/rfcs/2021-09-01-8547-accept-metrics-in-datadog-agent-source.md @@ -78,7 +78,7 @@ https://vector.mycompany.tld` to forward metrics to a Vector deployment. The current `dd_url` endpoint configuration has a [conditional behavior](https://github.com/DataDog/datadog-agent/blob/main/pkg/config/config.go#L1199-L1201) (also -[here](https://github.com/DataDog/datadog-agent/blob/main/pkg/forwarder/forwarder_health.go#L131-L143)). I.e. if +[in the forwarder health check](https://github.com/DataDog/datadog-agent/blob/main/pkg/forwarder/forwarder_health.go#L131-L143)). I.e. if `dd_url` contains a known pattern (i.e. it has a suffix that matches a Datadog site) some extra hostname manipulation happens. But overall, the following paths are expected to be supported on the host behind `dd_url`: @@ -100,20 +100,20 @@ A few details about the Datadog Agents & [Datadog metrics](https://docs.datadogh [`MetricSample`](https://github.com/DataDog/datadog-agent/blob/main/pkg/metrics/metric_sample.go#L81-L94) and can be of [several types](https://github.com/DataDog/datadog-agent/blob/main/pkg/metrics/metric_sample.go#L20-L31) * Major Agent usecases: - * Metrics are send from corechecks (i.e. go code) - [here](https://github.com/DataDog/datadog-agent/blob/main/pkg/aggregator/sender.go#L227-L252) - * Dogstatsd metrics are converted to the `MetricSample` structure - [here](https://github.com/DataDog/datadog-agent/blob/main/pkg/dogstatsd/enrich.go#L87-L137) However Datadog Agents + * Metrics are send from corechecks (i.e. go code) via the + [aggregator sender](https://github.com/DataDog/datadog-agent/blob/main/pkg/aggregator/sender.go#L227-L252) + * Dogstatsd metrics are converted to the `MetricSample` structure in the + [dogstatsd enrich module](https://github.com/DataDog/datadog-agent/blob/main/pkg/dogstatsd/enrich.go#L87-L137). However Datadog Agents metrics are transformed before being sent, ultimately metrics accounts for two different kind of payload: * The count, gauge and rate series kind of payload, sent to `/api/v1/series` using the [JSON schema officially documented](https://docs.datadoghq.com/api/latest/metrics) with few undocumented [additional fields](https://github.com/DataDog/datadog-agent/blob/main/pkg/metrics/series.go#L45-L57), but this align very well with the existing `datadog_metrics` sinks. -* The sketches kind of payload, sent to `/api/beta/sketches` and serialized as protobuf as shown - [here](https://github.com/DataDog/datadog-agent/blob/main/pkg/serializer/serializer.go#L315-L338) (it ultimately lands - [here](https://github.com/DataDog/datadog-agent/blob/main/pkg/metrics/sketch_series.go#L103-L269)). Public `.proto` - definition can be found - [here](https://github.com/DataDog/agent-payload/blob/master/proto/metrics/agent_payload.proto#L47-L81). +* The sketches kind of payload, sent to `/api/beta/sketches` and serialized as protobuf as shown in the + [serializer](https://github.com/DataDog/datadog-agent/blob/main/pkg/serializer/serializer.go#L315-L338) (it ultimately lands in the + [sketch_series module](https://github.com/DataDog/datadog-agent/blob/main/pkg/metrics/sketch_series.go#L103-L269)). Public `.proto` + definition can be found in the + [agent-payload proto](https://github.com/DataDog/agent-payload/blob/master/proto/metrics/agent_payload.proto#L47-L81). Vector has a nice description of its [metrics data model](https://vector.dev/docs/architecture/data-model/metric/) and a [concise enum for diff --git a/rfcs/2021-10-29-8621-framing-and-codecs-sinks.md b/rfcs/2021-10-29-8621-framing-and-codecs-sinks.md index 4b2c7ace88349..87a28793d93ec 100644 --- a/rfcs/2021-10-29-8621-framing-and-codecs-sinks.md +++ b/rfcs/2021-10-29-8621-framing-and-codecs-sinks.md @@ -94,37 +94,37 @@ Incremental steps to execute this change. These will be converted to issues afte Overview for the current state of sinks regarding encoding: -|sink|encoding config|`.apply_rules`|notes| -|-|-|-|-| -|`aws_cloudwatch_logs`| `EncodingConfig` | ✔︎ | Enveloped in `rusoto_logs::InputLogEvent`. `Text` reads message_key() -|`aws_kinesis_firehose`| `EncodingConfig` | ✔︎ | Enveloped in `rusoto_firehose::Record` that serializes to base64. `Text` reads `message_key()` | - -|`aws_kinesis_streams`| `EncodingConfig` | ✔︎ | Enveloped in `rusoto_kinesis::PutRecordsRequestEntry`. `Text` reads `message_key()` -|`aws_s3`| `EncodingConfig` | ✔︎ | Uses `util::{RequestBuilder, Encoder, Compressor}`. `Text` reads `message_key()` -|`aws_sqs`| `EncodingConfig` | ✔︎ | Enveloped in `EncodedEvent`. `Text` reads `message_key()` -|`azure_blob`| `EncodingConfig` | ✔︎ | Enveloped in `EncodedEvent`. `Text` reads `message_key()` -|`azure_monitor_logs`| `EncodingConfigWithDefault` | ✔︎ | Serializes to JSON. Enveloped in HTTP request -|`blackhole`| - | - | - -|`clickhouse`| `EncodingConfigWithDefault` | ✔︎ | Serializes to JSON. Enveloped in HTTP request -|`console`| `EncodingConfig` | ✔︎ | `Text` reads `message_key()` -|`datadog_logs`| `EncodingConfigFixed` | ✔︎ | Doesn't provide options to encode the event payload separately from the protocol -|`datadog_events`| - | ✗ | - -|`datadog_archives`| - | ✗ | Uses custom `DatadogArchivesEncoding`, which has a field `inner: StandardEncodings` which is not user-configurable -|`elasticsearch`| `EncodingConfigFixed` | ✔︎ | Reshapes event internally and implements custom `ElasticsearchEncoder` to serialize for the Elasticsearch protocol -|`file`| `EncodingConfig` | ✔︎ | `Text` reads `message_key()` -|`gcp`| `EncodingConfig` | ✔︎ | Enveloped in HTTP request via `sinks::util::request_builder::RequestBuild`. Sets HTTP request header depending on encoding config -|`honeycomb`| - | ✗ | Embeds event as JSON under a `data` key. Enveloped in HTTP request -|`http`| `EncodingConfig` | ✔︎ | Enveloped in HTTP request. Request-level compression. Sets HTTP request header depending on encoding config -|`humio`| `EncodingConfig` | ✔︎ | Wrapper, see `splunk_hec` for more information -|`influxdb`| `EncodingConfigWithDefault` | ✔︎ | Encoding (for the protocol envelope) is done by routing event fields either into a "tags" or "fields" map that are passed into an internal function `influx_line_protocol` -|`kafka`| `EncodingConfig` | ✔︎ | Uses `Encoder` for `StandardEncodings` in `encode_input`, enveloped in `KafkaRequest` -|`logdna`| `EncodingConfigWithDefault` | ✔︎ | Builds a message by manually picking fields from the event. Enveloped in HTTP request -|`loki`| `EncodingConfig` | ✔︎ | Uses reshaping. `Text` reads `message_key()`, `Logfmt` build a key-value string. Sink has config to preprocess event by adding/removing label fields and timestamp. Enveloped in HTTP request -|`nats`| `EncodingConfig` | ✔︎ | `Text` reads `message_key()` -|`new_relic_logs`| `EncodingConfigWithDefault` | ✔︎ | Defers to HTTP sink, uses encoding config to reshape only and convert to JSON -|`papertrail`| `EncodingConfig` | ✔︎ | `Text` reads `message_key()`. Serializes event using syslog and sends buffer via TCP -|`pulsar`| `EncodingConfig` | ✔︎ | `Text` reads `message_key()`, `Avro` expects another dedicated key for the serialization schema. Serialized buffer is sent to Pulsar producer -|`redis`| `EncodingConfig` | ✔︎ | `Text` reads `message_key()`. Encoded message is serialized to buffer -|`sematext`| `EncodingConfigFixed` | ✔︎ | Wrapper, see `elasticsearch` for more information -|`socket`| `EncodingConfig` | ✔︎ | `Text` reads `message_key()` -|`splunk_hec`| `EncodingConfig` | ✔︎ | Encoding is used to create a message according to the Splunk HEC protocol. There is no separate control over encoding the payload itself -|`vector`| - | - | - +| sink | encoding config | `.apply_rules` | notes | +| - | - | - | - | +| `aws_cloudwatch_logs` | `EncodingConfig` | ✔︎ | Enveloped in `rusoto_logs::InputLogEvent`. `Text` reads message_key() | +| `aws_kinesis_firehose` | `EncodingConfig` | ✔︎ | Enveloped in `rusoto_firehose::Record` that serializes to base64. `Text` reads `message_key()` | +| `aws_kinesis_streams` | `EncodingConfig` | ✔︎ | Enveloped in `rusoto_kinesis::PutRecordsRequestEntry`. `Text` reads `message_key()` | +| `aws_s3` | `EncodingConfig` | ✔︎ | Uses `util::{RequestBuilder, Encoder, Compressor}`. `Text` reads `message_key()` | +| `aws_sqs` | `EncodingConfig` | ✔︎ | Enveloped in `EncodedEvent`. `Text` reads `message_key()` | +| `azure_blob` | `EncodingConfig` | ✔︎ | Enveloped in `EncodedEvent`. `Text` reads `message_key()` | +| `azure_monitor_logs` | `EncodingConfigWithDefault` | ✔︎ | Serializes to JSON. Enveloped in HTTP request | +| `blackhole` | - | - | - | +| `clickhouse` | `EncodingConfigWithDefault` | ✔︎ | Serializes to JSON. Enveloped in HTTP request | +| `console` | `EncodingConfig` | ✔︎ | `Text` reads `message_key()` | +| `datadog_logs` | `EncodingConfigFixed` | ✔︎ | Doesn't provide options to encode the event payload separately from the protocol | +| `datadog_events` | - | ✗ | - | +| `datadog_archives` | - | ✗ | Uses custom `DatadogArchivesEncoding`, which has a field `inner: StandardEncodings` which is not user-configurable | +| `elasticsearch` | `EncodingConfigFixed` | ✔︎ | Reshapes event internally and implements custom `ElasticsearchEncoder` to serialize for the Elasticsearch protocol | +| `file` | `EncodingConfig` | ✔︎ | `Text` reads `message_key()` | +| `gcp` | `EncodingConfig` | ✔︎ | Enveloped in HTTP request via `sinks::util::request_builder::RequestBuild`. Sets HTTP request header depending on encoding config | +| `honeycomb` | - | ✗ | Embeds event as JSON under a `data` key. Enveloped in HTTP request | +| `http` | `EncodingConfig` | ✔︎ | Enveloped in HTTP request. Request-level compression. Sets HTTP request header depending on encoding config | +| `humio` | `EncodingConfig` | ✔︎ | Wrapper, see `splunk_hec` for more information | +| `influxdb` | `EncodingConfigWithDefault` | ✔︎ | Encoding (for the protocol envelope) is done by routing event fields either into a "tags" or "fields" map that are passed into an internal function `influx_line_protocol` | +| `kafka` | `EncodingConfig` | ✔︎ | Uses `Encoder` for `StandardEncodings` in `encode_input`, enveloped in `KafkaRequest` | +| `logdna` | `EncodingConfigWithDefault` | ✔︎ | Builds a message by manually picking fields from the event. Enveloped in HTTP request | +| `loki` | `EncodingConfig` | ✔︎ | Uses reshaping. `Text` reads `message_key()`, `Logfmt` build a key-value string. Sink has config to preprocess event by adding/removing label fields and timestamp. Enveloped in HTTP request | +| `nats` | `EncodingConfig` | ✔︎ | `Text` reads `message_key()` | +| `new_relic_logs` | `EncodingConfigWithDefault` | ✔︎ | Defers to HTTP sink, uses encoding config to reshape only and convert to JSON | +| `papertrail` | `EncodingConfig` | ✔︎ | `Text` reads `message_key()`. Serializes event using syslog and sends buffer via TCP | +| `pulsar` | `EncodingConfig` | ✔︎ | `Text` reads `message_key()`, `Avro` expects another dedicated key for the serialization schema. Serialized buffer is sent to Pulsar producer | +| `redis` | `EncodingConfig` | ✔︎ | `Text` reads `message_key()`. Encoded message is serialized to buffer | +| `sematext` | `EncodingConfigFixed` | ✔︎ | Wrapper, see `elasticsearch` for more information | +| `socket` | `EncodingConfig` | ✔︎ | `Text` reads `message_key()` | +| `splunk_hec` | `EncodingConfig` | ✔︎ | Encoding is used to create a message according to the Splunk HEC protocol. There is no separate control over encoding the payload itself | +| `vector` | - | - | - | diff --git a/rfcs/2021-11-03-9862-ingest-apm-stats-along-traces-in-dd-agent-source.md b/rfcs/2021-11-03-9862-ingest-apm-stats-along-traces-in-dd-agent-source.md index 0dd290232319a..c34109a576289 100644 --- a/rfcs/2021-11-03-9862-ingest-apm-stats-along-traces-in-dd-agent-source.md +++ b/rfcs/2021-11-03-9862-ingest-apm-stats-along-traces-in-dd-agent-source.md @@ -100,7 +100,7 @@ found in the trace support [RFC][trace-support-pr] and may provide relevant cont ## Cross cutting concerns * Ongoing work on transforms to add `named_outputs` that is laying the ground for the same feature but on `sources`, - [one PR][named-outputs-pr] has already be merged while scheduled work is tracked [here][named-outputs-improvements]. + [one PR][named-outputs-pr] has already be merged while scheduled work is tracked in the [named outputs improvements issue][named-outputs-improvements]. * [Ongoing work on schemas][schema-rfc] will ultimately offer a programatic way of validating required fields and express constrains on incoming event for a given sink. Traces & APM stats are a good fit for that because they will be represented as standard Vector events, but sinks handling thos will expect some mandatory information. diff --git a/rfcs/2021-11-12-9811-vrl-vm.md b/rfcs/2021-11-12-9811-vrl-vm.md index 0cb103fa9cf6a..6f6450cf3903d 100644 --- a/rfcs/2021-11-12-9811-vrl-vm.md +++ b/rfcs/2021-11-12-9811-vrl-vm.md @@ -323,8 +323,8 @@ functionality. The implementation needs to occur in well defined stages to prevent dumping a massive PR that never gets over the line. -- [ ] Submit a PR with spike-level code _roughly_ demonstrating the change for - the VM. [here](https://github.com/vectordotdev/vector/pull/9829) +- [ ] Submit a [PR with spike-level code](https://github.com/vectordotdev/vector/pull/9829) _roughly_ demonstrating the change for + the VM. - [ ] Incorporate (unit and property) tests. - [ ] Document and comment the VM and the functions used to emit OpCodes. - [ ] Develop a safe API surrounding the VM (in particular function calls). diff --git a/rfcs/2021-12-06-10298-kubernetes_logs-rewrite.md b/rfcs/2021-12-06-10298-kubernetes_logs-rewrite.md index d2dccc735fd1d..f66ab9824613a 100644 --- a/rfcs/2021-12-06-10298-kubernetes_logs-rewrite.md +++ b/rfcs/2021-12-06-10298-kubernetes_logs-rewrite.md @@ -126,8 +126,8 @@ enough to not warrant a version split between the old and new code. - Replace contents of `src/kubernetes` with equivalents from `kube` 1. `src/kubernetes/client` replaced with `kube-client` - 1. `src/kubernetes/reflector` and dependencies replaced with `kube-runtime::reflector` - 1. `src/kubernetes/state` updates to minimize in-house code + 2. `src/kubernetes/reflector` and dependencies replaced with `kube-runtime::reflector` + 3. `src/kubernetes/state` updates to minimize in-house code - Ensure unit tests and integration tests show matching behavior before and after rewrite ## Future Improvements diff --git a/rfcs/2022-05-17-11532-chronicle-sink.md b/rfcs/2022-05-17-11532-chronicle-sink.md index 146008b8b9b3a..a65596fdfa3d7 100644 --- a/rfcs/2022-05-17-11532-chronicle-sink.md +++ b/rfcs/2022-05-17-11532-chronicle-sink.md @@ -128,16 +128,16 @@ The encoding of the message (text or json) can be set via an `encoding` field. ##### Schema A UDM message has the following sections. (To keep this RFC succinct, this is -not a comprehensive list, the full list can be found -[here](https://cloud.google.com/chronicle/docs/reference/udm-field-list#udm_event_data_model).) +not a comprehensive list, the full list can be found in the +[UDM event data model reference](https://cloud.google.com/chronicle/docs/reference/udm-field-list#udm_event_data_model).) - *Metadata* Contains metadata about the event. The fields available for metadata can be - found [here](https://cloud.google.com/chronicle/docs/reference/udm-field-list#metadata). + found in the [UDM metadata field reference](https://cloud.google.com/chronicle/docs/reference/udm-field-list#metadata). Mandatory fields are `event_type` and `event_timestamp`. The value of `event_type` has repercussions for which fields are mandatory - in the rest of the event. See [here](https://cloud.google.com/chronicle/docs/unified-data-model/udm-usage#required_and_optional_fields_based_on_event_type) + in the rest of the event. See the [required and optional fields by event type](https://cloud.google.com/chronicle/docs/unified-data-model/udm-usage#required_and_optional_fields_based_on_event_type) for the list. For example, if `event_type == "FILE_COPY"` then `src.file` becomes mandatory. **Currently it is not possible to validate a field based on the value of another field within Vector schema, so this will need to @@ -145,18 +145,18 @@ not a comprehensive list, the full list can be found - *Principal* The principal is the entity that originates the event. This field is mandatory - and can contain any of the fields defined - [here](https://cloud.google.com/chronicle/docs/reference/udm-field-list#noun). + and can contain any of the fields defined in the + [UDM noun field reference](https://cloud.google.com/chronicle/docs/reference/udm-field-list#noun). - *Src* The source entity being acted upon. Depending on the `event_type` this field - is optional. Can contain any of the fields defined - [here](https://cloud.google.com/chronicle/docs/reference/udm-field-list#noun). + is optional. Can contain any of the fields defined in the + [UDM noun field reference](https://cloud.google.com/chronicle/docs/reference/udm-field-list#noun). - *Target* The target entity being acted upon. Depending on the `event_type` this field - is optional. Can contain any of the fields defined - [here](https://cloud.google.com/chronicle/docs/reference/udm-field-list#noun). + is optional. Can contain any of the fields defined in the + [UDM noun field reference](https://cloud.google.com/chronicle/docs/reference/udm-field-list#noun). ### Implementation diff --git a/rfcs/2022-10-12-14742-flexible-metric-tags.md b/rfcs/2022-10-12-14742-flexible-metric-tags.md index ccb0a2340e167..283541491d255 100644 --- a/rfcs/2022-10-12-14742-flexible-metric-tags.md +++ b/rfcs/2022-10-12-14742-flexible-metric-tags.md @@ -127,9 +127,9 @@ supported in the presence of multi-valued tags, which will be selectable with co 1. The individual values are tracked as before. Events are dropped when any one tag's cardinality exceeds the limit, but only the tags that would exceed the limit are dropped. -1. The individual values are tracked as before. Events are dropped when any one tag's cardinality +2. The individual values are tracked as before. Events are dropped when any one tag's cardinality exceeds the limit, and all values of tags that would exceed the limit are dropped. -1. Values of multi-valued tags are combined before tracking. Events are dropped as before and all +3. Values of multi-valued tags are combined before tracking. Events are dropped as before and all values of tags that would exceed the limit are dropped. #### Sinks @@ -215,7 +215,7 @@ The use of an `IndexSet` for the tag value provides us with two useful invariant 1. Only unique values for each tag will be stored, which prevents repeated values from showing up in the output. -1. The values can be retrieved in the order they first appeared, which allows us to trivially +2. The values can be retrieved in the order they first appeared, which allows us to trivially retrieve either the first or last stored value. ## Drawbacks @@ -257,16 +257,16 @@ which will cause problems for users: 1. Unconditionally expose the tags as arrays of values using the existing naming, but still accept assignments using either single values or arrays of values. This will cause breakage to existing scripts that relies on the existing single value tag values. -1. For Lua or VRL scripts, conditionally expose the tags as single values or arrays, as described in +2. For Lua or VRL scripts, conditionally expose the tags as single values or arrays, as described in the proposal, but accept assignments following the native JSON codec scheme. In Lua, this could cause a breaking change where scripts that emit metrics that have the wrong tabs type to be accepted for transmission. In VRL, this would create headaches for type definitions, at best preventing proper validation of programs. -1. Expose the tags as single values using the existing naming, picking some arbitrary value when a +3. Expose the tags as single values using the existing naming, picking some arbitrary value when a tag has multiple values, and set up a secondary tags structure that exposes the arrays. This will lead to all kinds of confusion and conflicts when the same tag is assigned through different variables. -1. Add functions specifically for manipulating tag sets. This continues to make metrics management +4. Add functions specifically for manipulating tag sets. This continues to make metrics management look like a second-class afterthought, and doesn't ease any compatibility problems for existing scripts. @@ -284,8 +284,8 @@ that would support this feature but with different semantics: 1. `Vec` — Retains the ordering of tags as they appear, but allows for duplicate values and cannot support both bare tags and multiple values simultaneously. -1. `Vec>` — Same as above but supports bare tags and mutiple values simultaneously. -1. `BTreeSet>` — Duplicate values are merged but are sorted, likely putting the bare +2. `Vec>` — Same as above but supports bare tags and mutiple values simultaneously. +3. `BTreeSet>` — Duplicate values are merged but are sorted, likely putting the bare tag first for single-value uses. There are also at least two other container types that could possibly support this use case: diff --git a/rfcs/2022-10-31-15056-tooling-revamp.md b/rfcs/2022-10-31-15056-tooling-revamp.md index 70e83db6bb530..d0660f5361222 100644 --- a/rfcs/2022-10-31-15056-tooling-revamp.md +++ b/rfcs/2022-10-31-15056-tooling-revamp.md @@ -41,7 +41,7 @@ This RFC discusses improving Vector's developer tooling in order to ease mainten - Test failures are difficult to debug, esp. since [this change](https://github.com/vectordotdev/vector/pull/13128) - Windows is essentially unsupported since Make takes a great deal of effort to install and most [scripts](https://github.com/vectordotdev/vector/tree/v0.24.2/scripts) require Bash and utilities like `find` -- Adding tests for new integrations to CI ([here](https://github.com/vectordotdev/vector/blob/v0.24.2/Makefile#L333-L341) and [here](https://github.com/vectordotdev/vector/blob/v0.24.2/.github/workflows/integration-test.yml#L58-L91)) is a manual and occasionally forgotten step (see [outstanding questions](#outstanding-questions)) +- Adding tests for new integrations to CI ([Makefile](https://github.com/vectordotdev/vector/blob/v0.24.2/Makefile#L333-L341) and [integration-test workflow](https://github.com/vectordotdev/vector/blob/v0.24.2/.github/workflows/integration-test.yml#L58-L91)) is a manual and occasionally forgotten step (see [outstanding questions](#outstanding-questions)) - Makefiles and scripts can get messy fast and often are hard to scale well ## Proposal diff --git a/rfcs/2023-05-03-data-volume-metrics.md b/rfcs/2023-05-03-data-volume-metrics.md index 368fe874cd420..ba1b571653122 100644 --- a/rfcs/2023-05-03-data-volume-metrics.md +++ b/rfcs/2023-05-03-data-volume-metrics.md @@ -109,7 +109,7 @@ as the O(1) scan. #### `component_received_event_bytes_total` -This metric is emitted by the framework [here][source_sender], so it looks like +This metric is emitted by the [framework's source sender][source_sender], so it looks like the only change needed is to add the service tag. #### `component_sent_event_bytes_total` diff --git a/rfcs/README.md b/rfcs/README.md index 8c0f2a932f5f7..055819c005902 100644 --- a/rfcs/README.md +++ b/rfcs/README.md @@ -78,8 +78,8 @@ more info navigating your solution. 1. Search GitHub for [previous issues](https://github.com/vectordotdev/vector/issues) and [RFCs](https://github.com/vectordotdev/vector/tree/master/rfcs) on this topic. -1. If an RFC issue does not exist, [open one](https://github.com/vectordotdev/vector/issues/new/choose). -1. Use the issue to obtain consensus that an RFC is necessary. +2. If an RFC issue does not exist, [open one](https://github.com/vectordotdev/vector/issues/new/choose). +3. Use the issue to obtain consensus that an RFC is necessary. - The change might be quickly rejected. - The change might be on our long term roadmap and get deferred. - The change might be blocked by other work. @@ -87,22 +87,22 @@ more info navigating your solution. ### Creating an RFC 1. Create a new branch -1. Copy the [`rfcs/_YYYY-MM-DD-issue#-title.md`](rfcs/_YYYY-MM-DD-issue%23-title.md) template with the appropriate +2. Copy the [`rfcs/_YYYY-MM-DD-issue#-title.md`](rfcs/_YYYY-MM-DD-issue%23-title.md) template with the appropriate name. Be sure to use the issue number you created above. (e.g., `rfcs/2020-02-10-445-internal-observability.md`) -1. Fill in your RFC, pay attention the bullets and guidelines. Do not omit any sections. -1. Work with the Vector team to land on a confident solution. Allocate time for code-level spikes if necessary. -1. Submit your RFC as a pull request and tag reviewers for approval. +3. Fill in your RFC, pay attention the bullets and guidelines. Do not omit any sections. +4. Work with the Vector team to land on a confident solution. Allocate time for code-level spikes if necessary. +5. Submit your RFC as a pull request and tag reviewers for approval. ### Getting an RFC accepted 1. Schedule a "last call" meeting for your RFC. This should be 1 week after opening your pull request. The purpose is to efficiently obtain consensus. -1. At least 3 Vector team members must approve your RFC in the form of pull request approvals. -1. Once approved, self-merge your RFC, or ask a Vector team member to do it for you. +2. At least 3 Vector team members must approve your RFC in the form of pull request approvals. +3. Once approved, self-merge your RFC, or ask a Vector team member to do it for you. ### Implementing an RFC 1. Create issues from the "Plan Of Attack" section. Place them in an epic if necessary. -1. Coordinate with leadership to schedule your work. +2. Coordinate with leadership to schedule your work. ## FAQ diff --git a/vdev/src/commands/check/markdown.rs b/vdev/src/commands/check/markdown.rs index 002ed62908444..3ecf0c91f3201 100644 --- a/vdev/src/commands/check/markdown.rs +++ b/vdev/src/commands/check/markdown.rs @@ -14,16 +14,10 @@ impl Cli { return Ok(()); } - let args: Vec<&str> = vec![ - "--config", - "scripts/.markdownlintrc", - // We should fix these as well. Previously these files were not linted. - "--ignore", - ".github", - ] - .into_iter() - .chain(files.iter().map(String::as_str)) - .collect(); + let args: Vec<&str> = vec!["--config", ".markdownlint.jsonc"] + .into_iter() + .chain(files.iter().map(String::as_str)) + .collect(); app::exec("markdownlint", &args, true) } diff --git a/website/README.md b/website/README.md index 214ed0d4007e2..47475945dbf9d 100644 --- a/website/README.md +++ b/website/README.md @@ -139,8 +139,8 @@ When you make changes to the Markdown sources, Sass/CSS, or JavaScript, the site ### Add a new version of Vector 1. Add the new version to the `versions` list in [`cue/reference/versions.cue`](./cue/reference/versions.cue). Make sure to preserve reverse ordering. -1. Generate a new CUE file for the release by running `make release-prepare` in the root directory of the Vector repo. This generates a CUE file at `cue/releases/{VERSION}.cue`. -1. Add a new Markdown file to [`content/en/releases`](./content/en/releases), where the filename is `{version}.md` (e.g. `0.12.0.md`) and the file has metadata that looks like this: +2. Generate a new CUE file for the release by running `make release-prepare` in the root directory of the Vector repo. This generates a CUE file at `cue/releases/{VERSION}.cue`. +3. Add a new Markdown file to [`content/en/releases`](./content/en/releases), where the filename is `{version}.md` (e.g. `0.12.0.md`) and the file has metadata that looks like this: ```markdown --- diff --git a/website/content/en/blog/adaptive-request-concurrency.md b/website/content/en/blog/adaptive-request-concurrency.md index 9c9aff9597454..b2a701433d638 100644 --- a/website/content/en/blog/adaptive-request-concurrency.md +++ b/website/content/en/blog/adaptive-request-concurrency.md @@ -25,7 +25,7 @@ TRACE tower_limit::rate::service: rate limit exceeded, disabling service Users typically have two questions about this: 1. What does it mean? -1. How can I fix it? +2. How can I fix it? The answer to the first question is simple: Vector has _internally_ rate-limited processing to respect user-configured limits—[`request.rate_limit_duration_secs`][rate_limit_duration_secs] and [`request.rate_limit_num`][rate_limit_num]—for that particular [sink][sinks]. In other words, Vector has intentionally reduced performance to stay within static limits. @@ -66,12 +66,12 @@ We feel strongly that Vector's **Adaptive Request Concurrency** (ARC) feature pr Here's how that plays out in some example scenarios: -Change | | Response -:------|:-:|:-------- -**You deploy more Vector instances** | ➔ |Vector automatically redistributes HTTP throughput across both current and new instances -**You scale up your Elasticsearch cluster** | ➔ | Vector automatically increases concurrency to take full advantage of the new capacity -**You scale your Elasticsearch cluster back down** | ➔ | Vector lowers concurrency to avoid any risk of destabilizing the cluster (while still taking full of advantage of the now-decreased bandwidth) -**Your Elasticsearch cluster experiences a temporary outage** | ➔ |Vector lowers concurrency dramatically and provides backpressure by [buffering][buffer] events +| Change | | Response | +| :------- | :-: | :-------- | +| **You deploy more Vector instances** | ➔ | Vector automatically redistributes HTTP throughput across both current and new instances | +| **You scale up your Elasticsearch cluster** | ➔ | Vector automatically increases concurrency to take full advantage of the new capacity | +| **You scale your Elasticsearch cluster back down** | ➔ | Vector lowers concurrency to avoid any risk of destabilizing the cluster (while still taking full of advantage of the now-decreased bandwidth) | +| **Your Elasticsearch cluster experiences a temporary outage** | ➔ | Vector lowers concurrency dramatically and provides backpressure by [buffering][buffer] events | With ARC, these scenarios require no human intervention. Vector quietly hums along making these decisions for you with a speed and granularity that rate limits simply cannot provide. diff --git a/website/content/en/blog/highlights-february-2025.md b/website/content/en/blog/highlights-february-2025.md index 1adf7bce8787e..eabb4a8adda38 100644 --- a/website/content/en/blog/highlights-february-2025.md +++ b/website/content/en/blog/highlights-february-2025.md @@ -44,5 +44,5 @@ We supported generating a Vector config schema for a while. This guide demonstra popular IDEs in order to help with writing configs. We would love to this to VRL IDE support in the future. In the meantime, we would like to highlight -this [tree-sitter](https://github.com/tree-sitter/tree-sitter) plugin that was developed by https://github.com/belltoy. You can read more -[here](https://github.com/vectordotdev/vrl/issues/964). +this [tree-sitter](https://github.com/tree-sitter/tree-sitter) plugin that was developed by https://github.com/belltoy. You can read more about it in the +[VRL tree-sitter issue](https://github.com/vectordotdev/vrl/issues/964). diff --git a/website/content/en/blog/how-we-test-vector.md b/website/content/en/blog/how-we-test-vector.md index 2d1313ec0a66e..cd029c3a5116a 100644 --- a/website/content/en/blog/how-we-test-vector.md +++ b/website/content/en/blog/how-we-test-vector.md @@ -21,9 +21,9 @@ task for software like Vector: 1. It's relatively new and not as "battle tested" as more widely deployed software. -1. The vast majority of its functionality lives at the edges, interfacing with +2. The vast majority of its functionality lives at the edges, interfacing with various external systems. -1. Instead of a monolithic application designed for a single task, it is +3. Instead of a monolithic application designed for a single task, it is a collection of components that can be assembled into a near-infinite number of configurations. diff --git a/website/content/en/blog/tracking-allocations.md b/website/content/en/blog/tracking-allocations.md index 009ff33622c11..7fd2815af22b9 100644 --- a/website/content/en/blog/tracking-allocations.md +++ b/website/content/en/blog/tracking-allocations.md @@ -44,4 +44,4 @@ In our development and testing of this feature, we've observed ~20% reduction in We currently do not provide support for tracking memory ownership between components. For example, when a sink is batching events, and is configured to batch many events, you may observe high Vector memory usage. If you looked at the memory usage metrics, you would see most of it being attributed to components that either created the events (such as a source) or processed the events (such as any transforms that modified the event) rather than the sink which is batching the events. Adding support for shared ownership tracking provides further insights into the lifetimes of components, further easing the debugging process. -Please Let us know your feedback and suggestions [here](https://github.com/vectordotdev/vector/issues/15474)! +Please let us know your feedback and suggestions in the [allocation tracking feedback issue](https://github.com/vectordotdev/vector/issues/15474)! diff --git a/website/content/en/blog/vector-remap-language.md b/website/content/en/blog/vector-remap-language.md index d580e612ac4bb..368b113afbb3f 100644 --- a/website/content/en/blog/vector-remap-language.md +++ b/website/content/en/blog/vector-remap-language.md @@ -330,14 +330,14 @@ maintaining flexibility. This makes VRL ideal for always-up, performance-sensitive infrastructure like observability pipelines. To illustrate how we achieve this, below is a VRL feature matrix across these two principles: -| Feature | Safety | Performance | -|:--------------------------|:------:|:-----------:| -| [Progressive type safety] | ✅ | | -| [Fail safety] | ✅ | | -| [Memory safety] | ✅ | | -| [Ergonomic safety] | ✅ | ✅ | -| [Vector/Rust native] | ✅ | ✅ | -| [Stateless] | ✅ | ✅ | +| Feature | Safety | Performance | +| :-- | :----: | :---------: | +| [Progressive type safety] | ✅ | | +| [Fail safety] | ✅ | | +| [Memory safety] | ✅ | | +| [Ergonomic safety] | ✅ | ✅ | +| [Vector/Rust native] | ✅ | ✅ | +| [Stateless] | ✅ | ✅ | For more info on each click on the feature, but for the purposes of demonstrating how VRL is unique, let's touch on the first two: *progressive type @@ -411,7 +411,7 @@ either specify the type of the `.log` field or handle the error in the event that `.log` is not a string. To resolve this error, the user must do one of three things: -1. **Handle the error** +1. **Handle the error** ```coffee ., err = parse_common_log(.log) @@ -430,7 +430,7 @@ three things: handling this error is a *very* common mistake that would otherwise result in data loss and downtime. -2. **Raise the error and abort** +2. **Raise the error and abort** ```coffee . = parse_common_log!(.log) @@ -445,7 +445,7 @@ three things: operators. Again, this forces the users to decide how to handle errors instead of being surprised by them. -3. **Specify types** +3. **Specify types** ```coffee .log = to_string!(.log) diff --git a/website/content/en/docs/administration/management.md b/website/content/en/docs/administration/management.md index c1eb175139124..dff4be7153bbf 100644 --- a/website/content/en/docs/administration/management.md +++ b/website/content/en/docs/administration/management.md @@ -179,11 +179,11 @@ docker restart -f $(docker ps -aqf "name=vector") The commands above involve configuring Vector using TOML, but you can also use JSON or YAML. You can also use one of three image variants (the commands assume `alpine`): -Variant | Image basis -:-------|:----------- -`alpine` | [Alpine](https://hub.docker.com/_/alpine), a Linux distro built around [musl libc](https://www.musl-libc.org) and [BusyBox](https://busybox.net) -`debian` | The [`debian-slim`](https://hub.docker.com/_/debian) image, which is a smaller and more compact version of the standard `debian` image -`distroless` | The [Distroless](https://github.com/GoogleContainerTools/distroless) project, which provides extremely lean images with no package managers, shells, or other inessential utilities +| Variant | Image basis | +| :-------- | :----------- | +| `alpine` | [Alpine](https://hub.docker.com/_/alpine), a Linux distro built around [musl libc](https://www.musl-libc.org) and [BusyBox](https://busybox.net) | +| `debian` | The [`debian-slim`](https://hub.docker.com/_/debian) image, which is a smaller and more compact version of the standard `debian` image | +| `distroless` | The [Distroless](https://github.com/GoogleContainerTools/distroless) project, which provides extremely lean images with no package managers, shells, or other inessential utilities | ### Helm diff --git a/website/content/en/docs/administration/monitoring.md b/website/content/en/docs/administration/monitoring.md index d9947a1b55e67..76819ee25ab18 100644 --- a/website/content/en/docs/administration/monitoring.md +++ b/website/content/en/docs/administration/monitoring.md @@ -77,14 +77,14 @@ sinks: Vector logs at the `info` level by default. You can set a different level when [starting] up your instance using either command-line flags or the `VECTOR_LOG` environment variable. The table below details these options: -Method | Description -:------|:----------- -`-v` flag | Drops the log level to `debug` -`-vv` flag | Drops the log level to `trace` -`-q` flag | Raises the log level to `warn` -`-qq` flag | Raises the log level to `error` -`-qqq` flag | Disables logging -`VECTOR_LOG=` environment variable | Set the log level. Must be one of `trace`, `debug`, `info`, `warn`, `error`, `off`. +| Method | Description | +| :------- | :----------- | +| `-v` flag | Drops the log level to `debug` | +| `-vv` flag | Drops the log level to `trace` | +| `-q` flag | Raises the log level to `warn` | +| `-qq` flag | Raises the log level to `error` | +| `-qqq` flag | Disables logging | +| `VECTOR_LOG=` environment variable | Set the log level. Must be one of `trace`, `debug`, `info`, `warn`, `error`, `off`. | #### Stack traces diff --git a/website/content/en/docs/administration/validating.md b/website/content/en/docs/administration/validating.md index bfdaec1f98189..0519d971fced4 100644 --- a/website/content/en/docs/administration/validating.md +++ b/website/content/en/docs/administration/validating.md @@ -36,10 +36,10 @@ files, including: These checks verify that the configuration file contains a valid topology: 1. At least one [source][sources] is defined -1. At least one [sink][sinks] is defined -1. All inputs for each topology component (specified using the `inputs` parameter) contain at least +2. At least one [sink][sinks] is defined +3. All inputs for each topology component (specified using the `inputs` parameter) contain at least one value. -1. All inputs refer to valid and upstream [sources] or [transforms]. +4. All inputs refer to valid and upstream [sources] or [transforms]. ### Environment checks @@ -47,7 +47,7 @@ Finally, these checks ensure that Vector is running in an environment that can s configured topology: 1. All components have the pre-requisites to run, e.g. data directories exist and are writable. -1. All sinks can connect to their specified targets. +2. All sinks can connect to their specified targets. These environment checks can be disabled using the [`--no-environment`][no_environment] flag: diff --git a/website/content/en/docs/introduction/concepts.md b/website/content/en/docs/introduction/concepts.md index ad2dadddc6135..237b4fa903d50 100644 --- a/website/content/en/docs/introduction/concepts.md +++ b/website/content/en/docs/introduction/concepts.md @@ -40,7 +40,7 @@ A **trace** event can be thought of as a special kind of log event. The componen **Note**: Support for traces is limited and is in alpha. If you’re interested in using traces with a Vector component that doesn’t yet support them, -please check the list of open issues [here](https://github.com/vectordotdev/vector/issues?q=is%3Aissue+state%3Aopen+label%3A%22domain%3A%20traces%22). +please check the [list of open traces issues](https://github.com/vectordotdev/vector/issues?q=is%3Aissue+state%3Aopen+label%3A%22domain%3A%20traces%22). If you don’t see your use case covered, feel free to [open a new issue](https://github.com/vectordotdev/vector/issues/new?template=feature.yml). @@ -88,7 +88,7 @@ This is the default behavior. When a buffer fills up, backpressure will be appli `buffer.when_full = drop_newest` When a buffer fills up, new events will be dropped. This does _not_ provide backpressure. -View the full configuration options for buffers [here](/docs/reference/configuration/sinks/vector/#buffer). +View the [full buffer configuration options](/docs/reference/configuration/sinks/vector/#buffer). ## Backpressure diff --git a/website/content/en/docs/reference/configuration/unit-tests.md b/website/content/en/docs/reference/configuration/unit-tests.md index c838086f50286..5e11240c9136f 100644 --- a/website/content/en/docs/reference/configuration/unit-tests.md +++ b/website/content/en/docs/reference/configuration/unit-tests.md @@ -14,9 +14,9 @@ work just like unit tests in most programming languages: 1. Provide a set of [inputs](#inputs) to a transform (or to [multiple transforms](#multiple) chained together). -1. Specify the expected [outputs](#outputs) from the changes made by the transform (or multiple +2. Specify the expected [outputs](#outputs) from the changes made by the transform (or multiple transforms). -1. Receive directly actionable feedback from any test failures. +3. Receive directly actionable feedback from any test failures. Unit tests can serve as a useful guardrail when running in Vector in production settings where you need to ensure that your topology doesn't exhibit unexpected behavior and generally improve the @@ -219,14 +219,14 @@ Inside each test definition, you need to specify two things: In the `inputs` array for the test, you have these options: -Parameter | Type | Description -:---------|:-----|:----------- -`type` | string | The type of input you're providing. [`vrl`](#logs), [`log`](#logs), [`raw`](#logs), or [`metric`](#metrics) are currently the only valid values. -`insert_at` | string (name of transform) | The name of the transform into which the test input is inserted. This is particularly useful when you want to test only a subset of a transform pipeline. -`value` | string (raw event value) | A raw string value to act as an input event. Use only in cases where events are raw strings and not structured objects with event fields. -`log_fields` | object | If the transform handles [log events](#logs), these are the key/value pairs that comprise the input event. -`metric` | object | If the transform handles [metric events](#metrics), these are the fields that comprise that metric. Subfields include `name`, `tags`, `kind`, and others. -`source` | string (vrl program) | If the transform handles [log events](#logs), the result of the vrl program will be the input event. +| Parameter | Type | Description | +| :--------- | :---- | :----------- | +| `type` | string | The type of input you're providing. [`vrl`](#logs), [`log`](#logs), [`raw`](#logs), or [`metric`](#metrics) are currently the only valid values. | +| `insert_at` | string (name of transform) | The name of the transform into which the test input is inserted. This is particularly useful when you want to test only a subset of a transform pipeline. | +| `value` | string (raw event value) | A raw string value to act as an input event. Use only in cases where events are raw strings and not structured objects with event fields. | +| `log_fields` | object | If the transform handles [log events](#logs), these are the key/value pairs that comprise the input event. | +| `metric` | object | If the transform handles [metric events](#metrics), these are the fields that comprise that metric. Subfields include `name`, `tags`, `kind`, and others. | +| `source` | string (vrl program) | If the transform handles [log events](#logs), the result of the vrl program will be the input event. | Here's an example `inputs` declaration: @@ -248,17 +248,17 @@ message = "<102>1 2020-12-22T15:22:31.111Z vector-user.biz su 2666 ID389 - Somet In the `outputs` array of your unit testing configuration, you specify two things: -Parameter | Type | Description -:---------|:-----|:----------- -`extract_from` | string (name of transform) | The transform whose output you want to test. -`conditions` | array of objects | The [VRL conditions](#verifying) to run against the output. +| Parameter | Type | Description | +| :--------- | :---- | :----------- | +| `extract_from` | string (name of transform) | The transform whose output you want to test. | +| `conditions` | array of objects | The [VRL conditions](#verifying) to run against the output. | Each condition in the `conditions` array has two fields: -Parameter | Type | Description -:---------|:-----|:----------- -`type` | string | The type of condition you're providing. [`vrl`][vrl] is currently the only valid value. -`source` | string (VRL Boolean expression) | Explained in detail [above](#verifying). +| Parameter | Type | Description | +| :--------- | :---- | :----------- | +| `type` | string | The type of condition you're providing. [`vrl`][vrl] is currently the only valid value. | +| `source` | string (VRL Boolean expression) | Explained in detail [above](#verifying). | Here's an example `outputs` declaration: diff --git a/website/content/en/docs/reference/vrl/_index.md b/website/content/en/docs/reference/vrl/_index.md index a0b2cfb426efe..9685d6eddfdbf 100644 --- a/website/content/en/docs/reference/vrl/_index.md +++ b/website/content/en/docs/reference/vrl/_index.md @@ -166,14 +166,14 @@ flexibility. This makes VRL ideal for critical, performance-sensitive infrastructure, like observability pipelines. To illustrate how we achieve these, below is a VRL feature matrix across these principles: -| Feature | Safety | Performance | -| :-------------------------------------------- | :----- | :---------- | -| [Compilation](#compilation) | ✅ | ✅ | -| [Ergonomic safety](#ergonomic-safety) | ✅ | ✅ | -| [Fail safety](#fail-safety) | ✅ | | -| [Memory safety](#memory-safety) | ✅ | | -| [Vector and Rust native](#vector-rust-native) | ✅ | ✅ | -| [Statelessness](#stateless) | ✅ | ✅ | +| Feature | Safety | Performance | +| :-- | :----- | :---------- | +| [Compilation](#compilation) | ✅ | ✅ | +| [Ergonomic safety](#ergonomic-safety) | ✅ | ✅ | +| [Fail safety](#fail-safety) | ✅ | | +| [Memory safety](#memory-safety) | ✅ | | +| [Vector and Rust native](#vector-rust-native) | ✅ | ✅ | +| [Statelessness](#stateless) | ✅ | ✅ | ## Concepts diff --git a/website/content/en/docs/reference/vrl/errors.md b/website/content/en/docs/reference/vrl/errors.md index e6da9cbec8245..ba6fe86ae6d88 100644 --- a/website/content/en/docs/reference/vrl/errors.md +++ b/website/content/en/docs/reference/vrl/errors.md @@ -64,17 +64,17 @@ of an infallible assignment. ##### Empty values -Type | Empty value -:----|:----------- -String | `""` -Integer | `0` -Float | `0.0` -Boolean | `false` -Object | `{}` -Array | `[]` -Timestamp | `t'1970-01-01T00:00:00Z'` (Unix epoch) -Regular expression | `r''` -Null | `null` +| Type | Empty value | +| :---- | :----------- | +| String | `""` | +| Integer | `0` | +| Float | `0.0` | +| Boolean | `false` | +| Object | `{}` | +| Array | `[]` | +| Timestamp | `t'1970-01-01T00:00:00Z'` (Unix epoch) | +| Regular expression | `r''` | +| Null | `null` | #### Coalescing diff --git a/website/content/en/docs/setup/going-to-prod/arch/aggregator.md b/website/content/en/docs/setup/going-to-prod/arch/aggregator.md index 3bf337c7f4ac6..6211e8b5f14c9 100644 --- a/website/content/en/docs/setup/going-to-prod/arch/aggregator.md +++ b/website/content/en/docs/setup/going-to-prod/arch/aggregator.md @@ -111,4 +111,4 @@ This eliminates the need to deploy a single monolith aggregator, creating an unn ## Support -For easy setup and maintenance of this architecture, consider the Vector’s [discussions](https://discussions.vector.dev) or [chat](https://chat.vector.dev). These are free best effort channels. For enterprise needs, consider Datadog Observability Pipelines, which comes with enterprise-level support. Read more about that product [here](https://www.datadoghq.com/product/observability-pipelines/). +For easy setup and maintenance of this architecture, consider the Vector’s [discussions](https://discussions.vector.dev) or [chat](https://chat.vector.dev). These are free best effort channels. For enterprise needs, consider Datadog Observability Pipelines, which comes with enterprise-level support. Read more about [Datadog Observability Pipelines](https://www.datadoghq.com/product/observability-pipelines/). diff --git a/website/content/en/docs/setup/going-to-prod/arch/unified.md b/website/content/en/docs/setup/going-to-prod/arch/unified.md index 59be6fa2a779a..710d1d4bdc6a5 100644 --- a/website/content/en/docs/setup/going-to-prod/arch/unified.md +++ b/website/content/en/docs/setup/going-to-prod/arch/unified.md @@ -23,4 +23,4 @@ We recommend this architecture for Vector users that have already deployed the a ## Support -For easy setup and maintenance of this architecture, consider the Vector’s [discussions](https://discussions.vector.dev) or [chat](https://chat.vector.dev). These are free best effort channels. For enterprise needs, consider Datadog Observability Pipelines, which comes with enterprise-level support. Read more about that [here](https://www.datadoghq.com/product/observability-pipelines/). +For easy setup and maintenance of this architecture, consider the Vector’s [discussions](https://discussions.vector.dev) or [chat](https://chat.vector.dev). These are free best effort channels. For enterprise needs, consider Datadog Observability Pipelines, which comes with enterprise-level support. Read more about [Datadog Observability Pipelines](https://www.datadoghq.com/product/observability-pipelines/). diff --git a/website/content/en/docs/setup/installation/manual/from-source.md b/website/content/en/docs/setup/installation/manual/from-source.md index 5ad19d46bce5c..0af130387f999 100644 --- a/website/content/en/docs/setup/installation/manual/from-source.md +++ b/website/content/en/docs/setup/installation/manual/from-source.md @@ -235,7 +235,7 @@ To update Vector, follow the same [installation](#installation) instructions abo Vector supports many feature flags to customize which features are included in a build. By default, all sources, transforms, and sinks are enabled. To view a complete list of features, they are listed -under "[features]" [here](https://github.com/vectordotdev/vector/blob/master/Cargo.toml). +under "[features]" in [Cargo.toml](https://github.com/vectordotdev/vector/blob/master/Cargo.toml). Example of building with only specific components: diff --git a/website/content/en/docs/setup/installation/package-managers/helm.md b/website/content/en/docs/setup/installation/package-managers/helm.md index 87c49e56140cb..0c38f086781fd 100644 --- a/website/content/en/docs/setup/installation/package-managers/helm.md +++ b/website/content/en/docs/setup/installation/package-managers/helm.md @@ -27,7 +27,7 @@ To check available Helm chart configuration options: helm show values vector/vector ``` -This example configuration file deploys Vector as an Agent, the full default configuration can be found [here](https://github.com/vectordotdev/helm-charts/blob/develop/charts/vector/templates/configmap.yaml). For more information about configuration options, see the [configuration] docs page. +This example configuration file deploys Vector as an Agent, the [full default configuration](https://github.com/vectordotdev/helm-charts/blob/develop/charts/vector/templates/configmap.yaml) is available in the helm-charts repository. For more information about configuration options, see the [configuration] docs page. ```yaml cat <<-'VALUES' > values.yaml @@ -69,7 +69,7 @@ To check available Helm chart configuration options: helm show values vector/vector ``` -The chart deploys an Aggregator by default, the full configuration can be found [here](https://github.com/vectordotdev/helm-charts/blob/develop/charts/vector/templates/configmap.yaml). For more information about configuration options, see the [Configuration] docs page. +The chart deploys an Aggregator by default, the [full configuration](https://github.com/vectordotdev/helm-charts/blob/develop/charts/vector/templates/configmap.yaml) is available in the helm-charts repository. For more information about configuration options, see the [Configuration] docs page. ### Installing diff --git a/website/content/en/docs/setup/installation/platforms/kubernetes.md b/website/content/en/docs/setup/installation/platforms/kubernetes.md index b86e3ad251e48..8629db632ca19 100644 --- a/website/content/en/docs/setup/installation/platforms/kubernetes.md +++ b/website/content/en/docs/setup/installation/platforms/kubernetes.md @@ -40,7 +40,7 @@ kubectl create namespace --dry-run=client -o yaml vector > namespace.yaml ##### Prepare your kustomization file -This example configuration file deploys Vector as an Agent, the full default configuration can be found [here](https://github.com/vectordotdev/helm-charts/blob/develop/charts/vector/templates/configmap.yaml). For more information about configuration options, see the [configuration] docs page. +This example configuration file deploys Vector as an Agent, the [full default configuration](https://github.com/vectordotdev/helm-charts/blob/develop/charts/vector/templates/configmap.yaml) is available in the helm-charts repository. For more information about configuration options, see the [configuration] docs page. ```shell cat <<-'KUSTOMIZATION' > kustomization.yaml @@ -99,7 +99,7 @@ kubectl create namespace --dry-run=client -o yaml vector > namespace.yaml ##### Prepare your kustomization file -This example configuration deploys Vector as an Aggregator, the full configuration can be found [here](https://github.com/vectordotdev/helm-charts/blob/develop/charts/vector/templates/configmap.yaml). For more information about configuration options, see the [Configuration] docs page. +This example configuration deploys Vector as an Aggregator, the [full configuration](https://github.com/vectordotdev/helm-charts/blob/develop/charts/vector/templates/configmap.yaml) is available in the helm-charts repository. For more information about configuration options, see the [Configuration] docs page. ```shell cat <<-'KUSTOMIZATION' > kustomization.yaml diff --git a/website/content/en/guides/developer/config-autocompletion.md b/website/content/en/guides/developer/config-autocompletion.md index e092a023f20f5..d98486e1c4ae2 100644 --- a/website/content/en/guides/developer/config-autocompletion.md +++ b/website/content/en/guides/developer/config-autocompletion.md @@ -37,11 +37,11 @@ vector generate-schema -o vector-v0.45.0-schema.json 1. `Settings | Languages & Frameworks | Schemas and DTDs | JSON Schema Mappings` 2. Import `vector-v0.45.0-schema.json` -You can find more details [here][jetbrains]. +You can find more details in the [JetBrains JSON schema documentation][jetbrains]. ### Visual Studio Code -Follow the guide [here][vscode]. +Follow the [VS Code YAML schema validation guide][vscode]. ## Example diff --git a/website/content/en/guides/developer/debugging.md b/website/content/en/guides/developer/debugging.md index 4e84514264e02..af1081377f2d5 100644 --- a/website/content/en/guides/developer/debugging.md +++ b/website/content/en/guides/developer/debugging.md @@ -35,7 +35,7 @@ You can set different verbosity levels for specific components: VECTOR_LOG=info,vector::sources::aws_s3=warn vector --config path/to/config.yaml ``` -You can find more information on the syntax [here](https://docs.rs/tracing-subscriber/latest/tracing_subscriber/filter/struct.EnvFilter.html#usage-notes). +You can find more information on the [EnvFilter syntax](https://docs.rs/tracing-subscriber/latest/tracing_subscriber/filter/struct.EnvFilter.html#usage-notes). ### Vector Tools diff --git a/website/content/en/highlights/2020-10-27-kubernetes-integration.md b/website/content/en/highlights/2020-10-27-kubernetes-integration.md index b94d1eb296e19..505f86f217c55 100644 --- a/website/content/en/highlights/2020-10-27-kubernetes-integration.md +++ b/website/content/en/highlights/2020-10-27-kubernetes-integration.md @@ -23,14 +23,14 @@ Kubernetes observability data. ## Feature highlights -1. [**A new `kubernetes_logs` source**][kubernetes_logs_source] - A new source +1. [**A new `kubernetes_logs` source**][kubernetes_logs_source] - A new source designed to handle the intricacies of Kubernetes log collection. It'll collect all Pod logs, merge split logs together, and enrich them with k8s metadata. -2. [**YAML config support**][config_formats_highlight] - +2. [**YAML config support**][config_formats_highlight] - To ensure Vector fits cleanly into your existing K8s workflows, Vector now accepts YAML and JSON config formats. -3. [**Adaptive Request Currency (ARC)**][adaptive_concurrency_post] - +3. [**Adaptive Request Currency (ARC)**][adaptive_concurrency_post] - A new Vector feature designed to automatically optimize HTTP communication in the face of ever changing environments like Kubernetes. It does away with static rate limits and raises the performance and reliability of your entire diff --git a/website/content/en/highlights/2020-11-19-prometheus-remote-integrations.md b/website/content/en/highlights/2020-11-19-prometheus-remote-integrations.md index 760d069821723..604b85bfc4bf7 100644 --- a/website/content/en/highlights/2020-11-19-prometheus-remote-integrations.md +++ b/website/content/en/highlights/2020-11-19-prometheus-remote-integrations.md @@ -66,7 +66,7 @@ Kubernetes where Prometheus is tightly integrated. To get started, setup the new [`prometheus_remote_write` source][prometheus_remote_write_source] and send your metrics to [Datadog][datadog], [New Relic][new_relic], [Influx][influx], -[Elasticsearch][elastic], and [more][sinks]: +[Elasticsearch][elastic], and [many other sinks][sinks]: ```toml title="vector.toml" [sources.prometheus] diff --git a/website/content/en/highlights/2020-12-01-0-11-upgrade-guide.md b/website/content/en/highlights/2020-12-01-0-11-upgrade-guide.md index dc37a0f044e8d..4c51c5dba4eb8 100644 --- a/website/content/en/highlights/2020-12-01-0-11-upgrade-guide.md +++ b/website/content/en/highlights/2020-12-01-0-11-upgrade-guide.md @@ -13,12 +13,12 @@ badges: 0.11 includes some minor breaking changes: 1. [The metrics emitted by the `internal_metrics` source have changed names.](#second) -1. [The `statsd` sink now supports all socket types.](#third) -1. [The `source_type` field is now explicit in the `splunk_hec` sink.](#fifth) -1. [Remove forwarding to syslog from distributed systemd unit.](#sixth) -1. [The `http` source no longer dedots JSON fields.](#seventh) -1. [The `prometheus` sink has been renamed to `prometheus_exporter`](#first) -1. [The `reduce` transform `identifier_fields` was renamed to `group_by`.](#fourth) +2. [The `statsd` sink now supports all socket types.](#third) +3. [The `source_type` field is now explicit in the `splunk_hec` sink.](#fifth) +4. [Remove forwarding to syslog from distributed systemd unit.](#sixth) +5. [The `http` source no longer dedots JSON fields.](#seventh) +6. [The `prometheus` sink has been renamed to `prometheus_exporter`](#first) +7. [The `reduce` transform `identifier_fields` was renamed to `group_by`.](#fourth) We cover each below to help you upgrade quickly: diff --git a/website/content/en/highlights/2021-02-16-0-12-upgrade-guide.md b/website/content/en/highlights/2021-02-16-0-12-upgrade-guide.md index e0e21c3bba265..9f05bda354d2f 100644 --- a/website/content/en/highlights/2021-02-16-0-12-upgrade-guide.md +++ b/website/content/en/highlights/2021-02-16-0-12-upgrade-guide.md @@ -14,10 +14,10 @@ badges: painlessly. If you have questions, [hop in our chat][chat] and we'll help you upgrade. 1. [Breaking: The `encoding.codec` option is now required for all relevant sinks](#first) -1. [Breaking: Vector `check_fields` conditions now require the `type` option](#second) -1. [Breaking: The `generator` source requires a `format` option](#third) -1. [Deprecation: Many transforms have been deprecated in favor of the new `remap` transform](#fourth) -1. [Deprecation: The `file` source `start_at_beginning` has been deprecated](#fifth) +2. [Breaking: Vector `check_fields` conditions now require the `type` option](#second) +3. [Breaking: The `generator` source requires a `format` option](#third) +4. [Deprecation: Many transforms have been deprecated in favor of the new `remap` transform](#fourth) +5. [Deprecation: The `file` source `start_at_beginning` has been deprecated](#fifth) ## Upgrade Guide diff --git a/website/content/en/highlights/2021-08-24-vector-aggregator.md b/website/content/en/highlights/2021-08-24-vector-aggregator.md index c55f7aecbd19e..95a2c790e3b69 100644 --- a/website/content/en/highlights/2021-08-24-vector-aggregator.md +++ b/website/content/en/highlights/2021-08-24-vector-aggregator.md @@ -37,7 +37,7 @@ installing a HAProxy Deployment for load balancing across multiple aggregators. ## Setup and installation -Instructions on how setup and install the `vector-aggregator` chart can be found [here][setup]. +Instructions on how setup and install the `vector-aggregator` chart can be found in the [Helm installation guide][setup]. ## Looking forward diff --git a/website/content/en/highlights/2021-08-25-0-16-upgrade-guide.md b/website/content/en/highlights/2021-08-25-0-16-upgrade-guide.md index 9061612d5d8b2..96c30410f344c 100644 --- a/website/content/en/highlights/2021-08-25-0-16-upgrade-guide.md +++ b/website/content/en/highlights/2021-08-25-0-16-upgrade-guide.md @@ -13,10 +13,10 @@ badges: Vector's 0.16.0 release includes three **breaking changes**: 1. [Component name field renamed to ID](#name-to-id) -1. [Datadog Log sink encoding option removed](#encoding) -1. [Renaming of `memory_use_bytes` internal metric](#memory_use_bytes) -1. [`datadog_logs` source renamed to `datadog_agent`](#datadog_logs_rename) -1. [`kubernetes_logs` source's new RBAC](#kubernetes_logs_rbac) +2. [Datadog Log sink encoding option removed](#encoding) +3. [Renaming of `memory_use_bytes` internal metric](#memory_use_bytes) +4. [`datadog_logs` source renamed to `datadog_agent`](#datadog_logs_rename) +5. [`kubernetes_logs` source's new RBAC](#kubernetes_logs_rbac) We cover them below to help you upgrade quickly: diff --git a/website/content/en/highlights/2021-10-08-0-17-upgrade-guide.md b/website/content/en/highlights/2021-10-08-0-17-upgrade-guide.md index 2af26a0c1b6d5..a6cf5ba705354 100644 --- a/website/content/en/highlights/2021-10-08-0-17-upgrade-guide.md +++ b/website/content/en/highlights/2021-10-08-0-17-upgrade-guide.md @@ -13,12 +13,12 @@ badges: Vector's 0.17.0 release includes several **breaking changes**: 1. [Blackhole sink configuration changes](#blackhole) -1. [Datadog Logs sink loses `batch.max_bytes` setting](#datadog_logs_max_bytes) -1. [Vector now logs to stderr](#logging) -1. [The `generator` source now has a default `interval` setting](#interval) -1. [The deprecated `wasm` transform was removed](#wasm) -1. [The `exec` source now has a `decoding` setting](#exec_source) -1. [The algorithm underlying ARC has been optimized](#arc) +2. [Datadog Logs sink loses `batch.max_bytes` setting](#datadog_logs_max_bytes) +3. [Vector now logs to stderr](#logging) +4. [The `generator` source now has a default `interval` setting](#interval) +5. [The deprecated `wasm` transform was removed](#wasm) +6. [The `exec` source now has a `decoding` setting](#exec_source) +7. [The algorithm underlying ARC has been optimized](#arc) We cover them below to help you upgrade quickly: diff --git a/website/content/en/highlights/2021-11-18-0-18-0-upgrade-guide.md b/website/content/en/highlights/2021-11-18-0-18-0-upgrade-guide.md index 66c49480594a4..2270990762970 100644 --- a/website/content/en/highlights/2021-11-18-0-18-0-upgrade-guide.md +++ b/website/content/en/highlights/2021-11-18-0-18-0-upgrade-guide.md @@ -13,10 +13,10 @@ badges: Vector's 0.18.0 release includes **breaking changes**: 1. [`batch.max_size` no longer valid for sinks](#batch-max-size) -1. [`request.in_flight_limit` no longer valid for sources and sinks](#request-in-flight-limit) -1. [`http_client_responses_total` now labels status with only numeric code](#http_client_responses_total) -1. [Field name change for aggregated summaries in `metric_to_log` transform](#agg_summary_metric_to_log) -1. [Removal of deprecated configuration fields for the Datadog metrics sink: `host` and `namespace`](#dd_metrics_sink_deprecated_fields) +2. [`request.in_flight_limit` no longer valid for sources and sinks](#request-in-flight-limit) +3. [`http_client_responses_total` now labels status with only numeric code](#http_client_responses_total) +4. [Field name change for aggregated summaries in `metric_to_log` transform](#agg_summary_metric_to_log) +5. [Removal of deprecated configuration fields for the Datadog metrics sink: `host` and `namespace`](#dd_metrics_sink_deprecated_fields) We cover them below to help you upgrade quickly: diff --git a/website/content/en/highlights/2021-12-28-0-19-0-upgrade-guide.md b/website/content/en/highlights/2021-12-28-0-19-0-upgrade-guide.md index fcfa8c9c8aff7..e23e7b54c9f5e 100644 --- a/website/content/en/highlights/2021-12-28-0-19-0-upgrade-guide.md +++ b/website/content/en/highlights/2021-12-28-0-19-0-upgrade-guide.md @@ -13,9 +13,9 @@ badges: Vector's 0.19.0 release includes **breaking changes**: 1. [Removal of deprecated configuration fields for the Splunk HEC Logs sink: `host`](#splunk-hec-logs-sink-deprecated-fields) -1. [Updated internal metrics for the Splunk HEC sinks](#splunk-hec-sinks-metrics-update) -1. [Removal of deprecated configuration fields for the Elasticsearch sink](#elasticsearch-sink-deprecated-fields) -1. [Removal of default for `version` field for Vector source and sink](#vector-version) +2. [Updated internal metrics for the Splunk HEC sinks](#splunk-hec-sinks-metrics-update) +3. [Removal of deprecated configuration fields for the Elasticsearch sink](#elasticsearch-sink-deprecated-fields) +4. [Removal of default for `version` field for Vector source and sink](#vector-version) And **deprecations**: diff --git a/website/content/en/highlights/2022-02-08-0-20-0-upgrade-guide.md b/website/content/en/highlights/2022-02-08-0-20-0-upgrade-guide.md index 9bb01a8f6026a..997aefb340c72 100644 --- a/website/content/en/highlights/2022-02-08-0-20-0-upgrade-guide.md +++ b/website/content/en/highlights/2022-02-08-0-20-0-upgrade-guide.md @@ -13,12 +13,12 @@ badges: Vector's 0.20.0 release includes **breaking changes**: 1. [Change to set expiration behavior in `prometheus_exporter` sink](#prom-exporter-set-expiration) -1. [New metrics behavior for `route` transform](#metrics-route-transform) -1. [New internal events for `loki` sink](#events-loki-sink) -1. [`if` statement predicates in `VRL` need to handle errors](#vrl-fallible-predicates) -1. [Division by a literal nonzero number in `VRL` is no longer fallible](#vrl-infallible-division) -1. ['syslog' decoder will raise an error if the incoming message cannot be decoded successfully](#syslog-decoder) -1. [Unit tests no longer allow nonexistent targets in `extract_from`/`no_outputs_from` configurations](#unit-test-nonexistent-build-error) +2. [New metrics behavior for `route` transform](#metrics-route-transform) +3. [New internal events for `loki` sink](#events-loki-sink) +4. [`if` statement predicates in `VRL` need to handle errors](#vrl-fallible-predicates) +5. [Division by a literal nonzero number in `VRL` is no longer fallible](#vrl-infallible-division) +6. ['syslog' decoder will raise an error if the incoming message cannot be decoded successfully](#syslog-decoder) +7. [Unit tests no longer allow nonexistent targets in `extract_from`/`no_outputs_from` configurations](#unit-test-nonexistent-build-error) And **deprecations**: diff --git a/website/content/en/highlights/2022-03-22-0-21-0-upgrade-guide.md b/website/content/en/highlights/2022-03-22-0-21-0-upgrade-guide.md index 855cfa2ddd81a..761e6d861455b 100644 --- a/website/content/en/highlights/2022-03-22-0-21-0-upgrade-guide.md +++ b/website/content/en/highlights/2022-03-22-0-21-0-upgrade-guide.md @@ -58,10 +58,10 @@ Here are some examples that require migrating | old syntax | new syntax | comment | | ------------- | --------------- | -------------- | -| foo\\.bar.baz | "foo.bar".baz | The `.` field separator needs to be escaped if used as part of a field name. The old syntax allowed individual character escaping. The new syntax requires quotes around the field name. -| headers.User-Agent | headers."User-Agent" | `-` requires quotes with the new syntax -| foo with spaces | "foo with spaces" | Spaces also need to be quoted -| foo\\"bar | "foo\\"bar" | Double quotes and backlashes must be escaped _inside_ quotes +| foo\\.bar.baz | "foo.bar".baz | The `.` field separator needs to be escaped if used as part of a field name. The old syntax allowed individual character escaping. The new syntax requires quotes around the field name. | +| headers.User-Agent | headers."User-Agent" | `-` requires quotes with the new syntax | +| foo with spaces | "foo with spaces" | Spaces also need to be quoted | +| foo\\"bar | "foo\\"bar" | Double quotes and backlashes must be escaped _inside_ quotes | ### TOML transform example @@ -84,7 +84,7 @@ inputs = ["input"] fields.match = ["message.\"user-identifier\""] ``` -For more information on the new syntax, you can review the documentation [here](https://vector.dev/docs/reference/vrl/expressions/#path) +For more information on the new syntax, you can review the [VRL path expressions documentation](https://vector.dev/docs/reference/vrl/expressions/#path) diff --git a/website/content/en/highlights/2022-03-31-native-event-codecs.md b/website/content/en/highlights/2022-03-31-native-event-codecs.md index b31b000e6bdfa..0a10938afffea 100644 --- a/website/content/en/highlights/2022-03-31-native-event-codecs.md +++ b/website/content/en/highlights/2022-03-31-native-event-codecs.md @@ -63,9 +63,9 @@ a `timestamp` key like: ``` The specific JSON schema here is subject to change, but you can find an initial -schema [here][cue schema]. The protobuf schema for the `native` codec is the -same used in the `vector` source and sink, and the definition can be found -[here][proto schema]. +[CUE schema][cue schema]. The protobuf schema for the `native` codec is the +same used in the `vector` source and sink, and the definition can be found in the +[proto schema][proto schema]. We will be providing more thorough guidance for using each as the feature matures. One current limitation of the JSON-based native codec is that timestamp diff --git a/website/content/en/highlights/2022-05-03-0-22-0-upgrade-guide.md b/website/content/en/highlights/2022-05-03-0-22-0-upgrade-guide.md index 0d1fcc0645fcc..b7be9fad5dd50 100644 --- a/website/content/en/highlights/2022-05-03-0-22-0-upgrade-guide.md +++ b/website/content/en/highlights/2022-05-03-0-22-0-upgrade-guide.md @@ -13,8 +13,8 @@ badges: Vector's 0.22.0 release includes **breaking changes**: 1. [`gcp_stackdriver_metrics` configuration change](#stackdriver-metrics) -3. [VRL now supports template strings](#vrl-template-strings) -4. [`encode_key_value` and `encode_logfmt` quote wrapping behavior change](#encode-key-value-quote-wrapping) +2. [VRL now supports template strings](#vrl-template-strings) +3. [`encode_key_value` and `encode_logfmt` quote wrapping behavior change](#encode-key-value-quote-wrapping) and **deprecations**: @@ -68,7 +68,7 @@ project_id = "vector-123456" zone = "us-central1-a" ``` -For more information on the new syntax, you can review the documentation [here](https://vector.dev/docs/reference/configuration/sinks/gcp_stackdriver_metrics/) +For more information on the new syntax, you can review the [GCP Stackdriver Metrics sink documentation](https://vector.dev/docs/reference/configuration/sinks/gcp_stackdriver_metrics/) #### VRL now supports template strings {#vrl-template-strings} diff --git a/website/content/en/highlights/2022-10-04-0-25-0-upgrade-guide.md b/website/content/en/highlights/2022-10-04-0-25-0-upgrade-guide.md index b017db9ccda3c..8464524b01c45 100644 --- a/website/content/en/highlights/2022-10-04-0-25-0-upgrade-guide.md +++ b/website/content/en/highlights/2022-10-04-0-25-0-upgrade-guide.md @@ -21,8 +21,8 @@ Vector's 0.25.0 release includes **breaking changes**: and **deprecations**: 1. [Deprecation of VRL metadata functions](#metadata-function-deprecation) -1. [Deprecation of `endpoint` option in Elasticsearch sink](#elasticsearch-endpoint-deprecation) -1. [Deprecation of the Lua version 1 API](#lua-v1-api-deprecation) +2. [Deprecation of `endpoint` option in Elasticsearch sink](#elasticsearch-endpoint-deprecation) +3. [Deprecation of the Lua version 1 API](#lua-v1-api-deprecation) We cover them below to help you upgrade quickly: diff --git a/website/content/en/highlights/2023-02-28-0-28-0-upgrade-guide.md b/website/content/en/highlights/2023-02-28-0-28-0-upgrade-guide.md index 847f9e796cfb2..8d221c3062213 100644 --- a/website/content/en/highlights/2023-02-28-0-28-0-upgrade-guide.md +++ b/website/content/en/highlights/2023-02-28-0-28-0-upgrade-guide.md @@ -15,7 +15,7 @@ Vector's 0.28.0 release includes **breaking changes**: 2. [Removal of metadata functions](#metadata-functions-removal) 3. [Removal of the `apex` sink](#apex-removal) 4. [Removal of the `disk_v1` buffer type](#disk_v1-removal) -4. [Renaming of `host` to `hostname` in `datadog_logs`](#host-rename-dd-logs) +5. [Renaming of `host` to `hostname` in `datadog_logs`](#host-rename-dd-logs) and **deprecations**: diff --git a/website/content/en/highlights/2023-03-26-0-37-0-upgrade-guide.md b/website/content/en/highlights/2023-03-26-0-37-0-upgrade-guide.md index cc9b211403d01..116e1a96027c4 100644 --- a/website/content/en/highlights/2023-03-26-0-37-0-upgrade-guide.md +++ b/website/content/en/highlights/2023-03-26-0-37-0-upgrade-guide.md @@ -12,7 +12,7 @@ badges: Vector's 0.37.0 release includes **breaking changes**: 1. [Vector defaults to requiring non-optional environment variables](#strict-env-vars) -1. [The `dnstap` source now requires the `mode` parameter](#dnstap-mode) +2. [The `dnstap` source now requires the `mode` parameter](#dnstap-mode) and **potentially impactful changes**: diff --git a/website/content/en/highlights/2023-07-05-0-31-0-upgrade-guide.md b/website/content/en/highlights/2023-07-05-0-31-0-upgrade-guide.md index 7214b6d8a7612..9f4f2976f261e 100644 --- a/website/content/en/highlights/2023-07-05-0-31-0-upgrade-guide.md +++ b/website/content/en/highlights/2023-07-05-0-31-0-upgrade-guide.md @@ -12,7 +12,7 @@ badges: Vector's 0.31.0 release includes **breaking changes**: 1. [Removal of various deprecated internal metrics](#deprecated-internal-metrics) -1. [`component_received_event_bytes_total` and `component_sent_event_bytes_total` consistently use estimated JSON size of the event](#event_json_size) +2. [`component_received_event_bytes_total` and `component_sent_event_bytes_total` consistently use estimated JSON size of the event](#event_json_size) and **potentially impactful changes**: diff --git a/website/content/en/highlights/2023-09-27-0-33-0-upgrade-guide.md b/website/content/en/highlights/2023-09-27-0-33-0-upgrade-guide.md index a081b80ee22b7..c965c41aae404 100644 --- a/website/content/en/highlights/2023-09-27-0-33-0-upgrade-guide.md +++ b/website/content/en/highlights/2023-09-27-0-33-0-upgrade-guide.md @@ -12,18 +12,18 @@ badges: Vector's 0.33.0 release includes **breaking changes**: 1. [Behavior of the `datadog_logs` sink's `endpoint` setting](#datadog-logs-endpoint) -1. [Disable OpenSSL legacy provider by default](#openssl-legacy-provider) +2. [Disable OpenSSL legacy provider by default](#openssl-legacy-provider) and **deprecations**: 1. [Default config location change](#default-config-location-change) -1. [Renaming the `armv7` rpm package](#armv7-rename) -1. [Metadata field in the Vector protobuf definition](#vector-proto-metadata) +2. [Renaming the `armv7` rpm package](#armv7-rename) +3. [Metadata field in the Vector protobuf definition](#vector-proto-metadata) and **potentially impactful changes**: 1. [Async runtime default number of worker threads](#runtime-worker-threads) -1. [Setting `request.concurrency = "none"`](#request-concurrency) +2. [Setting `request.concurrency = "none"`](#request-concurrency) We cover them below to help you upgrade quickly: diff --git a/website/content/en/highlights/2023-11-07-0-34-0-upgrade-guide.md b/website/content/en/highlights/2023-11-07-0-34-0-upgrade-guide.md index abd78eb132d99..0eb2e166d664d 100644 --- a/website/content/en/highlights/2023-11-07-0-34-0-upgrade-guide.md +++ b/website/content/en/highlights/2023-11-07-0-34-0-upgrade-guide.md @@ -12,18 +12,18 @@ badges: Vector's 0.34.0 release includes **breaking changes**: 1. [Removal of Deprecated Datadog Component Config Options](#datadog-deprecated-config-options) -1. [Removal of Deprecated `component_name` Metric Tag](#deprecated-component-name) -1. [Removal of Deprecated Metrics Replaced by `component_errors_total`](#deprecated-component-errors-total-metrics) -1. [Removal of `peer_addr` Metric Tag](#remove-peer-addr) -1. [Blackhole sink no longer reports by default](#blackhole-sink-reporting) -1. [Remove direct OpenSSL legacy provider support](#openssl-legacy-provider) -1. [Removal of the deprecated `armv7` rpm package](#armv7-rename) -1. [Default config location change](#default-config-location-change) +2. [Removal of Deprecated `component_name` Metric Tag](#deprecated-component-name) +3. [Removal of Deprecated Metrics Replaced by `component_errors_total`](#deprecated-component-errors-total-metrics) +4. [Removal of `peer_addr` Metric Tag](#remove-peer-addr) +5. [Blackhole sink no longer reports by default](#blackhole-sink-reporting) +6. [Remove direct OpenSSL legacy provider support](#openssl-legacy-provider) +7. [Removal of the deprecated `armv7` rpm package](#armv7-rename) +8. [Default config location change](#default-config-location-change) and **deprecations**: 1. [Deprecation of `requests_completed_total`, `request_duration_seconds`, and `requests_received_total` Internal Metrics](#deprecate-obsolete-http-metrics) -1. [Deprecation of `repositories.timber.io` package repositories](#timber-package-deprecation) +2. [Deprecation of `repositories.timber.io` package repositories](#timber-package-deprecation) We cover them below to help you upgrade quickly: diff --git a/website/content/en/highlights/2023-11-07-new-linux-repos.md b/website/content/en/highlights/2023-11-07-new-linux-repos.md index 83ad54738fca4..b437833a51a8d 100644 --- a/website/content/en/highlights/2023-11-07-new-linux-repos.md +++ b/website/content/en/highlights/2023-11-07-new-linux-repos.md @@ -54,14 +54,14 @@ existing repository to your discretion. rm "/etc/apt/sources.list.d/timber-vector.list" ``` -2. Run the following commands to set up APT to download through HTTPS: +1. Run the following commands to set up APT to download through HTTPS: ```sh sudo apt-get update sudo apt-get install apt-transport-https curl gnupg ``` -3. Run the following commands to set up the Vector `deb` repo on your system +1. Run the following commands to set up the Vector `deb` repo on your system and create a Datadog archive keyring: ```sh @@ -73,7 +73,7 @@ curl https://keys.datadoghq.com/DATADOG_APT_KEY_F14F620E.public | sudo gpg --no- curl https://keys.datadoghq.com/DATADOG_APT_KEY_C0962C7D.public | sudo gpg --no-default-keyring --keyring /usr/share/keyrings/datadog-archive-keyring.gpg --import --batch ``` -4. Run the following commands to update your local `apt` repo and install Vector: +1. Run the following commands to update your local `apt` repo and install Vector: ```sh sudo apt-get update @@ -91,7 +91,7 @@ sudo apt-get install vector rm "/etc/yum.repos.d/timber-vector.repo" ``` -2. Run the following commands to set up the Vector `rpm` repo on your system: +1. Run the following commands to set up the Vector `rpm` repo on your system: ```sh cat < /etc/yum.repos.d/vector.repo @@ -109,7 +109,7 @@ EOF **Note:** If you are running RHEL 8.1 or CentOS 8.1, use `repo_gpgcheck=0` instead of `repo_gpgcheck=1` in the configuration above. -3. Update your packages and install Vector: +1. Update your packages and install Vector: ```sh sudo yum makecache diff --git a/website/content/en/highlights/2023-12-19-0-35-0-upgrade-guide.md b/website/content/en/highlights/2023-12-19-0-35-0-upgrade-guide.md index de9408043cac2..f4c5081b7f9e8 100644 --- a/website/content/en/highlights/2023-12-19-0-35-0-upgrade-guide.md +++ b/website/content/en/highlights/2023-12-19-0-35-0-upgrade-guide.md @@ -12,9 +12,9 @@ badges: Vector's 0.35.0 release includes **breaking changes**: 1. [The Throttle transform's `events_discarded_total` internal metric is now opt-in](#events-discarded-total-opt-in) -1. [The `file` internal metric tag is now opt-in for file-based components](#file-tag-opt-in) -1. [Datadog sinks now default the API key and Site to the value of environment variables](#datadog-env-vars) -1. [Removal of `requests_completed_total`, `request_duration_seconds`, and `requests_received_total` Internal Metrics](#remove-obsolete-http-metrics) +2. [The `file` internal metric tag is now opt-in for file-based components](#file-tag-opt-in) +3. [Datadog sinks now default the API key and Site to the value of environment variables](#datadog-env-vars) +4. [Removal of `requests_completed_total`, `request_duration_seconds`, and `requests_received_total` Internal Metrics](#remove-obsolete-http-metrics) and **deprecations**: @@ -23,9 +23,9 @@ and **deprecations**: and **potentially impactful changes**: 1. [HTTP server-based sources now include a `keepalive.max_connection_age_secs` config option that defaults to 5 minutes](#http-keepalive-max-connection-age) -1. [Update `component_sent_bytes_total` to correctly report uncompressed bytes for all sinks](#component-sent-bytes-total) -1. [Update `component_received_bytes_total` to correctly report decompressed bytes for all sources](#component-received-bytes-total) -1. [Update default values for the `request.retry_max_duration_secs` and `request.rate_limit_num` sink configuration options](#request-config-options) +2. [Update `component_sent_bytes_total` to correctly report uncompressed bytes for all sinks](#component-sent-bytes-total) +3. [Update `component_received_bytes_total` to correctly report decompressed bytes for all sources](#component-received-bytes-total) +4. [Update default values for the `request.retry_max_duration_secs` and `request.rate_limit_num` sink configuration options](#request-config-options) We cover them below to help you upgrade quickly: diff --git a/website/content/en/highlights/2024-05-07-0-38-0-upgrade-guide.md b/website/content/en/highlights/2024-05-07-0-38-0-upgrade-guide.md index 90a7239d71ebf..379b7ad197ee2 100644 --- a/website/content/en/highlights/2024-05-07-0-38-0-upgrade-guide.md +++ b/website/content/en/highlights/2024-05-07-0-38-0-upgrade-guide.md @@ -16,7 +16,7 @@ Vector's 0.38.0 release includes a **breaking change**: and **deprecations**: 1. [Deprecation of the path coalescing operator](#path-coalescing) -1. [Deprecation of `enterprise` configuration](#enterprise-configuration) +2. [Deprecation of `enterprise` configuration](#enterprise-configuration) We cover them below to help you upgrade quickly: diff --git a/website/content/en/highlights/2024-06-17-0-39-0-upgrade-guide.md b/website/content/en/highlights/2024-06-17-0-39-0-upgrade-guide.md index 784d2602f2592..dae63b3e6da72 100644 --- a/website/content/en/highlights/2024-06-17-0-39-0-upgrade-guide.md +++ b/website/content/en/highlights/2024-06-17-0-39-0-upgrade-guide.md @@ -12,7 +12,7 @@ badges: Vector's 0.39.0 release includes two **breaking changes**: 1. [Removal of the path coalescing operator](#path-coalescing) -1. [Removal of `enterprise` configuration](#enterprise-configuration) +2. [Removal of `enterprise` configuration](#enterprise-configuration) We cover them below to help you upgrade quickly: diff --git a/website/content/en/highlights/2024-07-29-0-40-0-upgrade-guide.md b/website/content/en/highlights/2024-07-29-0-40-0-upgrade-guide.md index a2f604eeaf66f..5813dfc248a40 100644 --- a/website/content/en/highlights/2024-07-29-0-40-0-upgrade-guide.md +++ b/website/content/en/highlights/2024-07-29-0-40-0-upgrade-guide.md @@ -12,8 +12,8 @@ badges: Vector's 0.40.0 release includes three **breaking changes**: 1. [GELF codec defaults to null-delimited messages for stream-based sources](#gelf) -1. [Reduce transforms now properly aggregate nested fields](#reduce-aggregate) -1. [Vector no longer supports CentOS 7](#centos7) +2. [Reduce transforms now properly aggregate nested fields](#reduce-aggregate) +3. [Vector no longer supports CentOS 7](#centos7) We cover them below to help you upgrade quickly: From 7dc3ce3f34de6b79c0f09b4e44fe0e9d87cc09c2 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Wed, 25 Mar 2026 18:32:34 -0400 Subject: [PATCH 055/326] chore(dev): add links to design issues (#25046) * chore: add TODO comments linking to log_namespace design issues https://github.com/vectordotdev/vector/issues/25044 https://github.com/vectordotdev/vector/issues/25045 Co-Authored-By: Claude Sonnet 4.6 (1M context) * docs fix --------- Co-authored-by: Claude Sonnet 4.6 (1M context) --- lib/codecs/src/decoding/format/mod.rs | 2 ++ lib/codecs/src/decoding/format/otlp.rs | 1 + 2 files changed, 3 insertions(+) diff --git a/lib/codecs/src/decoding/format/mod.rs b/lib/codecs/src/decoding/format/mod.rs index 701d11c8403d5..cbb2172c3392d 100644 --- a/lib/codecs/src/decoding/format/mod.rs +++ b/lib/codecs/src/decoding/format/mod.rs @@ -50,6 +50,8 @@ pub trait Deserializer: DynClone + Send + Sync { /// by not requiring heap allocations for it. /// /// **Note**: The type of the produced events depends on the implementation. + /// + /// TODO: fn parse( &self, bytes: Bytes, diff --git a/lib/codecs/src/decoding/format/otlp.rs b/lib/codecs/src/decoding/format/otlp.rs index 2004df92269ca..29b3477e4398f 100644 --- a/lib/codecs/src/decoding/format/otlp.rs +++ b/lib/codecs/src/decoding/format/otlp.rs @@ -181,6 +181,7 @@ impl Deserializer for OtlpDeserializer { } } OtlpSignalType::Traces => { + // TODO: if let Ok(mut events) = self.traces_deserializer.parse(bytes.clone(), log_namespace) && let Some(Event::Log(log)) = events.first() From 4fba57e9c4d6561c126851df8c4065221b44e992 Mon Sep 17 00:00:00 2001 From: Oleksandr Zanichkovskyi Date: Wed, 25 Mar 2026 23:51:02 +0100 Subject: [PATCH 056/326] feat(opentelemetry source): allow to add headers to metadata for OpenTelemetry metrics and traces (#24942) * feat: allow to add headers to metadata for OpenTelemetry metrics and traces * chore: code formatting * chore: refactored the implementation and tests * chore: added changelog * chore: fixed formatting * chore: updated docs * make generate-docs * feat: pass log_namespace when enriching metrics with header for use_otlp_decoding.metrics = true * chore: removed log_namespace comment related to metrics and traces * chore: fixed review comments * update comment and regen docs * chore: revert log_namespace threading in build_warp_trace_filter The semantics of log_namespace for traces have not been defined yet. The correct fix will be addressed in a dedicated follow-up branch. Co-Authored-By: Claude Sonnet 4.6 (1M context) * chore: revert log_namespace threading in build_warp_trace_filter The semantics of log_namespace for traces have not been defined yet. The correct fix will be addressed in a dedicated follow-up branch. Co-Authored-By: Claude Sonnet 4.6 (1M context) --------- Co-authored-by: Pavlos Rontidis Co-authored-by: Claude Sonnet 4.6 (1M context) --- ...telemetry-header-enrichment.enhancement.md | 7 + src/sources/opentelemetry/config.rs | 7 +- src/sources/opentelemetry/http.rs | 25 +- src/sources/opentelemetry/tests.rs | 288 +++++++++++++++++- src/sources/util/http/headers.rs | 176 +++++++++-- .../sources/generated/opentelemetry.cue | 7 +- 6 files changed, 463 insertions(+), 47 deletions(-) create mode 100644 changelog.d/24619-opentelemetry-header-enrichment.enhancement.md diff --git a/changelog.d/24619-opentelemetry-header-enrichment.enhancement.md b/changelog.d/24619-opentelemetry-header-enrichment.enhancement.md new file mode 100644 index 0000000000000..e0b0b33a5caa3 --- /dev/null +++ b/changelog.d/24619-opentelemetry-header-enrichment.enhancement.md @@ -0,0 +1,7 @@ +`opentelemetry` source: Implemented header enrichment for OTLP metrics and traces. Unlike logs, which support enriching +the event itself or its metadata, depending on `log_namespace` settings, for metrics and traces this setting is ignored +and header values are added to the event metadata. + +Issue: https://github.com/vectordotdev/vector/issues/24619 + +authors: ozanichkovsky diff --git a/src/sources/opentelemetry/config.rs b/src/sources/opentelemetry/config.rs index c7eed8f301b50..89f6163da0595 100644 --- a/src/sources/opentelemetry/config.rs +++ b/src/sources/opentelemetry/config.rs @@ -202,13 +202,14 @@ pub struct HttpConfig { #[serde(default)] pub keepalive: KeepaliveConfig, - /// A list of HTTP headers to include in the log event. + /// A list of HTTP headers to include in the event. /// /// Accepts the wildcard (`*`) character for headers matching a specified pattern. /// - /// Specifying "*" results in all headers included in the log event. + /// Specifying "*" results in all headers included in the event. /// - /// These headers are not included in the JSON payload if a field with a conflicting name exists. + /// For log events in legacy namespace mode, headers are not included if a field with a conflicting name exists. + /// For metrics and traces, headers are always added to event metadata. #[serde(default)] #[configurable(metadata(docs::examples = "User-Agent"))] #[configurable(metadata(docs::examples = "X-My-Custom-Header"))] diff --git a/src/sources/opentelemetry/http.rs b/src/sources/opentelemetry/http.rs index f15e44bbd0926..42be681f1cb90 100644 --- a/src/sources/opentelemetry/http.rs +++ b/src/sources/opentelemetry/http.rs @@ -109,9 +109,11 @@ pub(crate) fn build_warp_filter( ); let metrics_filters = build_warp_metrics_filter( acknowledgements, + log_namespace, out.clone(), bytes_received.clone(), events_received.clone(), + headers.clone(), metrics_deserializer, ); let trace_filters = build_warp_trace_filter( @@ -119,6 +121,7 @@ pub(crate) fn build_warp_filter( out.clone(), bytes_received, events_received, + headers.clone(), traces_deserializer, ); log_filters @@ -257,12 +260,14 @@ fn build_warp_log_filter( } fn build_warp_metrics_filter( acknowledgements: bool, + log_namespace: LogNamespace, source_sender: SourceSender, bytes_received: Registered, events_received: Registered, + headers_cfg: Vec, deserializer: Option, ) -> BoxedFilter<(Response,)> { - let make_events = move |encoding_header: Option, _headers: HeaderMap, body: Bytes| { + let make_events = move |encoding_header: Option, headers: HeaderMap, body: Bytes| { decompress_body(encoding_header.as_deref(), body) .inspect_err(|err| { // Other status codes are already handled by `sources::util::decompress_body` (tech debt). @@ -276,15 +281,14 @@ fn build_warp_metrics_filter( .and_then(|decoded_body| { bytes_received.emit(ByteSize(decoded_body.len())); if let Some(d) = deserializer.as_ref() { - parse_with_deserializer( - d, - decoded_body, - LogNamespace::default(), - &events_received, - ) + parse_with_deserializer(d, decoded_body, log_namespace, &events_received) } else { decode_metrics_body(decoded_body, &events_received) } + .map(|mut events| { + enrich_events(&mut events, &headers_cfg, &headers, log_namespace); + events + }) }) }; @@ -301,9 +305,10 @@ fn build_warp_trace_filter( source_sender: SourceSender, bytes_received: Registered, events_received: Registered, + headers_cfg: Vec, deserializer: Option, ) -> BoxedFilter<(Response,)> { - let make_events = move |encoding_header: Option, _headers: HeaderMap, body: Bytes| { + let make_events = move |encoding_header: Option, headers: HeaderMap, body: Bytes| { decompress_body(encoding_header.as_deref(), body) .inspect_err(|err| { // Other status codes are already handled by `sources::util::decompress_body` (tech debt). @@ -326,6 +331,10 @@ fn build_warp_trace_filter( } else { decode_trace_body(decoded_body, &events_received) } + .map(|mut events| { + enrich_events(&mut events, &headers_cfg, &headers, LogNamespace::default()); + events + }) }) }; diff --git a/src/sources/opentelemetry/tests.rs b/src/sources/opentelemetry/tests.rs index de5781756dba1..9fb0bac71a1cd 100644 --- a/src/sources/opentelemetry/tests.rs +++ b/src/sources/opentelemetry/tests.rs @@ -4,12 +4,31 @@ use std::{ time::{SystemTime, UNIX_EPOCH}, }; +use crate::{ + SourceSender, + config::{OutputId, SourceConfig, SourceContext}, + event::{ + Event, EventStatus, LogEvent, Metric as MetricEvent, MetricKind, MetricTags, MetricValue, + ObjectMap, Value, into_event_stream, + metric::{Bucket, Quantile}, + }, + sources::opentelemetry::config::{ + GrpcConfig, HttpConfig, LOGS, METRICS, OpentelemetryConfig, TRACES, + }, + test_util::{ + self, + addr::next_addr, + components::{SOURCE_TAGS, assert_source_compliance}, + }, +}; use chrono::{DateTime, TimeZone, Utc}; use futures::Stream; use futures_util::StreamExt; use prost::Message; use similar_asserts::assert_eq; use tonic::Request; +use vector_lib::opentelemetry::proto::collector::trace::v1::ExportTraceServiceRequest; +use vector_lib::opentelemetry::proto::trace::v1::{ResourceSpans, ScopeSpans, Span}; use vector_lib::{ config::LogNamespace, lookup::path, @@ -33,22 +52,6 @@ use vector_lib::{ }; use vrl::value; -use crate::{ - SourceSender, - config::{OutputId, SourceConfig, SourceContext}, - event::{ - Event, EventStatus, LogEvent, Metric as MetricEvent, MetricKind, MetricTags, MetricValue, - ObjectMap, Value, into_event_stream, - metric::{Bucket, Quantile}, - }, - sources::opentelemetry::config::{GrpcConfig, HttpConfig, LOGS, METRICS, OpentelemetryConfig}, - test_util::{ - self, - addr::next_addr, - components::{SOURCE_TAGS, assert_source_compliance}, - }, -}; - fn create_test_logs_request() -> Request { Request::new(ExportLogsServiceRequest { resource_logs: vec![ResourceLogs { @@ -100,6 +103,104 @@ fn create_test_logs_request() -> Request { }) } +fn create_test_metrics_request() -> ExportMetricsServiceRequest { + ExportMetricsServiceRequest { + resource_metrics: vec![ResourceMetrics { + resource: Some(Resource { + attributes: vec![KeyValue { + key: "service.name".to_string(), + value: Some(AnyValue { + value: Some(StringValue("vector-collector".to_string())), + }), + }], + dropped_attributes_count: 0, + }), + schema_url: "".to_string(), + scope_metrics: vec![ScopeMetrics { + scope: Some(InstrumentationScope { + name: "vector-collector-instrumentation".to_string(), + version: "0.111.0".to_string(), + attributes: vec![], + dropped_attributes_count: 0, + }), + schema_url: "".to_string(), + metrics: vec![Metric { + name: "some.random.metric".to_string(), + description: "Some random metric we use for test".to_string(), + unit: "1".to_string(), + data: Some(Data::Summary(Summary { + data_points: vec![SummaryDataPoint { + attributes: vec![ + KeyValue { + key: "host".to_string(), + value: Some(AnyValue { + value: Some(StringValue("localhost".to_string())), + }), + }, + KeyValue { + key: "service".to_string(), + value: Some(AnyValue { + value: Some(StringValue("vector-collector".to_string())), + }), + }, + ], + start_time_unix_nano: 0, + time_unix_nano: 0, + count: 5, + sum: 122.5, + quantile_values: vec![ + ValueAtQuantile { + quantile: 0.5, + value: 24.5, + }, + ValueAtQuantile { + quantile: 0.9, + value: 45.0, + }, + ValueAtQuantile { + quantile: 1.0, + value: 60.0, + }, + ], + flags: 0, + }], + })), + }], + }], + }], + } +} + +fn create_test_traces_request() -> ExportTraceServiceRequest { + ExportTraceServiceRequest { + resource_spans: vec![ResourceSpans { + resource: None, + scope_spans: vec![ScopeSpans { + scope: None, + spans: vec![Span { + trace_id: (1..17).collect::>(), + span_id: (1..9).collect::>(), + parent_span_id: (1..9).collect::>(), + name: "span".to_string(), + kind: 1, + start_time_unix_nano: 1713525203000000000, + end_time_unix_nano: 1713525205000000000, + attributes: vec![], + dropped_attributes_count: 0, + events: vec![], + dropped_events_count: 0, + links: vec![], + dropped_links_count: 0, + status: None, + trace_state: "".to_string(), + }], + schema_url: "".to_string(), + }], + schema_url: "".to_string(), + }], + } +} + #[test] fn generate_config() { test_util::test_generate_config::(); @@ -1091,6 +1192,39 @@ fn get_source_config_with_headers( } } +async fn send_and_collect_otel_event( + use_otlp_decoding: bool, + output_port: &str, + endpoint: &str, + body: Vec, +) -> Event { + let (_guard_0, grpc_addr) = next_addr(); + let (_guard_1, http_addr) = next_addr(); + + let source = get_source_config_with_headers(grpc_addr, http_addr, use_otlp_decoding); + + let (sender, output, _) = new_source(EventStatus::Delivered, output_port.to_string()); + let server = source + .build(SourceContext::new_test(sender, None)) + .await + .unwrap(); + tokio::spawn(server); + test_util::wait_for_tcp(http_addr).await; + + let _res = reqwest::Client::new() + .post(format!("http://{http_addr}/{endpoint}")) + .header("Content-Type", "application/x-protobuf") + .header("User-Agent", "Test") + .body(body) + .send() + .await + .expect("Failed to send request to OpenTelemetry source."); + + let mut events = test_util::collect_ready(output).await; + assert_eq!(events.len(), 1); + events.pop().unwrap() +} + #[tokio::test] async fn http_headers_logs_use_otlp_decoding_false() { assert_source_compliance(&SOURCE_TAGS, async { @@ -1237,6 +1371,128 @@ async fn http_headers_logs_use_otlp_decoding_true() { .await; } +#[tokio::test] +async fn http_headers_metrics_use_otlp_decoding_false() { + assert_source_compliance(&SOURCE_TAGS, async { + let event = send_and_collect_otel_event( + false, + METRICS, + "v1/metrics", + create_test_metrics_request().encode_to_vec(), + ) + .await; + let metric = event.as_metric(); + assert_eq!( + metric + .metadata() + .value() + .get(path!("opentelemetry", "headers")) + .unwrap() + .get("AbsentHeader") + .unwrap(), + &Value::Null + ); + assert_eq!( + metric + .metadata() + .value() + .get(path!("opentelemetry", "headers")) + .unwrap() + .get("User-Agent") + .unwrap(), + &value!("Test") + ); + }) + .await; +} + +#[tokio::test] +async fn http_headers_metrics_use_otlp_decoding_true() { + assert_source_compliance(&SOURCE_TAGS, async { + let event = send_and_collect_otel_event( + true, + METRICS, + "v1/metrics", + create_test_metrics_request().encode_to_vec(), + ) + .await; + let log = event.as_log(); + assert_eq!(log["AbsentHeader"], Value::Null); + assert_eq!(log["User-Agent"], "Test".into()); + }) + .await; +} + +#[tokio::test] +async fn http_headers_traces_use_otlp_decoding_false() { + assert_source_compliance(&SOURCE_TAGS, async { + let event = send_and_collect_otel_event( + false, + TRACES, + "v1/traces", + create_test_traces_request().encode_to_vec(), + ) + .await; + let trace = event.as_trace(); + assert_eq!( + trace + .metadata() + .value() + .get(path!("opentelemetry", "headers")) + .unwrap() + .get("AbsentHeader") + .unwrap(), + &Value::Null + ); + assert_eq!( + trace + .metadata() + .value() + .get(path!("opentelemetry", "headers")) + .unwrap() + .get("User-Agent") + .unwrap(), + &value!("Test") + ); + }) + .await; +} + +#[tokio::test] +async fn http_headers_traces_use_otlp_decoding_true() { + assert_source_compliance(&SOURCE_TAGS, async { + let event = send_and_collect_otel_event( + true, + TRACES, + "v1/traces", + create_test_traces_request().encode_to_vec(), + ) + .await; + let trace = event.as_trace(); + assert_eq!( + trace + .metadata() + .value() + .get(path!("opentelemetry", "headers")) + .unwrap() + .get("AbsentHeader") + .unwrap(), + &Value::Null + ); + assert_eq!( + trace + .metadata() + .value() + .get(path!("opentelemetry", "headers")) + .unwrap() + .get("User-Agent") + .unwrap(), + &value!("Test") + ); + }) + .await; +} + pub struct OTelTestEnv { pub grpc_addr: String, pub config: OpentelemetryConfig, diff --git a/src/sources/util/http/headers.rs b/src/sources/util/http/headers.rs index 8a4b73ff212df..ef8376f695723 100644 --- a/src/sources/util/http/headers.rs +++ b/src/sources/util/http/headers.rs @@ -24,14 +24,22 @@ pub fn add_headers( let value = headers.get(header_name).map(HeaderValue::as_bytes); for event in events.iter_mut() { - if let Event::Log(log) = event { - log_namespace.insert_source_metadata( - source_name, - log, - Some(LegacyKey::InsertIfEmpty(path!(header_name))), - path!("headers", header_name), - Value::from(value.map(Bytes::copy_from_slice)), - ); + match event { + Event::Log(log) => { + log_namespace.insert_source_metadata( + source_name, + log, + Some(LegacyKey::InsertIfEmpty(path!(header_name))), + path!("headers", header_name), + Value::from(value.map(Bytes::copy_from_slice)), + ); + } + Event::Metric(_) | Event::Trace(_) => { + event.metadata_mut().value_mut().insert( + path!(source_name, "headers", header_name), + Value::from(value.map(Bytes::copy_from_slice)), + ); + } } } } @@ -45,14 +53,22 @@ pub fn add_headers( let value = headers.get(header_name).map(HeaderValue::as_bytes); for event in events.iter_mut() { - if let Event::Log(log) = event { - log_namespace.insert_source_metadata( - source_name, - log, - Some(LegacyKey::InsertIfEmpty(path!(header_name.as_str()))), - path!("headers", header_name.as_str()), - Value::from(value.map(Bytes::copy_from_slice)), - ); + match event { + Event::Log(log) => { + log_namespace.insert_source_metadata( + source_name, + log, + Some(LegacyKey::InsertIfEmpty(path!(header_name.as_str()))), + path!("headers", header_name.as_str()), + Value::from(value.map(Bytes::copy_from_slice)), + ); + } + Event::Metric(_) | Event::Trace(_) => { + event.metadata_mut().value_mut().insert( + path!(source_name, "headers", header_name.as_str()), + Value::from(value.map(Bytes::copy_from_slice)), + ); + } } } } @@ -64,12 +80,15 @@ pub fn add_headers( #[cfg(test)] mod tests { + use chrono::{DateTime, Utc}; + use std::time::SystemTime; use vector_lib::config::LogNamespace; use vrl::{path, value}; use warp::http::HeaderMap; + use crate::event::{Event, MetricKind, MetricTags, MetricValue}; use crate::{ - event::LogEvent, + event::{LogEvent, Metric, TraceEvent}, sources::{http_server::HttpConfigParamKind, util::add_headers}, }; @@ -109,6 +128,53 @@ mod tests { .get(path!("test", "headers")) .unwrap() ); + + let mut metric = [Event::from( + Metric::new( + "some.random.metric", + MetricKind::Incremental, + MetricValue::Counter { value: 123.4 }, + ) + .with_timestamp(Some(DateTime::::from(SystemTime::now()))) + .with_tags(Some(MetricTags::default())), + )]; + + add_headers( + &mut metric, + &header_names, + &headers, + LogNamespace::default(), + "test", + ); + + let mut trace = [TraceEvent::from(btreemap! { + "span_id" => "abc123", + "trace_id" => "xyz789", + "span_name" => "test-span", + "service" => "my-service", + }) + .into()]; + + add_headers( + &mut trace, + &header_names, + &headers, + LogNamespace::default(), + "test", + ); + + assert_eq!( + metric[0] + .metadata() + .value() + .get(path!("test", "headers")) + .unwrap(), + trace[0] + .metadata() + .value() + .get(path!("test", "headers")) + .unwrap() + ); } #[test] @@ -162,5 +228,81 @@ mod tests { "gzip".into(), "Checking log contains Content-Encoding header" ); + + let mut metric = [Event::from( + Metric::new( + "some.random.metric", + MetricKind::Incremental, + MetricValue::Counter { value: 123.4 }, + ) + .with_timestamp(Some(DateTime::::from(SystemTime::now()))) + .with_tags(Some(MetricTags::default())), + )]; + + add_headers( + &mut metric, + &header_names, + &headers, + LogNamespace::default(), + "test", + ); + + let metric_headers = metric[0] + .metadata() + .value() + .get(path!("test", "headers")) + .unwrap(); + + assert_eq!( + metric_headers.get("content-type").unwrap(), + &value!("application/x-protobuf"), + "Checking metric contains Content-Type header" + ); + assert!( + !metric_headers.contains("user-agent"), + "Checking metric does not contain User-Agent header" + ); + assert_eq!( + metric_headers.get("content-encoding").unwrap(), + &value!("gzip"), + "Checking metric contains Content-Encoding header" + ); + + let mut trace = [TraceEvent::from(btreemap! { + "span_id" => "abc123", + "trace_id" => "xyz789", + "span_name" => "test-span", + "service" => "my-service", + }) + .into()]; + + add_headers( + &mut trace, + &header_names, + &headers, + LogNamespace::default(), + "test", + ); + + let trace_headers = trace[0] + .metadata() + .value() + .get(path!("test", "headers")) + .unwrap(); + + assert_eq!( + trace_headers.get("content-type").unwrap(), + &value!("application/x-protobuf"), + "Checking trace contains Content-Type header" + ); + assert!( + !trace_headers.contains("user-agent"), + "Checking trace does not contain User-Agent header" + ); + assert_eq!( + trace_headers.get("content-encoding").unwrap(), + &value!("gzip"), + "Checking trace contains Content-Encoding header" + ); } } diff --git a/website/cue/reference/components/sources/generated/opentelemetry.cue b/website/cue/reference/components/sources/generated/opentelemetry.cue index d3b498506a323..ca4150550c771 100644 --- a/website/cue/reference/components/sources/generated/opentelemetry.cue +++ b/website/cue/reference/components/sources/generated/opentelemetry.cue @@ -170,13 +170,14 @@ generated: components: sources: opentelemetry: configuration: { } headers: { description: """ - A list of HTTP headers to include in the log event. + A list of HTTP headers to include in the event. Accepts the wildcard (`*`) character for headers matching a specified pattern. - Specifying "*" results in all headers included in the log event. + Specifying "*" results in all headers included in the event. - These headers are not included in the JSON payload if a field with a conflicting name exists. + For log events in legacy namespace mode, headers are not included if a field with a conflicting name exists. + For metrics and traces, headers are always added to event metadata. """ required: false type: array: { From 498d1daa38432a329d168a2eec97a212922e71bc Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Wed, 25 Mar 2026 20:51:47 -0400 Subject: [PATCH 057/326] chore(ci): improve CLA instructions (#25039) * chore(cla): improve bot message with copy-pastable sign instruction Add a clearer `custom-notsigned-prcomment` that includes the sign phrase in a code block (one-click copy in GitHub UI) and a note for contributors whose commit email is not linked to their GitHub account. Co-Authored-By: Claude Sonnet 4.6 * repurpose --------- Co-authored-by: Claude Sonnet 4.6 --- .github/workflows/cla.yml | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/.github/workflows/cla.yml b/.github/workflows/cla.yml index 2f48f1f1d1edf..536e364a402b9 100644 --- a/.github/workflows/cla.yml +++ b/.github/workflows/cla.yml @@ -14,9 +14,9 @@ jobs: CLAAssistant: runs-on: ubuntu-latest permissions: - pull-requests: write # Required to comment on PRs - id-token: write # Required to federate tokens with dd-octo-sts-action - actions: write # Required to create/update workflow runs + pull-requests: write # Required to comment on PRs + id-token: write # Required to federate tokens with dd-octo-sts-action + actions: write # Required to create/update workflow runs steps: - name: CLA already verified on PR if: github.event_name == 'merge_group' @@ -36,17 +36,16 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} PERSONAL_ACCESS_TOKEN: ${{ steps.octo-sts.outputs.token }} with: - path-to-signatures: 'cla.json' - path-to-document: 'https://gist.github.com/bits-bot/55bdc97a4fdad52d97feb4d6c3d1d618' # e.g. a CLA or a DCO document - branch: 'vector' + path-to-signatures: "cla.json" + path-to-document: "https://gist.github.com/bits-bot/55bdc97a4fdad52d97feb4d6c3d1d618" # e.g. a CLA or a DCO document + branch: "vector" remote-repository-name: cla-signatures remote-organization-name: DataDog allowlist: step-security-bot - # the followings are the optional inputs - If the optional inputs are not given, then default values will be taken - #create-file-commit-message: 'For example: Creating file for storing CLA Signatures' - #signed-commit-message: 'For example: $contributorName has signed the CLA in $owner/$repo#$pullRequestNo' - #custom-notsigned-prcomment: 'pull request comment with Introductory message to ask new contributors to sign' - #custom-pr-sign-comment: 'The signature to be committed in order to sign the CLA' - #custom-allsigned-prcomment: 'pull request comment when all contributors has signed, defaults to **CLA Assistant Lite bot** All Contributors have signed the CLA.' - #lock-pullrequest-aftermerge: false - if you don't want this bot to automatically lock the pull request after merging (default - true) + custom-notsigned-prcomment: | + Thank you for your contribution! Before we can merge this PR, please sign our [Contributor License Agreement](https://gist.github.com/bits-bot/55bdc97a4fdad52d97feb4d6c3d1d618). + + To sign, copy and post the phrase below as a new comment on this PR. + + > **Note:** If the bot says your username was not found, the email used in your git commit may not be linked to your GitHub account. Fix this at [github.com/settings/emails](https://github.com/settings/emails), then comment `recheck` to retry. From bc1e6bdde6b07f51efd4764c65d75c32d056499c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 26 Mar 2026 15:10:51 +0000 Subject: [PATCH 058/326] chore(deps): bump uuid from 1.18.1 to 1.22.0 (#25025) * chore(deps): bump uuid from 1.18.1 to 1.22.0 Bumps [uuid](https://github.com/uuid-rs/uuid) from 1.18.1 to 1.22.0. - [Release notes](https://github.com/uuid-rs/uuid/releases) - [Commits](https://github.com/uuid-rs/uuid/compare/v1.18.1...v1.22.0) --- updated-dependencies: - dependency-name: uuid dependency-version: 1.22.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Update licenses --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Thomas --- Cargo.lock | 49 +++++++++++++++++++++++++++++++------------- Cargo.toml | 2 +- LICENSE-3rdparty.csv | 1 + 3 files changed, 37 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c5efc9802ce97..4a6ddc9d8b9e6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -32,7 +32,7 @@ checksum = "ac1f845298e95f983ff1944b728ae08b8cebab80d684f0a832ed0fc74dfa27e2" dependencies = [ "cfg-if", "cipher", - "cpufeatures", + "cpufeatures 0.2.11", ] [[package]] @@ -2215,7 +2215,18 @@ checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818" dependencies = [ "cfg-if", "cipher", - "cpufeatures", + "cpufeatures 0.2.11", +] + +[[package]] +name = "chacha20" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.0", ] [[package]] @@ -2225,7 +2236,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10cd79432192d1c0f4e1a0fef9527696cc039165d729fb41b3f4f4f354c2dc35" dependencies = [ "aead", - "chacha20", + "chacha20 0.9.1", "cipher", "poly1305", "zeroize", @@ -2768,6 +2779,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crc" version = "3.3.0" @@ -3019,7 +3039,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.11", "curve25519-dalek-derive", "digest", "fiat-crypto", @@ -5797,7 +5817,7 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb26cec98cce3a3d96cbb7bced3c4b16e3d13f27ec56dbd62cbc8f39cfb9d653" dependencies = [ - "cpufeatures", + "cpufeatures 0.2.11", ] [[package]] @@ -7815,7 +7835,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8159bd90725d2df49889a078b54f4f79e87f1f8a8444194cdca81d38f5393abf" dependencies = [ - "cpufeatures", + "cpufeatures 0.2.11", "opaque-debug", "universal-hash", ] @@ -8592,6 +8612,7 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc266eb313df6c5c09c1c7b1fbe2510961e5bcd3add930c1e31f7ed9da0feff8" dependencies = [ + "chacha20 0.10.0", "getrandom 0.4.2", "rand_core 0.10.0", ] @@ -9954,7 +9975,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f5058ada175748e33390e40e872bd0fe59a19f265d0158daa551c5a88a76009c" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.11", "digest", ] @@ -9965,7 +9986,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.11", "digest", ] @@ -9976,7 +9997,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.11", "digest", ] @@ -12092,14 +12113,14 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "uuid" -version = "1.18.1" +version = "1.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" +checksum = "a68d3c8f01c0cfa54a75291d83601161799e4a89a39e0929f4b0354d88757a37" dependencies = [ - "getrandom 0.3.4", + "getrandom 0.4.2", "js-sys", - "rand 0.9.2", - "serde", + "rand 0.10.0", + "serde_core", "wasm-bindgen", ] diff --git a/Cargo.toml b/Cargo.toml index 06b83efedb33e..5dc1dda8072da 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -204,7 +204,7 @@ tonic-reflection = { version = "0.11", default-features = false, features = ["se tracing = { version = "0.1.44", default-features = false } tracing-subscriber = { version = "0.3.22", default-features = false, features = ["fmt"] } url = { version = "2.5.4", default-features = false, features = ["serde"] } -uuid = { version = "1.18.1", features = ["v4", "v7", "serde", "fast-rng"] } +uuid = { version = "1.22.0", features = ["v4", "v7", "serde", "fast-rng"] } vector-config = { path = "lib/vector-config" } vector-config-common = { path = "lib/vector-config-common" } vector-config-macros = { path = "lib/vector-config-macros" } diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 8a5759685500e..8dbc10abc4f7e 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -144,6 +144,7 @@ cesu8,https://github.com/emk/cesu8-rs,Apache-2.0 OR MIT,Eric Kidd chacha20,https://github.com/RustCrypto/stream-ciphers,Apache-2.0 OR MIT,RustCrypto Developers +chacha20,https://github.com/RustCrypto/stream-ciphers,MIT OR Apache-2.0,RustCrypto Developers chacha20poly1305,https://github.com/RustCrypto/AEADs/tree/master/chacha20poly1305,Apache-2.0 OR MIT,RustCrypto Developers charset,https://github.com/hsivonen/charset,MIT OR Apache-2.0,Henri Sivonen chrono,https://github.com/chronotope/chrono,MIT OR Apache-2.0,The chrono Authors From 072f6284c83279cd920177b32114f10f8fd5164c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 26 Mar 2026 15:15:29 +0000 Subject: [PATCH 059/326] chore(deps): bump snafu from 0.8.9 to 0.9.0 (#25029) Bumps [snafu](https://github.com/shepmaster/snafu) from 0.8.9 to 0.9.0. - [Changelog](https://github.com/shepmaster/snafu/blob/main/CHANGELOG.md) - [Commits](https://github.com/shepmaster/snafu/compare/0.8.9...0.9.0) --- updated-dependencies: - dependency-name: snafu dependency-version: 0.9.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Thomas --- Cargo.lock | 45 +++++++++++++++++++++++++++++++++------------ Cargo.toml | 2 +- 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4a6ddc9d8b9e6..05055530add86 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2455,7 +2455,7 @@ dependencies = [ "serde_with", "similar-asserts", "smallvec", - "snafu 0.8.9", + "snafu 0.9.0", "strum 0.28.0", "syslog_loose 0.23.0", "tokio", @@ -3435,7 +3435,7 @@ dependencies = [ "criterion", "data-encoding", "hickory-proto 0.25.2", - "snafu 0.8.9", + "snafu 0.9.0", ] [[package]] @@ -3452,7 +3452,7 @@ dependencies = [ "pastey", "prost 0.12.6", "prost-build 0.12.6", - "snafu 0.8.9", + "snafu 0.9.0", "tracing 0.1.44", "vector-common", "vector-config", @@ -3474,7 +3474,7 @@ dependencies = [ "anyhow", "serde", "serde_json", - "snafu 0.8.9", + "snafu 0.9.0", "tracing 0.1.44", "vector-config", "vector-config-common", @@ -3681,7 +3681,7 @@ dependencies = [ "const-str", "dyn-clone", "indoc", - "snafu 0.8.9", + "snafu 0.9.0", "vector-vrl-category", "vrl", ] @@ -8096,7 +8096,7 @@ dependencies = [ "prost 0.12.6", "prost-build 0.12.6", "prost-types 0.12.6", - "snafu 0.8.9", + "snafu 0.9.0", "vector-common", ] @@ -10210,10 +10210,19 @@ name = "snafu" version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e84b3f4eacbf3a1ce05eac6763b4d629d60cbc94d632e4092c54ade71f1e1a2" +dependencies = [ + "snafu-derive 0.8.9", +] + +[[package]] +name = "snafu" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1d4bced6a69f90b2056c03dcff2c4737f98d6fb9e0853493996e1d253ca29c6" dependencies = [ "futures-core", "pin-project", - "snafu-derive 0.8.9", + "snafu-derive 0.9.0", ] [[package]] @@ -10240,6 +10249,18 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "snafu-derive" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54254b8531cafa275c5e096f62d48c81435d1015405a91198ddb11e967301d40" +dependencies = [ + "heck 0.5.0", + "proc-macro2 1.0.106", + "quote 1.0.44", + "syn 2.0.117", +] + [[package]] name = "snap" version = "1.1.1" @@ -12342,7 +12363,7 @@ dependencies = [ "similar-asserts", "smallvec", "smpl_jwt", - "snafu 0.8.9", + "snafu 0.9.0", "snap", "socket2 0.5.10", "sqlx", @@ -12402,7 +12423,7 @@ dependencies = [ "prost 0.12.6", "prost-build 0.12.6", "prost-types 0.12.6", - "snafu 0.8.9", + "snafu 0.9.0", "tokio", "tokio-stream", "tonic 0.11.0", @@ -12441,7 +12462,7 @@ dependencies = [ "rkyv", "serde", "serde_yaml", - "snafu 0.8.9", + "snafu 0.9.0", "temp-dir", "tokio", "tokio-test", @@ -12504,7 +12525,7 @@ dependencies = [ "serde", "serde_json", "serde_with", - "snafu 0.8.9", + "snafu 0.9.0", "toml", "tracing 0.1.44", "url", @@ -12598,7 +12619,7 @@ dependencies = [ "serde_yaml", "similar-asserts", "smallvec", - "snafu 0.8.9", + "snafu 0.9.0", "socket2 0.5.10", "tokio", "tokio-openssl", diff --git a/Cargo.toml b/Cargo.toml index 5dc1dda8072da..42e2b2c24a12f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -189,7 +189,7 @@ semver = { version = "1.0.27", default-features = false, features = ["serde", "s serde = { version = "1.0.219", default-features = false, features = ["alloc", "derive", "rc"] } serde_json = { version = "1.0.143", default-features = false, features = ["preserve_order", "raw_value", "std"] } serde_yaml = { version = "0.9.34", default-features = false } -snafu = { version = "0.8.9", default-features = false, features = ["futures", "std"] } +snafu = { version = "0.9.0", default-features = false, features = ["futures", "std"] } socket2 = { version = "0.5.10", default-features = false } tempfile = "3.23.0" tokio = { version = "1.49.0", default-features = false } From 041769359ba0a9d68017e10e6e84cb7a83b3df27 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 26 Mar 2026 15:22:32 +0000 Subject: [PATCH 060/326] chore(website deps): bump picomatch from 2.3.1 to 2.3.2 in /website (#25047) Bumps [picomatch](https://github.com/micromatch/picomatch) from 2.3.1 to 2.3.2. - [Release notes](https://github.com/micromatch/picomatch/releases) - [Changelog](https://github.com/micromatch/picomatch/blob/master/CHANGELOG.md) - [Commits](https://github.com/micromatch/picomatch/compare/2.3.1...2.3.2) --- updated-dependencies: - dependency-name: picomatch dependency-version: 2.3.2 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- website/yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/website/yarn.lock b/website/yarn.lock index 57cd6d50741c0..95702b3c49289 100644 --- a/website/yarn.lock +++ b/website/yarn.lock @@ -3055,9 +3055,9 @@ picocolors@^1.0.0, picocolors@^1.1.1: integrity sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA== picomatch@^2.0.4, picomatch@^2.2.1, picomatch@^2.3.1: - version "2.3.1" - resolved "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz" - integrity sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA== + version "2.3.2" + resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.2.tgz#5a942915e26b372dc0f0e6753149a16e6b1c5601" + integrity sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA== pify@^2.3.0: version "2.3.0" From d0c6cea89661577079d3eec790e4c054772de7d6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 26 Mar 2026 15:25:49 +0000 Subject: [PATCH 061/326] chore(website deps): bump yaml from 1.10.2 to 1.10.3 in /website (#25043) Bumps [yaml](https://github.com/eemeli/yaml) from 1.10.2 to 1.10.3. - [Release notes](https://github.com/eemeli/yaml/releases) - [Commits](https://github.com/eemeli/yaml/compare/v1.10.2...v1.10.3) --- updated-dependencies: - dependency-name: yaml dependency-version: 1.10.3 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- website/yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/website/yarn.lock b/website/yarn.lock index 95702b3c49289..f7cedc6f6748f 100644 --- a/website/yarn.lock +++ b/website/yarn.lock @@ -3876,9 +3876,9 @@ yallist@^3.0.2: integrity sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g== yaml@^1.10.0, yaml@^1.10.2: - version "1.10.2" - resolved "https://registry.npmjs.org/yaml/-/yaml-1.10.2.tgz" - integrity sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg== + version "1.10.3" + resolved "https://registry.yarnpkg.com/yaml/-/yaml-1.10.3.tgz#76e407ed95c42684fb8e14641e5de62fe65bbcb3" + integrity sha512-vIYeF1u3CjlhAFekPPAk2h/Kv4T3mAkMox5OymRiJQB0spDP10LHvt+K7G9Ny6NuuMAb25/6n1qyUjAcGNf/AA== yargs-parser@^20.2.2: version "20.2.9" From 5fcf1516151f659b5832fc433a4e10a219e31c7b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 26 Mar 2026 15:26:25 +0000 Subject: [PATCH 062/326] chore(deps): bump ipnet from 2.11.0 to 2.12.0 (#25030) Bumps [ipnet](https://github.com/krisprice/ipnet) from 2.11.0 to 2.12.0. - [Release notes](https://github.com/krisprice/ipnet/releases) - [Changelog](https://github.com/krisprice/ipnet/blob/master/RELEASES.md) - [Commits](https://github.com/krisprice/ipnet/compare/2.11.0...2.12.0) --- updated-dependencies: - dependency-name: ipnet dependency-version: 2.12.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Thomas --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 05055530add86..76f557baf7a8a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5542,9 +5542,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.11.0" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" dependencies = [ "serde", ] From 09629f02d60614a38b20b49cb10acbce47a9f6cc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 26 Mar 2026 15:49:33 +0000 Subject: [PATCH 063/326] chore(ci): bump docker/setup-buildx-action from 3.12.0 to 4.0.0 (#25003) Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from 3.12.0 to 4.0.0. - [Release notes](https://github.com/docker/setup-buildx-action/releases) - [Commits](https://github.com/docker/setup-buildx-action/compare/8d2750c68a42422c14e847fe6c8ac0403b4cbd6f...4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd) --- updated-dependencies: - dependency-name: docker/setup-buildx-action dependency-version: 4.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/environment.yml | 2 +- .github/workflows/publish.yml | 2 +- .github/workflows/regression.yml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/environment.yml b/.github/workflows/environment.yml index fce6b25c3c181..75680427712a5 100644 --- a/.github/workflows/environment.yml +++ b/.github/workflows/environment.yml @@ -43,7 +43,7 @@ jobs: - name: Set up QEMU uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3.7.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Login to DockerHub uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 if: env.SHOULD_PUBLISH == 'true' diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 3c533c593aeb6..9b5385a38b39f 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -347,7 +347,7 @@ jobs: platforms: all - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 with: version: latest - name: Download all Linux package artifacts diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index a8c6981e53a9d..75a84c0fc1afe 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -209,7 +209,7 @@ jobs: - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Build 'vector' target image uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 @@ -252,7 +252,7 @@ jobs: - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Build 'vector' target image uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 From 3255b66b2e8a2fb7ec3412904b856a8349a236ab Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 26 Mar 2026 15:49:57 +0000 Subject: [PATCH 064/326] chore(ci): bump actions/cache from 5.0.3 to 5.0.4 (#25002) Bumps [actions/cache](https://github.com/actions/cache) from 5.0.3 to 5.0.4. - [Release notes](https://github.com/actions/cache/releases) - [Changelog](https://github.com/actions/cache/blob/main/RELEASES.md) - [Commits](https://github.com/actions/cache/compare/cdf6c1fa76f9f475f3d7449005a359c84ca0f306...668228422ae6a00e4ad889ee87cd7109ec5666a7) --- updated-dependencies: - dependency-name: actions/cache dependency-version: 5.0.4 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/cross.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cross.yml b/.github/workflows/cross.yml index 8cf8706b1a42c..9519a7d79df94 100644 --- a/.github/workflows/cross.yml +++ b/.github/workflows/cross.yml @@ -35,7 +35,7 @@ jobs: with: ref: ${{ inputs.ref }} - - uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 + - uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 name: Cache Cargo registry + index with: path: | From 9337c80bb9d2491a8d617df408c20a88385d719a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 26 Mar 2026 15:50:14 +0000 Subject: [PATCH 065/326] chore(ci): bump docker/login-action from 3.7.0 to 4.0.0 (#25001) Bumps [docker/login-action](https://github.com/docker/login-action) from 3.7.0 to 4.0.0. - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/c94ce9fb468520275223c153574b00df6fe4bcc9...b45d80f862d83dbcd57f89517bcf500b2ab88fb2) --- updated-dependencies: - dependency-name: docker/login-action dependency-version: 4.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/build-test-runner.yml | 2 +- .github/workflows/environment.yml | 2 +- .github/workflows/publish.yml | 4 ++-- .github/workflows/regression.yml | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-test-runner.yml b/.github/workflows/build-test-runner.yml index 50a46fd4bd30f..7a7596651ed2c 100644 --- a/.github/workflows/build-test-runner.yml +++ b/.github/workflows/build-test-runner.yml @@ -36,7 +36,7 @@ jobs: vdev: true - name: Login to GitHub Container Registry - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 with: registry: ghcr.io username: ${{ github.actor }} diff --git a/.github/workflows/environment.yml b/.github/workflows/environment.yml index 75680427712a5..cf6313e4ce894 100644 --- a/.github/workflows/environment.yml +++ b/.github/workflows/environment.yml @@ -45,7 +45,7 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Login to DockerHub - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 if: env.SHOULD_PUBLISH == 'true' with: username: ${{ secrets.CI_DOCKER_USERNAME }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 9b5385a38b39f..0594151b2db23 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -331,12 +331,12 @@ jobs: with: ref: ${{ inputs.git_ref }} - name: Login to DockerHub - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 with: username: ${{ secrets.CI_DOCKER_USERNAME }} password: ${{ secrets.CI_DOCKER_PASSWORD }} - name: Login to GitHub Container Registry - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 with: registry: ghcr.io username: ${{ github.actor }} diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 75a84c0fc1afe..f2216df39e684 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -326,7 +326,7 @@ jobs: uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1 - name: Docker Login to ECR - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 with: registry: ${{ steps.login-ecr.outputs.registry }} @@ -366,7 +366,7 @@ jobs: uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1 - name: Docker Login to ECR - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 with: registry: ${{ steps.login-ecr.outputs.registry }} From 84abc68df859d58b2a7681757e52aca7a8094ad2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 26 Mar 2026 20:24:22 +0000 Subject: [PATCH 066/326] chore(deps): bump bollard from 0.19.2 to 0.20.2 (#25026) * chore(deps): bump bollard from 0.19.2 to 0.20.2 Bumps [bollard](https://github.com/fussybeaver/bollard) from 0.19.2 to 0.20.2. - [Release notes](https://github.com/fussybeaver/bollard/releases) - [Changelog](https://github.com/fussybeaver/bollard/blob/master/RELEASE.md) - [Commits](https://github.com/fussybeaver/bollard/compare/v0.19.2...v0.20.2) --- updated-dependencies: - dependency-name: bollard dependency-version: 0.20.2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Bump bollard in Cargo.toml * Fix incompatible changes --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Thomas --- Cargo.lock | 11 ++++------- Cargo.toml | 2 +- src/docker.rs | 5 ++--- src/sources/docker_logs/tests.rs | 4 ++-- 4 files changed, 9 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 76f557baf7a8a..be47e7ba18697 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1882,9 +1882,9 @@ dependencies = [ [[package]] name = "bollard" -version = "0.19.2" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8796b390a5b4c86f9f2e8173a68c2791f4fa6b038b84e96dbc01c016d1e6722c" +checksum = "ee04c4c84f1f811b017f2fbb7dd8815c976e7ca98593de9c1e2afad0f636bff4" dependencies = [ "base64 0.22.1", "bollard-stubs", @@ -1905,12 +1905,10 @@ dependencies = [ "pin-project-lite", "rustls 0.23.23", "rustls-native-certs 0.8.1", - "rustls-pemfile 2.1.0", "rustls-pki-types", "serde", "serde_derive", "serde_json", - "serde_repr", "serde_urlencoded", "thiserror 2.0.17", "tokio", @@ -1922,15 +1920,14 @@ dependencies = [ [[package]] name = "bollard-stubs" -version = "1.49.0-rc.28.3.3" +version = "1.52.1-rc.29.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e7814991259013d5a5bee4ae28657dae0747d843cf06c40f7fc0c2894d6fa38" +checksum = "0f0a8ca8799131c1837d1282c3f81f31e76ceb0ce426e04a7fe1ccee3287c066" dependencies = [ "chrono", "serde", "serde_json", "serde_repr", - "serde_with", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 42e2b2c24a12f..a139a477d5ab2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -354,7 +354,7 @@ arrow-schema = { version = "56.2.0", default-features = false, optional = true } axum = { version = "0.6.20", default-features = false } base64 = { workspace = true, optional = true } bloomy = { version = "1.2.0", default-features = false, optional = true } -bollard = { version = "0.19.1", default-features = false, features = ["pipe", "ssl", "chrono"], optional = true } +bollard = { version = "0.20", default-features = false, features = ["pipe", "ssl", "chrono"], optional = true } bytes = { workspace = true, features = ["serde"] } bytesize = { version = "2.0.1", default-features = false } chrono.workspace = true diff --git a/src/docker.rs b/src/docker.rs index cd13a4c4c54ce..381ffddac0219 100644 --- a/src/docker.rs +++ b/src/docker.rs @@ -4,12 +4,11 @@ use std::{collections::HashMap, env, path::PathBuf}; use bollard::{ API_DEFAULT_VERSION, Docker, errors::Error as DockerError, - models::HostConfig, + models::{ContainerCreateBody, HostConfig}, query_parameters::{ CreateContainerOptionsBuilder, CreateImageOptionsBuilder, ListImagesOptionsBuilder, RemoveContainerOptions, StartContainerOptions, StopContainerOptions, }, - secret::ContainerCreateBody, }; use futures::StreamExt; use http::uri::Uri; @@ -130,7 +129,7 @@ async fn pull_image(docker: &Docker, image: &str, tag: &str) { .create_image(options, None, None) .for_each(|item| async move { let info = item.unwrap(); - if let Some(error) = info.error { + if let Some(error) = info.error_detail { panic!("{error:?}"); } }) diff --git a/src/sources/docker_logs/tests.rs b/src/sources/docker_logs/tests.rs index f5af697c1b9f1..4270f45843122 100644 --- a/src/sources/docker_logs/tests.rs +++ b/src/sources/docker_logs/tests.rs @@ -30,12 +30,12 @@ mod integration_tests { } use bollard::{ + models::ContainerCreateBody, query_parameters::{ CreateContainerOptionsBuilder, CreateImageOptionsBuilder, KillContainerOptions, ListImagesOptionsBuilder, RemoveContainerOptions, StartContainerOptions, WaitContainerOptions, }, - secret::ContainerCreateBody, }; use futures::{FutureExt, stream::TryStreamExt}; use itertools::Itertools as _; @@ -182,7 +182,7 @@ mod integration_tests { .create_image(options, None, None) .for_each(|item| async move { let info = item.unwrap(); - if let Some(error) = info.error { + if let Some(error) = info.error_detail { panic!("{error:?}"); } }) From 6267fcc78d8cd347f6405ab03ca100b97944a5ef Mon Sep 17 00:00:00 2001 From: bas smit <474727+fbs@users.noreply.github.com> Date: Thu, 26 Mar 2026 22:31:32 +0000 Subject: [PATCH 067/326] fix(opentelemetry source): log error (#24708) * fix(source opentelemetry): log error If the HTTP opentelemetry source fails to build it is currently silently ignored, without logging any info to the 'user'. The only indication you have that something is wrong is that the log line about the http listener starting is _not_ there. With this change the error condition is logged the same way the gRPC error is logged. Before: ``` $ vector -c vector-otel.yaml ... 2026-02-23T09:15:50.053251Z INFO vector: Vector has started. debug="false" version="0.52.0" arch="aarch64" revision="ca5bf26 2025-12-16 14:56:07.290167996" 2026-02-23T09:15:50.053468Z INFO source{component_kind="source" component_id=otel component_type=opentelemetry}: vector::sources::util::grpc: Building gRPC server. address=127.0.0.1:4317 ``` now: ``` $ target/debug/vector --config vector-otel.yaml 2026-02-23T09:16:49.397436Z INFO vector: Vector has started. debug="true" version="0.54.0" arch="aarch64" revision="" 2026-02-23T09:16:49.398497Z INFO source{component_kind="source" component_id=otel component_type=opentelemetry}: vector::sources::util::grpc: Building gRPC server. address=127.0.0.1:4317 2026-02-23T09:16:49.400243Z ERROR source{component_kind="source" component_id=otel component_type=opentelemetry}: vector::sources::opentelemetry::config: Source future failed. error=TCP bind failed: Address family not supported by protocol family (os error 47) ``` * improve error messages * add changelog * Fix CI errors Co-authored-by: Thomas * fix(opentelemetry source): improve error message wording Co-Authored-By: Claude Sonnet 4.6 (1M context) --------- Co-authored-by: Pavlos Rontidis Co-authored-by: Thomas Co-authored-by: Claude Sonnet 4.6 (1M context) --- changelog.d/24708_log_error.fix.md | 5 +++++ src/sources/opentelemetry/config.rs | 7 +++++-- 2 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 changelog.d/24708_log_error.fix.md diff --git a/changelog.d/24708_log_error.fix.md b/changelog.d/24708_log_error.fix.md new file mode 100644 index 0000000000000..d53626b09af05 --- /dev/null +++ b/changelog.d/24708_log_error.fix.md @@ -0,0 +1,5 @@ +The `opentelemetry` source now logs an error if it fails to start up or during runtime. +This can happen when the configuration is invalid, for example trying to bind to the wrong +IP or when hitting the open file limit. + +authors: fbs diff --git a/src/sources/opentelemetry/config.rs b/src/sources/opentelemetry/config.rs index 89f6163da0595..827bbd213360d 100644 --- a/src/sources/opentelemetry/config.rs +++ b/src/sources/opentelemetry/config.rs @@ -328,7 +328,7 @@ impl SourceConfig for OpentelemetryConfig { cx.shutdown.clone(), ) .map_err(|error| { - error!(message = "Source future failed.", %error); + error!(message = "OpenTelemetry source gRPC server failed.", %error); }); let http_tls_settings = MaybeTlsSettings::from_config(self.http.tls.as_ref(), true)?; @@ -355,7 +355,10 @@ impl SourceConfig for OpentelemetryConfig { filters, cx.shutdown, self.http.keepalive.clone(), - ); + ) + .map_err(|error| { + error!(message = "OpenTelemetry source HTTP server failed.", %error); + }); Ok(join(grpc_source, http_source).map(|_| Ok(())).boxed()) } From e8526b2a3ef7ac257d0fb816f611cddbc494a478 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 26 Mar 2026 22:32:43 +0000 Subject: [PATCH 068/326] chore(deps): bump deadpool from 0.12.2 to 0.13.0 (#25014) * chore(deps): bump deadpool from 0.12.2 to 0.13.0 Bumps [deadpool](https://github.com/deadpool-rs/deadpool) from 0.12.2 to 0.13.0. - [Changelog](https://github.com/deadpool-rs/deadpool/blob/main/release.toml) - [Commits](https://github.com/deadpool-rs/deadpool/compare/deadpool-v0.12.2...deadpool-v0.13.0) --- updated-dependencies: - dependency-name: deadpool dependency-version: 0.13.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Update Cargo.toml and licenses --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Thomas --- Cargo.lock | 23 ++++++++++++++++++++--- Cargo.toml | 2 +- LICENSE-3rdparty.csv | 4 ++-- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index be47e7ba18697..0a3f2830ec51d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3196,7 +3196,18 @@ version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ed5957ff93768adf7a65ab167a17835c3d2c3c50d084fe305174c112f468e2f" dependencies = [ - "deadpool-runtime", + "deadpool-runtime 0.1.3", + "num_cpus", + "tokio", +] + +[[package]] +name = "deadpool" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "883466cb8db62725aee5f4a6011e8a5d42912b42632df32aad57fc91127c6e04" +dependencies = [ + "deadpool-runtime 0.3.1", "num_cpus", "tokio", ] @@ -3206,6 +3217,12 @@ name = "deadpool-runtime" version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63dfa964fe2a66f3fde91fc70b267fe193d822c7e603e2a675a49a7f46ad3f49" + +[[package]] +name = "deadpool-runtime" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2657f61fb1dd8bf37a8d51093cc7cee4e77125b22f7753f49b289f831bec2bae" dependencies = [ "tokio", ] @@ -12257,7 +12274,7 @@ dependencies = [ "criterion", "csv", "databend-client", - "deadpool", + "deadpool 0.13.0", "derivative", "dirs-next", "dnsmsg-parser", @@ -13844,7 +13861,7 @@ checksum = "08db1edfb05d9b3c1542e521aea074442088292f00b5f28e435c714a98f85031" dependencies = [ "assert-json-diff", "base64 0.22.1", - "deadpool", + "deadpool 0.12.2", "futures", "http 1.3.1", "http-body-util", diff --git a/Cargo.toml b/Cargo.toml index a139a477d5ab2..7676ebe174ea3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -336,7 +336,7 @@ smpl_jwt = { version = "0.8.0", default-features = false, optional = true } # AMQP lapin = { version = "4.3.0", default-features = false, features = ["tokio", "native-tls"], optional = true } async-rs = { version = "0.8", default-features = false, features = ["tokio"], optional = true } -deadpool = { version = "0.12.2", default-features = false, features = ["managed", "rt_tokio_1"], optional = true } +deadpool = { version = "0.13.0", default-features = false, features = ["managed", "rt_tokio_1"], optional = true } # Opentelemetry diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 8dbc10abc4f7e..9a135f0479f51 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -216,8 +216,8 @@ data-encoding,https://github.com/ia0/data-encoding,MIT,Julien Cretin databend-client,https://github.com/databendlabs/bendsql,Apache-2.0,Databend Authors dbl,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers -deadpool,https://github.com/bikeshedder/deadpool,MIT OR Apache-2.0,Michael P. Jung -deadpool-runtime,https://github.com/bikeshedder/deadpool,MIT OR Apache-2.0,Michael P. Jung +deadpool,https://github.com/deadpool-rs/deadpool,MIT OR Apache-2.0,Michael P. Jung +deadpool-runtime,https://github.com/deadpool-rs/deadpool,MIT OR Apache-2.0,Michael P. Jung der,https://github.com/RustCrypto/formats/tree/master/der,Apache-2.0 OR MIT,RustCrypto Developers deranged,https://github.com/jhpratt/deranged,MIT OR Apache-2.0,Jacob Pratt derivative,https://github.com/mcarton/rust-derivative,MIT OR Apache-2.0,mcarton From af15a5a595f342038fd3368854fa75c74c31e932 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Thu, 26 Mar 2026 21:09:12 -0400 Subject: [PATCH 069/326] fix(dev): vanila `cargo build` should run on windows (#24991) * fix(deps): gate Unix-only dependencies to enable Windows MSVC builds Move `tikv-jemallocator` to a Unix-only target dependency (it requires autoconf which is unavailable with MSVC). Enable `rdkafka/cmake_build` on Windows via a target-specific dependency entry so rdkafka builds using cmake instead of the Unix autotools path. Gate `sources-dnstap` module declarations on Unix to match the Unix-only `framestream` crate it depends on. Add `#[cfg(unix)]` to jemalloc allocator blocks in `src/lib.rs`. Tested on Windows with Cyrus SASL installed via MSYS2. Co-Authored-By: Claude Sonnet 4.6 * chore: apply rustfmt Co-Authored-By: Claude Sonnet 4.6 * refactor: make rdkafka a workspace dependency Avoids version string duplication across dependency sections. Co-Authored-By: Claude Sonnet 4.6 --------- Co-authored-by: pront Co-authored-by: Claude Sonnet 4.6 --- Cargo.toml | 6 ++++-- src/internal_events/mod.rs | 4 ++-- src/lib.rs | 8 ++++++-- src/sources/mod.rs | 2 +- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 7676ebe174ea3..4bc52384f0a06 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -182,6 +182,7 @@ prost-types = { version = "0.12", default-features = false } quickcheck = { version = "1.1.0" } rand = { version = "0.9.2", default-features = false, features = ["small_rng", "thread_rng"] } rand_distr = { version = "0.5.1", default-features = false } +rdkafka = { version = "0.39.0", default-features = false } regex = { version = "1.12.3", default-features = false, features = ["std", "perf"] } reqwest = { version = "0.11", features = ["json"] } rust_decimal = { version = "1.37.0", default-features = false, features = ["std"] } @@ -411,7 +412,7 @@ pulsar = { version = "6.7.0", default-features = false, features = ["tokio-runti quick-junit = { version = "0.6.0" } rand.workspace = true rand_distr.workspace = true -rdkafka = { version = "0.39.0", default-features = false, features = ["curl-static", "tokio", "libz", "ssl", "zstd"], optional = true } +rdkafka = { workspace = true, features = ["curl-static", "tokio", "libz", "ssl", "zstd"], optional = true } redis = { version = "0.32.4", default-features = false, features = ["connection-manager", "sentinel", "tokio-comp", "tokio-native-tls-comp"], optional = true } regex.workspace = true roaring = { version = "0.11.2", default-features = false, features = ["std"], optional = true } @@ -425,7 +426,6 @@ sqlx = { version = "0.8.6", default-features = false, features = ["derive", "pos stream-cancel = { version = "0.8.2", default-features = false } strip-ansi-escapes = { version = "0.2.1", default-features = false } syslog = { version = "6.1.1", default-features = false, optional = true } -tikv-jemallocator = { version = "0.6.0", default-features = false, features = ["unprefixed_malloc_on_supported_platforms"], optional = true } tokio-postgres = { version = "0.7.13", default-features = false, features = ["runtime", "with-chrono-0_4"], optional = true } tokio-tungstenite = { workspace = true, features = ["connect"], optional = true } toml.workspace = true @@ -453,9 +453,11 @@ byteorder = "1.5.0" windows-service = "0.8.0" windows = { version = "0.58", features = ["Win32_System_EventLog", "Win32_Foundation", "Win32_System_Com", "Win32_Security", "Win32_Security_Authorization", "Win32_System_Threading", "Win32_Storage_FileSystem"], optional = true } quick-xml = { version = "0.31", default-features = false, features = ["serialize"], optional = true } +rdkafka = { workspace = true, features = ["cmake_build"], optional = true } [target.'cfg(unix)'.dependencies] nix = { version = "0.31", default-features = false, features = ["socket", "signal", "fs"] } +tikv-jemallocator = { version = "0.6.0", default-features = false, features = ["unprefixed_malloc_on_supported_platforms"], optional = true } [target.'cfg(target_os = "linux")'.dependencies] procfs = { version = "0.18.0", default-features = false } diff --git a/src/internal_events/mod.rs b/src/internal_events/mod.rs index 02b7460856f2d..d111134e97adf 100644 --- a/src/internal_events/mod.rs +++ b/src/internal_events/mod.rs @@ -40,7 +40,7 @@ mod datadog_traces; mod dedupe; #[cfg(feature = "sources-demo_logs")] mod demo_logs; -#[cfg(feature = "sources-dnstap")] +#[cfg(all(unix, feature = "sources-dnstap"))] mod dnstap; #[cfg(feature = "sources-docker_logs")] mod docker_logs; @@ -194,7 +194,7 @@ pub(crate) use self::datadog_traces::*; pub(crate) use self::dedupe::*; #[cfg(feature = "sources-demo_logs")] pub(crate) use self::demo_logs::*; -#[cfg(feature = "sources-dnstap")] +#[cfg(all(unix, feature = "sources-dnstap"))] pub(crate) use self::dnstap::*; #[cfg(feature = "sources-docker_logs")] pub(crate) use self::docker_logs::*; diff --git a/src/lib.rs b/src/lib.rs index d4d4ab1eeb38a..8d26a3b080ecf 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -35,11 +35,15 @@ pub use indoc::indoc; // re-export codecs for convenience pub use vector_lib::codecs; -#[cfg(all(feature = "tikv-jemallocator", not(feature = "allocation-tracing")))] +#[cfg(all( + unix, + feature = "tikv-jemallocator", + not(feature = "allocation-tracing") +))] #[global_allocator] static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; -#[cfg(all(feature = "tikv-jemallocator", feature = "allocation-tracing"))] +#[cfg(all(unix, feature = "tikv-jemallocator", feature = "allocation-tracing"))] #[global_allocator] static ALLOC: self::internal_telemetry::allocations::Allocator = self::internal_telemetry::allocations::get_grouped_tracing_allocator( diff --git a/src/sources/mod.rs b/src/sources/mod.rs index cdc03f46fdcbb..2384b096f630b 100644 --- a/src/sources/mod.rs +++ b/src/sources/mod.rs @@ -17,7 +17,7 @@ pub mod aws_sqs; pub mod datadog_agent; #[cfg(feature = "sources-demo_logs")] pub mod demo_logs; -#[cfg(feature = "sources-dnstap")] +#[cfg(all(unix, feature = "sources-dnstap"))] pub mod dnstap; #[cfg(feature = "sources-docker_logs")] pub mod docker_logs; From 4e6b49d6951b1f6ed9e2851aec674577ccc4c2b2 Mon Sep 17 00:00:00 2001 From: Yoenn Burban <62966490+gwenaskell@users.noreply.github.com> Date: Fri, 27 Mar 2026 12:47:55 +0100 Subject: [PATCH 070/326] fix(metrics): deduct downstream utilization on task transforms (#24731) * add output utilization on task transforms * add changelog * fix formatting * avoid pin_project on OutputUtilization * replace wrap functions with constructors * Apply suggestions from code review Co-authored-by: Pavlos Rontidis * fix formatting * woops * improve changelog --------- Co-authored-by: Pavlos Rontidis --- .../24641-task-transform-utilization.fix.md | 5 + src/topology/builder.rs | 11 +- src/utilization.rs | 223 ++++++++++++++++-- 3 files changed, 221 insertions(+), 18 deletions(-) create mode 100644 changelog.d/24641-task-transform-utilization.fix.md diff --git a/changelog.d/24641-task-transform-utilization.fix.md b/changelog.d/24641-task-transform-utilization.fix.md new file mode 100644 index 0000000000000..f59a9c8a8eddf --- /dev/null +++ b/changelog.d/24641-task-transform-utilization.fix.md @@ -0,0 +1,5 @@ +Fixed utilization for task transforms to not account for time spent when downstream +is not polling. If the transform is frequently blocked on downstream components, +the reported utilization should be lower. + +authors: gwenaskell diff --git a/src/topology/builder.rs b/src/topology/builder.rs index 4fda56f9b35b3..8048b7a6073ed 100644 --- a/src/topology/builder.rs +++ b/src/topology/builder.rs @@ -52,7 +52,10 @@ use crate::{ spawn_named, topology::task::TaskError, transforms::{SyncTransform, TaskTransform, Transform, TransformOutputs, TransformOutputsBuf}, - utilization::{UtilizationComponentSender, UtilizationEmitter, UtilizationRegistry, wrap}, + utilization::{ + OutputUtilization, Utilization, UtilizationComponentSender, UtilizationEmitter, + UtilizationRegistry, + }, }; static ENRICHMENT_TABLES: LazyLock = @@ -641,7 +644,7 @@ impl<'a> Builder<'a> { .take() .expect("Task started but input has been taken."); - let mut rx = wrap(utilization_sender, component_key.clone(), rx); + let mut rx = Utilization::new(utilization_sender, component_key.clone(), rx); let events_received = register!(EventsReceived); sink.run( @@ -801,7 +804,8 @@ impl<'a> Builder<'a> { let sender = self .utilization_registry .add_component(key.clone(), gauge!("utilization")); - let input_rx = wrap(sender, key.clone(), input_rx.into_stream()); + let output_sender = sender.clone(); + let input_rx = Utilization::new(sender, key.clone(), input_rx.into_stream()); let events_received = register!(EventsReceived); let filtered = input_rx @@ -846,6 +850,7 @@ impl<'a> Builder<'a> { events.estimated_json_encoded_size_of(), )); }); + let stream = OutputUtilization::new(output_sender, stream); let transform = async move { debug!("Task transform starting."); diff --git a/src/utilization.rs b/src/utilization.rs index 413505b8fc9f5..b3496b998f1c5 100644 --- a/src/utilization.rs +++ b/src/utilization.rs @@ -24,6 +24,11 @@ use vector_lib::{id::ComponentKey, shutdown::ShutdownSignal, stats}; const UTILIZATION_EMITTER_DURATION: Duration = Duration::from_secs(5); +/// Stream wrappers used to approximate component utilization from poll timing. +/// +/// Current model: +/// +/// #[pin_project] pub(crate) struct Utilization { intervals: IntervalStream, @@ -33,6 +38,23 @@ pub(crate) struct Utilization { } impl Utilization { + /// Output-side utilization wrapper for task transforms. + /// + /// Measures the time after the wrapped stream yields an item until downstream + /// polls it again. + pub(crate) fn new( + timer_tx: UtilizationComponentSender, + component_key: ComponentKey, + inner: S, + ) -> Self { + Self { + intervals: IntervalStream::new(interval(Duration::from_secs(5))), + timer_tx, + component_key, + inner, + } + } + /// Consumes this wrapper and returns the inner stream. /// /// This can't be constant because destructors can't be run in a const context, and we're @@ -160,6 +182,7 @@ enum UtilizationTimerMessage { StopWait(ComponentKey, Instant), } +#[derive(Clone)] pub(crate) struct UtilizationComponentSender { component_key: ComponentKey, timer_tx: Sender, @@ -277,23 +300,43 @@ impl UtilizationEmitter { } } -/// Wrap a stream to emit stats about utilization. This is designed for use with -/// the input channels of transform and sinks components, and measures the -/// amount of time that the stream is waiting for input from upstream. We make -/// the simplifying assumption that this wait time is when the component is idle -/// and the rest of the time it is doing useful work. This is more true for -/// sinks than transforms, which can be blocked by downstream components, but -/// with knowledge of the config the data is still useful. -pub(crate) fn wrap( +/// Output-side counterpart of [`Utilization`]. Wraps the output stream of a +/// task transform to track time spent waiting for downstream to accept items. +/// +/// While [`Utilization`] measures idle time as "waiting for upstream input", +/// this wrapper measures the complementary case: after yielding an item, the +/// time until the consumer polls again is counted as downstream wait (idle). +/// Both wrappers share a single [`Timer`] per component, and the idempotent +/// `start_wait`/`stop_wait` transitions ensure no double-counting. +pub(crate) struct OutputUtilization { timer_tx: UtilizationComponentSender, - component_key: ComponentKey, inner: S, -) -> Utilization { - Utilization { - intervals: IntervalStream::new(interval(Duration::from_secs(5))), - timer_tx, - component_key, - inner, +} + +impl Stream for OutputUtilization +where + S: Stream + Unpin, +{ + type Item = S::Item; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = self.get_mut(); + this.timer_tx.try_send_stop_wait(); + let result = ready!(this.inner.poll_next_unpin(cx)); + if result.is_some() { + this.timer_tx.try_send_start_wait(); + } + Poll::Ready(result) + } +} + +impl OutputUtilization { + /// Wrap a task transform stream to track time spent waiting for downstream + /// to consume items. This is the output-side counterpart to + /// [`Utilization::new`], designed for use with task transforms where the + /// framework cannot otherwise detect downstream backpressure. + pub(crate) const fn new(timer_tx: UtilizationComponentSender, inner: S) -> Self { + Self { timer_tx, inner } } } @@ -429,4 +472,154 @@ mod tests { let avg = timer.ewma.average().unwrap(); assert_approx_eq(avg, 1.0, "near 1.0 (never waiting)"); } + + /// Mock task transform that passes events through unchanged, simulating + /// a configurable processing delay by advancing MockClock per item. + struct MockTaskTransform { + processing_time: Duration, + } + + use crate::event::EventArray; + use crate::transforms::TaskTransform; + + impl TaskTransform for MockTaskTransform { + fn transform( + self: Box, + task: Pin + Send>>, + ) -> Pin + Send>> { + let processing_time = self.processing_time; + task.map(move |events| { + MockClock::advance(processing_time); + events + }) + .boxed() + } + } + + /// End-to-end test exercising the Utilization (input) and + /// OutputUtilization (output) stream wrappers with a mock TaskTransform, + /// wired up the same way `build_task_transform` does in the builder. + /// + /// Pipeline: channel → Utilization(input) → TaskTransform → OutputUtilization(output) + /// + /// Timeline (10s): + /// T=100..103 waiting for input (3s wait) + /// T=103..105 transform processing (2s work) + /// T=105..108 blocked on downstream (3s wait, even though input has data) + /// T=108..110 transform processing (2s work) + /// + /// Expected utilization = 4/10 = 0.4 + #[tokio::test] + #[serial] + async fn test_task_transform_utilization_end_to_end() { + use crate::event::{EventArray, LogEvent}; + use crate::transforms::TaskTransform; + use futures::SinkExt; + use futures::channel::mpsc as futures_mpsc; + + MockClock::set_time(Duration::from_secs(100)); + + let (mut emitter, registry) = UtilizationEmitter::new(); + let key = ComponentKey::from("test_transform"); + + let sender = registry.add_component(key.clone(), metrics::gauge!("utilization")); + let output_sender = sender.clone(); + + // Upstream channel carrying EventArrays. + let (mut input_tx, input_rx) = futures_mpsc::channel::(10); + + // Wire up the pipeline exactly like build_task_transform: + // Utilization(input) → transform.transform() → OutputUtilization(output) + let input_wrapped = Utilization::new(sender, key.clone(), input_rx); + let transform = Box::new(MockTaskTransform { + processing_time: Duration::from_secs(2), + }); + let transform_output = transform.transform(Box::pin(input_wrapped)); + let mut pipeline = OutputUtilization::new(output_sender, transform_output); + + let waker = futures::task::noop_waker(); + let mut cx = std::task::Context::from_waker(&waker); + + // -- Phase 1: poll transform, no input available -- + // T=100: start_wait sent by input wrapper. + assert!(Pin::new(&mut pipeline).poll_next(&mut cx).is_pending()); + MockClock::advance(Duration::from_secs(3)); + // T=103: 3s of input wait. Period: 0s work / 3s → util = 0.0. + check_timer_utilization(&mut emitter, &key, 0.0, "0.0 (phase 1: all input wait)"); + + // -- Phase 2: send input, poll pipeline → transform processes -- + input_tx + .send(EventArray::from(LogEvent::default())) + .await + .unwrap(); + assert!(Pin::new(&mut pipeline).poll_next(&mut cx).is_ready()); + // T=105: MockClock advanced 2s inside transform. start_wait sent by + // output wrapper. Period: 2s work / 0s wait → util = 1.0. + // EWMA: 0.9×1.0 + 0.1×0.0 = 0.9 + check_timer_utilization(&mut emitter, &key, 0.9, "0.9 (phase 2: all work)"); + + // -- Phase 3: send another event, simulate downstream blocking -- + // Data is available in the input channel, but the transform is not + // polled so no progress is made: the time counts as downstream wait. + input_tx + .send(EventArray::from(LogEvent::default())) + .await + .unwrap(); + MockClock::advance(Duration::from_secs(3)); + // T=108: 3s downstream wait. Period: 0s work / 3s → util = 0.0. + // EWMA: 0.9×0.0 + 0.1×0.9 = 0.09 + check_timer_utilization( + &mut emitter, + &key, + 0.09, + "0.09 (phase 3: downstream wait despite buffered input)", + ); + + // -- Phase 4: poll pipeline, processes the queued event -- + assert!(Pin::new(&mut pipeline).poll_next(&mut cx).is_ready()); + // T=110: MockClock advanced 2s inside transform. Period: 2s work / 0s wait → util = 1.0. + // EWMA: 0.9×1.0 + 0.1×0.09 ≈ 0.909 + check_timer_utilization( + &mut emitter, + &key, + 0.909, + "0.909 (phase 4: queued event processed)", + ); + } + + /// Drain pending timer messages and assert the current EWMA utilization + /// for a single reporting period. + fn check_timer_utilization( + emitter: &mut UtilizationEmitter, + key: &ComponentKey, + expected: f64, + description: &str, + ) { + drain_emitter_messages(emitter); + let mut timers = emitter.timers.lock().expect("mutex poisoned"); + let timer = timers.get_mut(key).expect("timer should exist"); + timer.update_utilization(); + let avg = timer.ewma.average().unwrap(); + assert_approx_eq(avg, expected, description); + } + + /// Drain all pending messages from the emitter's channel into the timers, + /// simulating what `run_utilization` does in a loop. + fn drain_emitter_messages(emitter: &mut UtilizationEmitter) { + while let Ok(message) = emitter.timer_rx.try_recv() { + let mut timers = emitter.timers.lock().expect("mutex poisoned"); + match message { + UtilizationTimerMessage::StartWait(key, at) => { + if let Some(timer) = timers.get_mut(&key) { + timer.start_wait(at); + } + } + UtilizationTimerMessage::StopWait(key, at) => { + if let Some(timer) = timers.get_mut(&key) { + timer.stop_wait(at); + } + } + } + } + } } From a7450cdbba3eb86782d67bea39b81e5b641067d9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 27 Mar 2026 10:50:50 -0400 Subject: [PATCH 071/326] chore(deps): bump tempfile from 3.23.0 to 3.27.0 (#25016) Bumps [tempfile](https://github.com/Stebalien/tempfile) from 3.23.0 to 3.27.0. - [Changelog](https://github.com/Stebalien/tempfile/blob/master/CHANGELOG.md) - [Commits](https://github.com/Stebalien/tempfile/compare/v3.23.0...v3.27.0) --- updated-dependencies: - dependency-name: tempfile dependency-version: 3.27.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Thomas --- Cargo.lock | 26 +++++++++++++------------- Cargo.toml | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0a3f2830ec51d..5bbba19140aa9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -765,7 +765,7 @@ dependencies = [ "cfg-if", "event-listener 5.3.1", "futures-lite", - "rustix 1.0.1", + "rustix 1.1.4", ] [[package]] @@ -2925,7 +2925,7 @@ dependencies = [ "futures-core", "mio", "parking_lot", - "rustix 1.0.1", + "rustix 1.1.4", "signal-hook", "signal-hook-mio", "winapi", @@ -6213,9 +6213,9 @@ checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" [[package]] name = "linux-raw-sys" -version = "0.9.2" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db9c683daf087dc577b7506e9695b3d556a9f3849903fa28186283afd6809e9" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "listenfd" @@ -8088,7 +8088,7 @@ checksum = "25485360a54d6861439d60facef26de713b1e126bf015ec8f98239467a2b82f7" dependencies = [ "bitflags 2.10.0", "procfs-core", - "rustix 1.0.1", + "rustix 1.1.4", ] [[package]] @@ -9397,15 +9397,15 @@ dependencies = [ [[package]] name = "rustix" -version = "1.0.1" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dade4812df5c384711475be5fcd8c162555352945401aed22a35bffeab61f657" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ "bitflags 2.10.0", "errno", "libc", - "linux-raw-sys 0.9.2", - "windows-sys 0.59.0", + "linux-raw-sys 0.12.1", + "windows-sys 0.61.0", ] [[package]] @@ -10890,14 +10890,14 @@ checksum = "83176759e9416cf81ee66cb6508dbfe9c96f20b8b56265a39917551c23c70964" [[package]] name = "tempfile" -version = "3.23.0" +version = "3.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.2", "once_cell", - "rustix 1.0.1", + "rustix 1.1.4", "windows-sys 0.61.0", ] diff --git a/Cargo.toml b/Cargo.toml index 4bc52384f0a06..b2670983edbe2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -192,7 +192,7 @@ serde_json = { version = "1.0.143", default-features = false, features = ["prese serde_yaml = { version = "0.9.34", default-features = false } snafu = { version = "0.9.0", default-features = false, features = ["futures", "std"] } socket2 = { version = "0.5.10", default-features = false } -tempfile = "3.23.0" +tempfile = "3.27.0" tokio = { version = "1.49.0", default-features = false } tokio-stream = { version = "0.1.18", default-features = false } tokio-test = "0.4.5" From 7066adccecc24fe005f1fb7c934bbda4b557ea6c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 27 Mar 2026 10:51:39 -0400 Subject: [PATCH 072/326] chore(deps): bump proptest from 1.10.0 to 1.11.0 (#25021) * chore(deps): bump proptest from 1.10.0 to 1.11.0 Bumps [proptest](https://github.com/proptest-rs/proptest) from 1.10.0 to 1.11.0. - [Release notes](https://github.com/proptest-rs/proptest/releases) - [Changelog](https://github.com/proptest-rs/proptest/blob/main/CHANGELOG.md) - [Commits](https://github.com/proptest-rs/proptest/compare/v1.10.0...v1.11.0) --- updated-dependencies: - dependency-name: proptest dependency-version: 1.11.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Use workspace proptest --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Thomas --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- lib/vector-buffers/Cargo.toml | 2 +- lib/vector-core/Cargo.toml | 4 ++-- lib/vector-stream/Cargo.toml | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5bbba19140aa9..3c1cd0979e24d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8116,9 +8116,9 @@ dependencies = [ [[package]] name = "proptest" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37566cb3fdacef14c0737f9546df7cfeadbfbc9fef10991038bf5015d0c80532" +checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" dependencies = [ "bit-set", "bit-vec", diff --git a/Cargo.toml b/Cargo.toml index b2670983edbe2..d4c3fb4c46c65 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -173,7 +173,7 @@ nom = { version = "8.0.0", default-features = false } ordered-float = { version = "4.6.0", default-features = false } pastey = { version = "0.2", default-features = false } pin-project = { version = "1.1.11", default-features = false } -proptest = { version = "1.10" } +proptest = { version = "1.11" } proptest-derive = { version = "0.6.0" } prost = { version = "0.12", default-features = false, features = ["std"] } prost-build = { version = "0.12", default-features = false } diff --git a/lib/vector-buffers/Cargo.toml b/lib/vector-buffers/Cargo.toml index 04ad96436baf1..4f27f3ef369e1 100644 --- a/lib/vector-buffers/Cargo.toml +++ b/lib/vector-buffers/Cargo.toml @@ -42,7 +42,7 @@ crossbeam-queue = "0.3.12" hdrhistogram = "7.5.4" metrics-tracing-context.workspace = true metrics-util = { workspace = true, features = ["debugging"] } -proptest = "1.10" +proptest.workspace = true quickcheck.workspace = true rand.workspace = true serde_yaml.workspace = true diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml index a7e33377bc09c..41ad3d4b86b5b 100644 --- a/lib/vector-core/Cargo.toml +++ b/lib/vector-core/Cargo.toml @@ -37,7 +37,7 @@ ordered-float.workspace = true openssl = { version = "0.10.75", default-features = false, features = ["vendored"] } parking_lot = { version = "0.12.5", default-features = false } pin-project.workspace = true -proptest = { version = "1.10", optional = true } +proptest = { workspace = true, optional = true } prost-types.workspace = true prost.workspace = true quanta = { version = "0.12.6", default-features = false } @@ -82,7 +82,7 @@ criterion = { workspace = true, features = ["html_reports"] } env-test-util = "1.0.1" quickcheck.workspace = true quickcheck_macros = "1" -proptest = "1.10" +proptest.workspace = true similar-asserts = "1.7.0" tokio-test.workspace = true toml.workspace = true diff --git a/lib/vector-stream/Cargo.toml b/lib/vector-stream/Cargo.toml index b16138436eb82..21b9f1a9a2d7b 100644 --- a/lib/vector-stream/Cargo.toml +++ b/lib/vector-stream/Cargo.toml @@ -19,7 +19,7 @@ vector-common = { path = "../vector-common" } vector-core = { path = "../vector-core" } [dev-dependencies] -proptest = "1.10" +proptest.workspace = true rand.workspace = true rand_distr.workspace = true tokio = { workspace = true, features = ["test-util"] } From 3ca3374c5b0359a3839f6c8091e2c3632faec719 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 27 Mar 2026 15:02:57 +0000 Subject: [PATCH 073/326] chore(deps): bump rust_decimal from 1.39.0 to 1.40.0 (#24799) Bumps [rust_decimal](https://github.com/paupino/rust-decimal) from 1.39.0 to 1.40.0. - [Release notes](https://github.com/paupino/rust-decimal/releases) - [Changelog](https://github.com/paupino/rust-decimal/blob/master/CHANGELOG.md) - [Commits](https://github.com/paupino/rust-decimal/compare/1.39.0...1.40.0) --- updated-dependencies: - dependency-name: rust_decimal dependency-version: 1.40.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3c1cd0979e24d..a9809924cfd68 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9343,9 +9343,9 @@ dependencies = [ [[package]] name = "rust_decimal" -version = "1.39.0" +version = "1.40.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35affe401787a9bd846712274d97654355d21b2a2c092a3139aabe31e9022282" +checksum = "61f703d19852dbf87cbc513643fa81428361eb6940f1ac14fd58155d295a3eb0" dependencies = [ "arrayvec", "borsh", diff --git a/Cargo.toml b/Cargo.toml index d4c3fb4c46c65..21c40c793a9bf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -185,7 +185,7 @@ rand_distr = { version = "0.5.1", default-features = false } rdkafka = { version = "0.39.0", default-features = false } regex = { version = "1.12.3", default-features = false, features = ["std", "perf"] } reqwest = { version = "0.11", features = ["json"] } -rust_decimal = { version = "1.37.0", default-features = false, features = ["std"] } +rust_decimal = { version = "1.40.0", default-features = false, features = ["std"] } semver = { version = "1.0.27", default-features = false, features = ["serde", "std"] } serde = { version = "1.0.219", default-features = false, features = ["alloc", "derive", "rc"] } serde_json = { version = "1.0.143", default-features = false, features = ["preserve_order", "raw_value", "std"] } From c1f34ef803fe5365a5be54f46877590bc8c91ba6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 27 Mar 2026 16:44:05 +0000 Subject: [PATCH 074/326] chore(website deps): bump brace-expansion from 1.1.12 to 1.1.13 in /website (#25056) chore(website deps): bump brace-expansion in /website Bumps [brace-expansion](https://github.com/juliangruber/brace-expansion) from 1.1.12 to 1.1.13. - [Release notes](https://github.com/juliangruber/brace-expansion/releases) - [Commits](https://github.com/juliangruber/brace-expansion/compare/v1.1.12...v1.1.13) --- updated-dependencies: - dependency-name: brace-expansion dependency-version: 1.1.13 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- website/yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/website/yarn.lock b/website/yarn.lock index f7cedc6f6748f..da88eb1b5980f 100644 --- a/website/yarn.lock +++ b/website/yarn.lock @@ -1744,9 +1744,9 @@ boolbase@^1.0.0: integrity sha1-aN/1++YMUes3cl6p4+0xDcwed24= brace-expansion@^1.1.7: - version "1.1.12" - resolved "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz" - integrity sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg== + version "1.1.13" + resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.13.tgz#d37875c01dc9eff988dd49d112a57cb67b54efe6" + integrity sha512-9ZLprWS6EENmhEOpjCYW2c8VkmOvckIJZfkr7rBW6dObmfgJ/L1GpSYW5Hpo9lDz4D1+n0Ckz8rU7FwHDQiG/w== dependencies: balanced-match "^1.0.0" concat-map "0.0.1" From d29a524ea05dcbb6f823121ffdff449ecd960673 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 27 Mar 2026 16:49:17 +0000 Subject: [PATCH 075/326] chore(deps): bump security-framework from 3.5.1 to 3.6.0 (#24763) Bumps [security-framework](https://github.com/kornelski/rust-security-framework) from 3.5.1 to 3.6.0. - [Commits](https://github.com/kornelski/rust-security-framework/compare/v3.5.1...v3.6.0) --- updated-dependencies: - dependency-name: security-framework dependency-version: 3.6.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 12 ++++++------ lib/vector-core/Cargo.toml | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a9809924cfd68..a557cd2ced9e5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9471,7 +9471,7 @@ dependencies = [ "openssl-probe", "rustls-pki-types", "schannel", - "security-framework 3.5.1", + "security-framework 3.6.0", ] [[package]] @@ -9711,9 +9711,9 @@ dependencies = [ [[package]] name = "security-framework" -version = "3.5.1" +version = "3.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3297343eaf830f66ede390ea39da1d462b6b0c1b000f420d0a83f898bbbe6ef" +checksum = "d17b898a6d6948c3a8ee4372c17cb384f90d2e6e912ef00895b14fd7ab54ec38" dependencies = [ "bitflags 2.10.0", "core-foundation 0.10.1", @@ -9724,9 +9724,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.15.0" +version = "2.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" dependencies = [ "core-foundation-sys", "libc", @@ -12626,7 +12626,7 @@ dependencies = [ "regex", "ryu", "schannel", - "security-framework 3.5.1", + "security-framework 3.6.0", "serde", "serde_json", "serde_with", diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml index 41ad3d4b86b5b..8a512884b2c32 100644 --- a/lib/vector-core/Cargo.toml +++ b/lib/vector-core/Cargo.toml @@ -67,7 +67,7 @@ vrl.workspace = true cfg-if.workspace = true [target.'cfg(target_os = "macos")'.dependencies] -security-framework = "3.5.1" +security-framework = "3.6.0" [target.'cfg(windows)'.dependencies] schannel = "0.1.28" From 89e5c343c631e36090d2543106b88685d7a1c908 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 27 Mar 2026 16:55:57 -0400 Subject: [PATCH 076/326] chore(ci): bump juliangruber/read-file-action from 1.1.7 to 1.1.8 (#25000) Bumps [juliangruber/read-file-action](https://github.com/juliangruber/read-file-action) from 1.1.7 to 1.1.8. - [Release notes](https://github.com/juliangruber/read-file-action/releases) - [Commits](https://github.com/juliangruber/read-file-action/compare/b549046febe0fe86f8cb4f93c24e284433f9ab58...271ff311a4947af354c6abcd696a306553b9ec18) --- updated-dependencies: - dependency-name: juliangruber/read-file-action dependency-version: 1.1.8 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/regression.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index f2216df39e684..1b93487a54ec2 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -538,7 +538,7 @@ jobs: - name: Read regression report id: read-analysis - uses: juliangruber/read-file-action@b549046febe0fe86f8cb4f93c24e284433f9ab58 # v1.1.7 + uses: juliangruber/read-file-action@271ff311a4947af354c6abcd696a306553b9ec18 # v1.1.8 with: path: ${{ runner.temp }}/outputs/report.md From 55be10e5860afd47656a67a08335bb17a1d641c8 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 30 Mar 2026 12:14:44 -0400 Subject: [PATCH 077/326] fix(ci): add missing `contents: read` permission to integration review jobs (#25067) fix(ci): add missing contents:read permission to integration review jobs --- .github/workflows/ci-integration-review.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci-integration-review.yml b/.github/workflows/ci-integration-review.yml index 1195b4372d97b..a2a73c39e7d0e 100644 --- a/.github/workflows/ci-integration-review.yml +++ b/.github/workflows/ci-integration-review.yml @@ -85,6 +85,7 @@ jobs: build-test-runner: needs: prep-pr permissions: + contents: read packages: write # Required to push test runner image to GHCR uses: ./.github/workflows/build-test-runner.yml with: @@ -98,6 +99,7 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 90 permissions: + contents: read packages: read # Required to pull test runner image from GHCR strategy: fail-fast: false @@ -140,6 +142,7 @@ jobs: runs-on: ubuntu-24.04-8core timeout-minutes: 30 permissions: + contents: read packages: read # Required to pull test runner image from GHCR strategy: fail-fast: false From d906ba19487c0552be7557d49a3f1dbfcd2405d9 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 30 Mar 2026 17:12:33 -0400 Subject: [PATCH 078/326] chore(deps): upgrade rustls from 0.23.23 to 0.23.37 (#25075) Upgrade rustls from 0.23.23 to 0.23.37 --- Cargo.lock | 38 +++++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a557cd2ced9e5..76e4c912d09ee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1903,7 +1903,7 @@ dependencies = [ "hyperlocal", "log", "pin-project-lite", - "rustls 0.23.23", + "rustls 0.23.37", "rustls-native-certs 0.8.1", "rustls-pki-types", "serde", @@ -5147,7 +5147,7 @@ dependencies = [ "http 1.3.1", "hyper 1.7.0", "hyper-util", - "rustls 0.23.23", + "rustls 0.23.37", "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", @@ -6735,7 +6735,7 @@ dependencies = [ "percent-encoding", "rand 0.8.5", "rustc_version_runtime", - "rustls 0.23.23", + "rustls 0.23.37", "rustversion", "serde", "serde_bytes", @@ -8505,7 +8505,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash", - "rustls 0.23.23", + "rustls 0.23.37", "socket2 0.5.10", "thiserror 2.0.17", "tokio", @@ -8524,7 +8524,7 @@ dependencies = [ "rand 0.9.2", "ring", "rustc-hash", - "rustls 0.23.23", + "rustls 0.23.37", "rustls-pki-types", "slab", "thiserror 2.0.17", @@ -9094,7 +9094,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.23", + "rustls 0.23.37", "rustls-native-certs 0.8.1", "rustls-pki-types", "serde", @@ -9436,15 +9436,15 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.23" +version = "0.23.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47796c98c480fce5406ef69d1c76378375492c3b0a0de587be0c1d9feb12f395" +checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" dependencies = [ "log", "once_cell", "ring", "rustls-pki-types", - "rustls-webpki 0.102.8", + "rustls-webpki 0.103.10", "subtle", "zeroize", ] @@ -9495,11 +9495,12 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.10.1" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2bf47e6ff922db3825eb750c4e2ff784c6ff8fb9e13046ef6a1d1c5401b0b37" +checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" dependencies = [ "web-time", + "zeroize", ] [[package]] @@ -9523,6 +9524,17 @@ dependencies = [ "untrusted", ] +[[package]] +name = "rustls-webpki" +version = "0.103.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "rustversion" version = "1.0.22" @@ -10378,7 +10390,7 @@ dependencies = [ "memchr", "once_cell", "percent-encoding", - "rustls 0.23.23", + "rustls 0.23.37", "serde", "serde_json", "sha2", @@ -11230,7 +11242,7 @@ version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" dependencies = [ - "rustls 0.23.23", + "rustls 0.23.37", "tokio", ] From 6a50bd50c9d482511f5a2a21466a3427dd752263 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 30 Mar 2026 17:27:02 -0400 Subject: [PATCH 079/326] chore(internal docs): update security policy structure and content (#25074) * docs(security): update security policy structure and content * Re-add meta/review schedule * Add whitespace * Revert changes to User Privileges * Finish CI/CD paragraph --- SECURITY.md | 102 ++++++++++++++++++++++------------------------------ 1 file changed, 43 insertions(+), 59 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index 667a3f7b19d8e..e56f752785d91 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -19,9 +19,11 @@ possible on our security efforts. - [Open Source](#open-source) - [Workflow](#workflow) - [Version Control](#version-control) - - [Git](#git) + - [Pull Requests](#pull-requests) + - [Reviews & Approvals](#reviews--approvals) - [Signed Commits](#signed-commits) - [Protected Branches](#protected-branches) + - [Merge Policies](#merge-policies) - [Personnel](#personnel) - [Education](#education) - [Policies](#policies) @@ -34,26 +36,22 @@ possible on our security efforts. - [Unsafe Code](#unsafe-code) - [User Privileges](#user-privileges) - [Dependencies](#dependencies) - - [Change Control](#change-control) - - [Pull Requests](#pull-requests) - - [Reviews & Approvals](#reviews--approvals) - - [Merge Policies](#merge-policies) - [Automated Checks](#automated-checks) - [Vulnerability Scans & Security Advisories](#vulnerability-scans--security-advisories) - [Vulnerability Remediation](#vulnerability-remediation) - - [Fuzz Testing](#fuzz-testing) - [Infrastructure](#infrastructure) - [CI/CD](#cicd) - - [Runtime Isolation](#runtime-isolation) - [Network Security](#network-security) - - [Penetration Testing](#penetration-testing) - [Protocols](#protocols) - [Release Artifacts & Channels](#release-artifacts--channels) - [Asset Audit Logging](#asset-audit-logging) - [Asset Signatures & Checksums](#asset-signatures--checksums) +- [Vulnerability Reporting](#vulnerability-reporting) - [Meta](#meta) - [Review Schedule](#review-schedule) - [Vulnerability Reporting](#vulnerability-reporting) + - [Vector CI](#vector-ci) + - [Other reports](#other-reports) ## Project Structure @@ -84,10 +82,19 @@ are all publicly available. Version control ensures that all code changes are audited and authentic. -#### Git +Vector uses [Git][urls.git] to ensure that changes are auditable and traceable. + +#### Pull Requests + +All changes to Vector must go through a pull request review process. + +#### Reviews & Approvals + +All pull requests must be reviewed by at least one Vector team member. The +review process takes into account many factors, all of which are detailed in +our [Reviewing guide](REVIEWING.md). In exceptional circumstances, this +approval can be retroactive. -Vector leverages the [Git][urls.git] version-control system. This ensures all -changes are audited and traceable. #### Signed Commits @@ -107,6 +114,12 @@ are [protected][urls.github_protected_branches]. The exact requirements are: - Signed commits are required. - Administrators are included in these checks. +#### Merge Policies + +Vector requires pull requests to pass all [automated checks](#automated-checks). +Once passed, the pull request must be squashed and merged. This creates a clean +linear history with a Vector team member's co-sign. + ## Personnel ### Education @@ -116,7 +129,7 @@ the [contributing](CONTRIBUTING.md) and [reviewing](REVIEWING.md) documents. ### Policies -Vector maintains this security policy. Changed are communicated to all Vector +Vector maintains this security policy. Changes are communicated to all Vector team members. ### Two-factor Authentication @@ -150,8 +163,9 @@ catch many common sources of vulnerabilities at compile time. #### Unsafe Code -Vector does not allow the use of unsafe code except in circumstances where it -is required, such as dealing with CFFI. +Vector uses unsafe code sparingly. Unsafe is sometimes required, such as dealing +with CFFI. We may occasionally also use unsafe code for performance reasons but +those changes are kept to a minimum. #### User Privileges @@ -164,27 +178,6 @@ Vector aims to reduce the number of dependencies it relies on. If a dependency is added it goes through a comprehensive review process that is detailed in the [Reviewing guide](REVIEWING.md#dependencies). -### Change Control - -As noted above Vector uses the Git version control system on GitHub. - -#### Pull Requests - -All changes to Vector must go through a pull request review process. - -#### Reviews & Approvals - -All pull requests must be reviewed by at least one Vector team member. The -review process takes into account many factors, all of which are detailed in -our [Reviewing guide](REVIEWING.md). In exceptional circumstances, this -approval can be retroactive. - -#### Merge Policies - -Vector requires pull requests to pass all [automated checks](#automated-checks). -Once passed, the pull request must be squashed and merged. This creates a clean -linear history with a Vector team member's co-sign. - ### Automated Checks When possible, we'll create automated checks to enforce security policies. @@ -195,25 +188,21 @@ When possible, we'll create automated checks to enforce security policies. is part of the [Rust Security advisory database][urls.rust_sec]. The configuration, and a list of currently accepted advisories, are maintained in the [Cargo Deny configuration][urls.cargo_deny_configuration]. The check is run - [on every incoming PR][urls.cargo_deny_schedule] to the Vector project. + on every PR to the Vector project. - Vector implements [Dependabot][urls.dependabot] which performs automated upgrades on dependencies and [alerts][urls.dependabot_alerts] about any dependency-related security vulnerabilities. #### Vulnerability Remediation -If the advisory check fails then the PR will not be merged. We review each advisory to -determine what action to take. If possible, we update the dependency to a version -where the vulnerability has been addressed. If this isn't possible we either record -the acceptance of the vulnerability or replace the dependency. If we accept the -vulnerability we open a ticket to track its remediation, generally awaiting a fix -upstream. If the risk is deemed unacceptable we revisit the code and dependency -to find a more secure alternative. - -#### Fuzz Testing - -Vector implements automated fuzz testing to probe our code for other sources -of potential vulnerabilities. +If the advisory check fails due to changes made in the PR, it will not be +merged. We review each advisory to determine what action to take. Whenever +possible, we update the dependency to a version where the vulnerability has been +addressed. If this isn't possible we either record the acceptance of the +vulnerability or replace the dependency. If we accept the vulnerability we open +a ticket to track its remediation, generally awaiting a fix upstream. If the +risk is deemed unacceptable we revisit the code and dependency to find a more +secure alternative. ## Infrastructure @@ -223,16 +212,12 @@ Vector's infrastructure and how we secure them. ### CI/CD -#### Runtime Isolation - -All builds run in an isolated sandbox that is destroyed after each use. +All builds run in GitHub Actions runners which are ephemeral and don't maintain +state after the job is completed. We ensure we are following [OpenSSF best +practices](https://bestpractices.dev/) to minimize CI risk and exposure. ### Network Security -#### Penetration Testing - -Vector performs quarterly pen tests on vector.dev. - #### Protocols All network traffic is secured via TLS and SSH. This includes checking out @@ -260,7 +245,7 @@ Vector reviews this policy and all user access levels on a quarterly basis. We deeply appreciate any effort to discover and disclose security vulnerabilities responsibly. -## Vector CI +#### Vector CI If you would like to report a Vector CI vulnerability or have any security concerns with other Datadog products, please e-mail security@datadoghq.com. @@ -270,9 +255,9 @@ and verify the vulnerability before taking the necessary steps to fix it. After our initial reply to your disclosure, which should be directly after receiving it, we will periodically update you with the status of the fix. -## Other reports +#### Other reports -Due to the nature of a open-source project, Vector deployments are fully managed by users. Thus vulnerabilities in Vector deployments could +Due to the nature of an open-source project, Vector deployments are fully managed by users. Thus vulnerabilities in Vector deployments could potentially be exploited by malicious actors who already have access to the user’s infrastructure. We encourage responsible disclosure via opening an [open an issue][urls.new_security_report] so that risks can be properly assessed and mitigated. @@ -286,7 +271,6 @@ following when reporting: [urls.cargo_deny]: https://github.com/EmbarkStudios/cargo-deny [urls.cargo_deny_configuration]: https://github.com/vectordotdev/vector/blob/master/deny.toml -[urls.cargo_deny_schedule]: https://github.com/vectordotdev/vector/blob/master/.github/workflows/test.yml#L267 [urls.dependabot]: https://github.com/marketplace/dependabot-preview [urls.dependabot_alerts]: https://github.com/vectordotdev/vector/network/alerts [urls.git]: https://git-scm.com/ From 122cca4678172bbc0c067c65fcc909c6b7aa9ce6 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Mon, 30 Mar 2026 17:51:55 -0400 Subject: [PATCH 080/326] chore(ci): update GHCR cleanup workflow (#25037) * chore(ci): improve GHCR cleanup workflow - Add step to delete dated nightly tags older than 30 days, guarded against semver/release tags - Increase test-runner cleanup batch size from 50 to 100 - Add continue-on-error to test-runner step - Update workflow name and header comment Co-Authored-By: Claude Sonnet 4.6 * chore(ci): replace gh CLI step with action for dated nightly cleanup Co-Authored-By: Claude Sonnet 4.6 * use ubuntu-24.04 * keep 30 nightlies --------- Co-authored-by: Claude Sonnet 4.6 --- .github/workflows/cleanup-ghcr-images.yml | 32 +++++++++++++---------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/.github/workflows/cleanup-ghcr-images.yml b/.github/workflows/cleanup-ghcr-images.yml index c11ecbf6a0c9d..dced4f4f0d449 100644 --- a/.github/workflows/cleanup-ghcr-images.yml +++ b/.github/workflows/cleanup-ghcr-images.yml @@ -2,39 +2,43 @@ # # This workflow cleans up old images from GitHub Container Registry # to prevent unlimited storage growth. It runs weekly and removes: -# 1. Untagged images from all packages (intermediate build artifacts) -# 2. Old test-runner versions (keeps 50 most recent, deletes up to 50 oldest) +# 1. Old dated nightly tags for vector (keeps last ~8) +# 2. Old test-runner versions (keeps 5 most recent) -name: Cleanup Untagged GHCR Images +name: Cleanup GHCR Images on: schedule: # Run weekly on Sundays at 2 AM UTC - - cron: '0 2 * * 0' + - cron: "0 2 * * 0" workflow_dispatch: permissions: - contents: read # Restrictive default + contents: read # Restrictive default jobs: - cleanup: - runs-on: ubuntu-latest + cleanup-vector-nightlies: + runs-on: ubuntu-24.04 permissions: - packages: write # Required to delete package versions from GHCR + packages: write # Required to delete package versions from GHCR steps: - - name: Delete untagged vector images + - name: Delete old dated nightly vector images uses: actions/delete-package-versions@e5bc658cc4c965c472efe991f8beea3981499c55 # v5.0.0 with: package-name: vector package-type: container - min-versions-to-keep: 0 - delete-only-untagged-versions: true - continue-on-error: true + min-versions-to-keep: 30 + ignore-versions: '^(nightly$|\d+\.\d+)' - - name: Delete old vector-test-runner images + cleanup-test-runner: + runs-on: ubuntu-24.04 + permissions: + packages: write # Required to delete package versions from GHCR + steps: + - name: Delete old test-runner images uses: actions/delete-package-versions@e5bc658cc4c965c472efe991f8beea3981499c55 # v5.0.0 with: package-name: test-runner package-type: container min-versions-to-keep: 5 - num-old-versions-to-delete: 50 + num-old-versions-to-delete: 100 From 87ed519cc40011224208914f1dd79b7240f7e1d7 Mon Sep 17 00:00:00 2001 From: Thomas Date: Tue, 31 Mar 2026 09:36:52 -0400 Subject: [PATCH 081/326] chore(dev): consolidate advisory ignores with issue links (#25076) chore(deny): consolidate advisory ignores with issue links --- deny.toml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/deny.toml b/deny.toml index 5a3f7dee13c4d..b8e0be77f41c8 100644 --- a/deny.toml +++ b/deny.toml @@ -40,16 +40,8 @@ license-files = [ [advisories] ignore = [ - # Vulnerability in `rsa` crate: https://rustsec.org/advisories/RUSTSEC-2023-0071.html - # There is not fix available yet. - # https://github.com/vectordotdev/vector/issues/19262 - "RUSTSEC-2023-0071", - { id = "RUSTSEC-2024-0388", reason = "derivative is unmaintained" }, - { id = "RUSTSEC-2024-0384", reason = "instant is unmaintained" }, - { id = "RUSTSEC-2025-0012", reason = "backoff is unmaintained" }, - # rustls-pemfile is unmaintained. Blocked by both async-nats and http 1.0.0 upgrade. - { id = "RUSTSEC-2025-0134", reason = "rustls-pemfile is unmaintained" }, - # rustls-webpki 0.101.7 vulnerability. Fix requires upgrading rustls from 0.21 to 0.23+, - # which is a significant chain upgrade through aws-smithy-http-client, hyper-rustls, tokio-rustls, etc. - { id = "RUSTSEC-2026-0049", reason = "Fix requires major rustls upgrade (0.21 -> 0.23+); tracked for future upgrade" }, + { id = "RUSTSEC-2023-0071", reason = "rsa marvin attack - unpatched upstream (https://github.com/vectordotdev/vector/issues/19262)" }, + { id = "RUSTSEC-2024-0388", reason = "derivative is unmaintained (https://github.com/vectordotdev/vector/issues/24940)" }, + { id = "RUSTSEC-2025-0134", reason = "rustls-pemfile is unmaintained - unpatched crate (https://github.com/bytebeamio/rumqtt/issues/1010) & tonic/reqwest upgrade (https://github.com/vectordotdev/vector/issues/19179)" }, + { id = "RUSTSEC-2026-0049", reason = "rustls-webpki 0.102 is vulnerable - tonic upgrade (https://github.com/vectordotdev/vector/issues/19179)" }, ] From 84c7af2264deacb1afc0529671d53c8c6792bf88 Mon Sep 17 00:00:00 2001 From: Thomas Date: Tue, 31 Mar 2026 09:41:49 -0400 Subject: [PATCH 082/326] fix(ci): install vdev via setup action in ci-integration-review workflow (#25077) chore(ci): install vdev via setup action in ci-integration-review workflow --- .github/workflows/ci-integration-review.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-integration-review.yml b/.github/workflows/ci-integration-review.yml index a2a73c39e7d0e..30e359e366f30 100644 --- a/.github/workflows/ci-integration-review.yml +++ b/.github/workflows/ci-integration-review.yml @@ -117,14 +117,17 @@ jobs: submodules: "recursive" ref: ${{ github.event.review.commit_id }} + - uses: ./.github/actions/setup + with: + vdev: true + datadog-ci: true + - name: Pull test runner image uses: ./.github/actions/pull-test-runner with: github_token: ${{ secrets.GITHUB_TOKEN }} commit_sha: ${{ github.event.review.commit_id }} - - run: bash scripts/environment/prepare.sh --modules=datadog-ci - - name: Integration Tests - ${{ matrix.service }} if: ${{ startsWith(github.event.review.body, '/ci-run-integration-all') || startsWith(github.event.review.body, '/ci-run-all') @@ -156,14 +159,17 @@ jobs: submodules: "recursive" ref: ${{ github.event.review.commit_id }} + - uses: ./.github/actions/setup + with: + vdev: true + datadog-ci: true + - name: Pull test runner image uses: ./.github/actions/pull-test-runner with: github_token: ${{ secrets.GITHUB_TOKEN }} commit_sha: ${{ github.event.review.commit_id }} - - run: bash scripts/environment/prepare.sh --modules=datadog-ci - - name: E2E Tests - ${{ matrix.service }} if: ${{ startsWith(github.event.review.body, '/ci-run-e2e-all') || startsWith(github.event.review.body, '/ci-run-all') From d732259aaa6b5e156b001672ccf0e59be5e84802 Mon Sep 17 00:00:00 2001 From: StepSecurity Bot Date: Tue, 31 Mar 2026 10:08:30 -0700 Subject: [PATCH 083/326] chore(ci): pin GitHub actions to full sha (#25080) Signed-off-by: StepSecurity Bot --- .github/workflows/integration_windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration_windows.yml b/.github/workflows/integration_windows.yml index 5e89827f52e3a..39289d786fd24 100644 --- a/.github/workflows/integration_windows.yml +++ b/.github/workflows/integration_windows.yml @@ -16,7 +16,7 @@ jobs: windows: ${{ steps.filter.outputs.windows }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: dorny/paths-filter@v3 + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 id: filter with: filters: | From 0943dd938f247b0a1963e6bcb1a464ae49392c5f Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 31 Mar 2026 14:54:02 -0400 Subject: [PATCH 084/326] chore(ci): migrate from markdownlint-cli to markdownlint-cli2 (#25081) * chore(ci): migrate from markdownlint-cli to markdownlint-cli2 markdownlint-cli is the legacy CLI wrapper. markdownlint-cli2 is the modern replacement used by all major editors (Zed, VS Code, JetBrains) and auto-detects .markdownlint.jsonc without an explicit --config flag. This unifies the linting tool between CI and local editor environments, so developers see the same results in their IDE as in CI. Co-Authored-By: Claude Opus 4.6 (1M context) * chore(ci): rename markdownlint input to markdownlint-cli2 Co-Authored-By: Claude Opus 4.6 (1M context) --------- Co-authored-by: Claude Opus 4.6 (1M context) --- .github/actions/setup/action.yml | 8 ++++---- .github/workflows/test.yml | 2 +- .markdownlint.jsonc | 5 +---- Makefile | 2 +- scripts/environment/prepare.sh | 8 ++++---- vdev/src/commands/check/markdown.rs | 7 ++----- 6 files changed, 13 insertions(+), 19 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 379afd232341c..f9037aebb68e8 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -76,10 +76,10 @@ inputs: description: "Install vdev CLI tool (cached by vdev/ directory changes)." # prepare.sh - npm - markdownlint: + markdownlint-cli2: required: false default: false - description: "Install markdownlint (npm)." + description: "Install markdownlint-cli2 (npm)." datadog-ci: required: false default: false @@ -267,7 +267,7 @@ runs: ~/.cargo/bin/cargo-llvm-cov ~/.cargo/bin/dd-rust-license-tool ~/.cargo/bin/wasm-pack - /usr/local/bin/markdownlint + /usr/local/bin/markdownlint-cli2 /usr/local/bin/datadog-ci key: ${{ runner.os }}-prepare-binaries-${{ hashFiles('scripts/environment/*') }} restore-keys: | @@ -287,7 +287,7 @@ runs: [[ "${{ inputs.cargo-llvm-cov }}" == "true" ]] && mods+=("cargo-llvm-cov") [[ "${{ inputs.dd-rust-license-tool }}" == "true" ]] && mods+=("dd-rust-license-tool") [[ "${{ inputs.wasm-pack }}" == "true" ]] && mods+=("wasm-pack") - [[ "${{ inputs.markdownlint }}" == "true" ]] && mods+=("markdownlint") + [[ "${{ inputs.markdownlint-cli2 }}" == "true" ]] && mods+=("markdownlint-cli2") [[ "${{ inputs.datadog-ci }}" == "true" ]] && mods+=("datadog-ci") csm=$(IFS=,; echo "${mods[*]}") diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5572b2d3f8d9d..986b145305f78 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -147,7 +147,7 @@ jobs: - uses: ./.github/actions/setup with: rust: true - markdownlint: true + markdownlint-cli2: true - run: make check-markdown check-generated-docs: diff --git a/.markdownlint.jsonc b/.markdownlint.jsonc index 5bca38386599b..5e442a0162df4 100644 --- a/.markdownlint.jsonc +++ b/.markdownlint.jsonc @@ -1,7 +1,4 @@ -// This config is auto-detected by markdownlint-cli2 (used by Zed, VS Code, and most modern editors). -// The CI check currently uses markdownlint-cli (legacy). Migrate CI to markdownlint-cli2 to unify tooling: -// npm install -g markdownlint-cli2 -// markdownlint-cli2 "**/*.md" +// Auto-detected by markdownlint-cli2 and editors (Zed, VS Code, JetBrains). { "default": true, diff --git a/Makefile b/Makefile index 8b1b856647b9c..5c52c367c11b7 100644 --- a/Makefile +++ b/Makefile @@ -508,7 +508,7 @@ check-markdown: ## Check that markdown is styled properly .PHONY: fix-markdown fix-markdown: ## Auto-fix markdown style issues - ${MAYBE_ENVIRONMENT_EXEC} markdownlint --fix --config .markdownlint.jsonc $(shell git ls-files '*.md') + ${MAYBE_ENVIRONMENT_EXEC} markdownlint-cli2 --fix $(shell git ls-files '*.md') .PHONY: check-examples check-examples: ## Check that the config/examples files are valid diff --git a/scripts/environment/prepare.sh b/scripts/environment/prepare.sh index 5bd5a29a79ac2..397fefbb464a8 100755 --- a/scripts/environment/prepare.sh +++ b/scripts/environment/prepare.sh @@ -37,7 +37,7 @@ CARGO_HACK_VERSION="0.6.43" DD_RUST_LICENSE_TOOL_VERSION="1.0.6" CARGO_LLVM_COV_VERSION="0.8.4" WASM_PACK_VERSION="0.13.1" -MARKDOWNLINT_VERSION="0.45.0" +MARKDOWNLINT_CLI2_VERSION="0.22.0" DATADOG_CI_VERSION="5.9.0" VDEV_VERSION="0.3.0" @@ -52,7 +52,7 @@ ALL_MODULES=( cargo-llvm-cov dd-rust-license-tool wasm-pack - markdownlint + markdownlint-cli2 datadog-ci release-flags # Not a tool - sources release-flags.sh to set CI env vars vdev @@ -92,7 +92,7 @@ Modules: cargo-llvm-cov dd-rust-license-tool wasm-pack - markdownlint + markdownlint-cli2 datadog-ci vdev @@ -218,5 +218,5 @@ maybe_install_cargo_tool dd-rust-license-tool "${DD_RUST_LICENSE_TOOL_VERSION}" maybe_install_cargo_tool wasm-pack "${WASM_PACK_VERSION}" maybe_install_cargo_tool vdev "${VDEV_VERSION}" -maybe_install_npm_package markdownlint markdownlint-cli "${MARKDOWNLINT_VERSION}" +maybe_install_npm_package markdownlint-cli2 markdownlint-cli2 "${MARKDOWNLINT_CLI2_VERSION}" maybe_install_npm_package datadog-ci "@datadog/datadog-ci" "${DATADOG_CI_VERSION}" "v${DATADOG_CI_VERSION}" "version" diff --git a/vdev/src/commands/check/markdown.rs b/vdev/src/commands/check/markdown.rs index 3ecf0c91f3201..1e10126a14e5e 100644 --- a/vdev/src/commands/check/markdown.rs +++ b/vdev/src/commands/check/markdown.rs @@ -14,11 +14,8 @@ impl Cli { return Ok(()); } - let args: Vec<&str> = vec!["--config", ".markdownlint.jsonc"] - .into_iter() - .chain(files.iter().map(String::as_str)) - .collect(); + let args: Vec<&str> = files.iter().map(String::as_str).collect(); - app::exec("markdownlint", &args, true) + app::exec("markdownlint-cli2", &args, true) } } From 50345d5929ecb45d74e1355e612f044c24ff4a93 Mon Sep 17 00:00:00 2001 From: Thomas Date: Tue, 31 Mar 2026 14:27:15 -0400 Subject: [PATCH 085/326] chore(ci): skip test runner pull, datadog-ci install, and checkout when int tests won't run (#25068) * chore(ci): skip test runner pull, datadog-ci install, and checkout when int tests won't run * chore(ci): evaluate int/e2e run conditions once via step output --- .github/workflows/ci-integration-review.yml | 36 ++++-- .github/workflows/integration.yml | 118 ++++++++++---------- 2 files changed, 80 insertions(+), 74 deletions(-) diff --git a/.github/workflows/ci-integration-review.yml b/.github/workflows/ci-integration-review.yml index 30e359e366f30..72d2b4e660151 100644 --- a/.github/workflows/ci-integration-review.yml +++ b/.github/workflows/ci-integration-review.yml @@ -105,14 +105,19 @@ jobs: fail-fast: false matrix: service: [ - "amqp", "appsignal", "aws", "axiom", "azure", "clickhouse", "databend", "datadog-agent", - "datadog-logs", "datadog-metrics", "datadog-traces", "dnstap", "docker-logs", "elasticsearch", + "amqp", "appsignal", "axiom", "aws", "azure", "clickhouse", "databend", "datadog-agent", + "datadog-logs", "datadog-metrics", "datadog-traces", "dnstap", "docker-logs", "doris", "elasticsearch", "eventstoredb", "fluent", "gcp", "greptimedb", "http-client", "influxdb", "kafka", "logstash", - "loki", "mongodb", "nats", "nginx", "opentelemetry", "postgres", "prometheus", "pulsar", - "redis", "splunk", "webhdfs" + "loki", "mqtt", "mongodb", "nats", "nginx", "opentelemetry", "postgres", "prometheus", "pulsar", + "redis", "webhdfs" ] steps: + - name: Evaluate run condition + id: run_condition + run: echo "should_run=${{ startsWith(github.event.review.body, '/ci-run-integration-all') || startsWith(github.event.review.body, '/ci-run-all') || startsWith(github.event.review.body, format('/ci-run-integration-{0}', matrix.service)) }}" >> $GITHUB_OUTPUT + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + if: steps.run_condition.outputs.should_run == 'true' with: submodules: "recursive" ref: ${{ github.event.review.commit_id }} @@ -123,20 +128,21 @@ jobs: datadog-ci: true - name: Pull test runner image + if: steps.run_condition.outputs.should_run == 'true' uses: ./.github/actions/pull-test-runner with: github_token: ${{ secrets.GITHUB_TOKEN }} commit_sha: ${{ github.event.review.commit_id }} - name: Integration Tests - ${{ matrix.service }} - if: ${{ startsWith(github.event.review.body, '/ci-run-integration-all') - || startsWith(github.event.review.body, '/ci-run-all') - || startsWith(github.event.review.body, format('/ci-run-integration-{0}', matrix.service)) }} + if: steps.run_condition.outputs.should_run == 'true' uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 with: timeout_minutes: 30 max_attempts: 3 - command: bash scripts/run-integration-test.sh int ${{ matrix.service }} + command: | + bash scripts/environment/prepare.sh --modules=datadog-ci + bash scripts/run-integration-test.sh int ${{ matrix.service }} e2e-tests: needs: @@ -154,7 +160,12 @@ jobs: "datadog-logs", "datadog-metrics", "opentelemetry-logs", "opentelemetry-metrics" ] steps: + - name: Evaluate run condition + id: run_condition + run: echo "should_run=${{ startsWith(github.event.review.body, '/ci-run-e2e-all') || startsWith(github.event.review.body, '/ci-run-all') || startsWith(github.event.review.body, format('/ci-run-e2e-{0}', matrix.service)) }}" >> $GITHUB_OUTPUT + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + if: steps.run_condition.outputs.should_run == 'true' with: submodules: "recursive" ref: ${{ github.event.review.commit_id }} @@ -165,20 +176,21 @@ jobs: datadog-ci: true - name: Pull test runner image + if: steps.run_condition.outputs.should_run == 'true' uses: ./.github/actions/pull-test-runner with: github_token: ${{ secrets.GITHUB_TOKEN }} commit_sha: ${{ github.event.review.commit_id }} - name: E2E Tests - ${{ matrix.service }} - if: ${{ startsWith(github.event.review.body, '/ci-run-e2e-all') - || startsWith(github.event.review.body, '/ci-run-all') - || startsWith(github.event.review.body, format('/ci-run-e2e-{0}', matrix.service)) }} + if: steps.run_condition.outputs.should_run == 'true' uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 with: timeout_minutes: 35 max_attempts: 3 - command: bash scripts/run-integration-test.sh e2e ${{ matrix.service }} + command: | + bash scripts/environment/prepare.sh --modules=datadog-ci + bash scripts/run-integration-test.sh e2e ${{ matrix.service }} update-pr-status: diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 20fae17be16ae..861913c315cd4 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -84,59 +84,56 @@ jobs: ] timeout-minutes: 90 steps: + - name: Download JSON artifact from changes.yml + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + if: github.event_name == 'merge_group' + with: + name: int_tests_changes + + - name: Determine if test should run + id: check + run: | + if [[ "${{ github.event_name }}" == "workflow_dispatch" || \ + "${{ needs.changes.outputs.dependencies }}" == "true" || \ + "${{ needs.changes.outputs.integration-yml }}" == "true" ]]; then + echo "should_run=true" >> "$GITHUB_OUTPUT" + elif [[ "${{ needs.changes.outputs.website_only }}" == "true" ]]; then + echo "should_run=false" >> "$GITHUB_OUTPUT" + elif [[ -f int_tests_changes.json ]]; then + should_run=$(jq -r '."${{ matrix.service }}" // false' int_tests_changes.json) + echo "should_run=${should_run}" >> "$GITHUB_OUTPUT" + else + echo "should_run=false" >> "$GITHUB_OUTPUT" + fi + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + if: steps.check.outputs.should_run == 'true' with: submodules: "recursive" - uses: ./.github/actions/setup + if: steps.check.outputs.should_run == 'true' with: vdev: true mold: false cargo-cache: false - - - name: Download JSON artifact from changes.yml - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 - if: github.event_name == 'merge_group' - with: - name: int_tests_changes + datadog-ci: true - name: Pull test runner image + if: steps.check.outputs.should_run == 'true' uses: ./.github/actions/pull-test-runner with: github_token: ${{ secrets.GITHUB_TOKEN }} commit_sha: ${{ github.sha }} - name: Run Integration Tests for ${{ matrix.service }} + if: steps.check.outputs.should_run == 'true' uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 with: timeout_minutes: 30 max_attempts: 3 command: | - if [[ -f int_tests_changes.json ]]; then - # Parse the JSON and check if the specific integration test should run. - should_run=$(jq -r '."${{ matrix.service }}" // false' int_tests_changes.json) - else - # The `changes` job did not run (manual run) or the file is missing, default to false. - should_run=false - fi - - if [[ "${{ needs.changes.outputs.website_only }}" == "true" ]]; then - echo "Skipping ${{ matrix.service }} test since only website changes were detected" - exit 0 - fi - - # Check if any of the three conditions is true - if [[ "${{ github.event_name }}" == "workflow_dispatch" || \ - "${{ needs.changes.outputs.dependencies }}" == "true" || \ - "${{ needs.changes.outputs.integration-yml }}" == "true" || \ - "$should_run" == "true" ]]; then - # Only install dep if test runs - bash scripts/environment/prepare.sh --modules=datadog-ci - echo "Running test for ${{ matrix.service }}" - bash scripts/run-integration-test.sh int ${{ matrix.service }} - else - echo "Skipping ${{ matrix.service }} test as the value is false or conditions not met." - fi + bash scripts/run-integration-test.sh int ${{ matrix.service }} e2e-tests: @@ -152,59 +149,56 @@ jobs: timeout-minutes: 90 steps: + - name: Download JSON artifact from changes.yml + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + if: github.event_name == 'merge_group' + with: + name: e2e_tests_changes + + - name: Determine if test should run + id: check + run: | + if [[ "${{ github.event_name }}" == "workflow_dispatch" || \ + "${{ needs.changes.outputs.dependencies }}" == "true" || \ + "${{ needs.changes.outputs.integration-yml }}" == "true" ]]; then + echo "should_run=true" >> "$GITHUB_OUTPUT" + elif [[ "${{ needs.changes.outputs.website_only }}" == "true" ]]; then + echo "should_run=false" >> "$GITHUB_OUTPUT" + elif [[ -f e2e_tests_changes.json ]]; then + should_run=$(jq -r '."${{ matrix.service }}" // false' e2e_tests_changes.json) + echo "should_run=${should_run}" >> "$GITHUB_OUTPUT" + else + echo "should_run=false" >> "$GITHUB_OUTPUT" + fi + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + if: steps.check.outputs.should_run == 'true' with: submodules: "recursive" - uses: ./.github/actions/setup + if: steps.check.outputs.should_run == 'true' with: vdev: true mold: false cargo-cache: false - - - name: Download JSON artifact from changes.yml - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 - if: github.event_name == 'merge_group' - with: - name: e2e_tests_changes + datadog-ci: true - name: Pull test runner image + if: steps.check.outputs.should_run == 'true' uses: ./.github/actions/pull-test-runner with: github_token: ${{ secrets.GITHUB_TOKEN }} commit_sha: ${{ github.sha }} - name: Run E2E Tests for ${{ matrix.service }} + if: steps.check.outputs.should_run == 'true' uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 with: timeout_minutes: 30 max_attempts: 3 command: | - if [[ -f e2e_tests_changes.json ]]; then - # Parse the JSON and check if the specific e2e test should run. - should_run=$(jq -r '."${{ matrix.service }}" // false' e2e_tests_changes.json) - else - # The `changes` job did not run (manual run) or the file is missing, default to false. - should_run=false - fi - - if [[ "${{ needs.changes.outputs.website_only }}" == "true" ]]; then - echo "Skipping ${{ matrix.service }} test since only website changes were detected" - exit 0 - fi - - # Check if any of the three conditions is true - if [[ "${{ github.event_name }}" == "workflow_dispatch" || \ - "${{ needs.changes.outputs.dependencies }}" == "true" || \ - "${{ needs.changes.outputs.integration-yml }}" == "true" ]] || \ - "$should_run" == "true" ]]; then - # Only install dep if test runs - bash scripts/environment/prepare.sh --modules=datadog-ci - echo "Running test for ${{ matrix.service }}" - bash scripts/run-integration-test.sh e2e ${{ matrix.service }} - else - echo "Skipping ${{ matrix.service }} test as the value is false or conditions not met." - fi + bash scripts/run-integration-test.sh e2e ${{ matrix.service }} integration-test-suite: name: Integration Test Suite From 24d8db42e01e28b590e5484fede9279908805901 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 31 Mar 2026 18:04:57 -0400 Subject: [PATCH 086/326] chore(ci): remove unused publish-homebrew workflow (#25085) Co-authored-by: Claude Opus 4.6 (1M context) --- .github/workflows/publish-homebrew.yml | 36 -------------------------- 1 file changed, 36 deletions(-) delete mode 100644 .github/workflows/publish-homebrew.yml diff --git a/.github/workflows/publish-homebrew.yml b/.github/workflows/publish-homebrew.yml deleted file mode 100644 index 117fe087c8441..0000000000000 --- a/.github/workflows/publish-homebrew.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: Publish to Homebrew - -on: - workflow_dispatch: - inputs: - vector_version: - description: "Vector version to publish" - required: true - type: string - - workflow_call: - inputs: - git_ref: - required: true - type: string - vector_version: - required: true - type: string - -permissions: - contents: read - -jobs: - publish-homebrew: - runs-on: ubuntu-24.04 - timeout-minutes: 10 - env: - GITHUB_TOKEN: ${{ secrets.HOMEBREW_PAT }} - steps: - - name: Checkout Vector - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - ref: ${{ inputs.git_ref || github.ref_name }} - - - name: Publish update to Homebrew tap - run: make release-homebrew VECTOR_VERSION=${{ inputs.vector_version }} From df00c7eec91ecb35eb0a8db0249701562de7dc87 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Wed, 1 Apr 2026 15:37:50 -0400 Subject: [PATCH 087/326] fix(ci): pin Pulsar integration test image to 4.1.3 (#25097) The `apachepulsar/pulsar:latest` image moved to a version that breaks TLS connectivity, causing `pulsar_happy_tls` and `consumes_event_with_tls` to fail consistently in CI. Pin to 4.1.3 to unblock the merge queue. Re-enabling `latest` is tracked in #25096. Co-authored-by: Claude Opus 4.6 (1M context) --- tests/integration/pulsar/config/test.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integration/pulsar/config/test.yaml b/tests/integration/pulsar/config/test.yaml index 9d7f33f60dac5..a273fb4e9f163 100644 --- a/tests/integration/pulsar/config/test.yaml +++ b/tests/integration/pulsar/config/test.yaml @@ -7,7 +7,9 @@ env: PULSAR_HOST: pulsar matrix: - version: [latest] + # TODO: re-enable `latest` once TLS tests pass with newer Pulsar images. + # See https://github.com/vectordotdev/vector/issues/25096 + version: ["4.1.3"] # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch From 639c924c733442bcc7443817d16ea0dd737ded54 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Apr 2026 20:24:01 +0000 Subject: [PATCH 088/326] chore(deps): bump debian from `1d3c811` to `26f98cc` in /distribution/docker/debian in the docker-images group across 1 directory (#24996) chore(deps): bump debian Bumps the docker-images group in /distribution/docker/debian with 1 update: debian. Updates `debian` from `1d3c811` to `26f98cc` --- updated-dependencies: - dependency-name: debian dependency-version: trixie-slim dependency-type: direct:production dependency-group: docker-images ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- distribution/docker/debian/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/distribution/docker/debian/Dockerfile b/distribution/docker/debian/Dockerfile index a726f8a72004c..4922e12ef9a60 100644 --- a/distribution/docker/debian/Dockerfile +++ b/distribution/docker/debian/Dockerfile @@ -1,4 +1,4 @@ -FROM docker.io/debian:trixie-slim@sha256:1d3c811171a08a5adaa4a163fbafd96b61b87aa871bbc7aa15431ac275d3d430 AS builder +FROM docker.io/debian:trixie-slim@sha256:26f98ccd92fd0a44d6928ce8ff8f4921b4d2f535bfa07555ee5d18f61429cf0c AS builder WORKDIR /vector @@ -7,7 +7,7 @@ RUN dpkg -i vector_*_"$(dpkg --print-architecture)".deb RUN mkdir -p /var/lib/vector -FROM docker.io/debian:trixie-slim@sha256:1d3c811171a08a5adaa4a163fbafd96b61b87aa871bbc7aa15431ac275d3d430 +FROM docker.io/debian:trixie-slim@sha256:26f98ccd92fd0a44d6928ce8ff8f4921b4d2f535bfa07555ee5d18f61429cf0c # https://github.com/opencontainers/image-spec/blob/main/annotations.md LABEL org.opencontainers.image.url="https://vector.dev" From 9dbdb0662b668867ac8db6e444ac710d4346a037 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Apr 2026 20:25:08 +0000 Subject: [PATCH 089/326] chore(deps): bump distroless/static from `28efbe9` to `47b2d72` in /distribution/docker/distroless-static in the docker-images group across 1 directory (#24998) chore(deps): bump distroless/static Bumps the docker-images group in /distribution/docker/distroless-static with 1 update: distroless/static. Updates `distroless/static` from `28efbe9` to `47b2d72` --- updated-dependencies: - dependency-name: distroless/static dependency-version: latest dependency-type: direct:production dependency-group: docker-images ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- distribution/docker/distroless-static/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distribution/docker/distroless-static/Dockerfile b/distribution/docker/distroless-static/Dockerfile index 9518be0c4492c..312190cda5302 100644 --- a/distribution/docker/distroless-static/Dockerfile +++ b/distribution/docker/distroless-static/Dockerfile @@ -9,7 +9,7 @@ RUN mkdir -p /var/lib/vector # distroless doesn't use static tags # hadolint ignore=DL3007 -FROM gcr.io/distroless/static:latest@sha256:28efbe90d0b2f2a3ee465cc5b44f3f2cf5533514cf4d51447a977a5dc8e526d0 +FROM gcr.io/distroless/static:latest@sha256:47b2d72ff90843eb8a768b5c2f89b40741843b639d065b9b937b07cd59b479c6 # https://github.com/opencontainers/image-spec/blob/main/annotations.md LABEL org.opencontainers.image.url="https://vector.dev" From 3c879836d586695701e9cf60fed3e7ef4b408f2e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Apr 2026 20:25:30 +0000 Subject: [PATCH 090/326] chore(deps): bump debian from `1d3c811` to `26f98cc` in /distribution/docker/distroless-libc in the docker-images group across 1 directory (#24997) chore(deps): bump debian Bumps the docker-images group in /distribution/docker/distroless-libc with 1 update: debian. Updates `debian` from `1d3c811` to `26f98cc` --- updated-dependencies: - dependency-name: debian dependency-version: trixie-slim dependency-type: direct:production dependency-group: docker-images ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- distribution/docker/distroless-libc/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distribution/docker/distroless-libc/Dockerfile b/distribution/docker/distroless-libc/Dockerfile index 2e3988b7403fe..0ae19e00e17d8 100644 --- a/distribution/docker/distroless-libc/Dockerfile +++ b/distribution/docker/distroless-libc/Dockerfile @@ -1,4 +1,4 @@ -FROM docker.io/debian:trixie-slim@sha256:1d3c811171a08a5adaa4a163fbafd96b61b87aa871bbc7aa15431ac275d3d430 AS builder +FROM docker.io/debian:trixie-slim@sha256:26f98ccd92fd0a44d6928ce8ff8f4921b4d2f535bfa07555ee5d18f61429cf0c AS builder WORKDIR /vector From 9f484c2b9c1c219f44c2fd5289c40e3a4b63f6e3 Mon Sep 17 00:00:00 2001 From: Thomas Date: Wed, 1 Apr 2026 19:26:12 -0400 Subject: [PATCH 091/326] fix(ci): prevent script injection in integration review workflow (#25106) --- .github/workflows/ci-integration-review.yml | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-integration-review.yml b/.github/workflows/ci-integration-review.yml index 72d2b4e660151..029ea40e05963 100644 --- a/.github/workflows/ci-integration-review.yml +++ b/.github/workflows/ci-integration-review.yml @@ -114,7 +114,15 @@ jobs: steps: - name: Evaluate run condition id: run_condition - run: echo "should_run=${{ startsWith(github.event.review.body, '/ci-run-integration-all') || startsWith(github.event.review.body, '/ci-run-all') || startsWith(github.event.review.body, format('/ci-run-integration-{0}', matrix.service)) }}" >> $GITHUB_OUTPUT + env: + REVIEW_BODY: ${{ github.event.review.body }} + SERVICE: ${{ matrix.service }} + run: | + if [[ "$REVIEW_BODY" == /ci-run-integration-all* ]] || [[ "$REVIEW_BODY" == /ci-run-all* ]] || [[ "$REVIEW_BODY" == "/ci-run-integration-${SERVICE}"* ]]; then + echo "should_run=true" >> "$GITHUB_OUTPUT" + else + echo "should_run=false" >> "$GITHUB_OUTPUT" + fi - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 if: steps.run_condition.outputs.should_run == 'true' @@ -162,7 +170,15 @@ jobs: steps: - name: Evaluate run condition id: run_condition - run: echo "should_run=${{ startsWith(github.event.review.body, '/ci-run-e2e-all') || startsWith(github.event.review.body, '/ci-run-all') || startsWith(github.event.review.body, format('/ci-run-e2e-{0}', matrix.service)) }}" >> $GITHUB_OUTPUT + env: + REVIEW_BODY: ${{ github.event.review.body }} + SERVICE: ${{ matrix.service }} + run: | + if [[ "$REVIEW_BODY" == /ci-run-e2e-all* ]] || [[ "$REVIEW_BODY" == /ci-run-all* ]] || [[ "$REVIEW_BODY" == "/ci-run-e2e-${SERVICE}"* ]]; then + echo "should_run=true" >> "$GITHUB_OUTPUT" + else + echo "should_run=false" >> "$GITHUB_OUTPUT" + fi - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 if: steps.run_condition.outputs.should_run == 'true' From 9055388bc0c89a56177f6df60de7a7e09fe74de1 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Thu, 2 Apr 2026 09:48:09 -0400 Subject: [PATCH 092/326] chore(dev): add repo-wide prettier config for YAML, JS, TS, and JSON (#25082) * chore: add repo-wide prettier for YAML/JS/TS/JSON formatting Add a root .prettierrc.json (auto-detected by Zed, VS Code, JetBrains) and .prettierignore to skip Hugo templates, generated files, and VRL configs with triple-quoted strings. - Pin prettier@3.5.3 in prepare.sh and .github/actions/setup - Integrate prettier into vdev check fmt / vdev fmt - Add check-prettier and fix-prettier Makefile targets - Add prettier path filter in changes.yml so check-fmt triggers on YAML/JS/TS/JSON changes - Remove website/.prettierrc.json (superseded by root config) Co-Authored-By: Claude Opus 4.6 (1M context) * chore: format all YAML/JS/TS/JSON files with prettier Run make fix-prettier to establish a consistent formatting baseline. Changes are purely cosmetic: whitespace cleanup, quote normalization, and array formatting. Co-Authored-By: Claude Opus 4.6 (1M context) * chore: use git ls-files instead of globs for prettier file discovery Shell globs expand to include untracked directories. Use git ls-files to only format tracked files, consistent with how markdownlint works. Co-Authored-By: Claude Opus 4.6 (1M context) * chore: pass --ignore-path .prettierignore when using git ls-files When prettier receives explicit file paths it skips .prettierignore. Pass --ignore-path explicitly so exclusions (lib/codecs/tests/data, config examples with VRL triple-quotes, Hugo build output, etc.) are still respected. Co-Authored-By: Claude Opus 4.6 (1M context) * chore: suppress file list echo in prettier Makefile targets Co-Authored-By: Claude Opus 4.6 (1M context) * chore: invoke prettier per file extension to avoid ARG_MAX Running prettier with all tracked files in a single invocation can exceed macOS ARG_MAX (262 KB) since this repo has ~4400 matching paths. Invoke prettier once per extension type instead. Co-Authored-By: Claude Opus 4.6 (1M context) * chore: log per-extension prettier invocations in vdev Move info! inside the loop so each extension type logs its own progress line. Co-Authored-By: Claude Opus 4.6 (1M context) * chore: bump prettier to 3.8.1 to match local version Co-Authored-By: Claude Opus 4.6 (1M context) * chore: address PR feedback on prettier setup - Remove unused vendor/ entry from .prettierignore - Deduplicate PRETTIER_EXTENSIONS constant between fmt and check/fmt Co-Authored-By: Claude Opus 4.6 (1M context) --------- Co-authored-by: Claude Opus 4.6 (1M context) --- .github/DISCUSSION_TEMPLATE/q-a.yml | 2 +- .github/ISSUE_TEMPLATE/bug.yml | 152 +-- .github/ISSUE_TEMPLATE/config.yml | 1 - .github/ISSUE_TEMPLATE/feature.yml | 167 ++- .github/actions/install-vdev/action.yml | 2 +- .github/actions/pull-test-runner/action.yml | 4 +- .github/actions/setup/action.yml | 7 +- .github/dependabot.yml | 52 +- .github/labeler.yml | 24 +- .github/workflows/build-test-runner.yml | 8 +- .github/workflows/build_preview_sites.yml | 32 +- .github/workflows/changes.yml | 222 +-- .github/workflows/ci-integration-review.yml | 67 +- .github/workflows/ci-review-trigger.yml | 28 +- .github/workflows/component_features.yml | 6 +- .github/workflows/coverage.yml | 4 +- .github/workflows/create_preview_sites.yml | 12 +- .github/workflows/cross.yml | 2 +- .github/workflows/custom_builds.yml | 6 +- .github/workflows/deny.yml | 6 +- .github/workflows/environment.yml | 6 +- .github/workflows/install-sh.yml | 4 +- .github/workflows/integration-test.yml | 2 +- .github/workflows/integration.yml | 51 +- .github/workflows/k8s_e2e.yml | 6 +- .github/workflows/labeler.yml | 8 +- .github/workflows/nightly.yml | 8 +- .github/workflows/preview_site_trigger.yml | 6 +- .github/workflows/protobuf.yml | 10 +- .github/workflows/publish.yml | 16 +- .github/workflows/regression.yml | 27 +- .github/workflows/release.yml | 6 +- .github/workflows/scorecard.yml | 4 +- .github/workflows/spelling.yml | 105 +- .github/workflows/static-analysis.yml | 4 +- .github/workflows/test-make-command.yml | 11 +- .github/workflows/test.yml | 5 +- .github/workflows/unit_mac.yml | 3 +- .github/workflows/unit_windows.yml | 3 +- .github/workflows/vdev_publish.yml | 6 +- .prettierignore | 30 + .prettierrc.json | 4 + Makefile | 18 + config/examples/environment_variables.yaml | 6 +- .../examples/file_to_cloudwatch_metrics.yaml | 12 +- .../namespacing/sinks/es_cluster.yaml | 6 +- .../namespacing/sinks/s3_archives.yaml | 12 +- .../namespacing/transforms/apache_sample.yaml | 2 +- config/examples/stdio.yaml | 2 +- config/examples/wrapped_json.yaml | 6 +- .../collector/logs/v1/logs_service_http.yaml | 9 +- .../metrics/v1/metrics_service_http.yaml | 9 +- .../trace/v1/trace_service_http.yaml | 8 +- .../data/fixtures/log_event/malformed.json | 2 +- .../resources/json-schema_definition.json | 11 +- lib/vector-vrl/web-playground/public/index.js | 552 ++++---- .../web-playground/public/vrl-highlighter.js | 1199 ++++++++--------- .../lading/lading.yaml | 37 +- .../vector/vector.yaml | 36 +- .../lading/lading.yaml | 37 +- .../vector/vector.yaml | 36 +- .../lading/lading.yaml | 36 +- .../vector/vector.yaml | 30 +- .../lading/lading.yaml | 36 +- .../vector/vector.yaml | 30 +- .../file_100_to_blackhole/lading/lading.yaml | 37 +- .../file_100_to_blackhole/vector/vector.yaml | 10 +- .../file_to_blackhole/lading/lading.yaml | 37 +- .../file_to_blackhole/vector/vector.yaml | 10 +- .../fluent_elasticsearch/lading/lading.yaml | 37 +- .../fluent_elasticsearch/vector/vector.yaml | 16 +- .../http_elasticsearch/lading/lading.yaml | 37 +- .../http_elasticsearch/vector/vector.yaml | 16 +- .../http_text_to_http_json/lading/lading.yaml | 37 +- .../http_text_to_http_json/vector/vector.yaml | 10 +- .../http_to_http_acks/lading/lading.yaml | 37 +- .../http_to_http_acks/vector/vector.yaml | 16 +- .../http_to_http_disk_buffer/experiment.yaml | 1 - .../lading/lading.yaml | 38 +- .../vector/vector.yaml | 12 +- .../http_to_http_json/lading/lading.yaml | 37 +- .../http_to_http_json/vector/vector.yaml | 16 +- .../http_to_http_noack/lading/lading.yaml | 37 +- .../http_to_http_noack/vector/vector.yaml | 16 +- .../cases/http_to_s3/lading/lading.yaml | 36 +- .../cases/http_to_s3/vector/vector.yaml | 18 +- .../otlp_grpc_to_blackhole/lading/lading.yaml | 37 +- .../otlp_grpc_to_blackhole/vector/vector.yaml | 10 +- .../otlp_http_to_blackhole/lading/lading.yaml | 37 +- .../otlp_http_to_blackhole/vector/vector.yaml | 10 +- .../lading/lading.yaml | 37 +- .../vector/vector.yaml | 14 +- .../lading/lading.yaml | 37 +- .../vector/vector.yaml | 10 +- .../splunk_hec_route_s3/lading/lading.yaml | 36 +- .../splunk_hec_route_s3/vector/vector.yaml | 36 +- .../lading/lading.yaml | 36 +- .../vector/vector.yaml | 14 +- .../lading/lading.yaml | 36 +- .../vector/vector.yaml | 14 +- .../lading/lading.yaml | 36 +- .../syslog_humio_logs/lading/lading.yaml | 36 +- .../syslog_humio_logs/vector/vector.yaml | 16 +- .../lading/lading.yaml | 36 +- .../vector/vector.yaml | 20 +- .../lading/lading.yaml | 36 +- .../vector/vector.yaml | 20 +- .../lading/lading.yaml | 36 +- .../vector/vector.yaml | 22 +- .../cases/syslog_loki/lading/lading.yaml | 37 +- .../cases/syslog_loki/vector/vector.yaml | 16 +- .../lading/lading.yaml | 36 +- .../vector/vector.yaml | 26 +- .../syslog_splunk_hec_logs/lading/lading.yaml | 36 +- .../syslog_splunk_hec_logs/vector/vector.yaml | 16 +- regression/config.yaml | 1 - scripts/environment/prepare.sh | 4 + tests/data/cmd/config/config_3.json | 8 +- tests/data/secret-backends/file-secrets.json | 2 +- tests/e2e/datadog-logs/config/compose.yaml | 22 +- tests/e2e/datadog-logs/config/test.yaml | 25 +- tests/e2e/datadog-logs/data/agent_only.yaml | 4 +- tests/e2e/datadog-logs/data/agent_vector.yaml | 4 +- tests/e2e/datadog-metrics/config/compose.yaml | 15 +- tests/e2e/datadog-metrics/config/test.yaml | 26 +- .../e2e/datadog-metrics/data/agent_only.yaml | 2 +- .../datadog-metrics/data/agent_vector.yaml | 2 +- .../opentelemetry-logs/config/compose.yaml | 6 +- tests/e2e/opentelemetry-logs/config/test.yaml | 10 +- .../data/collector-sink.yaml | 10 +- .../data/collector-source.yaml | 16 +- .../data/vector_default.yaml | 3 +- .../opentelemetry-metrics/config/compose.yaml | 6 +- .../opentelemetry-metrics/config/test.yaml | 8 +- .../data/collector-sink.yaml | 10 +- .../data/collector-source.yaml | 10 +- tests/integration/amqp/config/compose.yaml | 2 +- tests/integration/amqp/config/test.yaml | 18 +- tests/integration/appsignal/config/test.yaml | 8 +- tests/integration/aws/config/compose.yaml | 12 +- tests/integration/aws/config/test.yaml | 18 +- tests/integration/axiom/config/test.yaml | 10 +- tests/integration/azure/config/compose.yaml | 4 +- tests/integration/azure/config/test.yaml | 10 +- .../clickhouse/config/compose.yaml | 2 +- tests/integration/clickhouse/config/test.yaml | 12 +- .../integration/databend/config/compose.yaml | 2 +- tests/integration/databend/config/test.yaml | 12 +- .../datadog-agent/config/compose.yaml | 40 +- .../datadog-agent/config/test.yaml | 14 +- .../integration/datadog-logs/config/test.yaml | 14 +- .../datadog-metrics/config/test.yaml | 14 +- .../datadog-traces/config/compose.yaml | 46 +- .../datadog-traces/config/test.yaml | 18 +- tests/integration/dnstap/config/test.yaml | 14 +- .../integration/docker-logs/config/test.yaml | 12 +- tests/integration/doris/config/test.yaml | 16 +- .../elasticsearch/config/compose.yaml | 30 +- .../elasticsearch/config/test.yaml | 10 +- .../eventstoredb/config/compose.yaml | 4 +- .../integration/eventstoredb/config/test.yaml | 12 +- tests/integration/fluent/config/test.yaml | 10 +- tests/integration/gcp/config/compose.yaml | 14 +- tests/integration/gcp/config/invalidauth.json | 2 +- tests/integration/gcp/config/test.yaml | 20 +- .../greptimedb/config/compose.yaml | 2 +- tests/integration/greptimedb/config/test.yaml | 8 +- .../http-client/config/compose.yaml | 34 +- .../integration/http-client/config/test.yaml | 10 +- .../http-client/data/serve/logs/json.json | 2 +- .../data/serve/metrics/native.json | 2 +- .../http-client/data/serve/traces/native.json | 2 +- tests/integration/humio/config/compose.yaml | 2 +- tests/integration/humio/config/test.yaml | 10 +- .../integration/influxdb/config/compose.yaml | 16 +- tests/integration/influxdb/config/test.yaml | 14 +- tests/integration/kafka/config/compose.yaml | 48 +- tests/integration/kafka/config/test.yaml | 18 +- .../integration/logstash/config/compose.yaml | 10 +- tests/integration/logstash/config/test.yaml | 10 +- tests/integration/logstash/data/heartbeat.yml | 10 +- tests/integration/loki/config/compose.yaml | 2 +- tests/integration/loki/config/test.yaml | 12 +- tests/integration/mongodb/config/compose.yaml | 38 +- tests/integration/mongodb/config/test.yaml | 12 +- tests/integration/mqtt/config/compose.yaml | 4 +- tests/integration/mqtt/config/test.yaml | 14 +- tests/integration/nats/config/compose.yaml | 44 +- tests/integration/nginx/config/compose.yaml | 16 +- tests/integration/nginx/config/test.yaml | 12 +- .../opentelemetry/config/compose.yaml | 4 +- .../opentelemetry/config/test.yaml | 10 +- .../opentelemetry/data/config.yaml | 6 +- .../integration/postgres/config/compose.yaml | 12 +- tests/integration/postgres/config/test.yaml | 16 +- .../prometheus/config/compose.yaml | 20 +- tests/integration/prometheus/config/test.yaml | 20 +- .../prometheus/data/prometheus.yaml | 8 +- tests/integration/pulsar/config/compose.yaml | 2 +- tests/integration/pulsar/config/test.yaml | 12 +- tests/integration/redis/config/compose.yaml | 2 +- tests/integration/redis/config/test.yaml | 14 +- .../integration/shutdown/config/compose.yaml | 48 +- tests/integration/shutdown/config/test.yaml | 6 +- tests/integration/splunk/config/compose.yaml | 16 +- tests/integration/splunk/config/test.yaml | 16 +- tests/integration/webhdfs/config/compose.yaml | 2 +- tests/integration/webhdfs/config/test.yaml | 12 +- .../windows-event-log/config/test.yaml | 10 +- .../ignore-invalid/sinks/es_cluster.json | 6 +- .../namespacing/success/sinks/es_cluster.json | 6 +- vdev/src/commands/check/fmt.rs | 19 +- vdev/src/commands/fmt.rs | 22 +- website/.htmltest.external.yml | 16 +- website/.prettierrc.json | 17 - website/algolia.json | 33 +- website/assets/js/below.js | 16 +- website/assets/js/home.tsx | 346 +++-- website/assets/js/search.tsx | 209 ++- website/babel.config.js | 18 +- website/data/redirects.yaml | 132 +- website/postcss.config.js | 16 +- website/scripts/create-config-examples.js | 143 +- website/scripts/typesense-index.ts | 50 +- website/scripts/typesense-sync.ts | 13 +- website/typesense.config.json | 2 +- 226 files changed, 4028 insertions(+), 2779 deletions(-) create mode 100644 .prettierignore create mode 100644 .prettierrc.json delete mode 100644 website/.prettierrc.json diff --git a/.github/DISCUSSION_TEMPLATE/q-a.yml b/.github/DISCUSSION_TEMPLATE/q-a.yml index 23cb1f8467745..cc4942b0976ce 100644 --- a/.github/DISCUSSION_TEMPLATE/q-a.yml +++ b/.github/DISCUSSION_TEMPLATE/q-a.yml @@ -1,5 +1,5 @@ title: "Q&A" -labels: [ q-a ] +labels: [q-a] body: - type: markdown attributes: diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml index 47354ec1ba30e..ff1c7df5e83ca 100644 --- a/.github/ISSUE_TEMPLATE/bug.yml +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -1,94 +1,94 @@ name: Bug description: 🐛 Let us know about an unexpected error, a crash, or an incorrect behavior. -type: 'Bug' +type: "Bug" body: -- type: markdown - attributes: - value: | - Thank you for opening 🐛 bug report! + - type: markdown + attributes: + value: | + Thank you for opening 🐛 bug report! -- type: textarea - attributes: - label: A note for the community - value: | - - * Please vote on this issue by adding a 👍 [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to the original issue to help the community and maintainers prioritize this request - * If you are interested in working on this issue or have submitted a pull request, please leave a comment - + - type: textarea + attributes: + label: A note for the community + value: | + + * Please vote on this issue by adding a 👍 [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to the original issue to help the community and maintainers prioritize this request + * If you are interested in working on this issue or have submitted a pull request, please leave a comment + -- type: textarea - id: problem - attributes: - label: Problem - description: > - Please provide a clear and concise description of what the bug is, - including what currently happens and what you expected to happen. - validations: - required: true + - type: textarea + id: problem + attributes: + label: Problem + description: > + Please provide a clear and concise description of what the bug is, + including what currently happens and what you expected to happen. + validations: + required: true -- type: textarea - id: config - attributes: - label: Configuration - description: | - Paste the relevant parts of your Vector configuration file. + - type: textarea + id: config + attributes: + label: Configuration + description: | + Paste the relevant parts of your Vector configuration file. - !! If your config files contain sensitive information please remove it !! - render: text + !! If your config files contain sensitive information please remove it !! + render: text -- type: input - id: version - attributes: - label: Version - description: | - Please paste the output of running `vector --version`. + - type: input + id: version + attributes: + label: Version + description: | + Please paste the output of running `vector --version`. - If you are not running the latest version of Vector, please try upgrading - because your issue may have already been fixed. - validations: - required: true + If you are not running the latest version of Vector, please try upgrading + because your issue may have already been fixed. + validations: + required: true -- type: textarea - id: debug - attributes: - label: Debug Output - description: | - Full debug output can be obtained by running Vector with the following: + - type: textarea + id: debug + attributes: + label: Debug Output + description: | + Full debug output can be obtained by running Vector with the following: - ``` - RUST_BACKTRACE=full vector -vvv - ``` + ``` + RUST_BACKTRACE=full vector -vvv + ``` - If the debug output is long, please create a GitHub Gist containing the debug output and paste the link here. + If the debug output is long, please create a GitHub Gist containing the debug output and paste the link here. - !! Debug output may contain sensitive information. Please review it before posting publicly. !! - render: text + !! Debug output may contain sensitive information. Please review it before posting publicly. !! + render: text -- type: textarea - id: data - attributes: - label: Example Data - description: | - Please provide any example data that will help debug the issue, for example: + - type: textarea + id: data + attributes: + label: Example Data + description: | + Please provide any example data that will help debug the issue, for example: - ``` - 201.69.207.46 - kemmer6752 [07/06/2019:14:53:55 -0400] "PATCH /innovative/interfaces" 301 669 - ``` + ``` + 201.69.207.46 - kemmer6752 [07/06/2019:14:53:55 -0400] "PATCH /innovative/interfaces" 301 669 + ``` -- type: textarea - id: context - attributes: - label: Additional Context - description: | - Is there anything atypical about your situation that we should know? For - example: is Vector running in Kubernetes? Are you passing any unusual command - line options or environment variables to opt-in to non-default behavior? + - type: textarea + id: context + attributes: + label: Additional Context + description: | + Is there anything atypical about your situation that we should know? For + example: is Vector running in Kubernetes? Are you passing any unusual command + line options or environment variables to opt-in to non-default behavior? -- type: textarea - id: references - attributes: - label: References - description: | - Are there any other GitHub issues (open or closed) or Pull Requests that should be linked here? For example: + - type: textarea + id: references + attributes: + label: References + description: | + Are there any other GitHub issues (open or closed) or Pull Requests that should be linked here? For example: - - #6017 + - #6017 diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index ea44ff1b06562..df12e12af4f76 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -12,4 +12,3 @@ contact_links: - name: Twitter url: https://twitter.com/vectordotdev about: Follow us and stay up to date with Vector. - diff --git a/.github/ISSUE_TEMPLATE/feature.yml b/.github/ISSUE_TEMPLATE/feature.yml index f2e066bb37303..8b7a6fc139185 100644 --- a/.github/ISSUE_TEMPLATE/feature.yml +++ b/.github/ISSUE_TEMPLATE/feature.yml @@ -1,87 +1,86 @@ name: Feature -type: 'Feature' +type: "Feature" description: 🚀 Suggest a new feature. body: -- type: markdown - attributes: - value: | - Thank you for opening 🚀 feature request! - - For general questions about Vector usage, please see - https://discussions.vector.dev. - -- type: textarea - attributes: - label: A note for the community - value: | - - * Please vote on this issue by adding a 👍 [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to the original issue to help the community and maintainers prioritize this request - * If you are interested in working on this issue or have submitted a pull request, please leave a comment - - -- type: textarea - id: use-cases - attributes: - label: Use Cases - description: | - In order to properly evaluate a feature request, it is necessary to - understand the use-cases for it. - - Please describe below the _end goal_ you are trying to achieve that has - led you to request this feature. - - Please keep this section focused on the problem and not on the suggested - solution. We'll get to that in a moment, below! - - -- type: textarea - id: attempted-solutions - attributes: - label: Attempted Solutions - description: | - If you've already tried to solve the problem within Vector's existing - features and found a limitation that prevented you from succeeding, please - describe it below in as much detail as possible. - - Ideally, this would include real configuration snippets that you tried - and what results you got in each case. - - Please remove any sensitive information such as passwords before sharing - configuration snippets and command lines. - -- type: textarea - id: proposal - attributes: - label: Proposal - description: | - If you have an idea for a way to address the problem via a change to - Vector features, please describe it below. - - In this section, it's helpful to include specific examples of how what - you are suggesting might look in configuration files, or on the command line, - since that allows us to understand the full picture of what you are proposing. - - If you're not sure of some details, don't worry! When we evaluate the - feature request we may suggest modifications as necessary to work within the - design constraints of Vector. - -- type: textarea - id: references - attributes: - label: References - description: | - Are there any other GitHub issues, whether open or closed, that are - related to the problem you've described above or to the suggested solution? If - so, please create a list below that mentions each of them. For example: - - - #7023 - -- type: input - id: version - attributes: - label: Version - description: | - Please paste the output of running `vector --version`. - - This will record which version was current at the time of your feature request, - to help manage the request backlog. + - type: markdown + attributes: + value: | + Thank you for opening 🚀 feature request! + + For general questions about Vector usage, please see + https://discussions.vector.dev. + + - type: textarea + attributes: + label: A note for the community + value: | + + * Please vote on this issue by adding a 👍 [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to the original issue to help the community and maintainers prioritize this request + * If you are interested in working on this issue or have submitted a pull request, please leave a comment + + + - type: textarea + id: use-cases + attributes: + label: Use Cases + description: | + In order to properly evaluate a feature request, it is necessary to + understand the use-cases for it. + + Please describe below the _end goal_ you are trying to achieve that has + led you to request this feature. + + Please keep this section focused on the problem and not on the suggested + solution. We'll get to that in a moment, below! + + - type: textarea + id: attempted-solutions + attributes: + label: Attempted Solutions + description: | + If you've already tried to solve the problem within Vector's existing + features and found a limitation that prevented you from succeeding, please + describe it below in as much detail as possible. + + Ideally, this would include real configuration snippets that you tried + and what results you got in each case. + + Please remove any sensitive information such as passwords before sharing + configuration snippets and command lines. + + - type: textarea + id: proposal + attributes: + label: Proposal + description: | + If you have an idea for a way to address the problem via a change to + Vector features, please describe it below. + + In this section, it's helpful to include specific examples of how what + you are suggesting might look in configuration files, or on the command line, + since that allows us to understand the full picture of what you are proposing. + + If you're not sure of some details, don't worry! When we evaluate the + feature request we may suggest modifications as necessary to work within the + design constraints of Vector. + + - type: textarea + id: references + attributes: + label: References + description: | + Are there any other GitHub issues, whether open or closed, that are + related to the problem you've described above or to the suggested solution? If + so, please create a list below that mentions each of them. For example: + + - #7023 + + - type: input + id: version + attributes: + label: Version + description: | + Please paste the output of running `vector --version`. + + This will record which version was current at the time of your feature request, + to help manage the request backlog. diff --git a/.github/actions/install-vdev/action.yml b/.github/actions/install-vdev/action.yml index da04417f67358..e98d9731f2a5d 100644 --- a/.github/actions/install-vdev/action.yml +++ b/.github/actions/install-vdev/action.yml @@ -9,7 +9,7 @@ inputs: skip-cache: description: "Skip cache lookup and force compilation" required: false - default: 'false' + default: "false" runs: using: "composite" diff --git a/.github/actions/pull-test-runner/action.yml b/.github/actions/pull-test-runner/action.yml index b46a47fde93e8..8cded209f8a6f 100644 --- a/.github/actions/pull-test-runner/action.yml +++ b/.github/actions/pull-test-runner/action.yml @@ -3,10 +3,10 @@ description: Login to GHCR and pull the pre-built test runner image for integrat inputs: github_token: - description: 'GitHub token for GHCR authentication' + description: "GitHub token for GHCR authentication" required: true commit_sha: - description: 'Commit SHA used to tag the test runner image' + description: "Commit SHA used to tag the test runner image" required: true runs: diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index f9037aebb68e8..fde57a4b33303 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -80,6 +80,10 @@ inputs: required: false default: false description: "Install markdownlint-cli2 (npm)." + prettier: + required: false + default: false + description: "Install prettier (npm)." datadog-ci: required: false default: false @@ -140,7 +144,6 @@ runs: echo "CARGO_TERM_COLOR=always" >> "$GITHUB_ENV" fi - - name: Enable rust matcher if: ${{ env.NEEDS_RUST == 'true' }} run: echo "::add-matcher::.github/matchers/rust.json" @@ -268,6 +271,7 @@ runs: ~/.cargo/bin/dd-rust-license-tool ~/.cargo/bin/wasm-pack /usr/local/bin/markdownlint-cli2 + /usr/local/bin/prettier /usr/local/bin/datadog-ci key: ${{ runner.os }}-prepare-binaries-${{ hashFiles('scripts/environment/*') }} restore-keys: | @@ -288,6 +292,7 @@ runs: [[ "${{ inputs.dd-rust-license-tool }}" == "true" ]] && mods+=("dd-rust-license-tool") [[ "${{ inputs.wasm-pack }}" == "true" ]] && mods+=("wasm-pack") [[ "${{ inputs.markdownlint-cli2 }}" == "true" ]] && mods+=("markdownlint-cli2") + [[ "${{ inputs.prettier }}" == "true" ]] && mods+=("prettier") [[ "${{ inputs.datadog-ci }}" == "true" ]] && mods+=("datadog-ci") csm=$(IFS=,; echo "${mods[*]}") diff --git a/.github/dependabot.yml b/.github/dependabot.yml index fd32d14deed15..609da972baa98 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -15,51 +15,51 @@ updates: patches: applies-to: version-updates patterns: - - "*" + - "*" update-types: - - "patch" + - "patch" amq: patterns: - - "amq-*" + - "amq-*" aws: patterns: - - "aws-*" + - "aws-*" aws-security: applies-to: security-updates patterns: - - "aws-*" + - "aws-*" azure: patterns: - - "azure_*" + - "azure_*" clap: patterns: - - "clap*" + - "clap*" crossbeam: patterns: - - "crossbeam*" + - "crossbeam*" csv: patterns: - - "csv*" + - "csv*" futures: patterns: - - "futures" - - "futures-util" + - "futures" + - "futures-util" metrics: patterns: - - "metrics" - - "metrics-tracing-context" - - "metrics-util" + - "metrics" + - "metrics-tracing-context" + - "metrics-util" netlink: patterns: - - "netlink-*" + - "netlink-*" phf: patterns: - - "phf*" + - "phf*" prost: patterns: - - "prost" - - "prost-*" + - "prost" + - "prost-*" serde: patterns: - "serde" @@ -70,22 +70,22 @@ updates: - "tokio-*" tonic: patterns: - - "tonic" - - "tonic-*" + - "tonic" + - "tonic-*" tower: patterns: - - "tower" - - "tower-*" + - "tower" + - "tower-*" tracing: patterns: - "tracing" - "tracing-*" wasm-bindgen: patterns: - - "wasm-bindgen-*" + - "wasm-bindgen-*" zstd: patterns: - - "zstd*" + - "zstd*" - package-ecosystem: "docker" directory: "/distribution/docker/alpine" schedule: @@ -158,8 +158,8 @@ updates: groups: artifact: patterns: - - "actions/download-artifact" - - "actions/upload-artifact" + - "actions/download-artifact" + - "actions/upload-artifact" - package-ecosystem: "npm" directory: "website/" schedule: diff --git a/.github/labeler.yml b/.github/labeler.yml index b5fe9e5e4eede..687dea74ce164 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -1,48 +1,48 @@ # Generic component-level labels and high-level groupings. "domain: sources": - changed-files: - - any-glob-to-any-file: ["src/sources/**/*"] + - any-glob-to-any-file: ["src/sources/**/*"] "domain: sinks": - changed-files: - - any-glob-to-any-file: ["src/sinks/**/*"] + - any-glob-to-any-file: ["src/sinks/**/*"] "domain: transforms": - changed-files: - - any-glob-to-any-file: ["src/transforms/**/*"] + - any-glob-to-any-file: ["src/transforms/**/*"] "domain: topology": - changed-files: - - any-glob-to-any-file: ["src/topology/**/*"] + - any-glob-to-any-file: ["src/topology/**/*"] "domain: codecs": - changed-files: - - any-glob-to-any-file: ["src/codecs/**/*"] + - any-glob-to-any-file: ["src/codecs/**/*"] "domain: core": - changed-files: - - any-glob-to-any-file: ["lib/vector-core/**/*"] + - any-glob-to-any-file: ["lib/vector-core/**/*"] "domain: vrl": - changed-files: - - any-glob-to-any-file: ["lib/vrl/**/*"] + - any-glob-to-any-file: ["lib/vrl/**/*"] "domain: ci": - changed-files: - - any-glob-to-any-file: [".github/workflows/*", ".github/actions/**/*", ".github/*.yml"] + - any-glob-to-any-file: [".github/workflows/*", ".github/actions/**/*", ".github/*.yml"] "domain: vdev": - changed-files: - - any-glob-to-any-file: ["vdev/**/*"] + - any-glob-to-any-file: ["vdev/**/*"] "domain: releasing": - changed-files: - - any-glob-to-any-file: ["distribution/**/*", "scripts/package-*", "scripts/release-*"] + - any-glob-to-any-file: ["distribution/**/*", "scripts/package-*", "scripts/release-*"] "domain: rfc": - changed-files: - - any-glob-to-any-file: ["rfcs/**/*"] + - any-glob-to-any-file: ["rfcs/**/*"] "domain: external docs": - changed-files: - - any-glob-to-any-file: ["website/cue/**/*"] + - any-glob-to-any-file: ["website/cue/**/*"] diff --git a/.github/workflows/build-test-runner.yml b/.github/workflows/build-test-runner.yml index 7a7596651ed2c..ca8e51b0b6944 100644 --- a/.github/workflows/build-test-runner.yml +++ b/.github/workflows/build-test-runner.yml @@ -9,11 +9,11 @@ on: workflow_call: inputs: commit_sha: - description: 'Commit SHA to use for image tag' + description: "Commit SHA to use for image tag" required: true type: string checkout_ref: - description: 'Git ref to checkout (defaults to commit_sha)' + description: "Git ref to checkout (defaults to commit_sha)" required: false type: string @@ -24,8 +24,8 @@ jobs: build: runs-on: ubuntu-24.04 permissions: - contents: read # Required by actions/checkout - packages: write # Required to push test runner image to GitHub Container Registry + contents: read # Required by actions/checkout + packages: write # Required to push test runner image to GitHub Container Registry steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: diff --git a/.github/workflows/build_preview_sites.yml b/.github/workflows/build_preview_sites.yml index 9df10ce781379..829571deda5da 100644 --- a/.github/workflows/build_preview_sites.yml +++ b/.github/workflows/build_preview_sites.yml @@ -7,17 +7,17 @@ on: - completed permissions: - contents: read # Restrictive default + contents: read # Restrictive default jobs: deploy_vector_preview_site: if: ${{ github.event.workflow_run.conclusion == 'success' && contains(github.event.workflow_run.head_branch, 'website') }} permissions: - contents: read # Required by the reusable workflow - actions: read # Required to download artifacts - issues: write # Required to post preview link comments - pull-requests: write # Required to post preview link comments - statuses: write # Required to update commit status + contents: read # Required by the reusable workflow + actions: read # Required to download artifacts + issues: write # Required to post preview link comments + pull-requests: write # Required to post preview link comments + statuses: write # Required to update commit status uses: ./.github/workflows/create_preview_sites.yml with: APP_ID: "d1a7j77663uxsc" @@ -30,11 +30,11 @@ jobs: deploy_rust_doc_preview_site: if: ${{ github.event.workflow_run.conclusion == 'success' && contains(github.event.workflow_run.head_branch, 'website') }} permissions: - contents: read # Required by the reusable workflow - actions: read # Required to download artifacts - issues: write # Required to post preview link comments - pull-requests: write # Required to post preview link comments - statuses: write # Required to update commit status + contents: read # Required by the reusable workflow + actions: read # Required to download artifacts + issues: write # Required to post preview link comments + pull-requests: write # Required to post preview link comments + statuses: write # Required to update commit status uses: ./.github/workflows/create_preview_sites.yml with: APP_ID: "d1hoyoksbulg25" @@ -47,11 +47,11 @@ jobs: deploy_vrl_playground_preview_site: if: ${{ github.event.workflow_run.conclusion == 'success' && contains(github.event.workflow_run.head_branch, 'website') }} permissions: - contents: read # Required by the reusable workflow - actions: read # Required to download artifacts - issues: write # Required to post preview link comments - pull-requests: write # Required to post preview link comments - statuses: write # Required to update commit status + contents: read # Required by the reusable workflow + actions: read # Required to download artifacts + issues: write # Required to post preview link comments + pull-requests: write # Required to post preview link comments + statuses: write # Required to update commit status uses: ./.github/workflows/create_preview_sites.yml with: APP_ID: "d2lr4eds605rpz" diff --git a/.github/workflows/changes.yml b/.github/workflows/changes.yml index 25599942b02c9..999548111e9a7 100644 --- a/.github/workflows/changes.yml +++ b/.github/workflows/changes.yml @@ -48,6 +48,8 @@ on: value: ${{ jobs.source.outputs.website_only }} install: value: ${{ jobs.source.outputs.install }} + prettier: + value: ${{ jobs.source.outputs.prettier }} test-yml: value: ${{ jobs.source.outputs.test-yml }} integration-yml: @@ -152,7 +154,7 @@ on: # Workflow-level permissions - read access to repository contents permissions: - contents: read # Required to checkout code + contents: read # Required to checkout code env: BASE_SHA: ${{ inputs.base_ref || (github.event_name == 'merge_group' && github.event.merge_group.base_sha) || github.event.pull_request.base.sha }} @@ -176,6 +178,7 @@ jobs: install: ${{ steps.filter.outputs.install }} k8s: ${{ steps.filter.outputs.k8s }} website_only: ${{ steps.filter.outputs.website == 'true' && steps.filter.outputs.not_website == 'false' }} + prettier: ${{ steps.filter.outputs.prettier }} test-yml: ${{ steps.filter.outputs.test-yml }} integration-yml: ${{ steps.filter.outputs.integration-yml }} test-make-command-yml: ${{ steps.filter.outputs.test-make-command-yml }} @@ -184,109 +187,118 @@ jobs: unit_mac-yml: ${{ steps.filter.outputs.unit_mac-yml }} unit_windows-yml: ${{ steps.filter.outputs.unit_windows-yml }} steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 - id: filter - with: - base: ${{ env.BASE_SHA }} - ref: ${{ env.HEAD_SHA }} - filters: | - source: - - ".cargo/**" - - "benches/**" - - "lib/**" - - "proto/**" - - "scripts/**" - - "src/**" - - "tests/**" - - "build.rs" - - "Cargo.lock" - - "Cargo.toml" - - "Makefile" - - "rust-toolchain.toml" - - "vdev/**" - - ".github/workflows/changes.yml" - dependencies: - - ".cargo/**" - - 'Cargo.toml' - - 'Cargo.lock' - - 'rust-toolchain.toml' - - 'Makefile' - - 'scripts/cross/**' - - "vdev/**" - - ".github/workflows/changes.yml" - deny: - - '**/Cargo.toml' - - 'Cargo.lock' - - ".github/workflows/deny.yml" - cue: - - 'website/cue/**' - - "vdev/**" - - ".github/workflows/changes.yml" - component_docs: - - 'scripts/generate-component-docs.rb' - - "vdev/**" - - 'website/cue/**/*.cue' - - 'docs/generated/**' - # If changes to the VRL sha is made the combined generated cue file will change which - # may cause issues - - 'Cargo.lock' - - ".github/workflows/changes.yml" - markdown: - - '**/**.md' - - "vdev/**" - - ".github/workflows/changes.yml" - scripts: - - '**/**.sh' - - ".github/workflows/changes.yml" - internal_events: - - 'src/internal_events/**' - - "vdev/**" - - ".github/workflows/changes.yml" - docker: - - 'distribution/docker/**' - - "vdev/**" - - ".github/workflows/changes.yml" - install: - - ".github/workflows/install-sh.yml" - - "distribution/install.sh" - - ".github/workflows/changes.yml" - k8s: - - "src/sources/kubernetes_logs/**" - - "lib/k8s-e2e-tests/**" - - "lib/k8s-test-framework/**" - - "scripts/test-e2e-kubernetes.sh" - - "scripts/build-docker.sh" - - "scripts/deploy-chart-test.sh" - - ".github/workflows/k8s_e2e.yml" - - ".github/workflows/changes.yml" - website: - - "website/**" - not_website: - - "!website/**" - test-yml: - - ".github/workflows/test.yml" - - ".github/workflows/changes.yml" - integration-yml: - - ".github/workflows/integration.yml" - - ".github/workflows/changes.yml" - test-make-command-yml: - - ".github/workflows/test-make-command.yml" - - ".github/workflows/changes.yml" - msrv-yml: - - ".github/workflows/msrv.yml" - - ".github/workflows/changes.yml" - cross-yml: - - ".github/workflows/cross.yml" - - ".github/workflows/changes.yml" - unit_mac-yml: - - ".github/workflows/unit_mac.yml" - - ".github/workflows/changes.yml" - unit_windows-yml: - - ".github/workflows/unit_windows.yml" - - ".github/workflows/changes.yml" + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + id: filter + with: + base: ${{ env.BASE_SHA }} + ref: ${{ env.HEAD_SHA }} + filters: | + source: + - ".cargo/**" + - "benches/**" + - "lib/**" + - "proto/**" + - "scripts/**" + - "src/**" + - "tests/**" + - "build.rs" + - "Cargo.lock" + - "Cargo.toml" + - "Makefile" + - "rust-toolchain.toml" + - "vdev/**" + - ".github/workflows/changes.yml" + dependencies: + - ".cargo/**" + - 'Cargo.toml' + - 'Cargo.lock' + - 'rust-toolchain.toml' + - 'Makefile' + - 'scripts/cross/**' + - "vdev/**" + - ".github/workflows/changes.yml" + deny: + - '**/Cargo.toml' + - 'Cargo.lock' + - ".github/workflows/deny.yml" + cue: + - 'website/cue/**' + - "vdev/**" + - ".github/workflows/changes.yml" + component_docs: + - 'scripts/generate-component-docs.rb' + - "vdev/**" + - 'website/cue/**/*.cue' + - 'docs/generated/**' + # If changes to the VRL sha is made the combined generated cue file will change which + # may cause issues + - 'Cargo.lock' + - ".github/workflows/changes.yml" + markdown: + - '**/**.md' + - "vdev/**" + - ".github/workflows/changes.yml" + scripts: + - '**/**.sh' + - ".github/workflows/changes.yml" + internal_events: + - 'src/internal_events/**' + - "vdev/**" + - ".github/workflows/changes.yml" + docker: + - 'distribution/docker/**' + - "vdev/**" + - ".github/workflows/changes.yml" + install: + - ".github/workflows/install-sh.yml" + - "distribution/install.sh" + - ".github/workflows/changes.yml" + k8s: + - "src/sources/kubernetes_logs/**" + - "lib/k8s-e2e-tests/**" + - "lib/k8s-test-framework/**" + - "scripts/test-e2e-kubernetes.sh" + - "scripts/build-docker.sh" + - "scripts/deploy-chart-test.sh" + - ".github/workflows/k8s_e2e.yml" + - ".github/workflows/changes.yml" + website: + - "website/**" + not_website: + - "!website/**" + prettier: + - "**/*.yml" + - "**/*.yaml" + - "**/*.js" + - "**/*.ts" + - "**/*.tsx" + - "**/*.json" + - ".prettierrc.json" + - ".prettierignore" + - ".github/workflows/changes.yml" + test-yml: + - ".github/workflows/test.yml" + - ".github/workflows/changes.yml" + integration-yml: + - ".github/workflows/integration.yml" + - ".github/workflows/changes.yml" + test-make-command-yml: + - ".github/workflows/test-make-command.yml" + - ".github/workflows/changes.yml" + msrv-yml: + - ".github/workflows/msrv.yml" + - ".github/workflows/changes.yml" + cross-yml: + - ".github/workflows/cross.yml" + - ".github/workflows/changes.yml" + unit_mac-yml: + - ".github/workflows/unit_mac.yml" + - ".github/workflows/changes.yml" + unit_windows-yml: + - ".github/workflows/unit_windows.yml" + - ".github/workflows/changes.yml" # Detects changes that are specific to integration tests int_tests: @@ -352,7 +364,7 @@ jobs: id: filter with: base: ${{ env.BASE_SHA }} - ref: ${{ env.HEAD_SHA }} + ref: ${{ env.HEAD_SHA }} filters: int_test_filters.yaml # This JSON hack was introduced because GitHub Actions does not support dynamic expressions in the @@ -446,7 +458,7 @@ jobs: id: filter with: base: ${{ env.BASE_SHA }} - ref: ${{ env.HEAD_SHA }} + ref: ${{ env.HEAD_SHA }} filters: int_test_filters.yaml # This JSON hack was introduced because GitHub Actions does not support dynamic expressions in the diff --git a/.github/workflows/ci-integration-review.yml b/.github/workflows/ci-integration-review.yml index 029ea40e05963..67fae9ea864d7 100644 --- a/.github/workflows/ci-integration-review.yml +++ b/.github/workflows/ci-integration-review.yml @@ -30,7 +30,7 @@ name: CI Integration Review Trigger on: pull_request_review: - types: [ submitted ] + types: [submitted] permissions: contents: read @@ -55,7 +55,7 @@ jobs: timeout-minutes: 5 if: startsWith(github.event.review.body, '/ci-run-integration') || startsWith(github.event.review.body, '/ci-run-e2e') || contains(github.event.review.body, '/ci-run-all') permissions: - statuses: write # Required to set commit status to pending + statuses: write # Required to set commit status to pending steps: - name: Generate authentication token id: generate_token @@ -68,7 +68,7 @@ jobs: uses: tspascoal/get-user-teams-membership@57e9f42acd78f4d0f496b3be4368fc5f62696662 # v3.0.0 with: username: ${{ github.actor }} - team: 'Vector' + team: "Vector" GITHUB_TOKEN: ${{ steps.generate_token.outputs.token }} - name: Validate author membership @@ -86,7 +86,7 @@ jobs: needs: prep-pr permissions: contents: read - packages: write # Required to push test runner image to GHCR + packages: write # Required to push test runner image to GHCR uses: ./.github/workflows/build-test-runner.yml with: commit_sha: ${{ github.event.review.commit_id }} @@ -100,17 +100,47 @@ jobs: timeout-minutes: 90 permissions: contents: read - packages: read # Required to pull test runner image from GHCR + packages: read # Required to pull test runner image from GHCR strategy: fail-fast: false matrix: - service: [ - "amqp", "appsignal", "axiom", "aws", "azure", "clickhouse", "databend", "datadog-agent", - "datadog-logs", "datadog-metrics", "datadog-traces", "dnstap", "docker-logs", "doris", "elasticsearch", - "eventstoredb", "fluent", "gcp", "greptimedb", "http-client", "influxdb", "kafka", "logstash", - "loki", "mqtt", "mongodb", "nats", "nginx", "opentelemetry", "postgres", "prometheus", "pulsar", - "redis", "webhdfs" - ] + service: + [ + "amqp", + "appsignal", + "axiom", + "aws", + "azure", + "clickhouse", + "databend", + "datadog-agent", + "datadog-logs", + "datadog-metrics", + "datadog-traces", + "dnstap", + "docker-logs", + "doris", + "elasticsearch", + "eventstoredb", + "fluent", + "gcp", + "greptimedb", + "http-client", + "influxdb", + "kafka", + "logstash", + "loki", + "mqtt", + "mongodb", + "nats", + "nginx", + "opentelemetry", + "postgres", + "prometheus", + "pulsar", + "redis", + "webhdfs" + ] steps: - name: Evaluate run condition id: run_condition @@ -160,13 +190,11 @@ jobs: timeout-minutes: 30 permissions: contents: read - packages: read # Required to pull test runner image from GHCR + packages: read # Required to pull test runner image from GHCR strategy: fail-fast: false matrix: - service: [ - "datadog-logs", "datadog-metrics", "opentelemetry-logs", "opentelemetry-metrics" - ] + service: ["datadog-logs", "datadog-metrics", "opentelemetry-logs", "opentelemetry-metrics"] steps: - name: Evaluate run condition id: run_condition @@ -208,7 +236,6 @@ jobs: bash scripts/environment/prepare.sh --modules=datadog-ci bash scripts/run-integration-test.sh e2e ${{ matrix.service }} - update-pr-status: name: Signal result to PR runs-on: ubuntu-24.04 @@ -218,7 +245,7 @@ jobs: - e2e-tests if: always() && (startsWith(github.event.review.body, '/ci-run-integration') || contains(github.event.review.body, '/ci-run-all') || contains(github.event.review.body, '/ci-run-e2e')) permissions: - statuses: write # Required to set final commit status + statuses: write # Required to set final commit status env: FAILED: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }} steps: @@ -234,7 +261,7 @@ jobs: uses: tspascoal/get-user-teams-membership@57e9f42acd78f4d0f496b3be4368fc5f62696662 # v3.0.0 with: username: ${{ github.actor }} - team: 'Vector' + team: "Vector" GITHUB_TOKEN: ${{ steps.generate_token.outputs.token }} - name: (PR review) Submit PR result as success @@ -243,7 +270,7 @@ jobs: with: sha: ${{ github.event.review.commit_id }} token: ${{ secrets.GITHUB_TOKEN }} - status: 'success' + status: "success" - name: Check all jobs status run: | diff --git a/.github/workflows/ci-review-trigger.yml b/.github/workflows/ci-review-trigger.yml index ce4e85f392b35..8a2eb206efdcd 100644 --- a/.github/workflows/ci-review-trigger.yml +++ b/.github/workflows/ci-review-trigger.yml @@ -54,19 +54,19 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 5 if: | - startsWith(github.event.review.body, '/ci-run-all') - || startsWith(github.event.review.body, '/ci-run-cli') - || startsWith(github.event.review.body, '/ci-run-test-vector-api') - || startsWith(github.event.review.body, '/ci-run-test-behavior') - || startsWith(github.event.review.body, '/ci-run-check-examples') - || startsWith(github.event.review.body, '/ci-run-test-docs') - || startsWith(github.event.review.body, '/ci-run-deny') - || startsWith(github.event.review.body, '/ci-run-component-features') - || startsWith(github.event.review.body, '/ci-run-cross') - || startsWith(github.event.review.body, '/ci-run-unit-mac') - || startsWith(github.event.review.body, '/ci-run-unit-windows') - || startsWith(github.event.review.body, '/ci-run-environment') - || startsWith(github.event.review.body, '/ci-run-k8s') + startsWith(github.event.review.body, '/ci-run-all') + || startsWith(github.event.review.body, '/ci-run-cli') + || startsWith(github.event.review.body, '/ci-run-test-vector-api') + || startsWith(github.event.review.body, '/ci-run-test-behavior') + || startsWith(github.event.review.body, '/ci-run-check-examples') + || startsWith(github.event.review.body, '/ci-run-test-docs') + || startsWith(github.event.review.body, '/ci-run-deny') + || startsWith(github.event.review.body, '/ci-run-component-features') + || startsWith(github.event.review.body, '/ci-run-cross') + || startsWith(github.event.review.body, '/ci-run-unit-mac') + || startsWith(github.event.review.body, '/ci-run-unit-windows') + || startsWith(github.event.review.body, '/ci-run-environment') + || startsWith(github.event.review.body, '/ci-run-k8s') steps: - name: Generate authentication token id: generate_token @@ -79,7 +79,7 @@ jobs: uses: tspascoal/get-user-teams-membership@57e9f42acd78f4d0f496b3be4368fc5f62696662 # v3.0.0 with: username: ${{ github.actor }} - team: 'Vector' + team: "Vector" GITHUB_TOKEN: ${{ steps.generate_token.outputs.token }} - name: Validate author membership diff --git a/.github/workflows/component_features.yml b/.github/workflows/component_features.yml index 9770b4ab81383..5803451f1dd6c 100644 --- a/.github/workflows/component_features.yml +++ b/.github/workflows/component_features.yml @@ -13,17 +13,17 @@ on: workflow_call: inputs: ref: - description: 'Git ref to checkout' + description: "Git ref to checkout" required: false type: string workflow_dispatch: inputs: ref: - description: 'Git ref to checkout' + description: "Git ref to checkout" required: false type: string schedule: - - cron: '0 10 * * 1' # 10:00 UTC Monday (6 AM EST NYC) + - cron: "0 10 * * 1" # 10:00 UTC Monday (6 AM EST NYC) permissions: contents: read diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index a4a4c39d4f760..db19d27ffd582 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -4,14 +4,14 @@ on: push: branches: - master - workflow_dispatch: # Allow manual trigger from GitHub UI + workflow_dispatch: # Allow manual trigger from GitHub UI permissions: contents: read env: CI: true - CARGO_INCREMENTAL: '0' # Disable incremental compilation for coverage + CARGO_INCREMENTAL: "0" # Disable incremental compilation for coverage jobs: coverage: diff --git a/.github/workflows/create_preview_sites.yml b/.github/workflows/create_preview_sites.yml index dbb93e6432a33..6cd7edf89bdb9 100644 --- a/.github/workflows/create_preview_sites.yml +++ b/.github/workflows/create_preview_sites.yml @@ -23,18 +23,18 @@ on: required: true permissions: - contents: read # Restrictive default + contents: read # Restrictive default jobs: create_preview_site: runs-on: ubuntu-24.04 timeout-minutes: 5 permissions: - contents: read # Required for repository context - actions: read # Required to download artifacts - issues: write # Required to post preview link comments - pull-requests: write # Required to post preview link comments - statuses: write # Required to update commit status + contents: read # Required for repository context + actions: read # Required to download artifacts + issues: write # Required to post preview link comments + pull-requests: write # Required to post preview link comments + statuses: write # Required to update commit status steps: # Get the artifacts with the PR number and branch name - name: Download artifact diff --git a/.github/workflows/cross.yml b/.github/workflows/cross.yml index 9519a7d79df94..b554a45d45c09 100644 --- a/.github/workflows/cross.yml +++ b/.github/workflows/cross.yml @@ -4,7 +4,7 @@ on: workflow_call: inputs: ref: - description: 'Git ref to checkout' + description: "Git ref to checkout" required: false type: string diff --git a/.github/workflows/custom_builds.yml b/.github/workflows/custom_builds.yml index 98a7b3647295f..60e1bc8dc6bed 100644 --- a/.github/workflows/custom_builds.yml +++ b/.github/workflows/custom_builds.yml @@ -1,7 +1,7 @@ name: Custom Builds permissions: - contents: read # Restrictive default + contents: read # Restrictive default on: workflow_dispatch: {} @@ -9,8 +9,8 @@ on: jobs: Custom: permissions: - contents: write # Required to create/update releases and tags - packages: write # Required to push container images to GHCR + contents: write # Required to create/update releases and tags + packages: write # Required to push container images to GHCR uses: ./.github/workflows/publish.yml with: git_ref: ${{ github.ref }} diff --git a/.github/workflows/deny.yml b/.github/workflows/deny.yml index 52b40d1e1d535..cef811743e716 100644 --- a/.github/workflows/deny.yml +++ b/.github/workflows/deny.yml @@ -14,14 +14,14 @@ on: workflow_call: inputs: ref: - description: 'Git ref to checkout' + description: "Git ref to checkout" required: false type: string workflow_dispatch: inputs: ref: - description: 'Git ref to checkout' + description: "Git ref to checkout" required: false type: string @@ -30,7 +30,7 @@ on: types: [checks_requested] schedule: # Same schedule as nightly.yml - - cron: "0 5 * * 2-6" # Runs at 5:00 AM UTC, Tuesday through Saturday + - cron: "0 5 * * 2-6" # Runs at 5:00 AM UTC, Tuesday through Saturday concurrency: group: ${{ github.workflow }}-${{ github.event.number || github.sha }} diff --git a/.github/workflows/environment.yml b/.github/workflows/environment.yml index cf6313e4ce894..de519e0cbc955 100644 --- a/.github/workflows/environment.yml +++ b/.github/workflows/environment.yml @@ -4,7 +4,7 @@ on: workflow_call: inputs: ref: - description: 'Git ref to checkout' + description: "Git ref to checkout" required: false type: string @@ -15,12 +15,12 @@ on: type: boolean default: false ref: - description: 'Git ref to checkout' + description: "Git ref to checkout" required: false type: string schedule: - - cron: '0 6 * * 1' # Every Monday at 06:00 UTC + - cron: "0 6 * * 1" # Every Monday at 06:00 UTC env: VERBOSE: true diff --git a/.github/workflows/install-sh.yml b/.github/workflows/install-sh.yml index 69777ead18d83..7d371baa27388 100644 --- a/.github/workflows/install-sh.yml +++ b/.github/workflows/install-sh.yml @@ -4,13 +4,13 @@ on: workflow_call: inputs: ref: - description: 'Git ref to checkout' + description: "Git ref to checkout" required: false type: string workflow_dispatch: inputs: ref: - description: 'Git ref to checkout' + description: "Git ref to checkout" required: false type: string diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 48b4deeb218d3..74b782aa7d336 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -16,7 +16,7 @@ on: # Workflow-level permissions - read access to repository contents permissions: - contents: read # Required to checkout code + contents: read # Required to checkout code env: AXIOM_TOKEN: ${{ secrets.AXIOM_TOKEN }} diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 861913c315cd4..64cfe8c6da004 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -10,7 +10,7 @@ on: workflow_dispatch: pull_request: merge_group: - types: [ checks_requested ] + types: [checks_requested] concurrency: # `github.event.number` exists for pull requests, otherwise fall back to SHA for merge queue @@ -58,7 +58,7 @@ jobs: needs.changes.outputs.integration-yml == 'true' || needs.changes.outputs.int-tests-any == 'true' || needs.changes.outputs.e2e-tests-any == 'true'))) - }} + }} uses: ./.github/workflows/build-test-runner.yml with: commit_sha: ${{ github.sha }} @@ -75,13 +75,43 @@ jobs: matrix: # TODO: Add "splunk" back once https://github.com/vectordotdev/vector/issues/23474 is fixed. # If you modify this list, please also update the `int_tests` job in changes.yml. - service: [ - "amqp", "appsignal", "axiom", "aws", "azure", "clickhouse", "databend", "datadog-agent", - "datadog-logs", "datadog-metrics", "datadog-traces", "dnstap", "docker-logs", "doris", "elasticsearch", - "eventstoredb", "fluent", "gcp", "greptimedb", "http-client", "influxdb", "kafka", "logstash", - "loki", "mqtt", "mongodb", "nats", "nginx", "opentelemetry", "postgres", "prometheus", "pulsar", - "redis", "webhdfs" - ] + service: + [ + "amqp", + "appsignal", + "axiom", + "aws", + "azure", + "clickhouse", + "databend", + "datadog-agent", + "datadog-logs", + "datadog-metrics", + "datadog-traces", + "dnstap", + "docker-logs", + "doris", + "elasticsearch", + "eventstoredb", + "fluent", + "gcp", + "greptimedb", + "http-client", + "influxdb", + "kafka", + "logstash", + "loki", + "mqtt", + "mongodb", + "nats", + "nginx", + "opentelemetry", + "postgres", + "prometheus", + "pulsar", + "redis", + "webhdfs" + ] timeout-minutes: 90 steps: - name: Download JSON artifact from changes.yml @@ -135,7 +165,6 @@ jobs: command: | bash scripts/run-integration-test.sh int ${{ matrix.service }} - e2e-tests: runs-on: ubuntu-24.04-8core needs: @@ -145,7 +174,7 @@ jobs: (github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch') }} strategy: matrix: - service: [ "datadog-logs", "datadog-metrics", "opentelemetry-logs", "opentelemetry-metrics" ] + service: ["datadog-logs", "datadog-metrics", "opentelemetry-logs", "opentelemetry-metrics"] timeout-minutes: 90 steps: diff --git a/.github/workflows/k8s_e2e.yml b/.github/workflows/k8s_e2e.yml index 0aa171f0daacd..afdcc01252e20 100644 --- a/.github/workflows/k8s_e2e.yml +++ b/.github/workflows/k8s_e2e.yml @@ -21,20 +21,20 @@ on: workflow_dispatch: inputs: ref: - description: 'Git ref to checkout' + description: "Git ref to checkout" required: false type: string workflow_call: inputs: ref: - description: 'Git ref to checkout' + description: "Git ref to checkout" required: false type: string pull_request: merge_group: types: [checks_requested] schedule: - - cron: '0 1 * * 2-6' # 01:00 UTC Tue-Sat + - cron: "0 1 * * 2-6" # 01:00 UTC Tue-Sat concurrency: # In flight runs will be canceled through re-trigger in the merge queue, scheduled run, or if diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index e94a4bb384ec6..8bfb4b5fb26c9 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -13,7 +13,7 @@ jobs: contents: read pull-requests: write steps: - - uses: actions/labeler@634933edcd8ababfe52f92936142cc22ac488b1b # v6.0.1 - with: - repo-token: "${{ secrets.GITHUB_TOKEN }}" - sync-labels: true + - uses: actions/labeler@634933edcd8ababfe52f92936142cc22ac488b1b # v6.0.1 + with: + repo-token: "${{ secrets.GITHUB_TOKEN }}" + sync-labels: true diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 848bed90f8299..b37bfdbc20e25 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -1,18 +1,18 @@ name: Nightly permissions: - contents: read # Restrictive default + contents: read # Restrictive default on: workflow_dispatch: schedule: - - cron: "0 5 * * 2-6" # Runs at 5:00 AM UTC, Tuesday through Saturday + - cron: "0 5 * * 2-6" # Runs at 5:00 AM UTC, Tuesday through Saturday jobs: Nightly: permissions: - contents: write # Required to create/update releases and tags - packages: write # Required to push container images to GHCR + contents: write # Required to create/update releases and tags + packages: write # Required to push container images to GHCR uses: ./.github/workflows/publish.yml with: git_ref: ${{ github.ref }} diff --git a/.github/workflows/preview_site_trigger.yml b/.github/workflows/preview_site_trigger.yml index 08ff26836ca20..98c5a6a2df720 100644 --- a/.github/workflows/preview_site_trigger.yml +++ b/.github/workflows/preview_site_trigger.yml @@ -5,15 +5,15 @@ on: # Restrictive default permissions permissions: - contents: read # Required for repository context + contents: read # Required for repository context jobs: approval_check: runs-on: ubuntu-24.04 timeout-minutes: 5 permissions: - contents: read # Required for repository context - actions: write # Required to upload artifacts that trigger the preview build workflow + contents: read # Required for repository context + actions: write # Required to upload artifacts that trigger the preview build workflow # Only run for PRs with 'website' in the branch name if: ${{ contains(github.head_ref, 'website') }} steps: diff --git a/.github/workflows/protobuf.yml b/.github/workflows/protobuf.yml index 06e8411a81541..bc9cca9baf473 100644 --- a/.github/workflows/protobuf.yml +++ b/.github/workflows/protobuf.yml @@ -6,15 +6,15 @@ permissions: on: pull_request: paths: - - "proto/**" - - "lib/vector-core/proto/**" + - "proto/**" + - "lib/vector-core/proto/**" merge_group: types: [checks_requested] concurrency: - # `github.ref` is unique for MQ runs and PRs - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + # `github.ref` is unique for MQ runs and PRs + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: validate-protos: diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 0594151b2db23..4c66ff585e173 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -59,14 +59,14 @@ jobs: strategy: matrix: include: - - target: x86_64-unknown-linux-musl # .tar.gz - - target: x86_64-unknown-linux-gnu # .tar.gz, .deb, .rpm - - target: aarch64-unknown-linux-musl # .tar.gz - - target: aarch64-unknown-linux-gnu # .tar.gz, .deb, .rpm - - target: armv7-unknown-linux-gnueabihf # .tar.gz, .deb, .rpm - - target: armv7-unknown-linux-musleabihf # .tar.gz - - target: arm-unknown-linux-gnueabi # .tar.gz, .deb - - target: arm-unknown-linux-musleabi # .tar.gz + - target: x86_64-unknown-linux-musl # .tar.gz + - target: x86_64-unknown-linux-gnu # .tar.gz, .deb, .rpm + - target: aarch64-unknown-linux-musl # .tar.gz + - target: aarch64-unknown-linux-gnu # .tar.gz, .deb, .rpm + - target: armv7-unknown-linux-gnueabihf # .tar.gz, .deb, .rpm + - target: armv7-unknown-linux-musleabihf # .tar.gz + - target: arm-unknown-linux-gnueabi # .tar.gz, .deb + - target: arm-unknown-linux-musleabi # .tar.gz steps: - name: Checkout Vector uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 1b93487a54ec2..167dc0186bada 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -32,11 +32,11 @@ on: description: "The SHA to use for comparison (optional). If not provided, it defaults to the current HEAD of the origin/master branch." required: false schedule: - - cron: '0 7 * * 1' # Runs at 7 AM UTC on Mondays + - cron: "0 7 * * 1" # Runs at 7 AM UTC on Mondays # Workflow-level permissions - read access to repository contents permissions: - contents: read # Required to checkout code + contents: read # Required to checkout code env: SINGLE_MACHINE_PERFORMANCE_API: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_API }} @@ -44,7 +44,6 @@ env: SMP_REPLICAS: 100 # default is 10 jobs: - resolve-inputs: runs-on: ubuntu-24.04 outputs: @@ -57,7 +56,7 @@ jobs: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - fetch-depth: 0 # need to pull repository history to find merge bases + fetch-depth: 0 # need to pull repository history to find merge bases - name: Set and Validate SHAs id: set_and_validate_shas @@ -195,8 +194,8 @@ jobs: - resolve-inputs # Job-level permissions for artifact upload permissions: - contents: read # Required to checkout code - actions: write # Required to upload artifacts + contents: read # Required to checkout code + actions: write # Required to upload artifacts steps: - uses: colpal/actions-clean@36e6ca1abd35efe61cb60f912bd7837f67887c8a # v1.1.1 @@ -238,8 +237,8 @@ jobs: - resolve-inputs # Job-level permissions for artifact upload permissions: - contents: read # Required to checkout code - actions: write # Required to upload artifacts + contents: read # Required to checkout code + actions: write # Required to upload artifacts steps: - uses: colpal/actions-clean@36e6ca1abd35efe61cb60f912bd7837f67887c8a # v1.1.1 @@ -305,7 +304,7 @@ jobs: - confirm-valid-credentials - build-baseline steps: - - name: 'Download baseline image' + - name: "Download baseline image" uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: name: baseline-image @@ -345,7 +344,7 @@ jobs: - confirm-valid-credentials - build-comparison steps: - - name: 'Download comparison image' + - name: "Download comparison image" uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: name: comparison-image @@ -386,8 +385,8 @@ jobs: - upload-comparison-image-to-ecr # Job-level permissions for artifact upload permissions: - contents: read # Required to checkout code - actions: write # Required to upload artifacts + contents: read # Required to checkout code + actions: write # Required to upload artifacts steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -502,8 +501,8 @@ jobs: - resolve-inputs # Job-level permissions for artifact upload permissions: - contents: read # Required to checkout code - actions: write # Required to upload artifacts + contents: read # Required to checkout code + actions: write # Required to upload artifacts steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e9965f4d6193c..04192306a1a77 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,7 +1,7 @@ name: Release Suite permissions: - contents: read # Restrictive default + contents: read # Restrictive default on: push: @@ -12,8 +12,8 @@ on: jobs: Release: permissions: - contents: write # Required to create/update releases and tags - packages: write # Required to push container images to GHCR + contents: write # Required to create/update releases and tags + packages: write # Required to push container images to GHCR uses: ./.github/workflows/publish.yml with: git_ref: ${{ github.ref }} diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index c43853e5a7d51..7a8fd8c9a7d47 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -10,9 +10,9 @@ on: # To guarantee Maintained check is occasionally updated. See # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained schedule: - - cron: '0 6 * * 1' # 6 AM UTC every Monday + - cron: "0 6 * * 1" # 6 AM UTC every Monday push: - branches: [ "master" ] + branches: ["master"] # Declare default permissions as read only. permissions: read-all diff --git a/.github/workflows/spelling.yml b/.github/workflows/spelling.yml index 425111667a5e8..e95e1183fc593 100644 --- a/.github/workflows/spelling.yml +++ b/.github/workflows/spelling.yml @@ -59,13 +59,13 @@ name: Check Spelling on: pull_request_target: branches: - - "**" + - "**" tags-ignore: - - "**" + - "**" types: - - 'opened' - - 'reopened' - - 'synchronize' + - "opened" + - "reopened" + - "synchronize" permissions: contents: read @@ -88,51 +88,50 @@ jobs: # note: If you use only_check_changed_files, you do not want cancel-in-progress cancel-in-progress: true steps: - - name: check-spelling - id: spelling - uses: check-spelling/check-spelling@c635c2f3f714eec2fcf27b643a1919b9a811ef2e # v0.0.25 - with: - suppress_push_for_open_pull_request: ${{ github.actor != 'dependabot[bot]' && 1 }} - checkout: true - check_file_names: 1 - spell_check_this: check-spelling/spell-check-this@prerelease - post_comment: 0 - use_sarif: ${{ (!github.event.pull_request || (github.event.pull_request.head.repo.full_name == github.repository)) && 1 }} - extra_dictionary_limit: 20 - extra_dictionaries: - cspell:aws/aws.txt - cspell:cpp/src/compiler-clang-attributes.txt - cspell:cpp/src/compiler-msvc.txt - cspell:cpp/src/ecosystem.txt - cspell:cpp/src/lang-jargon.txt - cspell:cpp/src/stdlib-cpp.txt - cspell:css/dict/css.txt - cspell:django/dict/django.txt - cspell:docker/src/docker-words.txt - cspell:dotnet/dict/dotnet.txt - cspell:elixir/dict/elixir.txt - cspell:filetypes/filetypes.txt - cspell:fullstack/dict/fullstack.txt - cspell:golang/dict/go.txt - cspell:html-symbol-entities/entities.txt - cspell:html/dict/html.txt - cspell:java/src/java-terms.txt - cspell:java/src/java.txt - cspell:k8s/dict/k8s.txt - cspell:lorem-ipsum/dictionary.txt - cspell:lua/dict/lua.txt - cspell:node/dict/node.txt - cspell:npm/dict/npm.txt - cspell:php/dict/php.txt - cspell:public-licenses/src/generated/public-licenses.txt - cspell:python/src/common/extra.txt - cspell:python/src/python/python-lib.txt - cspell:python/src/python/python.txt - cspell:r/src/r.txt - cspell:ruby/dict/ruby.txt - cspell:rust/dict/rust.txt - cspell:shell/dict/shell-all-words.txt - cspell:software-terms/dict/softwareTerms.txt - cspell:swift/src/swift.txt - cspell:typescript/dict/typescript.txt - check_extra_dictionaries: '' + - name: check-spelling + id: spelling + uses: check-spelling/check-spelling@c635c2f3f714eec2fcf27b643a1919b9a811ef2e # v0.0.25 + with: + suppress_push_for_open_pull_request: ${{ github.actor != 'dependabot[bot]' && 1 }} + checkout: true + check_file_names: 1 + spell_check_this: check-spelling/spell-check-this@prerelease + post_comment: 0 + use_sarif: ${{ (!github.event.pull_request || (github.event.pull_request.head.repo.full_name == github.repository)) && 1 }} + extra_dictionary_limit: 20 + extra_dictionaries: cspell:aws/aws.txt + cspell:cpp/src/compiler-clang-attributes.txt + cspell:cpp/src/compiler-msvc.txt + cspell:cpp/src/ecosystem.txt + cspell:cpp/src/lang-jargon.txt + cspell:cpp/src/stdlib-cpp.txt + cspell:css/dict/css.txt + cspell:django/dict/django.txt + cspell:docker/src/docker-words.txt + cspell:dotnet/dict/dotnet.txt + cspell:elixir/dict/elixir.txt + cspell:filetypes/filetypes.txt + cspell:fullstack/dict/fullstack.txt + cspell:golang/dict/go.txt + cspell:html-symbol-entities/entities.txt + cspell:html/dict/html.txt + cspell:java/src/java-terms.txt + cspell:java/src/java.txt + cspell:k8s/dict/k8s.txt + cspell:lorem-ipsum/dictionary.txt + cspell:lua/dict/lua.txt + cspell:node/dict/node.txt + cspell:npm/dict/npm.txt + cspell:php/dict/php.txt + cspell:public-licenses/src/generated/public-licenses.txt + cspell:python/src/common/extra.txt + cspell:python/src/python/python-lib.txt + cspell:python/src/python/python.txt + cspell:r/src/r.txt + cspell:ruby/dict/ruby.txt + cspell:rust/dict/rust.txt + cspell:shell/dict/shell-all-words.txt + cspell:software-terms/dict/softwareTerms.txt + cspell:swift/src/swift.txt + cspell:typescript/dict/typescript.txt + check_extra_dictionaries: "" diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml index d3330f07b7280..04ed6b19d9f25 100644 --- a/.github/workflows/static-analysis.yml +++ b/.github/workflows/static-analysis.yml @@ -4,8 +4,8 @@ on: push: branches: - master - - "**" # Run on all branches (includes PR branches) - workflow_dispatch: # Allow manual trigger from GitHub UI + - "**" # Run on all branches (includes PR branches) + workflow_dispatch: # Allow manual trigger from GitHub UI permissions: contents: read diff --git a/.github/workflows/test-make-command.yml b/.github/workflows/test-make-command.yml index f10ceaa3c45ad..79c27d54792f4 100644 --- a/.github/workflows/test-make-command.yml +++ b/.github/workflows/test-make-command.yml @@ -4,29 +4,28 @@ on: workflow_call: inputs: command: - description: 'Make command to run (e.g., test-behavior, check-examples, test-docs)' + description: "Make command to run (e.g., test-behavior, check-examples, test-docs)" required: true type: string job_name: - description: 'Display name for the job/status check' + description: "Display name for the job/status check" required: true type: string ref: - description: 'Git ref to checkout' + description: "Git ref to checkout" required: false type: string cargo_nextest: - description: 'Whether to install cargo-nextest' + description: "Whether to install cargo-nextest" required: false type: boolean default: false upload_test_results: - description: 'Whether to upload nextest test results (for commands that use cargo nextest)' + description: "Whether to upload nextest test results (for commands that use cargo nextest)" required: false type: boolean default: false - permissions: contents: read diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 986b145305f78..81ceef93ca5f1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,7 +7,7 @@ on: # Workflow-level permissions - read access to repository contents permissions: - contents: read # Required to checkout code + contents: read # Required to checkout code concurrency: # `github.ref` is unique for MQ runs and PRs @@ -31,13 +31,14 @@ jobs: check-fmt: name: Check code format runs-on: ubuntu-24.04 - if: ${{ needs.changes.outputs.source == 'true' || needs.changes.outputs.test-yml == 'true' }} + if: ${{ needs.changes.outputs.source == 'true' || needs.changes.outputs.test-yml == 'true' || needs.changes.outputs.prettier == 'true' }} needs: changes steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: ./.github/actions/setup with: rust: true + prettier: true - run: make check-fmt check-clippy: diff --git a/.github/workflows/unit_mac.yml b/.github/workflows/unit_mac.yml index b177b018d13ef..e0d367b530959 100644 --- a/.github/workflows/unit_mac.yml +++ b/.github/workflows/unit_mac.yml @@ -4,11 +4,10 @@ on: workflow_call: inputs: ref: - description: 'Git ref to checkout' + description: "Git ref to checkout" required: false type: string - permissions: contents: read diff --git a/.github/workflows/unit_windows.yml b/.github/workflows/unit_windows.yml index 831eec042f869..a4553b4a0b751 100644 --- a/.github/workflows/unit_windows.yml +++ b/.github/workflows/unit_windows.yml @@ -4,11 +4,10 @@ on: workflow_call: inputs: ref: - description: 'Git ref to checkout' + description: "Git ref to checkout" required: false type: string - permissions: contents: read diff --git a/.github/workflows/vdev_publish.yml b/.github/workflows/vdev_publish.yml index edb2f36674731..3a46822098beb 100644 --- a/.github/workflows/vdev_publish.yml +++ b/.github/workflows/vdev_publish.yml @@ -1,16 +1,16 @@ name: Publish vdev on: push: - tags: [ "vdev-v*.*.*" ] + tags: ["vdev-v*.*.*"] permissions: - contents: read # Restrictive default + contents: read # Restrictive default jobs: build: runs-on: ${{ matrix.os }} permissions: - contents: write # Required to upload release assets + contents: write # Required to upload release assets strategy: matrix: include: diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000000000..f7b79d1ba7d52 --- /dev/null +++ b/.prettierignore @@ -0,0 +1,30 @@ +# Build artifacts +target/ +node_modules/ + +# Generated / test fixture files +docs/generated/ +lib/codecs/tests/data/ + +# Hugo templates and assets (Go template syntax breaks prettier) +website/**/*.html +website/assets/js/app.js +website/assets/js/dd-browser-logs-rum.js +website/layouts/ + +# Hugo build output and generated resources +website/public/ +website/resources/ +website/data/ +website/assets/jsconfig.json + +# CUE files (not supported by prettier) +website/cue/ + +# Config examples using VRL triple-quoted strings +config/examples/docs_example.yaml +config/examples/file_to_prometheus.yaml + +# Lock files +Cargo.lock +website/yarn.lock diff --git a/.prettierrc.json b/.prettierrc.json new file mode 100644 index 0000000000000..a4643ee91d31c --- /dev/null +++ b/.prettierrc.json @@ -0,0 +1,4 @@ +{ + "trailingComma": "none", + "printWidth": 120 +} diff --git a/Makefile b/Makefile index 5c52c367c11b7..a923806254d9f 100644 --- a/Makefile +++ b/Makefile @@ -510,6 +510,24 @@ check-markdown: ## Check that markdown is styled properly fix-markdown: ## Auto-fix markdown style issues ${MAYBE_ENVIRONMENT_EXEC} markdownlint-cli2 --fix $(shell git ls-files '*.md') +.PHONY: check-prettier +check-prettier: ## Check that JS/TS/YAML/JSON files are formatted with prettier + @for ext in yml yaml js ts tsx json; do \ + files=$$(git ls-files "*.$$ext"); \ + if [ -n "$$files" ]; then \ + ${MAYBE_ENVIRONMENT_EXEC} prettier --ignore-path .prettierignore --check $$files || exit 1; \ + fi; \ + done + +.PHONY: fix-prettier +fix-prettier: ## Auto-fix JS/TS/YAML/JSON formatting with prettier + @for ext in yml yaml js ts tsx json; do \ + files=$$(git ls-files "*.$$ext"); \ + if [ -n "$$files" ]; then \ + ${MAYBE_ENVIRONMENT_EXEC} prettier --ignore-path .prettierignore --write $$files || exit 1; \ + fi; \ + done + .PHONY: check-examples check-examples: ## Check that the config/examples files are valid ${MAYBE_ENVIRONMENT_EXEC} $(VDEV) check examples diff --git a/config/examples/environment_variables.yaml b/config/examples/environment_variables.yaml index 2f73528397ad1..d8421c1cc8e01 100644 --- a/config/examples/environment_variables.yaml +++ b/config/examples/environment_variables.yaml @@ -13,7 +13,7 @@ data_dir: "/var/lib/vector" sources: apache_logs: type: "file" - include: [ "/var/log/apache2/*.log" ] + include: ["/var/log/apache2/*.log"] # ignore files older than 1 day ignore_older_secs: 86400 @@ -21,7 +21,7 @@ sources: # Docs: https://vector.dev/docs/reference/configuration/transforms/remap transforms: add_host: - inputs: [ "apache_logs" ] + inputs: ["apache_logs"] type: "remap" source: | ''' @@ -32,7 +32,7 @@ transforms: # Docs: https://vector.dev/docs/reference/configuration/sinks/console sinks: out: - inputs: [ "add_host" ] + inputs: ["add_host"] type: "console" encoding: codec: "json" diff --git a/config/examples/file_to_cloudwatch_metrics.yaml b/config/examples/file_to_cloudwatch_metrics.yaml index f322ab8f091b0..138f9d20ceac2 100644 --- a/config/examples/file_to_cloudwatch_metrics.yaml +++ b/config/examples/file_to_cloudwatch_metrics.yaml @@ -8,13 +8,13 @@ data_dir: "/var/lib/vector" sources: file: type: "file" - include: [ "sample.log" ] + include: ["sample.log"] start_at_beginning: true # Structure and parse the data transforms: remap: - inputs: [ "file" ] + inputs: ["file"] type: "remap" drop_on_error: false source: | @@ -22,7 +22,7 @@ transforms: # Transform into metrics log_to_metric: - inputs: [ "remap" ] + inputs: ["remap"] type: "log_to_metric" metrics: - type: "counter" @@ -35,19 +35,19 @@ transforms: # Output data sinks: console_metrics: - inputs: [ "log_to_metric" ] + inputs: ["log_to_metric"] type: "console" encoding: codec: "json" console_logs: - inputs: [ "remap" ] + inputs: ["remap"] type: "console" encoding: codec: "json" cloudwatch: - inputs: [ "log_to_metric" ] + inputs: ["log_to_metric"] type: "aws_cloudwatch_metrics" namespace: "vector" endpoint: "http://localhost:4566" diff --git a/config/examples/namespacing/sinks/es_cluster.yaml b/config/examples/namespacing/sinks/es_cluster.yaml index 820a49080b474..f916dac1b7180 100644 --- a/config/examples/namespacing/sinks/es_cluster.yaml +++ b/config/examples/namespacing/sinks/es_cluster.yaml @@ -1,6 +1,6 @@ # Send structured data to a short-term storage -inputs: ["apache_sample"] # only take sampled data +inputs: ["apache_sample"] # only take sampled data type: "elasticsearch" -endpoint: "http://79.12.221.222:9200" # local or external host +endpoint: "http://79.12.221.222:9200" # local or external host bulk: - index: "vector-%Y-%m-%d" # daily indices + index: "vector-%Y-%m-%d" # daily indices diff --git a/config/examples/namespacing/sinks/s3_archives.yaml b/config/examples/namespacing/sinks/s3_archives.yaml index a72c27027e8a1..6de3cd4df8f56 100644 --- a/config/examples/namespacing/sinks/s3_archives.yaml +++ b/config/examples/namespacing/sinks/s3_archives.yaml @@ -1,13 +1,13 @@ # Send structured data to a cost-effective long-term storage -inputs: ["apache_parser"] # don't sample for S3 +inputs: ["apache_parser"] # don't sample for S3 type: "aws_s3" region: "us-east-1" bucket: "my-log-archives" -key_prefix: "date=%Y-%m-%d" # daily partitions, hive friendly format -compression: "gzip" # compress final objects +key_prefix: "date=%Y-%m-%d" # daily partitions, hive friendly format +compression: "gzip" # compress final objects framing: - method: "newline_delimited" # new line delimited... + method: "newline_delimited" # new line delimited... encoding: - codec: "json" # ...JSON + codec: "json" # ...JSON batch: - max_bytes: 10000000 # 10mb uncompressed + max_bytes: 10000000 # 10mb uncompressed diff --git a/config/examples/namespacing/transforms/apache_sample.yaml b/config/examples/namespacing/transforms/apache_sample.yaml index 91b789e1016f2..cca25cf0989af 100644 --- a/config/examples/namespacing/transforms/apache_sample.yaml +++ b/config/examples/namespacing/transforms/apache_sample.yaml @@ -1,4 +1,4 @@ # Sample the data to save on cost inputs: ["apache_parser"] type: "sample" -rate: 2 # only keep 50% (1/`rate`) +rate: 2 # only keep 50% (1/`rate`) diff --git a/config/examples/stdio.yaml b/config/examples/stdio.yaml index cbe6429d8afae..7ca39a81876da 100644 --- a/config/examples/stdio.yaml +++ b/config/examples/stdio.yaml @@ -11,7 +11,7 @@ sources: sinks: out: - inputs: [ "in" ] + inputs: ["in"] type: "console" encoding: codec: "text" diff --git a/config/examples/wrapped_json.yaml b/config/examples/wrapped_json.yaml index 90fed711f2a18..4bd6ab0a79f58 100644 --- a/config/examples/wrapped_json.yaml +++ b/config/examples/wrapped_json.yaml @@ -12,14 +12,14 @@ data_dir: "/var/lib/vector" sources: logs: type: "file" - include: [ "/var/log/*.log" ] + include: ["/var/log/*.log"] ignore_older_secs: 86400 # 1 day # Parse the data as JSON # Docs: https://vector.dev/docs/reference/configuration/transforms/remap transforms: parse_json: - inputs: [ "logs" ] + inputs: ["logs"] type: "remap" drop_on_error: false source: | @@ -36,7 +36,7 @@ transforms: # Docs: https://vector.dev/docs/reference/configuration/sinks/console sinks: out: - inputs: [ "parse_json" ] + inputs: ["parse_json"] type: "console" encoding: codec: "json" diff --git a/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/logs/v1/logs_service_http.yaml b/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/logs/v1/logs_service_http.yaml index 4d913387252ef..4dd744f5d2c02 100644 --- a/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/logs/v1/logs_service_http.yaml +++ b/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/logs/v1/logs_service_http.yaml @@ -3,8 +3,7 @@ type: google.api.Service config_version: 3 http: - rules: - - selector: opentelemetry.proto.collector.logs.v1.LogsService.Export - post: /v1/logs - body: "*" - + rules: + - selector: opentelemetry.proto.collector.logs.v1.LogsService.Export + post: /v1/logs + body: "*" diff --git a/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/metrics/v1/metrics_service_http.yaml b/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/metrics/v1/metrics_service_http.yaml index 7ff39d4b406c6..7643be4125831 100644 --- a/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/metrics/v1/metrics_service_http.yaml +++ b/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/metrics/v1/metrics_service_http.yaml @@ -3,8 +3,7 @@ type: google.api.Service config_version: 3 http: - rules: - - selector: opentelemetry.proto.collector.metrics.v1.MetricsService.Export - post: /v1/metrics - body: "*" - + rules: + - selector: opentelemetry.proto.collector.metrics.v1.MetricsService.Export + post: /v1/metrics + body: "*" diff --git a/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/trace/v1/trace_service_http.yaml b/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/trace/v1/trace_service_http.yaml index d091b3a8d53d7..7eaa4e548c33a 100644 --- a/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/trace/v1/trace_service_http.yaml +++ b/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/trace/v1/trace_service_http.yaml @@ -3,7 +3,7 @@ type: google.api.Service config_version: 3 http: - rules: - - selector: opentelemetry.proto.collector.trace.v1.TraceService.Export - post: /v1/traces - body: "*" + rules: + - selector: opentelemetry.proto.collector.trace.v1.TraceService.Export + post: /v1/traces + body: "*" diff --git a/lib/vector-core/tests/data/fixtures/log_event/malformed.json b/lib/vector-core/tests/data/fixtures/log_event/malformed.json index 6f956957a7b81..32603adb1fb2f 100644 --- a/lib/vector-core/tests/data/fixtures/log_event/malformed.json +++ b/lib/vector-core/tests/data/fixtures/log_event/malformed.json @@ -1,4 +1,4 @@ { "foo": "bar", "foo": "You're not allowed duplicate keys..." -} \ No newline at end of file +} diff --git a/lib/vector-vrl/tests/resources/json-schema_definition.json b/lib/vector-vrl/tests/resources/json-schema_definition.json index 9c4c8a00d0b28..a2e51e99c51d1 100644 --- a/lib/vector-vrl/tests/resources/json-schema_definition.json +++ b/lib/vector-vrl/tests/resources/json-schema_definition.json @@ -1 +1,10 @@ -{"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"https://example.com/product.schema.json","title":"Product","description":"A product from Acme's catalog","type":"object","properties":{"productUser":{"description":"The unique identifier for a product user","type":"string","format":"email"}}} +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.com/product.schema.json", + "title": "Product", + "description": "A product from Acme's catalog", + "type": "object", + "properties": { + "productUser": { "description": "The unique identifier for a product user", "type": "string", "format": "email" } + } +} diff --git a/lib/vector-vrl/web-playground/public/index.js b/lib/vector-vrl/web-playground/public/index.js index 4cdc2bf4e0dcc..9f810ec29e74a 100644 --- a/lib/vector-vrl/web-playground/public/index.js +++ b/lib/vector-vrl/web-playground/public/index.js @@ -1,5 +1,5 @@ -import init, {run_vrl, vector_link, vector_version, vrl_link, vrl_version} from "./pkg/vector_vrl_web_playground.js"; -import {vrlLanguageDefinition, vrlThemeDefinition} from "./vrl-highlighter.js"; +import init, { run_vrl, vector_link, vector_version, vrl_link, vrl_version } from "./pkg/vector_vrl_web_playground.js"; +import { vrlLanguageDefinition, vrlThemeDefinition } from "./vrl-highlighter.js"; const PROGRAM_EDITOR_DEFAULT_VALUE = `# Remove some fields del(.foo) @@ -42,309 +42,313 @@ You can try validating your JSON here: https://jsonlint.com/ `; function loadMonaco() { - return new Promise((resolve, reject) => { - // require is provided by loader.min.js. - require.config({ - paths: {vs: "https://cdnjs.cloudflare.com/ajax/libs/monaco-editor/0.26.1/min/vs"}, - }); - require(["vs/editor/editor.main"], () => resolve(window.monaco), reject); + return new Promise((resolve, reject) => { + // require is provided by loader.min.js. + require.config({ + paths: { vs: "https://cdnjs.cloudflare.com/ajax/libs/monaco-editor/0.26.1/min/vs" } }); + require(["vs/editor/editor.main"], () => resolve(window.monaco), reject); + }); } export class VrlWebPlayground { - static async create() { - const instance = new VrlWebPlayground(true); - await instance._initAsync(); - return instance; + static async create() { + const instance = new VrlWebPlayground(true); + await instance._initAsync(); + return instance; + } + + constructor(_internal = false) { + if (!_internal) { + // Prefer factory: VrlWebPlayground.create() + this._initAsync(); // fire-and-forget fallback } - - constructor(_internal = false) { - if (!_internal) { - // Prefer factory: VrlWebPlayground.create() - this._initAsync(); // fire-and-forget fallback - } + } + + async _initAsync() { + // Load wasm/runtime + await init(); + + // Bind native funcs/versions + this.run_vrl = run_vrl; + this.vector_version = vector_version(); + this.vector_link = vector_link(); + this.vrl_version = vrl_version(); + this.vrl_link = vrl_link(); + + // Load Monaco + this.monaco = await loadMonaco(); + + // VRL lang + theme + this.monaco.languages.register({ id: "vrl" }); + this.monaco.editor.defineTheme("vrl-theme", vrlThemeDefinition); + this.monaco.languages.setMonarchTokensProvider("vrl", vrlLanguageDefinition); + + // Editors + this.eventEditor = this.createDefaultEditor("container-event", EVENT_EDITOR_DEFAULT_VALUE, "json", "vs-light"); + this.outputEditor = this.createDefaultEditor("container-output", OUTPUT_EDITOR_DEFAULT_VALUE, "json", "vs-light"); + this.programEditor = this.createDefaultEditor( + "container-program", + PROGRAM_EDITOR_DEFAULT_VALUE, + "vrl", + "vrl-theme" + ); + + // Versions + this.addVersions(); + + // Handle shared state from URL (if present) + this._maybeLoadFromUrl(); + } + + _maybeLoadFromUrl() { + const qs = window.location.search; + if (!qs) return; + + const urlParams = new URLSearchParams(qs); + const stateParam = urlParams.get("state"); + if (!stateParam) return; + + try { + const decoded = atob(decodeURIComponent(stateParam)); + const urlState = JSON.parse(decoded); + + if (typeof urlState.program === "string") { + this.programEditor.setValue(urlState.program); + } + + if (urlState.is_jsonl === true && typeof urlState.event === "string") { + this.eventEditor.setValue(urlState.event); + } else if (urlState.event != null) { + this.eventEditor.setValue(JSON.stringify(urlState.event, null, "\t")); + } + + // Run immediately with the provided state + this.handleRunCode(urlState); + } catch (e) { + this.disableJsonLinting(); + this.outputEditor.setValue(`Error reading the shared URL\n${e}`); } + } - async _initAsync() { - // Load wasm/runtime - await init(); - - // Bind native funcs/versions - this.run_vrl = run_vrl; - this.vector_version = vector_version(); - this.vector_link = vector_link(); - this.vrl_version = vrl_version(); - this.vrl_link = vrl_link(); - - // Load Monaco - this.monaco = await loadMonaco(); - - // VRL lang + theme - this.monaco.languages.register({id: "vrl"}); - this.monaco.editor.defineTheme("vrl-theme", vrlThemeDefinition); - this.monaco.languages.setMonarchTokensProvider("vrl", vrlLanguageDefinition); - - // Editors - this.eventEditor = this.createDefaultEditor("container-event", EVENT_EDITOR_DEFAULT_VALUE, "json", "vs-light"); - this.outputEditor = this.createDefaultEditor("container-output", OUTPUT_EDITOR_DEFAULT_VALUE, "json", "vs-light"); - this.programEditor = this.createDefaultEditor("container-program", PROGRAM_EDITOR_DEFAULT_VALUE, "vrl", "vrl-theme"); - - // Versions - this.addVersions(); - - // Handle shared state from URL (if present) - this._maybeLoadFromUrl(); + addVersions() { + const vectorLinkElement = document.getElementById("vector-version-link"); + if (vectorLinkElement) { + vectorLinkElement.text = (this.vector_version || "").toString().substring(0, 8); + vectorLinkElement.href = this.vector_link || "#"; } - _maybeLoadFromUrl() { - const qs = window.location.search; - if (!qs) return; - - const urlParams = new URLSearchParams(qs); - const stateParam = urlParams.get("state"); - if (!stateParam) return; - - try { - const decoded = atob(decodeURIComponent(stateParam)); - const urlState = JSON.parse(decoded); - - if (typeof urlState.program === "string") { - this.programEditor.setValue(urlState.program); - } - - if (urlState.is_jsonl === true && typeof urlState.event === "string") { - this.eventEditor.setValue(urlState.event); - } else if (urlState.event != null) { - this.eventEditor.setValue(JSON.stringify(urlState.event, null, "\t")); - } - - // Run immediately with the provided state - this.handleRunCode(urlState); - } catch (e) { - this.disableJsonLinting(); - this.outputEditor.setValue(`Error reading the shared URL\n${e}`); - } + const vrlLinkElement = document.getElementById("vrl-version-link"); + if (vrlLinkElement) { + vrlLinkElement.text = (this.vrl_version || "").toString().substring(0, 8); + vrlLinkElement.href = this.vrl_link || "#"; } + } - addVersions() { - const vectorLinkElement = document.getElementById("vector-version-link"); - if (vectorLinkElement) { - vectorLinkElement.text = (this.vector_version || "").toString().substring(0, 8); - vectorLinkElement.href = this.vector_link || "#"; - } - - const vrlLinkElement = document.getElementById("vrl-version-link"); - if (vrlLinkElement) { - vrlLinkElement.text = (this.vrl_version || "").toString().substring(0, 8); - vrlLinkElement.href = this.vrl_link || "#"; - } + createDefaultEditor(elementId, value, language, theme) { + const el = document.getElementById(elementId); + if (!el) { + console.warn(`Editor container #${elementId} not found`); + return null; } + return this.monaco.editor.create(el, { + value, + language, + theme, + minimap: { enabled: false }, + automaticLayout: true, + wordWrap: "on" + }); + } - createDefaultEditor(elementId, value, language, theme) { - const el = document.getElementById(elementId); - if (!el) { - console.warn(`Editor container #${elementId} not found`); - return null; - } - return this.monaco.editor.create(el, { - value, - language, - theme, - minimap: {enabled: false}, - automaticLayout: true, - wordWrap: 'on', - }); + _clearOutput() { + if (this.outputEditor) { + // wipe the buffer so stale values never linger + this.outputEditor.setValue(""); } - - _clearOutput() { - if (this.outputEditor) { - // wipe the buffer so stale values never linger - this.outputEditor.setValue(""); - } - const elapsedEl = document.getElementById("elapsed-time"); - if (elapsedEl) { - elapsedEl.textContent = ""; - } + const elapsedEl = document.getElementById("elapsed-time"); + if (elapsedEl) { + elapsedEl.textContent = ""; } + } - _formatRunResult(runResult) { - if (runResult?.target_value != null) { - const isJson = typeof runResult.target_value === "object"; - const text = isJson - ? JSON.stringify(runResult.target_value, null, "\t") - : String(runResult.target_value); - return {text, isJson}; - } - if (runResult?.msg != null) { - return {text: String(runResult.msg), isJson: false}; - } - return {text: "Error - VRL did not return a result.", isJson: false}; + _formatRunResult(runResult) { + if (runResult?.target_value != null) { + const isJson = typeof runResult.target_value === "object"; + const text = isJson ? JSON.stringify(runResult.target_value, null, "\t") : String(runResult.target_value); + return { text, isJson }; } - - _setElapsed(elapsed_time) { - const elapsedEl = document.getElementById("elapsed-time"); - if (elapsedEl && elapsed_time != null) { - const ms = elapsed_time.toFixed(4) - elapsedEl.textContent = `Duration: ${ms} milliseconds`; - } + if (runResult?.msg != null) { + return { text: String(runResult.msg), isJson: false }; } - - _safeGet(editor, fallback = "") { - return editor?.getValue?.() ?? fallback; + return { text: "Error - VRL did not return a result.", isJson: false }; + } + + _setElapsed(elapsed_time) { + const elapsedEl = document.getElementById("elapsed-time"); + if (elapsedEl && elapsed_time != null) { + const ms = elapsed_time.toFixed(4); + elapsedEl.textContent = `Duration: ${ms} milliseconds`; } - - getState() { - if (this.eventEditorIsJsonl()) { - return { - program: this._safeGet(this.programEditor), - event: this.eventEditor.getModel().getLinesContent().join("\n"), - is_jsonl: true, - error: null, - }; - } - - const editorValue = this._safeGet(this.eventEditor); - try { - return { - program: this._safeGet(this.programEditor), - event: JSON.parse(editorValue.length === 0 ? "{}" : editorValue), - is_jsonl: false, - error: null, - }; - } catch (_err) { - return { - program: this._safeGet(this.programEditor), - event: null, - is_jsonl: false, - error: `Could not parse JSON event:\n${editorValue}`, - }; - } + } + + _safeGet(editor, fallback = "") { + return editor?.getValue?.() ?? fallback; + } + + getState() { + if (this.eventEditorIsJsonl()) { + return { + program: this._safeGet(this.programEditor), + event: this.eventEditor.getModel().getLinesContent().join("\n"), + is_jsonl: true, + error: null + }; } - disableJsonLinting() { - this.monaco.languages.json.jsonDefaults.setDiagnosticsOptions({validate: false}); + const editorValue = this._safeGet(this.eventEditor); + try { + return { + program: this._safeGet(this.programEditor), + event: JSON.parse(editorValue.length === 0 ? "{}" : editorValue), + is_jsonl: false, + error: null + }; + } catch (_err) { + return { + program: this._safeGet(this.programEditor), + event: null, + is_jsonl: false, + error: `Could not parse JSON event:\n${editorValue}` + }; } - - enableJsonLinting() { - this.monaco.languages.json.jsonDefaults.setDiagnosticsOptions({validate: true}); + } + + disableJsonLinting() { + this.monaco.languages.json.jsonDefaults.setDiagnosticsOptions({ validate: false }); + } + + enableJsonLinting() { + this.monaco.languages.json.jsonDefaults.setDiagnosticsOptions({ validate: true }); + } + + tryJsonParse(str) { + try { + return JSON.parse(str); + } catch (_e) { + this.disableJsonLinting(); + const err = ERROR_INVALID_JSONL_EVENT_MSG.toString().replace("{{str}}", str); + this.outputEditor.setValue(err); + throw new Error(err); } - - tryJsonParse(str) { - try { - return JSON.parse(str); - } catch (_e) { - this.disableJsonLinting(); - const err = ERROR_INVALID_JSONL_EVENT_MSG.toString().replace("{{str}}", str); - this.outputEditor.setValue(err); - throw new Error(err); - } + } + + /** + * Treat as JSONL if there are >1 non-empty lines and at least the second non-empty + * line *appears* to be a JSON object. Robust to whitespace. + */ + eventEditorIsJsonl() { + const model = this.eventEditor?.getModel?.(); + if (!model) return false; + + const rawLines = model.getLinesContent(); + const lines = rawLines.map((l) => l.trim()).filter((l) => l.length > 0); + if (lines.length <= 1) return false; + + const second = lines[1]; + return second.startsWith("{") && second.endsWith("}"); + } + + _getTimezoneOrDefault() { + const tzEl = document.getElementById("timezone-input"); + return tzEl?.value && tzEl.value.trim().length > 0 ? tzEl.value.trim() : "Default"; + } + + handleRunCode(input) { + this._clearOutput(); + + // JSONL path short-circuit + if (this.eventEditorIsJsonl()) { + return this.handleRunCodeJsonl(); } - /** - * Treat as JSONL if there are >1 non-empty lines and at least the second non-empty - * line *appears* to be a JSON object. Robust to whitespace. - */ - eventEditorIsJsonl() { - const model = this.eventEditor?.getModel?.(); - if (!model) return false; - - const rawLines = model.getLinesContent(); - const lines = rawLines.map((l) => l.trim()).filter((l) => l.length > 0); - if (lines.length <= 1) return false; - - const second = lines[1]; - return second.startsWith("{") && second.endsWith("}"); + if (input == null) { + input = this.getState(); } - _getTimezoneOrDefault() { - const tzEl = document.getElementById("timezone-input"); - return tzEl?.value && tzEl.value.trim().length > 0 ? tzEl.value.trim() : "Default"; + if (input.error) { + console.error(input.error); + this.disableJsonLinting(); + this.outputEditor.setValue(input.error); + return input; } - handleRunCode(input) { - this._clearOutput(); - - // JSONL path short-circuit - if (this.eventEditorIsJsonl()) { - return this.handleRunCodeJsonl(); - } - - if (input == null) { - input = this.getState(); - } - - if (input.error) { - console.error(input.error); - this.disableJsonLinting(); - this.outputEditor.setValue(input.error); - return input; - } - - const timezone = this._getTimezoneOrDefault(); - console.debug("Selected timezone: ", timezone); - const runResult = this.run_vrl(input, timezone); - console.log("Run result: ", runResult); - - const {text, isJson} = this._formatRunResult(runResult); - if (isJson) this.enableJsonLinting(); else this.disableJsonLinting(); - this.outputEditor.setValue(text); - - this._setElapsed(runResult?.elapsed_time); - return runResult; - } - - handleRunCodeJsonl() { - this._clearOutput(); - - const program = this._safeGet(this.programEditor); - const model = this.eventEditor?.getModel?.(); - const rawLines = model ? model.getLinesContent() : []; - const lines = rawLines.map(l => l.trim()).filter(l => l.length > 0); - - const timezone = this._getTimezoneOrDefault(); - - // Build inputs while validating JSON per line - const inputs = lines.map(line => ({ - program, - event: this.tryJsonParse(line), - is_jsonl: true, - })); - - // Run and collect results - const results = inputs.map(input => this.run_vrl(input, timezone)); - - const outputs = results.map(r => this._formatRunResult(r).text); - - // Output is not pure JSON (multiple objects / possible errors) - this.disableJsonLinting(); - this.outputEditor.setValue(outputs.join("\n")); - - // Aggregate elapsed time, rounded - const total = results.reduce((sum, r) => sum + (typeof r?.elapsed_time === "number" ? r.elapsed_time : 0), 0); - this._setElapsed(total); - - return results; - } - - handleShareCode() { - const state = this.getState(); - try { - const encoded = encodeURIComponent(btoa(JSON.stringify(state))); - window.history.pushState(state, "", `?state=${encoded}`); - return true; - } catch (e) { - this.disableJsonLinting(); - this.outputEditor.setValue(`Error encoding state for URL\n${e}`); - return false; - } + const timezone = this._getTimezoneOrDefault(); + console.debug("Selected timezone: ", timezone); + const runResult = this.run_vrl(input, timezone); + console.log("Run result: ", runResult); + + const { text, isJson } = this._formatRunResult(runResult); + if (isJson) this.enableJsonLinting(); + else this.disableJsonLinting(); + this.outputEditor.setValue(text); + + this._setElapsed(runResult?.elapsed_time); + return runResult; + } + + handleRunCodeJsonl() { + this._clearOutput(); + + const program = this._safeGet(this.programEditor); + const model = this.eventEditor?.getModel?.(); + const rawLines = model ? model.getLinesContent() : []; + const lines = rawLines.map((l) => l.trim()).filter((l) => l.length > 0); + + const timezone = this._getTimezoneOrDefault(); + + // Build inputs while validating JSON per line + const inputs = lines.map((line) => ({ + program, + event: this.tryJsonParse(line), + is_jsonl: true + })); + + // Run and collect results + const results = inputs.map((input) => this.run_vrl(input, timezone)); + + const outputs = results.map((r) => this._formatRunResult(r).text); + + // Output is not pure JSON (multiple objects / possible errors) + this.disableJsonLinting(); + this.outputEditor.setValue(outputs.join("\n")); + + // Aggregate elapsed time, rounded + const total = results.reduce((sum, r) => sum + (typeof r?.elapsed_time === "number" ? r.elapsed_time : 0), 0); + this._setElapsed(total); + + return results; + } + + handleShareCode() { + const state = this.getState(); + try { + const encoded = encodeURIComponent(btoa(JSON.stringify(state))); + window.history.pushState(state, "", `?state=${encoded}`); + return true; + } catch (e) { + this.disableJsonLinting(); + this.outputEditor.setValue(`Error encoding state for URL\n${e}`); + return false; } + } } // Prefer the async factory to ensure everything is loaded before use: VrlWebPlayground.create() - .then((instance) => { - window.vrlPlayground = instance; - }) - .catch((err) => { - console.error("Failed to initialize VrlWebPlayground:", err); - }); + .then((instance) => { + window.vrlPlayground = instance; + }) + .catch((err) => { + console.error("Failed to initialize VrlWebPlayground:", err); + }); diff --git a/lib/vector-vrl/web-playground/public/vrl-highlighter.js b/lib/vector-vrl/web-playground/public/vrl-highlighter.js index 3efaed67a4a4b..527934106530f 100644 --- a/lib/vector-vrl/web-playground/public/vrl-highlighter.js +++ b/lib/vector-vrl/web-playground/public/vrl-highlighter.js @@ -1,636 +1,633 @@ // VRL Language Definition export let vrlLanguageDefinition = { - defaultToken: "invalid", - ignoreCase: true, - tokenPostfix: ".vrl", + defaultToken: "invalid", + ignoreCase: true, + tokenPostfix: ".vrl", - brackets: [ - { open: "{", close: "}", token: "delimiter.curly" }, - { open: "[", close: "]", token: "delimiter.square" }, - { open: "(", close: ")", token: "delimiter.parenthesis" }, - ], + brackets: [ + { open: "{", close: "}", token: "delimiter.curly" }, + { open: "[", close: "]", token: "delimiter.square" }, + { open: "(", close: ")", token: "delimiter.parenthesis" } + ], - regEx: /\/(?!\/\/)(?:[^\/\\]|\\.)*\/[igm]*/, + regEx: /\/(?!\/\/)(?:[^\/\\]|\\.)*\/[igm]*/, - keywords: [ - "abort", - "as", - "break", - "continue", - "else", - "false", - "for", - "if", - "impl", - "in", - "let", - "loop", - "null", - "return", - "self", - "std", - "then", - "this", - "true", - "type", - "until", - "use", - "while", - ], + keywords: [ + "abort", + "as", + "break", + "continue", + "else", + "false", + "for", + "if", + "impl", + "in", + "let", + "loop", + "null", + "return", + "self", + "std", + "then", + "this", + "true", + "type", + "until", + "use", + "while" + ], - // we include these common regular expressions - symbols: /[=>[a-z]+)\\.(?P[a-z]+)')" log2metric: type: "log_to_metric" - inputs: [ "remap" ] + inputs: ["remap"] metrics: - - type: "counter" + - type: "counter" field: "procid" tags: hostname: "{{ hostname }}" @@ -41,13 +41,13 @@ transforms: sinks: prometheus: - type: "prometheus_exporter" - inputs: [ "internal_metrics" ] + type: "prometheus_exporter" + inputs: ["internal_metrics"] address: "0.0.0.0:9090" datadog_metrics: - type: "datadog_metrics" - inputs: [ "log2metric" ] - endpoint: "http://0.0.0.0:8080" - default_api_key: "DEADBEEF" + type: "datadog_metrics" + inputs: ["log2metric"] + endpoint: "http://0.0.0.0:8080" + default_api_key: "DEADBEEF" default_namespace: "vector" diff --git a/regression/cases/syslog_splunk_hec_logs/lading/lading.yaml b/regression/cases/syslog_splunk_hec_logs/lading/lading.yaml index 8c8c616ea4488..170cc63205295 100644 --- a/regression/cases/syslog_splunk_hec_logs/lading/lading.yaml +++ b/regression/cases/syslog_splunk_hec_logs/lading/lading.yaml @@ -1,6 +1,40 @@ generator: - tcp: - seed: [2, 3, 5, 7, 11, 13, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131, 137] + seed: + [ + 2, + 3, + 5, + 7, + 11, + 13, + 19, + 23, + 29, + 31, + 37, + 41, + 43, + 47, + 53, + 59, + 61, + 67, + 71, + 73, + 79, + 83, + 89, + 97, + 101, + 103, + 107, + 109, + 113, + 127, + 131, + 137 + ] addr: "0.0.0.0:8282" variant: "syslog5424" bytes_per_second: "500 Mb" diff --git a/regression/cases/syslog_splunk_hec_logs/vector/vector.yaml b/regression/cases/syslog_splunk_hec_logs/vector/vector.yaml index e4d63f4e23b41..8c5874e22b718 100644 --- a/regression/cases/syslog_splunk_hec_logs/vector/vector.yaml +++ b/regression/cases/syslog_splunk_hec_logs/vector/vector.yaml @@ -9,10 +9,10 @@ sources: type: "internal_metrics" syslog: - type: "syslog" - address: "0.0.0.0:8282" + type: "syslog" + address: "0.0.0.0:8282" max_length: 1500000 - mode: "tcp" + mode: "tcp" ## ## Sinks @@ -20,16 +20,16 @@ sources: sinks: prometheus: - type: "prometheus_exporter" - inputs: [ "internal_metrics" ] + type: "prometheus_exporter" + inputs: ["internal_metrics"] address: "0.0.0.0:9090" splunk_hec_logs: - type: "splunk_hec_logs" - inputs: [ "syslog" ] + type: "splunk_hec_logs" + inputs: ["syslog"] endpoint: "http://0.0.0.0:8080" encoding: codec: "json" - token: "abcd1234" + token: "abcd1234" healthcheck: enabled: false diff --git a/regression/config.yaml b/regression/config.yaml index e7a3f5b1e5dab..9b80bae63a965 100644 --- a/regression/config.yaml +++ b/regression/config.yaml @@ -3,7 +3,6 @@ lading: target: - # Link templates for reports. # # Values may be removed to disable corresponding links in reports. diff --git a/scripts/environment/prepare.sh b/scripts/environment/prepare.sh index 397fefbb464a8..e74aba915ad5f 100755 --- a/scripts/environment/prepare.sh +++ b/scripts/environment/prepare.sh @@ -38,6 +38,7 @@ DD_RUST_LICENSE_TOOL_VERSION="1.0.6" CARGO_LLVM_COV_VERSION="0.8.4" WASM_PACK_VERSION="0.13.1" MARKDOWNLINT_CLI2_VERSION="0.22.0" +PRETTIER_VERSION="3.8.1" DATADOG_CI_VERSION="5.9.0" VDEV_VERSION="0.3.0" @@ -53,6 +54,7 @@ ALL_MODULES=( dd-rust-license-tool wasm-pack markdownlint-cli2 + prettier datadog-ci release-flags # Not a tool - sources release-flags.sh to set CI env vars vdev @@ -93,6 +95,7 @@ Modules: dd-rust-license-tool wasm-pack markdownlint-cli2 + prettier datadog-ci vdev @@ -219,4 +222,5 @@ maybe_install_cargo_tool wasm-pack "${WASM_PACK_VERSION}" maybe_install_cargo_tool vdev "${VDEV_VERSION}" maybe_install_npm_package markdownlint-cli2 markdownlint-cli2 "${MARKDOWNLINT_CLI2_VERSION}" +maybe_install_npm_package prettier prettier "${PRETTIER_VERSION}" maybe_install_npm_package datadog-ci "@datadog/datadog-ci" "${DATADOG_CI_VERSION}" "v${DATADOG_CI_VERSION}" "version" diff --git a/tests/data/cmd/config/config_3.json b/tests/data/cmd/config/config_3.json index 3517f16295177..a5e68a2ffd678 100644 --- a/tests/data/cmd/config/config_3.json +++ b/tests/data/cmd/config/config_3.json @@ -16,9 +16,7 @@ }, "transforms": { "transform0": { - "inputs": [ - "source0" - ], + "inputs": ["source0"], "drop_on_abort": false, "drop_on_error": false, "metric_tag_values": "single", @@ -29,9 +27,7 @@ }, "sinks": { "sink0": { - "inputs": [ - "transform0" - ], + "inputs": ["transform0"], "target": "stdout", "type": "console", "encoding": { diff --git a/tests/data/secret-backends/file-secrets.json b/tests/data/secret-backends/file-secrets.json index 8de9e8b673504..861c0cbdbc1a0 100644 --- a/tests/data/secret-backends/file-secrets.json +++ b/tests/data/secret-backends/file-secrets.json @@ -1 +1 @@ -{"ghi":"ghi.retrieved"} +{ "ghi": "ghi.retrieved" } diff --git a/tests/e2e/datadog-logs/config/compose.yaml b/tests/e2e/datadog-logs/config/compose.yaml index a144abed68cc5..774a47db4c729 100644 --- a/tests/e2e/datadog-logs/config/compose.yaml +++ b/tests/e2e/datadog-logs/config/compose.yaml @@ -1,4 +1,4 @@ -version: '3.8' +version: "3.8" services: # Generates random log data for consumption by the custom Agent check @@ -28,11 +28,11 @@ services: depends_on: - fakeintake-agent environment: - - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} - - DD_HOSTNAME=datadog-agent - - DD_ENABLE_PAYLOADS_EVENTS=false - - DD_ENABLE_PAYLOADS_SERVICE_CHECKS=false - - DD_CONTAINER_EXCLUDE="name:.*" + - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} + - DD_HOSTNAME=datadog-agent + - DD_ENABLE_PAYLOADS_EVENTS=false + - DD_ENABLE_PAYLOADS_SERVICE_CHECKS=false + - DD_CONTAINER_EXCLUDE="name:.*" volumes: # The Agent config file - type: bind @@ -58,11 +58,11 @@ services: depends_on: - vector environment: - - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} - - DD_HOSTNAME=datadog-agent-vector - - DD_ENABLE_PAYLOADS_EVENTS=false - - DD_ENABLE_PAYLOADS_SERVICE_CHECKS=false - - DD_CONTAINER_EXCLUDE="name:.*" + - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} + - DD_HOSTNAME=datadog-agent-vector + - DD_ENABLE_PAYLOADS_EVENTS=false + - DD_ENABLE_PAYLOADS_SERVICE_CHECKS=false + - DD_CONTAINER_EXCLUDE="name:.*" volumes: # The Agent config file - type: bind diff --git a/tests/e2e/datadog-logs/config/test.yaml b/tests/e2e/datadog-logs/config/test.yaml index 2ce4641b04e96..6582946497e7a 100644 --- a/tests/e2e/datadog-logs/config/test.yaml +++ b/tests/e2e/datadog-logs/config/test.yaml @@ -1,5 +1,5 @@ features: -- e2e-tests-datadog + - e2e-tests-datadog test: "e2e" @@ -7,22 +7,21 @@ test_filter: "datadog::logs::" runner: env: - EXPECTED_LOG_EVENTS: '1000' - VECTOR_RECEIVE_PORT: '8081' - FAKE_INTAKE_AGENT_ENDPOINT: 'http://fakeintake-agent:80' - FAKE_INTAKE_VECTOR_ENDPOINT: 'http://fakeintake-vector:80' + EXPECTED_LOG_EVENTS: "1000" + VECTOR_RECEIVE_PORT: "8081" + FAKE_INTAKE_AGENT_ENDPOINT: "http://fakeintake-agent:80" + FAKE_INTAKE_VECTOR_ENDPOINT: "http://fakeintake-vector:80" matrix: # validate against the latest Agent nightly and also stable v6 and v7 - agent_version: ['latest', '6', '7'] - + agent_version: ["latest", "6", "7"] # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/common/datadog.rs" -- "src/sources/datadog_agent/**" -- "src/internal_events/datadog_*" -- "src/sinks/datadog/logs/**" -- "src/sinks/util/**" -- "tests/e2e/datadog-logs/**" + - "src/common/datadog.rs" + - "src/sources/datadog_agent/**" + - "src/internal_events/datadog_*" + - "src/sinks/datadog/logs/**" + - "src/sinks/util/**" + - "tests/e2e/datadog-logs/**" diff --git a/tests/e2e/datadog-logs/data/agent_only.yaml b/tests/e2e/datadog-logs/data/agent_only.yaml index b92e08db48f05..5f97845ce9b0e 100644 --- a/tests/e2e/datadog-logs/data/agent_only.yaml +++ b/tests/e2e/datadog-logs/data/agent_only.yaml @@ -1,5 +1,5 @@ api_key: DEADBEEF -log_level: 'debug' +log_level: "debug" # disable bunch of stuff we don't need inventories_configuration_enabled: false @@ -29,4 +29,4 @@ logs_config: batch_wait: 1 # Required per https://github.com/DataDog/datadog-agent/tree/main/test/fakeintake#docker -dd_url: 'http://fakeintake-agent:80' +dd_url: "http://fakeintake-agent:80" diff --git a/tests/e2e/datadog-logs/data/agent_vector.yaml b/tests/e2e/datadog-logs/data/agent_vector.yaml index 57ca8bff2d5e7..dd5432566d7c0 100644 --- a/tests/e2e/datadog-logs/data/agent_vector.yaml +++ b/tests/e2e/datadog-logs/data/agent_vector.yaml @@ -1,5 +1,5 @@ api_key: DEADBEEF -log_level: 'debug' +log_level: "debug" # disable bunch of stuff we don't need inventories_configuration_enabled: false @@ -34,4 +34,4 @@ vector: url: "http://vector:8181" # Required per https://github.com/DataDog/datadog-agent/tree/main/test/fakeintake#docker -dd_url: 'http://fakeintake-agent:80' +dd_url: "http://fakeintake-agent:80" diff --git a/tests/e2e/datadog-metrics/config/compose.yaml b/tests/e2e/datadog-metrics/config/compose.yaml index 84c0f89278863..6a317c156182f 100644 --- a/tests/e2e/datadog-metrics/config/compose.yaml +++ b/tests/e2e/datadog-metrics/config/compose.yaml @@ -1,12 +1,11 @@ -version: '3' +version: "3" services: - # Emits metrics to the Agent only path dogstatsd-client-agent: build: ../dogstatsd_client environment: - - STATSD_HOST=datadog-agent + - STATSD_HOST=datadog-agent depends_on: - datadog-agent @@ -14,7 +13,7 @@ services: dogstatsd-client-vector: build: ../dogstatsd_client environment: - - STATSD_HOST=datadog-agent-vector + - STATSD_HOST=datadog-agent-vector depends_on: - datadog-agent-vector @@ -24,8 +23,8 @@ services: depends_on: - fakeintake-agent environment: - - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} - - DD_HOSTNAME=datadog-agent + - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} + - DD_HOSTNAME=datadog-agent volumes: # The Agent config file - ../data/agent_only.yaml:/etc/datadog-agent/datadog.yaml @@ -36,8 +35,8 @@ services: depends_on: - vector environment: - - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} - - DD_HOSTNAME=datadog-agent-vector + - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} + - DD_HOSTNAME=datadog-agent-vector volumes: # The Agent config file - ../data/agent_vector.yaml:/etc/datadog-agent/datadog.yaml diff --git a/tests/e2e/datadog-metrics/config/test.yaml b/tests/e2e/datadog-metrics/config/test.yaml index 2b551be47c147..87b30504ace2f 100644 --- a/tests/e2e/datadog-metrics/config/test.yaml +++ b/tests/e2e/datadog-metrics/config/test.yaml @@ -1,28 +1,28 @@ features: -- e2e-tests-datadog + - e2e-tests-datadog test: "e2e" -test_filter: 'datadog::metrics::' +test_filter: "datadog::metrics::" runner: env: - VECTOR_RECEIVE_PORT: '8081' - FAKE_INTAKE_AGENT_ENDPOINT: 'http://fakeintake-agent:80' - FAKE_INTAKE_VECTOR_ENDPOINT: 'http://fakeintake-vector:80' + VECTOR_RECEIVE_PORT: "8081" + FAKE_INTAKE_AGENT_ENDPOINT: "http://fakeintake-agent:80" + FAKE_INTAKE_VECTOR_ENDPOINT: "http://fakeintake-vector:80" matrix: # validate against the Agent latest nightly and also stable v6 and v7 - agent_version: ['latest', '6', '7'] + agent_version: ["latest", "6", "7"] # validate both series API versions - series_api_version: ['v1', 'v2'] + series_api_version: ["v1", "v2"] # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/common/datadog.rs" -- "src/sources/datadog_agent/**" -- "src/internal_events/datadog_*" -- "src/sinks/datadog/metrics/**" -- "src/sinks/util/**" -- "tests/e2e/datadog-metrics/**" + - "src/common/datadog.rs" + - "src/sources/datadog_agent/**" + - "src/internal_events/datadog_*" + - "src/sinks/datadog/metrics/**" + - "src/sinks/util/**" + - "tests/e2e/datadog-metrics/**" diff --git a/tests/e2e/datadog-metrics/data/agent_only.yaml b/tests/e2e/datadog-metrics/data/agent_only.yaml index 0ed0518c84b18..6b69a11cdf2cc 100644 --- a/tests/e2e/datadog-metrics/data/agent_only.yaml +++ b/tests/e2e/datadog-metrics/data/agent_only.yaml @@ -1,5 +1,5 @@ api_key: DEADBEEF -log_level: 'warn' +log_level: "warn" # disable bunch of stuff we don't need inventories_configuration_enabled: false diff --git a/tests/e2e/datadog-metrics/data/agent_vector.yaml b/tests/e2e/datadog-metrics/data/agent_vector.yaml index c1566946a96b6..22709586bbd7b 100644 --- a/tests/e2e/datadog-metrics/data/agent_vector.yaml +++ b/tests/e2e/datadog-metrics/data/agent_vector.yaml @@ -1,5 +1,5 @@ api_key: DEADBEEF -log_level: 'warn' +log_level: "warn" # disable bunch of stuff we don't need inventories_configuration_enabled: false diff --git a/tests/e2e/opentelemetry-logs/config/compose.yaml b/tests/e2e/opentelemetry-logs/config/compose.yaml index 62b3d48c44aa6..4066a553d2257 100644 --- a/tests/e2e/opentelemetry-logs/config/compose.yaml +++ b/tests/e2e/opentelemetry-logs/config/compose.yaml @@ -12,7 +12,7 @@ services: ports: - "${OTEL_COLLECTOR_SOURCE_GRPC_PORT:-4317}:4317" - "${OTEL_COLLECTOR_SOURCE_HTTP_PORT:-4318}:4318" - command: [ "--config=/etc/otelcol-contrib/config.yaml" ] + command: ["--config=/etc/otelcol-contrib/config.yaml"] logs-generator: container_name: logs-generator @@ -49,7 +49,7 @@ services: CONFIG_COLLECTOR_VERSION: ${CONFIG_COLLECTOR_VERSION} init: true user: "0:0" # test only, override special user with root - command: [ "--config", "/etc/otelcol-contrib/config.yaml" ] + command: ["--config", "/etc/otelcol-contrib/config.yaml"] volumes: - type: bind source: ../data/collector-sink.yaml @@ -76,7 +76,7 @@ services: environment: - VECTOR_LOG=${VECTOR_LOG:-info} - FEATURES=e2e-tests-opentelemetry - command: [ "vector", "-c", "/etc/vector/vector.yaml" ] + command: ["vector", "-c", "/etc/vector/vector.yaml"] volumes: vector_target: diff --git a/tests/e2e/opentelemetry-logs/config/test.yaml b/tests/e2e/opentelemetry-logs/config/test.yaml index 54aa7063e632c..e1c5cb85267d2 100644 --- a/tests/e2e/opentelemetry-logs/config/test.yaml +++ b/tests/e2e/opentelemetry-logs/config/test.yaml @@ -8,14 +8,14 @@ test_filter: "opentelemetry::logs::" runner: needs_docker_socket: true env: - OTEL_COLLECTOR_SOURCE_GRPC_PORT: '4317' - OTEL_COLLECTOR_SOURCE_HTTP_PORT: '4318' - OTEL_COLLECTOR_SINK_HTTP_PORT: '5318' + OTEL_COLLECTOR_SOURCE_GRPC_PORT: "4317" + OTEL_COLLECTOR_SOURCE_HTTP_PORT: "4318" + OTEL_COLLECTOR_SINK_HTTP_PORT: "5318" matrix: # Determines which `otel/opentelemetry-collector-contrib` version to use - collector_version: [ 'latest' ] - vector_config: [ 'vector_default.yaml', 'vector_otlp.yaml' ] + collector_version: ["latest"] + vector_config: ["vector_default.yaml", "vector_otlp.yaml"] # Only trigger this integration test if relevant OTEL source/sink files change paths: diff --git a/tests/e2e/opentelemetry-logs/data/collector-sink.yaml b/tests/e2e/opentelemetry-logs/data/collector-sink.yaml index e22f62f91d558..1da64319f6926 100644 --- a/tests/e2e/opentelemetry-logs/data/collector-sink.yaml +++ b/tests/e2e/opentelemetry-logs/data/collector-sink.yaml @@ -5,10 +5,10 @@ receivers: endpoint: "0.0.0.0:5318" processors: - batch: { } + batch: {} exporters: - debug: { } + debug: {} file: path: /output/opentelemetry-logs/collector-file-exporter.log rotation: @@ -18,9 +18,9 @@ exporters: service: pipelines: logs: - receivers: [ otlp ] - processors: [ batch ] - exporters: [ debug, file ] + receivers: [otlp] + processors: [batch] + exporters: [debug, file] telemetry: logs: level: "debug" diff --git a/tests/e2e/opentelemetry-logs/data/collector-source.yaml b/tests/e2e/opentelemetry-logs/data/collector-source.yaml index 1a41636209e78..d3f8427a47f33 100644 --- a/tests/e2e/opentelemetry-logs/data/collector-source.yaml +++ b/tests/e2e/opentelemetry-logs/data/collector-source.yaml @@ -6,7 +6,7 @@ receivers: grpc: processors: - batch: { } + batch: {} exporters: otlp/grpc: @@ -17,16 +17,16 @@ exporters: endpoint: http://vector:4318 tls: insecure: true - debug: { } + debug: {} service: pipelines: logs: - receivers: [ otlp ] - processors: [ batch ] - exporters: [ debug, otlp/grpc, otlphttp/vector ] + receivers: [otlp] + processors: [batch] + exporters: [debug, otlp/grpc, otlphttp/vector] metrics: - receivers: [ otlp ] - processors: [ batch ] - exporters: [ debug ] + receivers: [otlp] + processors: [batch] + exporters: [debug] diff --git a/tests/e2e/opentelemetry-logs/data/vector_default.yaml b/tests/e2e/opentelemetry-logs/data/vector_default.yaml index 1f3fd9b119d09..b8e3f6ef9dad3 100644 --- a/tests/e2e/opentelemetry-logs/data/vector_default.yaml +++ b/tests/e2e/opentelemetry-logs/data/vector_default.yaml @@ -46,10 +46,9 @@ transforms: ] } - sinks: otel_sink: - inputs: [ "remap_otel" ] + inputs: ["remap_otel"] type: opentelemetry protocol: type: http diff --git a/tests/e2e/opentelemetry-metrics/config/compose.yaml b/tests/e2e/opentelemetry-metrics/config/compose.yaml index fba9c2a19b247..44d17475c55b2 100644 --- a/tests/e2e/opentelemetry-metrics/config/compose.yaml +++ b/tests/e2e/opentelemetry-metrics/config/compose.yaml @@ -12,7 +12,7 @@ services: ports: - "${OTEL_COLLECTOR_SOURCE_GRPC_PORT:-4317}:4317" - "${OTEL_COLLECTOR_SOURCE_HTTP_PORT:-4318}:4318" - command: [ "--config=/etc/otelcol-contrib/config.yaml" ] + command: ["--config=/etc/otelcol-contrib/config.yaml"] metrics-generator: container_name: metrics-generator @@ -86,7 +86,7 @@ services: CONFIG_COLLECTOR_VERSION: ${CONFIG_COLLECTOR_VERSION} init: true user: "0:0" # test only, override special user with root - command: [ "--config", "/etc/otelcol-contrib/config.yaml" ] + command: ["--config", "/etc/otelcol-contrib/config.yaml"] volumes: - type: bind source: ../data/collector-sink.yaml @@ -113,7 +113,7 @@ services: environment: - VECTOR_LOG=${VECTOR_LOG:-info} - FEATURES=e2e-tests-opentelemetry - command: [ "vector", "-c", "/etc/vector/vector.yaml" ] + command: ["vector", "-c", "/etc/vector/vector.yaml"] volumes: vector_target: diff --git a/tests/e2e/opentelemetry-metrics/config/test.yaml b/tests/e2e/opentelemetry-metrics/config/test.yaml index 507cddfd2b21d..4d5b0bf4d5256 100644 --- a/tests/e2e/opentelemetry-metrics/config/test.yaml +++ b/tests/e2e/opentelemetry-metrics/config/test.yaml @@ -8,13 +8,13 @@ test_filter: "opentelemetry::metrics::" runner: needs_docker_socket: true env: - OTEL_COLLECTOR_SOURCE_GRPC_PORT: '4317' - OTEL_COLLECTOR_SOURCE_HTTP_PORT: '4318' - OTEL_COLLECTOR_SINK_HTTP_PORT: '5318' + OTEL_COLLECTOR_SOURCE_GRPC_PORT: "4317" + OTEL_COLLECTOR_SOURCE_HTTP_PORT: "4318" + OTEL_COLLECTOR_SINK_HTTP_PORT: "5318" matrix: # Determines which `otel/opentelemetry-collector-contrib` version to use - collector_version: [ 'latest' ] + collector_version: ["latest"] # Only trigger this integration test if relevant OTEL source/sink files change paths: diff --git a/tests/e2e/opentelemetry-metrics/data/collector-sink.yaml b/tests/e2e/opentelemetry-metrics/data/collector-sink.yaml index 120b29ff8e20f..6fb3eb25e1522 100644 --- a/tests/e2e/opentelemetry-metrics/data/collector-sink.yaml +++ b/tests/e2e/opentelemetry-metrics/data/collector-sink.yaml @@ -5,10 +5,10 @@ receivers: endpoint: "0.0.0.0:5318" processors: - batch: { } + batch: {} exporters: - debug: { } + debug: {} file: path: /output/opentelemetry-metrics/collector-file-exporter.log rotation: @@ -18,9 +18,9 @@ exporters: service: pipelines: metrics: - receivers: [ otlp ] - processors: [ batch ] - exporters: [ debug, file ] + receivers: [otlp] + processors: [batch] + exporters: [debug, file] telemetry: logs: level: "debug" diff --git a/tests/e2e/opentelemetry-metrics/data/collector-source.yaml b/tests/e2e/opentelemetry-metrics/data/collector-source.yaml index 6f1ba27560e87..bf40ac05aa78c 100644 --- a/tests/e2e/opentelemetry-metrics/data/collector-source.yaml +++ b/tests/e2e/opentelemetry-metrics/data/collector-source.yaml @@ -6,7 +6,7 @@ receivers: grpc: processors: - batch: { } + batch: {} exporters: otlp/grpc: @@ -17,11 +17,11 @@ exporters: endpoint: http://vector:4318 tls: insecure: true - debug: { } + debug: {} service: pipelines: metrics: - receivers: [ otlp ] - processors: [ batch ] - exporters: [ debug, otlp/grpc, otlphttp/vector ] + receivers: [otlp] + processors: [batch] + exporters: [debug, otlp/grpc, otlphttp/vector] diff --git a/tests/integration/amqp/config/compose.yaml b/tests/integration/amqp/config/compose.yaml index 3eb859f93aa60..643f0b13f083f 100644 --- a/tests/integration/amqp/config/compose.yaml +++ b/tests/integration/amqp/config/compose.yaml @@ -1,4 +1,4 @@ -version: '3' +version: "3" services: rabbitmq: diff --git a/tests/integration/amqp/config/test.yaml b/tests/integration/amqp/config/test.yaml index 94c080a18559d..40a43748cd12d 100644 --- a/tests/integration/amqp/config/test.yaml +++ b/tests/integration/amqp/config/test.yaml @@ -1,17 +1,17 @@ features: -- amqp-integration-tests + - amqp-integration-tests -test_filter: '::amqp::' +test_filter: "::amqp::" matrix: - version: ['3.8'] + version: ["3.8"] # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/amqp.rs" -- "src/internal_events/amqp.rs" -- "src/sinks/amqp/**" -- "src/sources/amqp.rs" -- "src/sources/util/**" -- "src/sinks/util/**" + - "src/amqp.rs" + - "src/internal_events/amqp.rs" + - "src/sinks/amqp/**" + - "src/sources/amqp.rs" + - "src/sources/util/**" + - "src/sinks/util/**" diff --git a/tests/integration/appsignal/config/test.yaml b/tests/integration/appsignal/config/test.yaml index d110306916df0..da826955f726a 100644 --- a/tests/integration/appsignal/config/test.yaml +++ b/tests/integration/appsignal/config/test.yaml @@ -1,7 +1,7 @@ features: -- appsignal-integration-tests + - appsignal-integration-tests -test_filter: '::appsignal::integration_tests::' +test_filter: "::appsignal::integration_tests::" runner: env: @@ -13,5 +13,5 @@ matrix: # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/sinks/appsignal/**" -- "src/sinks/util/**" + - "src/sinks/appsignal/**" + - "src/sinks/util/**" diff --git a/tests/integration/aws/config/compose.yaml b/tests/integration/aws/config/compose.yaml index 4fbbd62f6e120..a468d8e4c0a96 100644 --- a/tests/integration/aws/config/compose.yaml +++ b/tests/integration/aws/config/compose.yaml @@ -1,4 +1,4 @@ -version: '3' +version: "3" services: mock-ec2-metadata: @@ -6,15 +6,15 @@ services: mock-localstack: image: docker.io/localstack/localstack@sha256:46302bcb91a7e8008e6394be8afafdbfa40fb77a54d4046a38be35992042d5de environment: - - SERVICES=kinesis,s3,cloudwatch,es,firehose,kms,sqs,sns,logs + - SERVICES=kinesis,s3,cloudwatch,es,firehose,kms,sqs,sns,logs mock-ecs: image: docker.io/amazon/amazon-ecs-local-container-endpoints:latest environment: - # https://github.com/vectordotdev/vector/issues/24687 - - DOCKER_API_VERSION=1.44 + # https://github.com/vectordotdev/vector/issues/24687 + - DOCKER_API_VERSION=1.44 volumes: - - $DOCKER_SOCKET:/var/run/docker.sock - - $HOME/.aws/:/home/.aws/ + - $DOCKER_SOCKET:/var/run/docker.sock + - $HOME/.aws/:/home/.aws/ networks: default: diff --git a/tests/integration/aws/config/test.yaml b/tests/integration/aws/config/test.yaml index 956f724978fa3..d27214d9499de 100644 --- a/tests/integration/aws/config/test.yaml +++ b/tests/integration/aws/config/test.yaml @@ -1,5 +1,5 @@ features: -- aws-integration-tests + - aws-integration-tests test_filter: ::aws_ @@ -22,11 +22,11 @@ matrix: # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/aws/**" -- "src/internal_events/aws_*" -- "src/sources/aws_*/**" -- "src/sources/util/**" -- "src/sinks/aws_*/**" -- "src/sinks/util/**" -- "src/transforms/aws_*" -- "tests/integration/aws/**" + - "src/aws/**" + - "src/internal_events/aws_*" + - "src/sources/aws_*/**" + - "src/sources/util/**" + - "src/sinks/aws_*/**" + - "src/sinks/util/**" + - "src/transforms/aws_*" + - "tests/integration/aws/**" diff --git a/tests/integration/axiom/config/test.yaml b/tests/integration/axiom/config/test.yaml index 934bd739eeaca..ed542ece401d7 100644 --- a/tests/integration/axiom/config/test.yaml +++ b/tests/integration/axiom/config/test.yaml @@ -1,7 +1,7 @@ features: -- axiom-integration-tests + - axiom-integration-tests -test_filter: '::axiom::' +test_filter: "::axiom::" runner: env: @@ -16,6 +16,6 @@ matrix: # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/sinks/axiom.rs" -- "src/sinks/util/**" -- "tests/integration/axiom/**" + - "src/sinks/axiom.rs" + - "src/sinks/util/**" + - "tests/integration/axiom/**" diff --git a/tests/integration/azure/config/compose.yaml b/tests/integration/azure/config/compose.yaml index 933a72d235a24..fe3541bf2a07c 100644 --- a/tests/integration/azure/config/compose.yaml +++ b/tests/integration/azure/config/compose.yaml @@ -1,11 +1,11 @@ -version: '3' +version: "3" services: local-azure-blob: image: mcr.microsoft.com/azure-storage/azurite:${CONFIG_VERSION} command: azurite --blobHost 0.0.0.0 --loose volumes: - - /var/run:/var/run + - /var/run:/var/run networks: default: diff --git a/tests/integration/azure/config/test.yaml b/tests/integration/azure/config/test.yaml index 191bea7256cea..da00ceed79ad4 100644 --- a/tests/integration/azure/config/test.yaml +++ b/tests/integration/azure/config/test.yaml @@ -1,5 +1,5 @@ features: -- azure-integration-tests + - azure-integration-tests test_filter: ::azure_ @@ -9,11 +9,11 @@ env: LOGSTASH_ADDRESS: 0.0.0.0:8081 matrix: - version: [ 3.35.0 ] + version: [3.35.0] # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/sinks/azure_**" -- "src/sinks/util/**" -- "tests/integration/azure/**" + - "src/sinks/azure_**" + - "src/sinks/util/**" + - "tests/integration/azure/**" diff --git a/tests/integration/clickhouse/config/compose.yaml b/tests/integration/clickhouse/config/compose.yaml index fd823897ec4c2..e861c92c736f0 100644 --- a/tests/integration/clickhouse/config/compose.yaml +++ b/tests/integration/clickhouse/config/compose.yaml @@ -1,4 +1,4 @@ -version: '3' +version: "3" services: clickhouse: diff --git a/tests/integration/clickhouse/config/test.yaml b/tests/integration/clickhouse/config/test.yaml index 3418d810fa638..3f88eedf5a654 100644 --- a/tests/integration/clickhouse/config/test.yaml +++ b/tests/integration/clickhouse/config/test.yaml @@ -1,17 +1,17 @@ features: -- clickhouse-integration-tests + - clickhouse-integration-tests -test_filter: '::clickhouse::' +test_filter: "::clickhouse::" env: CLICKHOUSE_ADDRESS: http://clickhouse:8123 matrix: - version: ['23'] + version: ["23"] # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/sinks/clickhouse/**" -- "src/sinks/util/**" -- "tests/integration/clickhouse/**" + - "src/sinks/clickhouse/**" + - "src/sinks/util/**" + - "tests/integration/clickhouse/**" diff --git a/tests/integration/databend/config/compose.yaml b/tests/integration/databend/config/compose.yaml index 213fb20af70fe..e98305ba2ea01 100644 --- a/tests/integration/databend/config/compose.yaml +++ b/tests/integration/databend/config/compose.yaml @@ -1,4 +1,4 @@ -version: '3' +version: "3" services: minio: diff --git a/tests/integration/databend/config/test.yaml b/tests/integration/databend/config/test.yaml index eaa9f5ad50d86..0ec4b2c9d57f4 100644 --- a/tests/integration/databend/config/test.yaml +++ b/tests/integration/databend/config/test.yaml @@ -1,18 +1,18 @@ features: -- databend-integration-tests + - databend-integration-tests -test_filter: '::databend::' +test_filter: "::databend::" runner: env: DATABEND_ENDPOINT: databend://vector:vector@databend:8000?sslmode=disable&presign=detect matrix: - version: ['latest'] + version: ["latest"] # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/sinks/databend/**" -- "src/sinks/util/**" -- "tests/integration/databend/**" + - "src/sinks/databend/**" + - "src/sinks/util/**" + - "tests/integration/databend/**" diff --git a/tests/integration/datadog-agent/config/compose.yaml b/tests/integration/datadog-agent/config/compose.yaml index 4ccf95ff24e00..c63089ac620a9 100644 --- a/tests/integration/datadog-agent/config/compose.yaml +++ b/tests/integration/datadog-agent/config/compose.yaml @@ -1,32 +1,32 @@ -version: '3' +version: "3" services: datadog-agent: image: docker.io/datadog/agent:${CONFIG_VERSION} environment: - - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} - - DD_APM_ENABLED=false - - DD_LOGS_ENABLED=true - - DD_LOGS_CONFIG_LOGS_DD_URL=runner:8080 - - DD_LOGS_CONFIG_LOGS_NO_SSL=true - - DD_LOGS_CONFIG_USE_HTTP=true - - DD_HEALTH_PORT=8182 - - DD_CMD_PORT=5001 - - DD_USE_DOGSTATSD=false - - DD_HOSTNAME=datadog-agent - - DD_SERIALIZER_COMPRESSOR_KIND=zstd + - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} + - DD_APM_ENABLED=false + - DD_LOGS_ENABLED=true + - DD_LOGS_CONFIG_LOGS_DD_URL=runner:8080 + - DD_LOGS_CONFIG_LOGS_NO_SSL=true + - DD_LOGS_CONFIG_USE_HTTP=true + - DD_HEALTH_PORT=8182 + - DD_CMD_PORT=5001 + - DD_USE_DOGSTATSD=false + - DD_HOSTNAME=datadog-agent + - DD_SERIALIZER_COMPRESSOR_KIND=zstd volumes: - - ../data/conf.yaml:/etc/datadog-agent/conf.d/test.d/conf.yaml + - ../data/conf.yaml:/etc/datadog-agent/conf.d/test.d/conf.yaml datadog-trace-agent: image: docker.io/datadog/agent:7.31.0 environment: - - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} - - DD_APM_ENABLED=true - - DD_APM_DD_URL=http://runner:8081 - - DD_HEALTH_PORT=8183 - - DD_CMD_PORT=5002 - - DD_USE_DOGSTATSD=false - - DD_HOSTNAME=datadog-trace-agent + - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} + - DD_APM_ENABLED=true + - DD_APM_DD_URL=http://runner:8081 + - DD_HEALTH_PORT=8183 + - DD_CMD_PORT=5002 + - DD_USE_DOGSTATSD=false + - DD_HOSTNAME=datadog-trace-agent networks: default: diff --git a/tests/integration/datadog-agent/config/test.yaml b/tests/integration/datadog-agent/config/test.yaml index c1d3ec50d7592..353c1c9e02188 100644 --- a/tests/integration/datadog-agent/config/test.yaml +++ b/tests/integration/datadog-agent/config/test.yaml @@ -1,7 +1,7 @@ features: -- datadog-agent-integration-tests + - datadog-agent-integration-tests -test_filter: 'sources::datadog_agent::integration_tests::' +test_filter: "sources::datadog_agent::integration_tests::" env: AGENT_ADDRESS: datadog-agent:8181 @@ -11,12 +11,12 @@ env: TEST_DATADOG_API_KEY: matrix: - version: ['7'] + version: ["7"] # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/common/datadog.rs" -- "src/internal_events/datadog_*" -- "src/sources/datadog_agent/**" -- "tests/integration/datadog-agent/**" + - "src/common/datadog.rs" + - "src/internal_events/datadog_*" + - "src/sources/datadog_agent/**" + - "tests/integration/datadog-agent/**" diff --git a/tests/integration/datadog-logs/config/test.yaml b/tests/integration/datadog-logs/config/test.yaml index f10db04ef5661..3e33d0ffc8922 100644 --- a/tests/integration/datadog-logs/config/test.yaml +++ b/tests/integration/datadog-logs/config/test.yaml @@ -1,7 +1,7 @@ features: -- datadog-logs-integration-tests + - datadog-logs-integration-tests -test_filter: '::datadog::logs::integration_tests::' +test_filter: "::datadog::logs::integration_tests::" runner: env: @@ -13,8 +13,8 @@ matrix: # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/common/datadog.rs" -- "src/internal_events/datadog_*" -- "src/sinks/datadog/logs/**" -- "src/sinks/util/**" -- "tests/integration/datadog-logs/**" + - "src/common/datadog.rs" + - "src/internal_events/datadog_*" + - "src/sinks/datadog/logs/**" + - "src/sinks/util/**" + - "tests/integration/datadog-logs/**" diff --git a/tests/integration/datadog-metrics/config/test.yaml b/tests/integration/datadog-metrics/config/test.yaml index 55282a361eccb..a0cdf30b036a1 100644 --- a/tests/integration/datadog-metrics/config/test.yaml +++ b/tests/integration/datadog-metrics/config/test.yaml @@ -1,7 +1,7 @@ features: -- datadog-metrics-integration-tests + - datadog-metrics-integration-tests -test_filter: '::datadog::metrics::integration_tests' +test_filter: "::datadog::metrics::integration_tests" runner: env: @@ -13,8 +13,8 @@ matrix: # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/common/datadog.rs" -- "src/internal_events/datadog_*" -- "src/sinks/datadog/metrics/**" -- "src/sinks/util/**" -- "tests/integration/datadog-metrics/**" + - "src/common/datadog.rs" + - "src/internal_events/datadog_*" + - "src/sinks/datadog/metrics/**" + - "src/sinks/util/**" + - "tests/integration/datadog-metrics/**" diff --git a/tests/integration/datadog-traces/config/compose.yaml b/tests/integration/datadog-traces/config/compose.yaml index f94ee0b53d089..847db7b3c1387 100644 --- a/tests/integration/datadog-traces/config/compose.yaml +++ b/tests/integration/datadog-traces/config/compose.yaml @@ -1,34 +1,34 @@ -version: '3' +version: "3" services: datadog-trace-agent: image: docker.io/datadog/agent:${CONFIG_VERSION} environment: - - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} - - DD_APM_ENABLED=true - - DD_APM_DD_URL=http://runner:8082 - - DD_HEALTH_PORT=8183 - - DD_CMD_PORT=5002 - - DD_USE_DOGSTATSD=false - - DD_APM_MAX_TPS=999999 - - DD_APM_ERROR_TPS=999999 - - DD_APM_MAX_MEMORY=0 - - DD_APM_MAX_CPU_PERCENT=0 - - DD_HOSTNAME=datadog-trace-agent + - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} + - DD_APM_ENABLED=true + - DD_APM_DD_URL=http://runner:8082 + - DD_HEALTH_PORT=8183 + - DD_CMD_PORT=5002 + - DD_USE_DOGSTATSD=false + - DD_APM_MAX_TPS=999999 + - DD_APM_ERROR_TPS=999999 + - DD_APM_MAX_MEMORY=0 + - DD_APM_MAX_CPU_PERCENT=0 + - DD_HOSTNAME=datadog-trace-agent datadog-trace-agent-to-vector: image: docker.io/datadog/agent:${CONFIG_VERSION} environment: - - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} - - DD_APM_ENABLED=true - - DD_APM_DD_URL=http://runner:8081 - - DD_HEALTH_PORT=8183 - - DD_CMD_PORT=5002 - - DD_USE_DOGSTATSD=false - - DD_APM_MAX_TPS=999999 - - DD_APM_ERROR_TPS=999999 - - DD_APM_MAX_MEMORY=0 - - DD_APM_MAX_CPU_PERCENT=0 - - DD_HOSTNAME=datadog-trace-agent-to-vector + - DD_API_KEY=${TEST_DATADOG_API_KEY:?TEST_DATADOG_API_KEY required} + - DD_APM_ENABLED=true + - DD_APM_DD_URL=http://runner:8081 + - DD_HEALTH_PORT=8183 + - DD_CMD_PORT=5002 + - DD_USE_DOGSTATSD=false + - DD_APM_MAX_TPS=999999 + - DD_APM_ERROR_TPS=999999 + - DD_APM_MAX_MEMORY=0 + - DD_APM_MAX_CPU_PERCENT=0 + - DD_HOSTNAME=datadog-trace-agent-to-vector networks: default: diff --git a/tests/integration/datadog-traces/config/test.yaml b/tests/integration/datadog-traces/config/test.yaml index 2a3f162e61b30..7f72063f617c7 100644 --- a/tests/integration/datadog-traces/config/test.yaml +++ b/tests/integration/datadog-traces/config/test.yaml @@ -1,14 +1,14 @@ features: -- datadog-traces-integration-tests + - datadog-traces-integration-tests -test_filter: '::datadog::traces::' +test_filter: "::datadog::traces::" env: - AGENT_PORT: '8082' + AGENT_PORT: "8082" VECTOR_LOG: ${VECTOR_LOG} TRACE_AGENT_TO_VECTOR_URL: http://datadog-trace-agent-to-vector:8126/v0.3/traces TRACE_AGENT_URL: http://datadog-trace-agent:8126/v0.3/traces - VECTOR_PORT: '8081' + VECTOR_PORT: "8081" TEST_DATADOG_API_KEY: matrix: @@ -17,8 +17,8 @@ matrix: # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/common/datadog.rs" -- "src/internal_events/datadog_*" -- "src/sinks/datadog/traces/**" -- "src/sinks/util/**" -- "tests/integration/datadog-traces/**" + - "src/common/datadog.rs" + - "src/internal_events/datadog_*" + - "src/sinks/datadog/traces/**" + - "src/sinks/util/**" + - "tests/integration/datadog-traces/**" diff --git a/tests/integration/dnstap/config/test.yaml b/tests/integration/dnstap/config/test.yaml index ed0de1d3ea411..8ff55500cbf59 100644 --- a/tests/integration/dnstap/config/test.yaml +++ b/tests/integration/dnstap/config/test.yaml @@ -1,7 +1,7 @@ features: -- dnstap-integration-tests + - dnstap-integration-tests -test_filter: '::dnstap::' +test_filter: "::dnstap::" runner: env: @@ -11,12 +11,12 @@ runner: dnstap_dnstap-sockets: /run/bind/socket matrix: - version: ['latest'] + version: ["latest"] # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/internal_events/dnstap.rs" -- "src/sources/dnstap/**" -- "src/sources/util/**" -- "tests/integration/dnstap/**" + - "src/internal_events/dnstap.rs" + - "src/sources/dnstap/**" + - "src/sources/util/**" + - "tests/integration/dnstap/**" diff --git a/tests/integration/docker-logs/config/test.yaml b/tests/integration/docker-logs/config/test.yaml index 0ac64071bd516..2ddd643652e9c 100644 --- a/tests/integration/docker-logs/config/test.yaml +++ b/tests/integration/docker-logs/config/test.yaml @@ -1,5 +1,5 @@ features: -- docker-logs-integration-tests + - docker-logs-integration-tests test_filter: "::docker_logs::" @@ -12,8 +12,8 @@ matrix: # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/docker.rs" -- "src/internal_events/docker_logs.rs" -- "src/sources/docker_logs/**" -- "src/sources/util/**" -- "tests/integration/docker-logs/**" + - "src/docker.rs" + - "src/internal_events/docker_logs.rs" + - "src/sources/docker_logs/**" + - "src/sources/util/**" + - "tests/integration/docker-logs/**" diff --git a/tests/integration/doris/config/test.yaml b/tests/integration/doris/config/test.yaml index 799a369d19a2f..e09e964991eb4 100644 --- a/tests/integration/doris/config/test.yaml +++ b/tests/integration/doris/config/test.yaml @@ -1,19 +1,19 @@ features: -- sinks-doris -- doris-integration-tests + - sinks-doris + - doris-integration-tests -test_filter: 'sinks::doris::integration_test::' +test_filter: "sinks::doris::integration_test::" env: DORIS_ADDRESS: http://doris:8040 matrix: - version: ['2.1.7'] + version: ["2.1.7"] # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/internal_events/doris.rs" -- "src/sinks/doris/**" -- "src/sinks/util/**" -- "tests/integration/doris/**" + - "src/internal_events/doris.rs" + - "src/sinks/doris/**" + - "src/sinks/util/**" + - "tests/integration/doris/**" diff --git a/tests/integration/elasticsearch/config/compose.yaml b/tests/integration/elasticsearch/config/compose.yaml index 4dcb6636bf4cb..cb6d0bcc2f392 100644 --- a/tests/integration/elasticsearch/config/compose.yaml +++ b/tests/integration/elasticsearch/config/compose.yaml @@ -1,30 +1,30 @@ -version: '3' +version: "3" services: localstack: image: docker.io/localstack/localstack@sha256:f21f1fc770ee4bfd5012afdc902154c56b7fb18c14cf672de151b65569c8251e environment: - - SERVICES=elasticsearch:4571 + - SERVICES=elasticsearch:4571 elasticsearch: image: docker.io/elasticsearch:${CONFIG_VERSION} environment: - - discovery.type=single-node - - ES_JAVA_OPTS=-Xms400m -Xmx400m + - discovery.type=single-node + - ES_JAVA_OPTS=-Xms400m -Xmx400m elasticsearch-secure: image: docker.io/elasticsearch:${CONFIG_VERSION} environment: - - discovery.type=single-node - - xpack.security.enabled=true - - xpack.security.http.ssl.enabled=true - - xpack.security.http.ssl.certificate=certs/intermediate_server/certs/elasticsearch-secure-chain.cert.pem - - xpack.security.http.ssl.key=certs/intermediate_server/private/elasticsearch-secure.key.pem - - xpack.security.transport.ssl.enabled=true - - xpack.security.transport.ssl.certificate=certs/intermediate_server/certs/elasticsearch-secure-chain.cert.pem - - xpack.security.transport.ssl.key=certs/intermediate_server/private/elasticsearch-secure.key.pem - - ELASTIC_PASSWORD=vector - - ES_JAVA_OPTS=-Xms400m -Xmx400m + - discovery.type=single-node + - xpack.security.enabled=true + - xpack.security.http.ssl.enabled=true + - xpack.security.http.ssl.certificate=certs/intermediate_server/certs/elasticsearch-secure-chain.cert.pem + - xpack.security.http.ssl.key=certs/intermediate_server/private/elasticsearch-secure.key.pem + - xpack.security.transport.ssl.enabled=true + - xpack.security.transport.ssl.certificate=certs/intermediate_server/certs/elasticsearch-secure-chain.cert.pem + - xpack.security.transport.ssl.key=certs/intermediate_server/private/elasticsearch-secure.key.pem + - ELASTIC_PASSWORD=vector + - ES_JAVA_OPTS=-Xms400m -Xmx400m volumes: - - ../../../data/ca:/usr/share/elasticsearch/config/certs:ro + - ../../../data/ca:/usr/share/elasticsearch/config/certs:ro networks: default: diff --git a/tests/integration/elasticsearch/config/test.yaml b/tests/integration/elasticsearch/config/test.yaml index cdf1eedbbb4a7..984932cb08540 100644 --- a/tests/integration/elasticsearch/config/test.yaml +++ b/tests/integration/elasticsearch/config/test.yaml @@ -1,7 +1,7 @@ features: -- es-integration-tests + - es-integration-tests -test_filter: '::elasticsearch::integration_tests::' +test_filter: "::elasticsearch::integration_tests::" env: AWS_ACCESS_KEY_ID: dummy @@ -16,6 +16,6 @@ matrix: # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/sinks/elasticsearch/**" -- "src/sinks/util/**" -- "tests/integration/elasticsearch/**" + - "src/sinks/elasticsearch/**" + - "src/sinks/util/**" + - "tests/integration/elasticsearch/**" diff --git a/tests/integration/eventstoredb/config/compose.yaml b/tests/integration/eventstoredb/config/compose.yaml index 01e6a14ea4117..76d9c45a8528b 100644 --- a/tests/integration/eventstoredb/config/compose.yaml +++ b/tests/integration/eventstoredb/config/compose.yaml @@ -1,11 +1,11 @@ -version: '3' +version: "3" services: eventstoredb: image: docker.io/eventstore/eventstore:${CONFIG_VERSION} command: --insecure --stats-period-sec=1 volumes: - - ../../../data:/etc/vector:ro + - ../../../data:/etc/vector:ro networks: default: diff --git a/tests/integration/eventstoredb/config/test.yaml b/tests/integration/eventstoredb/config/test.yaml index 2dab755bfd032..60c6d8e148cc8 100644 --- a/tests/integration/eventstoredb/config/test.yaml +++ b/tests/integration/eventstoredb/config/test.yaml @@ -1,7 +1,7 @@ features: -- eventstoredb_metrics-integration-tests + - eventstoredb_metrics-integration-tests -test_filter: '::eventstoredb_metrics::' +test_filter: "::eventstoredb_metrics::" matrix: version: ["24.2", "24.6", "latest"] @@ -9,7 +9,7 @@ matrix: # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/internal_events/eventstoredb_metrics.rs" -- "src/sources/eventstoredb_metrics/**" -- "src/sources/util/**" -- "tests/integration/eventstoredb/**" + - "src/internal_events/eventstoredb_metrics.rs" + - "src/sources/eventstoredb_metrics/**" + - "src/sources/util/**" + - "tests/integration/eventstoredb/**" diff --git a/tests/integration/fluent/config/test.yaml b/tests/integration/fluent/config/test.yaml index f5f977aa1f637..41151d5abd8af 100644 --- a/tests/integration/fluent/config/test.yaml +++ b/tests/integration/fluent/config/test.yaml @@ -1,5 +1,5 @@ features: -- fluent-integration-tests + - fluent-integration-tests test_filter: "::fluent::" @@ -14,7 +14,7 @@ matrix: # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/internal_events/fluent.rs" -- "src/sources/fluent/**" -- "src/sources/util/**" -- "tests/integration/fluent/**" + - "src/internal_events/fluent.rs" + - "src/sources/fluent/**" + - "src/sources/util/**" + - "tests/integration/fluent/**" diff --git a/tests/integration/gcp/config/compose.yaml b/tests/integration/gcp/config/compose.yaml index f317c23a5604c..d911f9fad8872 100644 --- a/tests/integration/gcp/config/compose.yaml +++ b/tests/integration/gcp/config/compose.yaml @@ -1,20 +1,20 @@ -version: '3' +version: "3" services: gcloud-pubsub: image: docker.io/messagebird/gcloud-pubsub-emulator:${CONFIG_VERSION} environment: - - PUBSUB_PROJECT1=testproject,topic1:subscription1 - - PUBSUB_PROJECT2=sourceproject,topic2:subscription2 + - PUBSUB_PROJECT1=testproject,topic1:subscription1 + - PUBSUB_PROJECT2=sourceproject,topic2:subscription2 chronicle-emulator: image: docker.io/timberio/chronicle-emulator:${CONFIG_VERSION} ports: - - 3000:3000 + - 3000:3000 volumes: - - ./public.pem:/public.pem:ro + - ./public.pem:/public.pem:ro command: - - -p - - /public.pem + - -p + - /public.pem networks: default: diff --git a/tests/integration/gcp/config/invalidauth.json b/tests/integration/gcp/config/invalidauth.json index 7126a1b4afbf6..0963bfc473027 100644 --- a/tests/integration/gcp/config/invalidauth.json +++ b/tests/integration/gcp/config/invalidauth.json @@ -2,7 +2,7 @@ "type": "service_account", "project_id": "chronicle-test", "private_key_id": "1", - "private_key": "-----BEGIN RSA PRIVATE KEY-----\nMIICXgIBAAKBgQDouHdVDVz0/M6PGe60Kf/g0nyOxCvbZgiUAZNzFimXDU+RpZ54\n6/oETl6VpRkbp8a4Xb8avll2lsamdHvGcsgnjJXdpp7LfWYLqHEpn0/XFM+womXg\nvglWCDwAsXmrmwpZKEC82mmyFigheyPA/sfuN6z+wa7P5B65xzIdDQX7nQIDAQAB\nAoGBANID/rUDrTrtll8v8Oon6OH0MjIIuOdzKhSfY3h9rKTDf2YaB2xq0KLoMpVr\ne8AoZb5l45t34naR1M3M2xKY7SSDAVJFfg/3Vxeot86DQ23IGLXj7LnNxXnvklXa\nEXaD8LNz/MXxS7/Lu0R+lEtjEkf23+BRb11fL6Q/EDToNHnhAkEA/FnwHhKMc/Bm\nXsS8bENuZP3SV2v7TU6MFTtXJFmsoZBxHnsM8UUi0gq9gBnApmdhy7v2N/Mv9gFI\nviSdr7vm1QJBAOwV3cHAciRHVK71TweOWIJKZBM9ZVut0VDs5GrBYZxGMBiOr3BI\ns7+0ugTKxVimuei6c0KNXw1kg3Vtc5+utakCQQDklAbXBpAomJHxt5zBKBc/7VXx\nEANyk/p5ZOXbLEsdkXuVU3p2tNwEi+v4s9r4H97Kr3goV+SSnbkpWntm6fn9AkBn\nFnE7rlXpA4C12QYGTaDWW7dxM0j0DGUvChH/j6uYuok73+o5hHWAy2DCwOwFduAN\nAIVd1S9hQLeqaf2oB3jpAkEAnRT+bAlMjtUOBO6XPNO4IbYwWJvGMcIEO7zu6AdB\nPJy3/U+bLimxFuYdrs6SnIHIUVdl35AlckHqzT54a5YKqQ==\n-----END RSA PRIVATE KEY-----", + "private_key": "-----BEGIN RSA PRIVATE KEY-----\nMIICXgIBAAKBgQDouHdVDVz0/M6PGe60Kf/g0nyOxCvbZgiUAZNzFimXDU+RpZ54\n6/oETl6VpRkbp8a4Xb8avll2lsamdHvGcsgnjJXdpp7LfWYLqHEpn0/XFM+womXg\nvglWCDwAsXmrmwpZKEC82mmyFigheyPA/sfuN6z+wa7P5B65xzIdDQX7nQIDAQAB\nAoGBANID/rUDrTrtll8v8Oon6OH0MjIIuOdzKhSfY3h9rKTDf2YaB2xq0KLoMpVr\ne8AoZb5l45t34naR1M3M2xKY7SSDAVJFfg/3Vxeot86DQ23IGLXj7LnNxXnvklXa\nEXaD8LNz/MXxS7/Lu0R+lEtjEkf23+BRb11fL6Q/EDToNHnhAkEA/FnwHhKMc/Bm\nXsS8bENuZP3SV2v7TU6MFTtXJFmsoZBxHnsM8UUi0gq9gBnApmdhy7v2N/Mv9gFI\nviSdr7vm1QJBAOwV3cHAciRHVK71TweOWIJKZBM9ZVut0VDs5GrBYZxGMBiOr3BI\ns7+0ugTKxVimuei6c0KNXw1kg3Vtc5+utakCQQDklAbXBpAomJHxt5zBKBc/7VXx\nEANyk/p5ZOXbLEsdkXuVU3p2tNwEi+v4s9r4H97Kr3goV+SSnbkpWntm6fn9AkBn\nFnE7rlXpA4C12QYGTaDWW7dxM0j0DGUvChH/j6uYuok73+o5hHWAy2DCwOwFduAN\nAIVd1S9hQLeqaf2oB3jpAkEAnRT+bAlMjtUOBO6XPNO4IbYwWJvGMcIEO7zu6AdB\nPJy3/U+bLimxFuYdrs6SnIHIUVdl35AlckHqzT54a5YKqQ==\n-----END RSA PRIVATE KEY-----", "client_email": "test@test.com", "client_id": "1", "auth_uri": "http://chronicle-emulator:3000/o/oauth2/auth", diff --git a/tests/integration/gcp/config/test.yaml b/tests/integration/gcp/config/test.yaml index b700b8283f831..474826afd6aed 100644 --- a/tests/integration/gcp/config/test.yaml +++ b/tests/integration/gcp/config/test.yaml @@ -1,8 +1,8 @@ features: -- gcp-integration-tests -- chronicle-integration-tests + - gcp-integration-tests + - chronicle-integration-tests -test_filter: '::gcp' +test_filter: "::gcp" env: EMULATOR_ADDRESS: http://gcloud-pubsub:8681 @@ -14,10 +14,10 @@ matrix: # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/internal_events/gcp_pubsub.rs" -- "src/sources/gcp_pubsub.rs" -- "src/sources/util/**" -- "src/sinks/gcp/**" -- "src/sinks/util/**" -- "src/gcp.rs" -- "tests/integration/gcp/**" + - "src/internal_events/gcp_pubsub.rs" + - "src/sources/gcp_pubsub.rs" + - "src/sources/util/**" + - "src/sinks/gcp/**" + - "src/sinks/util/**" + - "src/gcp.rs" + - "tests/integration/gcp/**" diff --git a/tests/integration/greptimedb/config/compose.yaml b/tests/integration/greptimedb/config/compose.yaml index 02a68364e7a31..01abc946761e1 100644 --- a/tests/integration/greptimedb/config/compose.yaml +++ b/tests/integration/greptimedb/config/compose.yaml @@ -1,4 +1,4 @@ -version: '3' +version: "3" services: greptimedb: diff --git a/tests/integration/greptimedb/config/test.yaml b/tests/integration/greptimedb/config/test.yaml index 17593b891df8c..2ec3984938617 100644 --- a/tests/integration/greptimedb/config/test.yaml +++ b/tests/integration/greptimedb/config/test.yaml @@ -1,7 +1,7 @@ features: -- greptimedb-integration-tests + - greptimedb-integration-tests -test_filter: '::greptimedb::' +test_filter: "::greptimedb::" runner: env: @@ -17,5 +17,5 @@ matrix: # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/sinks/greptimedb/**" -- "tests/integration/greptimedb/**" + - "src/sinks/greptimedb/**" + - "tests/integration/greptimedb/**" diff --git a/tests/integration/http-client/config/compose.yaml b/tests/integration/http-client/config/compose.yaml index 7ad5dfc47386b..91eaa066886eb 100644 --- a/tests/integration/http-client/config/compose.yaml +++ b/tests/integration/http-client/config/compose.yaml @@ -1,34 +1,34 @@ -version: '3' +version: "3" services: dufs: image: docker.io/sigoden/dufs:${CONFIG_VERSION} command: - - /data + - /data volumes: - - ../data/serve:/data + - ../data/serve:/data dufs-auth: image: docker.io/sigoden/dufs:${CONFIG_VERSION} command: - - -a - - "user:pass@/" - - --auth-method - - basic - - /data + - -a + - "user:pass@/" + - --auth-method + - basic + - /data volumes: - - ../data/serve:/data + - ../data/serve:/data dufs-https: image: docker.io/sigoden/dufs:${CONFIG_VERSION} command: - - --tls-cert - - /certs/ca.cert.pem - - --tls-key - - /certs/ca.key.pem - - /data + - --tls-cert + - /certs/ca.cert.pem + - --tls-key + - /certs/ca.key.pem + - /data volumes: - - ../data/serve:/data - - ../../../data/ca/intermediate_server/certs/dufs-https-chain.cert.pem:/certs/ca.cert.pem - - ../../../data/ca/intermediate_server/private/dufs-https.key.pem:/certs/ca.key.pem + - ../data/serve:/data + - ../../../data/ca/intermediate_server/certs/dufs-https-chain.cert.pem:/certs/ca.cert.pem + - ../../../data/ca/intermediate_server/private/dufs-https.key.pem:/certs/ca.key.pem networks: default: diff --git a/tests/integration/http-client/config/test.yaml b/tests/integration/http-client/config/test.yaml index a56570dc5cf81..bf52050571683 100644 --- a/tests/integration/http-client/config/test.yaml +++ b/tests/integration/http-client/config/test.yaml @@ -1,7 +1,7 @@ features: -- http-client-integration-tests + - http-client-integration-tests -test_filter: 'sources::http_client::' +test_filter: "sources::http_client::" env: DUFS_ADDRESS: http://dufs:5000 @@ -14,6 +14,6 @@ matrix: # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/sources/http_client/**" -- "src/sources/util/**" -- "tests/integration/http-client/**" + - "src/sources/http_client/**" + - "src/sources/util/**" + - "tests/integration/http-client/**" diff --git a/tests/integration/http-client/data/serve/logs/json.json b/tests/integration/http-client/data/serve/logs/json.json index 9672096ad8861..9cc07fbc01e5d 100644 --- a/tests/integration/http-client/data/serve/logs/json.json +++ b/tests/integration/http-client/data/serve/logs/json.json @@ -1 +1 @@ -{ "foo" : "baz" } +{ "foo": "baz" } diff --git a/tests/integration/http-client/data/serve/metrics/native.json b/tests/integration/http-client/data/serve/metrics/native.json index 4aea8048614d2..76935bd2f32aa 100644 --- a/tests/integration/http-client/data/serve/metrics/native.json +++ b/tests/integration/http-client/data/serve/metrics/native.json @@ -1 +1 @@ -{ "metric" : { "name" : "a_metric", "kind" : "absolute" , "counter" : { "value" : 1 } } } +{ "metric": { "name": "a_metric", "kind": "absolute", "counter": { "value": 1 } } } diff --git a/tests/integration/http-client/data/serve/traces/native.json b/tests/integration/http-client/data/serve/traces/native.json index cc755b92f3bea..0d970b94d3e4f 100644 --- a/tests/integration/http-client/data/serve/traces/native.json +++ b/tests/integration/http-client/data/serve/traces/native.json @@ -1 +1 @@ -{ "trace" : { "name" : "a_trace", "foo" : 42 } } +{ "trace": { "name": "a_trace", "foo": 42 } } diff --git a/tests/integration/humio/config/compose.yaml b/tests/integration/humio/config/compose.yaml index bc1045d98445d..0fb6c2cb5ddbe 100644 --- a/tests/integration/humio/config/compose.yaml +++ b/tests/integration/humio/config/compose.yaml @@ -1,4 +1,4 @@ -version: '3' +version: "3" services: humio: diff --git a/tests/integration/humio/config/test.yaml b/tests/integration/humio/config/test.yaml index a8cdebeba3b19..9d068364bf917 100644 --- a/tests/integration/humio/config/test.yaml +++ b/tests/integration/humio/config/test.yaml @@ -1,7 +1,7 @@ features: -- humio-integration-tests + - humio-integration-tests -test_filter: '::humio::' +test_filter: "::humio::" runner: env: @@ -13,6 +13,6 @@ matrix: # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/sinks/humio/**" -- "src/sinks/util/**" -- "tests/integration/humio/**" + - "src/sinks/humio/**" + - "src/sinks/util/**" + - "tests/integration/humio/**" diff --git a/tests/integration/influxdb/config/compose.yaml b/tests/integration/influxdb/config/compose.yaml index b5c31c006ab06..09049db7c6a31 100644 --- a/tests/integration/influxdb/config/compose.yaml +++ b/tests/integration/influxdb/config/compose.yaml @@ -1,24 +1,24 @@ -version: '3' +version: "3" services: influxdb-v1: image: docker.io/influxdb:${CONFIG_VERSION} environment: - - INFLUXDB_REPORTING_DISABLED=true + - INFLUXDB_REPORTING_DISABLED=true influxdb-v1-tls: image: docker.io/influxdb:${CONFIG_VERSION} environment: - - INFLUXDB_REPORTING_DISABLED=true - - INFLUXDB_HTTP_HTTPS_ENABLED=true - - INFLUXDB_HTTP_HTTPS_CERTIFICATE=/etc/ssl/intermediate_server/certs/influxdb-v1-tls-chain.cert.pem - - INFLUXDB_HTTP_HTTPS_PRIVATE_KEY=/etc/ssl/intermediate_server/private/influxdb-v1-tls.key.pem + - INFLUXDB_REPORTING_DISABLED=true + - INFLUXDB_HTTP_HTTPS_ENABLED=true + - INFLUXDB_HTTP_HTTPS_CERTIFICATE=/etc/ssl/intermediate_server/certs/influxdb-v1-tls-chain.cert.pem + - INFLUXDB_HTTP_HTTPS_PRIVATE_KEY=/etc/ssl/intermediate_server/private/influxdb-v1-tls.key.pem volumes: - - ../../../data/ca:/etc/ssl:ro + - ../../../data/ca:/etc/ssl:ro influxdb-v2: image: docker.io/influxdb:2.0 command: influxd --reporting-disabled environment: - - INFLUXDB_REPORTING_DISABLED=true + - INFLUXDB_REPORTING_DISABLED=true networks: default: diff --git a/tests/integration/influxdb/config/test.yaml b/tests/integration/influxdb/config/test.yaml index 35bf2f72691bf..d6965c383f24e 100644 --- a/tests/integration/influxdb/config/test.yaml +++ b/tests/integration/influxdb/config/test.yaml @@ -1,7 +1,7 @@ features: -- influxdb-integration-tests + - influxdb-integration-tests -test_filter: '::influxdb::' +test_filter: "::influxdb::" env: INFLUXDB_V1_HTTPS_ADDRESS: https://influxdb-v1-tls:8086 @@ -9,12 +9,12 @@ env: INFLUXDB_V2_ADDRESS: http://influxdb-v2:8086 matrix: - version: ['1.8'] + version: ["1.8"] # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/internal_events/influxdb.rs" -- "src/sinks/influxdb/**" -- "src/sinks/util/**" -- "tests/integration/influxdb/**" + - "src/internal_events/influxdb.rs" + - "src/sinks/influxdb/**" + - "src/sinks/util/**" + - "tests/integration/influxdb/**" diff --git a/tests/integration/kafka/config/compose.yaml b/tests/integration/kafka/config/compose.yaml index 81a35dd64ec94..66c16b7fc5f3a 100644 --- a/tests/integration/kafka/config/compose.yaml +++ b/tests/integration/kafka/config/compose.yaml @@ -1,39 +1,39 @@ -version: '3' +version: "3" services: zookeeper: image: docker.io/confluentinc/cp-zookeeper:${CONFIG_VERSION} ports: - - 2181:2181 + - 2181:2181 environment: - - ZOOKEEPER_CLIENT_PORT=2181 + - ZOOKEEPER_CLIENT_PORT=2181 kafka: image: docker.io/confluentinc/cp-kafka:7.6.1 depends_on: - - zookeeper + - zookeeper environment: - - KAFKA_BROKER_ID=1 - - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - - ZOOKEEPER_SASL_ENABLED=false - - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 - - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0 - - KAFKA_LISTENERS=PLAINTEXT://:9091,SSL://:9092,SASL_PLAINTEXT://:9093 - - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9091,SSL://kafka:9092,SASL_PLAINTEXT://kafka:9093 - - KAFKA_SSL_KEYSTORE_TYPE=PKCS12 - - KAFKA_SSL_KEY_CREDENTIALS=kafka.pass - - KAFKA_SSL_KEYSTORE_CREDENTIALS=kafka.pass - - KAFKA_SSL_KEYSTORE_FILENAME=kafka.p12 - - KAFKA_SSL_CLIENT_AUTH=none - - KAFKA_OPTS=-Djava.security.auth.login.config=/etc/kafka/kafka_server_jaas.conf - - KAFKA_SASL_ENABLED_MECHANISMS=PLAIN + - KAFKA_BROKER_ID=1 + - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - ZOOKEEPER_SASL_ENABLED=false + - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 + - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0 + - KAFKA_LISTENERS=PLAINTEXT://:9091,SSL://:9092,SASL_PLAINTEXT://:9093 + - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9091,SSL://kafka:9092,SASL_PLAINTEXT://kafka:9093 + - KAFKA_SSL_KEYSTORE_TYPE=PKCS12 + - KAFKA_SSL_KEY_CREDENTIALS=kafka.pass + - KAFKA_SSL_KEYSTORE_CREDENTIALS=kafka.pass + - KAFKA_SSL_KEYSTORE_FILENAME=kafka.p12 + - KAFKA_SSL_CLIENT_AUTH=none + - KAFKA_OPTS=-Djava.security.auth.login.config=/etc/kafka/kafka_server_jaas.conf + - KAFKA_SASL_ENABLED_MECHANISMS=PLAIN ports: - - 9091:9091 - - 9092:9092 - - 9093:9093 + - 9091:9091 + - 9092:9092 + - 9093:9093 volumes: - - ../../../data/ca/intermediate_server/private/kafka.pass:/etc/kafka/secrets/kafka.pass:ro - - ../../../data/ca/intermediate_server/private/kafka.p12:/etc/kafka/secrets/kafka.p12:ro - - ../../shared/data/kafka_server_jaas.conf:/etc/kafka/kafka_server_jaas.conf + - ../../../data/ca/intermediate_server/private/kafka.pass:/etc/kafka/secrets/kafka.pass:ro + - ../../../data/ca/intermediate_server/private/kafka.p12:/etc/kafka/secrets/kafka.p12:ro + - ../../shared/data/kafka_server_jaas.conf:/etc/kafka/kafka_server_jaas.conf networks: default: diff --git a/tests/integration/kafka/config/test.yaml b/tests/integration/kafka/config/test.yaml index 9619b4d7ea0f1..7f91f693b25dd 100644 --- a/tests/integration/kafka/config/test.yaml +++ b/tests/integration/kafka/config/test.yaml @@ -1,7 +1,7 @@ features: -- kafka-integration-tests + - kafka-integration-tests -test_filter: '::kafka::' +test_filter: "::kafka::" env: KAFKA_HOST: kafka @@ -12,10 +12,10 @@ matrix: # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/internal_events/kafka.rs" -- "src/sinks/kafka/**" -- "src/sinks/util/**" -- "src/sources/kafka.rs" -- "src/sources/util/**" -- "src/kafka.rs" -- "tests/integration/kafka/**" + - "src/internal_events/kafka.rs" + - "src/sinks/kafka/**" + - "src/sinks/util/**" + - "src/sources/kafka.rs" + - "src/sources/util/**" + - "src/kafka.rs" + - "tests/integration/kafka/**" diff --git a/tests/integration/logstash/config/compose.yaml b/tests/integration/logstash/config/compose.yaml index af024d0855453..212ce4e1155be 100644 --- a/tests/integration/logstash/config/compose.yaml +++ b/tests/integration/logstash/config/compose.yaml @@ -1,17 +1,17 @@ -version: '3' +version: "3" services: beats-heartbeat: image: docker.elastic.co/beats/heartbeat:${CONFIG_VERSION} command: -environment=container -strict.perms=false volumes: - - ../data/heartbeat.yml:/usr/share/heartbeat/heartbeat.yml:ro + - ../data/heartbeat.yml:/usr/share/heartbeat/heartbeat.yml:ro logstash: image: docker.elastic.co/logstash/logstash:7.13.1 volumes: - - /dev/null:/usr/share/logstash/pipeline/logstash.yml - - ../../shared/data/host.docker.internal.crt:/tmp/logstash.crt - - ../data/logstash.conf:/usr/share/logstash/pipeline/logstash.conf + - /dev/null:/usr/share/logstash/pipeline/logstash.yml + - ../../shared/data/host.docker.internal.crt:/tmp/logstash.crt + - ../data/logstash.conf:/usr/share/logstash/pipeline/logstash.conf networks: default: diff --git a/tests/integration/logstash/config/test.yaml b/tests/integration/logstash/config/test.yaml index e0f26afbec7f7..490293912fbae 100644 --- a/tests/integration/logstash/config/test.yaml +++ b/tests/integration/logstash/config/test.yaml @@ -1,7 +1,7 @@ features: -- logstash-integration-tests + - logstash-integration-tests -test_filter: '::logstash::integration_tests::' +test_filter: "::logstash::integration_tests::" env: HEARTBEAT_ADDRESS: 0.0.0.0:8080 @@ -13,6 +13,6 @@ matrix: # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/sources/logstash.rs" -- "src/sources/util/**" -- "tests/integration/logstash/**" + - "src/sources/logstash.rs" + - "src/sources/util/**" + - "tests/integration/logstash/**" diff --git a/tests/integration/logstash/data/heartbeat.yml b/tests/integration/logstash/data/heartbeat.yml index ee2a9c1bca24d..154f8cb971737 100644 --- a/tests/integration/logstash/data/heartbeat.yml +++ b/tests/integration/logstash/data/heartbeat.yml @@ -1,8 +1,8 @@ heartbeat.monitors: -- type: http - schedule: '@every 1s' - urls: - - https://google.com + - type: http + schedule: "@every 1s" + urls: + - https://google.com output.logstash: - hosts: ['runner:8080'] + hosts: ["runner:8080"] diff --git a/tests/integration/loki/config/compose.yaml b/tests/integration/loki/config/compose.yaml index 4350e809f8961..187c1c274bf71 100644 --- a/tests/integration/loki/config/compose.yaml +++ b/tests/integration/loki/config/compose.yaml @@ -1,4 +1,4 @@ -version: '3' +version: "3" services: loki: diff --git a/tests/integration/loki/config/test.yaml b/tests/integration/loki/config/test.yaml index 9a86d9ce6c29d..3d3c0a9c9902f 100644 --- a/tests/integration/loki/config/test.yaml +++ b/tests/integration/loki/config/test.yaml @@ -1,7 +1,7 @@ features: -- loki-integration-tests + - loki-integration-tests -test_filter: '::loki::' +test_filter: "::loki::" env: LOKI_ADDRESS: http://loki:3100 @@ -12,7 +12,7 @@ matrix: # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/internal_events/loki.rs" -- "src/sinks/loki/**" -- "src/sinks/util/**" -- "tests/integration/loki/**" + - "src/internal_events/loki.rs" + - "src/sinks/loki/**" + - "src/sinks/util/**" + - "tests/integration/loki/**" diff --git a/tests/integration/mongodb/config/compose.yaml b/tests/integration/mongodb/config/compose.yaml index f7cd8b6517471..f06682b462763 100644 --- a/tests/integration/mongodb/config/compose.yaml +++ b/tests/integration/mongodb/config/compose.yaml @@ -1,35 +1,35 @@ -version: '3' +version: "3" services: mongodb-primary: image: docker.io/bitnamilegacy/mongodb:${CONFIG_VERSION} environment: - - MONGODB_ADVERTISED_HOSTNAME=mongodb-primary - - MONGODB_REPLICA_SET_MODE=primary - - MONGODB_ROOT_PASSWORD=toor - - MONGODB_REPLICA_SET_KEY=vector + - MONGODB_ADVERTISED_HOSTNAME=mongodb-primary + - MONGODB_REPLICA_SET_MODE=primary + - MONGODB_ROOT_PASSWORD=toor + - MONGODB_REPLICA_SET_KEY=vector mongodb-secondary: image: docker.io/bitnamilegacy/mongodb:${CONFIG_VERSION} depends_on: - - mongodb-primary + - mongodb-primary environment: - - MONGODB_ADVERTISED_HOSTNAME=mongodb-secondary - - MONGODB_REPLICA_SET_MODE=secondary - - MONGODB_INITIAL_PRIMARY_HOST=mongodb-primary - - MONGODB_INITIAL_PRIMARY_PORT_NUMBER=27017 - - MONGODB_INITIAL_PRIMARY_ROOT_PASSWORD=toor - - MONGODB_REPLICA_SET_KEY=vector + - MONGODB_ADVERTISED_HOSTNAME=mongodb-secondary + - MONGODB_REPLICA_SET_MODE=secondary + - MONGODB_INITIAL_PRIMARY_HOST=mongodb-primary + - MONGODB_INITIAL_PRIMARY_PORT_NUMBER=27017 + - MONGODB_INITIAL_PRIMARY_ROOT_PASSWORD=toor + - MONGODB_REPLICA_SET_KEY=vector mongodb-arbiter: image: docker.io/bitnamilegacy/mongodb:${CONFIG_VERSION} depends_on: - - mongodb-primary + - mongodb-primary environment: - - MONGODB_ADVERTISED_HOSTNAME=mongodb-arbiter - - MONGODB_REPLICA_SET_MODE=arbiter - - MONGODB_INITIAL_PRIMARY_HOST=mongodb-primary - - MONGODB_INITIAL_PRIMARY_PORT_NUMBER=27017 - - MONGODB_INITIAL_PRIMARY_ROOT_PASSWORD=toor - - MONGODB_REPLICA_SET_KEY=vector + - MONGODB_ADVERTISED_HOSTNAME=mongodb-arbiter + - MONGODB_REPLICA_SET_MODE=arbiter + - MONGODB_INITIAL_PRIMARY_HOST=mongodb-primary + - MONGODB_INITIAL_PRIMARY_PORT_NUMBER=27017 + - MONGODB_INITIAL_PRIMARY_ROOT_PASSWORD=toor + - MONGODB_REPLICA_SET_KEY=vector networks: default: diff --git a/tests/integration/mongodb/config/test.yaml b/tests/integration/mongodb/config/test.yaml index 10da02aca38dc..3668030a20fb5 100644 --- a/tests/integration/mongodb/config/test.yaml +++ b/tests/integration/mongodb/config/test.yaml @@ -1,7 +1,7 @@ features: -- mongodb_metrics-integration-tests + - mongodb_metrics-integration-tests -test_filter: '::mongodb_metrics::' +test_filter: "::mongodb_metrics::" env: PRIMARY_MONGODB_ADDRESS: mongodb://root:toor@mongodb-primary @@ -13,7 +13,7 @@ matrix: # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/internal_events/mongodb_metrics.rs" -- "src/sources/mongodb_metrics/**" -- "src/sources/util/**" -- "tests/integration/mongodb/**" + - "src/internal_events/mongodb_metrics.rs" + - "src/sources/mongodb_metrics/**" + - "src/sources/util/**" + - "tests/integration/mongodb/**" diff --git a/tests/integration/mqtt/config/compose.yaml b/tests/integration/mqtt/config/compose.yaml index cc238cadf050c..9ce016da98cb1 100644 --- a/tests/integration/mqtt/config/compose.yaml +++ b/tests/integration/mqtt/config/compose.yaml @@ -1,10 +1,10 @@ -version: '3' +version: "3" services: emqx: image: docker.io/emqx:${CONFIG_VERSION} ports: - - 1883:1883 + - 1883:1883 networks: default: diff --git a/tests/integration/mqtt/config/test.yaml b/tests/integration/mqtt/config/test.yaml index ec0c637e094d1..1119ca931faf4 100644 --- a/tests/integration/mqtt/config/test.yaml +++ b/tests/integration/mqtt/config/test.yaml @@ -1,13 +1,13 @@ features: -- mqtt-integration-tests + - mqtt-integration-tests -test_filter: '::mqtt::' +test_filter: "::mqtt::" matrix: - version: ['5.0.15'] + version: ["5.0.15"] paths: -- "src/internal_events/mqtt.rs" -- "src/sinks/mqtt/**" -- "src/sinks/util/**" -- "src/sources/mqtt/**" + - "src/internal_events/mqtt.rs" + - "src/sinks/mqtt/**" + - "src/sinks/util/**" + - "src/sources/mqtt/**" diff --git a/tests/integration/nats/config/compose.yaml b/tests/integration/nats/config/compose.yaml index f77020b57afee..a556ffd61f351 100644 --- a/tests/integration/nats/config/compose.yaml +++ b/tests/integration/nats/config/compose.yaml @@ -1,4 +1,4 @@ -version: '3' +version: "3" services: nats: @@ -6,50 +6,50 @@ services: nats-userpass: image: docker.io/library/nats:${CONFIG_VERSION} command: - - --user - - natsuser - - --pass - - natspass + - --user + - natsuser + - --pass + - natspass nats-token: image: docker.io/library/nats:${CONFIG_VERSION} command: - - --auth - - secret + - --auth + - secret nats-nkey: image: docker.io/library/nats:${CONFIG_VERSION} command: - - --config - - /usr/share/nats/config/nats-nkey.conf + - --config + - /usr/share/nats/config/nats-nkey.conf volumes: - - ../data:/usr/share/nats/config + - ../data:/usr/share/nats/config nats-tls: image: docker.io/library/nats:${CONFIG_VERSION} command: - - --config - - /usr/share/nats/config/nats-tls.conf + - --config + - /usr/share/nats/config/nats-tls.conf volumes: - - ../data:/usr/share/nats/config + - ../data:/usr/share/nats/config nats-tls-client-cert: image: docker.io/library/nats:${CONFIG_VERSION} command: - - --config - - /usr/share/nats/config/nats-tls-client-cert.conf + - --config + - /usr/share/nats/config/nats-tls-client-cert.conf volumes: - - ../data:/usr/share/nats/config + - ../data:/usr/share/nats/config nats-jwt: image: docker.io/library/nats:${CONFIG_VERSION} command: - - --config - - /usr/share/nats/config/nats-jwt.conf + - --config + - /usr/share/nats/config/nats-jwt.conf volumes: - - ../data:/usr/share/nats/config + - ../data:/usr/share/nats/config nats-jetstream-test: image: docker.io/library/nats:${CONFIG_VERSION} command: - - --config - - /usr/share/nats/config/nats-jetstream.conf + - --config + - /usr/share/nats/config/nats-jetstream.conf volumes: - - ../data:/usr/share/nats/config + - ../data:/usr/share/nats/config networks: default: diff --git a/tests/integration/nginx/config/compose.yaml b/tests/integration/nginx/config/compose.yaml index 3caf63a002f11..4b0f50d0d92ea 100644 --- a/tests/integration/nginx/config/compose.yaml +++ b/tests/integration/nginx/config/compose.yaml @@ -1,25 +1,25 @@ -version: '3' +version: "3" services: squid: image: docker.io/ubuntu/squid:latest depends_on: - - nginx-proxy + - nginx-proxy networks: - - default - - proxy + - default + - proxy nginx: image: docker.io/nginx:${CONFIG_VERSION} volumes: - - ../data/:/etc/nginx:ro + - ../data/:/etc/nginx:ro networks: - - default + - default nginx-proxy: image: docker.io/nginx:${CONFIG_VERSION} volumes: - - ../data/:/etc/nginx:ro + - ../data/:/etc/nginx:ro networks: - - proxy + - proxy networks: default: diff --git a/tests/integration/nginx/config/test.yaml b/tests/integration/nginx/config/test.yaml index 8d9014b657787..e23372f596139 100644 --- a/tests/integration/nginx/config/test.yaml +++ b/tests/integration/nginx/config/test.yaml @@ -1,7 +1,7 @@ features: -- nginx-integration-tests + - nginx-integration-tests -test_filter: '::nginx_metrics::' +test_filter: "::nginx_metrics::" runner: env: @@ -15,7 +15,7 @@ matrix: # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/internal_events/nginx_metrics.rs" -- "src/sources/nginx_metrics/**" -- "src/sources/util/**" -- "tests/integration/nginx/**" + - "src/internal_events/nginx_metrics.rs" + - "src/sources/nginx_metrics/**" + - "src/sources/util/**" + - "tests/integration/nginx/**" diff --git a/tests/integration/opentelemetry/config/compose.yaml b/tests/integration/opentelemetry/config/compose.yaml index d8f0c95d20fbc..f65495b9bc2ae 100644 --- a/tests/integration/opentelemetry/config/compose.yaml +++ b/tests/integration/opentelemetry/config/compose.yaml @@ -1,10 +1,10 @@ -version: '3' +version: "3" services: opentelemetry-collector: image: docker.io/otel/opentelemetry-collector-contrib:${CONFIG_VERSION} volumes: - - ../data/config.yaml:/etc/otelcol-contrib/config.yaml + - ../data/config.yaml:/etc/otelcol-contrib/config.yaml networks: default: diff --git a/tests/integration/opentelemetry/config/test.yaml b/tests/integration/opentelemetry/config/test.yaml index 91a2968fdfae0..0a3af636f56b5 100644 --- a/tests/integration/opentelemetry/config/test.yaml +++ b/tests/integration/opentelemetry/config/test.yaml @@ -1,7 +1,7 @@ features: -- opentelemetry-integration-tests + - opentelemetry-integration-tests -test_filter: '::opentelemetry::' +test_filter: "::opentelemetry::" runner: env: @@ -14,6 +14,6 @@ matrix: # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/sources/opentelemetry/**" -- "src/sources/util/**" -- "tests/integration/opentelemetry/**" + - "src/sources/opentelemetry/**" + - "src/sources/util/**" + - "tests/integration/opentelemetry/**" diff --git a/tests/integration/opentelemetry/data/config.yaml b/tests/integration/opentelemetry/data/config.yaml index 66c8fe34e6112..ee0fe7fa36126 100644 --- a/tests/integration/opentelemetry/data/config.yaml +++ b/tests/integration/opentelemetry/data/config.yaml @@ -25,10 +25,10 @@ service: pipelines: logs: receivers: [otlp] - exporters: [otlp,otlphttp] + exporters: [otlp, otlphttp] traces: receivers: [otlp] - exporters: [otlp,otlphttp] + exporters: [otlp, otlphttp] metrics: receivers: [otlp] - exporters: [otlp,otlphttp] + exporters: [otlp, otlphttp] diff --git a/tests/integration/postgres/config/compose.yaml b/tests/integration/postgres/config/compose.yaml index 017e497c81811..40970c5f3da28 100644 --- a/tests/integration/postgres/config/compose.yaml +++ b/tests/integration/postgres/config/compose.yaml @@ -1,16 +1,16 @@ -version: '3' +version: "3" services: postgres: image: docker.io/postgres:${CONFIG_VERSION} command: /postgres-init.sh environment: - - POSTGRES_USER=vector - - POSTGRES_PASSWORD=vector + - POSTGRES_USER=vector + - POSTGRES_PASSWORD=vector volumes: - - socket:/var/run/postgresql - - ../data/postgres-init.sh:/postgres-init.sh:ro - - ../../../data/ca:/certs:ro + - socket:/var/run/postgresql + - ../data/postgres-init.sh:/postgres-init.sh:ro + - ../../../data/ca:/certs:ro volumes: # Use external volume 'postgres_socket' that's shared with the test runner diff --git a/tests/integration/postgres/config/test.yaml b/tests/integration/postgres/config/test.yaml index 85c012a11fa5f..905d4f53ccda5 100644 --- a/tests/integration/postgres/config/test.yaml +++ b/tests/integration/postgres/config/test.yaml @@ -1,6 +1,6 @@ features: -- postgresql_metrics-integration-tests -- postgres_sink-integration-tests + - postgresql_metrics-integration-tests + - postgres_sink-integration-tests test_filter: ::postgres @@ -13,13 +13,13 @@ runner: postgres_socket: /pg_socket matrix: - version: ['13.1'] + version: ["13.1"] # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/internal_events/postgresql_metrics.rs" -- "src/sinks/postgres/**" -- "src/sources/postgresql_metrics.rs" -- "src/sources/util/**" -- "tests/integration/postgres/**" + - "src/internal_events/postgresql_metrics.rs" + - "src/sinks/postgres/**" + - "src/sources/postgresql_metrics.rs" + - "src/sources/util/**" + - "tests/integration/postgres/**" diff --git a/tests/integration/prometheus/config/compose.yaml b/tests/integration/prometheus/config/compose.yaml index a945f730ce834..3ff987c6d5189 100644 --- a/tests/integration/prometheus/config/compose.yaml +++ b/tests/integration/prometheus/config/compose.yaml @@ -1,26 +1,26 @@ -version: '3' +version: "3" services: influxdb-v1: image: docker.io/influxdb:${CONFIG_INFLUXDB} environment: - - INFLUXDB_REPORTING_DISABLED=true + - INFLUXDB_REPORTING_DISABLED=true influxdb-v1-tls: image: docker.io/influxdb:${CONFIG_INFLUXDB} environment: - - INFLUXDB_REPORTING_DISABLED=true - - INFLUXDB_HTTP_HTTPS_ENABLED=true - - INFLUXDB_HTTP_BIND_ADDRESS=:8087 - - INFLUXDB_BIND_ADDRESS=:8089 - - INFLUXDB_HTTP_HTTPS_CERTIFICATE=/etc/ssl/intermediate_server/certs/influxdb-v1-tls-chain.cert.pem - - INFLUXDB_HTTP_HTTPS_PRIVATE_KEY=/etc/ssl/intermediate_server/private/influxdb-v1-tls.key.pem + - INFLUXDB_REPORTING_DISABLED=true + - INFLUXDB_HTTP_HTTPS_ENABLED=true + - INFLUXDB_HTTP_BIND_ADDRESS=:8087 + - INFLUXDB_BIND_ADDRESS=:8089 + - INFLUXDB_HTTP_HTTPS_CERTIFICATE=/etc/ssl/intermediate_server/certs/influxdb-v1-tls-chain.cert.pem + - INFLUXDB_HTTP_HTTPS_PRIVATE_KEY=/etc/ssl/intermediate_server/private/influxdb-v1-tls.key.pem volumes: - - ../../../data/ca:/etc/ssl:ro + - ../../../data/ca:/etc/ssl:ro prometheus: image: docker.io/prom/prometheus:${CONFIG_PROMETHEUS} command: --config.file=/etc/vector/prometheus.yaml volumes: - - ../data/prometheus.yaml:/etc/vector/prometheus.yaml:ro + - ../data/prometheus.yaml:/etc/vector/prometheus.yaml:ro networks: default: diff --git a/tests/integration/prometheus/config/test.yaml b/tests/integration/prometheus/config/test.yaml index 455338053e2ff..ac79ea17c3d8e 100644 --- a/tests/integration/prometheus/config/test.yaml +++ b/tests/integration/prometheus/config/test.yaml @@ -1,21 +1,21 @@ -test_filter: '::prometheus::remote_write::' +test_filter: "::prometheus::remote_write::" features: -- prometheus-integration-tests + - prometheus-integration-tests env: REMOTE_WRITE_SOURCE_RECEIVE_ADDRESS: runner:9102 matrix: - prometheus: ['v2.33.4'] - influxdb: ['1.8'] + prometheus: ["v2.33.4"] + influxdb: ["1.8"] # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/internal_events/prometheus.rs" -- "src/sources/prometheus/**" -- "src/sources/util/**" -- "src/sinks/prometheus/**" -- "src/sinks/util/**" -- "tests/integration/prometheus/**" + - "src/internal_events/prometheus.rs" + - "src/sources/prometheus/**" + - "src/sources/util/**" + - "src/sinks/prometheus/**" + - "src/sinks/util/**" + - "tests/integration/prometheus/**" diff --git a/tests/integration/prometheus/data/prometheus.yaml b/tests/integration/prometheus/data/prometheus.yaml index 5e9a6ac09a312..79ce0d180acaa 100644 --- a/tests/integration/prometheus/data/prometheus.yaml +++ b/tests/integration/prometheus/data/prometheus.yaml @@ -3,12 +3,12 @@ global: scrape_configs: # sinks::prometheus::exporter::integration_tests - - job_name: 'integration-test-1' + - job_name: "integration-test-1" static_configs: - - targets: ['127.0.0.1:9101'] + - targets: ["127.0.0.1:9101"] remote_write: # sources::prometheus::remote_write::integration_tests - - url: 'http://runner:9102' + - url: "http://runner:9102" remote_timeout: 1s - name: 'integration-test-2' + name: "integration-test-2" diff --git a/tests/integration/pulsar/config/compose.yaml b/tests/integration/pulsar/config/compose.yaml index cca9c9de8d713..5d6c4e86249f9 100644 --- a/tests/integration/pulsar/config/compose.yaml +++ b/tests/integration/pulsar/config/compose.yaml @@ -1,4 +1,4 @@ -version: '3' +version: "3" services: pulsar: diff --git a/tests/integration/pulsar/config/test.yaml b/tests/integration/pulsar/config/test.yaml index a273fb4e9f163..51dbb1100c9da 100644 --- a/tests/integration/pulsar/config/test.yaml +++ b/tests/integration/pulsar/config/test.yaml @@ -1,7 +1,7 @@ features: -- pulsar-integration-tests + - pulsar-integration-tests -test_filter: '::pulsar::integration_tests::' +test_filter: "::pulsar::integration_tests::" env: PULSAR_HOST: pulsar @@ -14,7 +14,7 @@ matrix: # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/internal_events/pulsar.rs" -- "src/sinks/pulsar/**" -- "src/sinks/util/**" -- "tests/integration/pulsar/**" + - "src/internal_events/pulsar.rs" + - "src/sinks/pulsar/**" + - "src/sinks/util/**" + - "tests/integration/pulsar/**" diff --git a/tests/integration/redis/config/compose.yaml b/tests/integration/redis/config/compose.yaml index 4f0a909a6e455..845c99eff0a62 100644 --- a/tests/integration/redis/config/compose.yaml +++ b/tests/integration/redis/config/compose.yaml @@ -1,4 +1,4 @@ -version: '3' +version: "3" services: redis-primary: diff --git a/tests/integration/redis/config/test.yaml b/tests/integration/redis/config/test.yaml index fdfc54e0446ec..6129d66a052fb 100644 --- a/tests/integration/redis/config/test.yaml +++ b/tests/integration/redis/config/test.yaml @@ -1,5 +1,5 @@ features: -- redis-integration-tests + - redis-integration-tests test_filter: "::redis::" @@ -13,9 +13,9 @@ matrix: # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/internal_events/redis.rs" -- "src/sources/redis/**" -- "src/sources/util/**" -- "src/sinks/redis.rs" -- "src/sinks/util/**" -- "tests/integration/redis/**" + - "src/internal_events/redis.rs" + - "src/sources/redis/**" + - "src/sources/util/**" + - "src/sinks/redis.rs" + - "src/sinks/util/**" + - "tests/integration/redis/**" diff --git a/tests/integration/shutdown/config/compose.yaml b/tests/integration/shutdown/config/compose.yaml index 81a35dd64ec94..66c16b7fc5f3a 100644 --- a/tests/integration/shutdown/config/compose.yaml +++ b/tests/integration/shutdown/config/compose.yaml @@ -1,39 +1,39 @@ -version: '3' +version: "3" services: zookeeper: image: docker.io/confluentinc/cp-zookeeper:${CONFIG_VERSION} ports: - - 2181:2181 + - 2181:2181 environment: - - ZOOKEEPER_CLIENT_PORT=2181 + - ZOOKEEPER_CLIENT_PORT=2181 kafka: image: docker.io/confluentinc/cp-kafka:7.6.1 depends_on: - - zookeeper + - zookeeper environment: - - KAFKA_BROKER_ID=1 - - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - - ZOOKEEPER_SASL_ENABLED=false - - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 - - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0 - - KAFKA_LISTENERS=PLAINTEXT://:9091,SSL://:9092,SASL_PLAINTEXT://:9093 - - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9091,SSL://kafka:9092,SASL_PLAINTEXT://kafka:9093 - - KAFKA_SSL_KEYSTORE_TYPE=PKCS12 - - KAFKA_SSL_KEY_CREDENTIALS=kafka.pass - - KAFKA_SSL_KEYSTORE_CREDENTIALS=kafka.pass - - KAFKA_SSL_KEYSTORE_FILENAME=kafka.p12 - - KAFKA_SSL_CLIENT_AUTH=none - - KAFKA_OPTS=-Djava.security.auth.login.config=/etc/kafka/kafka_server_jaas.conf - - KAFKA_SASL_ENABLED_MECHANISMS=PLAIN + - KAFKA_BROKER_ID=1 + - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - ZOOKEEPER_SASL_ENABLED=false + - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 + - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0 + - KAFKA_LISTENERS=PLAINTEXT://:9091,SSL://:9092,SASL_PLAINTEXT://:9093 + - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9091,SSL://kafka:9092,SASL_PLAINTEXT://kafka:9093 + - KAFKA_SSL_KEYSTORE_TYPE=PKCS12 + - KAFKA_SSL_KEY_CREDENTIALS=kafka.pass + - KAFKA_SSL_KEYSTORE_CREDENTIALS=kafka.pass + - KAFKA_SSL_KEYSTORE_FILENAME=kafka.p12 + - KAFKA_SSL_CLIENT_AUTH=none + - KAFKA_OPTS=-Djava.security.auth.login.config=/etc/kafka/kafka_server_jaas.conf + - KAFKA_SASL_ENABLED_MECHANISMS=PLAIN ports: - - 9091:9091 - - 9092:9092 - - 9093:9093 + - 9091:9091 + - 9092:9092 + - 9093:9093 volumes: - - ../../../data/ca/intermediate_server/private/kafka.pass:/etc/kafka/secrets/kafka.pass:ro - - ../../../data/ca/intermediate_server/private/kafka.p12:/etc/kafka/secrets/kafka.p12:ro - - ../../shared/data/kafka_server_jaas.conf:/etc/kafka/kafka_server_jaas.conf + - ../../../data/ca/intermediate_server/private/kafka.pass:/etc/kafka/secrets/kafka.pass:ro + - ../../../data/ca/intermediate_server/private/kafka.p12:/etc/kafka/secrets/kafka.p12:ro + - ../../shared/data/kafka_server_jaas.conf:/etc/kafka/kafka_server_jaas.conf networks: default: diff --git a/tests/integration/shutdown/config/test.yaml b/tests/integration/shutdown/config/test.yaml index c32f2e010940a..0086db0d6f175 100644 --- a/tests/integration/shutdown/config/test.yaml +++ b/tests/integration/shutdown/config/test.yaml @@ -1,9 +1,9 @@ args: -- --test-threads -- '4' + - --test-threads + - "4" features: -- shutdown-tests + - shutdown-tests test: "integration" diff --git a/tests/integration/splunk/config/compose.yaml b/tests/integration/splunk/config/compose.yaml index a252674d2bffa..4cff3cab1ca0d 100644 --- a/tests/integration/splunk/config/compose.yaml +++ b/tests/integration/splunk/config/compose.yaml @@ -1,18 +1,18 @@ -version: '3' +version: "3" services: splunk-hec: image: docker.io/splunk/splunk:${CONFIG_VERSION} environment: - - SPLUNK_START_ARGS=--accept-license - - SPLUNK_PASSWORD=password - - SPLUNK_HEC_TOKEN=abcd1234 + - SPLUNK_START_ARGS=--accept-license + - SPLUNK_PASSWORD=password + - SPLUNK_HEC_TOKEN=abcd1234 volumes: - - ../data/splunk/default.yml:/tmp/defaults/default.yml + - ../data/splunk/default.yml:/tmp/defaults/default.yml ports: - - 8000:8000 - - 8088:8088 - - 8089:8089 + - 8000:8000 + - 8088:8088 + - 8089:8089 networks: default: diff --git a/tests/integration/splunk/config/test.yaml b/tests/integration/splunk/config/test.yaml index 71a5c7994fe4a..2e745fab67352 100644 --- a/tests/integration/splunk/config/test.yaml +++ b/tests/integration/splunk/config/test.yaml @@ -1,7 +1,7 @@ features: -- splunk-integration-tests + - splunk-integration-tests -test_filter: '::splunk_hec::' +test_filter: "::splunk_hec::" env: SPLUNK_API_ADDRESS: https://splunk-hec:8089 @@ -13,9 +13,9 @@ matrix: # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/internal_events/splunk_hec.rs" -- "src/sources/splunk_hec/**" -- "src/sources/util/**" -- "src/sinks/splunk_hec/**" -- "src/sinks/util/**" -- "tests/integration/splunk/**" + - "src/internal_events/splunk_hec.rs" + - "src/sources/splunk_hec/**" + - "src/sources/util/**" + - "src/sinks/splunk_hec/**" + - "src/sinks/util/**" + - "tests/integration/splunk/**" diff --git a/tests/integration/webhdfs/config/compose.yaml b/tests/integration/webhdfs/config/compose.yaml index f3b5a51519513..17fd3d9eb69b5 100644 --- a/tests/integration/webhdfs/config/compose.yaml +++ b/tests/integration/webhdfs/config/compose.yaml @@ -1,4 +1,4 @@ -version: '3' +version: "3" services: namenode: diff --git a/tests/integration/webhdfs/config/test.yaml b/tests/integration/webhdfs/config/test.yaml index f1bd9f88b2ff9..029f12a587b76 100644 --- a/tests/integration/webhdfs/config/test.yaml +++ b/tests/integration/webhdfs/config/test.yaml @@ -1,17 +1,17 @@ -test_filter: '::webhdfs::' +test_filter: "::webhdfs::" features: -- webhdfs-integration-tests + - webhdfs-integration-tests env: WEBHDFS_ENDPOINT: http://namenode.local:9870 matrix: - hadoop: ['2.0.0-hadoop3.2.1-java8'] + hadoop: ["2.0.0-hadoop3.2.1-java8"] # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/sinks/webhdfs/**" -- "src/sinks/util/**" -- "tests/integration/webhdfs/**" + - "src/sinks/webhdfs/**" + - "src/sinks/util/**" + - "tests/integration/webhdfs/**" diff --git a/tests/integration/windows-event-log/config/test.yaml b/tests/integration/windows-event-log/config/test.yaml index ed0555b2e6695..6c3c619c54111 100644 --- a/tests/integration/windows-event-log/config/test.yaml +++ b/tests/integration/windows-event-log/config/test.yaml @@ -1,7 +1,7 @@ features: -- windows-event-log-integration-tests + - windows-event-log-integration-tests -test_filter: '::windows_event_log::integration_tests::' +test_filter: "::windows_event_log::integration_tests::" runner: env: {} @@ -12,6 +12,6 @@ matrix: # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch paths: -- "src/sources/windows_event_log/**" -- "src/internal_events/windows_event_log.rs" -- "tests/integration/windows-event-log/**" + - "src/sources/windows_event_log/**" + - "src/internal_events/windows_event_log.rs" + - "tests/integration/windows-event-log/**" diff --git a/tests/namespacing/ignore-invalid/sinks/es_cluster.json b/tests/namespacing/ignore-invalid/sinks/es_cluster.json index 6dd5e392cf253..c72ff984f709a 100644 --- a/tests/namespacing/ignore-invalid/sinks/es_cluster.json +++ b/tests/namespacing/ignore-invalid/sinks/es_cluster.json @@ -1,8 +1,6 @@ { - "inputs": [ - "apache_sample" - ], + "inputs": ["apache_sample"], "type": "elasticsearch", "endpoint": "http://79.12.221.222:9200", - "bulk": {"index": "vector-%Y-%m-%d"} + "bulk": { "index": "vector-%Y-%m-%d" } } diff --git a/tests/namespacing/success/sinks/es_cluster.json b/tests/namespacing/success/sinks/es_cluster.json index 6dd5e392cf253..c72ff984f709a 100644 --- a/tests/namespacing/success/sinks/es_cluster.json +++ b/tests/namespacing/success/sinks/es_cluster.json @@ -1,8 +1,6 @@ { - "inputs": [ - "apache_sample" - ], + "inputs": ["apache_sample"], "type": "elasticsearch", "endpoint": "http://79.12.221.222:9200", - "bulk": {"index": "vector-%Y-%m-%d"} + "bulk": { "index": "vector-%Y-%m-%d" } } diff --git a/vdev/src/commands/check/fmt.rs b/vdev/src/commands/check/fmt.rs index b01f6622218a2..1547d7b015808 100644 --- a/vdev/src/commands/check/fmt.rs +++ b/vdev/src/commands/check/fmt.rs @@ -1,6 +1,6 @@ use anyhow::Result; -use crate::app; +use crate::{app, commands::fmt::PRETTIER_EXTENSIONS, utils::git::git_ls_files}; /// Check that all files are formatted properly #[derive(clap::Args, Debug)] @@ -13,6 +13,21 @@ impl Cli { app::exec("scripts/check-style.sh", ["--all"], true)?; info!("Checking Rust formatting..."); - app::exec("cargo", ["fmt", "--", "--check"], true) + app::exec("cargo", ["fmt", "--", "--check"], true)?; + + for ext in PRETTIER_EXTENSIONS { + let files = git_ls_files(Some(ext))?; + if files.is_empty() { + continue; + } + info!("Checking prettier formatting for {ext} files..."); + let args: Vec<&str> = ["--ignore-path", ".prettierignore", "--check"] + .into_iter() + .chain(files.iter().map(String::as_str)) + .collect(); + app::exec("prettier", &args, true)?; + } + + Ok(()) } } diff --git a/vdev/src/commands/fmt.rs b/vdev/src/commands/fmt.rs index d53c6758ca09d..f1f7353b970d6 100644 --- a/vdev/src/commands/fmt.rs +++ b/vdev/src/commands/fmt.rs @@ -1,6 +1,9 @@ use anyhow::Result; -use crate::app; +use crate::{app, utils::git::git_ls_files}; + +pub(crate) const PRETTIER_EXTENSIONS: &[&str] = + &["*.yml", "*.yaml", "*.js", "*.ts", "*.tsx", "*.json"]; /// Apply format changes across the repository #[derive(clap::Args, Debug)] @@ -13,6 +16,21 @@ impl Cli { app::exec("scripts/check-style.sh", ["--fix"], true)?; info!("Formatting Rust code..."); - app::exec("cargo", ["fmt", "--all"], true) + app::exec("cargo", ["fmt", "--all"], true)?; + + for ext in PRETTIER_EXTENSIONS { + let files = git_ls_files(Some(ext))?; + if files.is_empty() { + continue; + } + info!("Formatting {ext} files with prettier..."); + let args: Vec<&str> = ["--ignore-path", ".prettierignore", "--write"] + .into_iter() + .chain(files.iter().map(String::as_str)) + .collect(); + app::exec("prettier", &args, true)?; + } + + Ok(()) } } diff --git a/website/.htmltest.external.yml b/website/.htmltest.external.yml index 3508294079c6f..3df01b8f3309d 100644 --- a/website/.htmltest.external.yml +++ b/website/.htmltest.external.yml @@ -4,11 +4,11 @@ CheckExternal: true IgnoreAltMissing: true IgnoreEmptyHref: true IgnoreURLs: -# We have to ignore this due to rate limiting -- "github.com" -- "packages.timber.io" -- "localhost" -# Somehow this link allows shows up as broken, so we'll exclude it -- "https://www.microsoft.com/en-us/windows" -# Somehow these links always come up as 5xx even though they work fine in the browser -- "maxmind.com" + # We have to ignore this due to rate limiting + - "github.com" + - "packages.timber.io" + - "localhost" + # Somehow this link allows shows up as broken, so we'll exclude it + - "https://www.microsoft.com/en-us/windows" + # Somehow these links always come up as 5xx even though they work fine in the browser + - "maxmind.com" diff --git a/website/.prettierrc.json b/website/.prettierrc.json deleted file mode 100644 index 7b17711cfff7a..0000000000000 --- a/website/.prettierrc.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "plugins": ["prettier-plugin-go-template"], - "overrides": [ - { - "files": ["*.html"], - "options": { - "parser": "go-template" - } - } - ], - "tabWidth": 2, - "useTabs": false, - "semi": false, - "singleQuote": true, - "trailingComma": "none", - "printWidth": 120 -} diff --git a/website/algolia.json b/website/algolia.json index d0c7622b530bf..1a88269b7d9e4 100644 --- a/website/algolia.json +++ b/website/algolia.json @@ -3,47 +3,24 @@ "minWordSizefor2Typos": 8, "hitsPerPage": 10, "maxValuesPerFacet": 100, - "searchableAttributes": [ - "title", - "content", - "unordered(hierarchy)", - "unordered(tags)" - ], + "searchableAttributes": ["title", "content", "unordered(hierarchy)", "unordered(tags)"], "numericAttributesToIndex": null, "attributesToRetrieve": null, "unretrievableAttributes": null, "optionalWords": null, "attributesForFaceting": null, - "attributesToSnippet": [ - "title:10", - "content:10" - ], + "attributesToSnippet": ["title:10", "content:10"], "attributesToHighlight": null, "paginationLimitedTo": 1000, "attributeForDistinct": null, "exactOnSingleWordQuery": "attribute", - "ranking": [ - "typo", - "geo", - "words", - "filters", - "proximity", - "attribute", - "exact", - "custom" - ], - "customRanking": [ - "desc(level)", - "desc(ranking)" - ], + "ranking": ["typo", "geo", "words", "filters", "proximity", "attribute", "exact", "custom"], + "customRanking": ["desc(level)", "desc(ranking)"], "separatorsToIndex": "", "removeWordsIfNoResults": "none", "queryType": "prefixLast", "highlightPreTag": "", "highlightPostTag": "", "snippetEllipsisText": "…", - "alternativesAsExact": [ - "ignorePlurals", - "singleWordSynonym" - ] + "alternativesAsExact": ["ignorePlurals", "singleWordSynonym"] } diff --git a/website/assets/js/below.js b/website/assets/js/below.js index f61004ef2a746..0123d5eb2c36c 100644 --- a/website/assets/js/below.js +++ b/website/assets/js/below.js @@ -1,16 +1,16 @@ -import tocbot from 'tocbot'; +import tocbot from "tocbot"; // Table of contents for documentation pages const tableOfContents = () => { - if (document.getElementById('toc')) { + if (document.getElementById("toc")) { tocbot.init({ - tocSelector: '#toc', - contentSelector: '#page-content', - headingSelector: 'h2,h3,h4,h5', + tocSelector: "#toc", + contentSelector: "#page-content", + headingSelector: "h2,h3,h4,h5", scrollSmoothDuration: 400 }); } -} +}; const showCodeFilename = () => { var els = document.getElementsByClassName("highlight"); @@ -21,9 +21,9 @@ const showCodeFilename = () => { els[i].parentNode.insertBefore(newNode, els[i]); } } -} +}; -document.addEventListener('DOMContentLoaded', () => { +document.addEventListener("DOMContentLoaded", () => { // search.start(); tableOfContents(); diff --git a/website/assets/js/home.tsx b/website/assets/js/home.tsx index fb92927ac4fe8..e6271eb538d16 100644 --- a/website/assets/js/home.tsx +++ b/website/assets/js/home.tsx @@ -28,15 +28,12 @@ interface IGlobeProps { const markers: GeoJSON.Point[] = markerData.map((m) => ({ type: "Point", - coordinates: [m.coordinates[0], m.coordinates[1]], + coordinates: [m.coordinates[0], m.coordinates[1]] })); const topo: unknown = countryData; const topology = topo as Topology; -const geojson = feature( - topology, - topology.objects.land -) as GeoJSON.FeatureCollection; +const geojson = feature(topology, topology.objects.land) as GeoJSON.FeatureCollection; const countries = geojson.features; export const Globe: React.FC = animated( @@ -66,30 +63,20 @@ export const Globe: React.FC = animated( return ( - ) + ); }, - ...props, - }) + ...props + }); return () => { - search.destroy() - } - }, [props]) + search.destroy(); + }; + }, [props]); - return
-} + return
; +}; const Search = () => { return ( @@ -190,64 +166,69 @@ const Search = () => { defaultActiveItemId={0} getSources={({ query }) => [ { - sourceId: 'queryResults', + sourceId: "queryResults", async getItems() { - const results = (query) => searchClient.collections('vector_docs').documents().search({ - q: query, - preset: 'vector_docs_search', - exhaustive_search: true, - highlight_fields: 'content', - highlight_full_fields: 'content' - - }).then((result) => { - // order the hits by page group - // const hits = result.hits.sort((a, b) => (a.document.pageTitle < b.document.pageTitle ? -1 : 1)) - - // add page as category if there are duplicates - const hitsWithCategory = result.hits.map((h, i) => { - const prev = result.hits[i - 1] as any - const title = h.document.pageTitle - - // if no previous hit is in this category - if (!prev) { - return { ...h, category: title } - } - - // skip if there is already one in this category - if (prev && prev.document.pageTitle === title) { - return h - } - - // add category if needed - if (prev && prev.document.pageTitle !== title) { - return { ...h, category: title } - } - - return h - }) - return hitsWithCategory - }) - return await results(query) + const results = (query) => + searchClient + .collections("vector_docs") + .documents() + .search({ + q: query, + preset: "vector_docs_search", + exhaustive_search: true, + highlight_fields: "content", + highlight_full_fields: "content" + }) + .then((result) => { + // order the hits by page group + // const hits = result.hits.sort((a, b) => (a.document.pageTitle < b.document.pageTitle ? -1 : 1)) + + // add page as category if there are duplicates + const hitsWithCategory = result.hits.map((h, i) => { + const prev = result.hits[i - 1] as any; + const title = h.document.pageTitle; + + // if no previous hit is in this category + if (!prev) { + return { ...h, category: title }; + } + + // skip if there is already one in this category + if (prev && prev.document.pageTitle === title) { + return h; + } + + // add category if needed + if (prev && prev.document.pageTitle !== title) { + return { ...h, category: title }; + } + + return h; + }); + return hitsWithCategory; + }); + return await results(query); }, getItemUrl({ item }) { - return item.document.itemUrl + return item.document.itemUrl; }, templates: { item({ item, components }) { - const highlight = item.highlights.length && item.highlights.find(h => h.field === 'content' || {}).value || item.document['content'] - item['content'] = highlight - return + const highlight = + (item.highlights.length && item.highlights.find((h) => h.field === "content" || {}).value) || + item.document["content"]; + item["content"] = highlight; + return ; }, noResults() { - return 'No results found.'; - }, - }, + return "No results found."; + } + } } ]} /> - ) -} - + ); +}; -const searchRoot = createRoot(document.getElementById('site-search')) -searchRoot.render() +const searchRoot = createRoot(document.getElementById("site-search")); +searchRoot.render(); diff --git a/website/babel.config.js b/website/babel.config.js index cfe4e614d29c6..bf0e24707c871 100644 --- a/website/babel.config.js +++ b/website/babel.config.js @@ -1,6 +1,6 @@ -import presetEnv from '@babel/preset-env'; -import presetReact from '@babel/preset-react'; -import presetTypeScript from '@babel/preset-typescript'; +import presetEnv from "@babel/preset-env"; +import presetReact from "@babel/preset-react"; +import presetTypeScript from "@babel/preset-typescript"; export default function (api) { api.cache(true); @@ -9,22 +9,22 @@ export default function (api) { [ presetEnv, { - "useBuiltIns": 'entry', - "corejs": 3 + useBuiltIns: "entry", + corejs: 3 } ], [ presetReact, { - "flow": false, - "typescript": true + flow: false, + typescript: true } ], [ presetTypeScript, { - "isTSX": true, - "allExtensions": true + isTSX: true, + allExtensions: true } ] ]; diff --git a/website/data/redirects.yaml b/website/data/redirects.yaml index 4e7a94f467b56..e8ac227c4248b 100644 --- a/website/data/redirects.yaml +++ b/website/data/redirects.yaml @@ -1,74 +1,74 @@ platforms: -- docker -- kubernetes + - docker + - kubernetes sinks: -- aws_sqs -- aws_cloudwatch_logs -- aws_cloudwatch_metrics -- aws_kinesis_firehose -- aws_kinesis_streams -- aws_s3 -- axiom -- azure_monitor_logs -- clickhouse -- databend -- datadog_logs -- datadog_metrics -- elasticsearch -- file -- gcp_cloud_storage -- gcp_pubsub -- gcp_stackdriver_logs -- gcp_stackdriver_metrics -- honeycomb -- http -- humio_logs -- humio_metrics -- influxdb_logs -- influxdb_metrics -- kafka -- keep -- mezmo -- loki -- nats -- new_relic_logs -- new_relic -- papertrail -- postgres -- prometheus_exporter -- prometheus_remote_write -- pulsar -- sematext_logs -- sematext_metrics -- socket -- splunk_hec -- statsd + - aws_sqs + - aws_cloudwatch_logs + - aws_cloudwatch_metrics + - aws_kinesis_firehose + - aws_kinesis_streams + - aws_s3 + - axiom + - azure_monitor_logs + - clickhouse + - databend + - datadog_logs + - datadog_metrics + - elasticsearch + - file + - gcp_cloud_storage + - gcp_pubsub + - gcp_stackdriver_logs + - gcp_stackdriver_metrics + - honeycomb + - http + - humio_logs + - humio_metrics + - influxdb_logs + - influxdb_metrics + - kafka + - keep + - mezmo + - loki + - nats + - new_relic_logs + - new_relic + - papertrail + - postgres + - prometheus_exporter + - prometheus_remote_write + - pulsar + - sematext_logs + - sematext_metrics + - socket + - splunk_hec + - statsd sources: -- apache_metrics -- aws_ecs_metrics -- aws_kinesis_firehose -- aws_s3 -- datadog_logs -- exec -- file -- heroku_logs -- host_metrics -- http -- journald -- kafka -- mongodb_metrics -- nginx_metrics -- postgresql_metrics -- prometheus_remote_write -- prometheus_scrape -- pulsar -- socket -- splunk_hec -- statsd -- stdin -- syslog + - apache_metrics + - aws_ecs_metrics + - aws_kinesis_firehose + - aws_s3 + - datadog_logs + - exec + - file + - heroku_logs + - host_metrics + - http + - journald + - kafka + - mongodb_metrics + - nginx_metrics + - postgresql_metrics + - prometheus_remote_write + - prometheus_scrape + - pulsar + - socket + - splunk_hec + - statsd + - stdin + - syslog # /guides/integrate/platforms/${key}/* /docs/reference/configuration/sources/${source} multi: diff --git a/website/postcss.config.js b/website/postcss.config.js index 96e73589f6611..ccd51f20f80bc 100644 --- a/website/postcss.config.js +++ b/website/postcss.config.js @@ -1,7 +1,7 @@ -import postcssImport from 'postcss-import'; -import tailwindCss from 'tailwindcss'; -import autoprefixer from 'autoprefixer'; -import purgecssPlugin from '@fullhuman/postcss-purgecss'; +import postcssImport from "postcss-import"; +import tailwindCss from "tailwindcss"; +import autoprefixer from "autoprefixer"; +import purgecssPlugin from "@fullhuman/postcss-purgecss"; // These are classes for things that are applied by JS, and thus missed by Hugo. // See assets/js/*.js for places where this happens. @@ -40,12 +40,12 @@ const safeClasses = { "text-md", "text-sm", "w-2", - "w-3", + "w-3" ] }; const purgecss = purgecssPlugin({ - content: ['./hugo_stats.json'], + content: ["./hugo_stats.json"], safelist: safeClasses, defaultExtractor: (content) => { const broadMatches = content.match(/[^<>"'`\s]*[^<>"'`\s:]/g) || []; @@ -59,6 +59,6 @@ export default { postcssImport, tailwindCss, autoprefixer, - ...(process.env.HUGO_ENVIRONMENT === 'production' ? [purgecss] : []) + ...(process.env.HUGO_ENVIRONMENT === "production" ? [purgecss] : []) ] -} +}; diff --git a/website/scripts/create-config-examples.js b/website/scripts/create-config-examples.js index b9096fd4d3b1d..692340b359413 100755 --- a/website/scripts/create-config-examples.js +++ b/website/scripts/create-config-examples.js @@ -1,7 +1,7 @@ -import fs from 'fs'; -import chalk from 'chalk'; -import * as TOML from '@iarna/toml'; -import YAML from 'yaml'; +import fs from "fs"; +import chalk from "chalk"; +import * as TOML from "@iarna/toml"; +import YAML from "yaml"; const cueJsonOutput = "data/docs.json"; @@ -10,56 +10,54 @@ const getExampleValue = (param, deepFilter) => { let value; const getArrayValue = (obj) => { - const enumVal = (obj.enum != null) ? [Object.keys(obj.enum)[0]] : null; + const enumVal = obj.enum != null ? [Object.keys(obj.enum)[0]] : null; - const examplesVal = (obj.examples != null && obj.examples.length > 0) ? [obj.examples[0]] : null; + const examplesVal = obj.examples != null && obj.examples.length > 0 ? [obj.examples[0]] : null; return obj.default || examplesVal || enumVal || null; - } + }; const getValue = (obj) => { - const enumVal = (obj.enum != null) ? Object.keys(obj.enum)[0] : null; + const enumVal = obj.enum != null ? Object.keys(obj.enum)[0] : null; - const examplesVal = (obj.examples != null && obj.examples.length > 0) ? obj.examples[0] : null; + const examplesVal = obj.examples != null && obj.examples.length > 0 ? obj.examples[0] : null; return obj.default || examplesVal || enumVal || null; - } + }; - Object.keys(param.type).forEach(k => { + Object.keys(param.type).forEach((k) => { const p = param.type[k]; - if (['array', 'object'].includes(k)) { + if (["array", "object"].includes(k)) { const topType = k; if (p.items && p.items.type) { const typeInfo = p.items.type; - Object.keys(typeInfo).forEach(k => { - if (['array', 'object'].includes(k)) { + Object.keys(typeInfo).forEach((k) => { + if (["array", "object"].includes(k)) { const subType = k; const options = typeInfo[k].options; var subObj = {}; - Object - .keys(options) - .filter(k => deepFilter(options[k])) - .forEach(k => { - Object.keys(options[k].type).forEach(key => { + Object.keys(options) + .filter((k) => deepFilter(options[k])) + .forEach((k) => { + Object.keys(options[k].type).forEach((key) => { const deepTypeInfo = options[k].type[key]; - if (subType === 'array') { + if (subType === "array") { subObj[k] = getArrayValue(deepTypeInfo); } else { subObj[k] = getValue(deepTypeInfo); } - }); }); value = subObj; } else { - if (topType === 'array') { + if (topType === "array") { value = getArrayValue(typeInfo[k]); } else { value = getValue(typeInfo[k]); @@ -75,15 +73,14 @@ const getExampleValue = (param, deepFilter) => { }); return value; -} +}; Object.makeExampleParams = (params, filter, deepFilter) => { var obj = {}; - Object - .keys(params) - .filter(k => filter(params[k])) - .forEach(k => { + Object.keys(params) + .filter((k) => filter(params[k])) + .forEach((k) => { let value = getExampleValue(params[k], deepFilter); if (value) { obj[k] = value; @@ -91,22 +88,22 @@ Object.makeExampleParams = (params, filter, deepFilter) => { }); return obj; -} +}; // Convert object to TOML string const toToml = (obj) => { return TOML.stringify(obj); -} +}; // Convert object to YAML string const toYaml = (obj) => { return `${YAML.stringify(obj)}`; -} +}; // Convert object to JSON string (indented) const toJson = (obj) => { return JSON.stringify(obj, null, 2); -} +}; // Set the example value for a given config parameter const setExampleValue = (exampleConfig, paramName, param) => { @@ -125,7 +122,7 @@ const setExampleValue = (exampleConfig, paramName, param) => { exampleConfig[paramName] = p.examples[0]; } - if (['array', 'object'].includes(k)) { + if (["array", "object"].includes(k)) { if (p.items) { var obj = {}; @@ -157,7 +154,7 @@ const setExampleValue = (exampleConfig, paramName, param) => { } } }); -} +}; // Assemble the "minimal" params for an example config const makeMinimalParams = (configuration) => { @@ -175,7 +172,7 @@ const makeMinimalParams = (configuration) => { } return minimal; -} +}; // Assemble the "advanced" params for an example config const makeAllParams = (configuration) => { @@ -190,7 +187,7 @@ const makeAllParams = (configuration) => { } return optional; -} +}; // Convert the use case examples (`component.examples`) into multi-format const makeUseCaseExamples = (component) => { @@ -210,31 +207,31 @@ const makeUseCaseExamples = (component) => { exampleConfig = { [kindPlural]: { [keyName]: { - "type": component.type, - inputs: ['my-source-or-transform-id'], + type: component.type, + inputs: ["my-source-or-transform-id"], ...extra } } - } + }; } else { exampleConfig = { [kindPlural]: { [keyName]: { - "type": component.type, + type: component.type, ...extra } } - } + }; } // Strip the "log" or "metric" key in the example output let output; if (example.output) { - if (example.output['log']) { - output = example.output['log']; - } else if (example.output['metric']) { - output = example.output['metric']; + if (example.output["log"]) { + output = example.output["log"]; + } else if (example.output["metric"]) { + output = example.output["metric"]; } else { output = example.output; } @@ -248,11 +245,11 @@ const makeUseCaseExamples = (component) => { configuration: { toml: toToml(exampleConfig), yaml: toYaml(exampleConfig), - json: toJson(exampleConfig), + json: toJson(exampleConfig) }, input: example.input, - output: output, - } + output: output + }; useCases.push(useCase); }); @@ -261,12 +258,12 @@ const makeUseCaseExamples = (component) => { } else { return null; } -} +}; const main = () => { try { const debug = process.env.DEBUG === "true" || false; - const data = fs.readFileSync(cueJsonOutput, 'utf8'); + const data = fs.readFileSync(cueJsonOutput, "utf8"); const docs = JSON.parse(data); const components = docs.components; @@ -285,13 +282,13 @@ const main = () => { const minimalParams = Object.makeExampleParams( configuration, - p => p.required || p.minimal, - p => p.required || p.minimal, + (p) => p.required || p.minimal, + (p) => p.required || p.minimal ); const advancedParams = Object.makeExampleParams( configuration, - _ => true, - p => p.required || p.minimal || p.relevant_when, + (_) => true, + (p) => p.required || p.minimal || p.relevant_when ); const useCaseExamples = makeUseCaseExamples(component); @@ -300,13 +297,13 @@ const main = () => { let minimalExampleConfig, advancedExampleConfig; // Sinks and transforms are treated differently because they need an `inputs` field - if (['sinks', 'transforms'].includes(kind)) { + if (["sinks", "transforms"].includes(kind)) { minimalExampleConfig = { [kind]: { [keyName]: { - "type": componentType, - inputs: ['my-source-or-transform-id'], - ...minimalParams, + type: componentType, + inputs: ["my-source-or-transform-id"], + ...minimalParams } } }; @@ -314,9 +311,9 @@ const main = () => { advancedExampleConfig = { [kind]: { [keyName]: { - "type": componentType, - inputs: ['my-source-or-transform-id'], - ...advancedParams, + type: componentType, + inputs: ["my-source-or-transform-id"], + ...advancedParams } } }; @@ -324,8 +321,8 @@ const main = () => { minimalExampleConfig = { [kind]: { [keyName]: { - "type": componentType, - ...minimalParams, + type: componentType, + ...minimalParams } } }; @@ -333,49 +330,47 @@ const main = () => { advancedExampleConfig = { [kind]: { [keyName]: { - "type": componentType, - ...advancedParams, + type: componentType, + ...advancedParams } } }; } - docs['components'][kind][componentType]['examples'] = useCaseExamples; + docs["components"][kind][componentType]["examples"] = useCaseExamples; - docs['components'][kind][componentType]['example_configs'] = { + docs["components"][kind][componentType]["example_configs"] = { minimal: { toml: toToml(minimalExampleConfig), yaml: toYaml(minimalExampleConfig), - json: toJson(minimalExampleConfig), + json: toJson(minimalExampleConfig) }, advanced: { toml: toToml(advancedExampleConfig), yaml: toYaml(advancedExampleConfig), - json: toJson(advancedExampleConfig), - }, + json: toJson(advancedExampleConfig) + } }; } } - // A debugging statement to make sure things are going basically as planned if (debug) { - console.log(docs['components']['sources']['syslog']['examples']); + console.log(docs["components"]["sources"]["syslog"]["examples"]); } - console.log(chalk.green("Success. Finished generating examples for all components.")); console.log(chalk.blue(`Writing generated examples as JSON to ${cueJsonOutput}...`)); // Write back to the JSON file only when not in debug mode if (!debug) { - fs.writeFileSync(cueJsonOutput, JSON.stringify(docs), 'utf8'); + fs.writeFileSync(cueJsonOutput, JSON.stringify(docs), "utf8"); } console.log(chalk.green(`Success. Finished writing example configs to ${cueJsonOutput}.`)); } catch (err) { console.error(err); } -} +}; main(); diff --git a/website/scripts/typesense-index.ts b/website/scripts/typesense-index.ts index 7b14f07d0fbdd..f800e710d1754 100644 --- a/website/scripts/typesense-index.ts +++ b/website/scripts/typesense-index.ts @@ -54,7 +54,7 @@ const tagHierarchy = { h5: 2, h6: 1, li: 1, - p: 1, + p: 1 }; function getPageUrl(file: string) { @@ -75,7 +75,7 @@ function getItemUrl(file: string, { level, domId }: Payload[0]) { // Hash the file name to create a unique ID for the record using the same hash function as the one used in the synapse stream package. function hashString(fileName: string) { - return crypto.createHash('md5').update(fileName).digest('hex') + return crypto.createHash("md5").update(fileName).digest("hex"); } async function indexHTMLFiles( @@ -92,8 +92,8 @@ async function indexHTMLFiles( const $ = cheerio.load(html); const containers = $("#page-content"); const pageTitle = $("meta[name='algolia:title']").attr("content") || ""; - const pageTagsString = $("meta[name='keywords']").attr('content') || ""; - const pageTags: string[] = (pageTagsString === "") ? [] : pageTagsString.split(","); + const pageTagsString = $("meta[name='keywords']").attr("content") || ""; + const pageTags: string[] = pageTagsString === "" ? [] : pageTagsString.split(","); // @ts-ignore $(".algolia-no-index").each((_, d) => $(d).remove()); @@ -114,7 +114,7 @@ async function indexHTMLFiles( tagName: node.tagName, content: $(node) .text() - .replace(/[\n\t]/g, " "), + .replace(/[\n\t]/g, " ") }); } @@ -146,9 +146,10 @@ async function indexHTMLFiles( ranking, hierarchy: [], tags: pageTags, - content: "", + content: "" }; - } else if (item.level === 1) { // h1 logic + } else if (item.level === 1) { + // h1 logic activeRecord.content += item.content; } else if (item.level < activeRecord.level) { algoliaRecords.push({ ...activeRecord }); @@ -164,9 +165,10 @@ async function indexHTMLFiles( ranking, hierarchy: [...activeRecord.hierarchy, activeRecord.title.trim()], tags: pageTags, - content: "", + content: "" }; - } else { // h2-h6 logic + } else { + // h2-h6 logic algoliaRecords.push({ ...activeRecord }); const hierarchySize = activeRecord.hierarchy.length; @@ -184,12 +186,12 @@ async function indexHTMLFiles( ranking, hierarchy: [...activeRecord.hierarchy.slice(0, lastIndex)], tags: pageTags, - content: "", + content: "" }; } if (activeRecord) { - algoliaRecords.push({ ...activeRecord }) + algoliaRecords.push({ ...activeRecord }); } for (const rec of algoliaRecords) { @@ -211,9 +213,7 @@ async function indexHTMLFiles( } } - console.log( - chalk.green(`Success. Updated records for ${files.length} file(s).`) - ); + console.log(chalk.green(`Success. Updated records for ${files.length} file(s).`)); records.push(...algoliaRecords); } @@ -228,56 +228,56 @@ async function buildIndex() { name: "Docs", path: `${publicPath}/docs/introduction/**/**.html`, displayPath: "docs/introduction", - ranking: 50, + ranking: 50 }, { name: "Docs", path: `${publicPath}/docs/administration/**/**.html`, displayPath: "docs/administration", - ranking: 50, + ranking: 50 }, { name: "Docs", path: `${publicPath}/docs/reference/**/**.html`, displayPath: "docs/reference", - ranking: 50, + ranking: 50 }, { name: "Docs", path: `${publicPath}/docs/setup/**/**.html`, displayPath: "docs/setup", - ranking: 50, + ranking: 50 }, { name: "Advanced guides", path: `${publicPath}/guides/advanced/**/**.html`, displayPath: "guides/advanced", - ranking: 40, + ranking: 40 }, { name: "Level up guides", path: `${publicPath}/guides/level-up/**/**.html`, displayPath: "guides/level-up", - ranking: 40, + ranking: 40 }, { name: "Developer guides", path: `${publicPath}/guides/developer/**/**.html`, displayPath: "guides/developer", - ranking: 40, + ranking: 40 }, { name: "Vector highlights", path: `${publicPath}/highlights/**/**.html`, displayPath: "highlights/", - ranking: 40, + ranking: 40 }, { name: "Upgrade guides", path: `${publicPath}/highlights/*-upgrade-guide/**.html`, displayPath: "highlights/", - ranking: 10, - }, + ranking: 10 + } ]; // Recurse through each section and push the resulting records to `allRecords` @@ -286,7 +286,7 @@ async function buildIndex() { // We shouldn't index upgrade guides if (section.name === "Vector highlights") { - files = files.filter((path) => !path.includes("-upgrade-guide")) + files = files.filter((path) => !path.includes("-upgrade-guide")); } console.log(chalk.blue(`Indexing ${section.displayPath}...`)); diff --git a/website/scripts/typesense-sync.ts b/website/scripts/typesense-sync.ts index 65f867de7b98d..9590be0104fb8 100644 --- a/website/scripts/typesense-sync.ts +++ b/website/scripts/typesense-sync.ts @@ -1,19 +1,18 @@ // Will be running in the CI/CD pipeline -import process from 'node:process'; -import { typesenseSync } from 'typesense-sync'; -const configFilePath = './typesense.config.json'; +import process from "node:process"; +import { typesenseSync } from "typesense-sync"; +const configFilePath = "./typesense.config.json"; const config = { configFilePath, - shouldSaveSettings: true, + shouldSaveSettings: true }; typesenseSync(config) .then(() => { - console.log('Typesense sync completed'); + console.log("Typesense sync completed"); }) .catch((err) => { - console.error('An error occurred', err); + console.error("An error occurred", err); process.exit(1); }); - diff --git a/website/typesense.config.json b/website/typesense.config.json index 89b40e66a4ef8..249a303dc895e 100644 --- a/website/typesense.config.json +++ b/website/typesense.config.json @@ -54,7 +54,7 @@ "optional": true, "sort": false, "stem": false, - "type": "string", + "type": "string", "stem_dictionary": "", "store": true }, From b59c9a25b899223b6633776eeafe2e67fe418640 Mon Sep 17 00:00:00 2001 From: steveduan-IDME Date: Thu, 2 Apr 2026 11:32:25 -0400 Subject: [PATCH 093/326] chore(ci): Add http_server source to semantic PR scope list (#25078) Co-Authored-By: Claude Code --- .github/workflows/semantic.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/semantic.yml b/.github/workflows/semantic.yml index 79a36a94ec7cf..1e134a9f96007 100644 --- a/.github/workflows/semantic.yml +++ b/.github/workflows/semantic.yml @@ -160,6 +160,7 @@ jobs: heroku_logs source host_metrics source http_client source + http_server source http_scrape source internal_logs source internal_metrics source From 1a78c93520ab25377b7162c9aa5756f29492f433 Mon Sep 17 00:00:00 2001 From: Thomas Date: Thu, 2 Apr 2026 13:57:21 -0400 Subject: [PATCH 094/326] fix(ci): fix Pulsar TLS integration tests with latest image (#25099) * fix(ci): pin Pulsar integration test image to 4.1.3 The `apachepulsar/pulsar:latest` image moved to a version that breaks TLS connectivity, causing `pulsar_happy_tls` and `consumes_event_with_tls` to fail consistently in CI. Pin to 4.1.3 to unblock the merge queue. Re-enabling `latest` is tracked in #25096. Co-Authored-By: Claude Opus 4.6 (1M context) * Add PULSAR_PREFIX_advertisedAddress=pulsar * fix(ci): revert Pulsar image pin to latest * Revert to version: latest * Fix test.yml format * Delete obsolete comment --------- Co-authored-by: Pavlos Rontidis Co-authored-by: Claude Opus 4.6 (1M context) --- tests/integration/pulsar/config/compose.yaml | 4 ++++ tests/integration/pulsar/config/test.yaml | 4 +--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/integration/pulsar/config/compose.yaml b/tests/integration/pulsar/config/compose.yaml index 5d6c4e86249f9..58c2b9344fcad 100644 --- a/tests/integration/pulsar/config/compose.yaml +++ b/tests/integration/pulsar/config/compose.yaml @@ -8,6 +8,10 @@ services: - 6650:6650 - 6651:6651 environment: + # Since apache/pulsar#25238, standalone mode advertises the container's FQDN + # (the container ID) instead of localhost. Set explicitly so TLS hostname + # verification matches the certificate's DNS SAN. + - PULSAR_PREFIX_advertisedAddress=pulsar - PULSAR_PREFIX_brokerServicePortTls=6651 - PULSAR_PREFIX_tlsKeyFilePath=/etc/pulsar/certs/pulsar.key.pem - PULSAR_PREFIX_tlsCertificateFilePath=/etc/pulsar/certs/pulsar.cert.pem diff --git a/tests/integration/pulsar/config/test.yaml b/tests/integration/pulsar/config/test.yaml index 51dbb1100c9da..c136bffaaab24 100644 --- a/tests/integration/pulsar/config/test.yaml +++ b/tests/integration/pulsar/config/test.yaml @@ -7,9 +7,7 @@ env: PULSAR_HOST: pulsar matrix: - # TODO: re-enable `latest` once TLS tests pass with newer Pulsar images. - # See https://github.com/vectordotdev/vector/issues/25096 - version: ["4.1.3"] + version: [latest] # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch From dedb9667511691731c30c05235c319867d96580c Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Fri, 3 Apr 2026 11:19:53 -0400 Subject: [PATCH 095/326] fix(ci): pin npm transitive deps via package-lock.json and npm ci (#25115) Replace `npm install -g` (which resolves transitive dependencies live from the npm registry without a lockfile) with `npm ci` against a committed package-lock.json. This pins the entire dependency tree for markdownlint-cli2, prettier, and datadog-ci, closing the supply chain hole exposed by the axios compromise (2026-03-30). Changes: - Add scripts/environment/npm-tools/ with package.json and package-lock.json as the single source of truth for npm tool versions - Replace maybe_install_npm_package with maybe_install_npm_tools using npm ci --prefix for deterministic installs from the lockfile - Use $(npm config get prefix -g)/bin for symlinks instead of hardcoded /usr/local/bin to respect npm configuration - Use sudo for symlinks only when the target dir is not writable - Verify symlink targets point to our node_modules, not just that binaries exist, to prevent stale or system installs from being used - Early return when no npm tool is requested, so hosts without npm are not broken - Separate cargo and npm caches in CI for cleaner invalidation; npm cache keys hash all files in npm-tools/ - Bump datadog-ci to 5.11.0 to resolve critical simple-git RCE (GHSA-r275-fr43-pm7q) Co-authored-by: Claude Opus 4.6 (1M context) --- .github/actions/setup/action.yml | 18 +- scripts/environment/Dockerfile | 1 + .../environment/npm-tools/package-lock.json | 3378 +++++++++++++++++ scripts/environment/npm-tools/package.json | 9 + scripts/environment/prepare.sh | 67 +- 5 files changed, 3449 insertions(+), 24 deletions(-) create mode 100644 scripts/environment/npm-tools/package-lock.json create mode 100644 scripts/environment/npm-tools/package.json diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index fde57a4b33303..6faa185204ced 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -256,8 +256,8 @@ runs: with: skip-cache: ${{ env.VDEV_NEEDS_COMPILE == 'true' }} - - name: Cache prepare.sh binaries - id: cache-prepare-binaries + - name: Cache cargo binaries + id: cache-cargo-binaries uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 with: path: | @@ -270,12 +270,16 @@ runs: ~/.cargo/bin/cargo-llvm-cov ~/.cargo/bin/dd-rust-license-tool ~/.cargo/bin/wasm-pack - /usr/local/bin/markdownlint-cli2 - /usr/local/bin/prettier - /usr/local/bin/datadog-ci - key: ${{ runner.os }}-prepare-binaries-${{ hashFiles('scripts/environment/*') }} + key: ${{ runner.os }}-cargo-binaries-${{ hashFiles('scripts/environment/*.sh') }} restore-keys: | - ${{ runner.os }}-prepare-binaries- + ${{ runner.os }}-cargo-binaries- + + - name: Cache npm tools + id: cache-npm-tools + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 + with: + path: scripts/environment/npm-tools/node_modules + key: ${{ runner.os }}-npm-tools-${{ hashFiles('scripts/environment/npm-tools/*') }} - name: Run prepare.sh shell: bash diff --git a/scripts/environment/Dockerfile b/scripts/environment/Dockerfile index fa7d72075dae6..2798a296dd428 100644 --- a/scripts/environment/Dockerfile +++ b/scripts/environment/Dockerfile @@ -16,6 +16,7 @@ WORKDIR /git/vectordotdev/vector # Setup the env COPY scripts/environment/*.sh scripts/environment/ +COPY scripts/environment/npm-tools/ scripts/environment/npm-tools/ RUN ./scripts/environment/bootstrap-ubuntu-24.04.sh # Setup the toolchain diff --git a/scripts/environment/npm-tools/package-lock.json b/scripts/environment/npm-tools/package-lock.json new file mode 100644 index 0000000000000..e3d4efa122515 --- /dev/null +++ b/scripts/environment/npm-tools/package-lock.json @@ -0,0 +1,3378 @@ +{ + "name": "environment", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "dependencies": { + "@datadog/datadog-ci": "5.11.0", + "markdownlint-cli2": "0.22.0", + "prettier": "3.8.1" + } + }, + "node_modules/@antfu/install-pkg": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@antfu/install-pkg/-/install-pkg-1.1.0.tgz", + "integrity": "sha512-MGQsmw10ZyI+EJo45CdSER4zEb+p31LpDAFp2Z3gkSd1yqVZGi0Ebx++YTEMonJy4oChEMLsxZ64j8FH6sSqtQ==", + "license": "MIT", + "dependencies": { + "package-manager-detector": "^1.3.0", + "tinyexec": "^1.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/@datadog/datadog-ci": { + "version": "5.11.0", + "resolved": "https://registry.npmjs.org/@datadog/datadog-ci/-/datadog-ci-5.11.0.tgz", + "integrity": "sha512-4N/WsfnQVYJNxGtDIOhg4sUkZalDo8h+toSXR/GAw6XU81e1bOj3GvleL6IHUQ06KyPdurqPZKB4Za1CbCsT7g==", + "license": "Apache-2.0", + "dependencies": { + "@datadog/datadog-ci-base": "5.11.0", + "@datadog/datadog-ci-plugin-coverage": "5.11.0", + "@datadog/datadog-ci-plugin-deployment": "5.11.0", + "@datadog/datadog-ci-plugin-dora": "5.11.0", + "@datadog/datadog-ci-plugin-gate": "5.11.0", + "@datadog/datadog-ci-plugin-junit": "5.11.0", + "@datadog/datadog-ci-plugin-sarif": "5.11.0", + "@datadog/datadog-ci-plugin-sbom": "5.11.0", + "clipanion": "3.2.1", + "typanion": "3.14.0" + }, + "bin": { + "datadog-ci": "dist/cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@datadog/datadog-ci-base": { + "version": "5.11.0", + "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-base/-/datadog-ci-base-5.11.0.tgz", + "integrity": "sha512-NACS6zhf2qidgpgrhQH0EJn4x20pwyRAhw75jBRPmjzheaXrpU0438e3kutyNsDGZrw8gHjuRqiccoY3QU7PSQ==", + "license": "Apache-2.0", + "dependencies": { + "@antfu/install-pkg": "1.1.0", + "@types/datadog-metrics": "0.6.1", + "async-retry": "1.3.1", + "axios": "1.13.5", + "chalk": "3.0.0", + "clipanion": "3.2.1", + "datadog-metrics": "0.9.3", + "debug": "4.4.1", + "deep-extend": "0.6.0", + "fast-xml-parser": "5.3.7", + "form-data": "4.0.4", + "glob": "13.0.5", + "inquirer": "8.2.7", + "is-docker": "4.0.0", + "jest-diff": "30.2.0", + "js-yaml": "4.1.1", + "jszip": "3.10.1", + "proxy-agent": "6.4.0", + "semver": "7.6.3", + "simple-git": "3.33.0", + "terminal-link": "2.1.1", + "tiny-async-pool": "2.1.0", + "typanion": "3.14.0", + "upath": "2.0.1" + }, + "peerDependencies": { + "@datadog/datadog-ci-plugin-aas": "5.11.0", + "@datadog/datadog-ci-plugin-cloud-run": "5.11.0", + "@datadog/datadog-ci-plugin-container-app": "5.11.0", + "@datadog/datadog-ci-plugin-coverage": "5.11.0", + "@datadog/datadog-ci-plugin-deployment": "5.11.0", + "@datadog/datadog-ci-plugin-dora": "5.11.0", + "@datadog/datadog-ci-plugin-gate": "5.11.0", + "@datadog/datadog-ci-plugin-junit": "5.11.0", + "@datadog/datadog-ci-plugin-lambda": "5.11.0", + "@datadog/datadog-ci-plugin-sarif": "5.11.0", + "@datadog/datadog-ci-plugin-sbom": "5.11.0", + "@datadog/datadog-ci-plugin-stepfunctions": "5.11.0", + "@datadog/datadog-ci-plugin-synthetics": "5.11.0", + "@datadog/datadog-ci-plugin-terraform": "5.11.0" + }, + "peerDependenciesMeta": { + "@datadog/datadog-ci-plugin-aas": { + "optional": true + }, + "@datadog/datadog-ci-plugin-cloud-run": { + "optional": true + }, + "@datadog/datadog-ci-plugin-container-app": { + "optional": true + }, + "@datadog/datadog-ci-plugin-coverage": { + "optional": true + }, + "@datadog/datadog-ci-plugin-deployment": { + "optional": true + }, + "@datadog/datadog-ci-plugin-dora": { + "optional": true + }, + "@datadog/datadog-ci-plugin-gate": { + "optional": true + }, + "@datadog/datadog-ci-plugin-junit": { + "optional": true + }, + "@datadog/datadog-ci-plugin-lambda": { + "optional": true + }, + "@datadog/datadog-ci-plugin-sarif": { + "optional": true + }, + "@datadog/datadog-ci-plugin-sbom": { + "optional": true + }, + "@datadog/datadog-ci-plugin-stepfunctions": { + "optional": true + }, + "@datadog/datadog-ci-plugin-synthetics": { + "optional": true + }, + "@datadog/datadog-ci-plugin-terraform": { + "optional": true + } + } + }, + "node_modules/@datadog/datadog-ci-base/node_modules/debug": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz", + "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/@datadog/datadog-ci-plugin-coverage": { + "version": "5.11.0", + "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-coverage/-/datadog-ci-plugin-coverage-5.11.0.tgz", + "integrity": "sha512-aKzkjbd/NVt+7Sd4QL0qVqUrnyMG0UF63lvNxve7suWQDoIWowAgCPa0ta+jpMYUYCGtjt74H3wL+q0W0G+D+Q==", + "license": "Apache-2.0", + "dependencies": { + "chalk": "3.0.0", + "fast-xml-parser": "5.3.7", + "form-data": "4.0.4", + "upath": "2.0.1" + }, + "peerDependencies": { + "@datadog/datadog-ci-base": "5.11.0" + } + }, + "node_modules/@datadog/datadog-ci-plugin-deployment": { + "version": "5.11.0", + "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-deployment/-/datadog-ci-plugin-deployment-5.11.0.tgz", + "integrity": "sha512-FF9L7alDz1XN5L3nYR7xMeZd99U6kxkT+InSyZY21ZHpcuuEngseW1aMgB0iFn3lxZ+1H4z196uCckViTEkjog==", + "license": "Apache-2.0", + "dependencies": { + "axios": "1.13.5", + "chalk": "3.0.0", + "simple-git": "3.33.0" + }, + "peerDependencies": { + "@datadog/datadog-ci-base": "5.11.0" + } + }, + "node_modules/@datadog/datadog-ci-plugin-dora": { + "version": "5.11.0", + "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-dora/-/datadog-ci-plugin-dora-5.11.0.tgz", + "integrity": "sha512-DWSyT2tW23XOYCvTEHvD/F9FRIffTN+2wZGYojaD1LM9wJew4xievziBwyD46quCF0OdsbGMLvAyR2hyrrYeZA==", + "license": "Apache-2.0", + "dependencies": { + "chalk": "3.0.0", + "simple-git": "3.33.0" + }, + "peerDependencies": { + "@datadog/datadog-ci-base": "5.11.0" + } + }, + "node_modules/@datadog/datadog-ci-plugin-gate": { + "version": "5.11.0", + "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-gate/-/datadog-ci-plugin-gate-5.11.0.tgz", + "integrity": "sha512-WUNr6y292eH+sumAyD5rFhqlU/aJ8vN74h4EJj0SqDpnrDCzMnSDgRyb9GXhv8ZO4GOW6LTEcPmMZhyfYL3b5g==", + "license": "Apache-2.0", + "dependencies": { + "@types/uuid": "9.0.8", + "chalk": "3.0.0", + "uuid": "9.0.1" + }, + "peerDependencies": { + "@datadog/datadog-ci-base": "5.11.0" + } + }, + "node_modules/@datadog/datadog-ci-plugin-junit": { + "version": "5.11.0", + "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-junit/-/datadog-ci-plugin-junit-5.11.0.tgz", + "integrity": "sha512-I8H09jNR/LDYqiy7TOKv9Gu4+t/o0J6QH6cooKrNboULSDYijVo6w3knANsd2m3Hlcbi1QkTmNpPXIdeGR5Dig==", + "license": "Apache-2.0", + "dependencies": { + "chalk": "3.0.0", + "fast-xml-parser": "5.3.7", + "form-data": "4.0.4", + "upath": "2.0.1", + "uuid": "9.0.1" + }, + "peerDependencies": { + "@datadog/datadog-ci-base": "5.11.0" + } + }, + "node_modules/@datadog/datadog-ci-plugin-sarif": { + "version": "5.11.0", + "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-sarif/-/datadog-ci-plugin-sarif-5.11.0.tgz", + "integrity": "sha512-JZuqNzFsW9BZeStLM60f5zdEpG67kh7xqgbeJhqIcloEpDJIEmsJHH487PM0stakcqhhIfzUzvEW744LLQg5Ug==", + "license": "Apache-2.0", + "dependencies": { + "ajv": "8.18.0", + "ajv-formats": "3.0.1", + "chalk": "3.0.0", + "form-data": "4.0.4", + "upath": "2.0.1", + "uuid": "9.0.1" + }, + "peerDependencies": { + "@datadog/datadog-ci-base": "5.11.0" + } + }, + "node_modules/@datadog/datadog-ci-plugin-sbom": { + "version": "5.11.0", + "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-sbom/-/datadog-ci-plugin-sbom-5.11.0.tgz", + "integrity": "sha512-lHU65a1Hru7SiwhZtZ9OTd6vyPcRG2U/61imVpvLvhEMSwY2xSLTtZfxWbMi4VJxq81V2zhdifxP5BMJO1FnHw==", + "license": "Apache-2.0", + "dependencies": { + "ajv": "8.18.0", + "ajv-formats": "3.0.1", + "axios": "1.13.5", + "chalk": "3.0.0", + "packageurl-js": "2.0.1" + }, + "peerDependencies": { + "@datadog/datadog-ci-base": "5.11.0" + } + }, + "node_modules/@inquirer/external-editor": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@inquirer/external-editor/-/external-editor-1.0.3.tgz", + "integrity": "sha512-RWbSrDiYmO4LbejWY7ttpxczuwQyZLBUyygsA9Nsv95hpzUWwnNTVQmAq3xuh7vNwCp07UTmE5i11XAEExx4RA==", + "license": "MIT", + "dependencies": { + "chardet": "^2.1.1", + "iconv-lite": "^0.7.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@jest/diff-sequences": { + "version": "30.0.1", + "resolved": "https://registry.npmjs.org/@jest/diff-sequences/-/diff-sequences-30.0.1.tgz", + "integrity": "sha512-n5H8QLDJ47QqbCNn5SuFjCRDrOLEZ0h8vAHCK5RL9Ls7Xa8AQLa/YxAc9UjFqoEDM48muwtBGjtMY5cr0PLDCw==", + "license": "MIT", + "engines": { + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + } + }, + "node_modules/@jest/get-type": { + "version": "30.1.0", + "resolved": "https://registry.npmjs.org/@jest/get-type/-/get-type-30.1.0.tgz", + "integrity": "sha512-eMbZE2hUnx1WV0pmURZY9XoXPkUYjpc55mb0CrhtdWLtzMQPFvu/rZkTLZFTsdaVQa+Tr4eWAteqcUzoawq/uA==", + "license": "MIT", + "engines": { + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + } + }, + "node_modules/@jest/schemas": { + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.5.tgz", + "integrity": "sha512-DmdYgtezMkh3cpU8/1uyXakv3tJRcmcXxBOcO0tbaozPwpmh4YMsnWrQm9ZmZMfa5ocbxzbFk6O4bDPEc/iAnA==", + "license": "MIT", + "dependencies": { + "@sinclair/typebox": "^0.34.0" + }, + "engines": { + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + } + }, + "node_modules/@kwsites/file-exists": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@kwsites/file-exists/-/file-exists-1.1.1.tgz", + "integrity": "sha512-m9/5YGR18lIwxSFDwfE3oA7bWuq9kdau6ugN4H2rJeyhFQZcG9AgSHkQtSD15a8WvTgfz9aikZMrKPHvbpqFiw==", + "license": "MIT", + "dependencies": { + "debug": "^4.1.1" + } + }, + "node_modules/@kwsites/promise-deferred": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@kwsites/promise-deferred/-/promise-deferred-1.1.1.tgz", + "integrity": "sha512-GaHYm+c0O9MjZRu0ongGBRbinu8gVAMd2UZjji6jVmqKtZluZnptXGWhz1E8j8D2HJ3f/yMxKAUC0b+57wncIw==", + "license": "MIT" + }, + "node_modules/@nodelib/fs.scandir": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", + "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", + "license": "MIT", + "dependencies": { + "@nodelib/fs.stat": "2.0.5", + "run-parallel": "^1.1.9" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.stat": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", + "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.walk": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", + "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", + "license": "MIT", + "dependencies": { + "@nodelib/fs.scandir": "2.1.5", + "fastq": "^1.6.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@sinclair/typebox": { + "version": "0.34.49", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.34.49.tgz", + "integrity": "sha512-brySQQs7Jtn0joV8Xh9ZV/hZb9Ozb0pmazDIASBkYKCjXrXU3mpcFahmK/z4YDhGkQvP9mWJbVyahdtU5wQA+A==", + "license": "MIT" + }, + "node_modules/@sindresorhus/merge-streams": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/merge-streams/-/merge-streams-4.0.0.tgz", + "integrity": "sha512-tlqY9xq5ukxTUZBmoOp+m61cqwQD5pHJtFY3Mn8CA8ps6yghLH/Hw8UPdqg4OLmFW3IFlcXnQNmo/dh8HzXYIQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@tootallnate/quickjs-emscripten": { + "version": "0.23.0", + "resolved": "https://registry.npmjs.org/@tootallnate/quickjs-emscripten/-/quickjs-emscripten-0.23.0.tgz", + "integrity": "sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA==", + "license": "MIT" + }, + "node_modules/@types/datadog-metrics": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/@types/datadog-metrics/-/datadog-metrics-0.6.1.tgz", + "integrity": "sha512-p6zVpfmNcXwtcXjgpz7do/fKyfndGhU5sGJVtb5Gn5PvLDiQUAgD0mI/itf/99sBi9DRxeyhFQ9dQF6OxxQNbA==", + "license": "MIT" + }, + "node_modules/@types/debug": { + "version": "4.1.13", + "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.13.tgz", + "integrity": "sha512-KSVgmQmzMwPlmtljOomayoR89W4FynCAi3E8PPs7vmDVPe84hT+vGPKkJfThkmXs0x0jAaa9U8uW8bbfyS2fWw==", + "license": "MIT", + "dependencies": { + "@types/ms": "*" + } + }, + "node_modules/@types/katex": { + "version": "0.16.8", + "resolved": "https://registry.npmjs.org/@types/katex/-/katex-0.16.8.tgz", + "integrity": "sha512-trgaNyfU+Xh2Tc+ABIb44a5AYUpicB3uwirOioeOkNPPbmgRNtcWyDeeFRzjPZENO9Vq8gvVqfhaaXWLlevVwg==", + "license": "MIT" + }, + "node_modules/@types/ms": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@types/ms/-/ms-2.1.0.tgz", + "integrity": "sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==", + "license": "MIT" + }, + "node_modules/@types/unist": { + "version": "2.0.11", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-2.0.11.tgz", + "integrity": "sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==", + "license": "MIT" + }, + "node_modules/@types/uuid": { + "version": "9.0.8", + "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-9.0.8.tgz", + "integrity": "sha512-jg+97EGIcY9AGHJJRaaPVgetKDsrTgbRjQ5Msgjh/DQKEFl0DtyRr/VCOyD1T2R1MNeWPK/u7JoGhlDZnKBAfA==", + "license": "MIT" + }, + "node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/ajv": { + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz", + "integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/ansi-escapes": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz", + "integrity": "sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==", + "license": "MIT", + "dependencies": { + "type-fest": "^0.21.3" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "license": "Python-2.0" + }, + "node_modules/ast-types": { + "version": "0.13.4", + "resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.13.4.tgz", + "integrity": "sha512-x1FCFnFifvYDDzTaLII71vG5uvDwgtmDTEVWAxrgeiR8VjMONcCXJx7E+USjDtHlwFmt9MysbqgF9b9Vjr6w+w==", + "license": "MIT", + "dependencies": { + "tslib": "^2.0.1" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/async-retry": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/async-retry/-/async-retry-1.3.1.tgz", + "integrity": "sha512-aiieFW/7h3hY0Bq5d+ktDBejxuwR78vRu9hDUdR8rNhSaQ29VzPL4AoIRG7D/c7tdenwOcKvgPM6tIxB3cB6HA==", + "license": "MIT", + "dependencies": { + "retry": "0.12.0" + } + }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "license": "MIT" + }, + "node_modules/axios": { + "version": "1.13.5", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.13.5.tgz", + "integrity": "sha512-cz4ur7Vb0xS4/KUN0tPWe44eqxrIu31me+fbang3ijiNscE129POzipJJA6zniq2C/Z6sJCjMimjS8Lc/GAs8Q==", + "license": "MIT", + "dependencies": { + "follow-redirects": "^1.15.11", + "form-data": "^4.0.5", + "proxy-from-env": "^1.1.0" + } + }, + "node_modules/axios/node_modules/form-data": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz", + "integrity": "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==", + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "hasown": "^2.0.2", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/balanced-match": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", + "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", + "license": "MIT", + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/basic-ftp": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.2.0.tgz", + "integrity": "sha512-VoMINM2rqJwJgfdHq6RiUudKt2BV+FY5ZFezP/ypmwayk68+NzzAQy4XXLlqsGD4MCzq3DrmNFD/uUmBJuGoXw==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/bignumber.js": { + "version": "9.3.1", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.3.1.tgz", + "integrity": "sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ==", + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "license": "MIT", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, + "node_modules/bl/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/brace-expansion": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.5.tgz", + "integrity": "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==", + "license": "MIT", + "dependencies": { + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/braces": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", + "license": "MIT", + "dependencies": { + "fill-range": "^7.1.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/chalk": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-3.0.0.tgz", + "integrity": "sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/character-entities": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-2.0.2.tgz", + "integrity": "sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-entities-legacy": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/character-entities-legacy/-/character-entities-legacy-3.0.0.tgz", + "integrity": "sha512-RpPp0asT/6ufRm//AJVwpViZbGM/MkjQFxJccQRHmISF/22NBtsHqAWmL+/pmkPWoIUJdWyeVleTl1wydHATVQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-reference-invalid": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/character-reference-invalid/-/character-reference-invalid-2.0.1.tgz", + "integrity": "sha512-iBZ4F4wRbyORVsu0jPV7gXkOsGYjGHPmAyv+HiHG8gi5PtC9KI2j1+v8/tlibRvjoWX027ypmG/n0HtO5t7unw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/chardet": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/chardet/-/chardet-2.1.1.tgz", + "integrity": "sha512-PsezH1rqdV9VvyNhxxOW32/d75r01NY7TQCmOqomRo15ZSOKbpTFVsfjghxo6JloQUCGnH4k1LGu0R4yCLlWQQ==", + "license": "MIT" + }, + "node_modules/cli-cursor": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-3.1.0.tgz", + "integrity": "sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw==", + "license": "MIT", + "dependencies": { + "restore-cursor": "^3.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cli-spinners": { + "version": "2.9.2", + "resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-2.9.2.tgz", + "integrity": "sha512-ywqV+5MmyL4E7ybXgKys4DugZbX0FC6LnwrhjuykIjnK9k8OQacQ7axGKnjDXWNhns0xot3bZI5h55H8yo9cJg==", + "license": "MIT", + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/cli-width": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-3.0.0.tgz", + "integrity": "sha512-FxqpkPPwu1HjuN93Omfm4h8uIanXofW0RxVEW3k5RKx+mJJYSthzNhp32Kzxxy3YAEZ/Dc/EWN1vZRY0+kOhbw==", + "license": "ISC", + "engines": { + "node": ">= 10" + } + }, + "node_modules/clipanion": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/clipanion/-/clipanion-3.2.1.tgz", + "integrity": "sha512-dYFdjLb7y1ajfxQopN05mylEpK9ZX0sO1/RfMXdfmwjlIsPkbh4p7A682x++zFPLDCo1x3p82dtljHf5cW2LKA==", + "license": "MIT", + "workspaces": [ + "website" + ], + "dependencies": { + "typanion": "^3.8.0" + }, + "peerDependencies": { + "typanion": "*" + } + }, + "node_modules/clone": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz", + "integrity": "sha512-JQHZ2QMW6l3aH/j6xCqQThY/9OH4D/9ls34cgkUBiEeocRTU04tHfKPBsUK1PqZCUQM7GiA0IIXJSuXHI64Kbg==", + "license": "MIT", + "engines": { + "node": ">=0.8" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "license": "MIT" + }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "license": "MIT", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/commander": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-8.3.0.tgz", + "integrity": "sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, + "node_modules/core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", + "license": "MIT" + }, + "node_modules/data-uri-to-buffer": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-6.0.2.tgz", + "integrity": "sha512-7hvf7/GW8e86rW0ptuwS3OcBGDjIi6SZva7hCyWC0yYry2cOPmLIjXAUHI6DK2HsnwJd9ifmt57i8eV2n4YNpw==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/datadog-metrics": { + "version": "0.9.3", + "resolved": "https://registry.npmjs.org/datadog-metrics/-/datadog-metrics-0.9.3.tgz", + "integrity": "sha512-BVsBX2t+4yA3tHs7DnB5H01cHVNiGJ/bHA8y6JppJDyXG7s2DLm6JaozPGpgsgVGd42Is1CHRG/yMDQpt877Xg==", + "license": "MIT", + "dependencies": { + "debug": "3.1.0", + "dogapi": "2.8.4" + } + }, + "node_modules/datadog-metrics/node_modules/debug": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", + "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/datadog-metrics/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/decode-named-character-reference": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/decode-named-character-reference/-/decode-named-character-reference-1.3.0.tgz", + "integrity": "sha512-GtpQYB283KrPp6nRw50q3U9/VfOutZOe103qlN7BPP6Ad27xYnOIWv4lPzo8HCAL+mMZofJ9KEy30fq6MfaK6Q==", + "license": "MIT", + "dependencies": { + "character-entities": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "license": "MIT", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/defaults": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/defaults/-/defaults-1.0.4.tgz", + "integrity": "sha512-eFuaLoy/Rxalv2kr+lqMlUnrDWV+3j4pljOIJgLIhI058IQfWJ7vXhyEIHu+HtC738klGALYxOKDO0bQP3tg8A==", + "license": "MIT", + "dependencies": { + "clone": "^1.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/degenerator": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/degenerator/-/degenerator-5.0.1.tgz", + "integrity": "sha512-TllpMR/t0M5sqCXfj85i4XaAzxmS5tVA16dqvdkMwGmzI+dXLXnw3J+3Vdv7VKw+ThlTMboK6i9rnZ6Nntj5CQ==", + "license": "MIT", + "dependencies": { + "ast-types": "^0.13.4", + "escodegen": "^2.1.0", + "esprima": "^4.0.1" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/devlop": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/devlop/-/devlop-1.1.0.tgz", + "integrity": "sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==", + "license": "MIT", + "dependencies": { + "dequal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/dogapi": { + "version": "2.8.4", + "resolved": "https://registry.npmjs.org/dogapi/-/dogapi-2.8.4.tgz", + "integrity": "sha512-065fsvu5dB0o4+ENtLjZILvXMClDNH/yA9H6L8nsdcNiz9l0Hzpn7aQaCOPYXxqyzq4CRPOdwkFXUjDOXfRGbg==", + "license": "MIT", + "dependencies": { + "extend": "^3.0.2", + "json-bigint": "^1.0.0", + "lodash": "^4.17.21", + "minimist": "^1.2.5", + "rc": "^1.2.8" + }, + "bin": { + "dogapi": "bin/dogapi" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/entities": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-set-tostringtag": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", + "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "license": "MIT", + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/escodegen": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.1.0.tgz", + "integrity": "sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==", + "license": "BSD-2-Clause", + "dependencies": { + "esprima": "^4.0.1", + "estraverse": "^5.2.0", + "esutils": "^2.0.2" + }, + "bin": { + "escodegen": "bin/escodegen.js", + "esgenerate": "bin/esgenerate.js" + }, + "engines": { + "node": ">=6.0" + }, + "optionalDependencies": { + "source-map": "~0.6.1" + } + }, + "node_modules/esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "license": "BSD-2-Clause", + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", + "license": "MIT" + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "license": "MIT" + }, + "node_modules/fast-glob": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz", + "integrity": "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==", + "license": "MIT", + "dependencies": { + "@nodelib/fs.stat": "^2.0.2", + "@nodelib/fs.walk": "^1.2.3", + "glob-parent": "^5.1.2", + "merge2": "^1.3.0", + "micromatch": "^4.0.8" + }, + "engines": { + "node": ">=8.6.0" + } + }, + "node_modules/fast-uri": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.0.tgz", + "integrity": "sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/fast-xml-parser": { + "version": "5.3.7", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.3.7.tgz", + "integrity": "sha512-JzVLro9NQv92pOM/jTCR6mHlJh2FGwtomH8ZQjhFj/R29P2Fnj38OgPJVtcvYw6SuKClhgYuwUZf5b3rd8u2mA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "strnum": "^2.1.2" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, + "node_modules/fastq": { + "version": "1.20.1", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.1.tgz", + "integrity": "sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw==", + "license": "ISC", + "dependencies": { + "reusify": "^1.0.4" + } + }, + "node_modules/figures": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz", + "integrity": "sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg==", + "license": "MIT", + "dependencies": { + "escape-string-regexp": "^1.0.5" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/fill-range": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", + "license": "MIT", + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/follow-redirects": { + "version": "1.15.11", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.11.tgz", + "integrity": "sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "license": "MIT", + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/form-data": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.4.tgz", + "integrity": "sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow==", + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "hasown": "^2.0.2", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-east-asian-width": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/get-east-asian-width/-/get-east-asian-width-1.5.0.tgz", + "integrity": "sha512-CQ+bEO+Tva/qlmw24dCejulK5pMzVnUOFOijVogd3KQs07HnRIgp8TGipvCCRT06xeYEbpbgwaCxglFyiuIcmA==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/get-uri": { + "version": "6.0.5", + "resolved": "https://registry.npmjs.org/get-uri/-/get-uri-6.0.5.tgz", + "integrity": "sha512-b1O07XYq8eRuVzBNgJLstU6FYc1tS6wnMtF1I1D9lE8LxZSOGZ7LhxN54yPP6mGw5f2CkXY2BQUL9Fx41qvcIg==", + "license": "MIT", + "dependencies": { + "basic-ftp": "^5.0.2", + "data-uri-to-buffer": "^6.0.2", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/glob": { + "version": "13.0.5", + "resolved": "https://registry.npmjs.org/glob/-/glob-13.0.5.tgz", + "integrity": "sha512-BzXxZg24Ibra1pbQ/zE7Kys4Ua1ks7Bn6pKLkVPZ9FZe4JQS6/Q7ef3LG1H+k7lUf5l4T3PLSyYyYJVYUvfgTw==", + "license": "BlueOak-1.0.0", + "dependencies": { + "minimatch": "^10.2.1", + "minipass": "^7.1.2", + "path-scurry": "^2.0.0" + }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "license": "ISC", + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/globby": { + "version": "16.1.1", + "resolved": "https://registry.npmjs.org/globby/-/globby-16.1.1.tgz", + "integrity": "sha512-dW7vl+yiAJSp6aCekaVnVJxurRv7DCOLyXqEG3RYMYUg7AuJ2jCqPkZTA8ooqC2vtnkaMcV5WfFBMuEnTu1OQg==", + "license": "MIT", + "dependencies": { + "@sindresorhus/merge-streams": "^4.0.0", + "fast-glob": "^3.3.3", + "ignore": "^7.0.5", + "is-path-inside": "^4.0.0", + "slash": "^5.1.0", + "unicorn-magic": "^0.4.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-tostringtag": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "license": "MIT", + "dependencies": { + "has-symbols": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/iconv-lite": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.2.tgz", + "integrity": "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/ignore": { + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.5.tgz", + "integrity": "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/immediate": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", + "integrity": "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==", + "license": "MIT" + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", + "license": "ISC" + }, + "node_modules/inquirer": { + "version": "8.2.7", + "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-8.2.7.tgz", + "integrity": "sha512-UjOaSel/iddGZJ5xP/Eixh6dY1XghiBw4XK13rCCIJcJfyhhoul/7KhLLUGtebEj6GDYM6Vnx/mVsjx2L/mFIA==", + "license": "MIT", + "dependencies": { + "@inquirer/external-editor": "^1.0.0", + "ansi-escapes": "^4.2.1", + "chalk": "^4.1.1", + "cli-cursor": "^3.1.0", + "cli-width": "^3.0.0", + "figures": "^3.0.0", + "lodash": "^4.17.21", + "mute-stream": "0.0.8", + "ora": "^5.4.1", + "run-async": "^2.4.0", + "rxjs": "^7.5.5", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0", + "through": "^2.3.6", + "wrap-ansi": "^6.0.1" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/inquirer/node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/ip-address": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.1.0.tgz", + "integrity": "sha512-XXADHxXmvT9+CRxhXg56LJovE+bmWnEWB78LB83VZTprKTmaC5QfruXocxzTZ2Kl0DNwKuBdlIhjL8LeY8Sf8Q==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, + "node_modules/is-alphabetical": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-alphabetical/-/is-alphabetical-2.0.1.tgz", + "integrity": "sha512-FWyyY60MeTNyeSRpkM2Iry0G9hpr7/9kD40mD/cGQEuilcZYS4okz8SN2Q6rLCJ8gbCt6fN+rC+6tMGS99LaxQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/is-alphanumerical": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-alphanumerical/-/is-alphanumerical-2.0.1.tgz", + "integrity": "sha512-hmbYhX/9MUMF5uh7tOXyK/n0ZvWpad5caBA17GsC6vyuCqaWliRG5K1qS9inmUhEMaOBIW7/whAnSwveW/LtZw==", + "license": "MIT", + "dependencies": { + "is-alphabetical": "^2.0.0", + "is-decimal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/is-decimal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-decimal/-/is-decimal-2.0.1.tgz", + "integrity": "sha512-AAB9hiomQs5DXWcRB1rqsxGUstbRroFOPPVAomNk/3XHR5JyEZChOyTWe2oayKnsSsr/kcGqF+z6yuH6HHpN0A==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/is-docker": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-4.0.0.tgz", + "integrity": "sha512-LHE+wROyG/Y/0ZnbktRCoTix2c1RhgWaZraMZ8o1Q7zCh0VSrICJQO5oqIIISrcSBtrXv0o233w1IYwsWCjTzA==", + "license": "MIT", + "bin": { + "is-docker": "cli.js" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "license": "MIT", + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-hexadecimal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-hexadecimal/-/is-hexadecimal-2.0.1.tgz", + "integrity": "sha512-DgZQp241c8oO6cA1SbTEWiXeoxV42vlcJxgH+B3hi1AiqqKruZR3ZGF8In3fj4+/y/7rHvlOZLZtgJ/4ttYGZg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/is-interactive": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-interactive/-/is-interactive-1.0.0.tgz", + "integrity": "sha512-2HvIEKRoqS62guEC+qBjpvRubdX910WCMuJTZ+I9yvqKU2/12eSL549HMwtabb4oupdj2sMP50k+XJfB/8JE6w==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "license": "MIT", + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/is-path-inside": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-4.0.0.tgz", + "integrity": "sha512-lJJV/5dYS+RcL8uQdBDW9c9uWFLLBNRyFhnAKXw5tVqLlKZ4RMGZKv+YQ/IA3OhD+RpbJa1LLFM1FQPGyIXvOA==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-unicode-supported": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-0.1.0.tgz", + "integrity": "sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "license": "MIT" + }, + "node_modules/jest-diff": { + "version": "30.2.0", + "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-30.2.0.tgz", + "integrity": "sha512-dQHFo3Pt4/NLlG5z4PxZ/3yZTZ1C7s9hveiOj+GCN+uT109NC2QgsoVZsVOAvbJ3RgKkvyLGXZV9+piDpWbm6A==", + "license": "MIT", + "dependencies": { + "@jest/diff-sequences": "30.0.1", + "@jest/get-type": "30.1.0", + "chalk": "^4.1.2", + "pretty-format": "30.2.0" + }, + "engines": { + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + } + }, + "node_modules/jest-diff/node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/js-yaml": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", + "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/json-bigint": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", + "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", + "license": "MIT", + "dependencies": { + "bignumber.js": "^9.0.0" + } + }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "license": "MIT" + }, + "node_modules/jsonc-parser": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-3.3.1.tgz", + "integrity": "sha512-HUgH65KyejrUFPvHFPbqOY0rsFip3Bo5wb4ngvdi1EpCYWUQDC5V+Y7mZws+DLkr4M//zQJoanu1SP+87Dv1oQ==", + "license": "MIT" + }, + "node_modules/jsonpointer": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/jsonpointer/-/jsonpointer-5.0.1.tgz", + "integrity": "sha512-p/nXbhSEcu3pZRdkW1OfJhpsVtW1gd4Wa1fnQc9YLiTfAjn0312eMKimbdIQzuZl9aa9xUGaRlP9T/CJE/ditQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/jszip": { + "version": "3.10.1", + "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.10.1.tgz", + "integrity": "sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==", + "license": "(MIT OR GPL-3.0-or-later)", + "dependencies": { + "lie": "~3.3.0", + "pako": "~1.0.2", + "readable-stream": "~2.3.6", + "setimmediate": "^1.0.5" + } + }, + "node_modules/katex": { + "version": "0.16.44", + "resolved": "https://registry.npmjs.org/katex/-/katex-0.16.44.tgz", + "integrity": "sha512-EkxoDTk8ufHqHlf9QxGwcxeLkWRR3iOuYfRpfORgYfqc8s13bgb+YtRY59NK5ZpRaCwq1kqA6a5lpX8C/eLphQ==", + "funding": [ + "https://opencollective.com/katex", + "https://github.com/sponsors/katex" + ], + "license": "MIT", + "dependencies": { + "commander": "^8.3.0" + }, + "bin": { + "katex": "cli.js" + } + }, + "node_modules/lie": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz", + "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==", + "license": "MIT", + "dependencies": { + "immediate": "~3.0.5" + } + }, + "node_modules/linkify-it": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-5.0.0.tgz", + "integrity": "sha512-5aHCbzQRADcdP+ATqnDuhhJ/MRIqDkZX5pyjFHRRysS8vZ5AbqGEoFIb6pYHPZ+L/OC2Lc+xT8uHVVR5CAK/wQ==", + "license": "MIT", + "dependencies": { + "uc.micro": "^2.0.0" + } + }, + "node_modules/lodash": { + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz", + "integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==", + "license": "MIT" + }, + "node_modules/log-symbols": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.1.0.tgz", + "integrity": "sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg==", + "license": "MIT", + "dependencies": { + "chalk": "^4.1.0", + "is-unicode-supported": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/log-symbols/node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/lru-cache": { + "version": "11.2.7", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.2.7.tgz", + "integrity": "sha512-aY/R+aEsRelme17KGQa/1ZSIpLpNYYrhcrepKTZgE+W3WM16YMCaPwOHLHsmopZHELU0Ojin1lPVxKR0MihncA==", + "license": "BlueOak-1.0.0", + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/markdown-it": { + "version": "14.1.1", + "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-14.1.1.tgz", + "integrity": "sha512-BuU2qnTti9YKgK5N+IeMubp14ZUKUUw7yeJbkjtosvHiP0AZ5c8IAgEMk79D0eC8F23r4Ac/q8cAIFdm2FtyoA==", + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1", + "entities": "^4.4.0", + "linkify-it": "^5.0.0", + "mdurl": "^2.0.0", + "punycode.js": "^2.3.1", + "uc.micro": "^2.1.0" + }, + "bin": { + "markdown-it": "bin/markdown-it.mjs" + } + }, + "node_modules/markdownlint": { + "version": "0.40.0", + "resolved": "https://registry.npmjs.org/markdownlint/-/markdownlint-0.40.0.tgz", + "integrity": "sha512-UKybllYNheWac61Ia7T6fzuQNDZimFIpCg2w6hHjgV1Qu0w1TV0LlSgryUGzM0bkKQCBhy2FDhEELB73Kb0kAg==", + "license": "MIT", + "dependencies": { + "micromark": "4.0.2", + "micromark-core-commonmark": "2.0.3", + "micromark-extension-directive": "4.0.0", + "micromark-extension-gfm-autolink-literal": "2.1.0", + "micromark-extension-gfm-footnote": "2.1.0", + "micromark-extension-gfm-table": "2.1.1", + "micromark-extension-math": "3.1.0", + "micromark-util-types": "2.0.2", + "string-width": "8.1.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/DavidAnson" + } + }, + "node_modules/markdownlint-cli2": { + "version": "0.22.0", + "resolved": "https://registry.npmjs.org/markdownlint-cli2/-/markdownlint-cli2-0.22.0.tgz", + "integrity": "sha512-mOC9BY/XGtdX3M9n3AgERd79F0+S7w18yBBTNIQ453sI87etZfp1z4eajqSMV70CYjbxKe5ktKvT2HCpvcWx9w==", + "license": "MIT", + "dependencies": { + "globby": "16.1.1", + "js-yaml": "4.1.1", + "jsonc-parser": "3.3.1", + "jsonpointer": "5.0.1", + "markdown-it": "14.1.1", + "markdownlint": "0.40.0", + "markdownlint-cli2-formatter-default": "0.0.6", + "micromatch": "4.0.8", + "smol-toml": "1.6.0" + }, + "bin": { + "markdownlint-cli2": "markdownlint-cli2-bin.mjs" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/DavidAnson" + } + }, + "node_modules/markdownlint-cli2-formatter-default": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/markdownlint-cli2-formatter-default/-/markdownlint-cli2-formatter-default-0.0.6.tgz", + "integrity": "sha512-VVDGKsq9sgzu378swJ0fcHfSicUnMxnL8gnLm/Q4J/xsNJ4e5bA6lvAz7PCzIl0/No0lHyaWdqVD2jotxOSFMQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/DavidAnson" + }, + "peerDependencies": { + "markdownlint-cli2": ">=0.0.4" + } + }, + "node_modules/markdownlint/node_modules/ansi-regex": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", + "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/markdownlint/node_modules/string-width": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-8.1.0.tgz", + "integrity": "sha512-Kxl3KJGb/gxkaUMOjRsQ8IrXiGW75O4E3RPjFIINOVH8AMl2SQ/yWdTzWwF3FevIX9LcMAjJW+GRwAlAbTSXdg==", + "license": "MIT", + "dependencies": { + "get-east-asian-width": "^1.3.0", + "strip-ansi": "^7.1.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/markdownlint/node_modules/strip-ansi": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz", + "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^6.2.2" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/mdurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-2.0.0.tgz", + "integrity": "sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w==", + "license": "MIT" + }, + "node_modules/merge2": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", + "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/micromark": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/micromark/-/micromark-4.0.2.tgz", + "integrity": "sha512-zpe98Q6kvavpCr1NPVSCMebCKfD7CA2NqZ+rykeNhONIJBpc1tFKt9hucLGwha3jNTNI8lHpctWJWoimVF4PfA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "@types/debug": "^4.0.0", + "debug": "^4.0.0", + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "micromark-core-commonmark": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-combine-extensions": "^2.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-encode": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "micromark-util-subtokenize": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-core-commonmark": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/micromark-core-commonmark/-/micromark-core-commonmark-2.0.3.tgz", + "integrity": "sha512-RDBrHEMSxVFLg6xvnXmb1Ayr2WzLAWjeSATAoxwKYJV94TeNavgoIdA0a9ytzDSVzBy2YKFK+emCPOEibLeCrg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "micromark-factory-destination": "^2.0.0", + "micromark-factory-label": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-factory-title": "^2.0.0", + "micromark-factory-whitespace": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-classify-character": "^2.0.0", + "micromark-util-html-tag-name": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-subtokenize": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-extension-directive": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/micromark-extension-directive/-/micromark-extension-directive-4.0.0.tgz", + "integrity": "sha512-/C2nqVmXXmiseSSuCdItCMho7ybwwop6RrrRPk0KbOHW21JKoCldC+8rFOaundDoRBUWBnJJcxeA/Kvi34WQXg==", + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-factory-whitespace": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0", + "parse-entities": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-autolink-literal": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-autolink-literal/-/micromark-extension-gfm-autolink-literal-2.1.0.tgz", + "integrity": "sha512-oOg7knzhicgQ3t4QCjCWgTmfNhvQbDDnJeVu9v81r7NltNCVmhPy1fJRX27pISafdjL+SVc4d3l48Gb6pbRypw==", + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-footnote": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-footnote/-/micromark-extension-gfm-footnote-2.1.0.tgz", + "integrity": "sha512-/yPhxI1ntnDNsiHtzLKYnE3vf9JZ6cAisqVDauhp4CEHxlb4uoOTxOCJ+9s51bIB8U1N1FJ1RXOKTIlD5B/gqw==", + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-core-commonmark": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-table": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-table/-/micromark-extension-gfm-table-2.1.1.tgz", + "integrity": "sha512-t2OU/dXXioARrC6yWfJ4hqB7rct14e8f7m0cbI5hUmDyyIlwv5vEtooptH8INkbLzOatzKuVbQmAYcbWoyz6Dg==", + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-math": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-math/-/micromark-extension-math-3.1.0.tgz", + "integrity": "sha512-lvEqd+fHjATVs+2v/8kg9i5Q0AP2k85H0WUOwpIVvUML8BapsMvh1XAogmQjOCsLpoKRCVQqEkQBB3NhVBcsOg==", + "license": "MIT", + "dependencies": { + "@types/katex": "^0.16.0", + "devlop": "^1.0.0", + "katex": "^0.16.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-factory-destination": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-destination/-/micromark-factory-destination-2.0.1.tgz", + "integrity": "sha512-Xe6rDdJlkmbFRExpTOmRj9N3MaWmbAgdpSrBQvCFqhezUn4AHqJHbaEnfbVYYiexVSs//tqOdY/DxhjdCiJnIA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-label": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-label/-/micromark-factory-label-2.0.1.tgz", + "integrity": "sha512-VFMekyQExqIW7xIChcXn4ok29YE3rnuyveW3wZQWWqF4Nv9Wk5rgJ99KzPvHjkmPXF93FXIbBp6YdW3t71/7Vg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-space": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-space/-/micromark-factory-space-2.0.1.tgz", + "integrity": "sha512-zRkxjtBxxLd2Sc0d+fbnEunsTj46SWXgXciZmHq0kDYGnck/ZSGj9/wULTV95uoeYiK5hRXP2mJ98Uo4cq/LQg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-title": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-title/-/micromark-factory-title-2.0.1.tgz", + "integrity": "sha512-5bZ+3CjhAd9eChYTHsjy6TGxpOFSKgKKJPJxr293jTbfry2KDoWkhBb6TcPVB4NmzaPhMs1Frm9AZH7OD4Cjzw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-whitespace": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-whitespace/-/micromark-factory-whitespace-2.0.1.tgz", + "integrity": "sha512-Ob0nuZ3PKt/n0hORHyvoD9uZhr+Za8sFoP+OnMcnWK5lngSzALgQYKMr9RJVOWLqQYuyn6ulqGWSXdwf6F80lQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-character": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/micromark-util-character/-/micromark-util-character-2.1.1.tgz", + "integrity": "sha512-wv8tdUTJ3thSFFFJKtpYKOYiGP2+v96Hvk4Tu8KpCAsTMs6yi+nVmGh1syvSCsaxz45J6Jbw+9DD6g97+NV67Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-chunked": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-chunked/-/micromark-util-chunked-2.0.1.tgz", + "integrity": "sha512-QUNFEOPELfmvv+4xiNg2sRYeS/P84pTW0TCgP5zc9FpXetHY0ab7SxKyAQCNCc1eK0459uoLI1y5oO5Vc1dbhA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-classify-character": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-classify-character/-/micromark-util-classify-character-2.0.1.tgz", + "integrity": "sha512-K0kHzM6afW/MbeWYWLjoHQv1sgg2Q9EccHEDzSkxiP/EaagNzCm7T/WMKZ3rjMbvIpvBiZgwR3dKMygtA4mG1Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-combine-extensions": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-combine-extensions/-/micromark-util-combine-extensions-2.0.1.tgz", + "integrity": "sha512-OnAnH8Ujmy59JcyZw8JSbK9cGpdVY44NKgSM7E9Eh7DiLS2E9RNQf0dONaGDzEG9yjEl5hcqeIsj4hfRkLH/Bg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-chunked": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-decode-numeric-character-reference": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/micromark-util-decode-numeric-character-reference/-/micromark-util-decode-numeric-character-reference-2.0.2.tgz", + "integrity": "sha512-ccUbYk6CwVdkmCQMyr64dXz42EfHGkPQlBj5p7YVGzq8I7CtjXZJrubAYezf7Rp+bjPseiROqe7G6foFd+lEuw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-encode": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-encode/-/micromark-util-encode-2.0.1.tgz", + "integrity": "sha512-c3cVx2y4KqUnwopcO9b/SCdo2O67LwJJ/UyqGfbigahfegL9myoEFoDYZgkT7f36T0bLrM9hZTAaAyH+PCAXjw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-html-tag-name": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-html-tag-name/-/micromark-util-html-tag-name-2.0.1.tgz", + "integrity": "sha512-2cNEiYDhCWKI+Gs9T0Tiysk136SnR13hhO8yW6BGNyhOC4qYFnwF1nKfD3HFAIXA5c45RrIG1ub11GiXeYd1xA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-normalize-identifier": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-normalize-identifier/-/micromark-util-normalize-identifier-2.0.1.tgz", + "integrity": "sha512-sxPqmo70LyARJs0w2UclACPUUEqltCkJ6PhKdMIDuJ3gSf/Q+/GIe3WKl0Ijb/GyH9lOpUkRAO2wp0GVkLvS9Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-resolve-all": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-resolve-all/-/micromark-util-resolve-all-2.0.1.tgz", + "integrity": "sha512-VdQyxFWFT2/FGJgwQnJYbe1jjQoNTS4RjglmSjTUlpUMa95Htx9NHeYW4rGDJzbjvCsl9eLjMQwGeElsqmzcHg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-sanitize-uri": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-sanitize-uri/-/micromark-util-sanitize-uri-2.0.1.tgz", + "integrity": "sha512-9N9IomZ/YuGGZZmQec1MbgxtlgougxTodVwDzzEouPKo3qFWvymFHWcnDi2vzV1ff6kas9ucW+o3yzJK9YB1AQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-encode": "^2.0.0", + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-subtokenize": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-util-subtokenize/-/micromark-util-subtokenize-2.1.0.tgz", + "integrity": "sha512-XQLu552iSctvnEcgXw6+Sx75GflAPNED1qx7eBJ+wydBb2KCbRZe+NwvIEEMM83uml1+2WSXpBAcp9IUCgCYWA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-symbol": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-2.0.1.tgz", + "integrity": "sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-types": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/micromark-util-types/-/micromark-util-types-2.0.2.tgz", + "integrity": "sha512-Yw0ECSpJoViF1qTU4DC6NwtC4aWGt1EkzaQB8KPPyCRR8z9TWeV0HbEFGTO+ZY1wB22zmxnJqhPyTpOVCpeHTA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromatch": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", + "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", + "license": "MIT", + "dependencies": { + "braces": "^3.0.3", + "picomatch": "^2.3.1" + }, + "engines": { + "node": ">=8.6" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mimic-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", + "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/minimatch": { + "version": "10.2.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", + "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", + "license": "BlueOak-1.0.0", + "dependencies": { + "brace-expansion": "^5.0.5" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/minipass": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/mute-stream": { + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", + "integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==", + "license": "ISC" + }, + "node_modules/netmask": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/netmask/-/netmask-2.0.2.tgz", + "integrity": "sha512-dBpDMdxv9Irdq66304OLfEmQ9tbNRFnFTuZiLo+bD+r332bBmMJ8GBLXklIXXgxd3+v9+KUnZaUR5PJMa75Gsg==", + "license": "MIT", + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/onetime": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", + "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", + "license": "MIT", + "dependencies": { + "mimic-fn": "^2.1.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ora": { + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/ora/-/ora-5.4.1.tgz", + "integrity": "sha512-5b6Y85tPxZZ7QytO+BQzysW31HJku27cRIlkbAXaNx+BdcVi+LlRFmVXzeF6a7JCwJpyw5c4b+YSVImQIrBpuQ==", + "license": "MIT", + "dependencies": { + "bl": "^4.1.0", + "chalk": "^4.1.0", + "cli-cursor": "^3.1.0", + "cli-spinners": "^2.5.0", + "is-interactive": "^1.0.0", + "is-unicode-supported": "^0.1.0", + "log-symbols": "^4.1.0", + "strip-ansi": "^6.0.0", + "wcwidth": "^1.0.1" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ora/node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/pac-proxy-agent": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-7.2.0.tgz", + "integrity": "sha512-TEB8ESquiLMc0lV8vcd5Ql/JAKAoyzHFXaStwjkzpOpC5Yv+pIzLfHvjTSdf3vpa2bMiUQrg9i6276yn8666aA==", + "license": "MIT", + "dependencies": { + "@tootallnate/quickjs-emscripten": "^0.23.0", + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "get-uri": "^6.0.1", + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.6", + "pac-resolver": "^7.0.1", + "socks-proxy-agent": "^8.0.5" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/pac-resolver": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/pac-resolver/-/pac-resolver-7.0.1.tgz", + "integrity": "sha512-5NPgf87AT2STgwa2ntRMr45jTKrYBGkVU36yT0ig/n/GMAa3oPqhZfIQ2kMEimReg0+t9kZViDVZ83qfVUlckg==", + "license": "MIT", + "dependencies": { + "degenerator": "^5.0.0", + "netmask": "^2.0.2" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/package-manager-detector": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/package-manager-detector/-/package-manager-detector-1.6.0.tgz", + "integrity": "sha512-61A5ThoTiDG/C8s8UMZwSorAGwMJ0ERVGj2OjoW5pAalsNOg15+iQiPzrLJ4jhZ1HJzmC2PIHT2oEiH3R5fzNA==", + "license": "MIT" + }, + "node_modules/packageurl-js": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/packageurl-js/-/packageurl-js-2.0.1.tgz", + "integrity": "sha512-N5ixXjzTy4QDQH0Q9YFjqIWd6zH6936Djpl2m9QNFmDv5Fum8q8BjkpAcHNMzOFE0IwQrFhJWex3AN6kS0OSwg==", + "license": "MIT" + }, + "node_modules/pako": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", + "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==", + "license": "(MIT AND Zlib)" + }, + "node_modules/parse-entities": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/parse-entities/-/parse-entities-4.0.2.tgz", + "integrity": "sha512-GG2AQYWoLgL877gQIKeRPGO1xF9+eG1ujIb5soS5gPvLQ1y2o8FL90w2QWNdf9I361Mpp7726c+lj3U0qK1uGw==", + "license": "MIT", + "dependencies": { + "@types/unist": "^2.0.0", + "character-entities-legacy": "^3.0.0", + "character-reference-invalid": "^2.0.0", + "decode-named-character-reference": "^1.0.0", + "is-alphanumerical": "^2.0.0", + "is-decimal": "^2.0.0", + "is-hexadecimal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/path-scurry": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.2.tgz", + "integrity": "sha512-3O/iVVsJAPsOnpwWIeD+d6z/7PmqApyQePUtCndjatj/9I5LylHvt5qluFaBT3I5h3r1ejfR056c+FCv+NnNXg==", + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^11.0.0", + "minipass": "^7.1.2" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/picomatch": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", + "license": "MIT", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/prettier": { + "version": "3.8.1", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.8.1.tgz", + "integrity": "sha512-UOnG6LftzbdaHZcKoPFtOcCKztrQ57WkHDeRD9t/PTQtmT0NHSeWWepj6pS0z/N7+08BHFDQVUrfmfMRcZwbMg==", + "license": "MIT", + "bin": { + "prettier": "bin/prettier.cjs" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/prettier/prettier?sponsor=1" + } + }, + "node_modules/pretty-format": { + "version": "30.2.0", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-30.2.0.tgz", + "integrity": "sha512-9uBdv/B4EefsuAL+pWqueZyZS2Ba+LxfFeQ9DN14HU4bN8bhaxKdkpjpB6fs9+pSjIBu+FXQHImEg8j/Lw0+vA==", + "license": "MIT", + "dependencies": { + "@jest/schemas": "30.0.5", + "ansi-styles": "^5.2.0", + "react-is": "^18.3.1" + }, + "engines": { + "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" + } + }, + "node_modules/pretty-format/node_modules/ansi-styles": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", + "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", + "license": "MIT" + }, + "node_modules/proxy-agent": { + "version": "6.4.0", + "resolved": "https://registry.npmjs.org/proxy-agent/-/proxy-agent-6.4.0.tgz", + "integrity": "sha512-u0piLU+nCOHMgGjRbimiXmA9kM/L9EHh3zL81xCdp7m+Y2pHIsnmbdDoEDoAz5geaonNR6q6+yOPQs6n4T6sBQ==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.0.2", + "debug": "^4.3.4", + "http-proxy-agent": "^7.0.1", + "https-proxy-agent": "^7.0.3", + "lru-cache": "^7.14.1", + "pac-proxy-agent": "^7.0.1", + "proxy-from-env": "^1.1.0", + "socks-proxy-agent": "^8.0.2" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/proxy-agent/node_modules/lru-cache": { + "version": "7.18.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz", + "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", + "license": "MIT" + }, + "node_modules/punycode.js": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode.js/-/punycode.js-2.3.1.tgz", + "integrity": "sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/queue-microtask": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", + "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "license": "(BSD-2-Clause OR MIT OR Apache-2.0)", + "dependencies": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "bin": { + "rc": "cli.js" + } + }, + "node_modules/react-is": { + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", + "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", + "license": "MIT" + }, + "node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/restore-cursor": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-3.1.0.tgz", + "integrity": "sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA==", + "license": "MIT", + "dependencies": { + "onetime": "^5.1.0", + "signal-exit": "^3.0.2" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/reusify": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz", + "integrity": "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==", + "license": "MIT", + "engines": { + "iojs": ">=1.0.0", + "node": ">=0.10.0" + } + }, + "node_modules/run-async": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/run-async/-/run-async-2.4.1.tgz", + "integrity": "sha512-tvVnVv01b8c1RrA6Ep7JkStj85Guv/YrMcwqYQnwjsAS2cTmmPGBBjAjpCW7RrSodNSoE2/qg9O4bceNvUuDgQ==", + "license": "MIT", + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/run-parallel": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", + "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "queue-microtask": "^1.2.2" + } + }, + "node_modules/rxjs": { + "version": "7.8.2", + "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.2.tgz", + "integrity": "sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.1.0" + } + }, + "node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/semver": { + "version": "7.6.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", + "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/setimmediate": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", + "integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==", + "license": "MIT" + }, + "node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "license": "ISC" + }, + "node_modules/simple-git": { + "version": "3.33.0", + "resolved": "https://registry.npmjs.org/simple-git/-/simple-git-3.33.0.tgz", + "integrity": "sha512-D4V/tGC2sjsoNhoMybKyGoE+v8A60hRawKQ1iFRA1zwuDgGZCBJ4ByOzZ5J8joBbi4Oam0qiPH+GhzmSBwbJng==", + "license": "MIT", + "dependencies": { + "@kwsites/file-exists": "^1.1.1", + "@kwsites/promise-deferred": "^1.1.1", + "debug": "^4.4.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/steveukx/git-js?sponsor=1" + } + }, + "node_modules/slash": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-5.1.0.tgz", + "integrity": "sha512-ZA6oR3T/pEyuqwMgAKT0/hAv8oAXckzbkmR0UkUosQ+Mc4RxGoJkRmwHgHufaenlyAgE1Mxgpdcrf75y6XcnDg==", + "license": "MIT", + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/smart-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", + "license": "MIT", + "engines": { + "node": ">= 6.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/smol-toml": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.6.0.tgz", + "integrity": "sha512-4zemZi0HvTnYwLfrpk/CF9LOd9Lt87kAt50GnqhMpyF9U3poDAP2+iukq2bZsO/ufegbYehBkqINbsWxj4l4cw==", + "license": "BSD-3-Clause", + "engines": { + "node": ">= 18" + }, + "funding": { + "url": "https://github.com/sponsors/cyyynthia" + } + }, + "node_modules/socks": { + "version": "2.8.7", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.7.tgz", + "integrity": "sha512-HLpt+uLy/pxB+bum/9DzAgiKS8CX1EvbWxI4zlmgGCExImLdiad2iCwXT5Z4c9c3Eq8rP2318mPW2c+QbtjK8A==", + "license": "MIT", + "dependencies": { + "ip-address": "^10.0.1", + "smart-buffer": "^4.2.0" + }, + "engines": { + "node": ">= 10.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks-proxy-agent": { + "version": "8.0.5", + "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.5.tgz", + "integrity": "sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "socks": "^2.8.3" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "license": "BSD-3-Clause", + "optional": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/strnum": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.2.2.tgz", + "integrity": "sha512-DnR90I+jtXNSTXWdwrEy9FakW7UX+qUZg28gj5fk2vxxl7uS/3bpI4fjFYVmdK9etptYBPNkpahuQnEwhwECqA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT" + }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/supports-hyperlinks": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/supports-hyperlinks/-/supports-hyperlinks-2.3.0.tgz", + "integrity": "sha512-RpsAZlpWcDwOPQA22aCH4J0t7L8JmAvsCxfOSEwm7cQs3LshN36QaTkwd70DnBOXDWGssw2eUoc8CaRWT0XunA==", + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0", + "supports-color": "^7.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/terminal-link": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/terminal-link/-/terminal-link-2.1.1.tgz", + "integrity": "sha512-un0FmiRUQNr5PJqy9kP7c40F5BOfpGlYTrxonDChEZB7pzZxRNp/bt+ymiy9/npwXya9KH99nJ/GXFIiUkYGFQ==", + "license": "MIT", + "dependencies": { + "ansi-escapes": "^4.2.1", + "supports-hyperlinks": "^2.0.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/through": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==", + "license": "MIT" + }, + "node_modules/tiny-async-pool": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/tiny-async-pool/-/tiny-async-pool-2.1.0.tgz", + "integrity": "sha512-ltAHPh/9k0STRQqaoUX52NH4ZQYAJz24ZAEwf1Zm+HYg3l9OXTWeqWKyYsHu40wF/F0rxd2N2bk5sLvX2qlSvg==", + "license": "MIT" + }, + "node_modules/tinyexec": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.0.4.tgz", + "integrity": "sha512-u9r3uZC0bdpGOXtlxUIdwf9pkmvhqJdrVCH9fapQtgy/OeTTMZ1nqH7agtvEfmGui6e1XxjcdrlxvxJvc3sMqw==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "license": "MIT", + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, + "node_modules/typanion": { + "version": "3.14.0", + "resolved": "https://registry.npmjs.org/typanion/-/typanion-3.14.0.tgz", + "integrity": "sha512-ZW/lVMRabETuYCd9O9ZvMhAh8GslSqaUjxmK/JLPCh6l73CvLBiuXswj/+7LdnWOgYsQ130FqLzFz5aGT4I3Ug==", + "license": "MIT", + "workspaces": [ + "website" + ] + }, + "node_modules/type-fest": { + "version": "0.21.3", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz", + "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==", + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/uc.micro": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-2.1.0.tgz", + "integrity": "sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==", + "license": "MIT" + }, + "node_modules/unicorn-magic": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/unicorn-magic/-/unicorn-magic-0.4.0.tgz", + "integrity": "sha512-wH590V9VNgYH9g3lH9wWjTrUoKsjLF6sGLjhR4sH1LWpLmCOH0Zf7PukhDA8BiS7KHe4oPNkcTHqYkj7SOGUOw==", + "license": "MIT", + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/upath": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/upath/-/upath-2.0.1.tgz", + "integrity": "sha512-1uEe95xksV1O0CYKXo8vQvN1JEbtJp7lb7C5U9HMsIp6IVwntkH/oNUzyVNQSd4S1sYk2FpSSW44FqMc8qee5w==", + "license": "MIT", + "engines": { + "node": ">=4", + "yarn": "*" + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "license": "MIT" + }, + "node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/wcwidth": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/wcwidth/-/wcwidth-1.0.1.tgz", + "integrity": "sha512-XHPEwS0q6TaxcvG85+8EYkbiCux2XtWG2mkc47Ng2A77BQu9+DqIOJldST4HgPkuea7dvKSj5VgX3P1d4rW8Tg==", + "license": "MIT", + "dependencies": { + "defaults": "^1.0.3" + } + }, + "node_modules/wrap-ansi": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", + "integrity": "sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=8" + } + } + } +} diff --git a/scripts/environment/npm-tools/package.json b/scripts/environment/npm-tools/package.json new file mode 100644 index 0000000000000..30d63057b7fed --- /dev/null +++ b/scripts/environment/npm-tools/package.json @@ -0,0 +1,9 @@ +{ + "private": true, + "description": "Pinned npm tools for CI. Run 'npm ci' to install with exact versions from package-lock.json.", + "dependencies": { + "@datadog/datadog-ci": "5.11.0", + "markdownlint-cli2": "0.22.0", + "prettier": "3.8.1" + } +} diff --git a/scripts/environment/prepare.sh b/scripts/environment/prepare.sh index e74aba915ad5f..02e100999c8de 100755 --- a/scripts/environment/prepare.sh +++ b/scripts/environment/prepare.sh @@ -37,9 +37,8 @@ CARGO_HACK_VERSION="0.6.43" DD_RUST_LICENSE_TOOL_VERSION="1.0.6" CARGO_LLVM_COV_VERSION="0.8.4" WASM_PACK_VERSION="0.13.1" -MARKDOWNLINT_CLI2_VERSION="0.22.0" -PRETTIER_VERSION="3.8.1" -DATADOG_CI_VERSION="5.9.0" +# npm tool versions are defined in scripts/environment/npm-tools/package.json +# and pinned (including transitive deps) in npm-tools/package-lock.json. VDEV_VERSION="0.3.0" ALL_MODULES=( @@ -151,22 +150,58 @@ maybe_install_cargo_tool() { fi } -# Helper for NPM packages -# Usage: maybe_install_npm_package -maybe_install_npm_package() { - local tool="$1" - local package="$2" - local version="$3" - local version_pattern="${4:-$version}" - local version_cmd="${5:---version}" # Default to --version, can override with "version" etc. +# Install npm tools from the committed package-lock.json so that every +# transitive dependency version is pinned (no live registry resolution). +# Versions are defined in npm-tools/package.json; npm ci ensures exact lockfile match. +# Note: npm ci installs all packages in the lockfile even if only one tool +# is requested, since it does not support selective installation. +maybe_install_npm_tools() { + local npm_tools=(markdownlint-cli2 prettier datadog-ci) + + # Early return when no npm tool is requested, so hosts without npm + # (e.g. tests/e2e/Dockerfile calling prepare.sh --modules=cargo-nextest) + # are not broken by the npm commands below. + local any_requested=false + for tool in "${npm_tools[@]}"; do + if contains_module "$tool"; then + any_requested=true + break + fi + done + if [[ "$any_requested" == "false" ]]; then + return 0 + fi - if ! contains_module "$tool"; then + local npm_tools_dir="${SCRIPT_DIR}/npm-tools" + local npm_bin_dir + npm_bin_dir="$(npm config get prefix -g)/bin" + local need_install=false + + for tool in "${npm_tools[@]}"; do + if contains_module "$tool"; then + local expected="${npm_tools_dir}/node_modules/.bin/${tool}" + if [[ "$(readlink "${npm_bin_dir}/${tool}" 2>/dev/null)" != "$expected" ]] || [[ ! -x "$expected" ]]; then + need_install=true + break + fi + fi + done + + if [[ "$need_install" == "false" ]]; then return 0 fi - if [[ "$("$tool" "$version_cmd" 2>/dev/null)" != "$version_pattern" ]]; then - sudo npm install -g "${package}@${version}" + npm ci --prefix "${npm_tools_dir}" + + # Use sudo only when the target directory is not writable (e.g. /usr/local/bin + # on Linux CI runners is root-owned, but Homebrew dirs on macOS are user-owned). + local ln_cmd=(ln -sf) + if [[ ! -w "${npm_bin_dir}" ]]; then + ln_cmd=(sudo ln -sf) fi + for tool in "${npm_tools[@]}"; do + "${ln_cmd[@]}" "${npm_tools_dir}/node_modules/.bin/${tool}" "${npm_bin_dir}/${tool}" + done } # Always ensure git safe.directory is set @@ -221,6 +256,4 @@ maybe_install_cargo_tool dd-rust-license-tool "${DD_RUST_LICENSE_TOOL_VERSION}" maybe_install_cargo_tool wasm-pack "${WASM_PACK_VERSION}" maybe_install_cargo_tool vdev "${VDEV_VERSION}" -maybe_install_npm_package markdownlint-cli2 markdownlint-cli2 "${MARKDOWNLINT_CLI2_VERSION}" -maybe_install_npm_package prettier prettier "${PRETTIER_VERSION}" -maybe_install_npm_package datadog-ci "@datadog/datadog-ci" "${DATADOG_CI_VERSION}" "v${DATADOG_CI_VERSION}" "version" +maybe_install_npm_tools From 04f3b782e95f044fc7307fe4f6230dc48793800b Mon Sep 17 00:00:00 2001 From: Bruce Guenter Date: Fri, 3 Apr 2026 10:23:45 -0600 Subject: [PATCH 096/326] fix(buffers): Reverse performance regression in buffer metrics (#24995) * fix(buffers): Reverse performance regression in buffer metrics PR #23561 reworked the gauge metric handling for buffer metric reporting in order to get rid of a global shared hashmap. However, in doing so, the current occupancy calculation was moved into the hot path. This change moves that calculation out of the hot path and into the reporting loop, which executes on a fixed interval instead of per event array. * Address PR feedback * Refactor BufferUsage::install and add tests * chore: remove changelog fragment from PR 24995 --------- Co-authored-by: Pavlos Rontidis --- lib/vector-buffers/src/buffer_usage_data.rs | 329 +++++++++++++------- 1 file changed, 215 insertions(+), 114 deletions(-) diff --git a/lib/vector-buffers/src/buffer_usage_data.rs b/lib/vector-buffers/src/buffer_usage_data.rs index ce7a67648db3d..b418023657bb5 100644 --- a/lib/vector-buffers/src/buffer_usage_data.rs +++ b/lib/vector-buffers/src/buffer_usage_data.rs @@ -20,6 +20,7 @@ use crate::{ const ORDERING: Ordering = Ordering::Relaxed; /// Snapshot of category metrics. +#[derive(Clone, Copy, Debug, Default)] struct CategorySnapshot { event_count: u64, event_byte_size: u64, @@ -86,37 +87,52 @@ impl CategoryMetrics { } } -/// `CurrentMetrics` is a wrapper around a pair of `CategoryMetrics` that are used to track a -/// "current" value that may increment or decrement. The challenge this solves is that the -/// increments and decrements may race and result in underflows that are hard to handle using only -/// efficient atomic operations. By tracking the increments and decrements separately, we can ensure -/// that the current value is always accurate even if the increments and decrements race. -#[derive(Debug, Default)] -struct CurrentMetrics { - increments: CategoryMetrics, - decrements: CategoryMetrics, +/// Running totals of events that have entered and left a buffer stage. +/// +/// Each reporting tick consumes the latest deltas from the atomic counters and +/// folds them into these cumulative totals. The difference +/// (`total_entered - total_left`) gives the approximate current buffer +/// occupancy without requiring a separate "current size" counter that would +/// itself be subject to cross-thread races. +#[derive(Clone, Copy, Debug, Default)] +struct ReporterCurrentMetrics { + total_entered: CategorySnapshot, + total_left: CategorySnapshot, } -impl CurrentMetrics { - fn increment(&self, event_count: u64, event_byte_size: u64) { - self.increments.increment(event_count, event_byte_size); +impl ReporterCurrentMetrics { + fn add_received(&mut self, snapshot: CategorySnapshot) { + self.total_entered.event_count = self + .total_entered + .event_count + .saturating_add(snapshot.event_count); + self.total_entered.event_byte_size = self + .total_entered + .event_byte_size + .saturating_add(snapshot.event_byte_size); } - fn decrement(&self, event_count: u64, event_byte_size: u64) { - self.decrements.increment(event_count, event_byte_size); + fn add_left(&mut self, snapshot: CategorySnapshot) { + self.total_left.event_count = self + .total_left + .event_count + .saturating_add(snapshot.event_count); + self.total_left.event_byte_size = self + .total_left + .event_byte_size + .saturating_add(snapshot.event_byte_size); } - fn get(&self) -> CategorySnapshot { - let entered_total = self.increments.get(); - let left_total = self.decrements.get(); - + fn current(&self) -> CategorySnapshot { CategorySnapshot { - event_count: entered_total + event_count: self + .total_entered .event_count - .saturating_sub(left_total.event_count), - event_byte_size: entered_total + .saturating_sub(self.total_left.event_count), + event_byte_size: self + .total_entered .event_byte_size - .saturating_sub(left_total.event_byte_size), + .saturating_sub(self.total_left.event_byte_size), } } } @@ -161,7 +177,6 @@ impl BufferUsageHandle { pub fn increment_received_event_count_and_byte_size(&self, count: u64, byte_size: u64) { if count > 0 || byte_size > 0 { self.state.received.increment(count, byte_size); - self.state.current.increment(count, byte_size); } } @@ -171,7 +186,6 @@ impl BufferUsageHandle { pub fn increment_sent_event_count_and_byte_size(&self, count: u64, byte_size: u64) { if count > 0 || byte_size > 0 { self.state.sent.increment(count, byte_size); - self.state.current.decrement(count, byte_size); } } @@ -188,7 +202,6 @@ impl BufferUsageHandle { } else { self.state.dropped.increment(count, byte_size); } - self.state.current.decrement(count, byte_size); } } } @@ -201,7 +214,6 @@ struct BufferUsageData { dropped: CategoryMetrics, dropped_intentional: CategoryMetrics, max_size: CategoryMetrics, - current: CurrentMetrics, } impl BufferUsageData { @@ -235,6 +247,85 @@ impl BufferUsageData { .expect("should never be bigger than `usize`"), } } + + fn report(&self, current_metrics: &mut ReporterCurrentMetrics, buffer_id: &str) { + let max_size = self.max_size.get(); + emit(BufferCreated { + buffer_id: buffer_id.to_string(), + idx: self.idx, + max_size_bytes: max_size.event_byte_size, + max_size_events: max_size + .event_count + .try_into() + .expect("should never be bigger than `usize`"), + }); + + // Consume received before sent/dropped so that, in the presence of + // a race between producers and consumers, the computed current usage + // will err on the side of overcounting, which is more likely to be an + // accurate representation of the current usage than undercounting. + let received = self.received.consume(); + current_metrics.add_received(received); + + let sent = self.sent.consume(); + current_metrics.add_left(sent); + + let dropped = self.dropped.consume(); + current_metrics.add_left(dropped); + + let dropped_intentional = self.dropped_intentional.consume(); + current_metrics.add_left(dropped_intentional); + + let current = current_metrics.current(); + + if received.has_updates() { + emit(BufferEventsReceived { + buffer_id: buffer_id.to_string(), + idx: self.idx, + count: received.event_count, + byte_size: received.event_byte_size, + total_count: current.event_count, + total_byte_size: current.event_byte_size, + }); + } + + if sent.has_updates() { + emit(BufferEventsSent { + buffer_id: buffer_id.to_string(), + idx: self.idx, + count: sent.event_count, + byte_size: sent.event_byte_size, + total_count: current.event_count, + total_byte_size: current.event_byte_size, + }); + } + + if dropped.has_updates() { + emit(BufferEventsDropped { + buffer_id: buffer_id.to_string(), + idx: self.idx, + intentional: false, + reason: "corrupted_events", + count: dropped.event_count, + byte_size: dropped.event_byte_size, + total_count: current.event_count, + total_byte_size: current.event_byte_size, + }); + } + + if dropped_intentional.has_updates() { + emit(BufferEventsDropped { + buffer_id: buffer_id.to_string(), + idx: self.idx, + intentional: true, + reason: "drop_newest", + count: dropped_intentional.event_count, + byte_size: dropped_intentional.event_byte_size, + total_count: current.event_count, + total_byte_size: current.event_byte_size, + }); + } + } } /// Snapshot of buffer usage metrics. @@ -296,83 +387,29 @@ impl BufferUsage { pub fn install(self, buffer_id: &str) { let buffer_id = buffer_id.to_string(); let span = self.span; - let stages = self.stages; + let stages: Vec<_> = self + .stages + .into_iter() + .map(|stage| (stage, ReporterCurrentMetrics::default())) + .collect(); let task_name = format!("buffer usage reporter ({buffer_id})"); - let task = async move { - let mut interval = interval(Duration::from_secs(2)); - loop { - interval.tick().await; - - for stage in &stages { - let max_size = stage.max_size.get(); - emit(BufferCreated { - buffer_id: buffer_id.clone(), - idx: stage.idx, - max_size_bytes: max_size.event_byte_size, - max_size_events: max_size - .event_count - .try_into() - .expect("should never be bigger than `usize`"), - }); - - let current = stage.current.get(); - let received = stage.received.consume(); - if received.has_updates() { - emit(BufferEventsReceived { - buffer_id: buffer_id.clone(), - idx: stage.idx, - count: received.event_count, - byte_size: received.event_byte_size, - total_count: current.event_count, - total_byte_size: current.event_byte_size, - }); - } - - let sent = stage.sent.consume(); - if sent.has_updates() { - emit(BufferEventsSent { - buffer_id: buffer_id.clone(), - idx: stage.idx, - count: sent.event_count, - byte_size: sent.event_byte_size, - total_count: current.event_count, - total_byte_size: current.event_byte_size, - }); - } - - let dropped = stage.dropped.consume(); - if dropped.has_updates() { - emit(BufferEventsDropped { - buffer_id: buffer_id.clone(), - idx: stage.idx, - intentional: false, - reason: "corrupted_events", - count: dropped.event_count, - byte_size: dropped.event_byte_size, - total_count: current.event_count, - total_byte_size: current.event_byte_size, - }); - } - - let dropped_intentional = stage.dropped_intentional.consume(); - if dropped_intentional.has_updates() { - emit(BufferEventsDropped { - buffer_id: buffer_id.clone(), - idx: stage.idx, - intentional: true, - reason: "drop_newest", - count: dropped_intentional.event_count, - byte_size: dropped_intentional.event_byte_size, - total_count: current.event_count, - total_byte_size: current.event_byte_size, - }); - } - } - } - }; + let task = Self::report_buffer_usage(stages, buffer_id).instrument(span.or_current()); + spawn_named(task, task_name.as_str()); + } - spawn_named(task.instrument(span.or_current()), task_name.as_str()); + async fn report_buffer_usage( + mut stages: Vec<(Arc, ReporterCurrentMetrics)>, + buffer_id: String, + ) { + let mut interval = interval(Duration::from_secs(2)); + loop { + interval.tick().await; + + for (stage, current_metrics) in &mut stages { + stage.report(current_metrics, &buffer_id); + } + } } } @@ -381,27 +418,91 @@ mod tests { use super::*; #[test] - fn current_usage_is_derived_from_entered_and_left_totals() { - let handle = BufferUsageHandle { - state: Arc::new(BufferUsageData::new(0)), - }; - - handle.increment_received_event_count_and_byte_size(10, 1000); - handle.increment_sent_event_count_and_byte_size(3, 300); - handle.increment_dropped_event_count_and_byte_size(2, 200, false); + fn reporter_current_usage_is_derived_from_entered_and_left_totals() { + let mut current = ReporterCurrentMetrics::default(); + current.add_received(CategorySnapshot { + event_count: 10, + event_byte_size: 1000, + }); + current.add_left(CategorySnapshot { + event_count: 3, + event_byte_size: 300, + }); + current.add_left(CategorySnapshot { + event_count: 2, + event_byte_size: 200, + }); + + let current = current.current(); + assert_eq!(current.event_count, 5); + assert_eq!(current.event_byte_size, 500); + } - let current = handle.state.current.get(); + #[test] + fn reporter_current_usage_preserves_underflow_debt() { + let mut current = ReporterCurrentMetrics::default(); + current.add_left(CategorySnapshot { + event_count: 10, + event_byte_size: 1000, + }); + current.add_received(CategorySnapshot { + event_count: 15, + event_byte_size: 1500, + }); + + let current = current.current(); assert_eq!(current.event_count, 5); assert_eq!(current.event_byte_size, 500); } #[test] - fn current_usage_saturates_at_zero() { + fn consume_resets_deltas_between_ticks() { let data = BufferUsageData::new(0); - data.current.decrement(10, 1000); + let mut metrics = ReporterCurrentMetrics::default(); + + data.received.increment(10, 1000); + data.sent.increment(3, 300); + data.report(&mut metrics, "test"); + let current = metrics.current(); + assert_eq!(current.event_count, 7); + assert_eq!(current.event_byte_size, 700); + + // Second tick with no new activity should report the same totals. + data.report(&mut metrics, "test"); + let current = metrics.current(); + assert_eq!(current.event_count, 7); + assert_eq!(current.event_byte_size, 700); + } + + #[test] + fn accumulates_across_multiple_ticks() { + let data = BufferUsageData::new(0); + let mut metrics = ReporterCurrentMetrics::default(); + + data.received.increment(10, 1000); + data.report(&mut metrics, "test"); + + data.received.increment(5, 500); + data.sent.increment(8, 800); + data.report(&mut metrics, "test"); + let current = metrics.current(); + assert_eq!(current.event_count, 7); + assert_eq!(current.event_byte_size, 700); + } + + #[test] + fn drops_count_as_leaving_the_buffer() { + let data = BufferUsageData::new(0); + let mut metrics = ReporterCurrentMetrics::default(); + + data.received.increment(20, 2000); + data.sent.increment(5, 500); + data.dropped.increment(3, 300); + data.dropped_intentional.increment(2, 200); - let current = data.current.get(); - assert_eq!(current.event_count, 0); - assert_eq!(current.event_byte_size, 0); + data.report(&mut metrics, "test"); + let current = metrics.current(); + assert_eq!(current.event_count, 10); + assert_eq!(current.event_byte_size, 1000); } } From 51030d2aade15deae23297a1ed1994cb36c530e9 Mon Sep 17 00:00:00 2001 From: Thomas Date: Fri, 3 Apr 2026 12:36:36 -0400 Subject: [PATCH 097/326] fix(dev): make file source tests not flaky (#24957) Make file source tests not flaky --- src/sources/file.rs | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/src/sources/file.rs b/src/sources/file.rs index 62a95db362f10..df51a5a73788a 100644 --- a/src/sources/file.rs +++ b/src/sources/file.rs @@ -2462,12 +2462,23 @@ mod tests { inner: impl Future, ) -> Vec { assert_source_compliance(&FILE_SOURCE_TAGS, async move { - let (tx, rx) = if acking_mode == Acks { - let (tx, rx) = SourceSender::new_test_finalize(EventStatus::Delivered); - (tx, rx.boxed()) - } else { - let (tx, rx) = SourceSender::new_test(); - (tx, rx.boxed()) + let (tx, rx) = match acking_mode { + Acks => { + let (tx, rx) = SourceSender::new_test_finalize(EventStatus::Delivered); + (tx, rx.boxed()) + } + Unfinalized => { + // Use Rejected so that events are finalized but checkpoints + // are NOT updated (only Delivered triggers checkpoint updates). + // This avoids a race where the default Delivered status on drop + // could leak checkpoint writes into the next run. + let (tx, rx) = SourceSender::new_test_finalize(EventStatus::Rejected); + (tx, rx.boxed()) + } + NoAcks => { + let (tx, rx) = SourceSender::new_test(); + (tx, rx.boxed()) + } }; let (trigger_shutdown, shutdown, shutdown_done) = ShutdownSignal::new_wired(); From 2fe515ccd2dfc23d98b84300e4fd87bee27b35fa Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Fri, 3 Apr 2026 14:13:12 -0400 Subject: [PATCH 098/326] fix(ci): use helm-charts develop branch for K8s E2E tests (#25118) Instead of pulling the published chart from helm.vector.dev, clone the helm-charts repo (develop branch) and use the local chart path. This breaks the circular dependency between Vector and helm-charts releases and catches chart incompatibilities before release. - Add `helm_chart_repo()` helper with HELM_CHART_REPO env var override - Update deploy-chart-test.sh to support local chart paths - Clone helm-charts in CI and set HELM_CHART_REPO Fixes: #25111 Co-authored-by: Claude Opus 4.6 (1M context) --- .github/workflows/k8s_e2e.yml | 8 ++++++ lib/k8s-e2e-tests/src/lib.rs | 6 ++++ lib/k8s-e2e-tests/tests/vector-agent.rs | 28 +++++++++---------- lib/k8s-e2e-tests/tests/vector-aggregator.rs | 4 +-- .../tests/vector-dd-agent-aggregator.rs | 2 +- lib/k8s-e2e-tests/tests/vector.rs | 8 +++--- scripts/deploy-chart-test.sh | 12 ++++++-- 7 files changed, 44 insertions(+), 24 deletions(-) diff --git a/.github/workflows/k8s_e2e.yml b/.github/workflows/k8s_e2e.yml index afdcc01252e20..471783ed1488c 100644 --- a/.github/workflows/k8s_e2e.yml +++ b/.github/workflows/k8s_e2e.yml @@ -197,6 +197,13 @@ jobs: MINIKUBE_VERSION: ${{ matrix.minikube_version }} CONTAINER_RUNTIME: ${{ matrix.container_runtime }} + - name: Checkout helm-charts + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + repository: vectordotdev/helm-charts + ref: develop + path: helm-charts + # TODO: This job has been quite flakey. Need to investigate further and then remove the retries. - name: Run tests uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 @@ -204,6 +211,7 @@ jobs: USE_MINIKUBE_CACHE: "true" SKIP_PACKAGE_DEB: "true" CARGO_INCREMENTAL: 0 + HELM_CHART_REPO: ${{ github.workspace }}/helm-charts/charts/vector with: timeout_minutes: 45 max_attempts: 3 diff --git a/lib/k8s-e2e-tests/src/lib.rs b/lib/k8s-e2e-tests/src/lib.rs index 4d6d10f4a1875..8986b59ee03a1 100644 --- a/lib/k8s-e2e-tests/src/lib.rs +++ b/lib/k8s-e2e-tests/src/lib.rs @@ -20,6 +20,12 @@ pub mod metrics; pub const BUSYBOX_IMAGE: &str = "busybox:1.28"; +/// Returns the Helm chart repo to use for E2E tests. +/// Set `HELM_CHART_REPO` to override the default (e.g., a local chart path). +pub fn helm_chart_repo() -> String { + env::var("HELM_CHART_REPO").unwrap_or_else(|_| "https://helm.vector.dev".to_string()) +} + pub fn init() { _ = env_logger::builder().is_test(true).try_init(); } diff --git a/lib/k8s-e2e-tests/tests/vector-agent.rs b/lib/k8s-e2e-tests/tests/vector-agent.rs index 2bf56f32336c6..1a1176b8ce4a1 100644 --- a/lib/k8s-e2e-tests/tests/vector-agent.rs +++ b/lib/k8s-e2e-tests/tests/vector-agent.rs @@ -74,7 +74,7 @@ async fn default_agent() -> Result<(), Box> { &namespace, "vector", "vector", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![ &config_override_name(&override_name, true), @@ -199,7 +199,7 @@ async fn partial_merge() -> Result<(), Box> { &namespace, "vector", "vector", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![ &config_override_name(&override_name, true), @@ -351,7 +351,7 @@ async fn preexisting() -> Result<(), Box> { &namespace, "vector", "vector", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![ &config_override_name(&override_name, true), @@ -453,7 +453,7 @@ async fn multiple_lines() -> Result<(), Box> { &namespace, "vector", "vector", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![ &config_override_name(&override_name, true), @@ -580,7 +580,7 @@ async fn metadata_annotation() -> Result<(), Box> { &namespace, "vector", "vector", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![ &config_override_name(&override_name, true), @@ -769,7 +769,7 @@ async fn pod_filtering() -> Result<(), Box> { &namespace, "vector", "vector", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![ &config_override_name(&override_name, true), @@ -1012,7 +1012,7 @@ async fn custom_selectors() -> Result<(), Box> { &namespace, "vector", "vector", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![&config_override_name(&override_name, true), CONFIG], ..Default::default() @@ -1207,7 +1207,7 @@ async fn container_filtering() -> Result<(), Box> { &namespace, "vector", "vector", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![ &config_override_name(&override_name, true), @@ -1416,7 +1416,7 @@ async fn glob_pattern_filtering() -> Result<(), Box> { &namespace, "vector", "vector", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![&config_override_name(&override_name, true), CONFIG], ..Default::default() @@ -1576,7 +1576,7 @@ async fn multiple_ns() -> Result<(), Box> { &namespace, "vector", "vector", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![ &config_override_name(&override_name, true), @@ -1738,7 +1738,7 @@ async fn existing_config_file() -> Result<(), Box> { &namespace, "vector", "vector", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![ &config_override_name(&override_name, true), @@ -1839,7 +1839,7 @@ async fn metrics_pipeline() -> Result<(), Box> { &namespace, "vector", "vector", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![ &config_override_name(&override_name, true), @@ -1994,7 +1994,7 @@ async fn host_metrics() -> Result<(), Box> { &namespace, "vector", "vector", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![ &config_override_name(&override_name, true), @@ -2060,7 +2060,7 @@ async fn simple_checkpoint() -> Result<(), Box> { "test-vector", "vector", "vector", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![HELM_VALUES_AGENT], ..Default::default() diff --git a/lib/k8s-e2e-tests/tests/vector-aggregator.rs b/lib/k8s-e2e-tests/tests/vector-aggregator.rs index 07cb49ff4bf9d..d945fc5175051 100644 --- a/lib/k8s-e2e-tests/tests/vector-aggregator.rs +++ b/lib/k8s-e2e-tests/tests/vector-aggregator.rs @@ -19,7 +19,7 @@ async fn dummy_topology() -> Result<(), Box> { &namespace, "vector", "vector", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![&config_override_name(&override_name, false)], ..Default::default() @@ -54,7 +54,7 @@ async fn metrics_pipeline() -> Result<(), Box> { &namespace, "vector", "vector", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![&config_override_name(&override_name, false)], ..Default::default() diff --git a/lib/k8s-e2e-tests/tests/vector-dd-agent-aggregator.rs b/lib/k8s-e2e-tests/tests/vector-dd-agent-aggregator.rs index 25c7dc4ce6e0a..dde302827cd82 100644 --- a/lib/k8s-e2e-tests/tests/vector-dd-agent-aggregator.rs +++ b/lib/k8s-e2e-tests/tests/vector-dd-agent-aggregator.rs @@ -58,7 +58,7 @@ async fn datadog_to_vector() -> Result<(), Box> { &namespace, "vector", "vector", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![&config_override_name(&override_name, false)], ..Default::default() diff --git a/lib/k8s-e2e-tests/tests/vector.rs b/lib/k8s-e2e-tests/tests/vector.rs index 7d74442fb1146..e38f494236786 100644 --- a/lib/k8s-e2e-tests/tests/vector.rs +++ b/lib/k8s-e2e-tests/tests/vector.rs @@ -77,7 +77,7 @@ async fn logs() -> Result<(), Box> { &namespace, "vector", "aggregator", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { ..Default::default() }, @@ -97,7 +97,7 @@ async fn logs() -> Result<(), Box> { &namespace, "vector", "agent", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![&helm_values_stdout_sink(&agent_override_name)], ..Default::default() @@ -198,7 +198,7 @@ async fn haproxy() -> Result<(), Box> { &namespace, "vector", "aggregator", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![CONFIG], ..Default::default() @@ -227,7 +227,7 @@ async fn haproxy() -> Result<(), Box> { &namespace, "vector", "agent", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![&helm_values_haproxy(&agent_override_name)], ..Default::default() diff --git a/scripts/deploy-chart-test.sh b/scripts/deploy-chart-test.sh index 343b8030fc202..817161ce41c7b 100755 --- a/scripts/deploy-chart-test.sh +++ b/scripts/deploy-chart-test.sh @@ -61,8 +61,14 @@ up() { # A Vector container image to use. CONTAINER_IMAGE="${CONTAINER_IMAGE:?"You must assign CONTAINER_IMAGE variable with the Vector container image name"}" - $VECTOR_TEST_HELM repo add "$CUSTOM_HELM_REPO_LOCAL_NAME" "$CHART_REPO" --force-update || true - $VECTOR_TEST_HELM repo update + # Support both remote repos (https://...) and local chart paths + if [[ "$CHART_REPO" == http* ]]; then + $VECTOR_TEST_HELM repo add "$CUSTOM_HELM_REPO_LOCAL_NAME" "$CHART_REPO" --force-update || true + $VECTOR_TEST_HELM repo update + HELM_INSTALL_TARGET="$CUSTOM_HELM_REPO_LOCAL_NAME/$HELM_CHART" + else + HELM_INSTALL_TARGET="$CHART_REPO" + fi $VECTOR_TEST_KUBECTL create namespace "$NAMESPACE" --dry-run -o yaml | $VECTOR_TEST_KUBECTL apply -f - @@ -95,7 +101,7 @@ up() { --namespace "$NAMESPACE" \ "${HELM_VALUES[@]}" \ "$RELEASE_NAME" \ - "$CUSTOM_HELM_REPO_LOCAL_NAME/$HELM_CHART" + "$HELM_INSTALL_TARGET" { set +x; } &>/dev/null } From b428aba5dd4f26285ce907212acae648bdb6d8df Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Fri, 3 Apr 2026 15:39:14 -0400 Subject: [PATCH 099/326] fix(dev): resolve build.rs HEAD path in worktrees (#25120) * fix(dev): resolve build.rs HEAD path in worktrees * fix(dev): preserve whitespace in git HEAD path --- build.rs | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/build.rs b/build.rs index 5d6d533e86423..01cd4bea5e8ef 100644 --- a/build.rs +++ b/build.rs @@ -104,13 +104,30 @@ fn git_short_hash() -> std::io::Result { }) } +#[cfg(not(feature = "nightly"))] +fn git_path(path: &str) -> std::io::Result { + let output_result = Command::new("git") + .args(["rev-parse", "--git-path", path]) + .output(); + + output_result.map(|output| { + String::from_utf8(output.stdout) + .expect("valid UTF-8") + .trim_end_matches(['\r', '\n']) + .to_owned() + }) +} + fn main() { // Always rerun if the build script itself changes. println!("cargo:rerun-if-changed=build.rs"); // re-run if the HEAD has changed. This is only necessary for non-release and nightly builds. #[cfg(not(feature = "nightly"))] - println!("cargo:rerun-if-changed=.git/HEAD"); + println!( + "cargo:rerun-if-changed={}", + git_path("HEAD").expect("git HEAD path detection failed") + ); #[cfg(feature = "protobuf-build")] { From 738c1d7758024a074f4fbe9ba9fa6d6110b316d7 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Fri, 3 Apr 2026 21:14:48 -0400 Subject: [PATCH 100/326] chore(ci): run K8s E2E suite on a weekly schedule (#25119) chore(ci): run K8s E2E suite weekly instead of nightly Reduce the schedule from Tue-Sat to once a week (Monday 01:00 UTC). The suite also runs on PRs that touch K8s-related files and in the merge queue, so weekly is sufficient for catching regressions. Co-authored-by: Claude Opus 4.6 (1M context) --- .github/workflows/k8s_e2e.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/k8s_e2e.yml b/.github/workflows/k8s_e2e.yml index 471783ed1488c..5f9ba890ab88f 100644 --- a/.github/workflows/k8s_e2e.yml +++ b/.github/workflows/k8s_e2e.yml @@ -4,7 +4,7 @@ # - manual dispatch in GH UI # - on a PR commit if the kubernetes_logs source was changed # - in the merge queue -# - on a schedule at midnight UTC Tue-Sat +# - on a weekly schedule (Monday 01:00 UTC) # - on demand by either of the following comments in a PR: # - '/ci-run-k8s' # - '/ci-run-all' @@ -34,7 +34,7 @@ on: merge_group: types: [checks_requested] schedule: - - cron: "0 1 * * 2-6" # 01:00 UTC Tue-Sat + - cron: "0 1 * * 1" # 01:00 UTC every Monday concurrency: # In flight runs will be canceled through re-trigger in the merge queue, scheduled run, or if From 703590bff3c2e49aa722c493329c753b1943c4ff Mon Sep 17 00:00:00 2001 From: Caleb Metz <135133572+cmetz100@users.noreply.github.com> Date: Mon, 6 Apr 2026 09:25:55 -0400 Subject: [PATCH 101/326] chore(ci): Migrate smp regression workflow auth from static secrets to oidc (#25112) * migrate smp regression workflow auth to oidc Signed-off-by: Caleb Metz * increase token duration Signed-off-by: Caleb Metz * fmt Signed-off-by: Caleb Metz --------- Signed-off-by: Caleb Metz --- .github/workflows/regression.yml | 39 ++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 167dc0186bada..5a3e1d252b72a 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -278,13 +278,16 @@ jobs: needs: - should-run-gate - resolve-inputs + # Job level permissions for downloading SMP binary + permissions: + id-token: write # Required for GitHub OIDC token exchange steps: - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0 with: - aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} + role-to-assume: arn:aws:iam::850406765696:role/smp-regression-oidc aws-region: us-west-2 + role-duration-seconds: 14400 # 4 hours - name: Download SMP binary run: | @@ -303,6 +306,9 @@ jobs: - resolve-inputs - confirm-valid-credentials - build-baseline + # Job level permissions for uploading baseline image to SMP ECR + permissions: + id-token: write # Required for GitHub OIDC token exchange steps: - name: "Download baseline image" uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 @@ -316,9 +322,9 @@ jobs: - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0 with: - aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} + role-to-assume: arn:aws:iam::850406765696:role/smp-regression-oidc aws-region: us-west-2 + role-duration-seconds: 14400 # 4 hours - name: Login to Amazon ECR id: login-ecr @@ -343,6 +349,9 @@ jobs: - resolve-inputs - confirm-valid-credentials - build-comparison + # Job level permissions for uploading comparison image to SMP ECR + permissions: + id-token: write # Required for GitHub OIDC token exchange steps: - name: "Download comparison image" uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 @@ -356,9 +365,9 @@ jobs: - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0 with: - aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} + role-to-assume: arn:aws:iam::850406765696:role/smp-regression-oidc aws-region: us-west-2 + role-duration-seconds: 14400 # 4 hours - name: Login to Amazon ECR id: login-ecr @@ -387,6 +396,7 @@ jobs: permissions: contents: read # Required to checkout code actions: write # Required to upload artifacts + id-token: write # Required for GitHub OIDC token exchange steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -395,9 +405,9 @@ jobs: - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0 with: - aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} + role-to-assume: arn:aws:iam::850406765696:role/smp-regression-oidc aws-region: us-west-2 + role-duration-seconds: 14400 # 4 hours - name: Login to Amazon ECR id: login-ecr @@ -462,15 +472,19 @@ jobs: - submit-job - should-run-gate - resolve-inputs + # Job level permissions for downloading SMP results + permissions: + contents: read # Required to checkout code + id-token: write # Required for GitHub OIDC token exchange steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0 with: - aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} + role-to-assume: arn:aws:iam::850406765696:role/smp-regression-oidc aws-region: us-west-2 + role-duration-seconds: 14400 # 4 hours - name: Download SMP binary run: | @@ -503,6 +517,7 @@ jobs: permissions: contents: read # Required to checkout code actions: write # Required to upload artifacts + id-token: write # Required for GitHub OIDC token exchange steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -511,9 +526,9 @@ jobs: - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0 with: - aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} + role-to-assume: arn:aws:iam::850406765696:role/smp-regression-oidc aws-region: us-west-2 + role-duration-seconds: 14400 # 4 hours - name: Download SMP binary run: | From 2d6fea2dfe7536202c83305cff5021bfadc90442 Mon Sep 17 00:00:00 2001 From: Jed Laundry Date: Tue, 7 Apr 2026 01:29:11 +1200 Subject: [PATCH 102/326] chore(ci): update check-spelling (#25124) update check-spelling --- .github/workflows/spelling.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/spelling.yml b/.github/workflows/spelling.yml index e95e1183fc593..3b2c1f57f2d12 100644 --- a/.github/workflows/spelling.yml +++ b/.github/workflows/spelling.yml @@ -90,7 +90,7 @@ jobs: steps: - name: check-spelling id: spelling - uses: check-spelling/check-spelling@c635c2f3f714eec2fcf27b643a1919b9a811ef2e # v0.0.25 + uses: check-spelling/check-spelling@a35147f799f30f8739c33f92222c847214e82e67 # v0.0.26 (see check-spelling/check-spelling/issues/103) with: suppress_push_for_open_pull_request: ${{ github.actor != 'dependabot[bot]' && 1 }} checkout: true From 1152614bc73e74df27986d1bff7864a376025544 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Mon, 6 Apr 2026 09:32:37 -0400 Subject: [PATCH 103/326] chore(ci): collect K8s diagnostics on E2E test failure (#25114) Add a diagnostic step to the K8s E2E workflow that runs on failure. Captures pod logs, events, configs, and node resource usage to avoid deep manual investigation when tests fail. Co-authored-by: Claude Opus 4.6 (1M context) --- .github/workflows/k8s_e2e.yml | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/.github/workflows/k8s_e2e.yml b/.github/workflows/k8s_e2e.yml index 5f9ba890ab88f..1df064c5eebc9 100644 --- a/.github/workflows/k8s_e2e.yml +++ b/.github/workflows/k8s_e2e.yml @@ -164,7 +164,7 @@ jobs: test-e2e-kubernetes: name: K8s ${{ matrix.kubernetes_version.version }} / ${{ matrix.container_runtime }} (${{ matrix.kubernetes_version.role }}) runs-on: ubuntu-24.04 - timeout-minutes: 45 + timeout-minutes: 60 needs: - build-x86_64-unknown-linux-gnu - compute-k8s-test-plan @@ -217,6 +217,37 @@ jobs: max_attempts: 3 command: make test-e2e-kubernetes + - name: Collect K8s diagnostics on failure + if: ${{ !success() }} + run: | + set +e +o pipefail + # Best-effort diagnostics -- never fail the job + run_diag() { local label="$1"; shift; echo "--- $label ---"; "$@" 2>&1 || true; echo; } + # For commands with pipes that can't be passed as args + run_diag_sh() { echo "--- $1 ---"; bash -c "$2" 2>&1 || true; echo; } + + run_diag "Cluster-wide pods" kubectl get pods -A -o wide + run_diag "Cluster-wide events" kubectl get events -A --sort-by=.metadata.creationTimestamp + run_diag "Nodes" kubectl get nodes -o wide + + for ns in $(kubectl get namespaces -o jsonpath='{.items[*].metadata.name}' 2>/dev/null | tr ' ' '\n' | grep -E '^vector-' || true); do + echo "==========================================" + echo "=== Namespace: $ns ===" + echo "==========================================" + run_diag "Pods" kubectl get pods -n "$ns" -o wide + run_diag "Pod descriptions" kubectl describe pods -n "$ns" + run_diag "Events" kubectl get events -n "$ns" --sort-by=.metadata.creationTimestamp + run_diag "ConfigMaps" kubectl get configmaps -n "$ns" -o yaml + + for pod in $(kubectl get pods -n "$ns" -o jsonpath='{.items[*].metadata.name}' 2>/dev/null || true); do + run_diag "Logs: $pod" kubectl logs -n "$ns" "$pod" --all-containers=true --tail=100 + run_diag "Previous logs: $pod" kubectl logs -n "$ns" "$pod" --all-containers=true --previous --tail=50 + done + done + + run_diag_sh "Node resources" "kubectl describe nodes | grep -A20 'Allocated resources'" + run_diag "Minikube logs" minikube logs --length=100 + final-result: name: K8s E2E Suite runs-on: ubuntu-24.04 From cab8fab16b2da5478d495cd37091b525f527d8ba Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Apr 2026 15:37:04 -0400 Subject: [PATCH 104/326] chore(ci): bump the artifact group across 1 directory with 2 updates (#24999) chore(ci): bump the artifact group with 2 updates Bumps the artifact group with 2 updates: [actions/upload-artifact](https://github.com/actions/upload-artifact) and [actions/download-artifact](https://github.com/actions/download-artifact). Updates `actions/upload-artifact` from 6.0.0 to 7.0.0 - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v6...bbbca2ddaa5d8feaa63e36b76fdaad77386f024f) Updates `actions/download-artifact` from 7.0.0 to 8.0.1 - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/v7...3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-version: 7.0.0 dependency-type: direct:production update-type: version-update:semver-major dependency-group: artifact - dependency-name: actions/download-artifact dependency-version: 8.0.1 dependency-type: direct:production update-type: version-update:semver-major dependency-group: artifact ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/check_generated_vrl_docs.yml | 2 +- .../workflows/check_generated_vrl_docs_comment.yml | 2 +- .github/workflows/integration.yml | 4 ++-- .github/workflows/k8s_e2e.yml | 2 +- .github/workflows/publish.yml | 14 +++++++------- .github/workflows/regression.yml | 10 +++++----- 6 files changed, 17 insertions(+), 17 deletions(-) diff --git a/.github/workflows/check_generated_vrl_docs.yml b/.github/workflows/check_generated_vrl_docs.yml index 7407b28260685..e183e12578f6e 100644 --- a/.github/workflows/check_generated_vrl_docs.yml +++ b/.github/workflows/check_generated_vrl_docs.yml @@ -112,7 +112,7 @@ jobs: && steps.last-commit.outputs.is-auto != 'true' && github.event_name == 'pull_request' && steps.push.outcome != 'success' - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: vrl-docs-check-pr path: /tmp/docs-check/pr-number diff --git a/.github/workflows/check_generated_vrl_docs_comment.yml b/.github/workflows/check_generated_vrl_docs_comment.yml index 25eed42cfeb28..e455203f75a6b 100644 --- a/.github/workflows/check_generated_vrl_docs_comment.yml +++ b/.github/workflows/check_generated_vrl_docs_comment.yml @@ -19,7 +19,7 @@ jobs: pull-requests: write steps: - name: Download PR metadata - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: vrl-docs-check-pr run-id: ${{ github.event.workflow_run.id }} diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 64cfe8c6da004..cdbba4f19187c 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -115,7 +115,7 @@ jobs: timeout-minutes: 90 steps: - name: Download JSON artifact from changes.yml - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 if: github.event_name == 'merge_group' with: name: int_tests_changes @@ -179,7 +179,7 @@ jobs: steps: - name: Download JSON artifact from changes.yml - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 if: github.event_name == 'merge_group' with: name: e2e_tests_changes diff --git a/.github/workflows/k8s_e2e.yml b/.github/workflows/k8s_e2e.yml index 1df064c5eebc9..6432b3a8c810e 100644 --- a/.github/workflows/k8s_e2e.yml +++ b/.github/workflows/k8s_e2e.yml @@ -185,7 +185,7 @@ jobs: mold: false cargo-cache: false - - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: e2e-test-deb-package path: target/artifacts diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 4c66ff585e173..239b7f95388f6 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -221,7 +221,7 @@ jobs: with: ref: ${{ inputs.git_ref }} - name: Download staged package artifacts (x86_64-unknown-linux-gnu) - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-unknown-linux-gnu path: target/artifacts @@ -272,7 +272,7 @@ jobs: with: ref: ${{ inputs.git_ref }} - name: Download staged package artifacts (x86_64-unknown-linux-gnu) - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-unknown-linux-gnu path: target/artifacts @@ -301,7 +301,7 @@ jobs: with: ref: ${{ inputs.git_ref }} - name: Download staged package artifacts (${{ matrix.target }}) - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: vector-${{ env.VECTOR_VERSION }}-${{ matrix.target }} path: target/artifacts @@ -351,7 +351,7 @@ jobs: with: version: latest - name: Download all Linux package artifacts - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: pattern: vector-${{ env.VECTOR_VERSION }}-*-linux-* path: target/artifacts @@ -394,7 +394,7 @@ jobs: with: vdev: true - name: Download all package artifacts - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: pattern: vector-${{ env.VECTOR_VERSION }}-* path: target/artifacts @@ -434,7 +434,7 @@ jobs: with: vdev: true - name: Download all package artifacts - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: pattern: vector-${{ env.VECTOR_VERSION }}-* path: target/artifacts @@ -461,7 +461,7 @@ jobs: with: ref: ${{ inputs.git_ref }} - name: Download all package artifacts - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: pattern: vector-${{ env.VECTOR_VERSION }}-* path: target/artifacts diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 5a3e1d252b72a..cc1170a8aaa21 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -311,7 +311,7 @@ jobs: id-token: write # Required for GitHub OIDC token exchange steps: - name: "Download baseline image" - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: baseline-image @@ -354,7 +354,7 @@ jobs: id-token: write # Required for GitHub OIDC token exchange steps: - name: "Download comparison image" - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: comparison-image @@ -491,7 +491,7 @@ jobs: aws s3 cp s3://smp-cli-releases/v${{ needs.resolve-inputs.outputs.smp-version }}/x86_64-unknown-linux-musl/smp ${{ runner.temp }}/bin/smp - name: Download submission metadata - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: vector-submission-metadata path: ${{ runner.temp }}/ @@ -535,7 +535,7 @@ jobs: aws s3 cp s3://smp-cli-releases/v${{ needs.resolve-inputs.outputs.smp-version }}/x86_64-unknown-linux-musl/smp ${{ runner.temp }}/bin/smp - name: Download submission metadata - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: vector-submission-metadata path: ${{ runner.temp }}/ @@ -581,7 +581,7 @@ jobs: steps: - name: Download capture-artifacts continue-on-error: true - uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: capture-artifacts From 4b2353a35dd54b888cdee7d1ca70c8723e12873c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Apr 2026 15:46:52 -0400 Subject: [PATCH 105/326] chore(ci): bump nick-fields/retry from 3.0.2 to 4.0.0 (#25102) Bumps [nick-fields/retry](https://github.com/nick-fields/retry) from 3.0.2 to 4.0.0. - [Release notes](https://github.com/nick-fields/retry/releases) - [Commits](https://github.com/nick-fields/retry/compare/ce71cc2ab81d554ebbe88c79ab5975992d79ba08...ad984534de44a9489a53aefd81eb77f87c70dc60) --- updated-dependencies: - dependency-name: nick-fields/retry dependency-version: 4.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci-integration-review.yml | 4 ++-- .github/workflows/integration.yml | 4 ++-- .github/workflows/k8s_e2e.yml | 2 +- .github/workflows/publish.yml | 2 +- .github/workflows/unit_mac.yml | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci-integration-review.yml b/.github/workflows/ci-integration-review.yml index 67fae9ea864d7..77b20b347e318 100644 --- a/.github/workflows/ci-integration-review.yml +++ b/.github/workflows/ci-integration-review.yml @@ -174,7 +174,7 @@ jobs: - name: Integration Tests - ${{ matrix.service }} if: steps.run_condition.outputs.should_run == 'true' - uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 + uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0 with: timeout_minutes: 30 max_attempts: 3 @@ -228,7 +228,7 @@ jobs: - name: E2E Tests - ${{ matrix.service }} if: steps.run_condition.outputs.should_run == 'true' - uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 + uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0 with: timeout_minutes: 35 max_attempts: 3 diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index cdbba4f19187c..2c1e91c57a8e7 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -158,7 +158,7 @@ jobs: - name: Run Integration Tests for ${{ matrix.service }} if: steps.check.outputs.should_run == 'true' - uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 + uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0 with: timeout_minutes: 30 max_attempts: 3 @@ -222,7 +222,7 @@ jobs: - name: Run E2E Tests for ${{ matrix.service }} if: steps.check.outputs.should_run == 'true' - uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 + uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0 with: timeout_minutes: 30 max_attempts: 3 diff --git a/.github/workflows/k8s_e2e.yml b/.github/workflows/k8s_e2e.yml index 6432b3a8c810e..3b35ec0b7b9dc 100644 --- a/.github/workflows/k8s_e2e.yml +++ b/.github/workflows/k8s_e2e.yml @@ -206,7 +206,7 @@ jobs: # TODO: This job has been quite flakey. Need to investigate further and then remove the retries. - name: Run tests - uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 + uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0 env: USE_MINIKUBE_CACHE: "true" SKIP_PACKAGE_DEB: "true" diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 239b7f95388f6..d5093f229017c 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -363,7 +363,7 @@ jobs: env: PLATFORM: "linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v6" REPOS: "timberio/vector,ghcr.io/vectordotdev/vector" - uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 + uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0 with: timeout_minutes: 15 max_attempts: 3 diff --git a/.github/workflows/unit_mac.yml b/.github/workflows/unit_mac.yml index e0d367b530959..a44a84d368953 100644 --- a/.github/workflows/unit_mac.yml +++ b/.github/workflows/unit_mac.yml @@ -31,7 +31,7 @@ jobs: # Some tests e.g. `reader_exits_cleanly_when_writer_done_and_in_flight_acks` are flaky. - name: Run tests - uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2 + uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0 with: timeout_minutes: 45 max_attempts: 3 From 9e897388fbf772a23c48df93de923a05b2d3ecbe Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Apr 2026 20:05:11 +0000 Subject: [PATCH 106/326] chore(deps): bump fast-xml-parser from 5.3.7 to 5.5.9 in /scripts/environment/npm-tools in the npm_and_yarn group across 1 directory (#25127) chore(deps): bump fast-xml-parser Bumps the npm_and_yarn group with 1 update in the /scripts/environment/npm-tools directory: [fast-xml-parser](https://github.com/NaturalIntelligence/fast-xml-parser). Updates `fast-xml-parser` from 5.3.7 to 5.5.9 - [Release notes](https://github.com/NaturalIntelligence/fast-xml-parser/releases) - [Changelog](https://github.com/NaturalIntelligence/fast-xml-parser/blob/master/CHANGELOG.md) - [Commits](https://github.com/NaturalIntelligence/fast-xml-parser/compare/v5.3.7...v5.5.9) --- updated-dependencies: - dependency-name: fast-xml-parser dependency-version: 5.5.9 dependency-type: indirect dependency-group: npm_and_yarn ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Pavlos Rontidis --- .../environment/npm-tools/package-lock.json | 192 ++++++++++-------- scripts/environment/npm-tools/package.json | 2 +- 2 files changed, 113 insertions(+), 81 deletions(-) diff --git a/scripts/environment/npm-tools/package-lock.json b/scripts/environment/npm-tools/package-lock.json index e3d4efa122515..1047d5444f89a 100644 --- a/scripts/environment/npm-tools/package-lock.json +++ b/scripts/environment/npm-tools/package-lock.json @@ -5,7 +5,7 @@ "packages": { "": { "dependencies": { - "@datadog/datadog-ci": "5.11.0", + "@datadog/datadog-ci": "5.12.0", "markdownlint-cli2": "0.22.0", "prettier": "3.8.1" } @@ -24,19 +24,19 @@ } }, "node_modules/@datadog/datadog-ci": { - "version": "5.11.0", - "resolved": "https://registry.npmjs.org/@datadog/datadog-ci/-/datadog-ci-5.11.0.tgz", - "integrity": "sha512-4N/WsfnQVYJNxGtDIOhg4sUkZalDo8h+toSXR/GAw6XU81e1bOj3GvleL6IHUQ06KyPdurqPZKB4Za1CbCsT7g==", + "version": "5.12.0", + "resolved": "https://registry.npmjs.org/@datadog/datadog-ci/-/datadog-ci-5.12.0.tgz", + "integrity": "sha512-8zJOgx0ouVZtspXTGdx327Yt6UiRnaQSlMc7c343a1HaKC9nIlQhnd90G1frFZhopiqA1KD8h9CWBhygxAJW7w==", "license": "Apache-2.0", "dependencies": { - "@datadog/datadog-ci-base": "5.11.0", - "@datadog/datadog-ci-plugin-coverage": "5.11.0", - "@datadog/datadog-ci-plugin-deployment": "5.11.0", - "@datadog/datadog-ci-plugin-dora": "5.11.0", - "@datadog/datadog-ci-plugin-gate": "5.11.0", - "@datadog/datadog-ci-plugin-junit": "5.11.0", - "@datadog/datadog-ci-plugin-sarif": "5.11.0", - "@datadog/datadog-ci-plugin-sbom": "5.11.0", + "@datadog/datadog-ci-base": "5.12.0", + "@datadog/datadog-ci-plugin-coverage": "5.12.0", + "@datadog/datadog-ci-plugin-deployment": "5.12.0", + "@datadog/datadog-ci-plugin-dora": "5.12.0", + "@datadog/datadog-ci-plugin-gate": "5.12.0", + "@datadog/datadog-ci-plugin-junit": "5.12.0", + "@datadog/datadog-ci-plugin-sarif": "5.12.0", + "@datadog/datadog-ci-plugin-sbom": "5.12.0", "clipanion": "3.2.1", "typanion": "3.14.0" }, @@ -48,23 +48,23 @@ } }, "node_modules/@datadog/datadog-ci-base": { - "version": "5.11.0", - "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-base/-/datadog-ci-base-5.11.0.tgz", - "integrity": "sha512-NACS6zhf2qidgpgrhQH0EJn4x20pwyRAhw75jBRPmjzheaXrpU0438e3kutyNsDGZrw8gHjuRqiccoY3QU7PSQ==", + "version": "5.12.0", + "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-base/-/datadog-ci-base-5.12.0.tgz", + "integrity": "sha512-ZyobD2nbzwRksWiYvcxl3ZuYE4+AX1is26qjWAs1ufhVyGBCx1cVqiDFEAILeyqkncZYIFGppjCyjUcxev+CBg==", "license": "Apache-2.0", "dependencies": { "@antfu/install-pkg": "1.1.0", "@types/datadog-metrics": "0.6.1", - "async-retry": "1.3.1", + "async-retry": "1.3.3", "axios": "1.13.5", "chalk": "3.0.0", "clipanion": "3.2.1", "datadog-metrics": "0.9.3", "debug": "4.4.1", "deep-extend": "0.6.0", - "fast-xml-parser": "5.3.7", + "fast-xml-parser": "5.5.9", "form-data": "4.0.4", - "glob": "13.0.5", + "glob": "13.0.6", "inquirer": "8.2.7", "is-docker": "4.0.0", "jest-diff": "30.2.0", @@ -79,17 +79,17 @@ "upath": "2.0.1" }, "peerDependencies": { - "@datadog/datadog-ci-plugin-aas": "5.11.0", - "@datadog/datadog-ci-plugin-cloud-run": "5.11.0", - "@datadog/datadog-ci-plugin-container-app": "5.11.0", - "@datadog/datadog-ci-plugin-coverage": "5.11.0", - "@datadog/datadog-ci-plugin-deployment": "5.11.0", - "@datadog/datadog-ci-plugin-dora": "5.11.0", - "@datadog/datadog-ci-plugin-gate": "5.11.0", - "@datadog/datadog-ci-plugin-junit": "5.11.0", - "@datadog/datadog-ci-plugin-lambda": "5.11.0", - "@datadog/datadog-ci-plugin-sarif": "5.11.0", - "@datadog/datadog-ci-plugin-sbom": "5.11.0", + "@datadog/datadog-ci-plugin-aas": "5.12.0", + "@datadog/datadog-ci-plugin-cloud-run": "5.12.0", + "@datadog/datadog-ci-plugin-container-app": "5.12.0", + "@datadog/datadog-ci-plugin-coverage": "5.12.0", + "@datadog/datadog-ci-plugin-deployment": "5.12.0", + "@datadog/datadog-ci-plugin-dora": "5.12.0", + "@datadog/datadog-ci-plugin-gate": "5.12.0", + "@datadog/datadog-ci-plugin-junit": "5.12.0", + "@datadog/datadog-ci-plugin-lambda": "5.12.0", + "@datadog/datadog-ci-plugin-sarif": "5.12.0", + "@datadog/datadog-ci-plugin-sbom": "5.12.0", "@datadog/datadog-ci-plugin-stepfunctions": "5.11.0", "@datadog/datadog-ci-plugin-synthetics": "5.11.0", "@datadog/datadog-ci-plugin-terraform": "5.11.0" @@ -157,24 +157,24 @@ } }, "node_modules/@datadog/datadog-ci-plugin-coverage": { - "version": "5.11.0", - "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-coverage/-/datadog-ci-plugin-coverage-5.11.0.tgz", - "integrity": "sha512-aKzkjbd/NVt+7Sd4QL0qVqUrnyMG0UF63lvNxve7suWQDoIWowAgCPa0ta+jpMYUYCGtjt74H3wL+q0W0G+D+Q==", + "version": "5.12.0", + "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-coverage/-/datadog-ci-plugin-coverage-5.12.0.tgz", + "integrity": "sha512-ll4aBYF+FMOlOjAnNiPjO8iPmVB/qfBHrUOCk2EMnOaQn6Eve/FajCaSqeV2BoI9+3UXv04P6eH5nxPtpcM/cg==", "license": "Apache-2.0", "dependencies": { "chalk": "3.0.0", - "fast-xml-parser": "5.3.7", + "fast-xml-parser": "5.5.9", "form-data": "4.0.4", "upath": "2.0.1" }, "peerDependencies": { - "@datadog/datadog-ci-base": "5.11.0" + "@datadog/datadog-ci-base": "5.12.0" } }, "node_modules/@datadog/datadog-ci-plugin-deployment": { - "version": "5.11.0", - "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-deployment/-/datadog-ci-plugin-deployment-5.11.0.tgz", - "integrity": "sha512-FF9L7alDz1XN5L3nYR7xMeZd99U6kxkT+InSyZY21ZHpcuuEngseW1aMgB0iFn3lxZ+1H4z196uCckViTEkjog==", + "version": "5.12.0", + "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-deployment/-/datadog-ci-plugin-deployment-5.12.0.tgz", + "integrity": "sha512-6KDwUmdxwGxYoBEYqAY0uo7ocvtTq3p3cXW4KjpDXwZyr1wWstzXQxTx7qquZzMMNvsD6JCq+HDqwdTgaHSAog==", "license": "Apache-2.0", "dependencies": { "axios": "1.13.5", @@ -182,26 +182,26 @@ "simple-git": "3.33.0" }, "peerDependencies": { - "@datadog/datadog-ci-base": "5.11.0" + "@datadog/datadog-ci-base": "5.12.0" } }, "node_modules/@datadog/datadog-ci-plugin-dora": { - "version": "5.11.0", - "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-dora/-/datadog-ci-plugin-dora-5.11.0.tgz", - "integrity": "sha512-DWSyT2tW23XOYCvTEHvD/F9FRIffTN+2wZGYojaD1LM9wJew4xievziBwyD46quCF0OdsbGMLvAyR2hyrrYeZA==", + "version": "5.12.0", + "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-dora/-/datadog-ci-plugin-dora-5.12.0.tgz", + "integrity": "sha512-HrC6H5lLzRTv3tKiS2f5YMU/WWwgzMF2kHIJU6+pe0y/v1xUYCELV112XYldbg3dkENfpG/AwjUeO6B8rE3s+g==", "license": "Apache-2.0", "dependencies": { "chalk": "3.0.0", "simple-git": "3.33.0" }, "peerDependencies": { - "@datadog/datadog-ci-base": "5.11.0" + "@datadog/datadog-ci-base": "5.12.0" } }, "node_modules/@datadog/datadog-ci-plugin-gate": { - "version": "5.11.0", - "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-gate/-/datadog-ci-plugin-gate-5.11.0.tgz", - "integrity": "sha512-WUNr6y292eH+sumAyD5rFhqlU/aJ8vN74h4EJj0SqDpnrDCzMnSDgRyb9GXhv8ZO4GOW6LTEcPmMZhyfYL3b5g==", + "version": "5.12.0", + "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-gate/-/datadog-ci-plugin-gate-5.12.0.tgz", + "integrity": "sha512-YNZuRigmGDGK2tK6xRsaq0PPx/4CRVZ4U6q7xabIP+jmhoG7mbI1RFUjlXSnGfeU1SXm0bVQw8F69AcZCHtNOA==", "license": "Apache-2.0", "dependencies": { "@types/uuid": "9.0.8", @@ -209,29 +209,29 @@ "uuid": "9.0.1" }, "peerDependencies": { - "@datadog/datadog-ci-base": "5.11.0" + "@datadog/datadog-ci-base": "5.12.0" } }, "node_modules/@datadog/datadog-ci-plugin-junit": { - "version": "5.11.0", - "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-junit/-/datadog-ci-plugin-junit-5.11.0.tgz", - "integrity": "sha512-I8H09jNR/LDYqiy7TOKv9Gu4+t/o0J6QH6cooKrNboULSDYijVo6w3knANsd2m3Hlcbi1QkTmNpPXIdeGR5Dig==", + "version": "5.12.0", + "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-junit/-/datadog-ci-plugin-junit-5.12.0.tgz", + "integrity": "sha512-XPVULYU1RUN2kFmSWGfAgU+psGFhPu+5NxnHeAQubs/wYcGwdZUEAD2+tn4F3PxpCkTduEb4FO6mLpYbjBpuqA==", "license": "Apache-2.0", "dependencies": { "chalk": "3.0.0", - "fast-xml-parser": "5.3.7", + "fast-xml-parser": "5.5.9", "form-data": "4.0.4", "upath": "2.0.1", "uuid": "9.0.1" }, "peerDependencies": { - "@datadog/datadog-ci-base": "5.11.0" + "@datadog/datadog-ci-base": "5.12.0" } }, "node_modules/@datadog/datadog-ci-plugin-sarif": { - "version": "5.11.0", - "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-sarif/-/datadog-ci-plugin-sarif-5.11.0.tgz", - "integrity": "sha512-JZuqNzFsW9BZeStLM60f5zdEpG67kh7xqgbeJhqIcloEpDJIEmsJHH487PM0stakcqhhIfzUzvEW744LLQg5Ug==", + "version": "5.12.0", + "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-sarif/-/datadog-ci-plugin-sarif-5.12.0.tgz", + "integrity": "sha512-tmUEv6sJWsFG9vnriSu9RZSrLvYo0EfvRaHiiO3ufxBC+NvU0HZYhtTqXVT6NnsUSVv/2FBnzl0hCnPDHhyBFg==", "license": "Apache-2.0", "dependencies": { "ajv": "8.18.0", @@ -242,13 +242,13 @@ "uuid": "9.0.1" }, "peerDependencies": { - "@datadog/datadog-ci-base": "5.11.0" + "@datadog/datadog-ci-base": "5.12.0" } }, "node_modules/@datadog/datadog-ci-plugin-sbom": { - "version": "5.11.0", - "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-sbom/-/datadog-ci-plugin-sbom-5.11.0.tgz", - "integrity": "sha512-lHU65a1Hru7SiwhZtZ9OTd6vyPcRG2U/61imVpvLvhEMSwY2xSLTtZfxWbMi4VJxq81V2zhdifxP5BMJO1FnHw==", + "version": "5.12.0", + "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-sbom/-/datadog-ci-plugin-sbom-5.12.0.tgz", + "integrity": "sha512-F0c0gfX4O+Qm1heLEhNCTl/tUPlVQvyFGYTUqJ0Z2niqT4Y3z7Oi6RSodgUSwGCKl6bwacRjTTdCrzI3V0n3nA==", "license": "Apache-2.0", "dependencies": { "ajv": "8.18.0", @@ -258,7 +258,7 @@ "packageurl-js": "2.0.1" }, "peerDependencies": { - "@datadog/datadog-ci-base": "5.11.0" + "@datadog/datadog-ci-base": "5.12.0" } }, "node_modules/@inquirer/external-editor": { @@ -525,12 +525,12 @@ } }, "node_modules/async-retry": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/async-retry/-/async-retry-1.3.1.tgz", - "integrity": "sha512-aiieFW/7h3hY0Bq5d+ktDBejxuwR78vRu9hDUdR8rNhSaQ29VzPL4AoIRG7D/c7tdenwOcKvgPM6tIxB3cB6HA==", + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/async-retry/-/async-retry-1.3.3.tgz", + "integrity": "sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==", "license": "MIT", "dependencies": { - "retry": "0.12.0" + "retry": "0.13.1" } }, "node_modules/asynckit": { @@ -1178,10 +1178,25 @@ ], "license": "BSD-3-Clause" }, + "node_modules/fast-xml-builder": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.1.4.tgz", + "integrity": "sha512-f2jhpN4Eccy0/Uz9csxh3Nu6q4ErKxf0XIsasomfOihuSUa3/xw6w8dnOtCDgEItQFJG8KyXPzQXzcODDrrbOg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "path-expression-matcher": "^1.1.3" + } + }, "node_modules/fast-xml-parser": { - "version": "5.3.7", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.3.7.tgz", - "integrity": "sha512-JzVLro9NQv92pOM/jTCR6mHlJh2FGwtomH8ZQjhFj/R29P2Fnj38OgPJVtcvYw6SuKClhgYuwUZf5b3rd8u2mA==", + "version": "5.5.9", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.5.9.tgz", + "integrity": "sha512-jldvxr1MC6rtiZKgrFnDSvT8xuH+eJqxqOBThUVjYrxssYTo1avZLGql5l0a0BAERR01CadYzZ83kVEkbyDg+g==", "funding": [ { "type": "github", @@ -1190,7 +1205,9 @@ ], "license": "MIT", "dependencies": { - "strnum": "^2.1.2" + "fast-xml-builder": "^1.1.4", + "path-expression-matcher": "^1.2.0", + "strnum": "^2.2.2" }, "bin": { "fxparser": "src/cli/cli.js" @@ -1341,17 +1358,17 @@ } }, "node_modules/glob": { - "version": "13.0.5", - "resolved": "https://registry.npmjs.org/glob/-/glob-13.0.5.tgz", - "integrity": "sha512-BzXxZg24Ibra1pbQ/zE7Kys4Ua1ks7Bn6pKLkVPZ9FZe4JQS6/Q7ef3LG1H+k7lUf5l4T3PLSyYyYJVYUvfgTw==", + "version": "13.0.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-13.0.6.tgz", + "integrity": "sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw==", "license": "BlueOak-1.0.0", "dependencies": { - "minimatch": "^10.2.1", - "minipass": "^7.1.2", - "path-scurry": "^2.0.0" + "minimatch": "^10.2.2", + "minipass": "^7.1.3", + "path-scurry": "^2.0.2" }, "engines": { - "node": "20 || >=22" + "node": "18 || 20 || >=22" }, "funding": { "url": "https://github.com/sponsors/isaacs" @@ -1884,9 +1901,9 @@ } }, "node_modules/lru-cache": { - "version": "11.2.7", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.2.7.tgz", - "integrity": "sha512-aY/R+aEsRelme17KGQa/1ZSIpLpNYYrhcrepKTZgE+W3WM16YMCaPwOHLHsmopZHELU0Ojin1lPVxKR0MihncA==", + "version": "11.3.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.3.0.tgz", + "integrity": "sha512-sr8xPKE25m6vJVcrdn6NxtC0fVfuPowbscLypegRgOm0yXSqr5JNHCAY3hnusdJ7HRBW04j6Ip4khvHU778DuQ==", "license": "BlueOak-1.0.0", "engines": { "node": "20 || >=22" @@ -2768,6 +2785,21 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/path-expression-matcher": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/path-expression-matcher/-/path-expression-matcher-1.2.1.tgz", + "integrity": "sha512-d7gQQmLvAKXKXE2GeP9apIGbMYKz88zWdsn/BN2HRWVQsDFdUY36WSLTY0Jvd4HWi7Fb30gQ62oAOzdgJA6fZw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/path-scurry": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.2.tgz", @@ -2965,9 +2997,9 @@ } }, "node_modules/retry": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", - "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==", + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", + "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", "license": "MIT", "engines": { "node": ">= 4" diff --git a/scripts/environment/npm-tools/package.json b/scripts/environment/npm-tools/package.json index 30d63057b7fed..a7ec2311e71d2 100644 --- a/scripts/environment/npm-tools/package.json +++ b/scripts/environment/npm-tools/package.json @@ -2,7 +2,7 @@ "private": true, "description": "Pinned npm tools for CI. Run 'npm ci' to install with exact versions from package-lock.json.", "dependencies": { - "@datadog/datadog-ci": "5.11.0", + "@datadog/datadog-ci": "5.12.0", "markdownlint-cli2": "0.22.0", "prettier": "3.8.1" } From 0c5fd58228bfab0dd9f6e81a1dc651da2ea8e670 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 7 Apr 2026 12:14:58 -0400 Subject: [PATCH 107/326] chore(api top): improve dashboard column layout (#25135) enhancement(vector top): improve dashboard column layout - Move Errors column to always be last (after Memory Used when allocation-tracing is enabled) - Slightly increase Kind column width (+1%) and decrease Errors (-1%) Co-authored-by: Claude Opus 4.6 (1M context) --- lib/vector-top/src/dashboard.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/vector-top/src/dashboard.rs b/lib/vector-top/src/dashboard.rs index 1b73d5b89e9c8..8f54c48534f71 100644 --- a/lib/vector-top/src/dashboard.rs +++ b/lib/vector-top/src/dashboard.rs @@ -170,9 +170,9 @@ static HEADER: [&str; NUM_COLUMNS] = [ columns::BYTES_IN, columns::EVENTS_OUT, columns::BYTES_OUT, - columns::ERRORS, #[cfg(feature = "allocation-tracing")] columns::MEMORY_USED, + columns::ERRORS, ]; struct Widgets<'a> { @@ -348,13 +348,13 @@ impl<'a> Widgets<'a> { r.sent_bytes_throughput_sec, self.human_metrics, ), + #[cfg(feature = "allocation-tracing")] + r.allocated_bytes.human_format_bytes(), if self.human_metrics { r.errors.human_format() } else { r.errors.thousands_format() }, - #[cfg(feature = "allocation-tracing")] - r.allocated_bytes.human_format_bytes(), ]; data.extend_from_slice(&formatted_metrics); @@ -383,26 +383,26 @@ impl<'a> Widgets<'a> { &[ Constraint::Percentage(13), // ID Constraint::Percentage(8), // Output - Constraint::Percentage(4), // Kind + Constraint::Percentage(5), // Kind Constraint::Percentage(9), // Type Constraint::Percentage(10), // Events In Constraint::Percentage(12), // Bytes In Constraint::Percentage(10), // Events Out Constraint::Percentage(12), // Bytes Out - Constraint::Percentage(8), // Errors - Constraint::Percentage(14), // Allocated Bytes + Constraint::Percentage(14), // Memory Used + Constraint::Percentage(7), // Errors ] } else { &[ Constraint::Percentage(13), // ID Constraint::Percentage(12), // Output - Constraint::Percentage(9), // Kind + Constraint::Percentage(10), // Kind Constraint::Percentage(6), // Type Constraint::Percentage(12), // Events In Constraint::Percentage(14), // Bytes In Constraint::Percentage(12), // Events Out Constraint::Percentage(14), // Bytes Out - Constraint::Percentage(8), // Errors + Constraint::Percentage(7), // Errors ] }; let w = Table::new(items, widths) From 02f281d6ad2786f8bbfad9ece820b8d9f4925571 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 7 Apr 2026 15:38:29 -0400 Subject: [PATCH 108/326] enhancement(api top): show "disabled" in Memory Used when allocation tracing is off (#25138) * enhancement(vector top): show "disabled" in Memory Used when allocation tracing is off Add GetAllocationTracingStatus gRPC endpoint that reports whether the connected Vector instance has --allocation-tracing active. vector-top queries this on startup and shows "disabled" in the Memory Used column when tracing is off, instead of misleading zeros. Changes: - proto: add GetAllocationTracingStatus RPC and messages - service.rs: implement the endpoint using is_allocation_tracing_enabled() - client.rs: add get_allocation_tracing_status() method - metrics.rs: query the endpoint on startup, store result in State - state.rs: add allocation_tracing_active field to State - dashboard.rs: show "disabled" when allocation_tracing_active is false Co-Authored-By: Claude Opus 4.6 (1M context) * chore: add changelog fragment for Memory Used disabled label Co-Authored-By: Claude Opus 4.6 (1M context) * add authors * fix(api): guard allocation tracing RPC behind cfg feature flag The `get_allocation_tracing_status` RPC handler unconditionally called `is_allocation_tracing_enabled()`, which only exists behind the `allocation-tracing` feature gate. This broke builds on feature sets that lack it (e.g. `default-msvc`). Add cfg guards so the handler returns `false` when the feature is absent. Co-Authored-By: Claude Opus 4.6 (1M context) --------- Co-authored-by: Claude Opus 4.6 (1M context) --- ...tor_top_disabled_memory_used.enhancement.md | 3 +++ lib/vector-api-client/src/client.rs | 12 ++++++++++++ lib/vector-top/src/dashboard.rs | 6 +++++- lib/vector-top/src/metrics.rs | 18 +++++++++++++++++- lib/vector-top/src/state.rs | 6 ++++++ proto/vector/observability.proto | 9 +++++++++ src/api/grpc/service.rs | 13 +++++++++++++ 7 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 changelog.d/vector_top_disabled_memory_used.enhancement.md diff --git a/changelog.d/vector_top_disabled_memory_used.enhancement.md b/changelog.d/vector_top_disabled_memory_used.enhancement.md new file mode 100644 index 0000000000000..e929a065019ef --- /dev/null +++ b/changelog.d/vector_top_disabled_memory_used.enhancement.md @@ -0,0 +1,3 @@ +`vector top` terminal UI now shows `disabled` in the Memory Used column when the connected Vector instance was not started with `--allocation-tracing`, instead of displaying misleading zeros. A new `GetAllocationTracingStatus` gRPC endpoint is queried on connect to determine the status. + +authors: pront diff --git a/lib/vector-api-client/src/client.rs b/lib/vector-api-client/src/client.rs index d4991efef0894..fcba2702a3e2a 100644 --- a/lib/vector-api-client/src/client.rs +++ b/lib/vector-api-client/src/client.rs @@ -5,6 +5,7 @@ use tonic::transport::{Channel, Endpoint}; use crate::{ error::{Error, Result}, proto::{ + GetAllocationTracingStatusRequest, GetAllocationTracingStatusResponse, GetComponentsRequest, GetComponentsResponse, GetMetaRequest, GetMetaResponse, HealthRequest, HealthResponse, MetricName, StreamComponentAllocatedBytesRequest, StreamComponentAllocatedBytesResponse, StreamComponentMetricsRequest, @@ -77,6 +78,17 @@ impl Client { Ok(response.into_inner()) } + /// Check whether allocation tracing is active on the connected Vector instance + pub async fn get_allocation_tracing_status( + &mut self, + ) -> Result { + let client = self.ensure_connected()?; + let response = client + .get_allocation_tracing_status(GetAllocationTracingStatusRequest {}) + .await?; + Ok(response.into_inner()) + } + // ========== Streaming RPCs ========== /// Stream periodic heartbeat timestamps diff --git a/lib/vector-top/src/dashboard.rs b/lib/vector-top/src/dashboard.rs index 8f54c48534f71..9e6a1907dff36 100644 --- a/lib/vector-top/src/dashboard.rs +++ b/lib/vector-top/src/dashboard.rs @@ -349,7 +349,11 @@ impl<'a> Widgets<'a> { self.human_metrics, ), #[cfg(feature = "allocation-tracing")] - r.allocated_bytes.human_format_bytes(), + if state.allocation_tracing_active { + r.allocated_bytes.human_format_bytes() + } else { + "disabled".to_string() + }, if self.human_metrics { r.errors.human_format() } else { diff --git a/lib/vector-top/src/metrics.rs b/lib/vector-top/src/metrics.rs index 52cf46beff244..63328a62825f0 100644 --- a/lib/vector-top/src/metrics.rs +++ b/lib/vector-top/src/metrics.rs @@ -569,5 +569,21 @@ pub async fn init_components( }) .collect::>(); - Ok(state::State::new(rows)) + let mut state = state::State::new(rows); + + #[cfg(feature = "allocation-tracing")] + { + // Allocation tracing is a compile-time + startup-time setting on the + // server, so querying once per connection is sufficient. On error + // (e.g. older server without this RPC) we default to false, matching + // pre-existing behavior. This is re-evaluated on every reconnect via + // the retry loop in `subscription()`. + state.allocation_tracing_active = client + .get_allocation_tracing_status() + .await + .map(|r| r.enabled) + .unwrap_or(false); + } + + Ok(state) } diff --git a/lib/vector-top/src/state.rs b/lib/vector-top/src/state.rs index a954c804ce467..30fc05ef76c13 100644 --- a/lib/vector-top/src/state.rs +++ b/lib/vector-top/src/state.rs @@ -122,6 +122,10 @@ pub struct State { pub sort_state: SortState, pub filter_state: FilterState, pub ui: UiState, + /// Set to `true` once we receive the first `AllocatedBytes` event, + /// indicating the connected Vector instance has allocation tracing active. + #[cfg(feature = "allocation-tracing")] + pub allocation_tracing_active: bool, } #[derive(Debug, Clone, Copy)] @@ -343,6 +347,8 @@ impl State { ui: UiState::default(), sort_state: SortState::default(), filter_state: FilterState::default(), + #[cfg(feature = "allocation-tracing")] + allocation_tracing_active: false, } } diff --git a/proto/vector/observability.proto b/proto/vector/observability.proto index 1c1d508e5d26c..9996a47908949 100644 --- a/proto/vector/observability.proto +++ b/proto/vector/observability.proto @@ -17,6 +17,9 @@ service ObservabilityService { // Get information about configured components (sources, transforms, sinks) rpc GetComponents(GetComponentsRequest) returns (GetComponentsResponse); + // Check whether allocation tracing is active on this instance + rpc GetAllocationTracingStatus(GetAllocationTracingStatusRequest) returns (GetAllocationTracingStatusResponse); + // ========== Real-time Metric Streams ========== // All streaming RPCs send periodic updates at the specified interval @@ -54,6 +57,12 @@ message GetMetaResponse { string hostname = 2; } +message GetAllocationTracingStatusRequest {} + +message GetAllocationTracingStatusResponse { + bool enabled = 1; +} + // ========== Component Messages ========== message GetComponentsRequest { diff --git a/src/api/grpc/service.rs b/src/api/grpc/service.rs index 171f65853cc91..580ebb69b5219 100644 --- a/src/api/grpc/service.rs +++ b/src/api/grpc/service.rs @@ -436,6 +436,19 @@ impl observability::Service for ObservabilityService { Ok(Response::new(GetMetaResponse { version, hostname })) } + async fn get_allocation_tracing_status( + &self, + _request: Request, + ) -> Result, Status> { + #[cfg(feature = "allocation-tracing")] + let enabled = crate::internal_telemetry::allocations::is_allocation_tracing_enabled(); + #[cfg(not(feature = "allocation-tracing"))] + let enabled = false; + Ok(Response::new(GetAllocationTracingStatusResponse { + enabled, + })) + } + async fn get_components( &self, request: Request, From 256db9c12951987ea235a0c5a4a88898d5187642 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Wed, 8 Apr 2026 11:32:50 -0400 Subject: [PATCH 109/326] fix(api top): avoid unused-mut warning on Windows builds (#25142) The `allocation-tracing` feature is gated behind `unix`, so on Windows `state` was never mutated, triggering `-D unused-mut`. Split into two cfg branches so each path only declares what it needs. Co-authored-by: Claude Opus 4.6 (1M context) --- lib/vector-top/src/metrics.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/vector-top/src/metrics.rs b/lib/vector-top/src/metrics.rs index 63328a62825f0..75aae4456fce6 100644 --- a/lib/vector-top/src/metrics.rs +++ b/lib/vector-top/src/metrics.rs @@ -569,8 +569,6 @@ pub async fn init_components( }) .collect::>(); - let mut state = state::State::new(rows); - #[cfg(feature = "allocation-tracing")] { // Allocation tracing is a compile-time + startup-time setting on the @@ -578,12 +576,15 @@ pub async fn init_components( // (e.g. older server without this RPC) we default to false, matching // pre-existing behavior. This is re-evaluated on every reconnect via // the retry loop in `subscription()`. + let mut state = state::State::new(rows); state.allocation_tracing_active = client .get_allocation_tracing_status() .await .map(|r| r.enabled) .unwrap_or(false); + Ok(state) } - Ok(state) + #[cfg(not(feature = "allocation-tracing"))] + Ok(state::State::new(rows)) } From 711b039fe7a8ddac2296dfe1e78685bb901795e6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 8 Apr 2026 13:30:51 -0400 Subject: [PATCH 110/326] chore(website deps): bump lodash from 4.17.23 to 4.18.1 in /website (#25113) Bumps [lodash](https://github.com/lodash/lodash) from 4.17.23 to 4.18.1. - [Release notes](https://github.com/lodash/lodash/releases) - [Commits](https://github.com/lodash/lodash/compare/4.17.23...4.18.1) --- updated-dependencies: - dependency-name: lodash dependency-version: 4.18.1 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- website/yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/website/yarn.lock b/website/yarn.lock index da88eb1b5980f..055e51daa030a 100644 --- a/website/yarn.lock +++ b/website/yarn.lock @@ -2815,9 +2815,9 @@ lodash.uniq@^4.5.0: integrity sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ== lodash@^4.17.21: - version "4.17.23" - resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.23.tgz#f113b0378386103be4f6893388c73d0bde7f2c5a" - integrity sha512-LgVTMpQtIopCi79SJeDiP0TfWi5CNEc/L/aRdTh3yIvmZXTnheWpKjSZhnvMl8iXbC1tFg9gdHHDMLoV7CnG+w== + version "4.18.1" + resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.18.1.tgz#ff2b66c1f6326d59513de2407bf881439812771c" + integrity sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q== logform@^2.6.0, logform@^2.6.1: version "2.6.1" From dcf98a7ccf54080a74626e9277ac5b2f8340ded8 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Wed, 8 Apr 2026 13:37:57 -0400 Subject: [PATCH 111/326] chore: fix spell check CI failures (#25144) chore: fix spell check failures for gzip'd and schema'd Replace `gzip'd` with `gzip'ed` (already in the allow list) and `schema'd` with `schema-based` to fix the Check Spelling CI job. Co-authored-by: Claude Opus 4.6 (1M context) --- rfcs/2021-01-08-5843-encoding-decoding-format-vector.md | 2 +- website/content/en/guides/aws/cloudwatch-logs-firehose.md | 2 +- website/cue/reference/releases/0.28.0.cue | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/rfcs/2021-01-08-5843-encoding-decoding-format-vector.md b/rfcs/2021-01-08-5843-encoding-decoding-format-vector.md index 9687c50e06992..59bfd92566555 100644 --- a/rfcs/2021-01-08-5843-encoding-decoding-format-vector.md +++ b/rfcs/2021-01-08-5843-encoding-decoding-format-vector.md @@ -248,7 +248,7 @@ Per the data format itself - I haven't found any single conglomerate conclusion ## Drawbacks -The only drawback to adopting protobuf as the encoding format is that Protobufs can be slightly slower than other schema'd data formats and we don't shed the protobuf tooling overhead. +The only drawback to adopting protobuf as the encoding format is that Protobufs can be slightly slower than other schema-based data formats and we don't shed the protobuf tooling overhead. Drawbacks to adopting HTTP/2 are that (like literally any other transport we could pick) it alters our kubernetes deployment handling and if we don't decide to continue to maintain other transports it means needing to appropriately handle the deprecation and removal of our TCP and HTTP/1.1 implementations fairly soon afterwards. diff --git a/website/content/en/guides/aws/cloudwatch-logs-firehose.md b/website/content/en/guides/aws/cloudwatch-logs-firehose.md index 8323be561df9c..02c4e48b40146 100644 --- a/website/content/en/guides/aws/cloudwatch-logs-firehose.md +++ b/website/content/en/guides/aws/cloudwatch-logs-firehose.md @@ -457,7 +457,7 @@ The [`aws_kinesis_firehose`][aws_kinesis_firehose] source: access_key = "my secret key" # this will also be set in the Firehose config ``` -will accept this request, decode the record (which is gzip'd and then base64 encoded), to produce an event that looks like: +will accept this request, decode the record (which is gzip'ed and then base64 encoded), to produce an event that looks like: ```json { diff --git a/website/cue/reference/releases/0.28.0.cue b/website/cue/reference/releases/0.28.0.cue index 91032cae68694..7d44ba72e215b 100644 --- a/website/cue/reference/releases/0.28.0.cue +++ b/website/cue/reference/releases/0.28.0.cue @@ -144,7 +144,7 @@ releases: "0.28.0": { type: "feat" scopes: ["vrl: stdlib"] description: """ - `encode_gzip` and `decode_gzip` functions were added to VRL to interact with gzip'd + `encode_gzip` and `decode_gzip` functions were added to VRL to interact with gzip'ed data. """ contributors: ["zamazan4ik"] From 30d9a58669556a9bc014ba0b2ea4f33f8e372d98 Mon Sep 17 00:00:00 2001 From: Jed Laundry Date: Thu, 9 Apr 2026 05:43:50 +1200 Subject: [PATCH 112/326] feat(azure_blob sink): Expand support for Azure authentication types (#24729) * start by moving auth config to azure_common * add auth to azure_blob Signed-off-by: Jed Laundry * initial auth switch Signed-off-by: Jed Laundry * integration test WIP Signed-off-by: Jed Laundry * doc updates Signed-off-by: Jed Laundry * add feature changelog Signed-off-by: Jed Laundry * clippy changes Signed-off-by: Jed Laundry * add account_name and blob_endpoint options Signed-off-by: Jed Laundry * remove block_in_place * add ClientCertificateCredential Signed-off-by: Jed Laundry * remake certificate with required attributes * refactor integration tests to support both connection string and oauth Signed-off-by: Jed Laundry * update doc examples Signed-off-by: Jed Laundry * make clippy happy Signed-off-by: Jed Laundry * fix tests Signed-off-by: Jed Laundry * make "client_secret_credential" explicitly configured Signed-off-by: Jed Laundry * add azure_logs_ingestion to Azure integration tests Signed-off-by: Jed Laundry * Err out the other TlsConfig options Signed-off-by: Jed Laundry * Add warning message to connection_string Signed-off-by: Jed Laundry * Simplify TlsConfig Signed-off-by: Jed Laundry * add workload identity config options * Move warning into docs::warnings * Properly regenerate cue file * Fix spaces * tidy options creation Signed-off-by: Jed Laundry * add UAMI type option Signed-off-by: Jed Laundry * add note about breaking change * propagate credential errors instead of panicking * automatic Cargo lock change Signed-off-by: Jed Laundry * clarify PEM cert file Signed-off-by: Jed Laundry * action test fixes Signed-off-by: Jed Laundry * allow sinks-azure_logs_ingestion to build without sinks-azure_blob Signed-off-by: Jed Laundry * require auth when using account_name or blob_endpoint Signed-off-by: Jed Laundry * add dep:base64 Signed-off-by: Jed Laundry * check-fmt changes * space diff fix --------- Signed-off-by: Jed Laundry Co-authored-by: Thomas Co-authored-by: Pavlos Rontidis --- .github/actions/spelling/allow.txt | 2 + .github/actions/spelling/expect.txt | 1 + Cargo.lock | 1 + Cargo.toml | 10 +- ...24729_azure_blob_authentication.feature.md | 3 + .../24729_azure_logs_ingestion.breaking.md | 3 + src/sinks/azure_blob/config.rs | 88 ++- src/sinks/azure_blob/integration_tests.rs | 174 ++++-- src/sinks/azure_blob/test.rs | 320 +++++++++++ src/sinks/azure_common/config.rs | 534 +++++++++++++++++- src/sinks/azure_logs_ingestion/config.rs | 200 +------ src/sinks/azure_logs_ingestion/tests.rs | 62 +- src/sinks/mod.rs | 2 +- tests/data/ClientCertificateAuth.pfx | Bin 0 -> 2467 bytes tests/data/Makefile | 21 + .../certs/azurite-chain.cert.pem | 98 ++++ .../certs/azurite.cert.pem | 32 ++ .../intermediate_server/csr/azurite.csr.pem | 18 + tests/data/ca/intermediate_server/index.txt | 1 + .../data/ca/intermediate_server/index.txt.old | 1 + .../ca/intermediate_server/newcerts/1009.pem | 32 ++ .../private/azurite.key.pem | 28 + tests/data/ca/intermediate_server/serial | 2 +- tests/data/ca/intermediate_server/serial.old | 2 +- tests/integration/azure/config/compose.yaml | 8 + tests/integration/azure/config/test.yaml | 3 +- .../components/sinks/generated/azure_blob.cue | 155 ++++- .../sinks/generated/azure_logs_ingestion.cue | 83 ++- 28 files changed, 1542 insertions(+), 342 deletions(-) create mode 100644 changelog.d/24729_azure_blob_authentication.feature.md create mode 100644 changelog.d/24729_azure_logs_ingestion.breaking.md create mode 100644 tests/data/ClientCertificateAuth.pfx create mode 100644 tests/data/ca/intermediate_server/certs/azurite-chain.cert.pem create mode 100644 tests/data/ca/intermediate_server/certs/azurite.cert.pem create mode 100644 tests/data/ca/intermediate_server/csr/azurite.csr.pem create mode 100644 tests/data/ca/intermediate_server/newcerts/1009.pem create mode 100644 tests/data/ca/intermediate_server/private/azurite.key.pem diff --git a/.github/actions/spelling/allow.txt b/.github/actions/spelling/allow.txt index a3f2dae3ceea2..b43abdd6fba66 100644 --- a/.github/actions/spelling/allow.txt +++ b/.github/actions/spelling/allow.txt @@ -210,6 +210,7 @@ greptimedb GSM gvisor gws +gzip'd hadoop Haier Haipad @@ -460,6 +461,7 @@ rumqttc Salesforce Samsung SBT +schema'd scriptblock Sega Segoe diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt index 60715b9c96c0e..ddd945006f1c4 100644 --- a/.github/actions/spelling/expect.txt +++ b/.github/actions/spelling/expect.txt @@ -379,6 +379,7 @@ mycie mycorp mydatabase mylabel +mylogstorage mypod myvalue Namazu diff --git a/Cargo.lock b/Cargo.lock index 76e4c912d09ee..45f4a998276fc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1668,6 +1668,7 @@ dependencies = [ "async-trait", "azure_core", "futures", + "openssl", "pin-project", "serde", "time", diff --git a/Cargo.toml b/Cargo.toml index 21c40c793a9bf..25ca1f9e0dec9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -302,7 +302,7 @@ aws-smithy-types = { version = "1.2.11", default-features = false, features = [" # Azure azure_core = { version = "0.30", features = ["reqwest", "hmac_openssl"], optional = true } -azure_identity = { version = "0.30", optional = true } +azure_identity = { version = "0.30", features = ["client_certificate"], optional = true } # Azure Storage azure_storage_blob = { version = "0.7", optional = true } @@ -875,8 +875,8 @@ sinks-aws_s3 = ["dep:base64", "dep:md-5", "aws-core", "dep:aws-sdk-s3"] sinks-aws_sqs = ["aws-core", "dep:aws-sdk-sqs"] sinks-aws_sns = ["aws-core", "dep:aws-sdk-sns"] sinks-axiom = ["sinks-http"] -sinks-azure_blob = ["dep:azure_core", "dep:azure_storage_blob"] -sinks-azure_logs_ingestion = ["dep:azure_core", "dep:azure_identity"] +sinks-azure_blob = ["dep:azure_core", "dep:azure_identity", "dep:azure_storage_blob", "dep:base64"] +sinks-azure_logs_ingestion = ["dep:azure_core", "dep:azure_identity", "dep:azure_storage_blob", "dep:base64"] sinks-azure_monitor_logs = [] sinks-blackhole = [] sinks-chronicle = [] @@ -987,7 +987,8 @@ aws-integration-tests = [ ] azure-integration-tests = [ - "azure-blob-integration-tests" + "azure-blob-integration-tests", + "azure-logs-ingestion-integration-tests", ] aws-cloudwatch-logs-integration-tests = ["sinks-aws_cloudwatch_logs"] @@ -1001,6 +1002,7 @@ aws-sqs-integration-tests = ["sinks-aws_sqs"] aws-sns-integration-tests = ["sinks-aws_sns"] axiom-integration-tests = ["sinks-axiom"] azure-blob-integration-tests = ["sinks-azure_blob"] +azure-logs-ingestion-integration-tests = ["sinks-azure_logs_ingestion"] chronicle-integration-tests = ["sinks-gcp"] clickhouse-integration-tests = ["sinks-clickhouse"] databend-integration-tests = ["sinks-databend"] diff --git a/changelog.d/24729_azure_blob_authentication.feature.md b/changelog.d/24729_azure_blob_authentication.feature.md new file mode 100644 index 0000000000000..7b224172563b7 --- /dev/null +++ b/changelog.d/24729_azure_blob_authentication.feature.md @@ -0,0 +1,3 @@ +Re-introduced Azure authentication support to `azure_blob`, including Azure CLI, Managed Identity, Workload Identity, and Managed Identity-based Client Assertion authentication types. + +authors: jlaundry diff --git a/changelog.d/24729_azure_logs_ingestion.breaking.md b/changelog.d/24729_azure_logs_ingestion.breaking.md new file mode 100644 index 0000000000000..ce0230a352f38 --- /dev/null +++ b/changelog.d/24729_azure_logs_ingestion.breaking.md @@ -0,0 +1,3 @@ +If using the `azure_logs_ingestion` sink (added in Vector 0.54.0) with Client Secret credentials, add `azure_credential_kind = "client_secret_credential"` to your sink config (this was previously the default, and now must be explicitly configured). + +authors: jlaundry diff --git a/src/sinks/azure_blob/config.rs b/src/sinks/azure_blob/config.rs index db0521b4fa137..1311e09edd4f7 100644 --- a/src/sinks/azure_blob/config.rs +++ b/src/sinks/azure_blob/config.rs @@ -16,7 +16,8 @@ use crate::{ sinks::{ Healthcheck, VectorSink, azure_common::{ - self, config::AzureBlobRetryLogic, service::AzureBlobService, sink::AzureBlobSink, + self, config::AzureAuthentication, config::AzureBlobRetryLogic, + config::AzureBlobTlsConfig, service::AzureBlobService, sink::AzureBlobSink, }, util::{ BatchConfig, BulkSizeBasedDefaultBatchSettings, Compression, ServiceBuilderExt, @@ -41,6 +42,10 @@ impl TowerRequestConfigDefaults for AzureBlobTowerRequestConfigDefaults { #[derive(Clone, Debug)] #[serde(deny_unknown_fields)] pub struct AzureBlobSinkConfig { + #[configurable(derived)] + #[serde(default)] + pub auth: Option, + /// The Azure Blob Storage Account connection string. /// /// Authentication with an access key or shared access signature (SAS) @@ -55,13 +60,33 @@ pub struct AzureBlobSinkConfig { /// | Allowed services | Blob | /// | Allowed resource types | Container & Object | /// | Allowed permissions | Read & Create | + #[configurable(metadata( + docs::warnings = "Access keys and SAS tokens can be used to gain unauthorized access to Azure Blob Storage \ + resources. Numerous security breaches have occurred due to leaked connection strings. It is important to keep \ + connection strings secure and not expose them in logs, error messages, or version control systems." + ))] #[configurable(metadata( docs::examples = "DefaultEndpointsProtocol=https;AccountName=mylogstorage;AccountKey=storageaccountkeybase64encoded;EndpointSuffix=core.windows.net" ))] #[configurable(metadata( docs::examples = "BlobEndpoint=https://mylogstorage.blob.core.windows.net/;SharedAccessSignature=generatedsastoken" ))] - pub connection_string: SensitiveString, + #[configurable(metadata(docs::examples = "AccountName=mylogstorage"))] + pub connection_string: Option, + + /// The Azure Blob Storage Account name. + /// + /// If provided, this will be used instead of the `connection_string`. + /// This is useful for authenticating with an Azure credential. + #[configurable(metadata(docs::examples = "mylogstorage"))] + pub(super) account_name: Option, + + /// The Azure Blob Storage endpoint. + /// + /// If provided, this will be used instead of the `connection_string`. + /// This is useful for authenticating with an Azure credential. + #[configurable(metadata(docs::examples = "https://mylogstorage.blob.core.windows.net/"))] + pub(super) blob_endpoint: Option, /// The Azure Blob Storage Account container name. #[configurable(metadata(docs::examples = "my-logs"))] @@ -138,6 +163,10 @@ pub struct AzureBlobSinkConfig { skip_serializing_if = "crate::serde::is_default" )] pub(super) acknowledgements: AcknowledgementsConfig, + + #[configurable(derived)] + #[serde(default)] + pub tls: Option, } pub fn default_blob_prefix() -> Template { @@ -147,7 +176,10 @@ pub fn default_blob_prefix() -> Template { impl GenerateConfig for AzureBlobSinkConfig { fn generate_config() -> toml::Value { toml::Value::try_from(Self { - connection_string: String::from("DefaultEndpointsProtocol=https;AccountName=some-account-name;AccountKey=some-account-key;").into(), + auth: None, + connection_string: Some(String::from("DefaultEndpointsProtocol=https;AccountName=some-account-name;AccountKey=some-account-key;").into()), + account_name: None, + blob_endpoint: None, container_name: String::from("logs"), blob_prefix: default_blob_prefix(), blob_time_format: Some(String::from("%s")), @@ -157,6 +189,7 @@ impl GenerateConfig for AzureBlobSinkConfig { batch: BatchConfig::default(), request: TowerRequestConfig::default(), acknowledgements: Default::default(), + tls: None, }) .unwrap() } @@ -166,11 +199,56 @@ impl GenerateConfig for AzureBlobSinkConfig { #[typetag::serde(name = "azure_blob")] impl SinkConfig for AzureBlobSinkConfig { async fn build(&self, cx: SinkContext) -> Result<(VectorSink, Healthcheck)> { + let connection_string: String = match ( + &self.connection_string, + &self.account_name, + &self.blob_endpoint, + ) { + (Some(connstr), None, None) => connstr.inner().into(), + (None, Some(account_name), None) => { + if self.auth.is_none() { + return Err( + "`auth` configuration must be provided when using `account_name`".into(), + ); + } + format!("AccountName={}", account_name) + } + (None, None, Some(blob_endpoint)) => { + if self.auth.is_none() { + return Err( + "`auth` configuration must be provided when using `blob_endpoint`".into(), + ); + } + // BlobEndpoint must always end in a trailing slash + let blob_endpoint = if blob_endpoint.ends_with('/') { + blob_endpoint.clone() + } else { + format!("{}/", blob_endpoint) + }; + format!("BlobEndpoint={}", blob_endpoint) + } + (None, None, None) => { + return Err("One of `connection_string`, `account_name`, or `blob_endpoint` must be provided".into()); + } + (Some(_), Some(_), _) => { + return Err("Cannot provide both `connection_string` and `account_name`".into()); + } + (Some(_), _, Some(_)) => { + return Err("Cannot provide both `connection_string` and `blob_endpoint`".into()); + } + (_, Some(_), Some(_)) => { + return Err("Cannot provide both `account_name` and `blob_endpoint`".into()); + } + }; + let client = azure_common::config::build_client( - self.connection_string.clone().into(), + self.auth.clone(), + connection_string.clone(), self.container_name.clone(), cx.proxy(), - )?; + self.tls.clone(), + ) + .await?; let healthcheck = azure_common::config::build_healthcheck( self.container_name.clone(), diff --git a/src/sinks/azure_blob/integration_tests.rs b/src/sinks/azure_blob/integration_tests.rs index 63943312fe66d..dbe20a884ff86 100644 --- a/src/sinks/azure_blob/integration_tests.rs +++ b/src/sinks/azure_blob/integration_tests.rs @@ -1,6 +1,8 @@ use std::io::{BufRead, BufReader}; +use std::sync::Arc; use azure_core::http::StatusCode; +use azure_storage_blob::BlobContainerClient; use bytes::{Buf, BytesMut}; use flate2::read::GzDecoder; @@ -24,17 +26,24 @@ use crate::{ components::{SINK_TAGS, assert_sink_compliance}, random_events_with_stream, random_lines, random_lines_with_stream, random_string, }, + tls, }; #[tokio::test] async fn azure_blob_healthcheck_passed() { let config = AzureBlobSinkConfig::new_emulator().await; - let client = azure_common::config::build_client( - config.connection_string.clone().into(), - config.container_name.clone(), - &crate::config::ProxyConfig::default(), - ) - .expect("Failed to create client"); + let client = config.build_test_client().await; + + azure_common::config::build_healthcheck(config.container_name, client) + .expect("Failed to build healthcheck") + .await + .expect("Failed to pass healthcheck"); +} + +#[tokio::test] +async fn azure_blob_healthcheck_passed_with_oauth() { + let config = AzureBlobSinkConfig::new_emulator_with_oauth().await; + let client = config.build_test_client().await; azure_common::config::build_healthcheck(config.container_name, client) .expect("Failed to build healthcheck") @@ -49,12 +58,7 @@ async fn azure_blob_healthcheck_unknown_container() { container_name: String::from("other-container-name"), ..config }; - let client = azure_common::config::build_client( - config.connection_string.clone().into(), - config.container_name.clone(), - &crate::config::ProxyConfig::default(), - ) - .expect("Failed to create client"); + let client = config.build_test_client().await; assert_eq!( azure_common::config::build_healthcheck(config.container_name, client) @@ -66,10 +70,8 @@ async fn azure_blob_healthcheck_unknown_container() { ); } -#[tokio::test] -async fn azure_blob_insert_lines_into_blob() { +async fn assert_insert_lines_into_blob(config: AzureBlobSinkConfig) { let blob_prefix = format!("lines/into/blob/{}", random_string(10)); - let config = AzureBlobSinkConfig::new_emulator().await; let config = AzureBlobSinkConfig { blob_prefix: blob_prefix.clone().try_into().unwrap(), ..config @@ -88,9 +90,17 @@ async fn azure_blob_insert_lines_into_blob() { } #[tokio::test] -async fn azure_blob_insert_json_into_blob() { +async fn azure_blob_insert_lines_into_blob() { + assert_insert_lines_into_blob(AzureBlobSinkConfig::new_emulator().await).await; +} + +#[tokio::test] +async fn azure_blob_insert_lines_into_blob_with_oauth() { + assert_insert_lines_into_blob(AzureBlobSinkConfig::new_emulator_with_oauth().await).await; +} + +async fn assert_insert_json_into_blob(config: AzureBlobSinkConfig) { let blob_prefix = format!("json/into/blob/{}", random_string(10)); - let config = AzureBlobSinkConfig::new_emulator().await; let config = AzureBlobSinkConfig { blob_prefix: blob_prefix.clone().try_into().unwrap(), encoding: ( @@ -117,6 +127,16 @@ async fn azure_blob_insert_json_into_blob() { assert_eq!(expected, blob_lines); } +#[tokio::test] +async fn azure_blob_insert_json_into_blob() { + assert_insert_json_into_blob(AzureBlobSinkConfig::new_emulator().await).await; +} + +#[tokio::test] +async fn azure_blob_insert_json_into_blob_with_oauth() { + assert_insert_json_into_blob(AzureBlobSinkConfig::new_emulator_with_oauth().await).await; +} + #[ignore] #[tokio::test] // This test fails to get the posted blob with "header not found content-length". @@ -177,14 +197,12 @@ async fn azure_blob_insert_json_into_blob_gzip() { assert_eq!(expected, blob_lines); } -#[tokio::test] -async fn azure_blob_rotate_files_after_the_buffer_size_is_reached() { +async fn assert_rotate_files_after_the_buffer_size_is_reached(mut config: AzureBlobSinkConfig) { let groups = 3; let (lines, size, input) = random_lines_with_stream_with_group_key(100, 30, groups); let size_per_group = (size / groups) + 10; let blob_prefix = format!("lines-rotate/into/blob/{}", random_string(10)); - let mut config = AzureBlobSinkConfig::new_emulator().await; config.batch.max_bytes = Some(size_per_group); let config = AzureBlobSinkConfig { @@ -211,52 +229,104 @@ async fn azure_blob_rotate_files_after_the_buffer_size_is_reached() { } } +#[tokio::test] +async fn azure_blob_rotate_files_after_the_buffer_size_is_reached() { + assert_rotate_files_after_the_buffer_size_is_reached(AzureBlobSinkConfig::new_emulator().await) + .await; +} + +#[tokio::test] +async fn azure_blob_rotate_files_after_the_buffer_size_is_reached_with_oauth() { + assert_rotate_files_after_the_buffer_size_is_reached( + AzureBlobSinkConfig::new_emulator_with_oauth().await, + ) + .await; +} + impl AzureBlobSinkConfig { pub async fn new_emulator() -> AzureBlobSinkConfig { - let address = std::env::var("AZURE_ADDRESS").unwrap_or_else(|_| "localhost".into()); + let address = std::env::var("AZURITE_ADDRESS").unwrap_or_else(|_| "localhost".into()); let config = AzureBlobSinkConfig { - connection_string: format!("UseDevelopmentStorage=true;DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://{address}:10000/devstoreaccount1;QueueEndpoint=http://{address}:10001/devstoreaccount1;TableEndpoint=http://{address}:10002/devstoreaccount1;").into(), - container_name: "logs".to_string(), - blob_prefix: Default::default(), - blob_time_format: None, - blob_append_uuid: None, - encoding: (None::, TextSerializerConfig::default()).into(), - compression: Compression::None, - batch: Default::default(), - request: TowerRequestConfig::default(), - acknowledgements: Default::default(), - }; + auth: None, + connection_string: Some(format!("UseDevelopmentStorage=true;DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://{address}:10000/devstoreaccount1;QueueEndpoint=http://{address}:10001/devstoreaccount1;TableEndpoint=http://{address}:10002/devstoreaccount1;").into()), + account_name: None, + blob_endpoint: None, + container_name: "logs".to_string(), + blob_prefix: Default::default(), + blob_time_format: None, + blob_append_uuid: None, + encoding: (None::, TextSerializerConfig::default()).into(), + compression: Compression::None, + batch: Default::default(), + request: TowerRequestConfig::default(), + acknowledgements: Default::default(), + tls: None, + }; config.ensure_container().await; config } - fn to_sink(&self) -> VectorSink { - let client = azure_common::config::build_client( - self.connection_string.clone().into(), + pub async fn new_emulator_with_oauth() -> AzureBlobSinkConfig { + let address = std::env::var("AZURITE_OAUTH_ADDRESS").unwrap_or_else(|_| "localhost".into()); + let config = AzureBlobSinkConfig { + auth: Some(azure_common::config::AzureAuthentication::MockCredential), + connection_string: Some(format!("DefaultEndpointsProtocol=https;AccountName=devstoreaccount1;BlobEndpoint=https://{address}:14430/devstoreaccount1;QueueEndpoint=https://{address}:14431/devstoreaccount1;TableEndpoint=https://{address}:14432/devstoreaccount1;").into()), + account_name: None, + blob_endpoint: None, + container_name: "logs".to_string(), + blob_prefix: Default::default(), + blob_time_format: None, + blob_append_uuid: None, + encoding: (None::, TextSerializerConfig::default()).into(), + compression: Compression::None, + batch: Default::default(), + request: TowerRequestConfig::default(), + acknowledgements: Default::default(), + tls: Some(azure_common::config::AzureBlobTlsConfig { + ca_file: Some(tls::TEST_PEM_CA_PATH.into()), + }), + }; + + config.ensure_container().await; + + config + } + + async fn build_test_client(&self) -> Arc { + azure_common::config::build_client( + self.auth.clone(), + self.connection_string + .clone() + .expect("failed to unwrap connection_string") + .inner() + .to_string(), self.container_name.clone(), &crate::config::ProxyConfig::default(), + self.tls.clone(), ) - .expect("Failed to create client"); + .await + .expect("Failed to create client") + } + async fn to_sink(&self) -> VectorSink { + let client = self.build_test_client().await; self.build_processor(client).expect("Failed to create sink") } async fn run_assert(&self, input: impl Stream + Send) { // `to_sink` needs to be inside the assertion check - assert_sink_compliance(&SINK_TAGS, async move { self.to_sink().run(input).await }) - .await - .expect("Running sink failed"); + assert_sink_compliance( + &SINK_TAGS, + async move { self.to_sink().await.run(input).await }, + ) + .await + .expect("Running sink failed"); } pub async fn list_blobs(&self, prefix: String) -> Vec { - let client = azure_common::config::build_client( - self.connection_string.clone().into(), - self.container_name.clone(), - &crate::config::ProxyConfig::default(), - ) - .unwrap(); + let client = self.build_test_client().await; // Iterate pager results and collect blob names. Filter by prefix server-side. let mut pager = client @@ -276,12 +346,7 @@ impl AzureBlobSinkConfig { } pub async fn get_blob(&self, blob: String) -> (Option, Option, Vec) { - let client = azure_common::config::build_client( - self.connection_string.clone().into(), - self.container_name.clone(), - &crate::config::ProxyConfig::default(), - ) - .unwrap(); + let client = self.build_test_client().await; let blob_client = client.blob_client(&blob); @@ -337,12 +402,7 @@ impl AzureBlobSinkConfig { } async fn ensure_container(&self) { - let client = azure_common::config::build_client( - self.connection_string.clone().into(), - self.container_name.clone(), - &crate::config::ProxyConfig::default(), - ) - .unwrap(); + let client = self.build_test_client().await; let result = client.create_container(None).await; let response = match result { diff --git a/src/sinks/azure_blob/test.rs b/src/sinks/azure_blob/test.rs index b7f1a9689229f..2dd6aaf69bdd4 100644 --- a/src/sinks/azure_blob/test.rs +++ b/src/sinks/azure_blob/test.rs @@ -14,6 +14,8 @@ use super::{config::AzureBlobSinkConfig, request_builder::AzureBlobRequestOption use crate::{ codecs::{Encoder, EncodingConfigWithFraming}, event::{Event, LogEvent}, + sinks::azure_common::config::{AzureAuthentication, SpecificAzureCredential}, + sinks::prelude::*, sinks::util::{ Compression, request_builder::{EncodeResult, RequestBuilder}, @@ -22,7 +24,10 @@ use crate::{ fn default_config(encoding: EncodingConfigWithFraming) -> AzureBlobSinkConfig { AzureBlobSinkConfig { + auth: Default::default(), connection_string: Default::default(), + account_name: Default::default(), + blob_endpoint: Default::default(), container_name: Default::default(), blob_prefix: Default::default(), blob_time_format: Default::default(), @@ -32,6 +37,7 @@ fn default_config(encoding: EncodingConfigWithFraming) -> AzureBlobSinkConfig { batch: Default::default(), request: Default::default(), acknowledgements: Default::default(), + tls: Default::default(), } } @@ -234,3 +240,317 @@ fn azure_blob_build_request_with_uuid() { assert_eq!(request.content_encoding, None); assert_eq!(request.content_type, "text/plain"); } + +#[tokio::test] +async fn azure_blob_build_config_with_null_auth() { + let config: Result = toml::from_str::( + r#" + connection_string = "AccountName=mylogstorage" + container_name = "my-logs" + + [encoding] + codec = "json" + + [auth] + "#, + ); + + match config { + Ok(_) => panic!("Config parsing should have failed due to invalid auth config"), + Err(e) => { + let err_str = e.to_string(); + assert!( + err_str.contains("data did not match any variant of untagged enum"), + "Config parsing did not complain about invalid auth config: {}", + err_str + ); + } + } +} + +#[tokio::test] +async fn azure_blob_build_config_with_client_id_and_secret() { + let config: AzureBlobSinkConfig = toml::from_str::( + r#" + connection_string = "AccountName=mylogstorage" + container_name = "my-logs" + + [encoding] + codec = "json" + + [auth] + azure_credential_kind = "client_secret_credential" + azure_tenant_id = "00000000-0000-0000-0000-000000000000" + azure_client_id = "mock-client-id" + azure_client_secret = "mock-client-secret" + "#, + ) + .unwrap_or_else(|error| panic!("Config parsing failed: {error:?}")); + + assert!(&config.auth.is_some()); + + match &config.auth.clone().unwrap() { + AzureAuthentication::Specific(SpecificAzureCredential::ClientSecretCredential { + azure_tenant_id, + azure_client_id, + azure_client_secret, + }) => { + assert_eq!(azure_tenant_id, "00000000-0000-0000-0000-000000000000"); + assert_eq!(azure_client_id, "mock-client-id"); + let secret: String = azure_client_secret.inner().into(); + assert_eq!(secret, "mock-client-secret"); + } + _ => panic!("Expected Specific(ClientSecretCredential) variant"), + } + + let cx = SinkContext::default(); + let _sink = config + .build(cx) + .await + .unwrap_or_else(|error| panic!("Failed to build sink: {error:?}")); +} + +#[tokio::test] +async fn azure_blob_build_config_with_client_certificate() { + let config: AzureBlobSinkConfig = toml::from_str::( + r#" + connection_string = "AccountName=mylogstorage" + container_name = "my-logs" + + [encoding] + codec = "json" + + [auth] + azure_credential_kind = "client_certificate_credential" + azure_tenant_id = "00000000-0000-0000-0000-000000000000" + azure_client_id = "mock-client-id" + certificate_path = "tests/data/ClientCertificateAuth.pfx" + certificate_password = "MockPassword123" + "#, + ) + .unwrap_or_else(|error| panic!("Config parsing failed: {error:?}")); + + assert!(&config.auth.is_some()); + + match &config.auth.clone().unwrap() { + AzureAuthentication::Specific(SpecificAzureCredential::ClientCertificateCredential { + .. + }) => { + // Expected variant + } + _ => panic!("Expected Specific(ClientCertificateCredential) variant"), + } + + let cx = SinkContext::default(); + let _sink = config + .build(cx) + .await + .unwrap_or_else(|error| panic!("Failed to build sink: {error:?}")); +} + +#[tokio::test] +async fn azure_blob_build_config_with_account_name() { + let config: AzureBlobSinkConfig = toml::from_str::( + r#" + account_name = "mylogstorage" + container_name = "my-logs" + + [encoding] + codec = "json" + + [auth] + azure_credential_kind = "client_secret_credential" + azure_tenant_id = "00000000-0000-0000-0000-000000000000" + azure_client_id = "mock-client-id" + azure_client_secret = "mock-client-secret" + "#, + ) + .unwrap_or_else(|error| panic!("Config parsing failed: {error:?}")); + + let cx = SinkContext::default(); + let _ = config + .build(cx) + .await + .unwrap_or_else(|error| panic!("Failed to build sink: {error:?}")); +} + +#[tokio::test] +async fn azure_blob_build_config_with_account_name_with_no_auth() { + let config: AzureBlobSinkConfig = toml::from_str::( + r#" + account_name = "mylogstorage" + container_name = "my-logs" + + [encoding] + codec = "json" + "#, + ) + .unwrap_or_else(|error| panic!("Config parsing failed: {error:?}")); + + let cx = SinkContext::default(); + let sink = config.build(cx).await; + match sink { + Ok(_) => panic!("Config build should have errored due to missing `auth`"), + Err(e) => { + let err_str = e.to_string(); + assert!( + err_str.contains("`auth` configuration must be provided"), + "Config build did not complain about missing `auth`: {}", + err_str + ); + } + } +} + +#[tokio::test] +async fn azure_blob_build_config_with_blob_endpoint() { + let config: AzureBlobSinkConfig = toml::from_str::( + r#" + blob_endpoint = "https://localhost:10000/devstoreaccount1" + container_name = "my-logs" + + [encoding] + codec = "json" + + [auth] + azure_credential_kind = "client_secret_credential" + azure_tenant_id = "00000000-0000-0000-0000-000000000000" + azure_client_id = "mock-client-id" + azure_client_secret = "mock-client-secret" + "#, + ) + .unwrap_or_else(|error| panic!("Config parsing failed: {error:?}")); + + let cx = SinkContext::default(); + let _ = config + .build(cx) + .await + .unwrap_or_else(|error| panic!("Failed to build sink: {error:?}")); +} + +#[tokio::test] +async fn azure_blob_build_config_with_blob_endpoint_with_no_auth() { + let config: AzureBlobSinkConfig = toml::from_str::( + r#" + blob_endpoint = "https://localhost:10000/devstoreaccount1" + container_name = "my-logs" + + [encoding] + codec = "json" + "#, + ) + .unwrap_or_else(|error| panic!("Config parsing failed: {error:?}")); + + let cx = SinkContext::default(); + let sink = config.build(cx).await; + match sink { + Ok(_) => panic!("Config build should have errored due to missing `auth`"), + Err(e) => { + let err_str = e.to_string(); + assert!( + err_str.contains("`auth` configuration must be provided"), + "Config build did not complain about missing `auth`: {}", + err_str + ); + } + } +} + +#[tokio::test] +async fn azure_blob_build_config_with_conflicting_connection_string_and_account_name() { + let config: AzureBlobSinkConfig = toml::from_str::( + r#" + connection_string = "AccountName=mylogstorage" + account_name = "mylogstorage" + container_name = "my-logs" + + [encoding] + codec = "json" + "#, + ) + .unwrap_or_else(|error| panic!("Config parsing failed: {error:?}")); + + let cx = SinkContext::default(); + let sink = config.build(cx).await; + match sink { + Ok(_) => panic!( + "Config build should have errored due to conflicting connection_string and account_name" + ), + Err(e) => { + let err_str = e.to_string(); + assert!( + err_str.contains("`connection_string` and `account_name`"), + "Config build did not complain about conflicting connection_string and account_name: {}", + err_str + ); + } + } +} + +#[tokio::test] +async fn azure_blob_build_config_with_conflicting_connection_string_and_client_id_and_secret() { + let config: AzureBlobSinkConfig = toml::from_str::( + r#" + connection_string = "AccountName=mylogstorage;AccountKey=mockkey" + container_name = "my-logs" + + [encoding] + codec = "json" + + [auth] + azure_credential_kind = "client_secret_credential" + azure_tenant_id = "00000000-0000-0000-0000-000000000000" + azure_client_id = "mock-client-id" + azure_client_secret = "mock-client-secret" + "#, + ) + .unwrap_or_else(|error| panic!("Config parsing failed: {error:?}")); + + assert!(&config.auth.is_some()); + + let cx = SinkContext::default(); + let sink = config.build(cx).await; + match sink { + Ok(_) => { + panic!("Config build should have errored due to conflicting Shared Key and Client ID") + } + Err(e) => { + let err_str = e.to_string(); + assert!( + err_str + .contains("Cannot use both Shared Key and another Azure Authentication method"), + "Config build did not complain about conflicting Shared Key and Client ID: {}", + err_str + ); + } + } +} + +#[tokio::test] +async fn azure_blob_build_config_with_custom_ca_certificate() { + let config: AzureBlobSinkConfig = toml::from_str::( + r#" + account_name = "mylogstorage" + container_name = "my-logs" + + [encoding] + codec = "json" + + [tls] + ca_file = "tests/data/ca/certs/ca.cert.pem" + + [auth] + azure_credential_kind = "client_secret_credential" + azure_tenant_id = "00000000-0000-0000-0000-000000000000" + azure_client_id = "mock-client-id" + azure_client_secret = "mock-client-secret" + "#, + ) + .unwrap_or_else(|error| panic!("Config parsing failed: {error:?}")); + + let cx = SinkContext::default(); + let _ = config + .build(cx) + .await + .unwrap_or_else(|error| panic!("Failed to build sink: {error:?}")); +} diff --git a/src/sinks/azure_common/config.rs b/src/sinks/azure_common/config.rs index 139608d9847ab..800e0a6eb3f23 100644 --- a/src/sinks/azure_common/config.rs +++ b/src/sinks/azure_common/config.rs @@ -1,19 +1,36 @@ +use std::fs::File; +use std::io::Read; +use std::path::PathBuf; use std::sync::Arc; +use base64::prelude::*; + use azure_core::error::Error as AzureCoreError; use crate::sinks::azure_common::connection_string::{Auth, ParsedConnectionString}; use crate::sinks::azure_common::shared_key_policy::SharedKeyAuthorizationPolicy; -use azure_core::http::Url; +use azure_core::http::{ClientMethodOptions, StatusCode, Url}; + +use azure_core::credentials::{TokenCredential, TokenRequestOptions}; +use azure_core::{Error, error::ErrorKind}; + +use azure_identity::{ + AzureCliCredential, ClientAssertion, ClientAssertionCredential, ClientCertificateCredential, + ClientCertificateCredentialOptions, ClientSecretCredential, ManagedIdentityCredential, + ManagedIdentityCredentialOptions, UserAssignedId, WorkloadIdentityCredential, + WorkloadIdentityCredentialOptions, +}; + use azure_storage_blob::{BlobContainerClient, BlobContainerClientOptions}; -use azure_core::http::StatusCode; use bytes::Bytes; use futures::FutureExt; use snafu::Snafu; use vector_lib::{ + configurable::configurable_component, json_size::JsonSize, request_metadata::{GroupedCountByteSize, MetaDescriptive, RequestMetadata}, + sensitive_string::SensitiveString, stream::DriverResponse, }; @@ -22,6 +39,374 @@ use crate::{ sinks::{Healthcheck, util::retries::RetryLogic}, }; +/// TLS configuration. +#[configurable_component] +#[configurable(metadata(docs::advanced))] +#[derive(Clone, Debug, Default)] +#[serde(deny_unknown_fields)] +pub struct AzureBlobTlsConfig { + /// Absolute path to an additional CA certificate file. + /// + /// The certificate must be in PEM (X.509) format. + #[serde(alias = "ca_path")] + #[configurable(metadata(docs::examples = "/path/to/certificate_authority.crt"))] + #[configurable(metadata(docs::human_name = "CA File Path"))] + pub ca_file: Option, +} + +/// Azure service principal authentication. +#[configurable_component] +#[derive(Clone, Debug, Eq, PartialEq)] +#[serde(deny_unknown_fields, untagged)] +pub enum AzureAuthentication { + #[configurable(metadata(docs::enum_tag_description = "The kind of Azure credential to use."))] + Specific(SpecificAzureCredential), + + /// Mock credential for testing — returns a static fake token + #[cfg(test)] + #[serde(skip)] + MockCredential, +} + +impl Default for AzureAuthentication { + // This should never be actually used. + // This is only needed when using Default::default() (such as unit tests), + // as serde requires `azure_credential_kind` to be specified. + fn default() -> Self { + Self::Specific(SpecificAzureCredential::ManagedIdentity { + user_assigned_managed_identity_id: None, + user_assigned_managed_identity_id_type: None, + }) + } +} + +#[configurable_component] +#[derive(Clone, Debug, Eq, PartialEq)] +#[serde(deny_unknown_fields, rename_all = "snake_case")] +#[derive(Default)] +/// User Assigned Managed Identity Types. +pub enum UserAssignedManagedIdentityIdType { + #[default] + /// Client ID + ClientId, + /// Object ID + ObjectId, + /// Resource ID + ResourceId, +} + +/// Specific Azure credential types. +#[configurable_component] +#[derive(Clone, Debug, Eq, PartialEq)] +#[serde( + tag = "azure_credential_kind", + rename_all = "snake_case", + deny_unknown_fields +)] +pub enum SpecificAzureCredential { + /// Use Azure CLI credentials + #[cfg(not(target_arch = "wasm32"))] + AzureCli {}, + + /// Use certificate credentials + ClientCertificateCredential { + /// The [Azure Tenant ID][azure_tenant_id]. + /// + /// [azure_tenant_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + #[configurable(metadata(docs::examples = "${AZURE_TENANT_ID:?err}"))] + azure_tenant_id: String, + + /// The [Azure Client ID][azure_client_id]. + /// + /// [azure_client_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + #[configurable(metadata(docs::examples = "${AZURE_CLIENT_ID:?err}"))] + azure_client_id: String, + + /// PKCS12 certificate with RSA private key. + #[configurable(metadata(docs::examples = "path/to/certificate.pfx"))] + #[configurable(metadata(docs::examples = "${AZURE_CLIENT_CERTIFICATE_PATH:?err}"))] + certificate_path: PathBuf, + + /// The password for the client certificate, if applicable. + #[configurable(metadata(docs::examples = "${AZURE_CLIENT_CERTIFICATE_PASSWORD}"))] + certificate_password: Option, + }, + + /// Use client ID/secret credentials + ClientSecretCredential { + /// The [Azure Tenant ID][azure_tenant_id]. + /// + /// [azure_tenant_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + #[configurable(metadata(docs::examples = "${AZURE_TENANT_ID:?err}"))] + azure_tenant_id: String, + + /// The [Azure Client ID][azure_client_id]. + /// + /// [azure_client_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + #[configurable(metadata(docs::examples = "${AZURE_CLIENT_ID:?err}"))] + azure_client_id: String, + + /// The [Azure Client Secret][azure_client_secret]. + /// + /// [azure_client_secret]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + #[configurable(metadata(docs::examples = "00-00~000000-0000000~0000000000000000000"))] + #[configurable(metadata(docs::examples = "${AZURE_CLIENT_SECRET:?err}"))] + azure_client_secret: SensitiveString, + }, + + /// Use Managed Identity credentials + ManagedIdentity { + /// The User Assigned Managed Identity to use. + #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + #[serde(default, skip_serializing_if = "Option::is_none")] + user_assigned_managed_identity_id: Option, + + /// The type of the User Assigned Managed Identity ID provided (Client ID, Object ID, + /// or Resource ID). Defaults to Client ID. + user_assigned_managed_identity_id_type: Option, + }, + + /// Use Managed Identity with Client Assertion credentials + ManagedIdentityClientAssertion { + /// The User Assigned Managed Identity to use for the managed identity. + #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + #[configurable(metadata( + docs::examples = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/rg-vector/providers/Microsoft.ManagedIdentity/userAssignedIdentities/id-vector-uami" + ))] + #[serde(default, skip_serializing_if = "Option::is_none")] + user_assigned_managed_identity_id: Option, + + /// The type of the User Assigned Managed Identity ID provided (Client ID, Object ID, or Resource ID). Defaults to Client ID. + user_assigned_managed_identity_id_type: Option, + + /// The target Tenant ID to use. + #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + client_assertion_tenant_id: String, + + /// The target Client ID to use. + #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + client_assertion_client_id: String, + }, + + /// Use Workload Identity credentials + WorkloadIdentity { + /// The [Azure Tenant ID][azure_tenant_id]. Defaults to the value of the environment variable `AZURE_TENANT_ID`. + /// + /// [azure_tenant_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + #[configurable(metadata(docs::examples = "${AZURE_TENANT_ID}"))] + tenant_id: Option, + + /// The [Azure Client ID][azure_client_id]. Defaults to the value of the environment variable `AZURE_CLIENT_ID`. + /// + /// [azure_client_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + #[configurable(metadata(docs::examples = "${AZURE_CLIENT_ID}"))] + client_id: Option, + + /// Path of a file containing a Kubernetes service account token. Defaults to the value of the environment variable `AZURE_FEDERATED_TOKEN_FILE`. + #[configurable(metadata( + docs::examples = "/var/run/secrets/azure/tokens/azure-identity-token" + ))] + #[configurable(metadata(docs::examples = "${AZURE_FEDERATED_TOKEN_FILE}"))] + token_file_path: Option, + }, +} + +#[derive(Debug)] +struct ManagedIdentityClientAssertion { + credential: Arc, + scope: String, +} + +#[cfg_attr(target_arch = "wasm32", async_trait::async_trait(?Send))] +#[cfg_attr(not(target_arch = "wasm32"), async_trait::async_trait)] +impl ClientAssertion for ManagedIdentityClientAssertion { + async fn secret(&self, options: Option>) -> azure_core::Result { + Ok(self + .credential + .get_token( + &[&self.scope], + Some(TokenRequestOptions { + method_options: options.unwrap_or_default(), + }), + ) + .await? + .token + .secret() + .to_string()) + } +} + +impl AzureAuthentication { + /// Returns the provider for the credentials based on the authentication mechanism chosen. + pub async fn credential(&self) -> azure_core::Result> { + match self { + Self::Specific(specific) => specific.credential().await, + + #[cfg(test)] + Self::MockCredential => Ok(Arc::new(MockTokenCredential) as Arc), + } + } +} + +impl SpecificAzureCredential { + /// Returns the provider for the credentials based on the specific credential type. + pub async fn credential(&self) -> azure_core::Result> { + let credential: Arc = match self { + #[cfg(not(target_arch = "wasm32"))] + Self::AzureCli {} => AzureCliCredential::new(None)?, + + // requires azure_identity feature 'client_certificate' + Self::ClientCertificateCredential { + azure_tenant_id, + azure_client_id, + certificate_path, + certificate_password, + } => { + let certificate_bytes: Vec = std::fs::read(certificate_path).map_err(|e| { + Error::with_message( + ErrorKind::Credential, + format!( + "Failed to read certificate file {}: {e}", + certificate_path.display() + ), + ) + })?; + + // Note: in azure_identity 0.33.0+, this changes to SecretBytes, and the base64 encoding is no longer needed + let certificate_base64: azure_core::credentials::Secret = + BASE64_STANDARD.encode(&certificate_bytes).into(); + + let mut options: ClientCertificateCredentialOptions = + ClientCertificateCredentialOptions::default(); + if let Some(password) = certificate_password { + options.password = Some(password.inner().to_string().into()); + } + + ClientCertificateCredential::new( + azure_tenant_id.clone(), + azure_client_id.clone(), + certificate_base64, + Some(options), + )? + } + + Self::ClientSecretCredential { + azure_tenant_id, + azure_client_id, + azure_client_secret, + } => { + if azure_tenant_id.is_empty() { + return Err(Error::with_message(ErrorKind::Credential, + "`auth.azure_tenant_id` is blank; either use `auth.azure_credential_kind`, or provide tenant ID, client ID, and secret.".to_string() + )); + } + if azure_client_id.is_empty() { + return Err(Error::with_message(ErrorKind::Credential, + "`auth.azure_client_id` is blank; either use `auth.azure_credential_kind`, or provide tenant ID, client ID, and secret.".to_string() + )); + } + if azure_client_secret.inner().is_empty() { + return Err(Error::with_message(ErrorKind::Credential, + "`auth.azure_client_secret` is blank; either use `auth.azure_credential_kind`, or provide tenant ID, client ID, and secret.".to_string() + )); + } + + let secret: String = azure_client_secret.inner().into(); + ClientSecretCredential::new( + &azure_tenant_id.clone(), + azure_client_id.clone(), + secret.into(), + None, + )? + } + + Self::ManagedIdentity { + user_assigned_managed_identity_id, + user_assigned_managed_identity_id_type, + } => { + let mut options = ManagedIdentityCredentialOptions::default(); + if let Some(id) = user_assigned_managed_identity_id { + options.user_assigned_id = match user_assigned_managed_identity_id_type + .as_ref() + .unwrap_or(&Default::default()) + { + UserAssignedManagedIdentityIdType::ClientId => { + Some(UserAssignedId::ClientId(id.clone())) + } + UserAssignedManagedIdentityIdType::ObjectId => { + Some(UserAssignedId::ObjectId(id.clone())) + } + UserAssignedManagedIdentityIdType::ResourceId => { + Some(UserAssignedId::ResourceId(id.clone())) + } + }; + } + ManagedIdentityCredential::new(Some(options))? + } + + Self::ManagedIdentityClientAssertion { + user_assigned_managed_identity_id, + user_assigned_managed_identity_id_type, + client_assertion_tenant_id, + client_assertion_client_id, + } => { + let mut options = ManagedIdentityCredentialOptions::default(); + if let Some(id) = user_assigned_managed_identity_id { + options.user_assigned_id = match user_assigned_managed_identity_id_type + .as_ref() + .unwrap_or(&Default::default()) + { + UserAssignedManagedIdentityIdType::ClientId => { + Some(UserAssignedId::ClientId(id.clone())) + } + UserAssignedManagedIdentityIdType::ObjectId => { + Some(UserAssignedId::ObjectId(id.clone())) + } + UserAssignedManagedIdentityIdType::ResourceId => { + Some(UserAssignedId::ResourceId(id.clone())) + } + }; + } + let msi: Arc = ManagedIdentityCredential::new(Some(options))?; + let assertion = ManagedIdentityClientAssertion { + credential: msi, + // Future: make this configurable for sovereign clouds? (no way to test...) + scope: "api://AzureADTokenExchange/.default".to_string(), + }; + + ClientAssertionCredential::new( + client_assertion_tenant_id.clone(), + client_assertion_client_id.clone(), + assertion, + None, + )? + } + + Self::WorkloadIdentity { + tenant_id, + client_id, + token_file_path, + } => { + let options = WorkloadIdentityCredentialOptions { + tenant_id: tenant_id.clone(), + client_id: client_id.clone(), + token_file_path: token_file_path.clone(), + ..Default::default() + }; + + WorkloadIdentityCredential::new(Some(options))? + } + }; + Ok(credential) + } +} + #[derive(Debug, Clone)] pub struct AzureBlobRequest { pub blob_data: Bytes, @@ -126,10 +511,12 @@ pub fn build_healthcheck( Ok(healthcheck.boxed()) } -pub fn build_client( +pub async fn build_client( + auth: Option, connection_string: String, container_name: String, proxy: &crate::config::ProxyConfig, + tls: Option, ) -> crate::Result> { // Parse connection string without legacy SDK let parsed = ParsedConnectionString::parse(&connection_string) @@ -140,16 +527,26 @@ pub fn build_client( .map_err(|e| format!("Failed to build container URL: {e}"))?; let url = Url::parse(&container_url).map_err(|e| format!("Invalid container URL: {e}"))?; + let mut credential: Option> = None; + // Prepare options; attach Shared Key policy if needed let mut options = BlobContainerClientOptions::default(); - match parsed.auth() { - Auth::Sas { .. } | Auth::None => { - // No extra policy; SAS is in the URL already (or anonymous) + match (parsed.auth(), &auth) { + (Auth::None, None) => { + warn!("No authentication method provided, requests will be anonymous."); + } + (Auth::Sas { .. }, None) => { + info!("Using SAS token authentication."); } - Auth::SharedKey { - account_name, - account_key, - } => { + ( + Auth::SharedKey { + account_name, + account_key, + }, + None, + ) => { + info!("Using Shared Key authentication."); + let policy = SharedKeyAuthorizationPolicy::new( account_name, account_key, @@ -162,6 +559,41 @@ pub fn build_client( .per_call_policies .push(Arc::new(policy)); } + (Auth::None, Some(AzureAuthentication::Specific(..))) => { + info!("Using Azure Authentication method."); + let credential_result: Arc = + auth.unwrap().credential().await.map_err(|e| { + Error::with_message( + ErrorKind::Credential, + format!("Failed to configure Azure Authentication: {e}"), + ) + })?; + credential = Some(credential_result); + } + (Auth::Sas { .. }, Some(AzureAuthentication::Specific(..))) => { + return Err(Box::new(Error::with_message( + ErrorKind::Credential, + "Cannot use both SAS token and another Azure Authentication method at the same time", + ))); + } + (Auth::SharedKey { .. }, Some(AzureAuthentication::Specific(..))) => { + return Err(Box::new(Error::with_message( + ErrorKind::Credential, + "Cannot use both Shared Key and another Azure Authentication method at the same time", + ))); + } + #[cfg(test)] + (Auth::None, Some(AzureAuthentication::MockCredential)) => { + warn!("Using mock token credential authentication."); + credential = Some(auth.unwrap().credential().await.unwrap()); + } + #[cfg(test)] + (_, Some(AzureAuthentication::MockCredential)) => { + return Err(Box::new(Error::with_message( + ErrorKind::Credential, + "Cannot use both connection string auth and mock credential at the same time", + ))); + } } // Use reqwest v0.12 since Azure SDK only implements HttpClient for reqwest::Client v0.12 @@ -191,12 +623,90 @@ pub fn build_client( reqwest_builder = reqwest_builder.proxy(p); } } + + if let Some(AzureBlobTlsConfig { ca_file }) = &tls + && let Some(ca_file) = ca_file + { + let mut buf = Vec::new(); + File::open(ca_file)?.read_to_end(&mut buf)?; + let cert = reqwest_12::Certificate::from_pem(&buf)?; + + warn!("Adding TLS root certificate from {}", ca_file.display()); + reqwest_builder = reqwest_builder.add_root_certificate(cert); + } + options.client_options.transport = Some(azure_core::http::Transport::new(std::sync::Arc::new( reqwest_builder .build() .map_err(|e| format!("Failed to build reqwest client: {e}"))?, ))); - let client = - BlobContainerClient::from_url(url, None, Some(options)).map_err(|e| format!("{e}"))?; + let client = BlobContainerClient::from_url(url, credential, Some(options)) + .map_err(|e| format!("{e}"))?; Ok(Arc::new(client)) } + +#[cfg(test)] +#[derive(Debug)] +struct MockTokenCredential; + +#[cfg(test)] +#[async_trait::async_trait] +impl TokenCredential for MockTokenCredential { + async fn get_token( + &self, + scopes: &[&str], + _options: Option>, + ) -> azure_core::Result { + let Some(scope) = scopes.first() else { + return Err(Error::with_message( + ErrorKind::Credential, + "no scopes were provided", + )); + }; + + // serde_json sometimes does and sometimes doesn't preserve order, be careful to sort + // the claims in alphabetical order to ensure a consistent base64 encoding for testing + let jwt = serde_json::json!({ + "aud": scope.strip_suffix("/.default").unwrap_or(*scope), + "exp": 2147483647, + "iat": 0, + "iss": "https://sts.windows.net/", + "nbf": 0, + }); + + // JWTs do not include standard base64 padding. + // this seemed cleaner than importing a new crates just for this function + let jwt_base64 = format!( + "e30.{}.", + BASE64_STANDARD + .encode(serde_json::to_string(&jwt).unwrap()) + .trim_end_matches("=") + ) + .to_string(); + + warn!( + "Using mock token credential, JWT: {}, base64: {}", + serde_json::to_string(&jwt).unwrap(), + jwt_base64 + ); + + Ok(azure_core::credentials::AccessToken::new( + jwt_base64, + azure_core::time::OffsetDateTime::now_utc() + std::time::Duration::from_secs(3600), + )) + } +} + +#[cfg(test)] +#[tokio::test] +async fn azure_mock_token_credential_test() { + let credential = MockTokenCredential; + let access_token = credential + .get_token(&["https://example.com/.default"], None) + .await + .expect("valid credential should return a token"); + assert_eq!( + access_token.token.secret(), + "e30.eyJhdWQiOiJodHRwczovL2V4YW1wbGUuY29tIiwiZXhwIjoyMTQ3NDgzNjQ3LCJpYXQiOjAsImlzcyI6Imh0dHBzOi8vc3RzLndpbmRvd3MubmV0LyIsIm5iZiI6MH0." + ); +} diff --git a/src/sinks/azure_logs_ingestion/config.rs b/src/sinks/azure_logs_ingestion/config.rs index 4e876a18a26bd..3b105d549d0c1 100644 --- a/src/sinks/azure_logs_ingestion/config.rs +++ b/src/sinks/azure_logs_ingestion/config.rs @@ -1,20 +1,14 @@ use std::sync::Arc; -use azure_core::credentials::{TokenCredential, TokenRequestOptions}; -use azure_core::http::ClientMethodOptions; -use azure_core::{Error, error::ErrorKind}; - -use azure_identity::{ - AzureCliCredential, ClientAssertion, ClientAssertionCredential, ClientSecretCredential, - ManagedIdentityCredential, ManagedIdentityCredentialOptions, UserAssignedId, - WorkloadIdentityCredential, -}; -use vector_lib::{configurable::configurable_component, schema, sensitive_string::SensitiveString}; +use azure_core::credentials::TokenCredential; + +use vector_lib::{configurable::configurable_component, schema}; use vrl::value::Kind; use crate::{ http::{HttpClient, get_http_scheme_from_uri}, sinks::{ + azure_common::config::AzureAuthentication, prelude::*, util::{RealtimeSizeBasedDefaultBatchSettings, UriSerde, http::HttpStatusRetryLogic}, }, @@ -65,7 +59,6 @@ pub struct AzureLogsIngestionConfig { pub stream_name: String, #[configurable(derived)] - #[serde(default)] pub auth: AzureAuthentication, /// [Token scope][token_scope] for dedicated Azure regions. @@ -129,191 +122,6 @@ impl Default for AzureLogsIngestionConfig { } } -/// Configuration of the authentication strategy for interacting with Azure services. -#[configurable_component] -#[derive(Clone, Debug, Derivative, Eq, PartialEq)] -#[derivative(Default)] -#[serde(deny_unknown_fields, untagged)] -pub enum AzureAuthentication { - /// Use client credentials - #[derivative(Default)] - ClientSecretCredential { - /// The [Azure Tenant ID][azure_tenant_id]. - /// - /// [azure_tenant_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal - #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] - azure_tenant_id: String, - - /// The [Azure Client ID][azure_client_id]. - /// - /// [azure_client_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal - #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] - azure_client_id: String, - - /// The [Azure Client Secret][azure_client_secret]. - /// - /// [azure_client_secret]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal - #[configurable(metadata(docs::examples = "00-00~000000-0000000~0000000000000000000"))] - azure_client_secret: SensitiveString, - }, - - /// Use credentials from environment variables - #[configurable(metadata(docs::enum_tag_description = "The kind of Azure credential to use."))] - Specific(SpecificAzureCredential), -} - -/// Specific Azure credential types. -#[configurable_component] -#[derive(Clone, Debug, Eq, PartialEq)] -#[serde( - tag = "azure_credential_kind", - rename_all = "snake_case", - deny_unknown_fields -)] -pub enum SpecificAzureCredential { - /// Use Azure CLI credentials - #[cfg(not(target_arch = "wasm32"))] - AzureCli {}, - - /// Use Managed Identity credentials - ManagedIdentity { - /// The User Assigned Managed Identity (Client ID) to use. - #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] - #[serde(default, skip_serializing_if = "Option::is_none")] - user_assigned_managed_identity_id: Option, - }, - - /// Use Managed Identity with Client Assertion credentials - ManagedIdentityClientAssertion { - /// The User Assigned Managed Identity (Client ID) to use for the managed identity. - #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] - #[serde(default, skip_serializing_if = "Option::is_none")] - user_assigned_managed_identity_id: Option, - - /// The target Tenant ID to use. - #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] - client_assertion_tenant_id: String, - - /// The target Client ID to use. - #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] - client_assertion_client_id: String, - }, - - /// Use Workload Identity credentials - WorkloadIdentity {}, -} - -#[derive(Debug)] -struct ManagedIdentityClientAssertion { - credential: Arc, - scope: String, -} - -#[cfg_attr(target_arch = "wasm32", async_trait::async_trait(?Send))] -#[cfg_attr(not(target_arch = "wasm32"), async_trait::async_trait)] -impl ClientAssertion for ManagedIdentityClientAssertion { - async fn secret(&self, options: Option>) -> azure_core::Result { - Ok(self - .credential - .get_token( - &[&self.scope], - Some(TokenRequestOptions { - method_options: options.unwrap_or_default(), - }), - ) - .await? - .token - .secret() - .to_string()) - } -} - -impl AzureAuthentication { - /// Returns the provider for the credentials based on the authentication mechanism chosen. - pub async fn credential(&self) -> azure_core::Result> { - match self { - Self::ClientSecretCredential { - azure_tenant_id, - azure_client_id, - azure_client_secret, - } => { - if azure_tenant_id.is_empty() { - return Err(Error::with_message(ErrorKind::Credential, - "`auth.azure_tenant_id` is blank; either use `auth.azure_credential_kind`, or provide tenant ID, client ID, and secret.".to_string() - )); - } - if azure_client_id.is_empty() { - return Err(Error::with_message(ErrorKind::Credential, - "`auth.azure_client_id` is blank; either use `auth.azure_credential_kind`, or provide tenant ID, client ID, and secret.".to_string() - )); - } - if azure_client_secret.inner().is_empty() { - return Err(Error::with_message(ErrorKind::Credential, - "`auth.azure_client_secret` is blank; either use `auth.azure_credential_kind`, or provide tenant ID, client ID, and secret.".to_string() - )); - } - let secret: String = azure_client_secret.inner().into(); - let credential: Arc = ClientSecretCredential::new( - &azure_tenant_id.clone(), - azure_client_id.clone(), - secret.into(), - None, - )?; - Ok(credential) - } - - Self::Specific(specific) => specific.credential().await, - } - } -} - -impl SpecificAzureCredential { - /// Returns the provider for the credentials based on the specific credential type. - pub async fn credential(&self) -> azure_core::Result> { - let credential: Arc = match self { - #[cfg(not(target_arch = "wasm32"))] - Self::AzureCli {} => AzureCliCredential::new(None)?, - - Self::ManagedIdentity { - user_assigned_managed_identity_id, - } => { - let mut options = ManagedIdentityCredentialOptions::default(); - if let Some(id) = user_assigned_managed_identity_id { - options.user_assigned_id = Some(UserAssignedId::ClientId(id.clone())); - } - ManagedIdentityCredential::new(Some(options))? - } - - Self::ManagedIdentityClientAssertion { - user_assigned_managed_identity_id, - client_assertion_tenant_id, - client_assertion_client_id, - } => { - let mut options = ManagedIdentityCredentialOptions::default(); - if let Some(id) = user_assigned_managed_identity_id { - options.user_assigned_id = Some(UserAssignedId::ClientId(id.clone())); - } - let msi: Arc = ManagedIdentityCredential::new(Some(options))?; - let assertion = ManagedIdentityClientAssertion { - credential: msi, - // Future: make this configurable for sovereign clouds? (no way to test...) - scope: "api://AzureADTokenExchange/.default".to_string(), - }; - - ClientAssertionCredential::new( - client_assertion_tenant_id.clone(), - client_assertion_client_id.clone(), - assertion, - None, - )? - } - - Self::WorkloadIdentity {} => WorkloadIdentityCredential::new(None)?, - }; - Ok(credential) - } -} - impl AzureLogsIngestionConfig { #[allow(clippy::too_many_arguments)] pub(super) async fn build_inner( diff --git a/src/sinks/azure_logs_ingestion/tests.rs b/src/sinks/azure_logs_ingestion/tests.rs index 26eda91338836..46095071bd28c 100644 --- a/src/sinks/azure_logs_ingestion/tests.rs +++ b/src/sinks/azure_logs_ingestion/tests.rs @@ -8,6 +8,8 @@ use vector_lib::config::log_schema; use azure_core::credentials::{AccessToken, TokenCredential}; use azure_core::time::OffsetDateTime; +use crate::sinks::azure_common::config::{AzureAuthentication, SpecificAzureCredential}; + use super::config::AzureLogsIngestionConfig; use crate::{ @@ -26,50 +28,22 @@ fn generate_config() { #[tokio::test] async fn basic_config_error_with_no_auth() { - let config: AzureLogsIngestionConfig = toml::from_str::( - r#" + let config: Result = + toml::from_str::( + r#" endpoint = "https://my-dce-5kyl.eastus-1.ingest.monitor.azure.com" dcr_immutable_id = "dcr-00000000000000000000000000000000" stream_name = "Custom-UnitTest" "#, - ) - .expect("Config parsing failed"); - - assert_eq!( - config.endpoint, - "https://my-dce-5kyl.eastus-1.ingest.monitor.azure.com" - ); - assert_eq!( - config.dcr_immutable_id, - "dcr-00000000000000000000000000000000" - ); - assert_eq!(config.stream_name, "Custom-UnitTest"); - assert_eq!(config.token_scope, "https://monitor.azure.com/.default"); - assert_eq!(config.timestamp_field, "TimeGenerated"); - - match &config.auth { - crate::sinks::azure_logs_ingestion::config::AzureAuthentication::ClientSecretCredential { - azure_tenant_id, - azure_client_id, - azure_client_secret, - } => { - assert_eq!(azure_tenant_id, ""); - assert_eq!(azure_client_id, ""); - let secret: String = azure_client_secret.inner().into(); - assert_eq!(secret, ""); - } - _ => panic!("Expected ClientSecretCredential variant"), - } + ); - let cx = SinkContext::default(); - let sink = config.build(cx).await; - match sink { - Ok(_) => panic!("Config build should have errored due to missing auth info"), + match config { + Ok(_) => panic!("Config parsing should have failed due to missing auth config"), Err(e) => { let err_str = e.to_string(); assert!( - err_str.contains("`auth.azure_tenant_id` is blank"), - "Config build did not complain about azure_tenant_id being blank: {}", + err_str.contains("missing field `auth`"), + "Config parsing did not complain about missing auth field: {}", err_str ); } @@ -85,6 +59,7 @@ fn basic_config_with_client_credentials() { stream_name = "Custom-UnitTest" [auth] + azure_credential_kind = "client_secret_credential" azure_tenant_id = "00000000-0000-0000-0000-000000000000" azure_client_id = "mock-client-id" azure_client_secret = "mock-client-secret" @@ -105,17 +80,17 @@ fn basic_config_with_client_credentials() { assert_eq!(config.timestamp_field, "TimeGenerated"); match &config.auth { - crate::sinks::azure_logs_ingestion::config::AzureAuthentication::ClientSecretCredential { + AzureAuthentication::Specific(SpecificAzureCredential::ClientSecretCredential { azure_tenant_id, azure_client_id, azure_client_secret, - } => { + }) => { assert_eq!(azure_tenant_id, "00000000-0000-0000-0000-000000000000"); assert_eq!(azure_client_id, "mock-client-id"); let secret: String = azure_client_secret.inner().into(); assert_eq!(secret, "mock-client-secret"); } - _ => panic!("Expected ClientSecretCredential variant"), + _ => panic!("Expected Specific(ClientSecretCredential) variant"), } } @@ -146,11 +121,7 @@ fn basic_config_with_managed_identity() { assert_eq!(config.timestamp_field, "TimeGenerated"); match &config.auth { - crate::sinks::azure_logs_ingestion::config::AzureAuthentication::Specific( - crate::sinks::azure_logs_ingestion::config::SpecificAzureCredential::ManagedIdentity { - .. - }, - ) => { + AzureAuthentication::Specific(SpecificAzureCredential::ManagedIdentity { .. }) => { // Expected variant } _ => panic!("Expected Specific(ManagedIdentity) variant"), @@ -182,6 +153,7 @@ async fn correct_request() { stream_name = "Custom-UnitTest" [auth] + azure_credential_kind = "client_secret_credential" azure_tenant_id = "00000000-0000-0000-0000-000000000000" azure_client_id = "mock-client-id" azure_client_secret = "mock-client-secret" @@ -292,6 +264,7 @@ async fn mock_healthcheck_with_400_response() { stream_name = "Custom-UnitTest" [auth] + azure_credential_kind = "client_secret_credential" azure_tenant_id = "00000000-0000-0000-0000-000000000000" azure_client_id = "mock-client-id" azure_client_secret = "mock-client-secret" @@ -361,6 +334,7 @@ async fn mock_healthcheck_with_403_response() { stream_name = "Custom-UnitTest" [auth] + azure_credential_kind = "client_secret_credential" azure_tenant_id = "00000000-0000-0000-0000-000000000000" azure_client_id = "mock-client-id" azure_client_secret = "mock-client-secret" diff --git a/src/sinks/mod.rs b/src/sinks/mod.rs index 3a9c3c049a0fa..e5584de61dd02 100644 --- a/src/sinks/mod.rs +++ b/src/sinks/mod.rs @@ -26,7 +26,7 @@ pub mod aws_s_s; pub mod axiom; #[cfg(feature = "sinks-azure_blob")] pub mod azure_blob; -#[cfg(feature = "sinks-azure_blob")] +#[cfg(any(feature = "sinks-azure_blob", feature = "sinks-azure_logs_ingestion",))] pub mod azure_common; #[cfg(feature = "sinks-azure_logs_ingestion")] pub mod azure_logs_ingestion; diff --git a/tests/data/ClientCertificateAuth.pfx b/tests/data/ClientCertificateAuth.pfx new file mode 100644 index 0000000000000000000000000000000000000000..e289b55375a4ac01132352e54813ffd76405bcc0 GIT binary patch literal 2467 zcmai$X*3iH8^_JYjO|(m*)GNs8rRtOeN7nIn_)7@zGSB`jBHaP(Y2&TlOjuK24hcl zDceP|zPhL+GMI$>y61gQ_v8KWoaa3M|9Q^y>GwYmiU@lM1b|RPmgra;t>(+DF#`Fr0R=8`-8d_*|TW-N^ zbl*q?#EcG5mTh$AF9my5!#w>BN%mvzvYCMHjMl9IEW7mY*aR|^%vM|mk z>D$T_eLDX7@~?mtF+1NH>3DzL7G&!OEmFHc3r%yrn8ag zH8OA8F}bG=p>N?#M!pyvyu4cQ#Qn?L7u)AQZ+B7!Q>u$)TiA@enyp?*wo7RZbgbNr z&9_oG>09(IIZv_)5y+kBwFf*c{#`)C3Juaoj0>HebqYigAvb>V_czdx7%&kMaVm$N zRv;_K|5@i`2LMk^l~Yma-vGXH&|ttURI>b6fZ?ozpeNh!`Rl5i<{r?J`@ux;7!I5> zk~oIIy(wTih5{n}gchSthP}|EWtvrAb1wwuF6w4`MtP(6ZzlOy;D-whH6j*woj-bk z)v4+nR`TDo8;<5`qfGI)l#GY2W8vLL+qKq31G%0>Uz1m#-SPChsR%41Y%^EtCv>o!YV;iZjxy@}h{{#bOtDMheWo zw(4F>xS3U(D%6LzC{1e0{q;x_u^s9p{A(?5C~05GqJXvpEEI}q(C|}I>7$HpPTa;( zrebEj=W6qY%t~uLkEblEA|;}Cy=kE5_?bzyuZr-)^J(I+G-#qWHO=2a1)KPk4-x38 z>=FAgP}b@({qg}lYNQf-c}`5X>Fr+!zr!Jbn*O(J*30vA{65;tEAe7zed!atQ<2X3 z#17Kh&D-g=(PI1y>@p-1C(HK!inEb}_oaA>{NkD#QW%M@eom+@KXV%@A#!?&T}rD< zd0yKg;gPnNgH3^^q=a#VR}ih@({t}!#?1yiEy;M0UXBJT9$34IMNi(SJ;TYLnr+tk z*c(E97tZ-w>DIX5PPWM^{Zd9y^_Z4iD?Fk7*@xJ;EWY?T^6!?b!bNqx<> zC%b<-DHuB|936ZiiCucg_3C?4^B>+chJ0pT;DP3e+L7ry8H^|a7^7h7O5T{@p4eRK zdXJ3Kb?FhAD+sCCqY7rE5>n&Kc7x_9>@w+kq)nE)XnBIt9mRn{-n-@f!Rwo%8sAp; z`G7ryL63GLEZ?Q$bVhwSc+|UTM%u5xBI7fmr&&j7eD#MYwMWopHu>PmHzWgMvdW7! zTeBA{-vzd?b+Y#GBWkjPq4owL1{?ak(~%HdqsX(sVK_!hiGkb$FvLH8pi|->j?aRt zs+J{|Z@0&nhxZgax?n>9Zo{>iq`l6pl zESYILy?h-=+#aO1HPU*Di&}%zsdiGF+oKSstNb?mjUSkxHeCO+WGgXJK4Rzdob#%S zo?N4OC+z_dCiFoKYwe?Ek-1%UmYwfp+*QKsb%~(F&Z`K1u8>Sg)r)PTx3td8i&(1F zvzK4x@G5O7&%dzVRJmnWgVdFg)8wg8DMaeWaeWut0Qw&nV@ZrO%V$}4UIx81Gk@E* z;39mM#(E!rt_hFo7t+-=p4R^`KKh%MSVeS38RG9F! zYl0t@52PjH7ZfO3M~=Hg4Pu^`L)OV>4BM%Ed!SMfMzq&s|D?1q`Z!vw|a*UpR2qV1*M+9 zLztw&9}G=BTDO3B;VLfa6)@9bE?Qz51B!p-smP+Fe+Dc}Kt&J?NABjzJC%MW3RI`X za2d$ig=@Blp-fOJD6XIH4G;if0ZaN8uILs-Ob3Hud6PjPmwKTu_?mrd<`jRwu)MFc Tg&i2m^(Z6qB2fO!-% ca/intermediate_server/certs/localhost-chain.cert.pem +ca/intermediate_server/private/azurite.key.pem: + openssl genrsa -out ca/intermediate_server/private/azurite.key.pem 2048 + +ca/intermediate_server/csr/azurite.csr.pem: ca/intermediate_server/private/azurite.key.pem + openssl req -config ca/intermediate_server/openssl.cnf \ + -key ca/intermediate_server/private/azurite.key.pem \ + -subj '/CN=azurite/OU=Vector/O=Datadog/ST=New York/L=New York/C=US' \ + -addext 'subjectAltName=DNS:azurite' \ + -addext 'basicConstraints=critical,CA:FALSE' \ + -addext 'extendedKeyUsage=serverAuth' \ + -new -sha256 -out ca/intermediate_server/csr/azurite.csr.pem + +ca/intermediate_server/certs/azurite.cert.pem: ca/intermediate_server/csr/azurite.csr.pem + openssl ca -batch -config ca/intermediate_server/openssl.cnf \ + -extensions server_cert -days 3650 -notext -md sha256 \ + -in ca/intermediate_server/csr/azurite.csr.pem \ + -out ca/intermediate_server/certs/azurite.cert.pem + +ca/intermediate_server/certs/azurite-chain.cert.pem: ca/intermediate_server/certs/ca-chain.cert.pem ca/intermediate_server/certs/azurite.cert.pem + cat ca/intermediate_server/certs/azurite.cert.pem ca/intermediate_server/certs/ca-chain.cert.pem > ca/intermediate_server/certs/azurite-chain.cert.pem + ca/intermediate_server/private/elasticsearch-secure.key.pem: openssl genrsa -out ca/intermediate_server/private/elasticsearch-secure.key.pem 2048 diff --git a/tests/data/ca/intermediate_server/certs/azurite-chain.cert.pem b/tests/data/ca/intermediate_server/certs/azurite-chain.cert.pem new file mode 100644 index 0000000000000..f646a0fe3c20a --- /dev/null +++ b/tests/data/ca/intermediate_server/certs/azurite-chain.cert.pem @@ -0,0 +1,98 @@ +-----BEGIN CERTIFICATE----- +MIIFhDCCA2ygAwIBAgICEAkwDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCVVMx +ETAPBgNVBAgMCE5ldyBZb3JrMRAwDgYDVQQKDAdEYXRhZG9nMQ8wDQYDVQQLDAZW +ZWN0b3IxJjAkBgNVBAMMHVZlY3RvciBJbnRlcm1lZGlhdGUgU2VydmVyIENBMB4X +DTI2MDMwNDIwNTI1M1oXDTM2MDMwMTIwNTI1M1owaDELMAkGA1UEBhMCVVMxETAP +BgNVBAgMCE5ldyBZb3JrMREwDwYDVQQHDAhOZXcgWW9yazEQMA4GA1UECgwHRGF0 +YWRvZzEPMA0GA1UECwwGVmVjdG9yMRAwDgYDVQQDDAdhenVyaXRlMIIBIjANBgkq +hkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvnAyCpCvDorySCNastVW9x3+31Ta4SVP +bGW0LqD1WO42dSmUMQ2iRGsFeHvM4guXrhXloBVf35L1OyWRp/bltHlleLrr58bR +RInmuyocDTvm4t7VU+ybnPD7MhdNsbMmo2HBn12cEY7PszxVOwcZ8j0XOHtI+ve3 +QZ4loS61BR5TxrCdnpE++gxh7KhUG1yTiKmEEt587vRIuVBWNrLFVhYQN6mc+2JJ +63PaXXvGpGiDwPqUqi0WhIYp73XVyIqbHivI27Tiuv/n9gcHPAt2UwVbv1AqqFFn +Vr8lMVavcDcoTrHdJ8PXN4EipgvssJF14gFFg7L0U4AOj4xu31e+iQIDAQABo4IB +MzCCAS8wCQYDVR0TBAIwADARBglghkgBhvhCAQEEBAMCBkAwMwYJYIZIAYb4QgEN +BCYWJE9wZW5TU0wgR2VuZXJhdGVkIFNlcnZlciBDZXJ0aWZpY2F0ZTAdBgNVHQ4E +FgQUvRxz2qJo5NhsNm51lOVK0woSO2UwgZUGA1UdIwSBjTCBioAUPD06L8zVggN9 +mcRY8eHbNu+tDUGhbqRsMGoxEjAQBgNVBAMMCVZlY3RvciBDQTEPMA0GA1UECwwG +VmVjdG9yMRAwDgYDVQQKDAdEYXRhZG9nMREwDwYDVQQIDAhOZXcgWW9yazERMA8G +A1UEBwwITmV3IFlvcmsxCzAJBgNVBAYTAlVTggIQADAOBgNVHQ8BAf8EBAMCBaAw +EwYDVR0lBAwwCgYIKwYBBQUHAwEwDQYJKoZIhvcNAQELBQADggIBAFZJBzkVVJQ8 +pjs1qRrjsIgr3w1ylDJD4YTTEh96jLvQTf/PzhwG+W8ylr8a7EPzaJ9SOnLimEat +sw8ZCZJlRNdALL7o1tDAfOUkva45IQzvpzvDjy7iLQ488Cv4or/wWBd56JqWlYyA +OcyaCAkslI4RRoeIlKczbmhF/hTXlqe4z8q/89tHwFINOksbCj5VWYKzmYMijluw +X6RfcTyyaPlhz/1TJERWxyYVs+RN9TLNWvgPAeNmSZvnOXJ7b0hiIwRN+Hj+0M9L +tSfHq7Nrz8ls9+u7gyfkYtXz7KahI6CVKATIJl18X7zTLRCh/2DRyISSs+hbXlBa +fSG/rL92F/rerzw0w6PP7cs0TnKuEnnwIBZV+q5mkg/c15mTL5vgzRCiZfE8E/bJ +UwV+eX7s1yd67UU9LgNanYYTCNpAnb9HZwm+65Q+wEO7M3puxL2/DqbJ3IQVHDRx +6/2raCYBszz6p4FPalRXmUqa5JogIxc5kVPHje330JwY5JKwIfe/GtT5AkeRqHx+ +KlbEBQYfIh66BPB+y60sPBUtJslPfDV+PabDHq6u9wJtZi12RbxLFBjQtCMAV3vx +AvTfgxzbZ8rqg7mL8JB1SupkMiw8eObC/zrYGMv8O3IOoSWSRNb/KrYjYf3oVRQ5 +CDjEDNgIlI4ue4WLySdt1Fped5jCf/+P +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIFtzCCA5+gAwIBAgICEAAwDQYJKoZIhvcNAQELBQAwajESMBAGA1UEAwwJVmVj +dG9yIENBMQ8wDQYDVQQLDAZWZWN0b3IxEDAOBgNVBAoMB0RhdGFkb2cxETAPBgNV +BAgMCE5ldyBZb3JrMREwDwYDVQQHDAhOZXcgWW9yazELMAkGA1UEBhMCVVMwHhcN +MjIwNjA3MjIyNzUzWhcNMzIwNjA0MjIyNzUzWjBrMQswCQYDVQQGEwJVUzERMA8G +A1UECAwITmV3IFlvcmsxEDAOBgNVBAoMB0RhdGFkb2cxDzANBgNVBAsMBlZlY3Rv +cjEmMCQGA1UEAwwdVmVjdG9yIEludGVybWVkaWF0ZSBTZXJ2ZXIgQ0EwggIiMA0G +CSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCy/mB0/ZwfgKrSZPQIFaGPtRA9xL2N +o2SsHndZ8k2XOCV225Hb2fzNH+o2WGNSjwmGjLP/uXb47KH0cHCAyFGzSjp++8/O +zoZaFiO0P5El02hQxmoabO3Cqu/N62EFsLfpSM828JM6YOn9p+WXUDn1+YPNoOOE +H142p4/RjFnXNHkzR3geXU4Pfi3KXDrMi8vK42lDqXPLPs6rhreBAfQ2dsYyqhz6 +tg6FzZuXxxzEYyYtNgGh+zTji99WCBMLbCmRcDurRjdTDO7m4O3PrwbGUy0xdLeb +HJiNGvUDCPH4bfwLiNqwVIZY38RBCAqbCnrqRhDaZIfAUev4mq3Kqh6KUeO/U7Vx +/5J5rL5ApREKOfWPATHMprBuEU2rs3N+MPBA04HoiFlu311urCxVEA1qsZCTkoCg +GHuDIVSU4E4hT4co95/J0to4zWgPlfPg1+cXyU8lAIMe7JdCGkG9cDe7Umw/GSbt +ZdoCMQZ6WyyiW2Hw+7sFD3V3VzYa5YA/rjKZRduPmGWKrs+mAa5J5pM2M22rrjbd +EpfTHWLS9s6cPN3/jxpCxn6Hv/KhIYRAcIterugag1+clvS1ajVjxBRavOxPBsf+ +hYdh7S5NTZnT98gjkc3yOuGQm7BPtXau+IYZRlWcB0dJ4/E2P69hmWQezSo9VVWh +5/K1RkbPvqTGZQIDAQABo2YwZDAdBgNVHQ4EFgQUPD06L8zVggN9mcRY8eHbNu+t +DUEwHwYDVR0jBBgwFoAURTWK6ARqnZkz8rktUc5PrtasIh8wEgYDVR0TAQH/BAgw +BgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwDQYJKoZIhvcNAQELBQADggIBAGqaGBuL +2J6Na8RHx/GmSeuZFiVcWhmd/I9bVpeMSYHSZujA2nay6OGaUYs0Lq/G5OKgsuT9 +AIHnsn7VUE1zqoDfXac/K8sXlOig8if7rTb+06jgymaP1YSELg3R+pBsdkZnXVil +izh/9FvzoyV+QQlIhojqCIybVFgxa1XFHq4QCPhDfwkg+tp9RctfwNmWgsJ63H19 +RmxN+H2xIrySvObwXnB4j6D4wvgu468QXQMEuSsnLcIQFg6Zteqe8fixbqTiOTBf +Dk1k+EpB9VMEkIPvMdfa48vseXdBEe6Ma9zGuJC76q4q1ZapVLTvOUP5Y24khlgd +cj5tfP7o7yc6HqymfXAcD1lzP2JQhqaRxA4I18Nrd+aHi+G1EM2c3cicuD3n6Iw9 +9oqdCwmMfS25fv5cyA5B6hRusIZ9wRopTi7at+JHl0GIt/FelaTYI7kRmAqgakQe +oEKLpXcH8lRJW802DmXm7ka4eQzwxa7Ngyf8O+JOFtGO0+EshuLJovxiPl6IyLyG +NJ/dHq3ad+46YVManbHdyjHxgT5PSvJFkq0Yluvf44NIyP5QRTCAvfH76bu7hXgS +QoQj5t5ILn6meQRTR79r2iwpQTanPLTEdoZvmrE4TeUBev9BA5KpiPPA3i3ZF/oV +0EYorXCNri7M/jylGW7AuWvNUyaVR6xgxAn6 +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIFujCCA6KgAwIBAgIJAKhPL9BkNaFGMA0GCSqGSIb3DQEBCwUAMGoxEjAQBgNV +BAMMCVZlY3RvciBDQTEPMA0GA1UECwwGVmVjdG9yMRAwDgYDVQQKDAdEYXRhZG9n +MREwDwYDVQQIDAhOZXcgWW9yazERMA8GA1UEBwwITmV3IFlvcmsxCzAJBgNVBAYT +AlVTMB4XDTIyMDYwNzIyMjc1MloXDTQyMDYwMjIyMjc1MlowajESMBAGA1UEAwwJ +VmVjdG9yIENBMQ8wDQYDVQQLDAZWZWN0b3IxEDAOBgNVBAoMB0RhdGFkb2cxETAP +BgNVBAgMCE5ldyBZb3JrMREwDwYDVQQHDAhOZXcgWW9yazELMAkGA1UEBhMCVVMw +ggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQC9c1T+NXTNmqiiV36NSEJt +7mo0cyv8Byk2ZGdC85vHBm45QDY5USoh0vgonzPpWgSMggPn1WbR0f1y+LBwXdlM ++ZyZh2RVVeUrSjJ88lLHVn4DfywpdDkwQaFj1VmOsj2I9rMMrgc5x5n1Hj7lwZ+t +uPVSAGmgKp4iFfzLph9r/rjP1TUAnVUComfTUVS+Gd7zoGPOc14cMJXG6g2P2aAU +P6dg5uQlTxRmagnlx7bwm3lRwv6LMtnAdnjwBDBxr933nucAnk21GgE92GejiO3Z +OwlzIdzBI23lPcWi5pq+vCTgAArNq24W1Ha+7Jn5QewNTGKFyyYAJetZAwCUR8QS +Ip++2GE2pNhaGqcV5u1Tbwl02eD6p2qRqjfgLxmb+aC6xfl0n9kiFGPZppjCqDEW +sw+gX66nf+qxZVRWpJon2kWcFvhTnLqoa3T3+9+KIeamz2lW6wxMnki/Co2EA1Wa +mmedaUUcRPCgMx9aCktRkMyH6bEY8/vfJ07juxUsszOc46T00Scmn6Vkuo9Uc3Kf +2Q2N6Wo4jtyAiMO4gAwq5kzzpBAhNgRfLHOb83r2gAUj2Y4Vln/UUR/KR8ZbJi4i +r1BjX16Lz3yblJXXb1lp4uZynlbHNaAevXyGlRqHddM2ykKtAX/vgJcZRGSvms11 +uce/cqzrzx60AhpLRma5CwIDAQABo2MwYTAdBgNVHQ4EFgQURTWK6ARqnZkz8rkt +Uc5PrtasIh8wHwYDVR0jBBgwFoAURTWK6ARqnZkz8rktUc5PrtasIh8wDwYDVR0T +AQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAYYwDQYJKoZIhvcNAQELBQADggIBAEf5 +TR3hq/DtSAmsYotu1lAWz/OlTpG+7AdqSOHB878X4ETN3xaQ+KWvSwvf0K70ZDTV +tFOTh/r43cpzPifPKd1P+2ctnQEzrBtAacvyETLq1ABRK9VJOtfJ6Xk5KZXPhKdY +t353PQgBgW8YzQ2adq2B7FtgIlX7f1DIndjcMZBbolETR6xt9QwB/UnPI7Mwt01T ++bCBhr1fWAbZ4YAMlQ0xRam4qUOTjxgfmePrmSrv4HO7cXHMsRMLiXk+BLcx959/ +K/B6xzpzn6366Eqnqlo/uDiMpo5ud2I/Snz5PduB6oLztPMEf/8RmkG5tpHXYdWr +tM64WqNGO+ikluIrrtYvtyZS4DfsLAMfMYZcxX/Uw56gHo0i2c8I6+6JvGWdvOJ0 +FjrsKeIQoRlV77z025kI4V9jKi3XNMEsAIH+W7KNSut0X80yX7SugvQGoe0GDkXu +0fy8hMC3uTN2LEycYFRRfoIeKPLi6OZFK0PdS2E15d8PEU3n3W4eBCPgMtmiOKLY +d8QNBC8XLAuBoK9R8luCJpOJWUcFXjLpjcDab4V2hKTuAs+GQyDh/Xx4wF1yHX0r +zIkyN0EkOD/SvD8X4uFaM4mdsAh+ucn4ryUV7i5PgvDM9z4InHAMAee1ebBl0U+h ++NzMWF5c5OwxD5o6/Wh1HopmzJiVNT2v9u0kHT/f +-----END CERTIFICATE----- diff --git a/tests/data/ca/intermediate_server/certs/azurite.cert.pem b/tests/data/ca/intermediate_server/certs/azurite.cert.pem new file mode 100644 index 0000000000000..3e2f680ef37c0 --- /dev/null +++ b/tests/data/ca/intermediate_server/certs/azurite.cert.pem @@ -0,0 +1,32 @@ +-----BEGIN CERTIFICATE----- +MIIFhDCCA2ygAwIBAgICEAkwDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCVVMx +ETAPBgNVBAgMCE5ldyBZb3JrMRAwDgYDVQQKDAdEYXRhZG9nMQ8wDQYDVQQLDAZW +ZWN0b3IxJjAkBgNVBAMMHVZlY3RvciBJbnRlcm1lZGlhdGUgU2VydmVyIENBMB4X +DTI2MDMwNDIwNTI1M1oXDTM2MDMwMTIwNTI1M1owaDELMAkGA1UEBhMCVVMxETAP +BgNVBAgMCE5ldyBZb3JrMREwDwYDVQQHDAhOZXcgWW9yazEQMA4GA1UECgwHRGF0 +YWRvZzEPMA0GA1UECwwGVmVjdG9yMRAwDgYDVQQDDAdhenVyaXRlMIIBIjANBgkq +hkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvnAyCpCvDorySCNastVW9x3+31Ta4SVP +bGW0LqD1WO42dSmUMQ2iRGsFeHvM4guXrhXloBVf35L1OyWRp/bltHlleLrr58bR +RInmuyocDTvm4t7VU+ybnPD7MhdNsbMmo2HBn12cEY7PszxVOwcZ8j0XOHtI+ve3 +QZ4loS61BR5TxrCdnpE++gxh7KhUG1yTiKmEEt587vRIuVBWNrLFVhYQN6mc+2JJ +63PaXXvGpGiDwPqUqi0WhIYp73XVyIqbHivI27Tiuv/n9gcHPAt2UwVbv1AqqFFn +Vr8lMVavcDcoTrHdJ8PXN4EipgvssJF14gFFg7L0U4AOj4xu31e+iQIDAQABo4IB +MzCCAS8wCQYDVR0TBAIwADARBglghkgBhvhCAQEEBAMCBkAwMwYJYIZIAYb4QgEN +BCYWJE9wZW5TU0wgR2VuZXJhdGVkIFNlcnZlciBDZXJ0aWZpY2F0ZTAdBgNVHQ4E +FgQUvRxz2qJo5NhsNm51lOVK0woSO2UwgZUGA1UdIwSBjTCBioAUPD06L8zVggN9 +mcRY8eHbNu+tDUGhbqRsMGoxEjAQBgNVBAMMCVZlY3RvciBDQTEPMA0GA1UECwwG +VmVjdG9yMRAwDgYDVQQKDAdEYXRhZG9nMREwDwYDVQQIDAhOZXcgWW9yazERMA8G +A1UEBwwITmV3IFlvcmsxCzAJBgNVBAYTAlVTggIQADAOBgNVHQ8BAf8EBAMCBaAw +EwYDVR0lBAwwCgYIKwYBBQUHAwEwDQYJKoZIhvcNAQELBQADggIBAFZJBzkVVJQ8 +pjs1qRrjsIgr3w1ylDJD4YTTEh96jLvQTf/PzhwG+W8ylr8a7EPzaJ9SOnLimEat +sw8ZCZJlRNdALL7o1tDAfOUkva45IQzvpzvDjy7iLQ488Cv4or/wWBd56JqWlYyA +OcyaCAkslI4RRoeIlKczbmhF/hTXlqe4z8q/89tHwFINOksbCj5VWYKzmYMijluw +X6RfcTyyaPlhz/1TJERWxyYVs+RN9TLNWvgPAeNmSZvnOXJ7b0hiIwRN+Hj+0M9L +tSfHq7Nrz8ls9+u7gyfkYtXz7KahI6CVKATIJl18X7zTLRCh/2DRyISSs+hbXlBa +fSG/rL92F/rerzw0w6PP7cs0TnKuEnnwIBZV+q5mkg/c15mTL5vgzRCiZfE8E/bJ +UwV+eX7s1yd67UU9LgNanYYTCNpAnb9HZwm+65Q+wEO7M3puxL2/DqbJ3IQVHDRx +6/2raCYBszz6p4FPalRXmUqa5JogIxc5kVPHje330JwY5JKwIfe/GtT5AkeRqHx+ +KlbEBQYfIh66BPB+y60sPBUtJslPfDV+PabDHq6u9wJtZi12RbxLFBjQtCMAV3vx +AvTfgxzbZ8rqg7mL8JB1SupkMiw8eObC/zrYGMv8O3IOoSWSRNb/KrYjYf3oVRQ5 +CDjEDNgIlI4ue4WLySdt1Fped5jCf/+P +-----END CERTIFICATE----- diff --git a/tests/data/ca/intermediate_server/csr/azurite.csr.pem b/tests/data/ca/intermediate_server/csr/azurite.csr.pem new file mode 100644 index 0000000000000..09cf2dd0bc3f7 --- /dev/null +++ b/tests/data/ca/intermediate_server/csr/azurite.csr.pem @@ -0,0 +1,18 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIC9TCCAd0CAQAwaDEQMA4GA1UEAwwHYXp1cml0ZTEPMA0GA1UECwwGVmVjdG9y +MRAwDgYDVQQKDAdEYXRhZG9nMREwDwYDVQQIDAhOZXcgWW9yazERMA8GA1UEBwwI +TmV3IFlvcmsxCzAJBgNVBAYTAlVTMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB +CgKCAQEAvnAyCpCvDorySCNastVW9x3+31Ta4SVPbGW0LqD1WO42dSmUMQ2iRGsF +eHvM4guXrhXloBVf35L1OyWRp/bltHlleLrr58bRRInmuyocDTvm4t7VU+ybnPD7 +MhdNsbMmo2HBn12cEY7PszxVOwcZ8j0XOHtI+ve3QZ4loS61BR5TxrCdnpE++gxh +7KhUG1yTiKmEEt587vRIuVBWNrLFVhYQN6mc+2JJ63PaXXvGpGiDwPqUqi0WhIYp +73XVyIqbHivI27Tiuv/n9gcHPAt2UwVbv1AqqFFnVr8lMVavcDcoTrHdJ8PXN4Ei +pgvssJF14gFFg7L0U4AOj4xu31e+iQIDAQABoEgwRgYJKoZIhvcNAQkOMTkwNzAS +BgNVHREECzAJggdhenVyaXRlMAwGA1UdEwEB/wQCMAAwEwYDVR0lBAwwCgYIKwYB +BQUHAwEwDQYJKoZIhvcNAQELBQADggEBAIbILz+IbvX9126CRLqmtlp4s1q2L+A2 +jYz+qk1G1UhImumfEcLMBfj6X/ZQ5IRIrH8BZZtW9ungXUS/9eYKUCQP52GXPPZJ +WkS3sERjvktEP7orNrTLo7v36f9P0J9eMjN8XYhcqHmZB5OyoQ4S3ndqgE4+HV8j +OyP9lZC1Urjrre/S7pyxxnkVftxXtVHXzB0MgoQa77U/ukqeaMYIw3qWRqGwwl44 +JnZm05mukTpYnDVmo+J2Ra1uNOJf9/SIEnawyL1ROYCINxgIcdodrpuxz+dnYT0K +zvsbMGQ1MTacJK61Pw0EBbxERB9ZhWnApRdeuhrF5oq9uGEzD/76wTg= +-----END CERTIFICATE REQUEST----- diff --git a/tests/data/ca/intermediate_server/index.txt b/tests/data/ca/intermediate_server/index.txt index 0402ad0946130..b2a0481603b78 100644 --- a/tests/data/ca/intermediate_server/index.txt +++ b/tests/data/ca/intermediate_server/index.txt @@ -6,3 +6,4 @@ V 320613195253Z 1005 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/C V 320731200837Z 1006 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/CN=dufs-https V 330412000039Z 1007 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/CN=rabbitmq V 341228053159Z 1008 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/CN=pulsar +V 360301205253Z 1009 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/CN=azurite diff --git a/tests/data/ca/intermediate_server/index.txt.old b/tests/data/ca/intermediate_server/index.txt.old index ab4efe2c2f6bd..0402ad0946130 100644 --- a/tests/data/ca/intermediate_server/index.txt.old +++ b/tests/data/ca/intermediate_server/index.txt.old @@ -5,3 +5,4 @@ V 320613195026Z 1004 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/C V 320613195253Z 1005 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/CN=kafka V 320731200837Z 1006 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/CN=dufs-https V 330412000039Z 1007 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/CN=rabbitmq +V 341228053159Z 1008 unknown /C=US/ST=New York/L=New York/O=Datadog/OU=Vector/CN=pulsar diff --git a/tests/data/ca/intermediate_server/newcerts/1009.pem b/tests/data/ca/intermediate_server/newcerts/1009.pem new file mode 100644 index 0000000000000..3e2f680ef37c0 --- /dev/null +++ b/tests/data/ca/intermediate_server/newcerts/1009.pem @@ -0,0 +1,32 @@ +-----BEGIN CERTIFICATE----- +MIIFhDCCA2ygAwIBAgICEAkwDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCVVMx +ETAPBgNVBAgMCE5ldyBZb3JrMRAwDgYDVQQKDAdEYXRhZG9nMQ8wDQYDVQQLDAZW +ZWN0b3IxJjAkBgNVBAMMHVZlY3RvciBJbnRlcm1lZGlhdGUgU2VydmVyIENBMB4X +DTI2MDMwNDIwNTI1M1oXDTM2MDMwMTIwNTI1M1owaDELMAkGA1UEBhMCVVMxETAP +BgNVBAgMCE5ldyBZb3JrMREwDwYDVQQHDAhOZXcgWW9yazEQMA4GA1UECgwHRGF0 +YWRvZzEPMA0GA1UECwwGVmVjdG9yMRAwDgYDVQQDDAdhenVyaXRlMIIBIjANBgkq +hkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvnAyCpCvDorySCNastVW9x3+31Ta4SVP +bGW0LqD1WO42dSmUMQ2iRGsFeHvM4guXrhXloBVf35L1OyWRp/bltHlleLrr58bR +RInmuyocDTvm4t7VU+ybnPD7MhdNsbMmo2HBn12cEY7PszxVOwcZ8j0XOHtI+ve3 +QZ4loS61BR5TxrCdnpE++gxh7KhUG1yTiKmEEt587vRIuVBWNrLFVhYQN6mc+2JJ +63PaXXvGpGiDwPqUqi0WhIYp73XVyIqbHivI27Tiuv/n9gcHPAt2UwVbv1AqqFFn +Vr8lMVavcDcoTrHdJ8PXN4EipgvssJF14gFFg7L0U4AOj4xu31e+iQIDAQABo4IB +MzCCAS8wCQYDVR0TBAIwADARBglghkgBhvhCAQEEBAMCBkAwMwYJYIZIAYb4QgEN +BCYWJE9wZW5TU0wgR2VuZXJhdGVkIFNlcnZlciBDZXJ0aWZpY2F0ZTAdBgNVHQ4E +FgQUvRxz2qJo5NhsNm51lOVK0woSO2UwgZUGA1UdIwSBjTCBioAUPD06L8zVggN9 +mcRY8eHbNu+tDUGhbqRsMGoxEjAQBgNVBAMMCVZlY3RvciBDQTEPMA0GA1UECwwG +VmVjdG9yMRAwDgYDVQQKDAdEYXRhZG9nMREwDwYDVQQIDAhOZXcgWW9yazERMA8G +A1UEBwwITmV3IFlvcmsxCzAJBgNVBAYTAlVTggIQADAOBgNVHQ8BAf8EBAMCBaAw +EwYDVR0lBAwwCgYIKwYBBQUHAwEwDQYJKoZIhvcNAQELBQADggIBAFZJBzkVVJQ8 +pjs1qRrjsIgr3w1ylDJD4YTTEh96jLvQTf/PzhwG+W8ylr8a7EPzaJ9SOnLimEat +sw8ZCZJlRNdALL7o1tDAfOUkva45IQzvpzvDjy7iLQ488Cv4or/wWBd56JqWlYyA +OcyaCAkslI4RRoeIlKczbmhF/hTXlqe4z8q/89tHwFINOksbCj5VWYKzmYMijluw +X6RfcTyyaPlhz/1TJERWxyYVs+RN9TLNWvgPAeNmSZvnOXJ7b0hiIwRN+Hj+0M9L +tSfHq7Nrz8ls9+u7gyfkYtXz7KahI6CVKATIJl18X7zTLRCh/2DRyISSs+hbXlBa +fSG/rL92F/rerzw0w6PP7cs0TnKuEnnwIBZV+q5mkg/c15mTL5vgzRCiZfE8E/bJ +UwV+eX7s1yd67UU9LgNanYYTCNpAnb9HZwm+65Q+wEO7M3puxL2/DqbJ3IQVHDRx +6/2raCYBszz6p4FPalRXmUqa5JogIxc5kVPHje330JwY5JKwIfe/GtT5AkeRqHx+ +KlbEBQYfIh66BPB+y60sPBUtJslPfDV+PabDHq6u9wJtZi12RbxLFBjQtCMAV3vx +AvTfgxzbZ8rqg7mL8JB1SupkMiw8eObC/zrYGMv8O3IOoSWSRNb/KrYjYf3oVRQ5 +CDjEDNgIlI4ue4WLySdt1Fped5jCf/+P +-----END CERTIFICATE----- diff --git a/tests/data/ca/intermediate_server/private/azurite.key.pem b/tests/data/ca/intermediate_server/private/azurite.key.pem new file mode 100644 index 0000000000000..545d8904a3d73 --- /dev/null +++ b/tests/data/ca/intermediate_server/private/azurite.key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC+cDIKkK8OivJI +I1qy1Vb3Hf7fVNrhJU9sZbQuoPVY7jZ1KZQxDaJEawV4e8ziC5euFeWgFV/fkvU7 +JZGn9uW0eWV4uuvnxtFEiea7KhwNO+bi3tVT7Juc8PsyF02xsyajYcGfXZwRjs+z +PFU7BxnyPRc4e0j697dBniWhLrUFHlPGsJ2ekT76DGHsqFQbXJOIqYQS3nzu9Ei5 +UFY2ssVWFhA3qZz7Yknrc9pde8akaIPA+pSqLRaEhinvddXIipseK8jbtOK6/+f2 +Bwc8C3ZTBVu/UCqoUWdWvyUxVq9wNyhOsd0nw9c3gSKmC+ywkXXiAUWDsvRTgA6P +jG7fV76JAgMBAAECggEAEXeC73sjw0a1QC6V9A8jQdkrdlp3FO1yInQVma1Ds5tt +vUNKB1HDz1itkMZyHU2I5Pu3Kv6q43u8KGeiu3Am023LA87JMmIG8a1gT0xmdERJ +QgfEM0VhZHyp3YdLpf/TjGq78p0IYofhvwPKoXZeR9yYk6KjJ/mugkM6GlWJXuWn +sE2ZYL4Wl6d//1/9D6xWfNVU1zCH7ScbFFvQKkBxURLpnQlgkfhr8HMrCQyV4Lps +6n166zcQKJO4mwGrWyu75HU8vcBYv1+6WbJ1Rgke6lL3dzjIhzE7C61i3dI636Aq +l0wZ3NVeysyrn5Xzu5vfq0D9/pt+Of1PE0FmmlRogQKBgQDhaY8OyUafMYhGnike +7K1ho60tFHE5vgxMpoUTinLyARgW8bA9VRtQ++PW90Z1sHzXAHjb30YZcYEuKaCW +SvIWg49/L4bORX2jY4UPZvESemJtnIXZkuuiGoYaXjIwNudorT1siCD/GYM24Uko +ax0QbFsNRtq89QE/ibKdFLRfUQKBgQDYR7JfauKmsY6OQjB8RvFe50AnnPfiJSZ+ +rD7H7gXWlt3q9zpsU3xBK+fwI9QsvTlUNsIkFhJpu6znCzK26nLIJCqGcm0W+9Ah +rJR+0TXyafQn1d1dTrtn7XVIL6QQEuUY9jggsAXWBejdPCeaVxplEiJa1i9x3YOp +OQWmMTvNuQKBgCiGF6fq231nJD691Fqw5gK1sD54fFqLJh7pmOcIbt2/AJuvW6XL +FRwcDLvqvIoP7oGgnhm5LBsK4tRvu2UJmDgf8r5ExxFyQMIM9DDuqsxNoEBgcVfK +J/5+kjlPUeqFFFknO/G1D2mNJp/JJKPVjeYT9NKQOGbcDRtlH+1JeZvhAoGAZkOg +Z9WWTdNu4H0Th+/TeVhG0XQ7EUcXqJWxKb+2Kv0y+ULk8QuYmQg1pyqJzI28acFq +kr2M/0mqO6Tj2fGJTHEtWl0Ij/GJPCLqI/ywUWsf8yYAgXoUytNQvU0peiA1C1SA +vZP9bnFk5hbncub0qA2nCOR1kpV3B7DapvZonKECgYEAjzpfOhaULZgwnkYOkoFw +VS+tDOVNVfjDOjuV82B/0zCgCW8dv2EeTCobEkIbSl8GYMzukNZQR8r3ZAxbs1By +Ieliad2FD/s0MtsPiGcxJN4twbDQO/SvLFtjQ+jCysgOVON6KiqpGVRcEL+6U4ZD +NiqT36xmXpmkZ05bMxOXPLM= +-----END PRIVATE KEY----- diff --git a/tests/data/ca/intermediate_server/serial b/tests/data/ca/intermediate_server/serial index 6cb3869343bf8..4e75b247d2247 100644 --- a/tests/data/ca/intermediate_server/serial +++ b/tests/data/ca/intermediate_server/serial @@ -1 +1 @@ -1009 +100A diff --git a/tests/data/ca/intermediate_server/serial.old b/tests/data/ca/intermediate_server/serial.old index 617ba1c154075..6cb3869343bf8 100644 --- a/tests/data/ca/intermediate_server/serial.old +++ b/tests/data/ca/intermediate_server/serial.old @@ -1 +1 @@ -1008 +1009 diff --git a/tests/integration/azure/config/compose.yaml b/tests/integration/azure/config/compose.yaml index fe3541bf2a07c..384a65ab0e42d 100644 --- a/tests/integration/azure/config/compose.yaml +++ b/tests/integration/azure/config/compose.yaml @@ -7,6 +7,14 @@ services: volumes: - /var/run:/var/run + azurite: + image: mcr.microsoft.com/azure-storage/azurite:${CONFIG_VERSION} + command: azurite --blobHost 0.0.0.0 --blobPort 14430 --queuePort 14431 --tablePort 14432 --loose --oauth basic --cert /certs/azurite-chain.cert.pem --key /private/azurite.key.pem + volumes: + - /var/run:/var/run + - ../../../data/ca/intermediate_server/certs:/certs + - ../../../data/ca/intermediate_server/private/azurite.key.pem:/private/azurite.key.pem + networks: default: name: ${VECTOR_NETWORK} diff --git a/tests/integration/azure/config/test.yaml b/tests/integration/azure/config/test.yaml index da00ceed79ad4..1b3a0e4cbe151 100644 --- a/tests/integration/azure/config/test.yaml +++ b/tests/integration/azure/config/test.yaml @@ -4,7 +4,8 @@ features: test_filter: ::azure_ env: - AZURE_ADDRESS: local-azure-blob + AZURITE_ADDRESS: local-azure-blob + AZURITE_OAUTH_ADDRESS: azurite HEARTBEAT_ADDRESS: 0.0.0.0:8080 LOGSTASH_ADDRESS: 0.0.0.0:8081 diff --git a/website/cue/reference/components/sinks/generated/azure_blob.cue b/website/cue/reference/components/sinks/generated/azure_blob.cue index 5ea0c1dd20221..8d962c30adbdb 100644 --- a/website/cue/reference/components/sinks/generated/azure_blob.cue +++ b/website/cue/reference/components/sinks/generated/azure_blob.cue @@ -1,6 +1,16 @@ package metadata generated: components: sinks: azure_blob: configuration: { + account_name: { + description: """ + The Azure Blob Storage Account name. + + If provided, this will be used instead of the `connection_string`. + This is useful for authenticating with an Azure credential. + """ + required: false + type: string: examples: ["mylogstorage"] + } acknowledgements: { description: """ Controls how acknowledgements are handled for this sink. @@ -27,6 +37,123 @@ generated: components: sinks: azure_blob: configuration: { type: bool: {} } } + auth: { + description: "Azure service principal authentication." + required: false + type: object: options: { + azure_client_id: { + description: """ + The [Azure Client ID][azure_client_id]. + + [azure_client_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + """ + relevant_when: "azure_credential_kind = \"client_certificate_credential\" or azure_credential_kind = \"client_secret_credential\"" + required: true + type: string: examples: ["00000000-0000-0000-0000-000000000000", "${AZURE_CLIENT_ID:?err}"] + } + azure_client_secret: { + description: """ + The [Azure Client Secret][azure_client_secret]. + + [azure_client_secret]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + """ + relevant_when: "azure_credential_kind = \"client_secret_credential\"" + required: true + type: string: examples: ["00-00~000000-0000000~0000000000000000000", "${AZURE_CLIENT_SECRET:?err}"] + } + azure_credential_kind: { + description: "The kind of Azure credential to use." + required: true + type: string: enum: { + azure_cli: "Use Azure CLI credentials" + client_certificate_credential: "Use certificate credentials" + client_secret_credential: "Use client ID/secret credentials" + managed_identity: "Use Managed Identity credentials" + managed_identity_client_assertion: "Use Managed Identity with Client Assertion credentials" + workload_identity: "Use Workload Identity credentials" + } + } + azure_tenant_id: { + description: """ + The [Azure Tenant ID][azure_tenant_id]. + + [azure_tenant_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + """ + relevant_when: "azure_credential_kind = \"client_certificate_credential\" or azure_credential_kind = \"client_secret_credential\"" + required: true + type: string: examples: ["00000000-0000-0000-0000-000000000000", "${AZURE_TENANT_ID:?err}"] + } + certificate_password: { + description: "The password for the client certificate, if applicable." + relevant_when: "azure_credential_kind = \"client_certificate_credential\"" + required: false + type: string: examples: ["${AZURE_CLIENT_CERTIFICATE_PASSWORD}"] + } + certificate_path: { + description: "PKCS12 certificate with RSA private key." + relevant_when: "azure_credential_kind = \"client_certificate_credential\"" + required: true + type: string: examples: ["path/to/certificate.pfx", "${AZURE_CLIENT_CERTIFICATE_PATH:?err}"] + } + client_assertion_client_id: { + description: "The target Client ID to use." + relevant_when: "azure_credential_kind = \"managed_identity_client_assertion\"" + required: true + type: string: examples: ["00000000-0000-0000-0000-000000000000"] + } + client_assertion_tenant_id: { + description: "The target Tenant ID to use." + relevant_when: "azure_credential_kind = \"managed_identity_client_assertion\"" + required: true + type: string: examples: ["00000000-0000-0000-0000-000000000000"] + } + client_id: { + description: """ + The [Azure Client ID][azure_client_id]. Defaults to the value of the environment variable `AZURE_CLIENT_ID`. + + [azure_client_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + """ + relevant_when: "azure_credential_kind = \"workload_identity\"" + required: false + type: string: examples: ["00000000-0000-0000-0000-000000000000", "${AZURE_CLIENT_ID}"] + } + tenant_id: { + description: """ + The [Azure Tenant ID][azure_tenant_id]. Defaults to the value of the environment variable `AZURE_TENANT_ID`. + + [azure_tenant_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + """ + relevant_when: "azure_credential_kind = \"workload_identity\"" + required: false + type: string: examples: ["00000000-0000-0000-0000-000000000000", "${AZURE_TENANT_ID}"] + } + token_file_path: { + description: "Path of a file containing a Kubernetes service account token. Defaults to the value of the environment variable `AZURE_FEDERATED_TOKEN_FILE`." + relevant_when: "azure_credential_kind = \"workload_identity\"" + required: false + type: string: examples: ["/var/run/secrets/azure/tokens/azure-identity-token", "${AZURE_FEDERATED_TOKEN_FILE}"] + } + user_assigned_managed_identity_id: { + description: "The User Assigned Managed Identity to use." + relevant_when: "azure_credential_kind = \"managed_identity\" or azure_credential_kind = \"managed_identity_client_assertion\"" + required: false + type: string: examples: ["00000000-0000-0000-0000-000000000000"] + } + user_assigned_managed_identity_id_type: { + description: """ + The type of the User Assigned Managed Identity ID provided (Client ID, Object ID, + or Resource ID). Defaults to Client ID. + """ + relevant_when: "azure_credential_kind = \"managed_identity\" or azure_credential_kind = \"managed_identity_client_assertion\"" + required: false + type: string: enum: { + client_id: "Client ID" + object_id: "Object ID" + resource_id: "Resource ID" + } + } + } + } batch: { description: "Event batching behavior." required: false @@ -74,6 +201,16 @@ generated: components: sinks: azure_blob: configuration: { required: false type: bool: {} } + blob_endpoint: { + description: """ + The Azure Blob Storage endpoint. + + If provided, this will be used instead of the `connection_string`. + This is useful for authenticating with an Azure credential. + """ + required: false + type: string: examples: ["https://mylogstorage.blob.core.windows.net/"] + } blob_prefix: { description: """ A prefix to apply to all blob keys. @@ -165,8 +302,9 @@ generated: components: sinks: azure_blob: configuration: { | Allowed resource types | Container & Object | | Allowed permissions | Read & Create | """ - required: true - type: string: examples: ["DefaultEndpointsProtocol=https;AccountName=mylogstorage;AccountKey=storageaccountkeybase64encoded;EndpointSuffix=core.windows.net", "BlobEndpoint=https://mylogstorage.blob.core.windows.net/;SharedAccessSignature=generatedsastoken"] + required: false + type: string: examples: ["DefaultEndpointsProtocol=https;AccountName=mylogstorage;AccountKey=storageaccountkeybase64encoded;EndpointSuffix=core.windows.net", "BlobEndpoint=https://mylogstorage.blob.core.windows.net/;SharedAccessSignature=generatedsastoken", "AccountName=mylogstorage"] + warnings: ["Access keys and SAS tokens can be used to gain unauthorized access to Azure Blob Storage resources. Numerous security breaches have occurred due to leaked connection strings. It is important to keep connection strings secure and not expose them in logs, error messages, or version control systems."] } container_name: { description: "The Azure Blob Storage Account container name." @@ -862,4 +1000,17 @@ generated: components: sinks: azure_blob: configuration: { } } } + tls: { + description: "TLS configuration." + required: false + type: object: options: ca_file: { + description: """ + Absolute path to an additional CA certificate file. + + The certificate must be in PEM (X.509) format. + """ + required: false + type: string: examples: ["/path/to/certificate_authority.crt"] + } + } } diff --git a/website/cue/reference/components/sinks/generated/azure_logs_ingestion.cue b/website/cue/reference/components/sinks/generated/azure_logs_ingestion.cue index ec03373e5354e..2aaf9b628e3c6 100644 --- a/website/cue/reference/components/sinks/generated/azure_logs_ingestion.cue +++ b/website/cue/reference/components/sinks/generated/azure_logs_ingestion.cue @@ -28,8 +28,8 @@ generated: components: sinks: azure_logs_ingestion: configuration: { } } auth: { - description: "Configuration of the authentication strategy for interacting with Azure services." - required: false + description: "Azure service principal authentication." + required: true type: object: options: { azure_client_id: { description: """ @@ -37,11 +37,9 @@ generated: components: sinks: azure_logs_ingestion: configuration: { [azure_client_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal """ - required: false - type: string: { - default: "" - examples: ["00000000-0000-0000-0000-000000000000"] - } + relevant_when: "azure_credential_kind = \"client_certificate_credential\" or azure_credential_kind = \"client_secret_credential\"" + required: true + type: string: examples: ["00000000-0000-0000-0000-000000000000", "${AZURE_CLIENT_ID:?err}"] } azure_client_secret: { description: """ @@ -49,17 +47,17 @@ generated: components: sinks: azure_logs_ingestion: configuration: { [azure_client_secret]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal """ - required: false - type: string: { - default: "" - examples: ["00-00~000000-0000000~0000000000000000000"] - } + relevant_when: "azure_credential_kind = \"client_secret_credential\"" + required: true + type: string: examples: ["00-00~000000-0000000~0000000000000000000", "${AZURE_CLIENT_SECRET:?err}"] } azure_credential_kind: { description: "The kind of Azure credential to use." required: true type: string: enum: { azure_cli: "Use Azure CLI credentials" + client_certificate_credential: "Use certificate credentials" + client_secret_credential: "Use client ID/secret credentials" managed_identity: "Use Managed Identity credentials" managed_identity_client_assertion: "Use Managed Identity with Client Assertion credentials" workload_identity: "Use Workload Identity credentials" @@ -71,11 +69,21 @@ generated: components: sinks: azure_logs_ingestion: configuration: { [azure_tenant_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal """ - required: false - type: string: { - default: "" - examples: ["00000000-0000-0000-0000-000000000000"] - } + relevant_when: "azure_credential_kind = \"client_certificate_credential\" or azure_credential_kind = \"client_secret_credential\"" + required: true + type: string: examples: ["00000000-0000-0000-0000-000000000000", "${AZURE_TENANT_ID:?err}"] + } + certificate_password: { + description: "The password for the client certificate, if applicable." + relevant_when: "azure_credential_kind = \"client_certificate_credential\"" + required: false + type: string: examples: ["${AZURE_CLIENT_CERTIFICATE_PASSWORD}"] + } + certificate_path: { + description: "PKCS12 certificate with RSA private key." + relevant_when: "azure_credential_kind = \"client_certificate_credential\"" + required: true + type: string: examples: ["path/to/certificate.pfx", "${AZURE_CLIENT_CERTIFICATE_PATH:?err}"] } client_assertion_client_id: { description: "The target Client ID to use." @@ -89,12 +97,51 @@ generated: components: sinks: azure_logs_ingestion: configuration: { required: true type: string: examples: ["00000000-0000-0000-0000-000000000000"] } + client_id: { + description: """ + The [Azure Client ID][azure_client_id]. Defaults to the value of the environment variable `AZURE_CLIENT_ID`. + + [azure_client_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + """ + relevant_when: "azure_credential_kind = \"workload_identity\"" + required: false + type: string: examples: ["00000000-0000-0000-0000-000000000000", "${AZURE_CLIENT_ID}"] + } + tenant_id: { + description: """ + The [Azure Tenant ID][azure_tenant_id]. Defaults to the value of the environment variable `AZURE_TENANT_ID`. + + [azure_tenant_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + """ + relevant_when: "azure_credential_kind = \"workload_identity\"" + required: false + type: string: examples: ["00000000-0000-0000-0000-000000000000", "${AZURE_TENANT_ID}"] + } + token_file_path: { + description: "Path of a file containing a Kubernetes service account token. Defaults to the value of the environment variable `AZURE_FEDERATED_TOKEN_FILE`." + relevant_when: "azure_credential_kind = \"workload_identity\"" + required: false + type: string: examples: ["/var/run/secrets/azure/tokens/azure-identity-token", "${AZURE_FEDERATED_TOKEN_FILE}"] + } user_assigned_managed_identity_id: { - description: "The User Assigned Managed Identity (Client ID) to use." + description: "The User Assigned Managed Identity to use." relevant_when: "azure_credential_kind = \"managed_identity\" or azure_credential_kind = \"managed_identity_client_assertion\"" required: false type: string: examples: ["00000000-0000-0000-0000-000000000000"] } + user_assigned_managed_identity_id_type: { + description: """ + The type of the User Assigned Managed Identity ID provided (Client ID, Object ID, + or Resource ID). Defaults to Client ID. + """ + relevant_when: "azure_credential_kind = \"managed_identity\" or azure_credential_kind = \"managed_identity_client_assertion\"" + required: false + type: string: enum: { + client_id: "Client ID" + object_id: "Object ID" + resource_id: "Resource ID" + } + } } } batch: { From 985f669ffae07c1104ae9cec1b097e74130e0341 Mon Sep 17 00:00:00 2001 From: fcfangcc Date: Thu, 9 Apr 2026 01:50:38 +0800 Subject: [PATCH 113/326] fix(file source): high CPU usage after async file server migration (#25064) * fixed reached_eof * add changelog * update changlog --------- Co-authored-by: Pavlos Rontidis --- benches/files.rs | 113 +++++++++++++++++- ...fixed_high_cpu_usage_in_file_source.fix.md | 3 + lib/file-source-common/src/buffer.rs | 9 +- lib/file-source/src/file_watcher/mod.rs | 30 ++++- lib/file-source/src/file_watcher/tests/mod.rs | 68 ++++++++++- 5 files changed, 212 insertions(+), 11 deletions(-) create mode 100644 changelog.d/25064_fixed_high_cpu_usage_in_file_source.fix.md diff --git a/benches/files.rs b/benches/files.rs index 703c4a984c8cd..6c6c00f367a66 100644 --- a/benches/files.rs +++ b/benches/files.rs @@ -1,9 +1,9 @@ use std::{convert::TryInto, path::PathBuf, time::Duration}; use bytes::Bytes; -use criterion::{BatchSize, Criterion, SamplingMode, Throughput, criterion_group}; +use criterion::{BatchSize, BenchmarkId, Criterion, SamplingMode, Throughput, criterion_group}; use futures::{SinkExt, StreamExt, stream}; -use tempfile::tempdir; +use tempfile::{TempDir, tempdir}; use tokio::fs::OpenOptions; use tokio_util::codec::{BytesCodec, FramedWrite}; use vector::{ @@ -12,6 +12,70 @@ use vector::{ }; use vector_lib::codecs::{TextSerializerConfig, encoding::FramingConfig}; +fn build_file_benchmark_environment( + idle_files: usize, +) -> ( + tokio::runtime::Runtime, + vector::topology::RunningTopology, + tokio::fs::File, + TempDir, +) { + let temp = tempdir().unwrap(); + let directory = temp.path().to_path_buf(); + + let mut data_dir = directory.clone(); + data_dir.push("data"); + std::fs::create_dir(&data_dir).unwrap(); + + let active_file = directory.join("active.log"); + std::fs::write(&active_file, []).unwrap(); + + for index in 0..idle_files { + let idle_file = directory.join(format!("idle-{index}.log")); + std::fs::write(idle_file, []).unwrap(); + } + + let output = directory.join("output.txt"); + + let mut source = sources::file::FileConfig::default(); + source.include = vec![directory.join("*.log")]; + source.data_dir = Some(data_dir); + + let mut config = config::Config::builder(); + config.add_source("in", source); + config.add_sink( + "out", + &["in"], + sinks::file::FileSinkConfig { + path: output.try_into().unwrap(), + idle_timeout: Duration::from_secs(30), + encoding: (None::, TextSerializerConfig::default()).into(), + compression: sinks::file::Compression::None, + acknowledgements: Default::default(), + timezone: Default::default(), + internal_metrics: Default::default(), + truncate: Default::default(), + }, + ); + + let rt = runtime(); + let (topology, input) = rt.block_on(async move { + let (topology, _crash) = start_topology(config.build().unwrap(), false).await; + + let mut options = OpenOptions::new(); + options.create(true).append(true).write(true); + + let input = options.open(active_file).await.unwrap(); + + // Give the source enough time to discover files and drive the idle watchers to EOF. + tokio::time::sleep(Duration::from_millis(250)).await; + + (topology, input) + }); + + (rt, topology, input, temp) +} + fn benchmark_files_no_partitions(c: &mut Criterion) { let num_lines: usize = 10_000; let line_size: usize = 100; @@ -99,10 +163,53 @@ fn benchmark_files_no_partitions(c: &mut Criterion) { group.finish(); } +fn benchmark_files_with_idle_watchers(c: &mut Criterion) { + let num_lines: usize = 10_000; + let line_size: usize = 100; + + let mut group = c.benchmark_group("files/idle_watchers"); + group.throughput(Throughput::Bytes((num_lines * line_size) as u64)); + group.sampling_mode(SamplingMode::Flat); + group.sample_size(10); + + for idle_files in [0usize, 128, 512] { + group.bench_with_input( + BenchmarkId::from_parameter(idle_files), + &idle_files, + |b, idle_files| { + b.iter_batched( + || build_file_benchmark_environment(*idle_files), + |(rt, topology, input, _temp)| { + rt.block_on(async move { + let mut sink = FramedWrite::new(input, BytesCodec::new()); + let raw_lines = + random_lines(line_size).take(num_lines).map(|mut line| { + line.push('\n'); + Bytes::from(line) + }); + let mut lines = stream::iter(raw_lines); + while let Some(line) = lines.next().await { + sink.send(line).await.unwrap(); + } + + // Keep the topology alive briefly so the benchmark includes watcher polling work. + tokio::time::sleep(Duration::from_millis(250)).await; + topology.stop().await; + }); + }, + BatchSize::LargeInput, + ) + }, + ); + } + + group.finish(); +} + criterion_group!( name = benches; // encapsulates inherent CI noise we saw in // https://github.com/vectordotdev/vector/issues/5394 config = Criterion::default().noise_threshold(0.05); - targets = benchmark_files_no_partitions + targets = benchmark_files_no_partitions, benchmark_files_with_idle_watchers ); diff --git a/changelog.d/25064_fixed_high_cpu_usage_in_file_source.fix.md b/changelog.d/25064_fixed_high_cpu_usage_in_file_source.fix.md new file mode 100644 index 0000000000000..8d64593e51b5c --- /dev/null +++ b/changelog.d/25064_fixed_high_cpu_usage_in_file_source.fix.md @@ -0,0 +1,3 @@ +Fixed an issue in the `file`/`kubernetes_logs` source that could cause unexpectedly high CPU usage after the async file server migration. + +authors: fcfangcc diff --git a/lib/file-source-common/src/buffer.rs b/lib/file-source-common/src/buffer.rs index 097331fbf2be3..099d896f4e1e6 100644 --- a/lib/file-source-common/src/buffer.rs +++ b/lib/file-source-common/src/buffer.rs @@ -1,5 +1,5 @@ use crate::FilePosition; -use std::{cmp::min, io, pin::Pin}; +use std::{cmp::min, io}; use bstr::Finder; use bytes::BytesMut; @@ -34,7 +34,7 @@ pub struct ReadResult { /// GiB/s range for buffers of length 1KiB. For buffers any smaller than this /// the overhead of setup dominates our benchmarks. pub async fn read_until_with_max_size<'a, R: AsyncBufRead + ?Sized + Unpin>( - reader: Pin>, + reader: &'a mut R, position: &'a mut FilePosition, delim: &'a [u8], buf: &'a mut BytesMut, @@ -45,7 +45,6 @@ pub async fn read_until_with_max_size<'a, R: AsyncBufRead + ?Sized + Unpin>( let delim_finder = Finder::new(delim); let delim_len = delim.len(); let mut discarded_for_size_and_truncated = Vec::new(); - let mut reader = Box::new(reader); // Used to track partial delimiter matches across buffer boundaries. // Data is read in chunks from the reader (see `fill_buf` below). @@ -294,7 +293,7 @@ mod test { let mut reader = BufReader::new(Cursor::new(&chunk)); match read_until_with_max_size( - Box::pin(&mut reader), + &mut reader, &mut position, &delimiter, &mut buffer, @@ -425,7 +424,7 @@ mod test { for (i, expected_line) in expected_lines.iter().enumerate() { let mut buffer = BytesMut::new(); let result = read_until_with_max_size( - Box::pin(&mut reader), + &mut reader, &mut position, delimiter, &mut buffer, diff --git a/lib/file-source/src/file_watcher/mod.rs b/lib/file-source/src/file_watcher/mod.rs index b750a81433886..6e103cf13f39b 100644 --- a/lib/file-source/src/file_watcher/mod.rs +++ b/lib/file-source/src/file_watcher/mod.rs @@ -19,6 +19,9 @@ use file_source_common::{ buffer::{ReadResult, read_until_with_max_size}, }; +const EOF_READ_BACKOFF_MIN: Duration = Duration::from_millis(1); +const EOF_READ_BACKOFF_MAX: Duration = Duration::from_millis(250); + #[cfg(test)] mod tests; @@ -57,6 +60,7 @@ pub struct FileWatcher { reached_eof: bool, last_read_attempt: Instant, last_read_success: Instant, + read_retry_delay: Duration, last_seen: Instant, max_line_bytes: usize, line_delimiter: Bytes, @@ -166,6 +170,7 @@ impl FileWatcher { reached_eof: false, last_read_attempt: ts, last_read_success: ts, + read_retry_delay: EOF_READ_BACKOFF_MIN, last_seen: ts, max_line_bytes, line_delimiter, @@ -196,6 +201,8 @@ impl FileWatcher { self.devno = file_info.portable_dev(); self.inode = file_info.portable_ino(); } + self.reached_eof = false; + self.read_retry_delay = EOF_READ_BACKOFF_MIN; self.path = path; Ok(()) } @@ -235,7 +242,7 @@ impl FileWatcher { let file_position = &mut self.file_position; let initial_position = *file_position; match read_until_with_max_size( - Box::pin(reader), + reader.as_mut(), file_position, self.line_delimiter.as_ref(), &mut self.buf, @@ -283,7 +290,7 @@ impl FileWatcher { }) } } else { - self.reached_eof = true; + self.track_read_eof(); Ok(RawLineResult { raw_line: None, discarded_for_size_and_truncated, @@ -306,9 +313,24 @@ impl FileWatcher { #[inline] fn track_read_success(&mut self) { + self.reached_eof = false; + self.read_retry_delay = EOF_READ_BACKOFF_MIN; self.last_read_success = Instant::now(); } + #[inline] + fn track_read_eof(&mut self) { + self.read_retry_delay = if self.reached_eof { + std::cmp::min( + self.read_retry_delay.saturating_mul(2), + EOF_READ_BACKOFF_MAX, + ) + } else { + EOF_READ_BACKOFF_MIN + }; + self.reached_eof = true; + } + #[inline] pub fn last_read_success(&self) -> Instant { self.last_read_success @@ -316,6 +338,10 @@ impl FileWatcher { #[inline] pub fn should_read(&self) -> bool { + if self.reached_eof && self.last_read_attempt.elapsed() < self.read_retry_delay { + return false; + } + self.last_read_success.elapsed() < Duration::from_secs(10) || self.last_read_attempt.elapsed() > Duration::from_secs(10) } diff --git a/lib/file-source/src/file_watcher/tests/mod.rs b/lib/file-source/src/file_watcher/tests/mod.rs index 7a9cc92909db0..5384a43802d24 100644 --- a/lib/file-source/src/file_watcher/tests/mod.rs +++ b/lib/file-source/src/file_watcher/tests/mod.rs @@ -1,9 +1,13 @@ mod experiment; mod experiment_no_truncations; -use std::str; +use std::{path::PathBuf, str, thread}; +use bytes::{Bytes, BytesMut}; use quickcheck::{Arbitrary, Gen}; +use tokio::time::Instant; + +use super::{EOF_READ_BACKOFF_MAX, EOF_READ_BACKOFF_MIN, FileWatcher, null_reader}; // Welcome. // @@ -171,6 +175,68 @@ impl Arbitrary for FileWatcherAction { } } +fn watcher_for_timing() -> FileWatcher { + let now = Instant::now(); + + FileWatcher { + path: PathBuf::new(), + findable: true, + reader: Box::new(null_reader()), + file_position: 0, + devno: 0, + inode: 0, + is_dead: false, + reached_eof: false, + last_read_attempt: now, + last_read_success: now, + read_retry_delay: EOF_READ_BACKOFF_MIN, + last_seen: now, + max_line_bytes: 1024, + line_delimiter: Bytes::from_static(b"\n"), + buf: BytesMut::new(), + } +} + +#[test] +fn backs_off_after_eof() { + let mut watcher = watcher_for_timing(); + + watcher.track_read_attempt(); + watcher.track_read_eof(); + + assert_eq!(watcher.read_retry_delay, EOF_READ_BACKOFF_MIN); + assert!(!watcher.should_read()); + + thread::sleep(EOF_READ_BACKOFF_MIN); + + assert!(watcher.should_read()); + + watcher.track_read_attempt(); + watcher.track_read_eof(); + + assert_eq!( + watcher.read_retry_delay, + EOF_READ_BACKOFF_MIN.saturating_mul(2) + ); +} + +#[test] +fn caps_and_resets_eof_backoff() { + let mut watcher = watcher_for_timing(); + + for _ in 0..16 { + watcher.track_read_attempt(); + watcher.track_read_eof(); + } + + assert_eq!(watcher.read_retry_delay, EOF_READ_BACKOFF_MAX); + + watcher.track_read_success(); + + assert_eq!(watcher.read_retry_delay, EOF_READ_BACKOFF_MIN); + assert!(!watcher.reached_eof()); +} + #[inline] pub fn delay(attempts: u32) { let delay = match attempts { From 9c4abf31a91d888d0b4ca204643b0e6e1a559466 Mon Sep 17 00:00:00 2001 From: tronboto <142882846+tronboto@users.noreply.github.com> Date: Wed, 8 Apr 2026 21:26:19 +0100 Subject: [PATCH 114/326] fix(shutdown): emit VectorStopped after topology drains (#25083) * move VectorStopped event * add changelog * Add VectorStopping event * Update changelog.d/24532_vector_stopped_log_ordering.fix.md * Apply suggestion from @pront --------- Co-authored-by: Pavlos Rontidis --- .../24532_vector_stopped_log_ordering.fix.md | 5 ++++ src/app.rs | 25 +++++++++++-------- src/internal_events/process.rs | 12 +++++++++ 3 files changed, 32 insertions(+), 10 deletions(-) create mode 100644 changelog.d/24532_vector_stopped_log_ordering.fix.md diff --git a/changelog.d/24532_vector_stopped_log_ordering.fix.md b/changelog.d/24532_vector_stopped_log_ordering.fix.md new file mode 100644 index 0000000000000..45f1a82eea06f --- /dev/null +++ b/changelog.d/24532_vector_stopped_log_ordering.fix.md @@ -0,0 +1,5 @@ +Fixed log message ordering on shutdown where `Vector has stopped.` was logged before components had finished draining, causing confusing output interleaved with `Waiting on running components` messages. + +A new `VectorStoppping` event was added in the place of the `VectorStopped` event. + +authors: tronboto diff --git a/src/app.rs b/src/app.rs index 6f30f85de1d20..904d42be11968 100644 --- a/src/app.rs +++ b/src/app.rs @@ -28,7 +28,9 @@ use crate::{ config::{self, ComponentConfig, ComponentType, Config, ConfigPath}, extra_context::ExtraContext, heartbeat, - internal_events::{VectorConfigLoadError, VectorQuit, VectorStarted, VectorStopped}, + internal_events::{ + VectorConfigLoadError, VectorQuit, VectorStarted, VectorStopped, VectorStopping, + }, signal::{SignalHandler, SignalPair, SignalRx, SignalTo}, topology::{ ReloadOutcome, RunningTopology, SharedTopologyController, ShutdownErrorReceiver, @@ -493,16 +495,19 @@ impl FinishedApplication { } async fn stop(topology_controller: TopologyController, mut signal_rx: SignalRx) -> ExitStatus { - emit!(VectorStopped); + emit!(VectorStopping); tokio::select! { - _ = topology_controller.stop() => ExitStatus::from_raw({ - #[cfg(windows)] - { - exitcode::OK as u32 - } - #[cfg(unix)] - exitcode::OK - }), // Graceful shutdown finished + _ = topology_controller.stop() => { + emit!(VectorStopped); + ExitStatus::from_raw({ + #[cfg(windows)] + { + exitcode::OK as u32 + } + #[cfg(unix)] + exitcode::OK + }) + }, // Graceful shutdown finished _ = signal_rx.recv() => Self::quit(), } } diff --git a/src/internal_events/process.rs b/src/internal_events/process.rs index cc3609737aea9..987724c2da18d 100644 --- a/src/internal_events/process.rs +++ b/src/internal_events/process.rs @@ -38,6 +38,18 @@ impl InternalEvent for VectorReloaded<'_> { } } +#[derive(Debug, NamedInternalEvent)] +pub struct VectorStopping; + +impl InternalEvent for VectorStopping { + fn emit(self) { + info!( + target: "vector", + message = "Vector is stopping.", + ); + } +} + #[derive(Debug, NamedInternalEvent)] pub struct VectorStopped; From 26eef13087e1a463efa4bdcc9d3919b13936e06c Mon Sep 17 00:00:00 2001 From: Lisa Vu Date: Wed, 8 Apr 2026 16:26:27 -0400 Subject: [PATCH 115/326] fix(datadog_agent source): preserve `device` tag from v2 series resources (#25146) * fix(datadog_agent source): preserve `device` tag from v2 series resources The v2 series protobuf format encodes `device` in the `resources` array, but the decoder only special-cased `host`, causing `device` to be incorrectly remapped to `resource.device`. This broke tag continuity for disk, SNMP, and other integrations routing through OPW. * fix(changelog): use GitHub username for authors field The spell checker pattern for changelog authors only allows alphanumeric characters and hyphens. Changed from `lisa.vu` to `lisaqvu` (GitHub username) to match the expected format. --- .../datadog_agent_source_device_tag_v2.fix.md | 5 ++ src/sources/datadog_agent/metrics.rs | 4 ++ src/sources/datadog_agent/tests.rs | 60 +++++++++++++++++++ 3 files changed, 69 insertions(+) create mode 100644 changelog.d/datadog_agent_source_device_tag_v2.fix.md diff --git a/changelog.d/datadog_agent_source_device_tag_v2.fix.md b/changelog.d/datadog_agent_source_device_tag_v2.fix.md new file mode 100644 index 0000000000000..669d594f8c89f --- /dev/null +++ b/changelog.d/datadog_agent_source_device_tag_v2.fix.md @@ -0,0 +1,5 @@ +`datadog_agent` source: Preserve `device` as a plain tag when decoding v2 series metrics, +instead of incorrectly prefixing it as `resource.device`. This matches the v1 series behavior +and fixes tag remapping for disk, SNMP, and other integrations that use the `device` resource type. + +authors: lisaqvu diff --git a/src/sources/datadog_agent/metrics.rs b/src/sources/datadog_agent/metrics.rs index d931be05fcab7..6f377e826a8b8 100644 --- a/src/sources/datadog_agent/metrics.rs +++ b/src/sources/datadog_agent/metrics.rs @@ -289,6 +289,10 @@ pub(crate) fn decode_ddseries_v2( log_schema() .host_key() .and_then(|key| tags.replace(key.to_string(), r.name)); + } else if r.r#type.eq("device") { + // The `device` resource type is used by Agent checks (disk, SNMP/NDM, etc.) + // and must be preserved as a plain `device` tag to match the v1 series behavior. + tags.replace("device".into(), r.name); } else { // But to avoid losing information if this situation changes, any other resource type/name will be saved in the tags map tags.replace(format!("resource.{}", r.r#type), r.name); diff --git a/src/sources/datadog_agent/tests.rs b/src/sources/datadog_agent/tests.rs index 065b6ec5f78f5..b2dbf9691db00 100644 --- a/src/sources/datadog_agent/tests.rs +++ b/src/sources/datadog_agent/tests.rs @@ -2648,6 +2648,66 @@ async fn series_v2_split_metric_namespace_false() { .await; } +#[tokio::test] +async fn series_v2_device_resource_preserved_as_tag() { + assert_source_compliance(&HTTP_PUSH_SOURCE_TAGS, async { + let (rx, _, _, addr, _guard) = + source(EventStatus::Delivered, true, true, false, false).await; + + let series = vec![ddmetric_proto::metric_payload::MetricSeries { + resources: vec![ + ddmetric_proto::metric_payload::Resource { + r#type: "host".to_string(), + name: "test_host".to_string(), + }, + ddmetric_proto::metric_payload::Resource { + r#type: "device".to_string(), + name: "sda".to_string(), + }, + ], + metric: "system.disk.free".to_string(), + tags: vec!["env:prod".to_string()], + points: vec![ddmetric_proto::metric_payload::MetricPoint { + value: 100.0, + timestamp: 1542182950, + }], + r#type: ddmetric_proto::metric_payload::MetricType::Gauge as i32, + unit: "".to_string(), + source_type_name: "".to_string(), + interval: 0, + metadata: None, + }]; + + let series_payload = ddmetric_proto::MetricPayload { series }; + let mut buf = Vec::new(); + series_payload.encode(&mut buf).unwrap(); + let body = unsafe { String::from_utf8_unchecked(buf) }; + + let events = send_and_collect( + addr, + body, + dd_api_key_headers(), + DD_API_SERIES_V2_PATH, + rx, + 1, + ) + .await; + + let metric = events[0].as_metric(); + let tags = metric.tags().unwrap(); + + // The `device` resource type must be preserved as a plain `device` tag, + // NOT as `resource.device`. This matches v1 series behavior. + assert_eq!(tags.get("device"), Some("sda")); + assert!( + tags.get("resource.device").is_none(), + "device should not be prefixed with 'resource.'" + ); + assert_eq!(tags.get("env"), Some("prod")); + }) + .await; +} + async fn test_sketches_split_metric_namespace_impl( split: bool, expected_name: &str, From b57d8b0ef0ec12a1645b556a6b1bf080dc66e50b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 8 Apr 2026 20:53:19 +0000 Subject: [PATCH 116/326] chore(deps): bump basic-ftp from 5.2.0 to 5.2.1 in /scripts/environment/npm-tools in the npm_and_yarn group across 1 directory (#25147) chore(deps): bump basic-ftp Bumps the npm_and_yarn group with 1 update in the /scripts/environment/npm-tools directory: [basic-ftp](https://github.com/patrickjuchli/basic-ftp). Updates `basic-ftp` from 5.2.0 to 5.2.1 - [Release notes](https://github.com/patrickjuchli/basic-ftp/releases) - [Changelog](https://github.com/patrickjuchli/basic-ftp/blob/master/CHANGELOG.md) - [Commits](https://github.com/patrickjuchli/basic-ftp/compare/v5.2.0...v5.2.1) --- updated-dependencies: - dependency-name: basic-ftp dependency-version: 5.2.1 dependency-type: indirect dependency-group: npm_and_yarn ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- scripts/environment/npm-tools/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/environment/npm-tools/package-lock.json b/scripts/environment/npm-tools/package-lock.json index 1047d5444f89a..708206c20ade0 100644 --- a/scripts/environment/npm-tools/package-lock.json +++ b/scripts/environment/npm-tools/package-lock.json @@ -596,9 +596,9 @@ "license": "MIT" }, "node_modules/basic-ftp": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.2.0.tgz", - "integrity": "sha512-VoMINM2rqJwJgfdHq6RiUudKt2BV+FY5ZFezP/ypmwayk68+NzzAQy4XXLlqsGD4MCzq3DrmNFD/uUmBJuGoXw==", + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.2.1.tgz", + "integrity": "sha512-0yaL8JdxTknKDILitVpfYfV2Ob6yb3udX/hK97M7I3jOeznBNxQPtVvTUtnhUkyHlxFWyr5Lvknmgzoc7jf+1Q==", "license": "MIT", "engines": { "node": ">=10.0.0" From 51f6fce6d850dc4c6833d910013a189119068734 Mon Sep 17 00:00:00 2001 From: Jonathan Davies Date: Thu, 9 Apr 2026 13:35:17 +0000 Subject: [PATCH 117/326] chore(shutdown): fix typo in 24532 changelog fragment (#25155) --- changelog.d/24532_vector_stopped_log_ordering.fix.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/24532_vector_stopped_log_ordering.fix.md b/changelog.d/24532_vector_stopped_log_ordering.fix.md index 45f1a82eea06f..c5e395c37ab6f 100644 --- a/changelog.d/24532_vector_stopped_log_ordering.fix.md +++ b/changelog.d/24532_vector_stopped_log_ordering.fix.md @@ -1,5 +1,5 @@ Fixed log message ordering on shutdown where `Vector has stopped.` was logged before components had finished draining, causing confusing output interleaved with `Waiting on running components` messages. -A new `VectorStoppping` event was added in the place of the `VectorStopped` event. +A new `VectorStopping` event was added in the place of the `VectorStopped` event. authors: tronboto From a7f6487c94b98766bdff802aac0cd1372b0311f6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Apr 2026 13:07:40 +0000 Subject: [PATCH 118/326] chore(deps): bump basic-ftp from 5.2.1 to 5.2.2 in /scripts/environment/npm-tools in the npm_and_yarn group across 1 directory (#25170) chore(deps): bump basic-ftp Bumps the npm_and_yarn group with 1 update in the /scripts/environment/npm-tools directory: [basic-ftp](https://github.com/patrickjuchli/basic-ftp). Updates `basic-ftp` from 5.2.1 to 5.2.2 - [Release notes](https://github.com/patrickjuchli/basic-ftp/releases) - [Changelog](https://github.com/patrickjuchli/basic-ftp/blob/master/CHANGELOG.md) - [Commits](https://github.com/patrickjuchli/basic-ftp/compare/v5.2.1...v5.2.2) --- updated-dependencies: - dependency-name: basic-ftp dependency-version: 5.2.2 dependency-type: indirect dependency-group: npm_and_yarn ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- scripts/environment/npm-tools/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/environment/npm-tools/package-lock.json b/scripts/environment/npm-tools/package-lock.json index 708206c20ade0..9e25b17830d18 100644 --- a/scripts/environment/npm-tools/package-lock.json +++ b/scripts/environment/npm-tools/package-lock.json @@ -596,9 +596,9 @@ "license": "MIT" }, "node_modules/basic-ftp": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.2.1.tgz", - "integrity": "sha512-0yaL8JdxTknKDILitVpfYfV2Ob6yb3udX/hK97M7I3jOeznBNxQPtVvTUtnhUkyHlxFWyr5Lvknmgzoc7jf+1Q==", + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.2.2.tgz", + "integrity": "sha512-1tDrzKsdCg70WGvbFss/ulVAxupNauGnOlgpyjKzeQxzyllBLS0CGLV7tjIXTK3ZQA9/FBEm9qyFFN1bciA6pw==", "license": "MIT", "engines": { "node": ">=10.0.0" From 5ab0ba0d27392c70995a3d5e182a8307e65c641f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Apr 2026 10:37:29 -0400 Subject: [PATCH 119/326] chore(website deps): bump axios from 1.13.5 to 1.15.0 in /website (#25172) Bumps [axios](https://github.com/axios/axios) from 1.13.5 to 1.15.0. - [Release notes](https://github.com/axios/axios/releases) - [Changelog](https://github.com/axios/axios/blob/v1.x/CHANGELOG.md) - [Commits](https://github.com/axios/axios/compare/v1.13.5...v1.15.0) --- updated-dependencies: - dependency-name: axios dependency-version: 1.15.0 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- website/yarn.lock | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/website/yarn.lock b/website/yarn.lock index 055e51daa030a..31254815a65f1 100644 --- a/website/yarn.lock +++ b/website/yarn.lock @@ -1641,13 +1641,13 @@ autoprefixer@^10.2.5: postcss-value-parser "^4.2.0" axios@^1.6.0, axios@^1.8.4: - version "1.13.5" - resolved "https://registry.yarnpkg.com/axios/-/axios-1.13.5.tgz#5e464688fa127e11a660a2c49441c009f6567a43" - integrity sha512-cz4ur7Vb0xS4/KUN0tPWe44eqxrIu31me+fbang3ijiNscE129POzipJJA6zniq2C/Z6sJCjMimjS8Lc/GAs8Q== + version "1.15.0" + resolved "https://registry.yarnpkg.com/axios/-/axios-1.15.0.tgz#0fcee91ef03d386514474904b27863b2c683bf4f" + integrity sha512-wWyJDlAatxk30ZJer+GeCWS209sA42X+N5jU2jy6oHTp7ufw8uzUTVFBX9+wTfAlhiJXGS0Bq7X6efruWjuK9Q== dependencies: follow-redirects "^1.15.11" form-data "^4.0.5" - proxy-from-env "^1.1.0" + proxy-from-env "^2.1.0" babel-plugin-macros@^3.1.0: version "3.1.0" @@ -3190,10 +3190,10 @@ prop-types@^15.7.2: object-assign "^4.1.1" react-is "^16.13.1" -proxy-from-env@^1.1.0: - version "1.1.0" - resolved "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz" - integrity sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg== +proxy-from-env@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-2.1.0.tgz#a7487568adad577cfaaa7e88c49cab3ab3081aba" + integrity sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA== purgecss@^4.0.3, purgecss@^4.1.3: version "4.1.3" From 480ca58f52a019454a36e9c9b754791224fcee57 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 13 Apr 2026 11:22:13 -0400 Subject: [PATCH 120/326] chore(vdev): extract vector-vrl-doc-builder into unpublished crate (#25034) * chore(vdev): extract vector-vrl-doc-builder into unpublished crate * fix(ci): address issues caught by Codex code review Allow clippy::print_stdout on the doc-builder binary (the workspace-wide lint denial would otherwise fail clippy) and explicitly request Rust in the VRL docs CI workflow so it does not rely on cache state. Co-Authored-By: Claude Opus 4.6 (1M context) * fix(ci): update VRL doc-builder workflow filters, caching, and CLI modes - Replace stale vdev/** path filter with Makefile, drop vdev: true - Add install-vrl-doc-builder action with binary caching (mirrors vdev pattern) - Use clap conflicts_with/requires for mutually exclusive CLI flags - Makefile auto-detects installed binary, falls back to cargo run Co-Authored-By: Claude Opus 4.6 (1M context) * fix(ci): add protoc to VRL doc-builder workflow vector-core's build.rs requires protoc for proto compilation. Previously protoc was installed implicitly via VDEV_NEEDS_COMPILE; now that vdev is disabled we need to request it explicitly. Co-Authored-By: Claude Opus 4.6 (1M context) * chore(ci): drop redundant false overrides from setup action call mold and cargo-cache default to true which is fine; no need to explicitly disable them. Co-Authored-By: Claude Opus 4.6 (1M context) --------- Co-authored-by: Pavlos Rontidis Co-authored-by: Claude Opus 4.6 (1M context) --- .../install-vrl-doc-builder/action.yml | 25 ++++++++++ .../workflows/check_generated_vrl_docs.yml | 10 ++-- Cargo.lock | 13 ++++- Cargo.toml | 1 + Makefile | 9 +++- lib/vector-vrl/doc-builder/Cargo.toml | 14 ++++++ lib/vector-vrl/doc-builder/src/main.rs | 49 +++++++++++++++++++ vdev/Cargo.toml | 2 - vdev/src/commands/build/mod.rs | 2 - vdev/src/commands/build/vector_vrl_docs.rs | 47 ------------------ 10 files changed, 114 insertions(+), 58 deletions(-) create mode 100644 .github/actions/install-vrl-doc-builder/action.yml create mode 100644 lib/vector-vrl/doc-builder/Cargo.toml create mode 100644 lib/vector-vrl/doc-builder/src/main.rs delete mode 100644 vdev/src/commands/build/vector_vrl_docs.rs diff --git a/.github/actions/install-vrl-doc-builder/action.yml b/.github/actions/install-vrl-doc-builder/action.yml new file mode 100644 index 0000000000000..3748e02e8b295 --- /dev/null +++ b/.github/actions/install-vrl-doc-builder/action.yml @@ -0,0 +1,25 @@ +name: "Install vector-vrl-doc-builder" +description: "Install vector-vrl-doc-builder CLI tool with caching based on source code changes" + +branding: + icon: tool + color: purple + +runs: + using: "composite" + steps: + - name: Cache vrl-doc-builder binary + id: cache-vrl-doc-builder + uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 + with: + path: ~/.cargo/bin/vector-vrl-doc-builder + key: ${{ runner.os }}-vrl-doc-builder-${{ hashFiles('lib/vector-vrl/**', 'Cargo.toml', 'Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-vrl-doc-builder- + + - name: Build and install vector-vrl-doc-builder + if: steps.cache-vrl-doc-builder.outputs.cache-hit != 'true' + shell: bash + run: | + echo "Building vector-vrl-doc-builder from source (cache miss)" + cargo install --path lib/vector-vrl/doc-builder --locked --force diff --git a/.github/workflows/check_generated_vrl_docs.yml b/.github/workflows/check_generated_vrl_docs.yml index e183e12578f6e..c2a9852474461 100644 --- a/.github/workflows/check_generated_vrl_docs.yml +++ b/.github/workflows/check_generated_vrl_docs.yml @@ -34,7 +34,7 @@ jobs: filters: | docs: - "lib/vector-vrl/**" - - "vdev/**" + - "Makefile" - "Cargo.lock" - ".github/workflows/check_generated_vrl_docs.yml" @@ -51,9 +51,11 @@ jobs: - uses: ./.github/actions/setup with: - vdev: true - mold: false - cargo-cache: false + rust: true + protoc: true + vdev: false + + - uses: ./.github/actions/install-vrl-doc-builder - name: Regenerate VRL docs run: make generate-vector-vrl-docs diff --git a/Cargo.lock b/Cargo.lock index 45f4a998276fc..054d3afc66007 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12226,8 +12226,6 @@ dependencies = [ "tempfile", "toml", "toml_edit 0.23.9", - "vector-vrl-functions", - "vrl", ] [[package]] @@ -12788,6 +12786,17 @@ dependencies = [ "vrl", ] +[[package]] +name = "vector-vrl-doc-builder" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "serde_json", + "vector-vrl-functions", + "vrl", +] + [[package]] name = "vector-vrl-functions" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 25ca1f9e0dec9..9389c9420d248 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -127,6 +127,7 @@ members = [ "lib/vector-vrl/category", "lib/vector-vrl/cli", "lib/vector-vrl/dnstap-parser", + "lib/vector-vrl/doc-builder", "lib/vector-vrl/enrichment", "lib/vector-vrl/functions", "lib/vector-vrl/metrics", diff --git a/Makefile b/Makefile index a923806254d9f..34da1302487d9 100644 --- a/Makefile +++ b/Makefile @@ -734,9 +734,16 @@ generate-component-docs: ## Generate per-component Cue docs from the configurati $(if $(findstring true,$(CI)),>/dev/null,) ./scripts/cue.sh fmt +VRL_DOC_BUILDER := $(shell command -v vector-vrl-doc-builder 2>/dev/null) +ifndef VRL_DOC_BUILDER +VRL_DOC_BUILDER_CMD = cargo run -p vector-vrl-doc-builder -- +else +VRL_DOC_BUILDER_CMD = vector-vrl-doc-builder +endif + .PHONY: generate-vector-vrl-docs generate-vector-vrl-docs: ## Generate VRL function documentation from Rust source. - ${MAYBE_ENVIRONMENT_EXEC} $(VDEV) build vector-vrl-docs --output docs/generated/ \ + ${MAYBE_ENVIRONMENT_EXEC} $(VRL_DOC_BUILDER_CMD) --output docs/generated/ \ $(if $(findstring true,$(CI)),>/dev/null,) .PHONY: generate-vrl-docs diff --git a/lib/vector-vrl/doc-builder/Cargo.toml b/lib/vector-vrl/doc-builder/Cargo.toml new file mode 100644 index 0000000000000..75a1d262f4708 --- /dev/null +++ b/lib/vector-vrl/doc-builder/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "vector-vrl-doc-builder" +version = "0.1.0" +authors = ["Vector Contributors "] +edition = "2024" +publish = false +license = "MPL-2.0" + +[dependencies] +anyhow.workspace = true +clap.workspace = true +serde_json = { workspace = true, features = ["preserve_order"] } +vector-vrl-functions = { path = "../functions", features = ["dnstap", "vrl-metrics"] } +vrl = { workspace = true, features = ["docs"] } diff --git a/lib/vector-vrl/doc-builder/src/main.rs b/lib/vector-vrl/doc-builder/src/main.rs new file mode 100644 index 0000000000000..92c38dfea307d --- /dev/null +++ b/lib/vector-vrl/doc-builder/src/main.rs @@ -0,0 +1,49 @@ +use anyhow::Result; +use std::path::PathBuf; +use vrl::docs::{build_functions_doc, document_functions_to_dir}; + +/// Generate Vector-specific VRL function documentation as JSON files. +/// +/// Two modes of operation: +/// --output Write one JSON file per function into DIR (uses --extension). +/// (no --output) Print all functions to stdout as a JSON array (uses --minify). +#[derive(clap::Parser, Debug)] +#[command()] +struct Cli { + /// Output directory to create JSON files. When omitted, output is written to stdout as a JSON + /// array. + #[arg(short, long, conflicts_with = "minify")] + output: Option, + + /// Whether to minify the JSON output (stdout mode only) + #[arg(short, long, default_value_t = false, conflicts_with = "output")] + minify: bool, + + /// File extension for generated files (directory mode only) + #[arg(short, long, default_value = "json", requires = "output")] + extension: String, +} + +#[allow(clippy::print_stdout)] +fn main() -> Result<()> { + let cli = ::parse(); + let functions = vector_vrl_functions::all_without_vrl_stdlib(); + if let Some(output) = &cli.output { + document_functions_to_dir(&functions, output, &cli.extension)?; + } else { + let built = build_functions_doc(&functions); + if cli.minify { + println!( + "{}", + serde_json::to_string(&built).expect("FunctionDoc serialization should not fail") + ); + } else { + println!( + "{}", + serde_json::to_string_pretty(&built) + .expect("FunctionDoc serialization should not fail") + ); + } + } + Ok(()) +} diff --git a/vdev/Cargo.toml b/vdev/Cargo.toml index 8fea6eb281d0a..7de3fc06d5a59 100644 --- a/vdev/Cargo.toml +++ b/vdev/Cargo.toml @@ -46,8 +46,6 @@ strum.workspace = true indoc.workspace = true git2 = { version = "0.20.4" } cfg-if.workspace = true -vector-vrl-functions = { path = "../lib/vector-vrl/functions", features = ["dnstap", "vrl-metrics"] } -vrl = { workspace = true, features = ["docs"] } [package.metadata.binstall] pkg-url = "{ repo }/releases/download/vdev-v{ version }/{ name }-{ target }-v{ version }.tgz" diff --git a/vdev/src/commands/build/mod.rs b/vdev/src/commands/build/mod.rs index 845ae928b84d9..f46d8a8ae210c 100644 --- a/vdev/src/commands/build/mod.rs +++ b/vdev/src/commands/build/mod.rs @@ -1,7 +1,6 @@ mod licenses; mod publish_metadata; mod vector; -mod vector_vrl_docs; mod vrl_docs; mod vrl_wasm; @@ -14,7 +13,6 @@ crate::cli_subcommands! { release_cue, vector, vrl_docs, - vector_vrl_docs, vrl_wasm, } diff --git a/vdev/src/commands/build/vector_vrl_docs.rs b/vdev/src/commands/build/vector_vrl_docs.rs deleted file mode 100644 index 7c25a23b139c3..0000000000000 --- a/vdev/src/commands/build/vector_vrl_docs.rs +++ /dev/null @@ -1,47 +0,0 @@ -use anyhow::Result; -use std::path::PathBuf; -use vrl::docs::{build_functions_doc, document_functions_to_dir}; - -/// Generate Vector-specific VRL function documentation as JSON files. -#[derive(clap::Args, Debug)] -#[command()] -pub struct Cli { - /// Output directory to create JSON files. If unspecified output is written to stdout as a JSON - /// array - #[arg(short, long)] - output: Option, - - /// Whether to pretty-print or minify - #[arg(short, long, default_value_t = false)] - minify: bool, - - /// File extension for generated files - #[arg(short, long, default_value = "json")] - extension: String, -} - -impl Cli { - pub fn exec(self) -> Result<()> { - let functions = vector_vrl_functions::all_without_vrl_stdlib(); - if let Some(output) = &self.output { - document_functions_to_dir(&functions, output, &self.extension)?; - } else { - let built = build_functions_doc(&functions); - #[allow(clippy::print_stdout)] - if self.minify { - println!( - "{}", - serde_json::to_string(&built) - .expect("FunctionDoc serialization should not fail") - ); - } else { - println!( - "{}", - serde_json::to_string_pretty(&built) - .expect("FunctionDoc serialization should not fail") - ); - } - } - Ok(()) - } -} From 3112033384d47b0eae9fa36a1297dab32cac1f88 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Mon, 13 Apr 2026 12:08:25 -0400 Subject: [PATCH 121/326] docs(website): convert TOML config examples to YAML (#25163) * docs(website): convert TOML config examples to YAML across website YAML is Vector's recommended and default configuration format. This converts all TOML-only config examples in website docs, guides, blog posts, and highlight files to YAML for consistency. Co-Authored-By: Claude Opus 4.6 (1M context) * docs(website): add YAML recommendation to multi-format references Update docs that mention TOML/YAML/JSON support to explicitly recommend YAML as the default configuration format. Co-Authored-By: Claude Opus 4.6 (1M context) * docs: add YAML-default guidance to AGENTS.md Instructs AI tools to generate Vector config in YAML unless explicitly asked otherwise. Co-Authored-By: Claude Opus 4.6 (1M context) * docs: remove version reference from AGENTS.md YAML guidance Co-Authored-By: Claude Opus 4.6 (1M context) * docs(website): fix issues from code review - Fix invalid YAML escape in template-syntax.md (use single quotes) - Fix `transform:` typo to `transforms:` in 0-25-0-upgrade-guide.md - Update remaining `vector.toml` prose references to `vector.yaml` - Remove anachronistic version reference from 2020 highlight Co-Authored-By: Claude Opus 4.6 (1M context) * docs(website): add YAML/TOML format picker tabs to template-syntax.md Add tabbed config format picker to all examples so users can toggle between YAML and TOML. Defaults to YAML. Co-Authored-By: Claude Opus 4.6 (1M context) --------- Co-authored-by: Claude Opus 4.6 (1M context) --- AGENTS.md | 4 + .../en/blog/adaptive-request-concurrency.md | 16 +- website/content/en/blog/graphql-api.md | 39 +- .../content/en/blog/kubernetes-integration.md | 20 +- website/content/en/blog/log-namespacing.md | 45 +- .../en/docs/administration/management.md | 2 +- .../en/docs/architecture/guarantees.md | 13 +- .../en/docs/architecture/pipeline-model.md | 2 +- .../en/docs/reference/configuration/_index.md | 5 +- .../configuration/template-syntax.md | 90 +++ .../reference/configuration/unit-tests.md | 528 ++++++++-------- .../setup/going-to-prod/high-availability.md | 37 +- .../advanced/custom-aggregations-with-lua.md | 245 ++++---- .../advanced/merge-multiline-logs-with-lua.md | 72 +-- .../advanced/parsing-csv-logs-with-lua.md | 193 +++--- .../en/guides/aws/cloudwatch-logs-firehose.md | 117 ++-- .../level-up/managing-complex-configs.md | 217 +++---- .../en/guides/level-up/managing-schemas.md | 232 +++---- .../en/guides/level-up/vector-tap-guide.md | 121 ++-- ...-11-25-unit-testing-vector-config-files.md | 57 +- .../en/highlights/2019-12-13-custom-dns.md | 4 +- .../en/highlights/2019-12-16-ec2-metadata.md | 26 +- .../2020-01-07-prometheus-source.md | 11 +- ...01-20-splunk-hec-specify-indexed-fields.md | 11 +- .../2020-02-05-merge-partial-docker-events.md | 9 +- .../2020-02-14-global-log-schema.md | 10 +- ...020-02-21-file-source-multiline-support.md | 20 +- .../2020-02-24-swimlanes-transform.md | 19 +- ...3-04-encoding-only-fields-except-fields.md | 16 +- .../highlights/2020-03-10-dedupe-transform.md | 16 +- ...0-03-11-tag-cardinality-limit-transform.md | 15 +- .../highlights/2020-03-31-filter-transform.md | 18 +- .../2020-04-01-more-condition-predicates.md | 10 +- ...27-add-support-for-loading-multiple-cas.md | 20 +- .../2020-07-10-add-reduce-transform.md | 79 +-- .../2020-09-18-adaptive-concurrency.md | 12 +- .../2020-10-27-metrics-integrations.md | 14 +- ...20-11-19-prometheus-remote-integrations.md | 39 +- .../2020-11-25-json-yaml-config-formats.md | 6 +- .../en/highlights/2020-12-23-graphql-api.md | 8 +- .../2020-12-23-internal-logs-source.md | 39 +- .../2021-01-10-kafka-sink-metrics.md | 17 +- .../2021-01-20-wildcard-identifiers.md | 34 +- .../2021-02-16-0-12-upgrade-guide.md | 12 +- .../2021-02-16-filter-remap-support.md | 34 +- .../en/highlights/2021-04-21-vector-tap.md | 29 +- .../en/highlights/2021-04-21-vrl-abort.md | 43 +- .../2021-07-14-0-15-upgrade-guide.md | 102 ++-- .../en/highlights/2021-07-14-vector-graph.md | 104 ++-- .../highlights/2021-07-16-remap-multiple.md | 65 +- .../en/highlights/2021-08-20-rate-limits.md | 7 +- .../2021-08-25-0-16-upgrade-guide.md | 32 +- .../en/highlights/2021-10-06-arc-default.md | 10 +- .../en/highlights/2021-10-06-source-codecs.md | 29 +- .../2021-11-12-event-throttle-transform.md | 13 +- .../2021-12-15-splunk-hec-improvements.md | 9 +- .../2021-12-28-0-19-0-upgrade-guide.md | 14 +- ...022-01-12-vector-unit-test-improvements.md | 44 +- .../2022-02-08-disk-buffer-v2-beta.md | 28 +- .../en/highlights/2022-03-15-vrl-vm-beta.md | 16 +- .../2022-03-22-0-21-0-upgrade-guide.md | 26 +- .../2022-03-31-native-event-codecs.md | 44 +- .../2022-05-03-0-22-0-upgrade-guide.md | 572 +++++++++--------- .../2022-07-07-0-23-0-upgrade-guide.md | 73 ++- .../2022-07-07-secrets-management.md | 25 +- .../en/highlights/2022-07-07-sink-codecs.md | 17 +- .../2022-10-04-0-25-0-upgrade-guide.md | 60 +- .../2024-07-29-0-40-0-upgrade-guide.md | 9 +- 68 files changed, 2096 insertions(+), 1829 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 8d25eca40144c..118ebb7461dbf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -147,6 +147,10 @@ make cue-build **Note:** Website changes use Hugo, CUE, Tailwind CSS, and TypeScript. See [website/README.md](website/README.md) for details. +## Configuration Format + +Always generate Vector configuration examples in **YAML** unless the user explicitly asks for TOML or JSON. YAML is Vector's recommended and default configuration format. + ## Common Patterns ### Development Tools diff --git a/website/content/en/blog/adaptive-request-concurrency.md b/website/content/en/blog/adaptive-request-concurrency.md index b2a701433d638..623d93d3d7749 100644 --- a/website/content/en/blog/adaptive-request-concurrency.md +++ b/website/content/en/blog/adaptive-request-concurrency.md @@ -94,13 +94,15 @@ This decision tree is always active and Vector always "knows" what to do, even i Vector never stops quietly making the linear up vs. exponential down decision in the background, and it works out of the box with zero configuration beyond enabling the feature, which is currently on an opt-in basis in version 0.11. You can enable ARC in an HTTP sink by setting the [`request.concurrency`][request_concurrency] parameter to `adaptive`. Here's an example for a Clickhouse sink: -```toml -[sinks.clickhouse_internal] -type = "clickhouse" -inputs = ["log_stream_1", "log_stream_2"] -host = "http://clickhouse-prod:8123" -table = "prod-log-data" -request.concurrency = "adaptive" +```yaml +sinks: + clickhouse_internal: + type: "clickhouse" + inputs: ["log_stream_1", "log_stream_2"] + host: "http://clickhouse-prod:8123" + table: "prod-log-data" + request: + concurrency: "adaptive" ``` There's also room for fine-tuning if you find yourself needing additional knobs: diff --git a/website/content/en/blog/graphql-api.md b/website/content/en/blog/graphql-api.md index f623c4caa921d..6d459dda9de95 100644 --- a/website/content/en/blog/graphql-api.md +++ b/website/content/en/blog/graphql-api.md @@ -36,19 +36,22 @@ tuned! Start up Vector locally with the following config: -```toml -[api] -enabled = true - -[sources.demo] -type = "demo_logs" -format = "json" -interval = 1.0 - -[sinks.console] -type = "console" -inputs = ["demo"] -encoding.codec = "text" +```yaml +api: + enabled: true + +sources: + demo: + type: "demo_logs" + format: "json" + interval: 1.0 + +sinks: + console: + type: "console" + inputs: ["demo"] + encoding: + codec: "text" ``` You can then access @@ -190,12 +193,12 @@ query { The API is an opt-in feature that spawns an HTTP server to accept queries. -To enable, open your `vector.toml`, and add the following: +To enable, open your `vector.yaml`, and add the following: -```toml -[api] - enabled = true - address = "127.0.0.1:8686" # optional. Change IP/port if required +```yaml +api: + enabled: true + address: "127.0.0.1:8686" # optional. Change IP/port if required ``` After restarting Vector, you can access an API playground at diff --git a/website/content/en/blog/kubernetes-integration.md b/website/content/en/blog/kubernetes-integration.md index bf60365d98ed1..bf9d7fbbb1cfa 100644 --- a/website/content/en/blog/kubernetes-integration.md +++ b/website/content/en/blog/kubernetes-integration.md @@ -27,15 +27,17 @@ The crux of our Kubernetes integration for Vector is the new [`kubernetes_logs`] This simple Vector configuration, for example, would suffice to send logs from *all* of your Nodes/Pods to [Elasticsearch]: -```toml title="vector.toml" -[sources.k8s_all] -type = "kubernetes_logs" - -[sinks.es_out] -type = "elasticsearch" -inputs = ["k8s_all"] -host = "http://your-elasticsearch-cluster:9200" -index = "vector-k8s-%F" +```yaml title="vector.yaml" +sources: + k8s_all: + type: "kubernetes_logs" + +sinks: + es_out: + type: "elasticsearch" + inputs: ["k8s_all"] + host: "http://your-elasticsearch-cluster:9200" + index: "vector-k8s-%F" ``` Using this simple topology as a starting point, you can [transform][transforms] your log data in a wide variety of ways, ship it to [other sinks][sinks], and much more. diff --git a/website/content/en/blog/log-namespacing.md b/website/content/en/blog/log-namespacing.md index d867f21c9411d..2d7ce9b314e58 100644 --- a/website/content/en/blog/log-namespacing.md +++ b/website/content/en/blog/log-namespacing.md @@ -39,27 +39,30 @@ so you can try out Log Namespacing on individual sources. The following example enables the `log_namespace` feature globally, then disables it for a single source. -```toml -schema.log_namespace = true - -[sources.input_with_log_namespace] -type = "demo_logs" -format = "shuffle" -lines = ["input_with_log_namespace"] -interval = 1 - -[sources.input_without_log_namespace] -type = "demo_logs" -format = "shuffle" -lines = ["input_without_log_namespace"] -interval = 1 -log_namespace = false - -[sinks.console] -type = "console" -inputs = ["input_with_log_namespace", "input_without_log_namespace"] -encoding.codec = "json" - +```yaml +schema: + log_namespace: true + +sources: + input_with_log_namespace: + type: "demo_logs" + format: "shuffle" + lines: ["input_with_log_namespace"] + interval: 1 + + input_without_log_namespace: + type: "demo_logs" + format: "shuffle" + lines: ["input_without_log_namespace"] + interval: 1 + log_namespace: false + +sinks: + console: + type: "console" + inputs: ["input_with_log_namespace", "input_without_log_namespace"] + encoding: + codec: "json" ``` ## How It Works diff --git a/website/content/en/docs/administration/management.md b/website/content/en/docs/administration/management.md index dff4be7153bbf..7f91c26cdd191 100644 --- a/website/content/en/docs/administration/management.md +++ b/website/content/en/docs/administration/management.md @@ -176,7 +176,7 @@ docker restart -f $(docker ps -aqf "name=vector") {{< /tab >}} {{< /tabs >}} -The commands above involve configuring Vector using TOML, but you can also use JSON or YAML. You can also use one of +The commands above involve configuring Vector using YAML, but you can also use TOML or JSON. You can also use one of three image variants (the commands assume `alpine`): | Variant | Image basis | diff --git a/website/content/en/docs/architecture/guarantees.md b/website/content/en/docs/architecture/guarantees.md index 76028d4e42fe1..14a1c0c9a72d8 100644 --- a/website/content/en/docs/architecture/guarantees.md +++ b/website/content/en/docs/architecture/guarantees.md @@ -61,12 +61,13 @@ delivery?](#faq-at-least-once) FAQ below). In order to achieve at-least-once delivery between restarts, your sink must be configured to use disk-based buffers: -```toml title="vector.toml" -[sinks.my_sink_id] - [sinks.my_sink_id.buffer] - type = "disk" - when_full = "block" - max_size = 104900000 # 100MiB +```yaml title="vector.yaml" +sinks: + my_sink_id: + buffer: + type: "disk" + when_full: "block" + max_size: 104900000 # 100MiB ``` Refer to each [sink's][sinks] documentation for further guidance on its buffer options. diff --git a/website/content/en/docs/architecture/pipeline-model.md b/website/content/en/docs/architecture/pipeline-model.md index 49cc846f7ac96..ddb6d84bc3b8c 100644 --- a/website/content/en/docs/architecture/pipeline-model.md +++ b/website/content/en/docs/architecture/pipeline-model.md @@ -10,7 +10,7 @@ Vector's pipeline model is based on a [directed acyclic graph][dag] of [componen ## Defining pipelines -A Vector pipeline is defined through a YAML, TOML, or JSON [configuration] file. For maintainability, many Vector users use configuration and data templating languages like [Jsonnet] or [CUE]. +A Vector pipeline is defined through a YAML, TOML, or JSON [configuration] file. We recommend using YAML as the default format. For maintainability, many Vector users use configuration and data templating languages like [Jsonnet] or [CUE]. Configuration is checked at pipeline compile time (when Vector boots). This prevents simple mistakes and enforces DAG properties. diff --git a/website/content/en/docs/reference/configuration/_index.md b/website/content/en/docs/reference/configuration/_index.md index 9e19281d9024c..65782a59b6000 100644 --- a/website/content/en/docs/reference/configuration/_index.md +++ b/website/content/en/docs/reference/configuration/_index.md @@ -250,8 +250,9 @@ Refer to the [Environment Variables](docs/reference/environment_variables/) page ### Formats Vector supports [YAML], [TOML], and [JSON] to ensure that Vector fits into your -workflow. A side benefit of supporting YAML and JSON is that they enable you to use -data templating languages such as [ytt], [Jsonnet] and [Cue]. +workflow. We recommend using YAML as the default configuration format. A side +benefit of supporting YAML and JSON is that they enable you to use data templating +languages such as [ytt], [Jsonnet] and [Cue]. #### Location diff --git a/website/content/en/docs/reference/configuration/template-syntax.md b/website/content/en/docs/reference/configuration/template-syntax.md index 43822e137c555..96135c0183745 100644 --- a/website/content/en/docs/reference/configuration/template-syntax.md +++ b/website/content/en/docs/reference/configuration/template-syntax.md @@ -12,6 +12,20 @@ data. Any option that supports this syntax will be clearly documented as such in Let's partition data on AWS S3 by "application_id" and "date". We can accomplish this with the `key_prefix` option in the `aws_s3` sink: +{{< tabs default="YAML" >}} +{{< tab title="YAML" >}} + +```yaml +sinks: + backup: + type: "aws_s3" + bucket: "all_application_logs" + key_prefix: "application_id={{ application_id }}/date=%F/" +``` + +{{< /tab >}} +{{< tab title="TOML" >}} + ```toml [sinks.backup] type = "aws_s3" @@ -19,6 +33,9 @@ the `aws_s3` sink: key_prefix = "application_id={{ application_id }}/date=%F/" ``` +{{< /tab >}} +{{< /tabs >}} + Notice that Vector allows direct field references as well as "strftime" specifiers. If we were to run the following log event through Vector: @@ -45,18 +62,44 @@ enables dynamic partitioning, something fundamental to storing log data in files Individual [log event][log] fields can be accessed using `{{ ... }}` to wrap a VRL [path expression][path_expression]: +{{< tabs default="YAML" >}} +{{< tab title="YAML" >}} + +```yaml +option: "{{ .parent.child }}" +``` + +{{< /tab >}} +{{< tab title="TOML" >}} + ```toml option = "{{ .parent.child }}" ``` +{{< /tab >}} +{{< /tabs >}} + ### Strftime specifiers In addition to directly accessing fields, Vector offers a shortcut for injecting [strftime specifiers][strftime]: +{{< tabs default="YAML" >}} +{{< tab title="YAML" >}} + +```yaml +option: "year=%Y/month=%m/day=%d/" +``` + +{{< /tab >}} +{{< tab title="TOML" >}} + ```toml option = "year=%Y/month=%m/day=%d/" ``` +{{< /tab >}} +{{< /tabs >}} + {{< info >}} The value is derived from the [`timestamp` field](/docs/architecture/data-model/log/#timestamps) and the name of this field can be changed via the [global `timestamp_key` option](/docs/reference/configuration/schema/#log_schema.timestamp_key). @@ -67,16 +110,42 @@ and the name of this field can be changed via the [global `timestamp_key` option You can escape this syntax by prefixing the character with a `\`. For example, you can escape the event field syntax like this: +{{< tabs default="YAML" >}} +{{< tab title="YAML" >}} + +```yaml +option: '\{{ field_name }}' +``` + +{{< /tab >}} +{{< tab title="TOML" >}} + ```toml option = "\{{ field_name }}" ``` +{{< /tab >}} +{{< /tabs >}} + And [strftime] specified like so: +{{< tabs default="YAML" >}} +{{< tab title="YAML" >}} + +```yaml +option: "year=\\%Y/month=\\%m/day=\\%d/" +``` + +{{< /tab >}} +{{< tab title="TOML" >}} + ```toml option = "year=\%Y/month=\%m/day=\%d/" ``` +{{< /tab >}} +{{< /tabs >}} + Each of the values above would be treated literally. ## How it works @@ -91,6 +160,24 @@ You can find additional examples for accessing fields in the Vector doesn't currently support fallback values, [issue 1692][1692] is open to add this functionality. In the interim, you can use the [`remap` transform][remap] to set a default value: +{{< tabs default="YAML" >}} +{{< tab title="YAML" >}} + +```yaml +transforms: + set_defaults: + type: "remap" + inputs: + - "my-source-id" + source: | + if !exists(.my_field) { + .my_field = "default" + } +``` + +{{< /tab >}} +{{< tab title="TOML" >}} + ```toml [transforms.set_defaults] type = "remap" @@ -102,6 +189,9 @@ you can use the [`remap` transform][remap] to set a default value: ''' ``` +{{< /tab >}} +{{< /tabs >}} + ### Missing fields If a field is missing, an error is logged and Vector drops the event. The `component_errors_total` internal diff --git a/website/content/en/docs/reference/configuration/unit-tests.md b/website/content/en/docs/reference/configuration/unit-tests.md index 5e11240c9136f..d470813252518 100644 --- a/website/content/en/docs/reference/configuration/unit-tests.md +++ b/website/content/en/docs/reference/configuration/unit-tests.md @@ -77,12 +77,11 @@ functions][type], functions like [`exists`][exists], [`includes`][includes], [`is_nullish`][is_nullish] and [`contains`][contains], and VRL [comparisons]. Here's an example usage of a Boolean expression passed to an `assert` function: -```toml -[[tests.outputs.conditions]] -type = "vrl" -source = ''' -assert!(is_string(.message) && is_timestamp(.timestamp) && !exists(.other)) -''' +```yaml +conditions: + - type: "vrl" + source: | + assert!(is_string(.message) && is_timestamp(.timestamp) && !exists(.other)) ``` In this case, the VRL program (under `source`) evaluates to a single Boolean that expresses the @@ -94,29 +93,27 @@ following: It's also possible to break a test up into multiple `assert` or `assert_eq` statements: -```toml -source = ''' -assert!(exists(.message), "no message field provided") -assert!(!is_nullish(.message), "message field is an empty string") -assert!(is_string(.message), "message field has as unexpected type") -assert_eq!(.message, "success", "message field had an unexpected value") -assert!(exists(.timestamp), "no timestamp provided") -assert!(is_timestamp(.timestamp), "timestamp is invalid") -assert!(!exists(.other), "extraneous other field present") -''' +```yaml +source: | + assert!(exists(.message), "no message field provided") + assert!(!is_nullish(.message), "message field is an empty string") + assert!(is_string(.message), "message field has as unexpected type") + assert_eq!(.message, "success", "message field had an unexpected value") + assert!(exists(.timestamp), "no timestamp provided") + assert!(is_timestamp(.timestamp), "timestamp is invalid") + assert!(!exists(.other), "extraneous other field present") ``` You can also store the Boolean expressions in variables rather than passing the entire statement to the `assert` function: -```toml -source = ''' -message_field_valid = exists(.message) && - !is_nullish(.message) && - .message == "success" +```yaml +source: | + message_field_valid = exists(.message) && + !is_nullish(.message) && + .message == "success" -assert!(message_field_valid) -''' + assert!(message_field_valid) ``` ## Example unit test configuration {#example} @@ -124,48 +121,48 @@ assert!(message_field_valid) Below is an annotated example of a unit test suite for a transform called `add_metadata`, which adds a unique ID and a timestamp to log events: -```toml -[sources.all_container_services] -type = "docker_logs" -docker_host = "http://localhost:2375" -include_images = ["web_frontend", "web_backend", "auth_service"] +```yaml +sources: + all_container_services: + type: "docker_logs" + docker_host: "http://localhost:2375" + include_images: ["web_frontend", "web_backend", "auth_service"] # The transform being tested is a Vector Remap Language (VRL) transform that # adds two fields to each incoming log event: a timestamp and a unique ID -[transforms.add_metadata] -type = "remap" -inputs = ["all_container_services"] -source = ''' -.timestamp = now() -.id = uuid_v4() -''' +transforms: + add_metadata: + type: "remap" + inputs: ["all_container_services"] + source: | + .timestamp = now() + .id = uuid_v4() # Here we begin configuring our test suite -[[tests]] -name = "Test for the add_metadata transform" - -# The inputs for the test -[[tests.inputs]] -insert_at = "add_metadata" # The transform into which the testing event is inserted -type = "log" # The event type (either log or metric) - -# The test log event that is passed to the `add_metadata` transform -[tests.inputs.log_fields] -message = "successful transaction" -code = 200 - -# The expected outputs of the test -[[tests.outputs]] -extract_from = "add_metadata" # The transform from which the resulting event is extracted - -# The declaration of what we expect -[[tests.outputs.conditions]] -type = "vrl" -source = ''' -assert!(is_timestamp(.timestamp)) -assert!(is_string(.id)) -assert_eq!(.message, "successful transaction") -''' +tests: + - name: "Test for the add_metadata transform" + + # The inputs for the test + inputs: + - insert_at: "add_metadata" # The transform into which the testing event is inserted + type: "log" # The event type (either log or metric) + + # The test log event that is passed to the `add_metadata` transform + log_fields: + message: "successful transaction" + code: 200 + + # The expected outputs of the test + outputs: + - extract_from: "add_metadata" # The transform from which the resulting event is extracted + + # The declaration of what we expect + conditions: + - type: "vrl" + source: | + assert!(is_timestamp(.timestamp)) + assert!(is_string(.id)) + assert_eq!(.message, "successful transaction") ``` This example represents a complete test of the `add_metadata` transform, include test `inputs` @@ -187,7 +184,7 @@ unit tests as a way of **mocking observability data sources** and ensuring that respond to those mock sources the way that you would expect. {{< success title="Multiple config formats available" >}} -The unit testing example above is in TOML but Vector also supports YAML and JSON as configuration +The unit testing example above is in YAML but Vector also supports TOML and JSON as configuration formats. {{< /success >}} @@ -198,16 +195,15 @@ same config file alongside your transform definitions or split them out into a s Unit tests need are specified inside of a `tests` array. Each test requires a `name`: -```toml -[[tests]] -name = "test 1" -# Other test config +```yaml +tests: + - name: "test 1" + # Other test config -[[tests]] -name = "test_2" -# Other test config + - name: "test_2" + # Other test config -# etc. + # etc. ``` Inside each test definition, you need to specify two things: @@ -230,18 +226,19 @@ In the `inputs` array for the test, you have these options: Here's an example `inputs` declaration: -```toml -[transforms.add_metadata] -# transform config +```yaml +transforms: + add_metadata: + # transform config -[[tests]] -name = "Test add_metadata transform" +tests: + - name: "Test add_metadata transform" -[[tests.inputs]] -insert_at = "add_metadata" + inputs: + - insert_at: "add_metadata" -[tests.inputs.log_fields] -message = "<102>1 2020-12-22T15:22:31.111Z vector-user.biz su 2666 ID389 - Something went wrong" + log_fields: + message: "<102>1 2020-12-22T15:22:31.111Z vector-user.biz su 2666 ID389 - Something went wrong" ``` ### Outputs @@ -262,16 +259,15 @@ Each condition in the `conditions` array has two fields: Here's an example `outputs` declaration: -```toml -[[tests.outputs]] -extract_from = "add_metadata" - -[[tests.outputs.conditions]] -type = "vrl" -source = ''' -assert!(is_string(.id)) -assert!(exists(.tags)) -''' +```yaml +outputs: + - extract_from: "add_metadata" + + conditions: + - type: "vrl" + source: | + assert!(is_string(.id)) + assert!(exists(.tags)) ``` #### Asserting no output @@ -280,10 +276,10 @@ In some cases, you may need to assert that _no_ event is output by a transform. this at the root level of a specific test's configuration using the `no_outputs_from` parameter, which takes a list of transform names. Here's an example: -```toml -[[tests]] -name = "Ensure no output" -no_outputs_from = ["log_filter", "metric_filter"] +```yaml +tests: + - name: "Ensure no output" + no_outputs_from: ["log_filter", "metric_filter"] ``` In this test configuration, Vector would expect that the `log_filter` and `metric_filter` transforms @@ -298,26 +294,27 @@ Some examples of use cases for `no_outputs_from`: Below is a full example of using `no_outputs_from` in a Vector unit test: -```toml -[transforms.log_filter] -type = "filter" -inputs = ["log_source"] -condition = '.env == "production"' - -[[tests]] -name = "Filter out non-production events" -no_outputs_from = ["log_filter"] - -[[tests.inputs]] -type = "log" -insert_at = "log_filter" - -[tests.inputs.log_fields] -message = "success" -code = 202 -endpoint = "/transactions" -method = "POST" -env = "staging" +```yaml +transforms: + log_filter: + type: "filter" + inputs: ["log_source"] + condition: '.env == "production"' + +tests: + - name: "Filter out non-production events" + no_outputs_from: ["log_filter"] + + inputs: + - type: "log" + insert_at: "log_filter" + + log_fields: + message: "success" + code: 202 + endpoint: "/transactions" + method: "POST" + env: "staging" ``` This unit test passes because the `env` field of the input event has a value of `staging`, which @@ -340,11 +337,11 @@ either a structured event [object](#object) or a raw [string](#raw-string-value) To specify a structured log event as your test input, use `log_fields`: -```toml -[tests.inputs.log_fields] -message = "successful transaction" -code = 200 -id = "38c5b0d0-5e7e-42aa-ae86-2b642ad2d1b8" +```yaml +log_fields: + message: "successful transaction" + code: 200 + id: "38c5b0d0-5e7e-42aa-ae86-2b642ad2d1b8" ``` If there are hyphens in the field name, you will need to quote this part (at least in YAML): @@ -362,38 +359,38 @@ If there are hyphens in the field name, you will need to quote this part (at lea To specify a raw string value for a log event, use `value`: -```toml -[[tests.inputs]] -insert_at = "add_metadata" -value = "<102>1 2020-12-22T15:22:31.111Z vector-user.biz su 2666 ID389 - Something went wrong" +```yaml +inputs: + - insert_at: "add_metadata" + value: "<102>1 2020-12-22T15:22:31.111Z vector-user.biz su 2666 ID389 - Something went wrong" ``` ##### VRL program To specify a program to construct the log event, use `source`: -```toml -[[tests.inputs]] - insert_at = "canary" - type = "vrl" - source = """ - . = {"a": {"b": "c"}, "d": now()} - """ +```yaml +inputs: + - insert_at: "canary" + type: "vrl" + source: | + . = {"a": {"b": "c"}, "d": now()} ``` #### Metrics You can specify the fields in a metric event to be unit tested using a `metric` object: -```toml -[[tests.inputs]] -insert_at = "my_metric_transform" -type = "metric" +```yaml +inputs: + - insert_at: "my_metric_transform" + type: "metric" -[tests.inputs.metric] -name = "count" -kind = "absolute" -counter = { value = 1 } + metric: + name: "count" + kind: "absolute" + counter: + value: 1 ``` Aggregated metrics are a little different: @@ -414,40 +411,40 @@ tests: Here's a full end-to-end example of unit testing a metric through a transform: -```toml -[transforms.add_env_to_metric] -type = "remap" -inputs = [] -source = ''' -env, err = get_env_var("ENV") -if err != null { - log(err, level: "error") -} -tags.environment = env -''' - -[[tests]] -name = "add_unique_id_test" - -[[tests.inputs]] -insert_at = "add_env_to_metric" -type = "metric" - -[tests.inputs.metric] -name = "website_hits" -kind = "absolute" -counter = { value = 1 } - -[[tests.outputs]] -extract_from = "add_env_to_metric" - -[[tests.outputs.conditions]] -type = "vrl" -source = ''' -assert_eq!(.name, "website_hits") -assert_eq!(.kind, "absolute") -assert_eq!(.tags.environment, "production") -''' +```yaml +transforms: + add_env_to_metric: + type: "remap" + inputs: [] + source: | + env, err = get_env_var("ENV") + if err != null { + log(err, level: "error") + } + tags.environment = env + +tests: + - name: "add_unique_id_test" + + inputs: + - insert_at: "add_env_to_metric" + type: "metric" + + metric: + name: "website_hits" + kind: "absolute" + counter: + value: 1 + + outputs: + - extract_from: "add_env_to_metric" + + conditions: + - type: "vrl" + source: | + assert_eq!(.name, "website_hits") + assert_eq!(.kind, "absolute") + assert_eq!(.tags.environment, "production") ``` ## Multiple transforms {#multiple} @@ -464,73 +461,71 @@ You may notice that the three transforms in this example could be combined into transform. Their separation into multiple transforms here is purely for demonstration purposes. {{< /warning >}} -```toml +```yaml # This source, like all sources, is ignored in the unit test itself -[sources.web_backend] -type = "docker_logs" -docker_host = "http://localhost:2375" -include_images = ["web_backend"] +sources: + web_backend: + type: "docker_logs" + docker_host: "http://localhost:2375" + include_images: ["web_backend"] # The first transform in the chain -[transforms.add_env_metadata] -type = "remap" -inputs = ["web_backend"] -source = ''' -.tags.environment = "production" -''' - -# The second transform in the chain -[transforms.sanitize] -type = "remap" -inputs = ["add_env_metadata"] -source = ''' -del(.username) -del(.email) -''' - -# The final transform in the chain -[transforms.add_host_metadata] -type = "remap" -inputs = ["sanitize"] -source = ''' -.tags.host = "web-backend1.vector-user.biz" -''' - -[[tests]] -name = "Multiple chained remap transforms" - -[[tests.inputs]] -type = "log" -# Insert test input events into the first transform -insert_at = "add_env_metadata" - -# The input event to insert into the first transform in the chain -[tests.inputs.log_fields] -message = "image successfully uploaded" -code = 202 -username = "tonydanza1337" -email = "tony@whostheboss.com" -transaction_id = "bcef6a6a-2b72-4a9a-99a0-97ae89d82815" - -[[tests.outputs]] -# Extract test outputs from the last transform -extract_from = "add_host_metadata" - -[[tests.outputs.conditions]] -type = "vrl" -# Our VRL assertions for the test output -source = ''' -assert_eq!(.tags.environment, "production", "incorrect environment tag") -assert_eq!(.tags.host, "web-backend1.vector-user.biz", "incorrect host tag") -assert!(!exists(.username)) -assert!(!exists(.email)) - -valid_transaction_id = exists(.transaction_id) && - is_string(.transaction_id) && - length!(.transaction_id) == 36 - -assert!(valid_transaction_id, "transaction ID invalid") -''' +transforms: + add_env_metadata: + type: "remap" + inputs: ["web_backend"] + source: | + .tags.environment = "production" + + # The second transform in the chain + sanitize: + type: "remap" + inputs: ["add_env_metadata"] + source: | + del(.username) + del(.email) + + # The final transform in the chain + add_host_metadata: + type: "remap" + inputs: ["sanitize"] + source: | + .tags.host = "web-backend1.vector-user.biz" + +tests: + - name: "Multiple chained remap transforms" + + inputs: + - type: "log" + # Insert test input events into the first transform + insert_at: "add_env_metadata" + + # The input event to insert into the first transform in the chain + log_fields: + message: "image successfully uploaded" + code: 202 + username: "tonydanza1337" + email: "tony@whostheboss.com" + transaction_id: "bcef6a6a-2b72-4a9a-99a0-97ae89d82815" + + outputs: + - # Extract test outputs from the last transform + extract_from: "add_host_metadata" + + conditions: + - type: "vrl" + # Our VRL assertions for the test output + source: | + assert_eq!(.tags.environment, "production", "incorrect environment tag") + assert_eq!(.tags.host, "web-backend1.vector-user.biz", "incorrect host tag") + assert!(!exists(.username)) + assert!(!exists(.email)) + + valid_transaction_id = exists(.transaction_id) && + is_string(.transaction_id) && + length!(.transaction_id) == 36 + + assert!(valid_transaction_id, "transaction ID invalid") ``` From a testing standpoint, all three transforms here can be thought of as a single unit. One example @@ -541,41 +536,40 @@ extracted from the end of the chain (`add_host_metadata`), and one set of VRL You could also test a subset of this transform chain. This configuration, for example, would test only the first two transforms (`add_env_metadata` and `sanitize`): -```toml -[[tests]] -name = "First two transforms" - -[[tests.inputs]] -type = "log" -# Insert test input into the first transform -insert_at = "add_env_metadata" - -# For comparison, we can use the same input event as above -[tests.inputs.log_fields] -message = "image successfully uploaded" -code = 202 -username = "tonydanza1337" -email = "tony@whostheboss.com" -transaction_id = "bcef6a6a-2b72-4a9a-99a0-97ae89d82815" - -[[tests.outputs]] -# Extract test output from the second transform rather than the last -extract_from = "sanitize" - -[[tests.outputs.conditions]] -type = "vrl" -source = ''' -assert_eq!(.tags.environment, "production", "incorrect environment tag") -assert!(!exists(.tags.host), "host tag included") -assert!(!exists(.username)) -assert!(!exists(.email)) +```yaml +tests: + - name: "First two transforms" -valid_transaction_id = exists(.transaction_id) && - is_string(.transaction_id) && - length!(.transaction_id) == 36 + inputs: + - type: "log" + # Insert test input into the first transform + insert_at: "add_env_metadata" -assert!(valid_transaction_id, "transaction ID invalid") -''' + # For comparison, we can use the same input event as above + log_fields: + message: "image successfully uploaded" + code: 202 + username: "tonydanza1337" + email: "tony@whostheboss.com" + transaction_id: "bcef6a6a-2b72-4a9a-99a0-97ae89d82815" + + outputs: + - # Extract test output from the second transform rather than the last + extract_from: "sanitize" + + conditions: + - type: "vrl" + source: | + assert_eq!(.tags.environment, "production", "incorrect environment tag") + assert!(!exists(.tags.host), "host tag included") + assert!(!exists(.username)) + assert!(!exists(.email)) + + valid_transaction_id = exists(.transaction_id) && + is_string(.transaction_id) && + length!(.transaction_id) == 36 + + assert!(valid_transaction_id, "transaction ID invalid") ``` In the VRL conditions for this two-transform test, notice that the assertion regarding the `host` diff --git a/website/content/en/docs/setup/going-to-prod/high-availability.md b/website/content/en/docs/setup/going-to-prod/high-availability.md index 768d40d16ea59..7fa71f52291a2 100644 --- a/website/content/en/docs/setup/going-to-prod/high-availability.md +++ b/website/content/en/docs/setup/going-to-prod/high-availability.md @@ -53,23 +53,26 @@ We recommend routing dropped events immediately to a backup destination to prior The following is an example configuration that routes failed events from a `remap` transform to a `aws_s3` sink: -```toml -[sources.input] - type = "datadog_agent" - -[transforms.parsing] - inputs = ["input"] - type = "remap" - reroute_dropped = true - source = "..." - -[sinks.analysis] - inputs = ["parsing"] - type = "datadog_logs" - -[sinks.backup] - inputs = ["**parsing.dropped**"] # dropped events from the `parsing` transform - type = "aws_s3" +```yaml +sources: + input: + type: "datadog_agent" + +transforms: + parsing: + inputs: ["input"] + type: "remap" + reroute_dropped: true + source: "..." + +sinks: + analysis: + inputs: ["parsing"] + type: "datadog_logs" + + backup: + inputs: ["parsing.dropped"] # dropped events from the `parsing` transform + type: "aws_s3" ``` {{< info >}} diff --git a/website/content/en/guides/advanced/custom-aggregations-with-lua.md b/website/content/en/guides/advanced/custom-aggregations-with-lua.md index 262f6b7a2d4c8..4cfc4578b2343 100644 --- a/website/content/en/guides/advanced/custom-aggregations-with-lua.md +++ b/website/content/en/guides/advanced/custom-aggregations-with-lua.md @@ -31,12 +31,12 @@ There are three types of hooks: `init`, `process`, and `shutdown`. The most important of them is `process`, which is called on each incoming events. It can be defined like this: -```toml -hooks.process = """ -function (event, emit) - -- do something -end -""" +```yaml +hooks: + process: | + function (event, emit) + -- do something + end ``` It takes a single event and can output one or many of them using the `emit` function provided as the second argument. @@ -76,54 +76,54 @@ variables. Using the knowledge from the previous section, it is possible to write down the following transform definition: -```toml title="vector.toml" -[transforms.aggregator] -type = "lua" -version = "2" -inputs = [] # add IDs of the input components here - -hooks.init = """ - function (emit) - count = 0 -- initialize state by setting a global variable - end -""" - -hooks.process = """ - function (event, emit) - count = count + 1 -- increment the counter and exit - end -""" - -timers = [{interval_seconds = 5, handler = """ - function (emit) - emit { - metric = { - name = "event_counter", - kind = "incremental", - timestamp = os.date("!*t"), - counter = { - value = counter - } - } - } - counter = 0 - end -"""}] - -hooks.shutdown = """ - function (emit) - emit { - metric = { - name = "event_counter", - kind = "incremental", - timestamp = os.date("!*t"), - counter = { - value = counter - } - } - } - end -""" +```yaml title="vector.yaml" +transforms: + aggregator: + type: "lua" + version: "2" + inputs: [] # add IDs of the input components here + + hooks: + init: | + function (emit) + count = 0 -- initialize state by setting a global variable + end + + process: | + function (event, emit) + count = count + 1 -- increment the counter and exit + end + + shutdown: | + function (emit) + emit { + metric = { + name = "event_counter", + kind = "incremental", + timestamp = os.date("!*t"), + counter = { + value = counter + } + } + } + end + + timers: + - interval_seconds: 5 + handler: | + function (emit) + emit { + metric = { + name = "event_counter", + kind = "incremental", + timestamp = os.date("!*t"), + counter = { + value = counter + } + } + } + counter = 0 + end ``` One could plug it into a [pipeline][docs.about.concepts#pipelines] and it would work! @@ -137,8 +137,8 @@ identical. It is possible make the config more [DRY][urls.dry_code] by extractin a dedicated function. Such a function can be placed into the [source][docs.transforms.lua#source] section of the config: -```toml -source = """ +```yaml +source: | function make_counter(value) return metric = { name = "event_counter", @@ -149,28 +149,28 @@ source = """ } } end -""" ``` and then adjusting the timer handler -```toml -timers = [{interval_seconds = 5, handler = """ - function (emit) - emit(make_counter(counter)) - counter = 0 - end -"""}] +```yaml +timers: + - interval_seconds: 5 + handler: | + function (emit) + emit(make_counter(counter)) + counter = 0 + end ``` and the `shutdown` hook: -```toml -hooks.shutdown = """ - function (emit) - emit(make_counter(counter)) - end -""" +```yaml +hooks: + shutdown: | + function (emit) + emit(make_counter(counter)) + end ``` ## Keep All Code Together @@ -178,45 +178,48 @@ hooks.shutdown = """ The new config looks tidier, but in order to make it more readable, it is also possible to gather implementations of all functions into the `source` section, resulting in the following component declaration: -```toml title="vector.toml" -[transforms.aggregator] -type = "lua" -version = "2" -inputs = [] # add IDs of the input components here -hooks.init = "init" -hooks.process = "process" -hooks.shutdown = "shutdown" -timers = [{interval_seconds = 5, handler = "timer_handler"}] - -source = """ - function init() - count = 0 - end - - function process() - count = count + 1 - end - - function timer_handler(emit) - emit(make_counter(counter)) - counter = 0 - end - - function shutdown(emit) - emit(make_counter(counter)) - end - - function make_counter(value) - return metric = { - name = "event_counter", - kind = "incremental", - timestamp = os.date("!*t"), - counter = { - value = value - } - } - end -""" +```yaml title="vector.yaml" +transforms: + aggregator: + type: "lua" + version: "2" + inputs: [] # add IDs of the input components here + hooks: + init: "init" + process: "process" + shutdown: "shutdown" + timers: + - interval_seconds: 5 + handler: "timer_handler" + + source: | + function init() + count = 0 + end + + function process() + count = count + 1 + end + + function timer_handler(emit) + emit(make_counter(counter)) + counter = 0 + end + + function shutdown(emit) + emit(make_counter(counter)) + end + + function make_counter(value) + return metric = { + name = "event_counter", + kind = "incremental", + timestamp = os.date("!*t"), + counter = { + value = value + } + } + end ``` ## A Loadable Module @@ -262,16 +265,20 @@ end and -```toml title="vector.toml" -[transforms.aggregator] -type = "lua" -version = "2" -inputs = [] # add IDs of the input components here -hooks.init = "init" -hooks.process = "process" -hooks.shutdown = "shutdown" -timers = [{interval_seconds = 5, handler = "timer_handler"}] -source = "require('aggregator')" +```yaml title="vector.yaml" +transforms: + aggregator: + type: "lua" + version: "2" + inputs: [] # add IDs of the input components here + hooks: + init: "init" + process: "process" + shutdown: "shutdown" + timers: + - interval_seconds: 5 + handler: "timer_handler" + source: "require('aggregator')" ``` There are also [other possibilities][urls.lua_modules_tutorial] to define Lua modules which do not require to use diff --git a/website/content/en/guides/advanced/merge-multiline-logs-with-lua.md b/website/content/en/guides/advanced/merge-multiline-logs-with-lua.md index c90fa50527457..45dd47ea4c7ff 100644 --- a/website/content/en/guides/advanced/merge-multiline-logs-with-lua.md +++ b/website/content/en/guides/advanced/merge-multiline-logs-with-lua.md @@ -43,44 +43,44 @@ in the [guide on parsing CSV logs][guides.parsing-csv-logs-with-lua]. The underl Such an algorithm can be implemented, for example, with the following transform config: -```toml title="vector.toml" -[transforms.lua] - inputs = [] - type = "lua" - version = "2" - source = """ - csv = require("csv") -- load the `lua-csv` module - expected_columns = 23 -- expected number of columns in incoming CSV lines - line_separator = "\\r\\n" -- note the double escaping required by the TOML format - """ - hooks.process = """ - function (event, emit) - merged_event = merge(event) - if merged_event == nil then -- a global variable containing the merged event - merged_event = event -- if it is empty, set it to the current event - else -- otherwise, concatenate the line in the stored merged event - -- with the next line - merged_event.log.message = merged_event.log.message .. - line_separator .. event.log.message - end - - fields = csv.openstring(event.log.message):lines()() -- parse CSV - if #fields < expected_columns then - return -- not all fields are present in the merged event yet - end - - -- do something with the array of the parsed fields - merged_event.log.csv_fields = fields -- for example, just store them in an - -- array field - - emit(merged_event) -- emit the resulting event - merged_event = nil -- clear the merged event - end - """ +```yaml title="vector.yaml" +transforms: + lua: + inputs: [] + type: "lua" + version: "2" + source: | + csv = require("csv") -- load the `lua-csv` module + expected_columns = 23 -- expected number of columns in incoming CSV lines + line_separator = "\r\n" + hooks: + process: | + function (event, emit) + merged_event = merge(event) + if merged_event == nil then -- a global variable containing the merged event + merged_event = event -- if it is empty, set it to the current event + else -- otherwise, concatenate the line in the stored merged event + -- with the next line + merged_event.log.message = merged_event.log.message .. + line_separator .. event.log.message + end + + fields = csv.openstring(event.log.message):lines()() -- parse CSV + if #fields < expected_columns then + return -- not all fields are present in the merged event yet + end + + -- do something with the array of the parsed fields + merged_event.log.csv_fields = fields -- for example, just store them in an + -- array field + + emit(merged_event) -- emit the resulting event + merged_event = nil -- clear the merged event + end ``` -In this code sample, the `source` option defines code that's executed when the transform is created. -while the `hooks.process` option defines a function that's called for each incoming event. +In this code sample, the `source` option defines code that is executed when the transform is created, +while the `hooks.process` option defines a function that is called for each incoming event. ## How It Works diff --git a/website/content/en/guides/advanced/parsing-csv-logs-with-lua.md b/website/content/en/guides/advanced/parsing-csv-logs-with-lua.md index 00123d1ace5d0..634b04f7e3006 100644 --- a/website/content/en/guides/advanced/parsing-csv-logs-with-lua.md +++ b/website/content/en/guides/advanced/parsing-csv-logs-with-lua.md @@ -36,36 +36,40 @@ log file: Let us draft an initial version of the Vector's configuration file: -```toml title="vector.toml" -data_dir = "." - -[sources.file] - type = "file" - include = ["*.csv"] - ignore_checkpoints = true - -[transforms.lua] - inputs = ["file"] - type = "lua" - version = "2" - hooks.process = """ - function (event, emit) - -- to be expanded - emit(event) - end - """ - -[sinks.console] - inputs = ["lua"] - type = "console" - encoding.codec = "json" +```yaml title="vector.yaml" +data_dir: "." + +sources: + file: + type: "file" + include: ["*.csv"] + ignore_checkpoints: true + +transforms: + lua: + inputs: ["file"] + type: "lua" + version: "2" + hooks: + process: | + function (event, emit) + -- to be expanded + emit(event) + end + +sinks: + console: + inputs: ["lua"] + type: "console" + encoding: + codec: "json" ``` This config sets up a [pipeline][docs.meta.glossary#pipeline] that reads log files, pipes them through the parsing transform (which currently is configured to just pass the events through), and displays the produced log events using [`console`][docs.sinks.console] sink. -At this point, running `vector --config vector.toml` results in the following output: +At this point, running `vector --config vector.yaml` results in the following output: ```json {"file":"log.csv","host":"localhost","message":"2020-04-09 12:48:49.661 UTC,,,1,,localhost.1,1,,2020-04-09 12:48:49 UTC,,0,LOG,00000,\"ending log output to stderr\",,\"Future log output will go to log destination \"\"csvlog\"\".\",,,,,,,\"\"","timestamp":"2020-04-09T14:33:28Z"} @@ -77,7 +81,7 @@ At this point, running `vector --config vector.toml` results in the following ou In order to perform actual parsing, it is possible to leverage [`lua-csv`][urls.lua_csv_repo]. Because it consists of a [single file][urls.lua_csv_view], it is possible to just download it to the same -directory where `vector.toml` is stored: +directory where `vector.yaml` is stored: ```bash curl -o csv.lua https://raw.githubusercontent.com/geoffleyland/lua-csv/d20cd42d61dc52e7f6bcb13b596ac7a7d4282fbf/lua/csv.lua @@ -86,10 +90,9 @@ curl -o csv.lua https://raw.githubusercontent.com/geoffleyland/lua-csv/d20cd42d6 Then it would be possible to load it by calling [`require`][urls.lua_require] Lua function in the [`source`][docs.transforms.lua#source] configuration section: -```toml -source = """ +```yaml +source: | csv = require("csv") -""" ``` With this `source` the `csv` module is loaded when Vector is started up (or if the `lua` transform is added later and the @@ -99,85 +102,85 @@ config is automatically reloaded) and can be used through the global variable `c With the `csv` module, the [`hooks.process`][docs.transforms.lua#process] can be changed to the following: -```toml -hooks.process = """ - function (event, emit) - fields = csv.openstring(event.log.message):lines()() -- parse the `message` field - event.log.message = nil -- drop the `message` field +```yaml +hooks: + process: | + function (event, emit) + fields = csv.openstring(event.log.message):lines()() -- parse the `message` field + event.log.message = nil -- drop the `message` field - column_names = { -- a sequence containing CSV column names - -- ... - } + column_names = { -- a sequence containing CSV column names + -- ... + } - for column, value in ipairs(fields) do -- iterate over CSV columns - column_name = column_names[column] -- get column name - event.log[column_name] = value -- set the corresponding field in the event - end + for column, value in ipairs(fields) do -- iterate over CSV columns + column_name = column_names[column] -- get column name + event.log[column_name] = value -- set the corresponding field in the event + end - emit(event) -- emit the transformed event - end -""" + emit(event) -- emit the transformed event + end ``` Note that the `column_names` can be created just once, in the `source` section instead to speed up processing. Putting it there and using the column names from the PostgreSQL documentation results in the following definition of the whole transform: -```toml title="vector.toml" +```yaml title="vector.yaml" +# ... +transforms: + lua: + inputs: ["file"] + type: "lua" + version: "2" + source: | + csv = require("csv") -- load external module for parsing CSV + column_names = { -- a sequence containing CSV column names + "log_time", + "user_name", + "database_name", + "process_id", + "connection_from", + "session_id", + "session_line_num", + "command_tag", + "session_start_time", + "virtual_transaction_id", + "transaction_id", + "error_severity", + "sql_state_code", + "message", + "detail", + "hint", + "internal_query", + "internal_query_pos", + "context", + "query", + "query_pos", + "location", + "application_name", + -- available only in postgres > 13, to remove for postgres <= 13 + "backend_type", + "leader_pid", + "query_id" + } + hooks: + process: | + function (event, emit) + fields = csv.openstring(event.log.message):lines()() -- parse the `message` field + event.log.message = nil -- drop the `message` field + + for column, column_name in ipairs(column_names) do -- iterate over column names + value = fields[column] -- get field value + event.log[column_name] = value -- set the corresponding field in the event + end + + emit(event) -- emit the transformed event + end # ... -[transforms.lua] - inputs = ["file"] - type = "lua" - version = "2" - source = """ - csv = require("csv") -- load external module for parsing CSV - column_names = { -- a sequence containing CSV column names - "log_time", - "user_name", - "database_name", - "process_id", - "connection_from", - "session_id", - "session_line_num", - "command_tag", - "session_start_time", - "virtual_transaction_id", - "transaction_id", - "error_severity", - "sql_state_code", - "message", - "detail", - "hint", - "internal_query", - "internal_query_pos", - "context", - "query", - "query_pos", - "location", - "application_name", - -- available only in postgres > 13, to remove for postgres <= 13 - "backend_type", - "leader_pid", - "query_id" - } - """ - hooks.process = """ - function (event, emit) - fields = csv.openstring(event.log.message):lines()() -- parse the `message` field - event.log.message = nil -- drop the `message` field - - for column, column_name in ipairs(column_names) do -- iterate over column names - value = fields[column] -- get field value - event.log[column_name] = value -- set the corresponding field in the event - end - - emit(event) -- emit the transformed event - end - """ -#... ``` -Trying to run `vector --config vector.toml` with the same input file results in structured events being output: +Trying to run `vector --config vector.yaml` with the same input file results in structured events being output: ```json {"application_name":"","backend_type":"not initialized","command_tag":"","connection_from":"","context":"","database_name":"","detail":"","error_severity":"LOG","file":"log.csv","hint":"Future log output will go to log destination \"csvlog\".","host":"localhost","internal_query":"","internal_query_pos":"","leader_pid":"","location":"","log_time":"2020-04-09 12:48:49.661 UTC","message":"ending log output to stderr","process_id":"1","query":"","query_id":"0","query_pos":"","session_id":"localhost.1","session_line_num":"1","session_start_time":"2020-04-09 12:48:49 UTC","sql_state_code":"00000","timestamp":"2020-04-09T19:49:07Z","transaction_id":"0","user_name":"","virtual_transaction_id":""} diff --git a/website/content/en/guides/aws/cloudwatch-logs-firehose.md b/website/content/en/guides/aws/cloudwatch-logs-firehose.md index 02c4e48b40146..0760a1e2d70e8 100644 --- a/website/content/en/guides/aws/cloudwatch-logs-firehose.md +++ b/website/content/en/guides/aws/cloudwatch-logs-firehose.md @@ -79,33 +79,36 @@ export FIREHOSE_S3_BUCKET="firehose-${AWS_ACCOUNT_ID}" # a bucket to write event Let's take a look at the configuration we will be using: -```toml title="vector.toml" -[sources.firehose] - type = "aws_kinesis_firehose" - address = "0.0.0.0:8080" # the public URL will be set when configuring Firehose - access_key = "${FIREHOSE_ACCESS_KEY} # this will also be set when configuring Firehose - -[transforms.parse] - type = "remap" - inputs = ["firehose"] - drop_on_error = false - source = ''' - parsed = parse_aws_cloudwatch_log_subscription_message!(.message) - . = unnest(parsed.log_events) - . = map_values(.) -> |value| { - event = del(value.log_events) - value |= event - message = del(.message) - . |= object!(parse_json!(message)) - } - ''' +```yaml title="vector.yaml" +sources: + firehose: + type: "aws_kinesis_firehose" + address: "0.0.0.0:8080" # the public URL will be set when configuring Firehose + access_key: "${FIREHOSE_ACCESS_KEY}" # this will also be set when configuring Firehose + +transforms: + parse: + type: "remap" + inputs: ["firehose"] + drop_on_error: false + source: | + parsed = parse_aws_cloudwatch_log_subscription_message!(.message) + . = unnest(parsed.log_events) + . = map_values(.) -> |value| { + event = del(value.log_events) + value |= event + message = del(.message) + . |= object!(parse_json!(message)) + } # you may want to add more transforms here -[sinks.console] - type = "console" - inputs = ["parse"] - encoding.codec = "json" +sinks: + console: + type: "console" + inputs: ["parse"] + encoding: + codec: "json" ``` This will configure `vector` to listen for Firehose messages on the configured @@ -352,16 +355,19 @@ delivery stream. To make sure everything is wired up correctly, let's send some logs to the log group we've setup. We can use Vector for this too! -```bash -[sources.stdin] - type = "stdin" -[sinks.cloudwatch] - type = "aws_cloudwatch_logs" - inputs = ["stdin"] - group_name = "${LOG_GROUP}" - stream_name = "test" - region = "us-east-1" - encoding.codec = "json" +```yaml +sources: + stdin: + type: "stdin" +sinks: + cloudwatch: + type: "aws_cloudwatch_logs" + inputs: ["stdin"] + group_name: "${LOG_GROUP}" + stream_name: "test" + region: "us-east-1" + encoding: + codec: "json" ``` This will read lines from `stdin` and write them to CloudWatch Logs. See the @@ -377,7 +383,7 @@ Let's send some logs. For this, I'm using log data. ```bash -flog -f json | vector --config config.toml +flog -f json | vector --config config.yaml ``` This will send some logs to your log group. Within 300 seconds (the default @@ -450,11 +456,12 @@ send it as an HTTP request: The [`aws_kinesis_firehose`][aws_kinesis_firehose] source: -```toml -[sources.firehose] - type = "aws_kinesis_firehose" - address = "0.0.0.0:8080" # the public URL will be set in the Firehose config - access_key = "my secret key" # this will also be set in the Firehose config +```yaml +sources: + firehose: + type: "aws_kinesis_firehose" + address: "0.0.0.0:8080" # the public URL will be set in the Firehose config + access_key: "my secret key" # this will also be set in the Firehose config ``` will accept this request, decode the record (which is gzip'ed and then base64 encoded), to produce an event that looks like: @@ -479,21 +486,21 @@ can use a [`remap`][remap] transform and leverage the [`parse_aws_cloudwatch_log [`unnest`][unnest], [`map_values`][map_values], and [`parse_json`][parse_json] functions: -```toml -[transforms.parse] - type = "remap" - inputs = ["firehose"] - drop_on_error = false - source = ''' - parsed = parse_aws_cloudwatch_log_subscription_message!(.message) - . = unnest(parsed.log_events) - . = map_values(.) -> |value| { - event = del(value.log_events) - value |= event - message = del(.message) - . |= object!(parse_json!(message)) - } - ''' +```yaml +transforms: + parse: + type: "remap" + inputs: ["firehose"] + drop_on_error: false + source: | + parsed = parse_aws_cloudwatch_log_subscription_message!(.message) + . = unnest(parsed.log_events) + . = map_values(.) -> |value| { + event = del(value.log_events) + value |= event + message = del(.message) + . |= object!(parse_json!(message)) + } ``` Let's step through this program one function at a time. diff --git a/website/content/en/guides/level-up/managing-complex-configs.md b/website/content/en/guides/level-up/managing-complex-configs.md index 5f55a70be97ae..88c7a58a93923 100644 --- a/website/content/en/guides/level-up/managing-complex-configs.md +++ b/website/content/en/guides/level-up/managing-complex-configs.md @@ -40,28 +40,32 @@ For example, if we wished to create a chain of three transforms; `remap`, `filte and `reduce`, we can run: ```bash -vector generate /remap,filter,reduce > vector.toml +vector generate /remap,filter,reduce > vector.yaml # Find out more with `vector generate --help` ``` And most of the boilerplate will be written for us, with each component printed with an `inputs` field that specifies the component before it: -```toml title="vector.toml" -[transforms.transform0] - inputs = [ "somewhere" ] - type = "remap" - # etc ... - -[transforms.transform1] - inputs = [ "transform0" ] - type = "filter" - # etc ... - -[transforms.transform2] - inputs = [ "transform1" ] - type = "reduce" - # etc ... +```yaml title="vector.yaml" +transforms: + transform0: + inputs: + - "somewhere" + type: "remap" + # etc ... + + transform1: + inputs: + - "transform0" + type: "filter" + # etc ... + + transform2: + inputs: + - "transform1" + type: "reduce" + # etc ... ``` The IDs of the generated components are sequential (`transform0`, @@ -79,18 +83,20 @@ Let's imagine we are in the process of building the config from the [unit test guide][guides.unit-testing], we might start off with our source and the grok parser: -```toml title="vector.toml" -[sources.over_tcp] - type = "socket" - mode = "tcp" - address = "0.0.0.0:9000" - -[transforms.foo] - inputs = ["over_tcp"] - type = "remap" - source = ''' - . = parse_grok!(.message, s'%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level} %{GREEDYDATA:message}') -''' +```yaml title="vector.yaml" +sources: + over_tcp: + type: "socket" + mode: "tcp" + address: "0.0.0.0:9000" + +transforms: + foo: + inputs: + - "over_tcp" + type: "remap" + source: | + . = parse_grok!(.message, s'%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level} %{GREEDYDATA:message}') ``` A common way to test this transform might be to temporarily change the source @@ -101,30 +107,28 @@ our config to run tests rather than focusing on features. Instead, we can leave our source as a `socket` type and add a unit test to the end of our config: -```toml title="vector.toml" -[[tests]] - name = "check_simple_log" - - [[tests.inputs]] - insert_at = "foo" - type = "raw" - value = "2019-11-28T12:00:00+00:00 info Sorry, I'm busy this week Cecil" - - [[tests.outputs]] - extract_from = "foo" +```yaml title="vector.yaml" +tests: + - name: "check_simple_log" + inputs: + - insert_at: "foo" + type: "raw" + value: "2019-11-28T12:00:00+00:00 info Sorry, I'm busy this week Cecil" + outputs: + - extract_from: "foo" ``` When we add a unit test output without any conditions it will simply print the input and output events of a transform, allowing us to inspect its behavior: ```sh -$ vector test ./vector.toml -Running vector.toml tests -test vector.toml: check_simple_log ... passed +$ vector test ./vector.yaml +Running vector.yaml tests +test vector.yaml: check_simple_log ... passed inspections: ---- vector.toml --- +--- vector.yaml --- test 'check_simple_log': @@ -137,28 +141,23 @@ As we introduce new transforms to our config we can change the test output to check the latest transform. Or, occasionally, we can add conditions to an output in order to turn it into a regression test: -```toml title="vector.toml" -[[tests]] - name = "check_simple_log" - - [[tests.inputs]] - insert_at = "foo" - type = "raw" - value = "2019-11-28T12:00:00+00:00 info Sorry, I'm busy this week Cecil" - - # This is now a regression test - [[tests.outputs]] - extract_from = "foo" - [[tests.outputs.conditions]] - type = "vrl" - source = """ - assert_eq!(.message, "Sorry, I'm busy this week Cecil") - """ - - # And we add a new output without conditions for inspecting - # a new transform - [[tests.outputs]] - extract_from = "bar" +```yaml title="vector.yaml" +tests: + - name: "check_simple_log" + inputs: + - insert_at: "foo" + type: "raw" + value: "2019-11-28T12:00:00+00:00 info Sorry, I'm busy this week Cecil" + outputs: + # This is now a regression test + - extract_from: "foo" + conditions: + - type: "vrl" + source: | + assert_eq!(.message, "Sorry, I'm busy this week Cecil") + # And we add a new output without conditions for inspecting + # a new transform + - extract_from: "bar" ``` How many tests you add is at your discretion, but you probably don't need to @@ -175,10 +174,10 @@ With Vector you can split a config down into as many files as you like and run them all as a larger topology: ```bash -# These three examples run the same two configs together: -vector -c ./configs/foo.toml -c ./configs/bar.toml -vector -c ./configs/*.toml -vector -c ./configs/foo.toml ./configs/bar.toml +# These three examples run the same two configs together: +vector -c ./configs/foo.yaml -c ./configs/bar.yaml +vector -c ./configs/*.yaml +vector -c ./configs/foo.yaml ./configs/bar.yaml ``` If you have a large chain of components it's a good idea to break them out into @@ -193,52 +192,58 @@ With Vector you can define a component configuration inside a component type fol Let's take an example with the following configuration file: -```toml title="vector.toml" -[sources.syslog] -type = "syslog" -address = "0.0.0.0:514" -max_length = 42000 -mode = "tcp" - -[transforms.change_fields] -type = "remap" -inputs = ["syslog"] -source = """ -.new_field = "some value" -""" - -[sinks.stdout] -type = "console" -inputs = ["change_fields"] -target = "stdout" -encoding.codec = "json" +```yaml title="vector.yaml" +sources: + syslog: + type: "syslog" + address: "0.0.0.0:514" + max_length: 42000 + mode: "tcp" + +transforms: + change_fields: + type: "remap" + inputs: + - "syslog" + source: | + .new_field = "some value" + +sinks: + stdout: + type: "console" + inputs: + - "change_fields" + target: "stdout" + encoding: + codec: "json" ``` -We can extract the `syslog` source in the file `/etc/vector/sources/syslog.toml` +We can extract the `syslog` source in the file `/etc/vector/sources/syslog.yaml` -```toml title="syslog.toml" -type = "syslog" -address = "0.0.0.0:514" -max_length = 42000 -mode = "tcp" +```yaml title="syslog.yaml" +type: "syslog" +address: "0.0.0.0:514" +max_length: 42000 +mode: "tcp" ``` -The `change_fields` transform in the file `/etc/vector/transforms/change_fields.toml` +The `change_fields` transform in the file `/etc/vector/transforms/change_fields.yaml` -```toml title="change_fields.toml" -type = "remap" -inputs = ["syslog"] -source = """ -.new_field = "some value" -""" +```yaml title="change_fields.yaml" +type: "remap" +inputs: + - "syslog" +source: | + .new_field = "some value" ``` -And the `stdout` sink in the file `/etc/vector/sinks/stdout.toml` +And the `stdout` sink in the file `/etc/vector/sinks/stdout.yaml` -```toml title="stdout.toml" -type = "console" -inputs = ["change_fields"] -target = "stdout" +```yaml title="stdout.yaml" +type: "console" +inputs: + - "change_fields" +target: "stdout" ``` And for Vector to look for the configuration in the component type related folders, diff --git a/website/content/en/guides/level-up/managing-schemas.md b/website/content/en/guides/level-up/managing-schemas.md index ab48bee63f6e8..168e330c8312d 100644 --- a/website/content/en/guides/level-up/managing-schemas.md +++ b/website/content/en/guides/level-up/managing-schemas.md @@ -41,13 +41,13 @@ By default, Vector primarily operates on three fields: `host`, `message`, and `t Vector sets these fields on logs as it ingests data (from a [source][docs.sources]). It may be that your data does not follow this convention. In this case you can modify the global defaults for all incoming data in the `log_schema` -section of your `vector.toml`. +section of your `vector.yaml`. -```toml title="vector.toml" -[log_schema] -host_key = "instance" # default "host" -message_key = "info" # default "message" -timestamp_key = "datetime" # default "timestamp" +```yaml title="vector.yaml" +log_schema: + host_key: "instance" # default "host" + message_key: "info" # default "message" + timestamp_key: "datetime" # default "timestamp" # Sources, transforms, and sinks... ``` @@ -65,15 +65,16 @@ and you'll likely need to configure this at a more fine-grained level. Some services will produce logs with the timestamp field mapped to `@timestamp` or some other value. If your vector pipeline is only working with data passing through these systems, you can add the following to your -`vector.toml`: +`vector.yaml`: -```toml title="vector.toml" -[log_schema] - timestamp_key = "@timestamp" # Applies to all sources, sinks, and transforms! +```yaml title="vector.yaml" +log_schema: + timestamp_key: "@timestamp" # Applies to all sources, sinks, and transforms! -[sources.my_naming_confused_source] - type = "logplex" - address = "0.0.0.0:8088" +sources: + my_naming_confused_source: + type: "logplex" + address: "0.0.0.0:8088" ``` @@ -91,24 +92,24 @@ Commonly you'll want to do this near either the source or sink of your pipeline. A transform of this type looks like this: -```toml title="vector.toml" -[transforms.strip_personal_details] -type = "remap" -inputs = ["my-source-id"] -source = ''' - del(.email, .passport_number) -''' +```yaml title="vector.yaml" +transforms: + strip_personal_details: + type: "remap" + inputs: ["my-source-id"] + source: | + del(.email, .passport_number) ``` The `remap` transform has a wealth of mapping functions, and in cases where we wish to flip this concept and drop all fields except for a list of exceptions we can do that with the `only_fields` function: -```toml title="vector.toml" -[transforms.strip_personal_details] -type = "remap" -inputs = ["my-source-id"] -source = ''' - only_fields(.timestamp, .message, .host, .user_id) -''' +```yaml title="vector.yaml" +transforms: + strip_personal_details: + type: "remap" + inputs: ["my-source-id"] + source: | + only_fields(.timestamp, .message, .host, .user_id) ``` ### Example: Filtering data for GDPR compliance @@ -127,51 +128,54 @@ config, just a little example!) We can build a config that will do the first part of this, but we'll just output to console for ease of this example. -```toml title="vector.toml" -data_dir = "./data" -dns_servers = [] - -[sources.application] -max_length = 102400 -type = "stdin" - -[transforms.parse] -inputs = ["application"] -type = "remap" -source = ''' -. = parse_json!(.message) -''' - -[transforms.not_gdpr] -type = "filter" -inputs = ["parse"] -condition = ".gdpr == false" - -[transforms.gdpr_to_strip] -type = "filter" -inputs = ["parse"] -condition = ".gdpr == true" - -[transforms.gdpr_stripped] -type = "remap" -inputs = ["gdpr_to_strip"] -source = "del(.email)" - -[sinks.console] -healthcheck = true -inputs = ["not_gdpr", "gdpr_stripped"] -type = "console" -encoding.codec = "json" -[sinks.console.buffer] -type = "memory" -max_events = 500 -when_full = "block" +```yaml title="vector.yaml" +data_dir: "./data" +dns_servers: [] + +sources: + application: + max_length: 102400 + type: "stdin" + +transforms: + parse: + inputs: ["application"] + type: "remap" + source: | + . = parse_json!(.message) + + not_gdpr: + type: "filter" + inputs: ["parse"] + condition: ".gdpr == false" + + gdpr_to_strip: + type: "filter" + inputs: ["parse"] + condition: ".gdpr == true" + + gdpr_stripped: + type: "remap" + inputs: ["gdpr_to_strip"] + source: "del(.email)" + +sinks: + console: + healthcheck: true + inputs: ["not_gdpr", "gdpr_stripped"] + type: "console" + encoding: + codec: "json" + buffer: + type: "memory" + max_events: 500 + when_full: "block" ``` Let's have a look: ```bash -$ cat <<-EOF | cargo run -- --config test.toml +$ cat <<-EOF | cargo run -- --config test.yaml { "id": "user1", "gdpr": false, "email": "us-user1@datadoghq.com" } { "id": "user2", "gdpr": false, "email": "us-user2@datadoghq.com" } { "id": "user3", "gdpr": true, "email": "eu-user3@datadoghq.com" } @@ -202,15 +206,17 @@ The applications for this include some of the reasons discussed in Lets take a look at what that might look like: -```toml title="vector.toml" -[sinks.output] - inputs = ["demo_logs"] - type = "kafka" - - # Put events in the host specific topic. - topic = "{{service}}" - encoding.except_fields = ["service"] # Remove this field now and save some bytes - # ... +```yaml title="vector.yaml" +sinks: + output: + inputs: ["demo_logs"] + type: "kafka" + + # Put events in the host specific topic. + topic: "{{service}}" + encoding: + except_fields: ["service"] # Remove this field now and save some bytes + # ... ``` {{< warning >}} @@ -222,14 +228,14 @@ a field which you want templatable open an issue and let us know. It's fairly common for one part of your pipeline to expect a field to be named differently than another part! The `remap` transform can also be used to slide data around for you. -```toml title="vector.toml" -[transforms.rename_timestamp] - type = "remap" - inputs = ["source0"] - source = ''' - ."@timestamp" = .timestamp - del(.timestamp) - ''' +```yaml title="vector.yaml" +transforms: + rename_timestamp: + type: "remap" + inputs: ["source0"] + source: | + ."@timestamp" = .timestamp + del(.timestamp) ``` Other times you might need to concatenate fields together, or perform arithmetic on their numerical values, the [`remap`][docs.transforms.remap] transform can be used to do all of these things. @@ -246,14 +252,14 @@ It's useful for when: Let's pretend one of your teammates falsely assumed folks always have first and last names, so we have a `first_name` and a `last_name` field coming from a source, and we'd like to output a `name` field to a sink. -```toml title="vector.toml" -[transforms.moosh_names] - type = "remap" - inputs = ["source0"] - source = ''' - .name = .first_name + " " + .last_name - del(.first_name, .last_name) - ''' +```yaml title="vector.yaml" +transforms: + moosh_names: + type: "remap" + inputs: ["source0"] + source: | + .name = .first_name + " " + .last_name + del(.first_name, .last_name) ``` ## Coercing Data Types @@ -263,14 +269,14 @@ should be a number, or vice versa. Gadzooks! The [`remap`][docs.transforms.remap] transform is also the correct tool for this job! -```toml title="vector.toml" -[transforms.correct_source_types] - type = "remap" - inputs = ["source0"] - source = ''' - .count = int(.count) - .date = timestamp(.date, "%F") - ''' +```yaml title="vector.yaml" +transforms: + correct_source_types: + type: "remap" + inputs: ["source0"] + source: | + .count = int(.count) + .date = timestamp(.date, "%F") ``` Remember that you can follow the coercion mappings with `del` or `only_fields` functions, empowering it to drop @@ -286,12 +292,12 @@ To do this we'll use the `timestamp` function with a format string argument. To [`strftime`](https://docs.rs/chrono/0.4.10/chrono/format/strftime/index.html) documentation. Let's ship some Canadian friendly logs up to the great white north! -```toml title="vector.toml" -[transforms.format_timestamp] - type = "remap" - source = ''' - .timestamp = timestamp(.timestamp, "%Y/%m/%d:%H:%M:%S %z") - ''' +```yaml title="vector.yaml" +transforms: + format_timestamp: + type: "remap" + source: | + .timestamp = timestamp(.timestamp, "%Y/%m/%d:%H:%M:%S %z") ``` ## Working with data formats @@ -305,12 +311,14 @@ supported. In these cases, you can use the `encoding` option. The [`console`][docs.sinks.console] sink supports both `json` and `text` as its output format -```toml title="vector.toml" -[sinks.print] - type = "console" - inputs = ["source0"] - target = "stdout" - encoding.codec = "json" +```yaml title="vector.yaml" +sinks: + print: + type: "console" + inputs: ["source0"] + target: "stdout" + encoding: + codec: "json" ``` You can also use a transform like [`remap`][docs.transforms.remap] or to parse out diff --git a/website/content/en/guides/level-up/vector-tap-guide.md b/website/content/en/guides/level-up/vector-tap-guide.md index c0622994311fa..0ace743aff5d3 100644 --- a/website/content/en/guides/level-up/vector-tap-guide.md +++ b/website/content/en/guides/level-up/vector-tap-guide.md @@ -31,27 +31,26 @@ To start, we'll reference the following base configuration, but feel free to substitute your own. Just note that the [Vector API] must be enabled for `vector tap` to work. See [under the hood](#under-the-hood) for more details. -```toml -[api] -enabled = true - -[sources.in] -type = "demo_logs" -format = "shuffle" -lines = [ - "test1", - "test2", -] - -[sinks.out] -type = "blackhole" -inputs = ["in*"] +```yaml +api: + enabled: true + +sources: + in: + type: "demo_logs" + format: "shuffle" + lines: ["test1", "test2"] + +sinks: + out: + type: "blackhole" + inputs: ["in*"] ``` Run Vector with this configuration and watch the configuration for changes. ```console -vector --config path/to/config.toml -w +vector --config path/to/config.yaml -w ``` Now run `vector tap`! You should start seeing a stream of @@ -125,14 +124,12 @@ accordingly by re-matching your provided patterns. With `vector tap` running, add the following `demo_logs` source to the base configuration. -```toml -[sources.in-2] -type = "demo_logs" -format = "shuffle" -lines = [ - "new test1", - "new test2", -] +```yaml +sources: + in-2: + type: "demo_logs" + format: "shuffle" + lines: ["new test1", "new test2"] ``` You'll now see events from component `in-2` appear in `tap` output. @@ -155,43 +152,45 @@ troubleshoot a pipeline. We'll use the following Vector configuration. -```toml -[api] -enabled = true - -[sources.in] -type = "demo_logs" -format = "shuffle" -lines = [ - '{ "type": "icecream", "flavor": "strawberry" }', - '{ "type": "icecream", "flavor": "chocolate" }', - '{ "type": "icecream", "flavor": "wasabi" }', -] - -[transforms.picky] -type = "remap" -inputs = ["in"] -drop_on_abort = true -reroute_dropped = true -source = ''' - if .flavor == "strawberry" { - .happiness = 10 - } else if .flavor == "chocolate" { - .happiness = 5 - } else { - abort - } -''' - -[sinks.store] -type = "console" -inputs = ["picky"] -target = "stdout" -encoding.codec = "json" - -[sinks.trash] -type = "blackhole" -inputs = ["picky.dropped"] +```yaml +api: + enabled: true + +sources: + in: + type: "demo_logs" + format: "shuffle" + lines: + - '{ "type": "icecream", "flavor": "strawberry" }' + - '{ "type": "icecream", "flavor": "chocolate" }' + - '{ "type": "icecream", "flavor": "wasabi" }' + +transforms: + picky: + type: "remap" + inputs: ["in"] + drop_on_abort: true + reroute_dropped: true + source: | + if .flavor == "strawberry" { + .happiness = 10 + } else if .flavor == "chocolate" { + .happiness = 5 + } else { + abort + } + +sinks: + store: + type: "console" + inputs: ["picky"] + target: "stdout" + encoding: + codec: "json" + + trash: + type: "blackhole" + inputs: ["picky.dropped"] ``` Running this configuration, we expect to see our favorite ice cream logs appear diff --git a/website/content/en/highlights/2019-11-25-unit-testing-vector-config-files.md b/website/content/en/highlights/2019-11-25-unit-testing-vector-config-files.md index 141437904cc99..90799c6d1ef9d 100644 --- a/website/content/en/highlights/2019-11-25-unit-testing-vector-config-files.md +++ b/website/content/en/highlights/2019-11-25-unit-testing-vector-config-files.md @@ -26,40 +26,39 @@ mission-critical production pipelines that are collaborated on. Let's look at a basic example that uses the [`regex_parser` transform][docs.transforms.regex_parser] to parse log lines: -```toml title="vector.toml" -[sources.my_logs] - type = "file" - include = ["/var/log/my-app.log"] - -[transforms.parser] - inputs = ["my_logs"] - type = "regex_parser" - regex = "^(?P[\\w\\-:\\+]+) (?P\\w+) (?P.*)$" - -[[tests]] - name = "verify_regex" - - [tests.input] - insert_at = "parser" - type = "raw" - value = "2019-11-28T12:00:00+00:00 info Hello world" - - [[tests.outputs]] - extract_from = "parser" - - [[tests.outputs.conditions]] - type = "check_fields" - "timestamp.equals" = "2019-11-28T12:00:00+00:00" - "level.equals" = "info" - "message.equals" = "Hello world" +```yaml title="vector.yaml" +sources: + my_logs: + type: "file" + include: ["/var/log/my-app.log"] + +transforms: + parser: + inputs: ["my_logs"] + type: "regex_parser" + regex: "^(?P[\\w\\-:\\+]+) (?P\\w+) (?P.*)$" + +tests: + - name: "verify_regex" + input: + insert_at: "parser" + type: "raw" + value: "2019-11-28T12:00:00+00:00 info Hello world" + outputs: + - extract_from: "parser" + conditions: + - type: "check_fields" + "timestamp.equals": "2019-11-28T12:00:00+00:00" + "level.equals": "info" + "message.equals": "Hello world" ``` And you can run the tests via the new `test` subcommand: ```sh -$ vector test ./vector.toml -Running ./vector.toml tests -Test ./vector.toml: verify_regex ... passed +$ vector test ./vector.yaml +Running ./vector.yaml tests +Test ./vector.yaml: verify_regex ... passed ``` ## Why? diff --git a/website/content/en/highlights/2019-12-13-custom-dns.md b/website/content/en/highlights/2019-12-13-custom-dns.md index 3fcbd88fd24c0..a9d0088aa4675 100644 --- a/website/content/en/highlights/2019-12-13-custom-dns.md +++ b/website/content/en/highlights/2019-12-13-custom-dns.md @@ -20,8 +20,8 @@ servers in your configs. The configuration isn't complicated, it's a global array field `dns_servers`: -```toml -dns_servers = ["0.0.0.0:53"] +```yaml +dns_servers: ["0.0.0.0:53"] ``` When `dns_servers` is set Vector will ignore the system configuration and use diff --git a/website/content/en/highlights/2019-12-16-ec2-metadata.md b/website/content/en/highlights/2019-12-16-ec2-metadata.md index 58d92e577ad63..117441d3db43d 100644 --- a/website/content/en/highlights/2019-12-16-ec2-metadata.md +++ b/website/content/en/highlights/2019-12-16-ec2-metadata.md @@ -21,19 +21,19 @@ our brand spanking new [`aws_ec2_metadata` transform][docs.transforms.aws_ec2_me Configuration isn't complicated, just add and hook up the transform. If you don't want all enrichments added then white-list them with the `fields` option: -```toml -[transforms.fill_me_up] - type = "aws_ec2_metadata" - inputs = ["my-source-id"] - fields = [ - "instance-id", - "local-hostname", - "public-hostname", - "public-ipv4", - "ami-id", - "availability-zone", - "region", - ] +```yaml +transforms: + fill_me_up: + type: "aws_ec2_metadata" + inputs: ["my-source-id"] + fields: + - "instance-id" + - "local-hostname" + - "public-hostname" + - "public-ipv4" + - "ami-id" + - "availability-zone" + - "region" ``` For more guidance get on the [reference page][docs.transforms.aws_ec2_metadata]. diff --git a/website/content/en/highlights/2020-01-07-prometheus-source.md b/website/content/en/highlights/2020-01-07-prometheus-source.md index 2d1797aff9749..45ca46ead7bc3 100644 --- a/website/content/en/highlights/2020-01-07-prometheus-source.md +++ b/website/content/en/highlights/2020-01-07-prometheus-source.md @@ -25,11 +25,12 @@ metrics data model and tested our interoperability between metrics sources. To use it simply add the source config and point it towards the hosts you wish to scrape: -```toml -[sources.my_source_id] - type = "prometheus_scrape" - hosts = ["http://localhost:9090"] - scrape_interval_secs = 1 +```yaml +sources: + my_source_id: + type: "prometheus_scrape" + hosts: ["http://localhost:9090"] + scrape_interval_secs: 1 ``` For more guidance get on the [reference page][docs.sources.prometheus]. diff --git a/website/content/en/highlights/2020-01-20-splunk-hec-specify-indexed-fields.md b/website/content/en/highlights/2020-01-20-splunk-hec-specify-indexed-fields.md index e8d6f6a018f61..d1aadc19bc248 100644 --- a/website/content/en/highlights/2020-01-20-splunk-hec-specify-indexed-fields.md +++ b/website/content/en/highlights/2020-01-20-splunk-hec-specify-indexed-fields.md @@ -21,9 +21,10 @@ will _not_ index any fields by default. In order to mark desired fields as indexed you can use the optional configuration option `indexed_fields`: -```toml title="vector.toml" - [sinks.my_sink_id] - type = "splunk_hec" - inputs = ["my-source-id"] -+ indexed_fields = ["foo", "bar"] +```yaml title="vector.yaml" + sinks: + my_sink_id: + type: "splunk_hec" + inputs: ["my-source-id"] ++ indexed_fields: ["foo", "bar"] ``` diff --git a/website/content/en/highlights/2020-02-05-merge-partial-docker-events.md b/website/content/en/highlights/2020-02-05-merge-partial-docker-events.md index 2dc7a049aba31..d1af3aaae1249 100644 --- a/website/content/en/highlights/2020-02-05-merge-partial-docker-events.md +++ b/website/content/en/highlights/2020-02-05-merge-partial-docker-events.md @@ -20,10 +20,11 @@ events. This can be a very difficult and frustrating problem to solve with other tools (we speak from experience). In this release, Vector solves this automatically with a new `auto_partial_merge` option in the `docker_logs` source. -```toml title="vector.toml" -[sources.my_source_id] - type = "docker_logs" - auto_partial_merge = true +```yaml title="vector.yaml" +sources: + my_source_id: + type: "docker_logs" + auto_partial_merge: true ``` We love assimilation and look forward to a future where our individualistic diff --git a/website/content/en/highlights/2020-02-14-global-log-schema.md b/website/content/en/highlights/2020-02-14-global-log-schema.md index 6949d0d409fae..213030fd5a3d5 100644 --- a/website/content/en/highlights/2020-02-14-global-log-schema.md +++ b/website/content/en/highlights/2020-02-14-global-log-schema.md @@ -24,11 +24,11 @@ the default names for the [`message_key`][docs.global-options#message_key], [`host_key`][docs.global-options#host_key], [`timestamp_key`][docs.global-options#host_key], and more: -```toml title="vector.toml" -[log_schema] - host_key = "host" # default - message_key = "message" # default - timestamp_key = "timestamp" # default +```yaml title="vector.yaml" +log_schema: + host_key: "host" # default + message_key: "message" # default + timestamp_key: "timestamp" # default ``` Why is this useful? diff --git a/website/content/en/highlights/2020-02-21-file-source-multiline-support.md b/website/content/en/highlights/2020-02-21-file-source-multiline-support.md index 762faf8c73e4f..f09c9f4789c95 100644 --- a/website/content/en/highlights/2020-02-21-file-source-multiline-support.md +++ b/website/content/en/highlights/2020-02-21-file-source-multiline-support.md @@ -34,16 +34,16 @@ foobar.rb:6:in `/': divided by 0 (ZeroDivisionError) You can merge them with the following config: -```toml title="vector.toml" -[sources.my_file_source] - type = "file" - # ... - - [sources.my_file_source.multiline] - start_pattern = "^[^\\s]" - mode = "continue_through" - condition_pattern = "^[\\s]+from" - timeout_ms = 1000 +```yaml title="vector.yaml" +sources: + my_file_source: + type: "file" + # ... + multiline: + start_pattern: "^[^\\s]" + mode: "continue_through" + condition_pattern: "^[\\s]+from" + timeout_ms: 1000 ``` And if this doesn't do it, you can always fall back to the [`lua` transform][docs.transforms.lua]. diff --git a/website/content/en/highlights/2020-02-24-swimlanes-transform.md b/website/content/en/highlights/2020-02-24-swimlanes-transform.md index 815694d51fea5..29e7c67cc280a 100644 --- a/website/content/en/highlights/2020-02-24-swimlanes-transform.md +++ b/website/content/en/highlights/2020-02-24-swimlanes-transform.md @@ -16,15 +16,16 @@ The new [`swimlanes` transform][docs.transforms.swimlanes] makes it much easier to configure conditional branches of transforms and sinks. For example, you can easily create [if/else pipelines][docs.transforms.swimlanes#examples]. -```toml title="vector.toml" -[transforms.lanes] - types = "swimlanes" - - [transforms.my_transform_id.lanes.errors] - "level.eq" = "error" - - [transforms.my_transform_id.lanes.not_errors] - "level.neq" = "error" +```yaml title="vector.yaml" +transforms: + lanes: + types: "swimlanes" + my_transform_id: + lanes: + errors: + "level.eq": "error" + not_errors: + "level.neq": "error" ``` Remember to occasionally let your branches mingle so that they don't completely diff --git a/website/content/en/highlights/2020-03-04-encoding-only-fields-except-fields.md b/website/content/en/highlights/2020-03-04-encoding-only-fields-except-fields.md index 08032a186b521..7f6e5833a431d 100644 --- a/website/content/en/highlights/2020-03-04-encoding-only-fields-except-fields.md +++ b/website/content/en/highlights/2020-03-04-encoding-only-fields-except-fields.md @@ -23,11 +23,13 @@ Vector has deprecated the root-level `encoding` option in favor of new Upgrading is easy: -```toml title="vector.toml" - [sinks.my-sink] - type = "..." -- encoding = "json" -+ encoding.codec = "json" -+ encoding.except_fields = ["_meta"] # optional -+ encoding.timestamp_format = "rfc3339" # optional +```yaml title="vector.yaml" + sinks: + my-sink: + type: "..." +- encoding: "json" ++ encoding: ++ codec: "json" ++ except_fields: ["_meta"] # optional ++ timestamp_format: "rfc3339" # optional ``` diff --git a/website/content/en/highlights/2020-03-10-dedupe-transform.md b/website/content/en/highlights/2020-03-10-dedupe-transform.md index 4a596b98f448b..44b68d9d489d1 100644 --- a/website/content/en/highlights/2020-03-10-dedupe-transform.md +++ b/website/content/en/highlights/2020-03-10-dedupe-transform.md @@ -22,14 +22,16 @@ mistakes that accidentally duplicate logs. This mistake can easily double Simply add the transform to your pipeline: -```toml -[transforms.my_transform_id] - # General - type = "dedupe" # required - inputs = ["my-source-id"] # required +```yaml +transforms: + my_transform_id: + # General + type: "dedupe" # required + inputs: ["my-source-id"] # required - # Fields - fields.match = ["timestamp", "host", "message"] # optional, default + # Fields + fields: + match: ["timestamp", "host", "message"] # optional, default ``` {{< success >}} diff --git a/website/content/en/highlights/2020-03-11-tag-cardinality-limit-transform.md b/website/content/en/highlights/2020-03-11-tag-cardinality-limit-transform.md index c6a053786906a..b184613dbbcfb 100644 --- a/website/content/en/highlights/2020-03-11-tag-cardinality-limit-transform.md +++ b/website/content/en/highlights/2020-03-11-tag-cardinality-limit-transform.md @@ -20,13 +20,14 @@ protect against this we built a new Getting started is easy. Simply add this component to your pipeline: -```toml title="vector.toml" -[transforms.tag_protection] - type = "tag_cardinality_limit" - inputs = ["my-source-id"] - limit_exceeded_action = "drop_tag" - mode = "exact" - value_limit = 500 +```yaml title="vector.yaml" +transforms: + tag_protection: + type: "tag_cardinality_limit" + inputs: ["my-source-id"] + limit_exceeded_action: "drop_tag" + mode: "exact" + value_limit: 500 ``` {{< success >}} diff --git a/website/content/en/highlights/2020-03-31-filter-transform.md b/website/content/en/highlights/2020-03-31-filter-transform.md index b1524f5d60910..5409db5a6b28a 100644 --- a/website/content/en/highlights/2020-03-31-filter-transform.md +++ b/website/content/en/highlights/2020-03-31-filter-transform.md @@ -20,15 +20,17 @@ transform since it is much more expressive. ## Get Started -```toml title="vector.toml" -[transforms.haproxy_errors] - # General - type = "filter" - inputs = ["my-source-id"] +```yaml title="vector.yaml" +transforms: + haproxy_errors: + # General + type: "filter" + inputs: ["my-source-id"] - # Conditions - condition."level.eq" = "error" - condition."service.eq" = "haproxy" + # Conditions + condition: + "level.eq": "error" + "service.eq": "haproxy" ``` Check out the [docs][docs.transforms.filter] for a fill list of available diff --git a/website/content/en/highlights/2020-04-01-more-condition-predicates.md b/website/content/en/highlights/2020-04-01-more-condition-predicates.md index 277e780274890..68d56c1b9d38e 100644 --- a/website/content/en/highlights/2020-04-01-more-condition-predicates.md +++ b/website/content/en/highlights/2020-04-01-more-condition-predicates.md @@ -31,10 +31,12 @@ following predicates were added: For example, you can filter all messages that contain the `error` term with the new `contains` predicate: -```toml -[transforms.errors] - type = "filter" - condition."message.contain" = "error" +```yaml +transforms: + errors: + type: "filter" + condition: + "message.contain": "error" ``` The world is your oyster. diff --git a/website/content/en/highlights/2020-05-27-add-support-for-loading-multiple-cas.md b/website/content/en/highlights/2020-05-27-add-support-for-loading-multiple-cas.md index dc5a36becf1a4..6124b4d055aa7 100644 --- a/website/content/en/highlights/2020-05-27-add-support-for-loading-multiple-cas.md +++ b/website/content/en/highlights/2020-05-27-add-support-for-loading-multiple-cas.md @@ -15,15 +15,17 @@ Working with `openssl` isn't very fun, and we don't want to inflict that on you. This is particularly useful if you have a socket source: -```toml title="vector.toml" -[sources.tls] - type = "socket" - address = "0.0.0.0:6514" - mode = "tcp" - tls.enabled = true - tls.crt_path = "cert.pfx" - tls.ca_path = "ca.pem" # Now supported: More complicated PEMS! - tls.verify_certificate = true +```yaml title="vector.yaml" +sources: + tls: + type: "socket" + address: "0.0.0.0:6514" + mode: "tcp" + tls: + enabled: true + crt_path: "cert.pfx" + ca_path: "ca.pem" # Now supported: More complicated PEMS! + verify_certificate: true ``` If it doesn't, that's a bug. [**Report it.**][urls.new_bug_report] We squash bugs. diff --git a/website/content/en/highlights/2020-07-10-add-reduce-transform.md b/website/content/en/highlights/2020-07-10-add-reduce-transform.md index 96b757c11f88c..fc7034c4cd871 100644 --- a/website/content/en/highlights/2020-07-10-add-reduce-transform.md +++ b/website/content/en/highlights/2020-07-10-add-reduce-transform.md @@ -54,42 +54,49 @@ Then output this (but not formatted so nicely!): We'll run this config: -```toml title=vector.toml -data_dir = "tmp" - -[sources.source0] - include = ["input.log"] - start_at_beginning = true - type = "file" - fingerprinting.strategy = "device_and_inode" - -[transforms.transform0] - inputs = ["source0"] - type = "json_parser" - field = "message" - -[transforms.transform1] - inputs = ["transform0"] - type = "reduce" - identifier_fields = ["request_id"] - ends_when.type = "check_fields" - ends_when."response_status.exists" = true - merge_strategies.message = "discard" - merge_strategies.query = "discard" - merge_strategies.template = "discard" - merge_strategies.query_duration_ms = "sum" - merge_strategies.render_duration_ms = "sum" - merge_strategies.response_duration_ms = "sum" - -[sinks.sink0] - healthcheck = true - inputs = ["transform1"] - type = "file" - path = "output.log" - encoding = "ndjson" - buffer.type = "memory" - buffer.max_events = 500 - buffer.when_full = "block"s +```yaml title=vector.yaml +data_dir: "tmp" + +sources: + source0: + include: ["input.log"] + start_at_beginning: true + type: "file" + fingerprinting: + strategy: "device_and_inode" + +transforms: + transform0: + inputs: ["source0"] + type: "json_parser" + field: "message" + + transform1: + inputs: ["transform0"] + type: "reduce" + identifier_fields: ["request_id"] + ends_when: + type: "check_fields" + "response_status.exists": true + merge_strategies: + message: "discard" + query: "discard" + template: "discard" + query_duration_ms: "sum" + render_duration_ms: "sum" + response_duration_ms: "sum" + +sinks: + sink0: + healthcheck: true + inputs: ["transform1"] + type: "file" + path: "output.log" + encoding: "ndjson" + buffer: + type: "memory" + max_events: 500 + when_full: "block" ``` We hope you find this useful! diff --git a/website/content/en/highlights/2020-09-18-adaptive-concurrency.md b/website/content/en/highlights/2020-09-18-adaptive-concurrency.md index a7236e736224d..925f66081b7b6 100644 --- a/website/content/en/highlights/2020-09-18-adaptive-concurrency.md +++ b/website/content/en/highlights/2020-09-18-adaptive-concurrency.md @@ -25,11 +25,13 @@ inspired by TCP congestion control algorithms. This feature, like all Vector features, will begin its life in public beta and be available on an opt-in basis. To get it, enable it for each sink: -```toml -[sinks.my-sink] -type = "..." # any http-based sink -request.concurrency = "adaptive" -# and remove the request.rate_limit_* settings +```yaml +sinks: + my-sink: + type: "..." # any http-based sink + request: + concurrency: "adaptive" + # and remove the request.rate_limit_* settings ``` [announcement]: /blog/adaptive-request-concurrency/ diff --git a/website/content/en/highlights/2020-10-27-metrics-integrations.md b/website/content/en/highlights/2020-10-27-metrics-integrations.md index 50bd6e8132ca1..6666f50fd247b 100644 --- a/website/content/en/highlights/2020-10-27-metrics-integrations.md +++ b/website/content/en/highlights/2020-10-27-metrics-integrations.md @@ -39,14 +39,16 @@ the single agent for all of your logs, metrics, and traces. To get started with these sources, define them and go: -```toml -[sources.host_metrics] -type = "host_metrics" # or apache_metrics, mongodb_metrics, or internal_metrics +```yaml +sources: + host_metrics: + type: "host_metrics" # or apache_metrics, mongodb_metrics, or internal_metrics # Then connect them to a sink: -[sinks.prometheus] -type = "prometheus" -inputs = ["host_metrics"] +sinks: + prometheus: + type: "prometheus" + inputs: ["host_metrics"] ``` Tada! One agent for all of your data. Check out the [docs][docs] for more diff --git a/website/content/en/highlights/2020-11-19-prometheus-remote-integrations.md b/website/content/en/highlights/2020-11-19-prometheus-remote-integrations.md index 604b85bfc4bf7..965b42c6b0365 100644 --- a/website/content/en/highlights/2020-11-19-prometheus-remote-integrations.md +++ b/website/content/en/highlights/2020-11-19-prometheus-remote-integrations.md @@ -31,17 +31,20 @@ Prometheus operators can route a stream of Prometheus data to the archiving solution of their choice. For this use case we recommend object stores for their cheap and durable qualities: -```toml title="vector.toml" -[sources.prometheus] - type = "prometheus_remote_write" - -[transforms.convert] - type = "metric_to_log" - inputs = ["prometheus"] - -[sinks.backup] - type = "aws_s3" - inputs = ["convert"] +```yaml title="vector.yaml" +sources: + prometheus: + type: "prometheus_remote_write" + +transforms: + convert: + type: "metric_to_log" + inputs: ["prometheus"] + +sinks: + backup: + type: "aws_s3" + inputs: ["convert"] ``` Swap `aws_s3` with `gcp_cloud_storage` or other object stores. @@ -68,13 +71,15 @@ To get started, setup the new your metrics to [Datadog][datadog], [New Relic][new_relic], [Influx][influx], [Elasticsearch][elastic], and [many other sinks][sinks]: -```toml title="vector.toml" -[sources.prometheus] - type = "prometheus_remote_write" +```yaml title="vector.yaml" +sources: + prometheus: + type: "prometheus_remote_write" -[sinks.datadog] - type = "datadog_metrics" - inputs = ["prometheus"] +sinks: + datadog: + type: "datadog_metrics" + inputs: ["prometheus"] ``` [chronosphere]: https://chronosphere.io/ diff --git a/website/content/en/highlights/2020-11-25-json-yaml-config-formats.md b/website/content/en/highlights/2020-11-25-json-yaml-config-formats.md index db44d80923daf..f977ab64a4de8 100644 --- a/website/content/en/highlights/2020-11-25-json-yaml-config-formats.md +++ b/website/content/en/highlights/2020-11-25-json-yaml-config-formats.md @@ -12,9 +12,9 @@ badges: --- To ensure Vector fits into existing workflows, like Kubernetes, we've added -support for JSON and YAML config formats in addition to TOML. TOML will -continue to be our default format, but users are welcome to use JSON and YAML -as they see fit. +support for JSON and YAML config formats in addition to TOML. YAML is now the +recommended and default configuration format, but users are welcome to use +TOML and JSON as they see fit. ## Use cases diff --git a/website/content/en/highlights/2020-12-23-graphql-api.md b/website/content/en/highlights/2020-12-23-graphql-api.md index 70c40ebfac28a..e749377c8dd05 100644 --- a/website/content/en/highlights/2020-12-23-graphql-api.md +++ b/website/content/en/highlights/2020-12-23-graphql-api.md @@ -28,10 +28,10 @@ The GraphQL API for Vector is **disabled by default**. We want to keep Vector's behavior as predictable and secure as possible, so we chose to make the feature opt-in. To enable the API, add this to your Vector config: -```toml -[api] -enabled = true -address = "127.0.0.1:8686" # optional. Change IP/port if required +```yaml +api: + enabled: true + address: "127.0.0.1:8686" # optional. Change IP/port if required ``` ## Read more diff --git a/website/content/en/highlights/2020-12-23-internal-logs-source.md b/website/content/en/highlights/2020-12-23-internal-logs-source.md index b90f1899975e8..fb8ae9aed167e 100644 --- a/website/content/en/highlights/2020-12-23-internal-logs-source.md +++ b/website/content/en/highlights/2020-12-23-internal-logs-source.md @@ -38,24 +38,27 @@ metrics and modify and ship them however you wish. Here's an example Vector configuration that ships Vector's logs to Splunk and allows its internal metrics to be scraped by [Prometheus]: -```toml -[sources.vector_logs] -type = "internal_logs" - -[sources.vector_metrics] -type = "internal_metrics" - -[sinks.splunk] -type = "splunk_hec" -inputs = ["vector_logs"] -endpoint = "https://my-account.splunkcloud.com" -token = "${SPLUNK_HEC_TOKEN}" -encoding.codec = "json" - -[sinks.prometheus] -type = "prometheus" -inputs = ["vector_metrics"] -address = "0.0.0.0:9090" +```yaml +sources: + vector_logs: + type: "internal_logs" + + vector_metrics: + type: "internal_metrics" + +sinks: + splunk: + type: "splunk_hec" + inputs: ["vector_logs"] + endpoint: "https://my-account.splunkcloud.com" + token: "${SPLUNK_HEC_TOKEN}" + encoding: + codec: "json" + + prometheus: + type: "prometheus" + inputs: ["vector_metrics"] + address: "0.0.0.0:9090" ``` [internal_logs]: /docs/reference/configuration/sources/internal_logs diff --git a/website/content/en/highlights/2021-01-10-kafka-sink-metrics.md b/website/content/en/highlights/2021-01-10-kafka-sink-metrics.md index 389e85dbf50ff..35e95c6444eba 100644 --- a/website/content/en/highlights/2021-01-10-kafka-sink-metrics.md +++ b/website/content/en/highlights/2021-01-10-kafka-sink-metrics.md @@ -18,14 +18,17 @@ metric events through Kafka. Metrics events are encoded into a format that mimics our [internal metrics data model], ideal for custom consumers on the other end. Getting started is easy: -```toml -[sources.host_metrics] -type = "host_metrics" +```yaml +sources: + host_metrics: + type: "host_metrics" -[sinks.kafka] -type = "kafka" -inputs = ["host_metrics"] -encoding.codec = "json" +sinks: + kafka: + type: "kafka" + inputs: ["host_metrics"] + encoding: + codec: "json" ``` ## Caveats diff --git a/website/content/en/highlights/2021-01-20-wildcard-identifiers.md b/website/content/en/highlights/2021-01-20-wildcard-identifiers.md index 9cff6a92702d1..88f82a0d75539 100644 --- a/website/content/en/highlights/2021-01-20-wildcard-identifiers.md +++ b/website/content/en/highlights/2021-01-20-wildcard-identifiers.md @@ -15,26 +15,28 @@ badges: [PR 6170][pr_6170] introduced wildcards when referencing component names in the `inputs` option. This allows you to build dynamic topologies. This feature comes with one limitation: the wildcard must be at the end of the string. -```toml -[sources.app1_logs] -type = "file" -includes = ["/var/log/app1.log"] +```yaml +sources: + app1_logs: + type: "file" + includes: ["/var/log/app1.log"] -[sources.app2_logs] -type = "file" -includes = ["/var/log/app.log"] + app2_logs: + type: "file" + includes: ["/var/log/app.log"] -[sources.system_logs] -type = "file" -includes = ["/var/log/system.log"] + system_logs: + type: "file" + includes: ["/var/log/system.log"] -[sinks.app_logs] -type = "datadog_logs" -inputs = ["app*"] +sinks: + app_logs: + type: "datadog_logs" + inputs: ["app*"] -[sinks.archive] -type = "aws_s3" -inputs = ["app*", "system_logs"] + archive: + type: "aws_s3" + inputs: ["app*", "system_logs"] ``` [pr_6170]: https://github.com/vectordotdev/vector/pull/6170 diff --git a/website/content/en/highlights/2021-02-16-0-12-upgrade-guide.md b/website/content/en/highlights/2021-02-16-0-12-upgrade-guide.md index 9f05bda354d2f..7f5a8f361039d 100644 --- a/website/content/en/highlights/2021-02-16-0-12-upgrade-guide.md +++ b/website/content/en/highlights/2021-02-16-0-12-upgrade-guide.md @@ -117,12 +117,12 @@ The following transforms have been deprecated in favor of the new [`remap` trans Deprecation notices have been placed on each of these transforms with example VRL programs that demonstrate how to migrate to the new `remap` transform. For example, migrating from the `json_parser` transform is as simple as: -```toml -[transforms.remap] -type = "remap" -source = ''' -. = merge(., parse_json!(.message)) -''' +```yaml +transforms: + remap: + type: "remap" + source: | + . = merge(., parse_json!(.message)) ``` **You do not need to upgrade immediately. These transforms will not be removed until Vector hits 1.0, a milestone that diff --git a/website/content/en/highlights/2021-02-16-filter-remap-support.md b/website/content/en/highlights/2021-02-16-filter-remap-support.md index 50d47fbabb7d5..26cc72dff7ce5 100644 --- a/website/content/en/highlights/2021-02-16-filter-remap-support.md +++ b/website/content/en/highlights/2021-02-16-filter-remap-support.md @@ -24,20 +24,26 @@ Previously, the `filter` transform required you to specify conditions using The example configuration below shows the same `filter` transform using the old system (`check_fields`) and the new system (`remap`): -```toml -[transforms.filter_out_non_critical] -type = "filter" -inputs = ["http-server-logs"]f - -# Using check_fields -condition.type = "check_fields" -condition.message.status_code.ne = 200 -condition.message.severity.ne = "info" -condition.message.severity.ne = "debug" - -# Using remap -condition.type = "remap" -condition.source = '.status_code != 200 && !includes(["info", "debug"], .severity)' +```yaml +transforms: + filter_out_non_critical: + type: "filter" + inputs: ["http-server-logs"] + + # Using check_fields + condition: + type: "check_fields" + message: + status_code: + ne: 200 + severity: + ne: "info" + # ne: "debug" + + # Using remap + condition: + type: "remap" + source: '.status_code != 200 && !includes(["info", "debug"], .severity)' ``` [filter]: /docs/reference/configuration/transforms/filter diff --git a/website/content/en/highlights/2021-04-21-vector-tap.md b/website/content/en/highlights/2021-04-21-vector-tap.md index be234b5ac9d89..39b89194cbab4 100644 --- a/website/content/en/highlights/2021-04-21-vector-tap.md +++ b/website/content/en/highlights/2021-04-21-vector-tap.md @@ -27,19 +27,22 @@ of this component. For example, given the configuration: -```toml -[api] - enabled = true -[sources.in] - type = "generator" - format = "shuffle" - interval = 1.0 - lines = ["Hello World"] - sequence = true - -[sinks.out] - type = "blackhole" - inputs = ["in"] +```yaml +api: + enabled: true + +sources: + in: + type: "generator" + format: "shuffle" + interval: 1.0 + lines: ["Hello World"] + sequence: true + +sinks: + out: + type: "blackhole" + inputs: ["in"] ``` If you were to run `vector` and then, in another terminal, run `vector tap in`, diff --git a/website/content/en/highlights/2021-04-21-vrl-abort.md b/website/content/en/highlights/2021-04-21-vrl-abort.md index 5e784ab56d505..234fa9a948c37 100644 --- a/website/content/en/highlights/2021-04-21-vrl-abort.md +++ b/website/content/en/highlights/2021-04-21-vrl-abort.md @@ -28,27 +28,32 @@ validation on the event before processing it, discarding any invalid events. Given a config of: -```toml -[sources.in] - type = "generator" - format = "shuffle" - interval = 1.0 - lines = ['{ "message": "valid message", "type": "ok"}', '{ "message": "invalid message", "type": "unknown"}'] +```yaml +sources: + in: + type: "generator" + format: "shuffle" + interval: 1.0 + lines: + - '{ "message": "valid message", "type": "ok"}' + - '{ "message": "invalid message", "type": "unknown"}' -[transforms.remap] - type = "remap" - inputs = ["in"] - source = """ - . |= object!(parse_json!(string!(.message))) - if .type != "ok" { - abort # unknown type - } - """ +transforms: + remap: + type: "remap" + inputs: ["in"] + source: | + . |= object!(parse_json!(string!(.message))) + if .type != "ok" { + abort # unknown type + } -[sinks.out] - type = "console" - inputs = ["remap"] - encoding.codec = "json" +sinks: + out: + type: "console" + inputs: ["remap"] + encoding: + codec: "json" ``` You would expect to see something like: diff --git a/website/content/en/highlights/2021-07-14-0-15-upgrade-guide.md b/website/content/en/highlights/2021-07-14-0-15-upgrade-guide.md index 38b39874a1b5f..10d03ca5a8f74 100644 --- a/website/content/en/highlights/2021-07-14-0-15-upgrade-guide.md +++ b/website/content/en/highlights/2021-07-14-0-15-upgrade-guide.md @@ -42,38 +42,42 @@ This means, if you are presently using the deprecated `check_fields` syntax, you For example, if you previously had: -```toml -[transforms.sample] -type = "sample" -inputs = ["in"] -rate = 10 -key_field = "message" -exclude."message.contains" = "error" +```yaml +transforms: + sample: + type: "sample" + inputs: ["in"] + rate: 10 + key_field: "message" + exclude: + "message.contains": "error" ``` You will need to add `exclude.type = "check_fields"` like: -```toml -[transforms.sample] -type = "sample" -inputs = ["in"] -rate = 10 -key_field = "message" -exclude."type" = "check_fields" -exclude."message.contains" = "error" +```yaml +transforms: + sample: + type: "sample" + inputs: ["in"] + rate: 10 + key_field: "message" + exclude: + "type": "check_fields" + "message.contains": "error" ``` To convert this to the new [VRL][VRL] conditions, you would write: -```toml -[transforms.sample] -type = "sample" -inputs = ["in"] -rate = 10 -key_field = "message" -exclude = """ - contains!(.message, "error") -""" +```yaml +transforms: + sample: + type: "sample" + inputs: ["in"] + rate: 10 + key_field: "message" + exclude: | + contains!(.message, "error") ``` We recommend upgrading to the [VRL][VRL] conditions as these are much more powerful than the legacy `check_fields`-style @@ -85,40 +89,42 @@ The `remap` condition type has been renamed `vrl` in this release to better high a [VRL][VRL] program. Most examples of using this condition type have the short-hand condition config of just specifying the [VRL][VRL] program without specifying a `type`. For example: -```toml -[transforms.filter_a] - inputs = ["stdin"] - type = "filter" - condition = ''' +```yaml +transforms: + filter_a: + inputs: ["stdin"] + type: "filter" + condition: | message = if exists(.tags) { .tags.message } else { .message } message == "test filter 1" - ''' ``` Which is automatically a [VRL][VRL] condition. However, if you were specifying the `type` like: -```toml -[transforms.filter_a] -inputs = ["stdin"] -type = "filter" -condition.type = "remap" -condition.source = ''' - message = if exists(.tags) { .tags.message } else { .message } - message == "test filter 1" -''' +```yaml +transforms: + filter_a: + inputs: ["stdin"] + type: "filter" + condition: + type: "remap" + source: | + message = if exists(.tags) { .tags.message } else { .message } + message == "test filter 1" ``` Then you will need to update `type = "remap"` to `type = "vrl"` like: -```toml -[transforms.filter_a] -inputs = ["stdin"] -type = "filter" -condition.type = "vrl" -condition.source = ''' - message = if exists(.tags) { .tags.message } else { .message } - message == "test filter 1" -''' +```yaml +transforms: + filter_a: + inputs: ["stdin"] + type: "filter" + condition: + type: "vrl" + source: | + message = if exists(.tags) { .tags.message } else { .message } + message == "test filter 1" ``` [vrl]: /docs/reference/vrl/ diff --git a/website/content/en/highlights/2021-07-14-vector-graph.md b/website/content/en/highlights/2021-07-14-vector-graph.md index 07a85dbd18f82..b8e7ddcdfc83c 100644 --- a/website/content/en/highlights/2021-07-14-vector-graph.md +++ b/website/content/en/highlights/2021-07-14-vector-graph.md @@ -14,74 +14,78 @@ badges: This release adds a new subcommand `vector graph` to output the topology specified by you Vector configuration as a graph in [DOT][DOT] format. This output can then be visualized using [Graphviz] to produce an image. -For example, if you had a config, `vector.toml`, like: +For example, if you had a config, `vector.yaml`, like: -```toml +```yaml ## ## Sources ## -[sources.internal_metrics] -type = "internal_metrics" +sources: + internal_metrics: + type: "internal_metrics" -[sources.dd_logs] -type = "datadog_logs" # required -acknowledgements = false # optional, default -address = "0.0.0.0:8282" # required + dd_logs: + type: "datadog_logs" # required + acknowledgements: false # optional, default + address: "0.0.0.0:8282" # required -[sources.file_gen] -type = "file" -include = ["/var/log/file_gen/**/*.log"] -read_from = "beginning" + file_gen: + type: "file" + include: ["/var/log/file_gen/**/*.log"] + read_from: "beginning" ## ## Transforms ## -[transforms.remap] -type = "remap" -inputs = ["file_gen"] -source = ''' -.agent_name = "vector" -parsed, err = parse_json(.message) -if err == null { - .message = parsed - .format = "json" -} else { - .format = "ascii" -} -matches = parse_regex!(.file, r'.*/(?P\d+)-(?P\w+).log') -.origin, err = .host + "/" + matches.name + "/" + matches.num -if err != null { - log("Failed to parse origin from file name", level: "error") -} -''' +transforms: + remap: + type: "remap" + inputs: ["file_gen"] + source: | + .agent_name = "vector" + parsed, err = parse_json(.message) + if err == null { + .message = parsed + .format = "json" + } else { + .format = "ascii" + } + matches = parse_regex!(.file, r'.*/(?P\d+)-(?P\w+).log') + .origin, err = .host + "/" + matches.name + "/" + matches.num + if err != null { + log("Failed to parse origin from file name", level: "error") + } ## ## Sinks ## -[sinks.prometheus] -type = "prometheus_exporter" -inputs = ["internal_metrics"] -address = "0.0.0.0:9598" - -[sinks.dd_logs_egress] -type = "datadog_logs" -inputs = ["dd_logs"] -default_api_key = "" -encoding.codec = "json" -request.concurrency = "adaptive" -batch.max_bytes = 5242880 -request.rate_limit_num = 1000 - - -[sinks.blackhole] -type = "blackhole" -inputs = ["remap"] +sinks: + prometheus: + type: "prometheus_exporter" + inputs: ["internal_metrics"] + address: "0.0.0.0:9598" + + dd_logs_egress: + type: "datadog_logs" + inputs: ["dd_logs"] + default_api_key: "" + encoding: + codec: "json" + request: + concurrency: "adaptive" + rate_limit_num: 1000 + batch: + max_bytes: 5242880 + + blackhole: + type: "blackhole" + inputs: ["remap"] ``` -And you ran the new `vector graph --config vector.toml` you would see: +And you ran the new `vector graph --config vector.yaml` you would see: ```dot digraph { @@ -102,7 +106,7 @@ digraph { To render this, if you have [Graphviz] installed, you could do: ```shell -vector graph --config vector.toml | dot -Tpng > graph.png +vector graph --config vector.yaml | dot -Tpng > graph.png ``` To get an image that looks like: diff --git a/website/content/en/highlights/2021-07-16-remap-multiple.md b/website/content/en/highlights/2021-07-16-remap-multiple.md index 5d23a59fabce9..80a0d5fa4d4a6 100644 --- a/website/content/en/highlights/2021-07-16-remap-multiple.md +++ b/website/content/en/highlights/2021-07-16-remap-multiple.md @@ -20,13 +20,13 @@ will create one log event for each event in the array. For example: -```toml -[transforms.remap] -type = "remap" -inputs = [] -source = """ -. = [{"message": "hello"}, {"message": "world"}] -""" +```yaml +transforms: + remap: + type: "remap" + inputs: [] + source: | + . = [{"message": "hello"}, {"message": "world"}] ``` Would generate two output events: @@ -45,15 +45,15 @@ Additionally, to make it easier to convert an incoming log event into an array, function to VRL that transforms an incoming event where one of the fields is an array into an array of events, each with one of the elements from the array field. This is easiest to see with an example: -```toml -[transforms.remap] -type = "remap" -inputs = [] -source = """ -. = {"host": "localhost", "events": [{"message": "hello"}, {"message": "world"}]} # to represent the incoming event +```yaml +transforms: + remap: + type: "remap" + inputs: [] + source: | + . = {"host": "localhost", "events": [{"message": "hello"}, {"message": "world"}]} # to represent the incoming event -. = unnest(.events) -""" + . = unnest(.events) ``` Would output the following log events: @@ -69,24 +69,23 @@ and another `remap` transform to receive each new event. An example of this: -```toml -[transforms.explode] -type = "remap" -inputs = [] -source = """ -. = {"host": "localhost", "events": [{"message": "hello"}, {"message": "world"}]} # to represent the incoming event - -. = unnest(.events) -""" - -[transforms.map] -type = "remap" -inputs = ["explode"] -source = """ -# example of pulling up the nested field to merge it into the top-level -. |= .events -del(.events) -""" +```yaml +transforms: + explode: + type: "remap" + inputs: [] + source: | + . = {"host": "localhost", "events": [{"message": "hello"}, {"message": "world"}]} # to represent the incoming event + + . = unnest(.events) + + map: + type: "remap" + inputs: ["explode"] + source: | + # example of pulling up the nested field to merge it into the top-level + . |= .events + del(.events) ``` Would output the following log events: diff --git a/website/content/en/highlights/2021-08-20-rate-limits.md b/website/content/en/highlights/2021-08-20-rate-limits.md index 5be8d63e4fa41..5668116faa5da 100644 --- a/website/content/en/highlights/2021-08-20-rate-limits.md +++ b/website/content/en/highlights/2021-08-20-rate-limits.md @@ -23,9 +23,10 @@ have no rate limiting and a maximum number of concurrent requests of 1024. To configure a request rate limit or maximum concurrency on a HTTP-based sink, you can set the `request` parameters like: -```toml -request.concurrency = 5 # limit to 5 in-flight requests -request.rate_limit_num = 10 # limit to 10 requests / second +```yaml +request: + concurrency: 5 # limit to 5 in-flight requests + rate_limit_num: 10 # limit to 10 requests / second ``` If you haven't already, we recommend trying out our [adaptive concurrency diff --git a/website/content/en/highlights/2021-08-25-0-16-upgrade-guide.md b/website/content/en/highlights/2021-08-25-0-16-upgrade-guide.md index 96c30410f344c..5a33295331e6c 100644 --- a/website/content/en/highlights/2021-08-25-0-16-upgrade-guide.md +++ b/website/content/en/highlights/2021-08-25-0-16-upgrade-guide.md @@ -30,11 +30,12 @@ components. For example, with the component config: -```toml -[transforms.parse_nginx] -type = "remap" -inputs = [] -source = "" +```yaml +transforms: + parse_nginx: + type: "remap" + inputs: [] + source: "" ``` The `parse_nginx` part of the config is now only referred to as `ID` in the documentation. @@ -57,19 +58,22 @@ limitations and is only useful for backward compatibility with older clients. We no longer allow you to set the encoding of the payloads in the Datadog logs sink. For instance, if your configuration looks like so: -```toml -[sinks.dd_logs_egress] -type = "datadog_logs" -inputs = ["datadog_agent"] -encoding.codec = "json" +```yaml +sinks: + dd_logs_egress: + type: "datadog_logs" + inputs: ["datadog_agent"] + encoding: + codec: "json" ``` You should remove `encoding.codec` entirely, leaving you with: -```toml -[sinks.dd_logs_egress] -type = "datadog_logs" -inputs = ["datadog_agent"] +```yaml +sinks: + dd_logs_egress: + type: "datadog_logs" + inputs: ["datadog_agent"] ``` Encoding fields other than `codec` are still valid. diff --git a/website/content/en/highlights/2021-10-06-arc-default.md b/website/content/en/highlights/2021-10-06-arc-default.md index 54cfb15f834be..49344701a0eb3 100644 --- a/website/content/en/highlights/2021-10-06-arc-default.md +++ b/website/content/en/highlights/2021-10-06-arc-default.md @@ -20,8 +20,9 @@ increased pressure downstream, to avoid overwhelming the sink destination. This feature was previously able to be opted into with the following configuration: -```toml -request.concurrency = "adaptive" +```yaml +request: + concurrency: "adaptive" ``` This is the new default for HTTP-based sinks. As mentioned in the [announcement @@ -35,8 +36,9 @@ If you would like to instead use a fixed concurrency as was previously the default, you can set a static value like: -```toml -request.concurrency = 5 +```yaml +request: + concurrency: 5 ``` This will tell Vector to limit to 5 concurrent requests. diff --git a/website/content/en/highlights/2021-10-06-source-codecs.md b/website/content/en/highlights/2021-10-06-source-codecs.md index 09967fc40923f..7b4b307c5b923 100644 --- a/website/content/en/highlights/2021-10-06-source-codecs.md +++ b/website/content/en/highlights/2021-10-06-source-codecs.md @@ -17,12 +17,14 @@ added new `decoding` options to [most sources][9404]. For example, if you have a `kafka` source that has JSON-encoded messages, now you can simply add `decoding.codec = "json"` to your source configuration like: -```toml -[sources.kafka] -type = "kafka" -bootstrap_servers = "localhost:9200" -topics = ["my_topic"] -decoding.codec = "json" +```yaml +sources: + kafka: + type: "kafka" + bootstrap_servers: "localhost:9200" + topics: ["my_topic"] + decoding: + codec: "json" ``` This will decode your messages from JSON, thus saving you from an additional @@ -35,12 +37,15 @@ separating messages). For example, if you have an `http` source where the messages are delimited by commas instead of newlines, you can configure this like: -```toml -[sources.http] -type = "http" -address = "0.0.0.0:8080" -framing.method = "character_delimited" -framing.character_delimited.delimiter = "," +```yaml +sources: + http: + type: "http" + address: "0.0.0.0:8080" + framing: + method: "character_delimited" + character_delimited: + delimiter: "," ``` To have Vector parse each comma-delimited element as a new message. This can be diff --git a/website/content/en/highlights/2021-11-12-event-throttle-transform.md b/website/content/en/highlights/2021-11-12-event-throttle-transform.md index 4f029774a0e84..c6e51d7307b9f 100644 --- a/website/content/en/highlights/2021-11-12-event-throttle-transform.md +++ b/website/content/en/highlights/2021-11-12-event-throttle-transform.md @@ -48,12 +48,13 @@ Given these incoming log events: ...and this configuration... -```toml -[transforms.my_transform_id] -type = "throttle" -inputs = [ "my-source-or-transform-id" ] -threshold = 1 -window_secs = 60 +```yaml +transforms: + my_transform_id: + type: "throttle" + inputs: ["my-source-or-transform-id"] + threshold: 1 + window_secs: 60 ``` ...only one event will be allowed through: diff --git a/website/content/en/highlights/2021-12-15-splunk-hec-improvements.md b/website/content/en/highlights/2021-12-15-splunk-hec-improvements.md index a4e4d0a87b766..54b5f2b85e1e3 100644 --- a/website/content/en/highlights/2021-12-15-splunk-hec-improvements.md +++ b/website/content/en/highlights/2021-12-15-splunk-hec-improvements.md @@ -31,8 +31,8 @@ would not work with Splunk senders that required them. Now, you can configure the `splunk_hec` source to use the indexer acknowledgements protocol by configuring: -```toml -acknowledgements = true +```yaml +acknowledgements: true ``` When enabled, responses to incoming requests will include an ID that can be used to query for acknowledgement status at the newly exposed `/services/collector/ack` endpoint ([learn more here][indexer how it works]). The acknowledgement status is wired into Vector's @@ -51,8 +51,9 @@ acknowledgements][indexer] part of the HEC protocol. This has defaulted to on to provide higher guarantees, but can be disabled to restore the previous behavior by configuring: -```toml -acknowledgements.indexer_acknowledgements_enabled = false +```yaml +acknowledgements: + indexer_acknowledgements_enabled: false ``` ## Passthrough Token Routing diff --git a/website/content/en/highlights/2021-12-28-0-19-0-upgrade-guide.md b/website/content/en/highlights/2021-12-28-0-19-0-upgrade-guide.md index e23e7b54c9f5e..7655ac4027eb9 100644 --- a/website/content/en/highlights/2021-12-28-0-19-0-upgrade-guide.md +++ b/website/content/en/highlights/2021-12-28-0-19-0-upgrade-guide.md @@ -165,11 +165,11 @@ source, you will now see a `splunk_channel` field in events output from the included channel information from requests in their events. If you want to remove this field, you can simply drop it with a `remap` transform like follows: -```toml -[transforms.foo] -type = "remap" -inputs = ["bar"] # where bar is the splunk_hec source -source = ''' - del(.splunk_channel) -''' +```yaml +transforms: + foo: + type: "remap" + inputs: ["bar"] # where bar is the splunk_hec source + source: | + del(.splunk_channel) ``` diff --git a/website/content/en/highlights/2022-01-12-vector-unit-test-improvements.md b/website/content/en/highlights/2022-01-12-vector-unit-test-improvements.md index 9444f10cbb47a..be461a4bc60e2 100644 --- a/website/content/en/highlights/2022-01-12-vector-unit-test-improvements.md +++ b/website/content/en/highlights/2022-01-12-vector-unit-test-improvements.md @@ -13,30 +13,28 @@ full support for testing task-style transforms (previously there may have been issues when using multiple inputs with task transforms). For example, you can now test `remap` transform's `dropped` output like so, -```toml -[transforms.foo] - type = "remap" - inputs = [] - drop_on_abort = true - reroute_dropped = true - source = "abort" +```yaml +transforms: + foo: + type: "remap" + inputs: [] + drop_on_abort: true + reroute_dropped: true + source: "abort" -[[tests]] - name = "remap_dropped_output" - no_outputs_from = [ "foo" ] - - [[tests.inputs]] - insert_at = "foo" - type = "log" - [tests.inputs.log_fields] - message = "I will be dropped" - - [[tests.outputs]] - extract_from = "foo.dropped" - - [[tests.outputs.conditions]] - type = "vrl" - source = 'assert_eq!(.message, "I will be dropped", "incorrect message")' +tests: + - name: "remap_dropped_output" + no_outputs_from: ["foo"] + inputs: + - insert_at: "foo" + type: "log" + log_fields: + message: "I will be dropped" + outputs: + - extract_from: "foo.dropped" + conditions: + - type: "vrl" + source: 'assert_eq!(.message, "I will be dropped", "incorrect message")' ``` Under-the-hood, we've reworked the unit testing implementation to more closely diff --git a/website/content/en/highlights/2022-02-08-disk-buffer-v2-beta.md b/website/content/en/highlights/2022-02-08-disk-buffer-v2-beta.md index 560c5389473cc..54c118ea395ff 100644 --- a/website/content/en/highlights/2022-02-08-disk-buffer-v2-beta.md +++ b/website/content/en/highlights/2022-02-08-disk-buffer-v2-beta.md @@ -33,10 +33,12 @@ persistence, regardless of issues with Vector and the host system. You may already be familiar with disk buffers if you have a sink configuration that looks like this: -```toml -[sinks.http] -# ... -buffer.type = "disk" +```yaml +sinks: + http: + # ... + buffer: + type: "disk" ``` ## Current disk buffer implementation @@ -74,16 +76,20 @@ for those who don't utilize a stateless deployment process as mentioned above. With all of that said, changing Vector to use the new disk buffer implementation is as simple as changing a single line: -```toml +```yaml # From this: -[sinks.http] -# ... -buffer.type = "disk" +sinks: + http: + # ... + buffer: + type: "disk" # To this: -[sinks.http] -# ... -buffer.type = "disk_v2" +sinks: + http: + # ... + buffer: + type: "disk_v2" ``` ## Let us know what you think! diff --git a/website/content/en/highlights/2022-03-15-vrl-vm-beta.md b/website/content/en/highlights/2022-03-15-vrl-vm-beta.md index f4bd959f259c6..399782a428435 100644 --- a/website/content/en/highlights/2022-03-15-vrl-vm-beta.md +++ b/website/content/en/highlights/2022-03-15-vrl-vm-beta.md @@ -20,14 +20,14 @@ the VRL program that is being run. You can give the VM a try by adding the `runtime` configuration option to your Remap transform: -```toml -[transforms.remap] -type = "remap" -inputs = [ "..." ] -runtime = "vm" -source = ''' -... -''' +```yaml +transforms: + remap: + type: "remap" + inputs: ["..."] + runtime: "vm" + source: | + ... ``` ## Let us know what you think! diff --git a/website/content/en/highlights/2022-03-22-0-21-0-upgrade-guide.md b/website/content/en/highlights/2022-03-22-0-21-0-upgrade-guide.md index 761e6d861455b..96714e842b4ce 100644 --- a/website/content/en/highlights/2022-03-22-0-21-0-upgrade-guide.md +++ b/website/content/en/highlights/2022-03-22-0-21-0-upgrade-guide.md @@ -64,24 +64,28 @@ Here are some examples that require migrating | foo\\"bar | "foo\\"bar" | Double quotes and backlashes must be escaped _inside_ quotes | -### TOML transform example +### YAML transform example Old syntax -```toml -[transforms.dedupe] -type = "dedupe" -inputs = ["input"] -fields.match = ["message.user-identifier"] +```yaml +transforms: + dedupe: + type: "dedupe" + inputs: ["input"] + fields: + match: ["message.user-identifier"] ``` New syntax (the dash requires the field name to be quoted) -```toml -[transforms.dedupe] -type = "dedupe" -inputs = ["input"] -fields.match = ["message.\"user-identifier\""] +```yaml +transforms: + dedupe: + type: "dedupe" + inputs: ["input"] + fields: + match: ['message."user-identifier"'] ``` For more information on the new syntax, you can review the [VRL path expressions documentation](https://vector.dev/docs/reference/vrl/expressions/#path) diff --git a/website/content/en/highlights/2022-03-31-native-event-codecs.md b/website/content/en/highlights/2022-03-31-native-event-codecs.md index 0a10938afffea..1f1b301caca82 100644 --- a/website/content/en/highlights/2022-03-31-native-event-codecs.md +++ b/website/content/en/highlights/2022-03-31-native-event-codecs.md @@ -23,12 +23,14 @@ transform. For example, an `exec` source can now be configured to receive events via the `native_json` codec: -```toml -[sources.in] -type = "exec" -mode = "scheduled" -command = ["./scrape.sh"] -decoding.codec = "native_json" +```yaml +sources: + in: + type: "exec" + mode: "scheduled" + command: ["./scrape.sh"] + decoding: + codec: "native_json" ``` If `scrape.sh` contained: @@ -82,24 +84,28 @@ deserialize directly to the same native representation within Vector. Example source configuration: -```toml -[sources.in] -type = "kafka" -bootstrap_servers = "localhost:9092" -topics = ["vector"] -decoding.codec = "native" +```yaml +sources: + in: + type: "kafka" + bootstrap_servers: "localhost:9092" + topics: ["vector"] + decoding: + codec: "native" ``` This would allow an instance of Vector to receive events from another Vector instance that has a `kafka` sink configured like: -```toml -[sinks.out] -type = "kafka" -inputs = ["..."] -bootstrap_servers = "localhost:9092" -topic = "vector" -encoding.codec = "native" +```yaml +sinks: + out: + type: "kafka" + inputs: ["..."] + bootstrap_servers: "localhost:9092" + topic: "vector" + encoding: + codec: "native" ``` Note that the new `native` encoding option is not yet documented on sinks as we diff --git a/website/content/en/highlights/2022-05-03-0-22-0-upgrade-guide.md b/website/content/en/highlights/2022-05-03-0-22-0-upgrade-guide.md index b7be9fad5dd50..f9a18b8e47cba 100644 --- a/website/content/en/highlights/2022-05-03-0-22-0-upgrade-guide.md +++ b/website/content/en/highlights/2022-05-03-0-22-0-upgrade-guide.md @@ -32,40 +32,39 @@ The `gcp_stackdriver_metrics` sink now matches the `gcp_stackdriver_logs` configuration, and doesn't require an additional `labels` section to add labels to submitted metrics. -##### TOML transform example +##### YAML transform example Old configuration -```toml -[sinks.my_sink_id] -type = "gcp_stackdriver_metrics" -inputs = [ "my-source-or-transform-id" ] -credentials_path = "/path/to/credentials.json" -project_id = "vector-123456" - - [sinks.my_sink_id.resource] - type = "global" - - [sinks.my_sink_id.resource.labels] - projectId = "vector-123456" - instanceId = "Twilight" - zone = "us-central1-a" +```yaml +sinks: + my_sink_id: + type: "gcp_stackdriver_metrics" + inputs: ["my-source-or-transform-id"] + credentials_path: "/path/to/credentials.json" + project_id: "vector-123456" + resource: + type: "global" + labels: + projectId: "vector-123456" + instanceId: "Twilight" + zone: "us-central1-a" ``` New configuration -```toml -[sinks.my_sink_id] -type = "gcp_stackdriver_metrics" -inputs = [ "my-source-or-transform-id" ] -credentials_path = "/path/to/credentials.json" -project_id = "vector-123456" - - [sinks.my_sink_id.resource] - type = "global" - projectId = "vector-123456" - instanceId = "Twilight" - zone = "us-central1-a" +```yaml +sinks: + my_sink_id: + type: "gcp_stackdriver_metrics" + inputs: ["my-source-or-transform-id"] + credentials_path: "/path/to/credentials.json" + project_id: "vector-123456" + resource: + type: "global" + projectId: "vector-123456" + instanceId: "Twilight" + zone: "us-central1-a" ``` For more information on the new syntax, you can review the [GCP Stackdriver Metrics sink documentation](https://vector.dev/docs/reference/configuration/sinks/gcp_stackdriver_metrics/) @@ -179,413 +178,442 @@ transform. Before: -```toml -[transforms.add_fields] -type = "add_fields" -inputs = ["some_input"] -fields.parent.child2 = "value2" +```yaml +transforms: + add_fields: + type: "add_fields" + inputs: ["some_input"] + fields: + parent: + child2: "value2" ``` After: -```toml -[transforms.add_fields] -type = "remap" -inputs = ["some_input"] -source = ''' -.parent.child2 = "value2" -''' +```yaml +transforms: + add_fields: + type: "remap" + inputs: ["some_input"] + source: | + .parent.child2 = "value2" ``` ##### `add_tags` Before: -```toml -[transforms.add_tags] -type = "add_tags" -inputs = ["some_input"] -tags.some_tag = "some_value" +```yaml +transforms: + add_tags: + type: "add_tags" + inputs: ["some_input"] + tags: + some_tag: "some_value" ``` After: -```toml -[transforms.add_tags] -type = "remap" -inputs = ["some_input"] -source = ''' -.tags.some_tag = "some_value" -''' +```yaml +transforms: + add_tags: + type: "remap" + inputs: ["some_input"] + source: | + .tags.some_tag = "some_value" ``` ##### `ansi_stripper` Before: -```toml -[transforms.ansi_stripper] -type = "ansi_stripper" -inputs = ["some_input"] +```yaml +transforms: + ansi_stripper: + type: "ansi_stripper" + inputs: ["some_input"] ``` After: -```toml -[transforms.ansi_stripper] -type = "remap" -inputs = ["some_input"] -drop_on_error = false -source = ''' -.message = strip_ansi_escape_codes(string!(.message)) -''' +```yaml +transforms: + ansi_stripper: + type: "remap" + inputs: ["some_input"] + drop_on_error: false + source: | + .message = strip_ansi_escape_codes(string!(.message)) ``` ##### `aws_cloudwatch_logs_subscription_parser` Before: -```toml -[transforms.aws_cloudwatch_logs_subscription_parser] -type = "aws_cloudwatch_logs_subscription_parser" -inputs = ["some_input"] +```yaml +transforms: + aws_cloudwatch_logs_subscription_parser: + type: "aws_cloudwatch_logs_subscription_parser" + inputs: ["some_input"] ``` After: -```toml -[transforms.aws_cloudwatch_logs_subscription_parser] -type = "remap" -inputs = ["some_input"] -drop_on_error = false -source = ''' -. |= parse_aws_cloudwatch_log_subscription_message!(.message) -''' +```yaml +transforms: + aws_cloudwatch_logs_subscription_parser: + type: "remap" + inputs: ["some_input"] + drop_on_error: false + source: | + . |= parse_aws_cloudwatch_log_subscription_message!(.message) ``` ##### `coercer` Before: -```toml -[transforms.coercer] -type = "coercer" -inputs = ["some_input"] -types.some_bool = "bool" -types.some_float = "float" -types.some_int = "int" -types.some_string = "string" -types.some_timestamp = "timestamp" +```yaml +transforms: + coercer: + type: "coercer" + inputs: ["some_input"] + types: + some_bool: "bool" + some_float: "float" + some_int: "int" + some_string: "string" + some_timestamp: "timestamp" ``` After: -```toml -[transforms.coercer] -type = "remap" -inputs = ["some_input"] -source = ''' -.some_bool = to_bool!(.some_bool) -.some_float = to_float!(.some_float) -.some_int = to_int!(.some_int) -.some_string = to_string!(.some_string) -.some_timestamp = to_timestamp!(.some_timestamp) -''' +```yaml +transforms: + coercer: + type: "remap" + inputs: ["some_input"] + source: | + .some_bool = to_bool!(.some_bool) + .some_float = to_float!(.some_float) + .some_int = to_int!(.some_int) + .some_string = to_string!(.some_string) + .some_timestamp = to_timestamp!(.some_timestamp) ``` ##### `concat` Before: -```toml -[transforms.concat] -type = "concat" -inputs = ["some_input"] -items = ["month", "day", "year"] -target = "date" -joiner = "/" +```yaml +transforms: + concat: + type: "concat" + inputs: ["some_input"] + items: ["month", "day", "year"] + target: "date" + joiner: "/" ``` After: -```toml -[transforms.concat] -type = "remap" -inputs = ["some_input"] -drop_on_error = false -source = ''' -.date = join!([.month, .day, .year], "/") -''' +```yaml +transforms: + concat: + type: "remap" + inputs: ["some_input"] + drop_on_error: false + source: | + .date = join!([.month, .day, .year], "/") ``` ##### `grok_parser` Before: -```toml -[transforms.grok_parser] -type = "grok_parser" -inputs = ["some_input"] -pattern = "%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level} %{GREEDYDATA:message}" -types.timestamp = "timestamp|%+" -types.level = "string" -types.message = "string" +```yaml +transforms: + grok_parser: + type: "grok_parser" + inputs: ["some_input"] + pattern: "%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level} %{GREEDYDATA:message}" + types: + timestamp: "timestamp|%+" + level: "string" + message: "string" ``` After: -```toml -[transforms.grok_parser] -type = "remap" -inputs = ["some_input"] -drop_on_error = false -source = ''' -. |= parse_grok!(.message, "%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level} %{GREEDYDATA:message}") -.timestamp = parse_timestamp!(.timestamp , format: "%+") -''' +```yaml +transforms: + grok_parser: + type: "remap" + inputs: ["some_input"] + drop_on_error: false + source: | + . |= parse_grok!(.message, "%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level} %{GREEDYDATA:message}") + .timestamp = parse_timestamp!(.timestamp , format: "%+") ``` ##### `json_parser` Before: -```toml -[transforms.json_parser] -type = "json_parser" -inputs = ["some_input"] +```yaml +transforms: + json_parser: + type: "json_parser" + inputs: ["some_input"] ``` After: -```toml -[transforms.json_parser] -type = "remap" -inputs = ["some_input"] -drop_on_error = false -source = ''' -. |= object!(parse_json(.message)) -''' +```yaml +transforms: + json_parser: + type: "remap" + inputs: ["some_input"] + drop_on_error: false + source: | + . |= object!(parse_json(.message)) ``` ##### `key_value_parser` Before: -```toml -[transforms.key_value_parser] -type = "key_value_parser" -inputs = ["some_input"] +```yaml +transforms: + key_value_parser: + type: "key_value_parser" + inputs: ["some_input"] ``` After: -```toml -[transforms.key_value_parser] -type = "remap" -inputs = ["some_input"] -drop_on_error = false -source = ''' -. |= parse_key_value!(.message) -''' +```yaml +transforms: + key_value_parser: + type: "remap" + inputs: ["some_input"] + drop_on_error: false + source: | + . |= parse_key_value!(.message) ``` ##### `logfmt_parser` Before: -```toml -[transforms.logfmt_parser] -type = "logfmt_parser" -inputs = ["some_input"] +```yaml +transforms: + logfmt_parser: + type: "logfmt_parser" + inputs: ["some_input"] ``` After: -```toml -[transforms.logfmt_parser] -type = "remap" -inputs = ["some_input"] -drop_on_error = false -source = ''' -. |= parse_logfmt!(.message) -''' +```yaml +transforms: + logfmt_parser: + type: "remap" + inputs: ["some_input"] + drop_on_error: false + source: | + . |= parse_logfmt!(.message) ``` ##### `merge` Before: -```toml -[transforms.merge] -type = "merge" -inputs = ["some_input"] +```yaml +transforms: + merge: + type: "merge" + inputs: ["some_input"] ``` After: -```toml -[transforms.merge] -type = "reduce" -inputs = ["some_input"] -starts_when = "._partial == true" -merge_strategies.message = "concat" +```yaml +transforms: + merge: + type: "reduce" + inputs: ["some_input"] + starts_when: "._partial == true" + merge_strategies: + message: "concat" ``` ##### `regex_parser` Before: -```toml -[transforms.regex_parser] -type = "regex_parser" -inputs = ["some_input"] -patterns = ['^(?P[\w\.]+) - (?P[\w]+) (?P[\d]+) \[(?P.*)\] "(?P[\w]+) (?P.*)" (?P[\d]+) (?P[\d]+)$'] -types.bytes_in = "int" -types.timestamp = "timestamp|%d/%m/%Y:%H:%M:%S %z" -types.status = "int" -types.bytes_out = "int" +```yaml +transforms: + regex_parser: + type: "regex_parser" + inputs: ["some_input"] + patterns: + - '^(?P[\w\.]+) - (?P[\w]+) (?P[\d]+) \[(?P.*)\] "(?P[\w]+) (?P.*)" (?P[\d]+) (?P[\d]+)$' + types: + bytes_in: "int" + timestamp: "timestamp|%d/%m/%Y:%H:%M:%S %z" + status: "int" + bytes_out: "int" ``` After: -```toml -[transforms.regex_parser] -type = "remap" -inputs = ["some_input"] -drop_on_error = false -source = ''' -. |= parse_regex!(.message, [#"^(?P[\w\.]+) - (?P[\w]+) (?P[\d]+) \[(?P.*)\] "(?P[\w]+) (?P.*)" (?P[\d]+) (?P[\d]+)$"#] -.bytes_in = to_int!(.bytes_in) -.some_timestamp = parse_timestamp!(.some_timestamp, "%d/%m/%Y:%H:%M:%S %z") -.status = to_int!(.status) -.bytes_out = to_int!(.bytes_out) -''' +```yaml +transforms: + regex_parser: + type: "remap" + inputs: ["some_input"] + drop_on_error: false + source: | + . |= parse_regex!(.message, [#"^(?P[\w\.]+) - (?P[\w]+) (?P[\d]+) \[(?P.*)\] "(?P[\w]+) (?P.*)" (?P[\d]+) (?P[\d]+)$"#] + .bytes_in = to_int!(.bytes_in) + .some_timestamp = parse_timestamp!(.some_timestamp, "%d/%m/%Y:%H:%M:%S %z") + .status = to_int!(.status) + .bytes_out = to_int!(.bytes_out) ``` ##### `remove_fields` Before: -```toml -[transforms.remove_fields] -type = "remove_fields" -inputs = ["some_input"] -fields = ["parent.child"] +```yaml +transforms: + remove_fields: + type: "remove_fields" + inputs: ["some_input"] + fields: ["parent.child"] ``` After: -```toml -[transforms.remove_fields] -type = "remap" -inputs = ["some_input"] -source = ''' -del(.parent.child) -''' +```yaml +transforms: + remove_fields: + type: "remap" + inputs: ["some_input"] + source: | + del(.parent.child) ``` ##### `remove_tags` Before: -```toml -[transforms.remove_tags] -type = "remove_tags" -inputs = ["some_input"] -tags = ["some_tag"] +```yaml +transforms: + remove_tags: + type: "remove_tags" + inputs: ["some_input"] + tags: ["some_tag"] ``` After: -```toml -[transforms.remove_tags] -type = "remap" -inputs = ["some_input"] -source = ''' -del(.tags.some_tag) -''' +```yaml +transforms: + remove_tags: + type: "remap" + inputs: ["some_input"] + source: | + del(.tags.some_tag) ``` ##### `rename_fields` Before: -```toml -[transforms.rename_fields] -type = "rename_fields" -inputs = ["some_input"] -fields.new_name = ["old_name"] +```yaml +transforms: + rename_fields: + type: "rename_fields" + inputs: ["some_input"] + fields: + new_name: ["old_name"] ``` After: -```toml -[transforms.rename_fields] -type = "remap" -inputs = ["some_input"] -source = ''' -.new_name = del(.old_name) -''' +```yaml +transforms: + rename_fields: + type: "remap" + inputs: ["some_input"] + source: | + .new_name = del(.old_name) ``` ##### `split` Before: -```toml -[transforms.split] -type = "split" -inputs = ["some_input"] -field_names = ["remote_addr", "user_id", "timestamp", "message", "status", "bytes"] -types.status = "int" -types.bytes = "int" +```yaml +transforms: + split: + type: "split" + inputs: ["some_input"] + field_names: ["remote_addr", "user_id", "timestamp", "message", "status", "bytes"] + types: + status: "int" + bytes: "int" ``` After: -```toml -[transforms.split] -type = "remap" -inputs = ["some_input"] -drop_on_error = false -source = ''' -values = split(.message) -.remote_addr = values[0] -.user_id = values[1] -.timestamp = values[2] -.message = values[3] -.status = to_int!(values[4]) -.bytes = to_int!(values[5]) -''' +```yaml +transforms: + split: + type: "remap" + inputs: ["some_input"] + drop_on_error: false + source: | + values = split(.message) + .remote_addr = values[0] + .user_id = values[1] + .timestamp = values[2] + .message = values[3] + .status = to_int!(values[4]) + .bytes = to_int!(values[5]) ``` ##### `tokenizer` Before: -```toml -[transforms.tokenizer] -type = "tokenizer" -inputs = ["some_input"] -field_names = ["remote_addr", "ident", "user_id", "timestamp", "message", "status", "bytes"] -.types.status = "int" -.types.bytes = "int" +```yaml +transforms: + tokenizer: + type: "tokenizer" + inputs: ["some_input"] + field_names: ["remote_addr", "ident", "user_id", "timestamp", "message", "status", "bytes"] + types: + status: "int" + bytes: "int" ``` After: -```toml -[transforms.tokenizer] -type = "remap" -inputs = ["some_input"] -drop_on_error = false -source = ''' -values = parse_tokens!(.message) -.remote_addr = values[0] -.user_id = values[1] -.timestamp = values[2] -.message = values[3] -.status = to_int!(values[4]) -.bytes = to_int!(values[5]) -''' +```yaml +transforms: + tokenizer: + type: "remap" + inputs: ["some_input"] + drop_on_error: false + source: | + values = parse_tokens!(.message) + .remote_addr = values[0] + .user_id = values[1] + .timestamp = values[2] + .message = values[3] + .status = to_int!(values[4]) + .bytes = to_int!(values[5]) ``` diff --git a/website/content/en/highlights/2022-07-07-0-23-0-upgrade-guide.md b/website/content/en/highlights/2022-07-07-0-23-0-upgrade-guide.md index 7fb2d3003977c..b71453f2e9cd1 100644 --- a/website/content/en/highlights/2022-07-07-0-23-0-upgrade-guide.md +++ b/website/content/en/highlights/2022-07-07-0-23-0-upgrade-guide.md @@ -135,15 +135,16 @@ will give a compile-time error if you try to mutate the event in a condition. Example filter transform config -```toml -[transforms.filter] -type = "filter" -inputs = [ "input" ] -condition.type = "vrl" -condition.source = """ -.foo = "bar" -true -""" +```yaml +transforms: + filter: + type: "filter" + inputs: ["input"] + condition: + type: "vrl" + source: | + .foo = "bar" + true ``` New error @@ -253,8 +254,9 @@ need to remove it to avoid Vector from erroring on startup. The `gcp_pubsub` sink now supports a variety of codecs. To encode your logs as JSON before publishing them to Cloud Pub/Sub, add the following encoding option -```toml -encoding.codec = "json" +```yaml +encoding: + codec: "json" ``` to the config of your `gcp_pubsub` sink. @@ -264,8 +266,9 @@ to the config of your `gcp_pubsub` sink. The `humio_metrics` sink configuration no longer expects an `encoding` option. If you previously used the encoding option -```toml -encoding.codec = "json" +```yaml +encoding: + codec: "json" ``` you need to remove the line from your `humio_metrics` config. Metrics are now @@ -277,8 +280,9 @@ We streamlined the encoding configuration for our sinks, enabling all applicable from a variety of codecs: `json`, `text`, `raw`, `logfmt`, `avro`, `native` or `native_json`, e.g. by setting -```toml -encoding.codec = "json" +```yaml +encoding: + codec: "json" ``` in your sink configuration. @@ -287,8 +291,9 @@ Additionally, some sinks now support configuring how encoded events should be se stream or batch: `bytes`, `character_delimited`, `length_delimited` or `newline_delimited`, e.g. by setting -```toml -framing.method = "newline_delimited" +```yaml +framing: + method: "newline_delimited" ``` in your sink configuration. @@ -353,10 +358,9 @@ This was introduced in order to better handle metrics sent from the `datadog_age The result is that configurations with VRL expressions that expect the namespace to be in the name will need to be adapted to either remove the namespace from the name, or join the namespace and the name, for example: -```toml -full_metric_name = """ -join([.namespace, .name], ".") -""" +```yaml +full_metric_name: | + join([.namespace, .name], ".") ``` ### Deprecations @@ -365,14 +369,15 @@ join([.namespace, .name], ".") We are deprecating setting encoding options by a shorthand string. E.g. when your sink encoding used -```toml -encoding = "json" +```yaml +encoding: "json" ``` it should now be replaced by explicitly setting the `codec` -```toml -encoding.codec = "json" +```yaml +encoding: + codec: "json" ``` #### Sink encoding value `ndjson` is now `json` encoding + `newline_delimited` framing {#sink-encoding-ndjson-json} @@ -382,20 +387,24 @@ can be set explicitly via a dedicated `framing` option. This affects all sink configurations that previously used -```toml -encoding.codec = "ndjson" +```yaml +encoding: + codec: "ndjson" ``` The `http`, `aws_s3`, `gcp_cloud_storage` and `azure_blob` sinks should be configured to use a combination of `json` encoding and `newline_delimited` framing instead -```toml -framing.method = "newline_delimited" -encoding.codec = "json" +```yaml +framing: + method: "newline_delimited" +encoding: + codec: "json" ``` For all other sinks, simply set the codec to `json` to maintain the current behavior -```toml -encoding.codec = "json" +```yaml +encoding: + codec: "json" ``` diff --git a/website/content/en/highlights/2022-07-07-secrets-management.md b/website/content/en/highlights/2022-07-07-secrets-management.md index 5f69e3783df04..6fc766134e671 100644 --- a/website/content/en/highlights/2022-07-07-secrets-management.md +++ b/website/content/en/highlights/2022-07-07-secrets-management.md @@ -19,21 +19,24 @@ these values can be read if a user on a host has access to read from the `/proc` A secret backend can be configured like this: -```toml -[secret.backend_1] -type = "exec" # exec is the only supported backend as of writing -command = ["/path/to/cmd1"] +```yaml +secret: + backend_1: + type: "exec" # exec is the only supported backend as of writing + command: ["/path/to/cmd1"] ``` You can then specify where secrets should be read via `SECRET[.]` in the config like: -```toml -[sources.my_source_id] -type = "aws_sqs" -region = "us-east-1" -queue_url = "https://sqs.us-east-2.amazonaws.com/123456789012/MyQueue" -auth.access_key_id = "SECRET[backend_1.aws_access_key_id]" -auth.secret_access_key = "SECRET[backend_1.aws_secret_access_key]" +```yaml +sources: + my_source_id: + type: "aws_sqs" + region: "us-east-1" + queue_url: "https://sqs.us-east-2.amazonaws.com/123456789012/MyQueue" + auth: + access_key_id: "SECRET[backend_1.aws_access_key_id]" + secret_access_key: "SECRET[backend_1.aws_secret_access_key]" ``` Here `auth.access_key_id` and `auth.secret_access_key` will use secrets provided by the `backend_1` secret backend. diff --git a/website/content/en/highlights/2022-07-07-sink-codecs.md b/website/content/en/highlights/2022-07-07-sink-codecs.md index 7e00290826a50..d6c0f75c47020 100644 --- a/website/content/en/highlights/2022-07-07-sink-codecs.md +++ b/website/content/en/highlights/2022-07-07-sink-codecs.md @@ -20,13 +20,16 @@ For example, if you have a `socket` sink that you want to send [length-delimited][length_delimited] JSON-encoded, messages, you can now do so with configuration like: -```toml -[sinks.socket] -type = "socket" -address = "92.12.333.224:5000" -mode = "tcp" -framing.method = "length_delimited" -encoding.codec = "json" +```yaml +sinks: + socket: + type: "socket" + address: "92.12.333.224:5000" + mode: "tcp" + framing: + method: "length_delimited" + encoding: + codec: "json" ``` This will encode messages flowing into this sink as JSON and frame them using diff --git a/website/content/en/highlights/2022-10-04-0-25-0-upgrade-guide.md b/website/content/en/highlights/2022-10-04-0-25-0-upgrade-guide.md index 8464524b01c45..61050ecefca6b 100644 --- a/website/content/en/highlights/2022-10-04-0-25-0-upgrade-guide.md +++ b/website/content/en/highlights/2022-10-04-0-25-0-upgrade-guide.md @@ -67,16 +67,17 @@ Additionally, the [account ID][nr_account_id] must also now be specified. All put together, here's an example of converting from a `new_relic_logs` sink configuration over to the `new_relic` sink configuration: -```toml -[sinks.new_relic_logs] -type = "new_relic_logs" -license_key = "xxxx" - -[sinks.new_relic] -type = "new_relic" -license_key = "xxxx" -account_id = "yyyy" -api = "logs" +```yaml +sinks: + new_relic_logs: + type: "new_relic_logs" + license_key: "xxxx" + + new_relic: + type: "new_relic" + license_key: "xxxx" + account_id: "yyyy" + api: "logs" ``` [0-24-0-upgrade-guide]: https://vector.dev/highlights/2022-08-16-0-24-0-upgrade-guide/#deprecated-components @@ -143,27 +144,28 @@ The older version 1 API is now deprecated and will be removed in a future versio For example, the following partial configuration: -```toml -[transform.example] -type = "lua" -version = 1 -source = """ - event["a"] = "some value" - event["b"] = nil -""" +```yaml +transforms: + example: + type: "lua" + version: 1 + source: | + event["a"] = "some value" + event["b"] = nil ``` would need to be converted to the following: -```toml -[transform.example] -type = "lua" -version = 2 -hooks.process = """ - function (event, emit) - event.log.a = "some value" - event.log.b = nil - emit(event) - end -""" +```yaml +transforms: + example: + type: "lua" + version: 2 + hooks: + process: | + function (event, emit) + event.log.a = "some value" + event.log.b = nil + emit(event) + end ``` diff --git a/website/content/en/highlights/2024-07-29-0-40-0-upgrade-guide.md b/website/content/en/highlights/2024-07-29-0-40-0-upgrade-guide.md index 5813dfc248a40..7160cfc402ca0 100644 --- a/website/content/en/highlights/2024-07-29-0-40-0-upgrade-guide.md +++ b/website/content/en/highlights/2024-07-29-0-40-0-upgrade-guide.md @@ -73,10 +73,11 @@ The new behavior is to use the default strategy based on the element type. ###### Config -```toml -group_by = [ "id" ] -merge_strategies.id = "discard" -merge_strategies."a.b[0]" = "array" +```yaml +group_by: ["id"] +merge_strategies: + id: "discard" + "a.b[0]": "array" ``` ###### Event 1 From 43f620435a353b048c40661f59a2d2d18f2419ed Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Mon, 13 Apr 2026 12:24:37 -0400 Subject: [PATCH 122/326] fix(observability): fix panic in allocation tracing dealloc path (#25136) * fix(observability): fix panic in allocation tracing when deallocating pre-tracking memory When `--allocation-tracing` is enabled at runtime, the custom allocator wraps every allocation with an extra byte to store the allocation group ID. Previously, allocations made before tracking was enabled used the original (unwrapped) layout. When those were later freed, `dealloc` read an out-of-bounds byte as the group ID, hitting `NonZeroU8::new_unchecked(0)` -- undefined behavior that recent Rust toolchains (>= ~1.78) turn into an abort in debug builds. Additionally, reentrant allocations (wrapped layout but tracing closure skipped) left the group ID header uninitialized, causing misattributed deallocations and skewed per-group memory accounting. Fix: always allocate with the wrapped layout, regardless of whether tracking is currently enabled. The group ID header byte is set to: - UNTRACKED (0): tracking was off at allocation time - UNTRACED (u8::MAX): tracking was on but the tracing closure was skipped due to reentrancy - A real group ID (1..254): normal traced allocation On deallocation, all paths free with the wrapped layout (which is always correct now). UNTRACKED and UNTRACED skip `trace_deallocation` to keep per-group accounting balanced. This eliminates: - The original panic/UB from `NonZero::new_unchecked(0)` - Layout mismatches for pre-tracking allocations (including realloc) - Accounting skew from uninitialized headers on reentrant allocations This bug has been latent since #15221 (Nov 2022) which introduced the runtime toggle. Co-Authored-By: Claude Opus 4.6 (1M context) * add authors * fix(observability): strengthen allocation tracing tests Harden test coverage based on code review feedback: - Assert exact ROOT group ID and trace counters in tracked alloc test - Add end-to-end reentrant allocation test that exercises the real reentrancy path via thread-local borrow - Remove redundant manual-sentinel test now covered by reentrant test - Rename sentinel collision test to reflect what it actually checks Co-Authored-By: Claude Opus 4.6 (1M context) --------- Co-authored-by: Claude Opus 4.6 (1M context) --- ...ix_allocation_tracing_dealloc_panic.fix.md | 3 + .../allocator/tracing_allocator.rs | 202 ++++++++++++++++-- 2 files changed, 192 insertions(+), 13 deletions(-) create mode 100644 changelog.d/fix_allocation_tracing_dealloc_panic.fix.md diff --git a/changelog.d/fix_allocation_tracing_dealloc_panic.fix.md b/changelog.d/fix_allocation_tracing_dealloc_panic.fix.md new file mode 100644 index 0000000000000..732c836eac0e8 --- /dev/null +++ b/changelog.d/fix_allocation_tracing_dealloc_panic.fix.md @@ -0,0 +1,3 @@ +Fixed a panic (abort) when running with `--allocation-tracing` in debug builds, caused by deallocating memory that was allocated before tracking was enabled. Also fixed per-group memory accounting skew for reentrant allocations whose tracing closure was skipped, which left the group ID header uninitialized and caused deallocations to be attributed to wrong groups. + +authors: pront diff --git a/src/internal_telemetry/allocations/allocator/tracing_allocator.rs b/src/internal_telemetry/allocations/allocator/tracing_allocator.rs index 8d1eff3e36d4e..d0a2027eeb2e4 100644 --- a/src/internal_telemetry/allocations/allocator/tracing_allocator.rs +++ b/src/internal_telemetry/allocations/allocator/tracing_allocator.rs @@ -9,8 +9,24 @@ use super::{ }; use crate::internal_telemetry::allocations::TRACK_ALLOCATIONS; +/// Header value for allocations made while tracking is disabled. +/// On deallocation we free with the wrapped layout but skip tracing. +const UNTRACKED: u8 = 0; + +/// Header value for allocations whose tracing closure was skipped due to +/// reentrancy. `register()` never hands out `u8::MAX`, so this cannot +/// collide with a real group ID. On deallocation we free with the wrapped +/// layout but skip `trace_deallocation` to keep accounting balanced. +const UNTRACED: u8 = u8::MAX; + /// A tracing allocator that groups allocation events by groups. /// +/// Every allocation made through this allocator uses the "wrapped" layout: +/// the requested layout extended by one byte to store an allocation group +/// ID. This byte is always present regardless of whether tracking is +/// enabled, which guarantees that `dealloc` can always read a valid header +/// and free with the correct (wrapped) layout. +/// /// This allocator can only be used when specified via `#[global_allocator]`. pub struct GroupedTraceableAllocator { allocator: A, @@ -29,11 +45,6 @@ unsafe impl GlobalAlloc for GroupedTraceableAllocator #[inline] unsafe fn alloc(&self, object_layout: Layout) -> *mut u8 { unsafe { - if !TRACK_ALLOCATIONS.load(Ordering::Relaxed) { - return self.allocator.alloc(object_layout); - } - - // Allocate our wrapped layout and make sure the allocation succeeded. let (actual_layout, offset_to_group_id) = get_wrapped_layout(object_layout); let actual_ptr = self.allocator.alloc(actual_layout); if actual_ptr.is_null() { @@ -42,6 +53,16 @@ unsafe impl GlobalAlloc for GroupedTraceableAllocator let group_id_ptr = actual_ptr.add(offset_to_group_id).cast::(); + if !TRACK_ALLOCATIONS.load(Ordering::Relaxed) { + group_id_ptr.write(UNTRACKED); + return actual_ptr; + } + + // Write the untraced sentinel so that `dealloc` always finds a + // valid header, even when the tracing closure below is skipped + // due to reentrancy. + group_id_ptr.write(UNTRACED); + let object_size = object_layout.size(); try_with_suspended_allocation_group( @@ -58,19 +79,19 @@ unsafe impl GlobalAlloc for GroupedTraceableAllocator #[inline] unsafe fn dealloc(&self, object_ptr: *mut u8, object_layout: Layout) { unsafe { - if !TRACK_ALLOCATIONS.load(Ordering::Relaxed) { - self.allocator.dealloc(object_ptr, object_layout); - return; - } - // Regenerate the wrapped layout so we know where we have to look, as the pointer we've given relates to the - // requested layout, not the wrapped layout that was actually allocated. let (wrapped_layout, offset_to_group_id) = get_wrapped_layout(object_layout); - let raw_group_id = object_ptr.add(offset_to_group_id).cast::().read(); - // Deallocate before tracking, just to make sure we're reclaiming memory as soon as possible. + // Always free with the wrapped layout since all allocations + // (tracked or not) use it. self.allocator.dealloc(object_ptr, wrapped_layout); + // Skip tracing for untracked (tracking was off) and untraced + // (reentrant, closure skipped) allocations. + if raw_group_id == UNTRACKED || raw_group_id == UNTRACED { + return; + } + let object_size = object_layout.size(); let source_group_id = AllocationGroupId::from_raw(raw_group_id); @@ -96,3 +117,158 @@ fn get_wrapped_layout(object_layout: Layout) -> (Layout, usize) { (actual_layout.pad_to_align(), offset_to_group_id) } + +#[cfg(test)] +mod tests { + use std::{ + alloc::{GlobalAlloc, Layout, System}, + sync::atomic::{AtomicUsize, Ordering}, + }; + + use serial_test::serial; + + use super::*; + use crate::internal_telemetry::allocations::allocator::{ + token::AllocationGroupId, tracer::Tracer, + }; + + /// RAII guard that enables `TRACK_ALLOCATIONS` on creation and + /// restores it to `false` on drop, ensuring cleanup even if the + /// test panics. + struct TrackingGuard; + + impl TrackingGuard { + fn enable() -> Self { + TRACK_ALLOCATIONS.store(true, Ordering::Release); + Self + } + } + + impl Drop for TrackingGuard { + fn drop(&mut self) { + TRACK_ALLOCATIONS.store(false, Ordering::Release); + } + } + + struct CountingTracer { + allocs: AtomicUsize, + deallocs: AtomicUsize, + } + + impl CountingTracer { + const fn new() -> Self { + Self { + allocs: AtomicUsize::new(0), + deallocs: AtomicUsize::new(0), + } + } + } + + impl Tracer for CountingTracer { + fn trace_allocation(&self, _size: usize, _group_id: AllocationGroupId) { + self.allocs.fetch_add(1, Ordering::Relaxed); + } + + fn trace_deallocation(&self, _size: usize, _source_group_id: AllocationGroupId) { + self.deallocs.fetch_add(1, Ordering::Relaxed); + } + } + + #[test] + fn sentinels_do_not_collide_with_root_id() { + assert_eq!(UNTRACED, u8::MAX); + assert_ne!(UNTRACED, AllocationGroupId::ROOT.as_raw()); + assert_ne!(UNTRACKED, AllocationGroupId::ROOT.as_raw()); + } + + /// Allocations made while tracking is disabled get UNTRACKED (0) in the + /// header. Deallocating them (whether tracking is on or off) must use + /// the wrapped layout and skip tracing. + #[test] + #[serial] + fn untracked_alloc_dealloc_skips_tracing() { + let allocator = GroupedTraceableAllocator::new(System, CountingTracer::new()); + let layout = Layout::from_size_align(64, 8).unwrap(); + + // Tracking starts off (default state). Allocate while disabled. + let ptr = unsafe { allocator.alloc(layout) }; + assert!(!ptr.is_null()); + + // Header must be UNTRACKED. + let (_, offset) = get_wrapped_layout(layout); + let raw_id = unsafe { ptr.add(offset).cast::().read() }; + assert_eq!(raw_id, UNTRACKED); + + // Enable tracking, then dealloc -- must not panic, no trace events. + let _guard = TrackingGuard::enable(); + unsafe { allocator.dealloc(ptr, layout) }; + + assert_eq!(allocator.tracer.allocs.load(Ordering::Relaxed), 0); + assert_eq!(allocator.tracer.deallocs.load(Ordering::Relaxed), 0); + } + + /// Tracked allocation: header is a valid non-zero, non-sentinel group + /// ID, tracing fires for both alloc and dealloc, and dealloc completes + /// without panic. + #[test] + #[serial] + fn tracked_alloc_dealloc_does_not_panic() { + let allocator = GroupedTraceableAllocator::new(System, CountingTracer::new()); + let layout = Layout::from_size_align(64, 8).unwrap(); + + let _guard = TrackingGuard::enable(); + let ptr = unsafe { allocator.alloc(layout) }; + assert!(!ptr.is_null()); + + let (_, offset) = get_wrapped_layout(layout); + let raw_id = unsafe { ptr.add(offset).cast::().read() }; + assert_eq!( + raw_id, + AllocationGroupId::ROOT.as_raw(), + "header must be the ROOT group ID" + ); + assert_eq!(allocator.tracer.allocs.load(Ordering::Relaxed), 1); + + unsafe { allocator.dealloc(ptr, layout) }; + + assert_eq!(allocator.tracer.deallocs.load(Ordering::Relaxed), 1); + } + + /// End-to-end reentrant allocation: hold a mutable borrow on the + /// thread-local group stack so `try_with_suspended_allocation_group` + /// skips the tracing closure. The header must be UNTRACED and both + /// trace counters must stay at zero through alloc + dealloc. + #[test] + #[serial] + fn reentrant_alloc_writes_untraced_and_skips_tracing() { + use crate::internal_telemetry::allocations::allocator::token::LOCAL_ALLOCATION_GROUP_STACK; + + let allocator = GroupedTraceableAllocator::new(System, CountingTracer::new()); + let layout = Layout::from_size_align(64, 8).unwrap(); + + let _guard = TrackingGuard::enable(); + + // Hold a mutable borrow to simulate reentrancy -- any nested + // `try_borrow_mut` inside `try_with_suspended_allocation_group` + // will fail, causing the tracing closure to be skipped. + LOCAL_ALLOCATION_GROUP_STACK.with(|group_stack| { + let _borrow = group_stack.borrow_mut(); + + let ptr = unsafe { allocator.alloc(layout) }; + assert!(!ptr.is_null()); + + let (_, offset) = get_wrapped_layout(layout); + let raw_id = unsafe { ptr.add(offset).cast::().read() }; + assert_eq!( + raw_id, UNTRACED, + "reentrant alloc must write UNTRACED sentinel" + ); + + assert_eq!(allocator.tracer.allocs.load(Ordering::Relaxed), 0); + + unsafe { allocator.dealloc(ptr, layout) }; + + assert_eq!(allocator.tracer.deallocs.load(Ordering::Relaxed), 0); + }); + } +} From c6574fd66c328ef57fba27d2ad5422666c3e18eb Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Mon, 13 Apr 2026 12:26:51 -0400 Subject: [PATCH 123/326] enhancement(api)!: replace custom Health RPC with standard gRPC health service (#25139) * enhancement(api)!: replace custom Health RPC with standard gRPC health service Replace the custom `ObservabilityService/Health` RPC with the standard `grpc.health.v1.Health` service on the observability API (port 8686). This enables native Kubernetes gRPC health probes, `grpc-health-probe`, and other standard tooling to work out of the box. The empty service name (`""`) is used for whole-server health, matching what Kubernetes probes and `grpc-health-probe` query by default. Key changes: - Remove Health RPC, HealthRequest, HealthResponse from the proto - Add tonic_health standard health service to the gRPC server - Switch vector-api-client to use tonic_health HealthClient; the client checks ServingStatus and returns NotServing error if not SERVING - Flip health to NOT_SERVING in TopologyController::stop before draining the topology, so Kubernetes removes the pod from endpoints early - Update docs, changelog, and add integration test Co-Authored-By: Claude Opus 4.6 (1M context) * fix gate * cleanup * Update src/topology/controller.rs Co-authored-by: Thomas --------- Co-authored-by: Claude Opus 4.6 (1M context) Co-authored-by: Thomas --- Cargo.lock | 1 + Cargo.toml | 1 + changelog.d/graphql_to_grpc_api.breaking.md | 4 +- lib/vector-api-client/Cargo.toml | 1 + lib/vector-api-client/src/client.rs | 51 ++++++++++++++++----- lib/vector-api-client/src/error.rs | 3 ++ lib/vector-api-client/src/lib.rs | 6 +-- proto/vector/observability.proto | 11 +---- src/api/grpc/service.rs | 21 ++------- src/api/grpc_server.rs | 34 +++++++++----- src/app.rs | 3 -- src/topology/controller.rs | 21 +++++---- src/topology/running.rs | 9 +--- tests/vector_api/health.rs | 25 ++++++++++ tests/vector_api/lib.rs | 1 + website/content/en/docs/reference/api.md | 4 +- 16 files changed, 119 insertions(+), 77 deletions(-) create mode 100644 tests/vector_api/health.rs diff --git a/Cargo.lock b/Cargo.lock index 054d3afc66007..d013bc09432ec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12453,6 +12453,7 @@ dependencies = [ "tokio-stream", "tonic 0.11.0", "tonic-build 0.11.0", + "tonic-health", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 9389c9420d248..84d89c50096f9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -561,6 +561,7 @@ docker = ["dep:bollard", "dep:dirs-next"] api = [ "dep:base64", "dep:tonic", + "dep:tonic-health", "dep:tonic-reflection", "dep:prost", "dep:prost-types", diff --git a/changelog.d/graphql_to_grpc_api.breaking.md b/changelog.d/graphql_to_grpc_api.breaking.md index 20ab9d67b35e3..2c9756ee16a89 100644 --- a/changelog.d/graphql_to_grpc_api.breaking.md +++ b/changelog.d/graphql_to_grpc_api.breaking.md @@ -27,8 +27,8 @@ vector top --url http://localhost:8686 [grpcurl](https://github.com/fullstorydev/grpcurl): ```bash -# Check health -grpcurl -plaintext localhost:8686 vector.observability.v1.ObservabilityService/Health +# Check health (standard gRPC health check, compatible with Kubernetes gRPC probes) +grpcurl -plaintext localhost:8686 grpc.health.v1.Health/Check # List components grpcurl -plaintext localhost:8686 vector.observability.v1.ObservabilityService/GetComponents diff --git a/lib/vector-api-client/Cargo.toml b/lib/vector-api-client/Cargo.toml index 00243fb9d0ecd..260a84a927e76 100644 --- a/lib/vector-api-client/Cargo.toml +++ b/lib/vector-api-client/Cargo.toml @@ -9,6 +9,7 @@ license = "MPL-2.0" [dependencies] # gRPC tonic.workspace = true +tonic-health.workspace = true prost.workspace = true prost-types.workspace = true diff --git a/lib/vector-api-client/src/client.rs b/lib/vector-api-client/src/client.rs index fcba2702a3e2a..0d8fda1b15aaf 100644 --- a/lib/vector-api-client/src/client.rs +++ b/lib/vector-api-client/src/client.rs @@ -1,17 +1,20 @@ use http::Uri; use tokio_stream::{Stream, StreamExt}; use tonic::transport::{Channel, Endpoint}; +use tonic_health::pb::{ + HealthCheckRequest, health_check_response::ServingStatus, health_client::HealthClient, +}; use crate::{ error::{Error, Result}, proto::{ GetAllocationTracingStatusRequest, GetAllocationTracingStatusResponse, - GetComponentsRequest, GetComponentsResponse, GetMetaRequest, GetMetaResponse, - HealthRequest, HealthResponse, MetricName, StreamComponentAllocatedBytesRequest, - StreamComponentAllocatedBytesResponse, StreamComponentMetricsRequest, - StreamComponentMetricsResponse, StreamHeartbeatRequest, StreamHeartbeatResponse, - StreamOutputEventsRequest, StreamOutputEventsResponse, StreamUptimeRequest, - StreamUptimeResponse, observability_service_client::ObservabilityServiceClient, + GetComponentsRequest, GetComponentsResponse, GetMetaRequest, GetMetaResponse, MetricName, + StreamComponentAllocatedBytesRequest, StreamComponentAllocatedBytesResponse, + StreamComponentMetricsRequest, StreamComponentMetricsResponse, StreamHeartbeatRequest, + StreamHeartbeatResponse, StreamOutputEventsRequest, StreamOutputEventsResponse, + StreamUptimeRequest, StreamUptimeResponse, + observability_service_client::ObservabilityServiceClient, }, }; @@ -19,6 +22,7 @@ use crate::{ #[derive(Debug, Clone)] pub struct Client { endpoint: Endpoint, + channel: Option, client: Option>, } @@ -33,6 +37,7 @@ impl Client { pub fn new(uri: Uri) -> Self { Self { endpoint: Endpoint::from(uri), + channel: None, client: None, } } @@ -40,7 +45,8 @@ impl Client { /// Connect to the gRPC server pub async fn connect(&mut self) -> Result<()> { let channel = self.endpoint.connect().await?; - self.client = Some(ObservabilityServiceClient::new(channel)); + self.client = Some(ObservabilityServiceClient::new(channel.clone())); + self.channel = Some(channel); Ok(()) } @@ -49,13 +55,34 @@ impl Client { self.client.as_mut().ok_or(Error::NotConnected) } + /// Get the underlying channel + fn channel(&self) -> Result<&Channel> { + self.channel.as_ref().ok_or(Error::NotConnected) + } + // ========== Unary RPCs ========== - /// Check if the API server is healthy - pub async fn health(&mut self) -> Result { - let client = self.ensure_connected()?; - let response = client.health(HealthRequest {}).await?; - Ok(response.into_inner()) + /// Check if the API server is healthy using the standard gRPC health check + /// protocol (grpc.health.v1.Health/Check). + /// + /// Queries the empty service name (`""`), which represents whole-server + /// health. This is the default used by Kubernetes gRPC probes and + /// `grpc-health-probe`. + /// + /// Returns `Ok(())` if the server is `SERVING`, or an error otherwise. + pub async fn health(&mut self) -> Result<()> { + let channel = self.channel()?.clone(); + let mut health_client = HealthClient::new(channel); + let response = health_client + .check(HealthCheckRequest { + service: String::new(), + }) + .await?; + let status = response.into_inner().status; + if status != ServingStatus::Serving as i32 { + return Err(Error::NotServing { status }); + } + Ok(()) } /// Get metadata about the Vector instance diff --git a/lib/vector-api-client/src/error.rs b/lib/vector-api-client/src/error.rs index 65fcfd80b511f..34eb347af12d8 100644 --- a/lib/vector-api-client/src/error.rs +++ b/lib/vector-api-client/src/error.rs @@ -16,6 +16,9 @@ pub enum Error { #[snafu(display("Not connected to gRPC server"))] NotConnected, + #[snafu(display("Server is not serving (status: {})", status))] + NotServing { status: i32 }, + #[snafu(display("Stream error: {}", message))] Stream { message: String }, } diff --git a/lib/vector-api-client/src/lib.rs b/lib/vector-api-client/src/lib.rs index a05002ebb3e61..74b72f2f0ea5b 100644 --- a/lib/vector-api-client/src/lib.rs +++ b/lib/vector-api-client/src/lib.rs @@ -11,9 +11,9 @@ //! let mut client = Client::new("http://localhost:9999".parse().unwrap()); //! client.connect().await?; //! -//! // Check health -//! let health = client.health().await?; -//! println!("Healthy: {}", health.healthy); +//! // Check health (standard gRPC health check) +//! client.health().await?; +//! println!("Server is healthy"); //! //! // Get components //! let components = client.get_components(0).await?; diff --git a/proto/vector/observability.proto b/proto/vector/observability.proto index 9996a47908949..57ce1a3f40002 100644 --- a/proto/vector/observability.proto +++ b/proto/vector/observability.proto @@ -8,9 +8,6 @@ import "event.proto"; service ObservabilityService { // ========== Simple Queries ========== - // Check if the API server is healthy and responding - rpc Health(HealthRequest) returns (HealthResponse); - // Get metadata about the Vector instance (version, hostname) rpc GetMeta(GetMetaRequest) returns (GetMetaResponse); @@ -42,13 +39,7 @@ service ObservabilityService { rpc StreamOutputEvents(StreamOutputEventsRequest) returns (stream StreamOutputEventsResponse); } -// ========== Health & Meta Messages ========== - -message HealthRequest {} - -message HealthResponse { - bool healthy = 1; -} +// ========== Meta Messages ========== message GetMetaRequest {} diff --git a/src/api/grpc/service.rs b/src/api/grpc/service.rs index 580ebb69b5219..110ef9d5ba441 100644 --- a/src/api/grpc/service.rs +++ b/src/api/grpc/service.rs @@ -4,10 +4,7 @@ use std::pin::Pin; // (only used in synchronous map updates inside IntervalStream closures), so the // cheaper std mutex is correct here. tokio::sync::Mutex is only needed when the // critical section itself contains .await. -use std::sync::{ - Arc, Mutex, - atomic::{AtomicBool, Ordering}, -}; +use std::sync::{Arc, Mutex}; use std::time::Duration; use futures::{StreamExt as FuturesStreamExt, stream}; @@ -399,12 +396,11 @@ fn ports_to_proto_outputs( /// gRPC observability service implementation. pub struct ObservabilityService { watch_rx: WatchRx, - running: Arc, } impl ObservabilityService { - pub const fn new(watch_rx: WatchRx, running: Arc) -> Self { - Self { watch_rx, running } + pub const fn new(watch_rx: WatchRx) -> Self { + Self { watch_rx } } } @@ -412,17 +408,6 @@ impl ObservabilityService { impl observability::Service for ObservabilityService { // ========== Simple Queries ========== - async fn health( - &self, - _request: Request, - ) -> Result, Status> { - if self.running.load(Ordering::Relaxed) { - Ok(Response::new(HealthResponse { healthy: true })) - } else { - Err(Status::unavailable("Vector is shutting down")) - } - } - async fn get_meta( &self, _request: Request, diff --git a/src/api/grpc_server.rs b/src/api/grpc_server.rs index 714459cd39b79..ce33b0da3f7d5 100644 --- a/src/api/grpc_server.rs +++ b/src/api/grpc_server.rs @@ -1,11 +1,8 @@ -use std::{ - error::Error as StdError, - net::SocketAddr, - sync::{Arc, atomic::AtomicBool}, -}; +use std::{error::Error as StdError, net::SocketAddr}; use tokio::sync::oneshot; use tokio_stream::wrappers::TcpListenerStream; use tonic::transport::Server as TonicServer; +use tonic_health::server::{HealthReporter, health_reporter}; use vector_lib::tap::topology::WatchRx; use super::grpc::ObservabilityService; @@ -14,6 +11,7 @@ use crate::{config::Config, proto::observability::Server as ObservabilityServer} /// gRPC API server for Vector observability. pub struct GrpcServer { _shutdown: oneshot::Sender<()>, + health_reporter: HealthReporter, addr: SocketAddr, } @@ -25,11 +23,7 @@ impl GrpcServer { /// is dropped. /// /// Returns an error if the server fails to bind to the configured address. - pub async fn start( - config: &Config, - watch_rx: WatchRx, - running: Arc, - ) -> crate::Result { + pub async fn start(config: &Config, watch_rx: WatchRx) -> crate::Result { let addr = config.api.address.ok_or_else(|| { crate::Error::from("API address not configured in config.api.address") })?; @@ -46,7 +40,11 @@ impl GrpcServer { info!("GRPC API server bound to {}.", actual_addr); - let service = ObservabilityService::new(watch_rx, running); + let service = ObservabilityService::new(watch_rx); + + // Create the standard gRPC health service (grpc.health.v1.Health). + // The empty service ("") is registered as SERVING by default. + let (health_reporter, health_service) = health_reporter(); let (_shutdown, rx) = oneshot::channel(); @@ -59,10 +57,12 @@ impl GrpcServer { .register_encoded_file_descriptor_set( crate::proto::observability::FILE_DESCRIPTOR_SET, ) + .register_encoded_file_descriptor_set(tonic_health::pb::FILE_DESCRIPTOR_SET) .build() .expect("Failed to build reflection service"); let result = TonicServer::builder() + .add_service(health_service) .add_service(ObservabilityServer::new(service)) .add_service(reflection_service) .serve_with_incoming_shutdown(incoming, async { @@ -85,10 +85,22 @@ impl GrpcServer { Ok(Self { _shutdown, + health_reporter, addr: actual_addr, }) } + /// Signal that the server is no longer serving. + /// + /// Call this **before** draining the topology so that Kubernetes gRPC + /// readiness probes fail early and the pod is removed from endpoints + /// before the process exits. + pub async fn set_not_serving(&mut self) { + self.health_reporter + .set_service_status("", tonic_health::ServingStatus::NotServing) + .await; + } + /// Get the address the server is listening on pub const fn addr(&self) -> SocketAddr { self.addr diff --git a/src/app.rs b/src/app.rs index 904d42be11968..580c06642a6e3 100644 --- a/src/app.rs +++ b/src/app.rs @@ -38,8 +38,6 @@ use crate::{ }, trace, }; -#[cfg(feature = "api")] -use std::sync::Arc; static WORKER_THREADS: AtomicUsize = AtomicUsize::new(0); @@ -143,7 +141,6 @@ impl ApplicationConfig { let api_server = handle.block_on(api::GrpcServer::start( self.topology.config(), self.topology.watch(), - Arc::clone(&self.topology.running), )); match api_server { Ok(server) => { diff --git a/src/topology/controller.rs b/src/topology/controller.rs index b77629828a577..efbe82cf594ac 100644 --- a/src/topology/controller.rs +++ b/src/topology/controller.rs @@ -76,13 +76,7 @@ impl TopologyController { } else if self.api_server.is_none() { debug!("Starting gRPC API server."); - match api::GrpcServer::start( - self.topology.config(), - self.topology.watch(), - Arc::clone(&self.topology.running), - ) - .await - { + match api::GrpcServer::start(self.topology.config(), self.topology.watch()).await { Ok(api_server) => { let addr = api_server.addr(); info!( @@ -156,7 +150,18 @@ impl TopologyController { } } - pub async fn stop(self) { + #[cfg_attr(not(feature = "api"), allow(unused_mut))] + pub async fn stop(mut self) { + // Phase 1: Mark the gRPC API as unavailable so that external probes + // (e.g. Kubernetes readiness) fail early and stop routing traffic + // to this instance. + #[cfg(feature = "api")] + if let Some(server) = self.api_server.as_mut() { + server.set_not_serving().await; + } + + // Phase 2: Drain the topology -- shuts down sources, waits for + // in-flight events to flush through transforms and sinks. self.topology.stop().await; } diff --git a/src/topology/running.rs b/src/topology/running.rs index 6839379cd0f1a..a1ecf7278ae89 100644 --- a/src/topology/running.rs +++ b/src/topology/running.rs @@ -1,9 +1,6 @@ use std::{ collections::{HashMap, HashSet}, - sync::{ - Arc, Mutex, - atomic::{AtomicBool, Ordering}, - }, + sync::{Arc, Mutex}, }; use futures::{Future, FutureExt, future}; @@ -66,7 +63,6 @@ pub struct RunningTopology { pub(crate) config: Config, pub(crate) abort_tx: mpsc::UnboundedSender, watch: (WatchTx, WatchRx), - pub(crate) running: Arc, graceful_shutdown_duration: Option, utilization_registry: Option, utilization_task: Option, @@ -90,7 +86,6 @@ impl RunningTopology { tasks: HashMap::new(), abort_tx, watch: watch::channel(TapResource::default()), - running: Arc::new(AtomicBool::new(true)), graceful_shutdown_duration: config.graceful_shutdown_duration, config, utilization_registry: None, @@ -148,8 +143,6 @@ impl RunningTopology { /// dropped then everything from this RunningTopology instance is fully /// dropped. pub fn stop(self) -> impl Future { - // Update the API's health endpoint to signal shutdown - self.running.store(false, Ordering::Relaxed); // Create handy handles collections of all tasks for the subsequent // operations. let mut wait_handles = Vec::new(); diff --git a/tests/vector_api/health.rs b/tests/vector_api/health.rs new file mode 100644 index 0000000000000..a17d16eecf913 --- /dev/null +++ b/tests/vector_api/health.rs @@ -0,0 +1,25 @@ +//! Integration tests for the standard gRPC health check on the observability API. + +use super::{common::*, harness::*}; + +// ============================================================================ +// Tests +// ============================================================================ + +/// Verifies that the standard gRPC health check (empty service = whole-server +/// health) reports SERVING on a running Vector instance. +#[tokio::test] +async fn health_check_reports_serving() { + let config = single_source_config("demo", 1.0, None); + let mut harness = TestHarness::new(&config) + .await + .expect("Vector should start"); + + harness + .api_client() + .health() + .await + .expect("health check should report SERVING"); + + assert!(harness.check_running(), "Vector should still be running"); +} diff --git a/tests/vector_api/lib.rs b/tests/vector_api/lib.rs index e82f39d41b42e..c6c7fbec35dd6 100644 --- a/tests/vector_api/lib.rs +++ b/tests/vector_api/lib.rs @@ -5,5 +5,6 @@ mod common; mod harness; +mod health; mod tap; mod top; diff --git a/website/content/en/docs/reference/api.md b/website/content/en/docs/reference/api.md index 11a3566f6acfa..949353241929b 100644 --- a/website/content/en/docs/reference/api.md +++ b/website/content/en/docs/reference/api.md @@ -24,8 +24,8 @@ You can interact with it using any standard gRPC tooling. ### Example using grpcurl ```bash -# Check health -grpcurl -plaintext localhost:8686 vector.observability.v1.ObservabilityService/Health +# Check health (standard gRPC health check, compatible with Kubernetes gRPC probes) +grpcurl -plaintext localhost:8686 grpc.health.v1.Health/Check # List components grpcurl -plaintext localhost:8686 vector.observability.v1.ObservabilityService/GetComponents From 44637c3056c8850752e6f1c6aef8ab5ead810b0f Mon Sep 17 00:00:00 2001 From: Thomas Date: Tue, 14 Apr 2026 11:04:29 -0400 Subject: [PATCH 124/326] chore(deps): bump datadog-ci from 5.12.0 to 5.13.0 (#25187) --- .../environment/npm-tools/package-lock.json | 2421 +---------------- scripts/environment/npm-tools/package.json | 2 +- 2 files changed, 145 insertions(+), 2278 deletions(-) diff --git a/scripts/environment/npm-tools/package-lock.json b/scripts/environment/npm-tools/package-lock.json index 9e25b17830d18..d9e098652b6ad 100644 --- a/scripts/environment/npm-tools/package-lock.json +++ b/scripts/environment/npm-tools/package-lock.json @@ -1,332 +1,27 @@ { - "name": "environment", + "name": "npm-tools", "lockfileVersion": 3, "requires": true, "packages": { "": { "dependencies": { - "@datadog/datadog-ci": "5.12.0", + "@datadog/datadog-ci": "5.13.0", "markdownlint-cli2": "0.22.0", "prettier": "3.8.1" } }, - "node_modules/@antfu/install-pkg": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@antfu/install-pkg/-/install-pkg-1.1.0.tgz", - "integrity": "sha512-MGQsmw10ZyI+EJo45CdSER4zEb+p31LpDAFp2Z3gkSd1yqVZGi0Ebx++YTEMonJy4oChEMLsxZ64j8FH6sSqtQ==", - "license": "MIT", - "dependencies": { - "package-manager-detector": "^1.3.0", - "tinyexec": "^1.0.1" - }, - "funding": { - "url": "https://github.com/sponsors/antfu" - } - }, "node_modules/@datadog/datadog-ci": { - "version": "5.12.0", - "resolved": "https://registry.npmjs.org/@datadog/datadog-ci/-/datadog-ci-5.12.0.tgz", - "integrity": "sha512-8zJOgx0ouVZtspXTGdx327Yt6UiRnaQSlMc7c343a1HaKC9nIlQhnd90G1frFZhopiqA1KD8h9CWBhygxAJW7w==", + "version": "5.13.0", + "resolved": "https://registry.npmjs.org/@datadog/datadog-ci/-/datadog-ci-5.13.0.tgz", + "integrity": "sha512-QEooWT2OE6J//OV4gka3F8Ca8QkmVGEyKR8zpJVZ6EufJVsNSnBX2V2VE3tzkvndcKRe2/Q9ZCiL9GUiZ4wTLA==", "license": "Apache-2.0", - "dependencies": { - "@datadog/datadog-ci-base": "5.12.0", - "@datadog/datadog-ci-plugin-coverage": "5.12.0", - "@datadog/datadog-ci-plugin-deployment": "5.12.0", - "@datadog/datadog-ci-plugin-dora": "5.12.0", - "@datadog/datadog-ci-plugin-gate": "5.12.0", - "@datadog/datadog-ci-plugin-junit": "5.12.0", - "@datadog/datadog-ci-plugin-sarif": "5.12.0", - "@datadog/datadog-ci-plugin-sbom": "5.12.0", - "clipanion": "3.2.1", - "typanion": "3.14.0" - }, "bin": { - "datadog-ci": "dist/cli.js" + "datadog-ci": "dist/bundle.js" }, "engines": { "node": ">=20" } }, - "node_modules/@datadog/datadog-ci-base": { - "version": "5.12.0", - "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-base/-/datadog-ci-base-5.12.0.tgz", - "integrity": "sha512-ZyobD2nbzwRksWiYvcxl3ZuYE4+AX1is26qjWAs1ufhVyGBCx1cVqiDFEAILeyqkncZYIFGppjCyjUcxev+CBg==", - "license": "Apache-2.0", - "dependencies": { - "@antfu/install-pkg": "1.1.0", - "@types/datadog-metrics": "0.6.1", - "async-retry": "1.3.3", - "axios": "1.13.5", - "chalk": "3.0.0", - "clipanion": "3.2.1", - "datadog-metrics": "0.9.3", - "debug": "4.4.1", - "deep-extend": "0.6.0", - "fast-xml-parser": "5.5.9", - "form-data": "4.0.4", - "glob": "13.0.6", - "inquirer": "8.2.7", - "is-docker": "4.0.0", - "jest-diff": "30.2.0", - "js-yaml": "4.1.1", - "jszip": "3.10.1", - "proxy-agent": "6.4.0", - "semver": "7.6.3", - "simple-git": "3.33.0", - "terminal-link": "2.1.1", - "tiny-async-pool": "2.1.0", - "typanion": "3.14.0", - "upath": "2.0.1" - }, - "peerDependencies": { - "@datadog/datadog-ci-plugin-aas": "5.12.0", - "@datadog/datadog-ci-plugin-cloud-run": "5.12.0", - "@datadog/datadog-ci-plugin-container-app": "5.12.0", - "@datadog/datadog-ci-plugin-coverage": "5.12.0", - "@datadog/datadog-ci-plugin-deployment": "5.12.0", - "@datadog/datadog-ci-plugin-dora": "5.12.0", - "@datadog/datadog-ci-plugin-gate": "5.12.0", - "@datadog/datadog-ci-plugin-junit": "5.12.0", - "@datadog/datadog-ci-plugin-lambda": "5.12.0", - "@datadog/datadog-ci-plugin-sarif": "5.12.0", - "@datadog/datadog-ci-plugin-sbom": "5.12.0", - "@datadog/datadog-ci-plugin-stepfunctions": "5.11.0", - "@datadog/datadog-ci-plugin-synthetics": "5.11.0", - "@datadog/datadog-ci-plugin-terraform": "5.11.0" - }, - "peerDependenciesMeta": { - "@datadog/datadog-ci-plugin-aas": { - "optional": true - }, - "@datadog/datadog-ci-plugin-cloud-run": { - "optional": true - }, - "@datadog/datadog-ci-plugin-container-app": { - "optional": true - }, - "@datadog/datadog-ci-plugin-coverage": { - "optional": true - }, - "@datadog/datadog-ci-plugin-deployment": { - "optional": true - }, - "@datadog/datadog-ci-plugin-dora": { - "optional": true - }, - "@datadog/datadog-ci-plugin-gate": { - "optional": true - }, - "@datadog/datadog-ci-plugin-junit": { - "optional": true - }, - "@datadog/datadog-ci-plugin-lambda": { - "optional": true - }, - "@datadog/datadog-ci-plugin-sarif": { - "optional": true - }, - "@datadog/datadog-ci-plugin-sbom": { - "optional": true - }, - "@datadog/datadog-ci-plugin-stepfunctions": { - "optional": true - }, - "@datadog/datadog-ci-plugin-synthetics": { - "optional": true - }, - "@datadog/datadog-ci-plugin-terraform": { - "optional": true - } - } - }, - "node_modules/@datadog/datadog-ci-base/node_modules/debug": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz", - "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/@datadog/datadog-ci-plugin-coverage": { - "version": "5.12.0", - "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-coverage/-/datadog-ci-plugin-coverage-5.12.0.tgz", - "integrity": "sha512-ll4aBYF+FMOlOjAnNiPjO8iPmVB/qfBHrUOCk2EMnOaQn6Eve/FajCaSqeV2BoI9+3UXv04P6eH5nxPtpcM/cg==", - "license": "Apache-2.0", - "dependencies": { - "chalk": "3.0.0", - "fast-xml-parser": "5.5.9", - "form-data": "4.0.4", - "upath": "2.0.1" - }, - "peerDependencies": { - "@datadog/datadog-ci-base": "5.12.0" - } - }, - "node_modules/@datadog/datadog-ci-plugin-deployment": { - "version": "5.12.0", - "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-deployment/-/datadog-ci-plugin-deployment-5.12.0.tgz", - "integrity": "sha512-6KDwUmdxwGxYoBEYqAY0uo7ocvtTq3p3cXW4KjpDXwZyr1wWstzXQxTx7qquZzMMNvsD6JCq+HDqwdTgaHSAog==", - "license": "Apache-2.0", - "dependencies": { - "axios": "1.13.5", - "chalk": "3.0.0", - "simple-git": "3.33.0" - }, - "peerDependencies": { - "@datadog/datadog-ci-base": "5.12.0" - } - }, - "node_modules/@datadog/datadog-ci-plugin-dora": { - "version": "5.12.0", - "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-dora/-/datadog-ci-plugin-dora-5.12.0.tgz", - "integrity": "sha512-HrC6H5lLzRTv3tKiS2f5YMU/WWwgzMF2kHIJU6+pe0y/v1xUYCELV112XYldbg3dkENfpG/AwjUeO6B8rE3s+g==", - "license": "Apache-2.0", - "dependencies": { - "chalk": "3.0.0", - "simple-git": "3.33.0" - }, - "peerDependencies": { - "@datadog/datadog-ci-base": "5.12.0" - } - }, - "node_modules/@datadog/datadog-ci-plugin-gate": { - "version": "5.12.0", - "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-gate/-/datadog-ci-plugin-gate-5.12.0.tgz", - "integrity": "sha512-YNZuRigmGDGK2tK6xRsaq0PPx/4CRVZ4U6q7xabIP+jmhoG7mbI1RFUjlXSnGfeU1SXm0bVQw8F69AcZCHtNOA==", - "license": "Apache-2.0", - "dependencies": { - "@types/uuid": "9.0.8", - "chalk": "3.0.0", - "uuid": "9.0.1" - }, - "peerDependencies": { - "@datadog/datadog-ci-base": "5.12.0" - } - }, - "node_modules/@datadog/datadog-ci-plugin-junit": { - "version": "5.12.0", - "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-junit/-/datadog-ci-plugin-junit-5.12.0.tgz", - "integrity": "sha512-XPVULYU1RUN2kFmSWGfAgU+psGFhPu+5NxnHeAQubs/wYcGwdZUEAD2+tn4F3PxpCkTduEb4FO6mLpYbjBpuqA==", - "license": "Apache-2.0", - "dependencies": { - "chalk": "3.0.0", - "fast-xml-parser": "5.5.9", - "form-data": "4.0.4", - "upath": "2.0.1", - "uuid": "9.0.1" - }, - "peerDependencies": { - "@datadog/datadog-ci-base": "5.12.0" - } - }, - "node_modules/@datadog/datadog-ci-plugin-sarif": { - "version": "5.12.0", - "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-sarif/-/datadog-ci-plugin-sarif-5.12.0.tgz", - "integrity": "sha512-tmUEv6sJWsFG9vnriSu9RZSrLvYo0EfvRaHiiO3ufxBC+NvU0HZYhtTqXVT6NnsUSVv/2FBnzl0hCnPDHhyBFg==", - "license": "Apache-2.0", - "dependencies": { - "ajv": "8.18.0", - "ajv-formats": "3.0.1", - "chalk": "3.0.0", - "form-data": "4.0.4", - "upath": "2.0.1", - "uuid": "9.0.1" - }, - "peerDependencies": { - "@datadog/datadog-ci-base": "5.12.0" - } - }, - "node_modules/@datadog/datadog-ci-plugin-sbom": { - "version": "5.12.0", - "resolved": "https://registry.npmjs.org/@datadog/datadog-ci-plugin-sbom/-/datadog-ci-plugin-sbom-5.12.0.tgz", - "integrity": "sha512-F0c0gfX4O+Qm1heLEhNCTl/tUPlVQvyFGYTUqJ0Z2niqT4Y3z7Oi6RSodgUSwGCKl6bwacRjTTdCrzI3V0n3nA==", - "license": "Apache-2.0", - "dependencies": { - "ajv": "8.18.0", - "ajv-formats": "3.0.1", - "axios": "1.13.5", - "chalk": "3.0.0", - "packageurl-js": "2.0.1" - }, - "peerDependencies": { - "@datadog/datadog-ci-base": "5.12.0" - } - }, - "node_modules/@inquirer/external-editor": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@inquirer/external-editor/-/external-editor-1.0.3.tgz", - "integrity": "sha512-RWbSrDiYmO4LbejWY7ttpxczuwQyZLBUyygsA9Nsv95hpzUWwnNTVQmAq3xuh7vNwCp07UTmE5i11XAEExx4RA==", - "license": "MIT", - "dependencies": { - "chardet": "^2.1.1", - "iconv-lite": "^0.7.0" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@types/node": ">=18" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - } - } - }, - "node_modules/@jest/diff-sequences": { - "version": "30.0.1", - "resolved": "https://registry.npmjs.org/@jest/diff-sequences/-/diff-sequences-30.0.1.tgz", - "integrity": "sha512-n5H8QLDJ47QqbCNn5SuFjCRDrOLEZ0h8vAHCK5RL9Ls7Xa8AQLa/YxAc9UjFqoEDM48muwtBGjtMY5cr0PLDCw==", - "license": "MIT", - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/@jest/get-type": { - "version": "30.1.0", - "resolved": "https://registry.npmjs.org/@jest/get-type/-/get-type-30.1.0.tgz", - "integrity": "sha512-eMbZE2hUnx1WV0pmURZY9XoXPkUYjpc55mb0CrhtdWLtzMQPFvu/rZkTLZFTsdaVQa+Tr4eWAteqcUzoawq/uA==", - "license": "MIT", - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/@jest/schemas": { - "version": "30.0.5", - "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-30.0.5.tgz", - "integrity": "sha512-DmdYgtezMkh3cpU8/1uyXakv3tJRcmcXxBOcO0tbaozPwpmh4YMsnWrQm9ZmZMfa5ocbxzbFk6O4bDPEc/iAnA==", - "license": "MIT", - "dependencies": { - "@sinclair/typebox": "^0.34.0" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/@kwsites/file-exists": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@kwsites/file-exists/-/file-exists-1.1.1.tgz", - "integrity": "sha512-m9/5YGR18lIwxSFDwfE3oA7bWuq9kdau6ugN4H2rJeyhFQZcG9AgSHkQtSD15a8WvTgfz9aikZMrKPHvbpqFiw==", - "license": "MIT", - "dependencies": { - "debug": "^4.1.1" - } - }, - "node_modules/@kwsites/promise-deferred": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@kwsites/promise-deferred/-/promise-deferred-1.1.1.tgz", - "integrity": "sha512-GaHYm+c0O9MjZRu0ongGBRbinu8gVAMd2UZjji6jVmqKtZluZnptXGWhz1E8j8D2HJ3f/yMxKAUC0b+57wncIw==", - "license": "MIT" - }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -362,12 +57,6 @@ "node": ">= 8" } }, - "node_modules/@sinclair/typebox": { - "version": "0.34.49", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.34.49.tgz", - "integrity": "sha512-brySQQs7Jtn0joV8Xh9ZV/hZb9Ozb0pmazDIASBkYKCjXrXU3mpcFahmK/z4YDhGkQvP9mWJbVyahdtU5wQA+A==", - "license": "MIT" - }, "node_modules/@sindresorhus/merge-streams": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/@sindresorhus/merge-streams/-/merge-streams-4.0.0.tgz", @@ -380,18 +69,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/@tootallnate/quickjs-emscripten": { - "version": "0.23.0", - "resolved": "https://registry.npmjs.org/@tootallnate/quickjs-emscripten/-/quickjs-emscripten-0.23.0.tgz", - "integrity": "sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA==", - "license": "MIT" - }, - "node_modules/@types/datadog-metrics": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/@types/datadog-metrics/-/datadog-metrics-0.6.1.tgz", - "integrity": "sha512-p6zVpfmNcXwtcXjgpz7do/fKyfndGhU5sGJVtb5Gn5PvLDiQUAgD0mI/itf/99sBi9DRxeyhFQ9dQF6OxxQNbA==", - "license": "MIT" - }, "node_modules/@types/debug": { "version": "4.1.13", "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.13.tgz", @@ -419,733 +96,127 @@ "integrity": "sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==", "license": "MIT" }, - "node_modules/@types/uuid": { - "version": "9.0.8", - "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-9.0.8.tgz", - "integrity": "sha512-jg+97EGIcY9AGHJJRaaPVgetKDsrTgbRjQ5Msgjh/DQKEFl0DtyRr/VCOyD1T2R1MNeWPK/u7JoGhlDZnKBAfA==", - "license": "MIT" + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "license": "Python-2.0" }, - "node_modules/agent-base": { - "version": "7.1.4", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", - "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "node_modules/braces": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", "license": "MIT", + "dependencies": { + "fill-range": "^7.1.1" + }, "engines": { - "node": ">= 14" + "node": ">=8" } }, - "node_modules/ajv": { - "version": "8.18.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz", - "integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==", + "node_modules/character-entities": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-2.0.2.tgz", + "integrity": "sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ==", "license": "MIT", - "dependencies": { - "fast-deep-equal": "^3.1.3", - "fast-uri": "^3.0.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2" - }, "funding": { "type": "github", - "url": "https://github.com/sponsors/epoberezkin" + "url": "https://github.com/sponsors/wooorm" } }, - "node_modules/ajv-formats": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", - "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "node_modules/character-entities-legacy": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/character-entities-legacy/-/character-entities-legacy-3.0.0.tgz", + "integrity": "sha512-RpPp0asT/6ufRm//AJVwpViZbGM/MkjQFxJccQRHmISF/22NBtsHqAWmL+/pmkPWoIUJdWyeVleTl1wydHATVQ==", "license": "MIT", - "dependencies": { - "ajv": "^8.0.0" - }, - "peerDependencies": { - "ajv": "^8.0.0" - }, - "peerDependenciesMeta": { - "ajv": { - "optional": true - } + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" } }, - "node_modules/ansi-escapes": { - "version": "4.3.2", - "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz", - "integrity": "sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==", + "node_modules/character-reference-invalid": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/character-reference-invalid/-/character-reference-invalid-2.0.1.tgz", + "integrity": "sha512-iBZ4F4wRbyORVsu0jPV7gXkOsGYjGHPmAyv+HiHG8gi5PtC9KI2j1+v8/tlibRvjoWX027ypmG/n0HtO5t7unw==", "license": "MIT", - "dependencies": { - "type-fest": "^0.21.3" - }, - "engines": { - "node": ">=8" - }, "funding": { - "url": "https://github.com/sponsors/sindresorhus" + "type": "github", + "url": "https://github.com/sponsors/wooorm" } }, - "node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "node_modules/commander": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-8.3.0.tgz", + "integrity": "sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww==", "license": "MIT", "engines": { - "node": ">=8" + "node": ">= 12" } }, - "node_modules/ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", "license": "MIT", "dependencies": { - "color-convert": "^2.0.1" + "ms": "^2.1.3" }, "engines": { - "node": ">=8" + "node": ">=6.0" }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } } }, - "node_modules/argparse": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "license": "Python-2.0" - }, - "node_modules/ast-types": { - "version": "0.13.4", - "resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.13.4.tgz", - "integrity": "sha512-x1FCFnFifvYDDzTaLII71vG5uvDwgtmDTEVWAxrgeiR8VjMONcCXJx7E+USjDtHlwFmt9MysbqgF9b9Vjr6w+w==", + "node_modules/decode-named-character-reference": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/decode-named-character-reference/-/decode-named-character-reference-1.3.0.tgz", + "integrity": "sha512-GtpQYB283KrPp6nRw50q3U9/VfOutZOe103qlN7BPP6Ad27xYnOIWv4lPzo8HCAL+mMZofJ9KEy30fq6MfaK6Q==", "license": "MIT", "dependencies": { - "tslib": "^2.0.1" + "character-entities": "^2.0.0" }, - "engines": { - "node": ">=4" - } - }, - "node_modules/async-retry": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/async-retry/-/async-retry-1.3.3.tgz", - "integrity": "sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==", - "license": "MIT", - "dependencies": { - "retry": "0.13.1" + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" } }, - "node_modules/asynckit": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", - "license": "MIT" - }, - "node_modules/axios": { - "version": "1.13.5", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.13.5.tgz", - "integrity": "sha512-cz4ur7Vb0xS4/KUN0tPWe44eqxrIu31me+fbang3ijiNscE129POzipJJA6zniq2C/Z6sJCjMimjS8Lc/GAs8Q==", + "node_modules/dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", "license": "MIT", - "dependencies": { - "follow-redirects": "^1.15.11", - "form-data": "^4.0.5", - "proxy-from-env": "^1.1.0" + "engines": { + "node": ">=6" } }, - "node_modules/axios/node_modules/form-data": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz", - "integrity": "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==", + "node_modules/devlop": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/devlop/-/devlop-1.1.0.tgz", + "integrity": "sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==", "license": "MIT", "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "es-set-tostringtag": "^2.1.0", - "hasown": "^2.0.2", - "mime-types": "^2.1.12" + "dequal": "^2.0.0" }, - "engines": { - "node": ">= 6" + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" } }, - "node_modules/balanced-match": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", - "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", - "license": "MIT", + "node_modules/entities": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "license": "BSD-2-Clause", "engines": { - "node": "18 || 20 || >=22" + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" } }, - "node_modules/base64-js": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", - "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT" - }, - "node_modules/basic-ftp": { - "version": "5.2.2", - "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.2.2.tgz", - "integrity": "sha512-1tDrzKsdCg70WGvbFss/ulVAxupNauGnOlgpyjKzeQxzyllBLS0CGLV7tjIXTK3ZQA9/FBEm9qyFFN1bciA6pw==", - "license": "MIT", - "engines": { - "node": ">=10.0.0" - } - }, - "node_modules/bignumber.js": { - "version": "9.3.1", - "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.3.1.tgz", - "integrity": "sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ==", - "license": "MIT", - "engines": { - "node": "*" - } - }, - "node_modules/bl": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", - "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", - "license": "MIT", - "dependencies": { - "buffer": "^5.5.0", - "inherits": "^2.0.4", - "readable-stream": "^3.4.0" - } - }, - "node_modules/bl/node_modules/readable-stream": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", - "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", - "license": "MIT", - "dependencies": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/brace-expansion": { - "version": "5.0.5", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.5.tgz", - "integrity": "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==", - "license": "MIT", - "dependencies": { - "balanced-match": "^4.0.2" - }, - "engines": { - "node": "18 || 20 || >=22" - } - }, - "node_modules/braces": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", - "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", - "license": "MIT", - "dependencies": { - "fill-range": "^7.1.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/buffer": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", - "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT", - "dependencies": { - "base64-js": "^1.3.1", - "ieee754": "^1.1.13" - } - }, - "node_modules/call-bind-apply-helpers": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", - "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/chalk": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-3.0.0.tgz", - "integrity": "sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg==", - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/character-entities": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-2.0.2.tgz", - "integrity": "sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ==", - "license": "MIT", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, - "node_modules/character-entities-legacy": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/character-entities-legacy/-/character-entities-legacy-3.0.0.tgz", - "integrity": "sha512-RpPp0asT/6ufRm//AJVwpViZbGM/MkjQFxJccQRHmISF/22NBtsHqAWmL+/pmkPWoIUJdWyeVleTl1wydHATVQ==", - "license": "MIT", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, - "node_modules/character-reference-invalid": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/character-reference-invalid/-/character-reference-invalid-2.0.1.tgz", - "integrity": "sha512-iBZ4F4wRbyORVsu0jPV7gXkOsGYjGHPmAyv+HiHG8gi5PtC9KI2j1+v8/tlibRvjoWX027ypmG/n0HtO5t7unw==", - "license": "MIT", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, - "node_modules/chardet": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/chardet/-/chardet-2.1.1.tgz", - "integrity": "sha512-PsezH1rqdV9VvyNhxxOW32/d75r01NY7TQCmOqomRo15ZSOKbpTFVsfjghxo6JloQUCGnH4k1LGu0R4yCLlWQQ==", - "license": "MIT" - }, - "node_modules/cli-cursor": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-3.1.0.tgz", - "integrity": "sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw==", - "license": "MIT", - "dependencies": { - "restore-cursor": "^3.1.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/cli-spinners": { - "version": "2.9.2", - "resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-2.9.2.tgz", - "integrity": "sha512-ywqV+5MmyL4E7ybXgKys4DugZbX0FC6LnwrhjuykIjnK9k8OQacQ7axGKnjDXWNhns0xot3bZI5h55H8yo9cJg==", - "license": "MIT", - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/cli-width": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-3.0.0.tgz", - "integrity": "sha512-FxqpkPPwu1HjuN93Omfm4h8uIanXofW0RxVEW3k5RKx+mJJYSthzNhp32Kzxxy3YAEZ/Dc/EWN1vZRY0+kOhbw==", - "license": "ISC", - "engines": { - "node": ">= 10" - } - }, - "node_modules/clipanion": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/clipanion/-/clipanion-3.2.1.tgz", - "integrity": "sha512-dYFdjLb7y1ajfxQopN05mylEpK9ZX0sO1/RfMXdfmwjlIsPkbh4p7A682x++zFPLDCo1x3p82dtljHf5cW2LKA==", - "license": "MIT", - "workspaces": [ - "website" - ], - "dependencies": { - "typanion": "^3.8.0" - }, - "peerDependencies": { - "typanion": "*" - } - }, - "node_modules/clone": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz", - "integrity": "sha512-JQHZ2QMW6l3aH/j6xCqQThY/9OH4D/9ls34cgkUBiEeocRTU04tHfKPBsUK1PqZCUQM7GiA0IIXJSuXHI64Kbg==", - "license": "MIT", - "engines": { - "node": ">=0.8" - } - }, - "node_modules/color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "license": "MIT", - "dependencies": { - "color-name": "~1.1.4" - }, - "engines": { - "node": ">=7.0.0" - } - }, - "node_modules/color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "license": "MIT" - }, - "node_modules/combined-stream": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", - "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", - "license": "MIT", - "dependencies": { - "delayed-stream": "~1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/commander": { - "version": "8.3.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-8.3.0.tgz", - "integrity": "sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww==", - "license": "MIT", - "engines": { - "node": ">= 12" - } - }, - "node_modules/core-util-is": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", - "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", - "license": "MIT" - }, - "node_modules/data-uri-to-buffer": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-6.0.2.tgz", - "integrity": "sha512-7hvf7/GW8e86rW0ptuwS3OcBGDjIi6SZva7hCyWC0yYry2cOPmLIjXAUHI6DK2HsnwJd9ifmt57i8eV2n4YNpw==", - "license": "MIT", - "engines": { - "node": ">= 14" - } - }, - "node_modules/datadog-metrics": { - "version": "0.9.3", - "resolved": "https://registry.npmjs.org/datadog-metrics/-/datadog-metrics-0.9.3.tgz", - "integrity": "sha512-BVsBX2t+4yA3tHs7DnB5H01cHVNiGJ/bHA8y6JppJDyXG7s2DLm6JaozPGpgsgVGd42Is1CHRG/yMDQpt877Xg==", - "license": "MIT", - "dependencies": { - "debug": "3.1.0", - "dogapi": "2.8.4" - } - }, - "node_modules/datadog-metrics/node_modules/debug": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", - "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", - "license": "MIT", - "dependencies": { - "ms": "2.0.0" - } - }, - "node_modules/datadog-metrics/node_modules/ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", - "license": "MIT" - }, - "node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/decode-named-character-reference": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/decode-named-character-reference/-/decode-named-character-reference-1.3.0.tgz", - "integrity": "sha512-GtpQYB283KrPp6nRw50q3U9/VfOutZOe103qlN7BPP6Ad27xYnOIWv4lPzo8HCAL+mMZofJ9KEy30fq6MfaK6Q==", - "license": "MIT", - "dependencies": { - "character-entities": "^2.0.0" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, - "node_modules/deep-extend": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", - "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", - "license": "MIT", - "engines": { - "node": ">=4.0.0" - } - }, - "node_modules/defaults": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/defaults/-/defaults-1.0.4.tgz", - "integrity": "sha512-eFuaLoy/Rxalv2kr+lqMlUnrDWV+3j4pljOIJgLIhI058IQfWJ7vXhyEIHu+HtC738klGALYxOKDO0bQP3tg8A==", - "license": "MIT", - "dependencies": { - "clone": "^1.0.2" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/degenerator": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/degenerator/-/degenerator-5.0.1.tgz", - "integrity": "sha512-TllpMR/t0M5sqCXfj85i4XaAzxmS5tVA16dqvdkMwGmzI+dXLXnw3J+3Vdv7VKw+ThlTMboK6i9rnZ6Nntj5CQ==", - "license": "MIT", - "dependencies": { - "ast-types": "^0.13.4", - "escodegen": "^2.1.0", - "esprima": "^4.0.1" - }, - "engines": { - "node": ">= 14" - } - }, - "node_modules/delayed-stream": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", - "license": "MIT", - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/dequal": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", - "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/devlop": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/devlop/-/devlop-1.1.0.tgz", - "integrity": "sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==", - "license": "MIT", - "dependencies": { - "dequal": "^2.0.0" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, - "node_modules/dogapi": { - "version": "2.8.4", - "resolved": "https://registry.npmjs.org/dogapi/-/dogapi-2.8.4.tgz", - "integrity": "sha512-065fsvu5dB0o4+ENtLjZILvXMClDNH/yA9H6L8nsdcNiz9l0Hzpn7aQaCOPYXxqyzq4CRPOdwkFXUjDOXfRGbg==", - "license": "MIT", - "dependencies": { - "extend": "^3.0.2", - "json-bigint": "^1.0.0", - "lodash": "^4.17.21", - "minimist": "^1.2.5", - "rc": "^1.2.8" - }, - "bin": { - "dogapi": "bin/dogapi" - } - }, - "node_modules/dunder-proto": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", - "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.1", - "es-errors": "^1.3.0", - "gopd": "^1.2.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "license": "MIT" - }, - "node_modules/entities": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", - "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", - "license": "BSD-2-Clause", - "engines": { - "node": ">=0.12" - }, - "funding": { - "url": "https://github.com/fb55/entities?sponsor=1" - } - }, - "node_modules/es-define-property": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", - "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-errors": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", - "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-object-atoms": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", - "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-set-tostringtag": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", - "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.6", - "has-tostringtag": "^1.0.2", - "hasown": "^2.0.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/escape-string-regexp": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", - "license": "MIT", - "engines": { - "node": ">=0.8.0" - } - }, - "node_modules/escodegen": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.1.0.tgz", - "integrity": "sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==", - "license": "BSD-2-Clause", - "dependencies": { - "esprima": "^4.0.1", - "estraverse": "^5.2.0", - "esutils": "^2.0.2" - }, - "bin": { - "escodegen": "bin/escodegen.js", - "esgenerate": "bin/esgenerate.js" - }, - "engines": { - "node": ">=6.0" - }, - "optionalDependencies": { - "source-map": "~0.6.1" - } - }, - "node_modules/esprima": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", - "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", - "license": "BSD-2-Clause", - "bin": { - "esparse": "bin/esparse.js", - "esvalidate": "bin/esvalidate.js" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/estraverse": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", - "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", - "license": "BSD-2-Clause", - "engines": { - "node": ">=4.0" - } - }, - "node_modules/esutils": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", - "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", - "license": "BSD-2-Clause", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/extend": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", - "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", - "license": "MIT" - }, - "node_modules/fast-deep-equal": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", - "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", - "license": "MIT" - }, "node_modules/fast-glob": { "version": "3.3.3", "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz", @@ -1162,57 +233,6 @@ "node": ">=8.6.0" } }, - "node_modules/fast-uri": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.0.tgz", - "integrity": "sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/fastify" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/fastify" - } - ], - "license": "BSD-3-Clause" - }, - "node_modules/fast-xml-builder": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.1.4.tgz", - "integrity": "sha512-f2jhpN4Eccy0/Uz9csxh3Nu6q4ErKxf0XIsasomfOihuSUa3/xw6w8dnOtCDgEItQFJG8KyXPzQXzcODDrrbOg==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/NaturalIntelligence" - } - ], - "license": "MIT", - "dependencies": { - "path-expression-matcher": "^1.1.3" - } - }, - "node_modules/fast-xml-parser": { - "version": "5.5.9", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.5.9.tgz", - "integrity": "sha512-jldvxr1MC6rtiZKgrFnDSvT8xuH+eJqxqOBThUVjYrxssYTo1avZLGql5l0a0BAERR01CadYzZ83kVEkbyDg+g==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/NaturalIntelligence" - } - ], - "license": "MIT", - "dependencies": { - "fast-xml-builder": "^1.1.4", - "path-expression-matcher": "^1.2.0", - "strnum": "^2.2.2" - }, - "bin": { - "fxparser": "src/cli/cli.js" - } - }, "node_modules/fastq": { "version": "1.20.1", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.1.tgz", @@ -1222,388 +242,69 @@ "reusify": "^1.0.4" } }, - "node_modules/figures": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz", - "integrity": "sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg==", - "license": "MIT", - "dependencies": { - "escape-string-regexp": "^1.0.5" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/fill-range": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", - "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", - "license": "MIT", - "dependencies": { - "to-regex-range": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/follow-redirects": { - "version": "1.15.11", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.11.tgz", - "integrity": "sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ==", - "funding": [ - { - "type": "individual", - "url": "https://github.com/sponsors/RubenVerborgh" - } - ], - "license": "MIT", - "engines": { - "node": ">=4.0" - }, - "peerDependenciesMeta": { - "debug": { - "optional": true - } - } - }, - "node_modules/form-data": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.4.tgz", - "integrity": "sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow==", - "license": "MIT", - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "es-set-tostringtag": "^2.1.0", - "hasown": "^2.0.2", - "mime-types": "^2.1.12" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/function-bind": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", - "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/get-east-asian-width": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/get-east-asian-width/-/get-east-asian-width-1.5.0.tgz", - "integrity": "sha512-CQ+bEO+Tva/qlmw24dCejulK5pMzVnUOFOijVogd3KQs07HnRIgp8TGipvCCRT06xeYEbpbgwaCxglFyiuIcmA==", - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/get-intrinsic": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", - "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.2", - "es-define-property": "^1.0.1", - "es-errors": "^1.3.0", - "es-object-atoms": "^1.1.1", - "function-bind": "^1.1.2", - "get-proto": "^1.0.1", - "gopd": "^1.2.0", - "has-symbols": "^1.1.0", - "hasown": "^2.0.2", - "math-intrinsics": "^1.1.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/get-proto": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", - "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", - "license": "MIT", - "dependencies": { - "dunder-proto": "^1.0.1", - "es-object-atoms": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/get-uri": { - "version": "6.0.5", - "resolved": "https://registry.npmjs.org/get-uri/-/get-uri-6.0.5.tgz", - "integrity": "sha512-b1O07XYq8eRuVzBNgJLstU6FYc1tS6wnMtF1I1D9lE8LxZSOGZ7LhxN54yPP6mGw5f2CkXY2BQUL9Fx41qvcIg==", - "license": "MIT", - "dependencies": { - "basic-ftp": "^5.0.2", - "data-uri-to-buffer": "^6.0.2", - "debug": "^4.3.4" - }, - "engines": { - "node": ">= 14" - } - }, - "node_modules/glob": { - "version": "13.0.6", - "resolved": "https://registry.npmjs.org/glob/-/glob-13.0.6.tgz", - "integrity": "sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw==", - "license": "BlueOak-1.0.0", - "dependencies": { - "minimatch": "^10.2.2", - "minipass": "^7.1.3", - "path-scurry": "^2.0.2" - }, - "engines": { - "node": "18 || 20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/glob-parent": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", - "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", - "license": "ISC", - "dependencies": { - "is-glob": "^4.0.1" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/globby": { - "version": "16.1.1", - "resolved": "https://registry.npmjs.org/globby/-/globby-16.1.1.tgz", - "integrity": "sha512-dW7vl+yiAJSp6aCekaVnVJxurRv7DCOLyXqEG3RYMYUg7AuJ2jCqPkZTA8ooqC2vtnkaMcV5WfFBMuEnTu1OQg==", - "license": "MIT", - "dependencies": { - "@sindresorhus/merge-streams": "^4.0.0", - "fast-glob": "^3.3.3", - "ignore": "^7.0.5", - "is-path-inside": "^4.0.0", - "slash": "^5.1.0", - "unicorn-magic": "^0.4.0" - }, - "engines": { - "node": ">=20" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/gopd": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", - "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/has-flag": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", - "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/has-symbols": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", - "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/has-tostringtag": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", - "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", - "license": "MIT", - "dependencies": { - "has-symbols": "^1.0.3" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/hasown": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", - "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", - "license": "MIT", - "dependencies": { - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/http-proxy-agent": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", - "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", - "license": "MIT", - "dependencies": { - "agent-base": "^7.1.0", - "debug": "^4.3.4" - }, - "engines": { - "node": ">= 14" - } - }, - "node_modules/https-proxy-agent": { - "version": "7.0.6", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", - "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", - "license": "MIT", - "dependencies": { - "agent-base": "^7.1.2", - "debug": "4" - }, - "engines": { - "node": ">= 14" - } - }, - "node_modules/iconv-lite": { - "version": "0.7.2", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.2.tgz", - "integrity": "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==", + "node_modules/fill-range": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", "license": "MIT", "dependencies": { - "safer-buffer": ">= 2.1.2 < 3.0.0" + "to-regex-range": "^5.0.1" }, "engines": { - "node": ">=0.10.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" + "node": ">=8" } }, - "node_modules/ieee754": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", - "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "BSD-3-Clause" - }, - "node_modules/ignore": { - "version": "7.0.5", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.5.tgz", - "integrity": "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg==", + "node_modules/get-east-asian-width": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/get-east-asian-width/-/get-east-asian-width-1.5.0.tgz", + "integrity": "sha512-CQ+bEO+Tva/qlmw24dCejulK5pMzVnUOFOijVogd3KQs07HnRIgp8TGipvCCRT06xeYEbpbgwaCxglFyiuIcmA==", "license": "MIT", "engines": { - "node": ">= 4" + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/immediate": { - "version": "3.0.6", - "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", - "integrity": "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==", - "license": "MIT" - }, - "node_modules/inherits": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "license": "ISC" - }, - "node_modules/ini": { - "version": "1.3.8", - "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", - "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", - "license": "ISC" - }, - "node_modules/inquirer": { - "version": "8.2.7", - "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-8.2.7.tgz", - "integrity": "sha512-UjOaSel/iddGZJ5xP/Eixh6dY1XghiBw4XK13rCCIJcJfyhhoul/7KhLLUGtebEj6GDYM6Vnx/mVsjx2L/mFIA==", - "license": "MIT", + "node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "license": "ISC", "dependencies": { - "@inquirer/external-editor": "^1.0.0", - "ansi-escapes": "^4.2.1", - "chalk": "^4.1.1", - "cli-cursor": "^3.1.0", - "cli-width": "^3.0.0", - "figures": "^3.0.0", - "lodash": "^4.17.21", - "mute-stream": "0.0.8", - "ora": "^5.4.1", - "run-async": "^2.4.0", - "rxjs": "^7.5.5", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0", - "through": "^2.3.6", - "wrap-ansi": "^6.0.1" + "is-glob": "^4.0.1" }, "engines": { - "node": ">=12.0.0" + "node": ">= 6" } }, - "node_modules/inquirer/node_modules/chalk": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", - "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "node_modules/globby": { + "version": "16.1.1", + "resolved": "https://registry.npmjs.org/globby/-/globby-16.1.1.tgz", + "integrity": "sha512-dW7vl+yiAJSp6aCekaVnVJxurRv7DCOLyXqEG3RYMYUg7AuJ2jCqPkZTA8ooqC2vtnkaMcV5WfFBMuEnTu1OQg==", "license": "MIT", "dependencies": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" + "@sindresorhus/merge-streams": "^4.0.0", + "fast-glob": "^3.3.3", + "ignore": "^7.0.5", + "is-path-inside": "^4.0.0", + "slash": "^5.1.0", + "unicorn-magic": "^0.4.0" }, "engines": { - "node": ">=10" + "node": ">=20" }, "funding": { - "url": "https://github.com/chalk/chalk?sponsor=1" + "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/ip-address": { - "version": "10.1.0", - "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.1.0.tgz", - "integrity": "sha512-XXADHxXmvT9+CRxhXg56LJovE+bmWnEWB78LB83VZTprKTmaC5QfruXocxzTZ2Kl0DNwKuBdlIhjL8LeY8Sf8Q==", + "node_modules/ignore": { + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.5.tgz", + "integrity": "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg==", "license": "MIT", "engines": { - "node": ">= 12" + "node": ">= 4" } }, "node_modules/is-alphabetical": { @@ -1640,21 +341,6 @@ "url": "https://github.com/sponsors/wooorm" } }, - "node_modules/is-docker": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-4.0.0.tgz", - "integrity": "sha512-LHE+wROyG/Y/0ZnbktRCoTix2c1RhgWaZraMZ8o1Q7zCh0VSrICJQO5oqIIISrcSBtrXv0o233w1IYwsWCjTzA==", - "license": "MIT", - "bin": { - "is-docker": "cli.js" - }, - "engines": { - "node": ">=20" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", @@ -1664,15 +350,6 @@ "node": ">=0.10.0" } }, - "node_modules/is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, "node_modules/is-glob": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", @@ -1695,15 +372,6 @@ "url": "https://github.com/sponsors/wooorm" } }, - "node_modules/is-interactive": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-interactive/-/is-interactive-1.0.0.tgz", - "integrity": "sha512-2HvIEKRoqS62guEC+qBjpvRubdX910WCMuJTZ+I9yvqKU2/12eSL549HMwtabb4oupdj2sMP50k+XJfB/8JE6w==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, "node_modules/is-number": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", @@ -1725,55 +393,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/is-unicode-supported": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-0.1.0.tgz", - "integrity": "sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==", - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/isarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", - "license": "MIT" - }, - "node_modules/jest-diff": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-30.2.0.tgz", - "integrity": "sha512-dQHFo3Pt4/NLlG5z4PxZ/3yZTZ1C7s9hveiOj+GCN+uT109NC2QgsoVZsVOAvbJ3RgKkvyLGXZV9+piDpWbm6A==", - "license": "MIT", - "dependencies": { - "@jest/diff-sequences": "30.0.1", - "@jest/get-type": "30.1.0", - "chalk": "^4.1.2", - "pretty-format": "30.2.0" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/jest-diff/node_modules/chalk": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", - "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/chalk?sponsor=1" - } - }, "node_modules/js-yaml": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", @@ -1786,21 +405,6 @@ "js-yaml": "bin/js-yaml.js" } }, - "node_modules/json-bigint": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", - "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", - "license": "MIT", - "dependencies": { - "bignumber.js": "^9.0.0" - } - }, - "node_modules/json-schema-traverse": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", - "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", - "license": "MIT" - }, "node_modules/jsonc-parser": { "version": "3.3.1", "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-3.3.1.tgz", @@ -1816,18 +420,6 @@ "node": ">=0.10.0" } }, - "node_modules/jszip": { - "version": "3.10.1", - "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.10.1.tgz", - "integrity": "sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==", - "license": "(MIT OR GPL-3.0-or-later)", - "dependencies": { - "lie": "~3.3.0", - "pako": "~1.0.2", - "readable-stream": "~2.3.6", - "setimmediate": "^1.0.5" - } - }, "node_modules/katex": { "version": "0.16.44", "resolved": "https://registry.npmjs.org/katex/-/katex-0.16.44.tgz", @@ -1844,15 +436,6 @@ "katex": "cli.js" } }, - "node_modules/lie": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz", - "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==", - "license": "MIT", - "dependencies": { - "immediate": "~3.0.5" - } - }, "node_modules/linkify-it": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-5.0.0.tgz", @@ -1862,53 +445,6 @@ "uc.micro": "^2.0.0" } }, - "node_modules/lodash": { - "version": "4.18.1", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz", - "integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==", - "license": "MIT" - }, - "node_modules/log-symbols": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.1.0.tgz", - "integrity": "sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg==", - "license": "MIT", - "dependencies": { - "chalk": "^4.1.0", - "is-unicode-supported": "^0.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/log-symbols/node_modules/chalk": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", - "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/chalk?sponsor=1" - } - }, - "node_modules/lru-cache": { - "version": "11.3.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.3.0.tgz", - "integrity": "sha512-sr8xPKE25m6vJVcrdn6NxtC0fVfuPowbscLypegRgOm0yXSqr5JNHCAY3hnusdJ7HRBW04j6Ip4khvHU778DuQ==", - "license": "BlueOak-1.0.0", - "engines": { - "node": "20 || >=22" - } - }, "node_modules/markdown-it": { "version": "14.1.1", "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-14.1.1.tgz", @@ -2030,15 +566,6 @@ "url": "https://github.com/chalk/strip-ansi?sponsor=1" } }, - "node_modules/math-intrinsics": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", - "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, "node_modules/mdurl": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-2.0.0.tgz", @@ -2552,220 +1079,38 @@ "node_modules/micromark-util-types": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/micromark-util-types/-/micromark-util-types-2.0.2.tgz", - "integrity": "sha512-Yw0ECSpJoViF1qTU4DC6NwtC4aWGt1EkzaQB8KPPyCRR8z9TWeV0HbEFGTO+ZY1wB22zmxnJqhPyTpOVCpeHTA==", - "funding": [ - { - "type": "GitHub Sponsors", - "url": "https://github.com/sponsors/unifiedjs" - }, - { - "type": "OpenCollective", - "url": "https://opencollective.com/unified" - } - ], - "license": "MIT" - }, - "node_modules/micromatch": { - "version": "4.0.8", - "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", - "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", - "license": "MIT", - "dependencies": { - "braces": "^3.0.3", - "picomatch": "^2.3.1" - }, - "engines": { - "node": ">=8.6" - } - }, - "node_modules/mime-db": { - "version": "1.52.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", - "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mime-types": { - "version": "2.1.35", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", - "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", - "license": "MIT", - "dependencies": { - "mime-db": "1.52.0" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mimic-fn": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", - "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/minimatch": { - "version": "10.2.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", - "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", - "license": "BlueOak-1.0.0", - "dependencies": { - "brace-expansion": "^5.0.5" - }, - "engines": { - "node": "18 || 20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/minimist": { - "version": "1.2.8", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", - "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/minipass": { - "version": "7.1.3", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", - "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", - "license": "BlueOak-1.0.0", - "engines": { - "node": ">=16 || 14 >=14.17" - } - }, - "node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" - }, - "node_modules/mute-stream": { - "version": "0.0.8", - "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", - "integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==", - "license": "ISC" - }, - "node_modules/netmask": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/netmask/-/netmask-2.0.2.tgz", - "integrity": "sha512-dBpDMdxv9Irdq66304OLfEmQ9tbNRFnFTuZiLo+bD+r332bBmMJ8GBLXklIXXgxd3+v9+KUnZaUR5PJMa75Gsg==", - "license": "MIT", - "engines": { - "node": ">= 0.4.0" - } - }, - "node_modules/onetime": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", - "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", - "license": "MIT", - "dependencies": { - "mimic-fn": "^2.1.0" - }, - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/ora": { - "version": "5.4.1", - "resolved": "https://registry.npmjs.org/ora/-/ora-5.4.1.tgz", - "integrity": "sha512-5b6Y85tPxZZ7QytO+BQzysW31HJku27cRIlkbAXaNx+BdcVi+LlRFmVXzeF6a7JCwJpyw5c4b+YSVImQIrBpuQ==", - "license": "MIT", - "dependencies": { - "bl": "^4.1.0", - "chalk": "^4.1.0", - "cli-cursor": "^3.1.0", - "cli-spinners": "^2.5.0", - "is-interactive": "^1.0.0", - "is-unicode-supported": "^0.1.0", - "log-symbols": "^4.1.0", - "strip-ansi": "^6.0.0", - "wcwidth": "^1.0.1" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/ora/node_modules/chalk": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", - "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/chalk?sponsor=1" - } - }, - "node_modules/pac-proxy-agent": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-7.2.0.tgz", - "integrity": "sha512-TEB8ESquiLMc0lV8vcd5Ql/JAKAoyzHFXaStwjkzpOpC5Yv+pIzLfHvjTSdf3vpa2bMiUQrg9i6276yn8666aA==", - "license": "MIT", - "dependencies": { - "@tootallnate/quickjs-emscripten": "^0.23.0", - "agent-base": "^7.1.2", - "debug": "^4.3.4", - "get-uri": "^6.0.1", - "http-proxy-agent": "^7.0.0", - "https-proxy-agent": "^7.0.6", - "pac-resolver": "^7.0.1", - "socks-proxy-agent": "^8.0.5" - }, - "engines": { - "node": ">= 14" - } + "integrity": "sha512-Yw0ECSpJoViF1qTU4DC6NwtC4aWGt1EkzaQB8KPPyCRR8z9TWeV0HbEFGTO+ZY1wB22zmxnJqhPyTpOVCpeHTA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" }, - "node_modules/pac-resolver": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/pac-resolver/-/pac-resolver-7.0.1.tgz", - "integrity": "sha512-5NPgf87AT2STgwa2ntRMr45jTKrYBGkVU36yT0ig/n/GMAa3oPqhZfIQ2kMEimReg0+t9kZViDVZ83qfVUlckg==", + "node_modules/micromatch": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", + "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", "license": "MIT", "dependencies": { - "degenerator": "^5.0.0", - "netmask": "^2.0.2" + "braces": "^3.0.3", + "picomatch": "^2.3.1" }, "engines": { - "node": ">= 14" + "node": ">=8.6" } }, - "node_modules/package-manager-detector": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/package-manager-detector/-/package-manager-detector-1.6.0.tgz", - "integrity": "sha512-61A5ThoTiDG/C8s8UMZwSorAGwMJ0ERVGj2OjoW5pAalsNOg15+iQiPzrLJ4jhZ1HJzmC2PIHT2oEiH3R5fzNA==", - "license": "MIT" - }, - "node_modules/packageurl-js": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/packageurl-js/-/packageurl-js-2.0.1.tgz", - "integrity": "sha512-N5ixXjzTy4QDQH0Q9YFjqIWd6zH6936Djpl2m9QNFmDv5Fum8q8BjkpAcHNMzOFE0IwQrFhJWex3AN6kS0OSwg==", + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "license": "MIT" }, - "node_modules/pako": { - "version": "1.0.11", - "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", - "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==", - "license": "(MIT AND Zlib)" - }, "node_modules/parse-entities": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/parse-entities/-/parse-entities-4.0.2.tgz", @@ -2785,37 +1130,6 @@ "url": "https://github.com/sponsors/wooorm" } }, - "node_modules/path-expression-matcher": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/path-expression-matcher/-/path-expression-matcher-1.2.1.tgz", - "integrity": "sha512-d7gQQmLvAKXKXE2GeP9apIGbMYKz88zWdsn/BN2HRWVQsDFdUY36WSLTY0Jvd4HWi7Fb30gQ62oAOzdgJA6fZw==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/NaturalIntelligence" - } - ], - "license": "MIT", - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/path-scurry": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.2.tgz", - "integrity": "sha512-3O/iVVsJAPsOnpwWIeD+d6z/7PmqApyQePUtCndjatj/9I5LylHvt5qluFaBT3I5h3r1ejfR056c+FCv+NnNXg==", - "license": "BlueOak-1.0.0", - "dependencies": { - "lru-cache": "^11.0.0", - "minipass": "^7.1.2" - }, - "engines": { - "node": "18 || 20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/picomatch": { "version": "2.3.2", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", @@ -2843,72 +1157,6 @@ "url": "https://github.com/prettier/prettier?sponsor=1" } }, - "node_modules/pretty-format": { - "version": "30.2.0", - "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-30.2.0.tgz", - "integrity": "sha512-9uBdv/B4EefsuAL+pWqueZyZS2Ba+LxfFeQ9DN14HU4bN8bhaxKdkpjpB6fs9+pSjIBu+FXQHImEg8j/Lw0+vA==", - "license": "MIT", - "dependencies": { - "@jest/schemas": "30.0.5", - "ansi-styles": "^5.2.0", - "react-is": "^18.3.1" - }, - "engines": { - "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" - } - }, - "node_modules/pretty-format/node_modules/ansi-styles": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", - "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/process-nextick-args": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", - "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", - "license": "MIT" - }, - "node_modules/proxy-agent": { - "version": "6.4.0", - "resolved": "https://registry.npmjs.org/proxy-agent/-/proxy-agent-6.4.0.tgz", - "integrity": "sha512-u0piLU+nCOHMgGjRbimiXmA9kM/L9EHh3zL81xCdp7m+Y2pHIsnmbdDoEDoAz5geaonNR6q6+yOPQs6n4T6sBQ==", - "license": "MIT", - "dependencies": { - "agent-base": "^7.0.2", - "debug": "^4.3.4", - "http-proxy-agent": "^7.0.1", - "https-proxy-agent": "^7.0.3", - "lru-cache": "^7.14.1", - "pac-proxy-agent": "^7.0.1", - "proxy-from-env": "^1.1.0", - "socks-proxy-agent": "^8.0.2" - }, - "engines": { - "node": ">= 14" - } - }, - "node_modules/proxy-agent/node_modules/lru-cache": { - "version": "7.18.3", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz", - "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==", - "license": "ISC", - "engines": { - "node": ">=12" - } - }, - "node_modules/proxy-from-env": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", - "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", - "license": "MIT" - }, "node_modules/punycode.js": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode.js/-/punycode.js-2.3.1.tgz", @@ -2938,73 +1186,6 @@ ], "license": "MIT" }, - "node_modules/rc": { - "version": "1.2.8", - "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", - "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", - "license": "(BSD-2-Clause OR MIT OR Apache-2.0)", - "dependencies": { - "deep-extend": "^0.6.0", - "ini": "~1.3.0", - "minimist": "^1.2.0", - "strip-json-comments": "~2.0.1" - }, - "bin": { - "rc": "cli.js" - } - }, - "node_modules/react-is": { - "version": "18.3.1", - "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", - "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", - "license": "MIT" - }, - "node_modules/readable-stream": { - "version": "2.3.8", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", - "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", - "license": "MIT", - "dependencies": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - }, - "node_modules/require-from-string": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", - "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/restore-cursor": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-3.1.0.tgz", - "integrity": "sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA==", - "license": "MIT", - "dependencies": { - "onetime": "^5.1.0", - "signal-exit": "^3.0.2" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/retry": { - "version": "0.13.1", - "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", - "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", - "license": "MIT", - "engines": { - "node": ">= 4" - } - }, "node_modules/reusify": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz", @@ -3015,15 +1196,6 @@ "node": ">=0.10.0" } }, - "node_modules/run-async": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/run-async/-/run-async-2.4.1.tgz", - "integrity": "sha512-tvVnVv01b8c1RrA6Ep7JkStj85Guv/YrMcwqYQnwjsAS2cTmmPGBBjAjpCW7RrSodNSoE2/qg9O4bceNvUuDgQ==", - "license": "MIT", - "engines": { - "node": ">=0.12.0" - } - }, "node_modules/run-parallel": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", @@ -3047,66 +1219,6 @@ "queue-microtask": "^1.2.2" } }, - "node_modules/rxjs": { - "version": "7.8.2", - "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.2.tgz", - "integrity": "sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.1.0" - } - }, - "node_modules/safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", - "license": "MIT" - }, - "node_modules/safer-buffer": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", - "license": "MIT" - }, - "node_modules/semver": { - "version": "7.6.3", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", - "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==", - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/setimmediate": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", - "integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==", - "license": "MIT" - }, - "node_modules/signal-exit": { - "version": "3.0.7", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", - "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", - "license": "ISC" - }, - "node_modules/simple-git": { - "version": "3.33.0", - "resolved": "https://registry.npmjs.org/simple-git/-/simple-git-3.33.0.tgz", - "integrity": "sha512-D4V/tGC2sjsoNhoMybKyGoE+v8A60hRawKQ1iFRA1zwuDgGZCBJ4ByOzZ5J8joBbi4Oam0qiPH+GhzmSBwbJng==", - "license": "MIT", - "dependencies": { - "@kwsites/file-exists": "^1.1.1", - "@kwsites/promise-deferred": "^1.1.1", - "debug": "^4.4.0" - }, - "funding": { - "type": "github", - "url": "https://github.com/steveukx/git-js?sponsor=1" - } - }, "node_modules/slash": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/slash/-/slash-5.1.0.tgz", @@ -3119,16 +1231,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/smart-buffer": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", - "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", - "license": "MIT", - "engines": { - "node": ">= 6.0.0", - "npm": ">= 3.0.0" - } - }, "node_modules/smol-toml": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.6.0.tgz", @@ -3141,162 +1243,6 @@ "url": "https://github.com/sponsors/cyyynthia" } }, - "node_modules/socks": { - "version": "2.8.7", - "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.7.tgz", - "integrity": "sha512-HLpt+uLy/pxB+bum/9DzAgiKS8CX1EvbWxI4zlmgGCExImLdiad2iCwXT5Z4c9c3Eq8rP2318mPW2c+QbtjK8A==", - "license": "MIT", - "dependencies": { - "ip-address": "^10.0.1", - "smart-buffer": "^4.2.0" - }, - "engines": { - "node": ">= 10.0.0", - "npm": ">= 3.0.0" - } - }, - "node_modules/socks-proxy-agent": { - "version": "8.0.5", - "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.5.tgz", - "integrity": "sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw==", - "license": "MIT", - "dependencies": { - "agent-base": "^7.1.2", - "debug": "^4.3.4", - "socks": "^2.8.3" - }, - "engines": { - "node": ">= 14" - } - }, - "node_modules/source-map": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "license": "BSD-3-Clause", - "optional": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "license": "MIT", - "dependencies": { - "safe-buffer": "~5.1.0" - } - }, - "node_modules/string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "license": "MIT", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-json-comments": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", - "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/strnum": { - "version": "2.2.2", - "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.2.2.tgz", - "integrity": "sha512-DnR90I+jtXNSTXWdwrEy9FakW7UX+qUZg28gj5fk2vxxl7uS/3bpI4fjFYVmdK9etptYBPNkpahuQnEwhwECqA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/NaturalIntelligence" - } - ], - "license": "MIT" - }, - "node_modules/supports-color": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", - "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "license": "MIT", - "dependencies": { - "has-flag": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/supports-hyperlinks": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/supports-hyperlinks/-/supports-hyperlinks-2.3.0.tgz", - "integrity": "sha512-RpsAZlpWcDwOPQA22aCH4J0t7L8JmAvsCxfOSEwm7cQs3LshN36QaTkwd70DnBOXDWGssw2eUoc8CaRWT0XunA==", - "license": "MIT", - "dependencies": { - "has-flag": "^4.0.0", - "supports-color": "^7.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/terminal-link": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/terminal-link/-/terminal-link-2.1.1.tgz", - "integrity": "sha512-un0FmiRUQNr5PJqy9kP7c40F5BOfpGlYTrxonDChEZB7pzZxRNp/bt+ymiy9/npwXya9KH99nJ/GXFIiUkYGFQ==", - "license": "MIT", - "dependencies": { - "ansi-escapes": "^4.2.1", - "supports-hyperlinks": "^2.0.0" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/through": { - "version": "2.3.8", - "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", - "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==", - "license": "MIT" - }, - "node_modules/tiny-async-pool": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/tiny-async-pool/-/tiny-async-pool-2.1.0.tgz", - "integrity": "sha512-ltAHPh/9k0STRQqaoUX52NH4ZQYAJz24ZAEwf1Zm+HYg3l9OXTWeqWKyYsHu40wF/F0rxd2N2bk5sLvX2qlSvg==", - "license": "MIT" - }, - "node_modules/tinyexec": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.0.4.tgz", - "integrity": "sha512-u9r3uZC0bdpGOXtlxUIdwf9pkmvhqJdrVCH9fapQtgy/OeTTMZ1nqH7agtvEfmGui6e1XxjcdrlxvxJvc3sMqw==", - "license": "MIT", - "engines": { - "node": ">=18" - } - }, "node_modules/to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", @@ -3309,33 +1255,6 @@ "node": ">=8.0" } }, - "node_modules/tslib": { - "version": "2.8.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", - "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "license": "0BSD" - }, - "node_modules/typanion": { - "version": "3.14.0", - "resolved": "https://registry.npmjs.org/typanion/-/typanion-3.14.0.tgz", - "integrity": "sha512-ZW/lVMRabETuYCd9O9ZvMhAh8GslSqaUjxmK/JLPCh6l73CvLBiuXswj/+7LdnWOgYsQ130FqLzFz5aGT4I3Ug==", - "license": "MIT", - "workspaces": [ - "website" - ] - }, - "node_modules/type-fest": { - "version": "0.21.3", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz", - "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==", - "license": "(MIT OR CC0-1.0)", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/uc.micro": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-2.1.0.tgz", @@ -3353,58 +1272,6 @@ "funding": { "url": "https://github.com/sponsors/sindresorhus" } - }, - "node_modules/upath": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/upath/-/upath-2.0.1.tgz", - "integrity": "sha512-1uEe95xksV1O0CYKXo8vQvN1JEbtJp7lb7C5U9HMsIp6IVwntkH/oNUzyVNQSd4S1sYk2FpSSW44FqMc8qee5w==", - "license": "MIT", - "engines": { - "node": ">=4", - "yarn": "*" - } - }, - "node_modules/util-deprecate": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", - "license": "MIT" - }, - "node_modules/uuid": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", - "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", - "funding": [ - "https://github.com/sponsors/broofa", - "https://github.com/sponsors/ctavan" - ], - "license": "MIT", - "bin": { - "uuid": "dist/bin/uuid" - } - }, - "node_modules/wcwidth": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/wcwidth/-/wcwidth-1.0.1.tgz", - "integrity": "sha512-XHPEwS0q6TaxcvG85+8EYkbiCux2XtWG2mkc47Ng2A77BQu9+DqIOJldST4HgPkuea7dvKSj5VgX3P1d4rW8Tg==", - "license": "MIT", - "dependencies": { - "defaults": "^1.0.3" - } - }, - "node_modules/wrap-ansi": { - "version": "6.2.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", - "integrity": "sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==", - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=8" - } } } } diff --git a/scripts/environment/npm-tools/package.json b/scripts/environment/npm-tools/package.json index a7ec2311e71d2..a40c7f7c3e2ae 100644 --- a/scripts/environment/npm-tools/package.json +++ b/scripts/environment/npm-tools/package.json @@ -2,7 +2,7 @@ "private": true, "description": "Pinned npm tools for CI. Run 'npm ci' to install with exact versions from package-lock.json.", "dependencies": { - "@datadog/datadog-ci": "5.12.0", + "@datadog/datadog-ci": "5.13.0", "markdownlint-cli2": "0.22.0", "prettier": "3.8.1" } From 16458a84ba60660bbdc4258e3245981102da2921 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 14 Apr 2026 11:48:09 -0400 Subject: [PATCH 125/326] chore(website deps): bump follow-redirects from 1.15.11 to 1.16.0 in /website (#25189) chore(website deps): bump follow-redirects in /website Bumps [follow-redirects](https://github.com/follow-redirects/follow-redirects) from 1.15.11 to 1.16.0. - [Release notes](https://github.com/follow-redirects/follow-redirects/releases) - [Commits](https://github.com/follow-redirects/follow-redirects/compare/v1.15.11...v1.16.0) --- updated-dependencies: - dependency-name: follow-redirects dependency-version: 1.16.0 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- website/yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/website/yarn.lock b/website/yarn.lock index 31254815a65f1..c2d1c9ed279dc 100644 --- a/website/yarn.lock +++ b/website/yarn.lock @@ -2383,9 +2383,9 @@ fn.name@1.x.x: integrity sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw== follow-redirects@^1.15.11: - version "1.15.11" - resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.11.tgz#777d73d72a92f8ec4d2e410eb47352a56b8e8340" - integrity sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ== + version "1.16.0" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.16.0.tgz#28474a159d3b9d11ef62050a14ed60e4df6d61bc" + integrity sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw== form-data@^4.0.5: version "4.0.5" From e32661adb9138ff721d882f059f35eaf792d355c Mon Sep 17 00:00:00 2001 From: Peter Ehikhuemen Date: Tue, 14 Apr 2026 18:53:04 -0400 Subject: [PATCH 126/326] feat(aws_s3 sink): add Parquet encoder with schema_file and auto infer schema support (#25156) * feat(codecs): add Parquet encoder with schema_file and schema_mode support * fix broken tests * remove benches stuff * fix the integration tests * Add compression level to compression type * update teh aws_s3.cue file * Minor updates based on my own PR review of szibis initial PR * fix minor issue from batch encoder * fix tests from szibis PR * Fix the tests * Fix cargo fmt and deny issues * fix cargo markdown and license errors * Revert cargo.lock serde_json back to what it was * rebuild cargo.lock, don't want to pull in non needed upgrades * update warning and licenses * revert weird test updates * merge internal_events * cargo fmt fix * Throw a validation error if someone tries to use arrow encoding with amazon s3 sink * update licenses after re-installing dd-rust-license-tool * Short circuit when there's a serialization error instead of just emitting an error * PR comments feedback * emit dropped events metric when there's an error in encoding * put JsonSerializationError under parquet feature * emit ArrowWriterError and dropped events * remove formatting changes in cargo.toml * Apply suggestions from May's code review Co-authored-by: May Lee * remove whitespace updates from lib/codecs/cargo.toml --------- Co-authored-by: May Lee --- .github/actions/spelling/expect.txt | 1 + .gitignore | 7 + Cargo.lock | 64 ++ Cargo.toml | 10 +- LICENSE-3rdparty.csv | 5 + .../aws_s3_parquet_encoding.feature.md | 10 + deny.toml | 1 + lib/codecs/Cargo.toml | 11 +- lib/codecs/src/encoding/encoder.rs | 16 +- lib/codecs/src/encoding/format/arrow.rs | 62 +- lib/codecs/src/encoding/format/mod.rs | 6 + lib/codecs/src/encoding/format/parquet.rs | 943 ++++++++++++++++++ lib/codecs/src/encoding/mod.rs | 4 + lib/codecs/src/encoding/serializer.rs | 30 +- lib/codecs/src/internal_events.rs | 101 +- lib/vector-lib/Cargo.toml | 1 + src/sinks/aws_s3/config.rs | 334 ++++++- src/sinks/aws_s3/integration_tests.rs | 106 ++ src/sinks/aws_s3/sink.rs | 8 +- src/sinks/clickhouse/config.rs | 12 +- .../cue/reference/components/sinks/aws_s3.cue | 129 +++ 21 files changed, 1818 insertions(+), 43 deletions(-) create mode 100644 changelog.d/aws_s3_parquet_encoding.feature.md create mode 100644 lib/codecs/src/encoding/format/parquet.rs diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt index ddd945006f1c4..3656732c51bba 100644 --- a/.github/actions/spelling/expect.txt +++ b/.github/actions/spelling/expect.txt @@ -604,6 +604,7 @@ transitioning Trauring Treemap trialled +Trino Trivago trivy trl diff --git a/.gitignore b/.gitignore index 1963fcca25fcc..a713eeb297f9a 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ cargo-timing*.html target tests/data/wasm/*/target bench_output.txt +rust-analyzer.toml # Python *.pyc @@ -64,3 +65,9 @@ volumes/ # LLM tools copilot-instructions.md + +# local files not checked in +local/ + +# vscode +.vscode/ diff --git a/Cargo.lock b/Cargo.lock index d013bc09432ec..ae4c753529775 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -382,6 +382,7 @@ dependencies = [ "arrow-data", "arrow-schema", "chrono", + "chrono-tz", "half", "hashbrown 0.16.0", "num", @@ -2428,6 +2429,7 @@ dependencies = [ "async-trait", "bytes", "chrono", + "chrono-tz", "csv-core", "derivative", "derive_more 2.1.1", @@ -2440,6 +2442,7 @@ dependencies = [ "metrics", "opentelemetry-proto", "ordered-float 4.6.0", + "parquet", "pin-project", "prost 0.12.6", "prost-reflect", @@ -5525,6 +5528,12 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "integer-encoding" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" + [[package]] name = "inventory" version = "0.3.22" @@ -7568,6 +7577,37 @@ dependencies = [ "windows-link 0.2.0", ] +[[package]] +name = "parquet" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0dbd48ad52d7dccf8ea1b90a3ddbfaea4f69878dd7683e51c507d4bc52b5b27" +dependencies = [ + "ahash 0.8.11", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-ipc", + "arrow-schema", + "arrow-select", + "base64 0.22.1", + "bytes", + "chrono", + "flate2", + "half", + "hashbrown 0.16.0", + "lz4_flex", + "num", + "num-bigint", + "paste", + "seq-macro", + "snap", + "thrift", + "twox-hash", + "zstd 0.13.2", +] + [[package]] name = "parse-size" version = "1.1.0" @@ -7580,6 +7620,12 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13242a5ce97f39a8095d03c8b273e91d09f2690c0b7d69a2af844941115bab24" +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + [[package]] name = "pastey" version = "0.2.1" @@ -9755,6 +9801,12 @@ dependencies = [ "serde_core", ] +[[package]] +name = "seq-macro" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" + [[package]] name = "serde" version = "1.0.228" @@ -11021,6 +11073,17 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "thrift" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" +dependencies = [ + "byteorder", + "integer-encoding", + "ordered-float 2.10.1", +] + [[package]] name = "tikv-jemalloc-sys" version = "0.6.1+5.3.0-1-ge13ca993e8ccb9ba9847cc330696e02839f328f7" @@ -12348,6 +12411,7 @@ dependencies = [ "openssl-probe", "openssl-src", "ordered-float 4.6.0", + "parquet", "pastey", "percent-encoding", "pin-project", diff --git a/Cargo.toml b/Cargo.toml index 84d89c50096f9..6b233580143c8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -204,7 +204,7 @@ tonic-build = { version = "0.11", default-features = false, features = ["transpo tonic-health = { version = "0.11", default-features = false } tonic-reflection = { version = "0.11", default-features = false, features = ["server"] } tracing = { version = "0.1.44", default-features = false } -tracing-subscriber = { version = "0.3.22", default-features = false, features = ["fmt"] } +tracing-subscriber = { version = "0.3.22", default-features = false, features = ["fmt"] } url = { version = "2.5.4", default-features = false, features = ["serde"] } uuid = { version = "1.22.0", features = ["v4", "v7", "serde", "fast-rng"] } vector-config = { path = "lib/vector-config" } @@ -353,6 +353,13 @@ async-compression = { version = "0.4.27", default-features = false, features = [ apache-avro = { version = "0.16.0", default-features = false, optional = true } arrow = { version = "56.2.0", default-features = false, features = ["ipc"], optional = true } arrow-schema = { version = "56.2.0", default-features = false, optional = true } +parquet = { version = "56.2.0", default-features = false, features = [ + "arrow", + "snap", + "zstd", + "lz4", + "flate2-rust_backened", +], optional = true } axum = { version = "0.6.20", default-features = false } base64 = { workspace = true, optional = true } bloomy = { version = "1.2.0", default-features = false, optional = true } @@ -609,6 +616,7 @@ enrichment-tables-memory = ["dep:evmap", "dep:evmap-derive", "dep:thread_local"] # Codecs codecs-arrow = ["dep:arrow", "dep:arrow-schema", "vector-lib/arrow"] +codecs-parquet = ["dep:parquet", "codecs-arrow", "vector-lib/parquet"] codecs-opentelemetry = ["vector-lib/opentelemetry"] codecs-syslog = ["vector-lib/syslog"] diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 9a135f0479f51..ac6f8bc2acb4a 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -382,6 +382,7 @@ inotify,https://github.com/hannobraun/inotify,ISC,"Hanno Braun inout,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers instability,https://github.com/ratatui-org/instability,MIT,"Stephen M. Coakley , Joshka" +integer-encoding,https://github.com/dermesser/integer-encoding-rs,MIT,Lewin Bormann inventory,https://github.com/dtolnay/inventory,MIT OR Apache-2.0,David Tolnay ipconfig,https://github.com/liranringel/ipconfig,MIT OR Apache-2.0,Liran Ringel ipcrypt-rs,https://github.com/jedisct1/rust-ipcrypt2,ISC,Frank Denis @@ -533,8 +534,10 @@ pad,https://github.com/ogham/rust-pad,MIT,Ben S parking,https://github.com/smol-rs/parking,Apache-2.0 OR MIT,"Stjepan Glavina , The Rust Project Developers" parking_lot,https://github.com/Amanieu/parking_lot,MIT OR Apache-2.0,Amanieu d'Antras parking_lot_core,https://github.com/Amanieu/parking_lot,MIT OR Apache-2.0,Amanieu d'Antras +parquet,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow parse-size,https://github.com/kennytm/parse-size,MIT,kennytm passt,https://github.com/kevingimbel/passt,MIT OR Apache-2.0,Kevin Gimbel +paste,https://github.com/dtolnay/paste,MIT OR Apache-2.0,David Tolnay pastey,https://github.com/as1100k/pastey,MIT OR Apache-2.0,"Aditya Kumar , David Tolnay " pbkdf2,https://github.com/RustCrypto/password-hashes/tree/master/pbkdf2,MIT OR Apache-2.0,RustCrypto Developers peeking_take_while,https://github.com/fitzgen/peeking_take_while,MIT OR Apache-2.0,Nick Fitzgerald @@ -674,6 +677,7 @@ secrecy,https://github.com/iqlusioninc/crates/tree/main/secrecy,Apache-2.0 OR MI security-framework,https://github.com/kornelski/rust-security-framework,MIT OR Apache-2.0,"Steven Fackler , Kornel " security-framework-sys,https://github.com/kornelski/rust-security-framework,MIT OR Apache-2.0,"Steven Fackler , Kornel " semver,https://github.com/dtolnay/semver,MIT OR Apache-2.0,David Tolnay +seq-macro,https://github.com/dtolnay/seq-macro,MIT OR Apache-2.0,David Tolnay serde,https://github.com/serde-rs/serde,MIT OR Apache-2.0,"Erick Tryzelaar , David Tolnay " serde-aux,https://github.com/iddm/serde-aux,MIT,Victor Polevoy serde-toml-merge,https://github.com/jdrouet/serde-toml-merge,MIT,Jeremie Drouet @@ -756,6 +760,7 @@ terminal_size,https://github.com/eminence/terminal-size,MIT OR Apache-2.0,Andrew thiserror,https://github.com/dtolnay/thiserror,MIT OR Apache-2.0,David Tolnay thiserror-impl,https://github.com/dtolnay/thiserror,MIT OR Apache-2.0,David Tolnay thread_local,https://github.com/Amanieu/thread_local-rs,MIT OR Apache-2.0,Amanieu d'Antras +thrift,https://github.com/apache/thrift/tree/master/lib/rs,Apache-2.0,Apache Thrift Developers tikv-jemalloc-sys,https://github.com/tikv/jemallocator,MIT OR Apache-2.0,"Alex Crichton , Gonzalo Brito Gadeschi , The TiKV Project Developers" tikv-jemallocator,https://github.com/tikv/jemallocator,MIT OR Apache-2.0,"Alex Crichton , Gonzalo Brito Gadeschi , Simon Sapin , Steven Fackler , The TiKV Project Developers" time,https://github.com/time-rs/time,MIT OR Apache-2.0,"Jacob Pratt , Time contributors" diff --git a/changelog.d/aws_s3_parquet_encoding.feature.md b/changelog.d/aws_s3_parquet_encoding.feature.md new file mode 100644 index 0000000000000..afaa01cc70c26 --- /dev/null +++ b/changelog.d/aws_s3_parquet_encoding.feature.md @@ -0,0 +1,10 @@ +Add Apache Parquet batch encoding support for the `aws_s3` sink with flexible schema definitions. + +Events can now be encoded as Parquet columnar files with multiple schema input options: + +- **Native Parquet schema** — automatically generate a schema or supply `.schema` file +- **Configurable compression** - (Snappy, ZSTD, GZIP, LZ4, None). + +Enable the `codecs-parquet` feature and configure `batch_encoding` with `codec = "parquet"` in the S3 sink configuration. + +authors: szibis petere-datadog diff --git a/deny.toml b/deny.toml index b8e0be77f41c8..c19b0a23f5ee6 100644 --- a/deny.toml +++ b/deny.toml @@ -44,4 +44,5 @@ ignore = [ { id = "RUSTSEC-2024-0388", reason = "derivative is unmaintained (https://github.com/vectordotdev/vector/issues/24940)" }, { id = "RUSTSEC-2025-0134", reason = "rustls-pemfile is unmaintained - unpatched crate (https://github.com/bytebeamio/rumqtt/issues/1010) & tonic/reqwest upgrade (https://github.com/vectordotdev/vector/issues/19179)" }, { id = "RUSTSEC-2026-0049", reason = "rustls-webpki 0.102 is vulnerable - tonic upgrade (https://github.com/vectordotdev/vector/issues/19179)" }, + { id = "RUSTSEC-2024-0436", reason = "paste crate is unmaintained - transitive dependency via parquet v56.2.0, no safe upgrade available" }, ] diff --git a/lib/codecs/Cargo.toml b/lib/codecs/Cargo.toml index 90a5fd2780109..b9043b7223d8a 100644 --- a/lib/codecs/Cargo.toml +++ b/lib/codecs/Cargo.toml @@ -15,9 +15,17 @@ path = "tests/bin/generate-avro-fixtures.rs" [dependencies] apache-avro = { version = "0.20.0", default-features = false } arrow = { version = "56.2.0", default-features = false, features = ["ipc", "json"], optional = true } +parquet = { version = "56.2.0", default-features = false, features = [ + "arrow", + "snap", + "zstd", + "lz4", + "flate2-rust_backened", +], optional = true } async-trait.workspace = true bytes.workspace = true chrono.workspace = true +chrono-tz.workspace = true rust_decimal.workspace = true csv-core = { version = "0.1.13", default-features = false } derivative.workspace = true @@ -69,7 +77,8 @@ uuid.workspace = true vrl.workspace = true [features] -arrow = ["dep:arrow"] +arrow = ["dep:arrow", "arrow/chrono-tz"] +parquet = ["dep:parquet", "arrow"] opentelemetry = ["dep:opentelemetry-proto"] syslog = ["dep:syslog_loose", "dep:strum", "dep:derive_more", "dep:serde-aux", "dep:toml"] test = [] diff --git a/lib/codecs/src/encoding/encoder.rs b/lib/codecs/src/encoding/encoder.rs index 4924dd05447b1..b3ca9a00216ec 100644 --- a/lib/codecs/src/encoding/encoder.rs +++ b/lib/codecs/src/encoding/encoder.rs @@ -5,6 +5,8 @@ use vector_core::event::Event; #[cfg(feature = "arrow")] use crate::encoding::ArrowStreamSerializer; +#[cfg(feature = "parquet")] +use crate::encoding::ParquetSerializer; use crate::{ encoding::{Error, Framer, Serializer}, internal_events::{EncoderFramingError, EncoderSerializeError}, @@ -16,6 +18,9 @@ pub enum BatchSerializer { /// Arrow IPC stream format serializer. #[cfg(feature = "arrow")] Arrow(ArrowStreamSerializer), + /// Parquet format serializer. + #[cfg(feature = "parquet")] + Parquet(Box), } /// An encoder that encodes batches of events. @@ -36,10 +41,12 @@ impl BatchEncoder { } /// Get the HTTP content type. - #[cfg(feature = "arrow")] + #[cfg(any(feature = "arrow", feature = "parquet"))] pub const fn content_type(&self) -> &'static str { match &self.serializer { BatchSerializer::Arrow(_) => "application/vnd.apache.arrow.stream", + #[cfg(feature = "parquet")] + BatchSerializer::Parquet(_) => "application/vnd.apache.parquet", } } } @@ -63,6 +70,11 @@ impl tokio_util::codec::Encoder> for BatchEncoder { } }) } + #[cfg(feature = "parquet")] + BatchSerializer::Parquet(serializer) => serializer + .encode(events, buffer) + .map_err(Error::SerializingError), + #[allow(unreachable_patterns)] _ => unreachable!("BatchSerializer cannot be constructed without encode()"), } } @@ -74,7 +86,7 @@ pub enum EncoderKind { /// Uses framing to encode individual events Framed(Box>), /// Encodes events in batches without framing - #[cfg(feature = "arrow")] + #[cfg(any(feature = "arrow", feature = "parquet"))] Batch(BatchEncoder), } diff --git a/lib/codecs/src/encoding/format/arrow.rs b/lib/codecs/src/encoding/format/arrow.rs index 236d6292d375e..b8c86239ddb6a 100644 --- a/lib/codecs/src/encoding/format/arrow.rs +++ b/lib/codecs/src/encoding/format/arrow.rs @@ -6,6 +6,7 @@ use arrow::{ datatypes::{DataType, Field, Fields, Schema, SchemaRef}, + error::ArrowError, ipc::writer::StreamWriter, json::reader::ReaderBuilder, record_batch::RecordBatch, @@ -203,7 +204,13 @@ pub fn encode_events_to_arrow_ipc_stream( return Err(ArrowEncodingError::NoEvents); } - let record_batch = build_record_batch(schema, events)?; + let json_values = vector_log_events_to_json_values(events).map_err(|e| { + ArrowEncodingError::RecordBatchCreation { + source: ArrowError::JsonError(e.to_string()), + } + })?; + + let record_batch = build_record_batch(schema, &json_values)?; let mut buffer = BytesMut::new().writer(); let mut writer = @@ -271,7 +278,7 @@ fn make_field_nullable(field: &Field) -> Result { /// Find non-nullable schema fields that are missing or null in any of the given events. pub fn find_null_non_nullable_fields<'a>( schema: &'a Schema, - values: &[&vrl::value::Value], + values: &[serde_json::Value], ) -> Vec<&'a str> { schema .fields() @@ -282,38 +289,41 @@ pub fn find_null_non_nullable_fields<'a>( value .as_object() .and_then(|map| map.get(field.name().as_str())) - .is_none_or(vrl::value::Value::is_null) + .is_none_or(serde_json::Value::is_null) }) }) .map(|field| field.name().as_str()) .collect() } -/// Build an Arrow RecordBatch from a slice of events using the provided schema. -fn build_record_batch( - schema: SchemaRef, +pub(crate) fn vector_log_events_to_json_values( events: &[Event], -) -> Result { - let values: Vec<_> = events +) -> Result, serde_json::Error> { + events .iter() .filter_map(Event::maybe_as_log) - .map(|log| log.value()) - .collect(); + .map(serde_json::to_value) + .collect() +} +/// Build an Arrow RecordBatch from a slice of events using the provided schema. +pub(crate) fn build_record_batch( + schema: SchemaRef, + values: &[serde_json::Value], +) -> Result { if values.is_empty() { return Err(ArrowEncodingError::NoEvents); } - let missing = find_null_non_nullable_fields(&schema, &values); + let missing = find_null_non_nullable_fields(&schema, values); if !missing.is_empty() { - for field_name in &missing { - let error: vector_common::Error = Box::new(ArrowEncodingError::NullConstraint { - field_name: field_name.to_string(), - }); - vector_common::internal_event::emit( - crate::internal_events::EncoderNullConstraintError { error: &error }, - ); - } + let error: vector_common::Error = Box::new(ArrowEncodingError::NullConstraint { + field_name: missing.join(", "), + }); + vector_common::internal_event::emit(crate::internal_events::EncoderNullConstraintError { + error: &error, + count: values.len(), + }); return Err(ArrowEncodingError::NullConstraint { field_name: missing.join(", "), }); @@ -323,7 +333,7 @@ fn build_record_batch( .build_decoder() .context(RecordBatchCreationSnafu)?; - decoder.serialize(&values).context(ArrowJsonDecodeSnafu)?; + decoder.serialize(values).context(ArrowJsonDecodeSnafu)?; decoder .flush() @@ -885,8 +895,10 @@ mod tests { ("a", Value::Bytes("val".into())), ("b", Value::Integer(42)), ]); - let value = event.as_log().value(); - let missing = find_null_non_nullable_fields(&schema, &[value]); + let missing = find_null_non_nullable_fields( + &schema, + &vector_log_events_to_json_values(&[event]).unwrap(), + ); assert!( missing.is_empty(), "Expected no missing fields, got: {missing:?}" @@ -898,8 +910,10 @@ mod tests { let schema = Schema::new(vec![Field::new("a", DataType::Utf8, false)]); let event = create_event(vec![("a", Value::Null)]); - let value = event.as_log().value(); - let missing = find_null_non_nullable_fields(&schema, &[value]); + let missing = find_null_non_nullable_fields( + &schema, + &vector_log_events_to_json_values(&[event]).unwrap(), + ); assert_eq!(missing, vec!["a"]); } } diff --git a/lib/codecs/src/encoding/format/mod.rs b/lib/codecs/src/encoding/format/mod.rs index 85bc094b26947..f760ea5c8dd83 100644 --- a/lib/codecs/src/encoding/format/mod.rs +++ b/lib/codecs/src/encoding/format/mod.rs @@ -16,6 +16,8 @@ mod native; mod native_json; #[cfg(feature = "opentelemetry")] mod otlp; +#[cfg(feature = "parquet")] +mod parquet; mod protobuf; mod raw_message; #[cfg(feature = "syslog")] @@ -24,6 +26,10 @@ mod text; use std::fmt::Debug; +#[cfg(feature = "parquet")] +pub use self::parquet::{ + ParquetCompression, ParquetSchemaMode, ParquetSerializer, ParquetSerializerConfig, +}; #[cfg(feature = "arrow")] pub use arrow::{ ArrowEncodingError, ArrowStreamSerializer, ArrowStreamSerializerConfig, SchemaProvider, diff --git a/lib/codecs/src/encoding/format/parquet.rs b/lib/codecs/src/encoding/format/parquet.rs new file mode 100644 index 0000000000000..7349119371a5b --- /dev/null +++ b/lib/codecs/src/encoding/format/parquet.rs @@ -0,0 +1,943 @@ +//! Parquet batch format codec for batched event encoding +//! +//! Provides Apache Parquet format encoding with schema file support and auto-inference. +//! Reuses the Arrow record batch building logic from the Arrow IPC codec, +//! then writes the batch as a complete Parquet file using `ArrowWriter`. + +use std::collections::HashSet; +use std::path::PathBuf; +use std::sync::Arc; + +use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; +use arrow::error::ArrowError; +use arrow::json::reader::infer_json_schema_from_iterator; +use arrow::record_batch::RecordBatch; +use bytes::{BufMut, BytesMut}; +use derivative::Derivative; +use parquet::arrow::ArrowWriter; +use parquet::basic::ZstdLevel; +use parquet::basic::{Compression as ParquetCodecCompression, GzipLevel}; +use parquet::file::properties::WriterProperties; +use std::io::{Error, ErrorKind}; +use tracing::warn; +use vector_common::internal_event::{ + ComponentEventsDropped, Count, InternalEventHandle, Registered, UNINTENTIONAL, emit, register, +}; +use vector_config::configurable_component; +use vector_core::event::Event; + +use super::arrow::{ArrowEncodingError, build_record_batch}; +use crate::encoding::format::arrow::vector_log_events_to_json_values; +use crate::internal_events::{ArrowWriterError, JsonSerializationError, SchemaGenerationError}; + +type EventsDroppedError = ComponentEventsDropped<'static, UNINTENTIONAL>; + +/// Compression algorithm and optional level for archive objects. +#[configurable_component] +#[derive(Default, Copy, Clone, Debug, PartialEq)] +#[serde(tag = "algorithm", rename_all = "snake_case")] +pub enum ParquetCompression { + /// Zstd compression. Level must be between 1 and 21. + Zstd { + /// Compression level (1–21). This is the range Vector currently supports; higher values compress more but are slower. + #[configurable(validation(range(min = 1, max = 21)))] + level: u8, + }, + /// Gzip compression. Level must be between 1 and 9. + Gzip { + /// Compression level (1–9). This is the range Vector currently supports; higher values compress more but are slower. + #[configurable(validation(range(min = 1, max = 9)))] + level: u8, + }, + + /// Snappy compression (no level). + #[default] + Snappy, + + /// LZ4 raw compression + Lz4, + + /// No compression + None, +} + +impl TryFrom for ParquetCodecCompression { + type Error = parquet::errors::ParquetError; + fn try_from( + value: ParquetCompression, + ) -> Result { + match value { + ParquetCompression::None => Ok(ParquetCodecCompression::UNCOMPRESSED), + ParquetCompression::Snappy => Ok(ParquetCodecCompression::SNAPPY), + ParquetCompression::Zstd { level } => Ok(ParquetCodecCompression::ZSTD( + ZstdLevel::try_new(level.into())?, + )), + ParquetCompression::Gzip { level } => Ok(ParquetCodecCompression::GZIP( + GzipLevel::try_new(level.into())?, + )), + ParquetCompression::Lz4 => Ok(ParquetCodecCompression::LZ4_RAW), + } + } +} + +/// Schema handling mode. +#[configurable_component] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ParquetSchemaMode { + /// Missing fields become null. Extra fields are silently dropped. + #[default] + Relaxed, + /// Missing fields become null. Extra fields cause an error. + Strict, + /// Auto infer schema based on the batch. No schema file needed. + AutoInfer, +} + +/// Configuration for the Parquet serializer. +/// +/// Encodes events as Apache Parquet columnar files, optimized for analytical queries +/// via Athena, Trino, Spark, and other columnar query engines. +/// +/// Either `schema_file` must be provided, or `schema_mode` must be set to `auto_infer`. +#[configurable_component] +#[derive(Clone, Debug, Default)] +pub struct ParquetSerializerConfig { + /// Path to a native Parquet schema file (`.schema`). + /// + /// Required unless `schema_mode` is `auto_infer`. The file must contain a valid + /// Parquet message type definition. + #[serde(default)] + pub schema_file: Option, + + /// Compression codec applied per column page inside the Parquet file. + #[serde(default)] + #[configurable(derived)] + pub compression: ParquetCompression, + + /// Controls how events with fields not present in the schema are handled. + #[serde(default)] + #[configurable(derived)] + pub schema_mode: ParquetSchemaMode, +} + +impl ParquetSerializerConfig { + /// Resolve the Arrow schema from the configured schema source. + fn resolve_schema(&self) -> Result> { + if self.schema_mode == ParquetSchemaMode::AutoInfer { + return Ok(Schema::empty()); + } + + let path = self + .schema_file + .as_ref() + .ok_or("schema_file is required unless schema_mode is auto_infer")?; + + let content = read_schema_file(path, "schema_file")?; + let parquet_type = parquet::schema::parser::parse_message_type(&content) + .map_err(|e| format!("Failed to parse Parquet schema: {e}"))?; + let schema_desc = parquet::schema::types::SchemaDescriptor::new(Arc::new(parquet_type)); + let arrow_schema = parquet::arrow::parquet_to_arrow_schema(&schema_desc, None) + .map_err(|e| format!("Failed to convert Parquet schema to Arrow: {e}"))?; + Ok(arrow_schema) + } + + /// The data type of events that are accepted by `ParquetSerializer`. + pub fn input_type(&self) -> vector_core::config::DataType { + vector_core::config::DataType::Log + } + + /// The schema required by the serializer. + pub fn schema_requirement(&self) -> vector_core::schema::Requirement { + vector_core::schema::Requirement::empty() + } +} + +fn read_schema_file( + path: &std::path::Path, + field_name: &str, +) -> Result> { + const MAX_SCHEMA_FILE_SIZE: u64 = 10 * 1024 * 1024; // 10 MB + let display = path.display(); + let metadata = std::fs::metadata(path) + .map_err(|e| format!("Failed to read {field_name} '{display}': {e}"))?; + if metadata.len() > MAX_SCHEMA_FILE_SIZE { + return Err(format!( + "{field_name} '{display}' is too large ({} bytes, max {MAX_SCHEMA_FILE_SIZE})", + metadata.len() + ) + .into()); + } + std::fs::read_to_string(path) + .map_err(|e| format!("Failed to read {field_name} '{display}': {e}").into()) +} + +/// Check the resolved Arrow schema for data types unsupported by the JSON-based +/// encode path (`arrow::json::reader::ReaderBuilder`). Binary variants are +/// accepted by Parquet/Arrow at the schema level but the JSON decoder rejects +/// them at runtime, so we fail fast here at config time. +fn reject_unsupported_arrow_types( + schema: &Schema, +) -> Result<(), Box> { + fn check_field(field: &Field, path: &str, bad: &mut Vec) { + let name = if path.is_empty() { + field.name().to_string() + } else { + format!("{path}.{}", field.name()) + }; + match field.data_type() { + DataType::Binary | DataType::LargeBinary | DataType::FixedSizeBinary(_) => { + bad.push(format!("'{name}' ({:?})", field.data_type())); + } + DataType::Struct(fields) => { + for f in fields { + check_field(f, &name, bad); + } + } + DataType::List(inner) | DataType::LargeList(inner) => { + check_field(inner, &name, bad); + } + DataType::Map(entries_field, _) => { + if let DataType::Struct(kv) = entries_field.data_type() { + for f in kv { + check_field(f, &name, bad); + } + } + } + _ => {} + } + } + + let mut bad = Vec::new(); + for field in schema.fields() { + check_field(field, "", &mut bad); + } + if !bad.is_empty() { + return Err(format!( + "Schema contains binary field(s) unsupported by the JSON-based Arrow encoder: {}. \ + Use Utf8 for base64/hex-encoded data instead.", + bad.join(", ") + ) + .into()); + } + Ok(()) +} + +/// Parquet batch serializer. +#[derive(Derivative)] +#[derivative(Debug, Clone)] +pub struct ParquetSerializer { + schema: SchemaRef, + writer_props: Arc, + schema_mode: ParquetSchemaMode, + /// Pre-built set of schema field names for O(1) strict-mode lookups. + schema_field_names: HashSet, + + #[derivative(Debug = "ignore")] + events_dropped_handle: Registered, +} + +impl ParquetSerializer { + /// Create a new `ParquetSerializer` from the given configuration. + pub fn new( + config: ParquetSerializerConfig, + ) -> Result> { + let schema = config.resolve_schema()?; + reject_unsupported_arrow_types(&schema)?; + let schema_ref = SchemaRef::new(schema); + + let schema_field_names = schema_ref + .fields() + .iter() + .map(|f| f.name().clone()) + .collect::>(); + + let writer_props = Arc::new( + WriterProperties::builder() + .set_compression(config.compression.try_into()?) + .build(), + ); + + Ok(Self { + schema: schema_ref, + writer_props, + schema_mode: config.schema_mode, + schema_field_names, + events_dropped_handle: register(EventsDroppedError::from( + "Events could not be serialized to parquet", + )), + }) + } + + /// Returns the MIME content type for Parquet data. + pub const fn content_type(&self) -> &'static str { + "application/vnd.apache.parquet" + } + + /// Writes `record_batch` into `buffer` as a complete Parquet file. + /// + /// On failure, emits an [`ArrowWriterError`] internal event (which also + /// increments `component_errors_total` and emits the events-dropped metric) + /// before returning the error. + fn write_record_batch( + &self, + record_batch: &RecordBatch, + buffer: &mut BytesMut, + event_count: usize, + ) -> Result<(), parquet::errors::ParquetError> { + let mut writer = ArrowWriter::try_new( + buffer.writer(), + Arc::clone(record_batch.schema_ref()), + Some((*self.writer_props).clone()), + ) + .inspect_err(|e| { + emit(ArrowWriterError { + error: e, + batch_count: event_count, + }); + })?; + + writer.write(record_batch).inspect_err(|e| { + emit(ArrowWriterError { + error: e, + batch_count: event_count, + }); + })?; + + writer.close().inspect_err(|e| { + emit(ArrowWriterError { + error: e, + batch_count: event_count, + }); + })?; + + Ok(()) + } +} + +impl tokio_util::codec::Encoder> for ParquetSerializer { + type Error = vector_common::Error; + + fn encode(&mut self, events: Vec, buffer: &mut BytesMut) -> Result<(), Self::Error> { + if events.is_empty() { + return Ok(()); + } + + let json_values = match vector_log_events_to_json_values(&events) { + Ok(values) => values, + Err(e) => { + emit(JsonSerializationError { + error: &e, + batch_count: events.len(), + }); + return Err(Box::new(e)); + } + }; + + let non_log_count = events.len() - json_values.len(); + + if non_log_count > 0 { + warn!( + message = "Non-log events dropped by Parquet encoder ", + %non_log_count, + internal_log_rate_secs = 10, + ); + self.events_dropped_handle.emit(Count(non_log_count)) + } + + if json_values.is_empty() { + return Ok(()); + } + + match self.schema_mode { + // In strict mode, check for extra top-level fields not in the schema. + ParquetSchemaMode::Strict => { + for event in &events { + if let Some(log) = event.maybe_as_log() + && let Some(object_map) = log.as_map() + { + for top_level in object_map.keys() { + if !self.schema_field_names.contains(top_level.as_str()) { + self.events_dropped_handle.emit(Count(events.len())); + return Err(Box::new(ArrowEncodingError::SchemaFetchError { + message: format!( + "Strict schema mode: event contains field '{top_level}' not in schema", + ), + })); + } + } + } + } + } + ParquetSchemaMode::AutoInfer => { + let schema = ParquetSchemaGenerator::infer_schema(&json_values)?; + self.schema = Arc::new(ParquetSchemaGenerator::try_normalize_schema( + &events, schema, + )); + } + ParquetSchemaMode::Relaxed => {} + } + + let record_batch = + build_record_batch(Arc::clone(&self.schema), &json_values).map_err(Box::new)?; + + self.write_record_batch(&record_batch, buffer, json_values.len()) + .map_err(Box::new)?; + + Ok(()) + } +} + +pub struct ParquetSchemaGenerator {} + +impl ParquetSchemaGenerator { + pub fn infer_schema(events: &[serde_json::Value]) -> Result { + let schema = infer_json_schema_from_iterator(events.iter().map(Ok::<_, ArrowError>)) + .map_err(|e| { + emit(SchemaGenerationError { + error: &e, + batch_count: events.len(), + }); + Error::new(ErrorKind::InvalidData, e.to_string()) + })?; + + Ok(schema) + } + + /// Attempt to modify schema to set timestamp fields as Timestamp instead of Utf8. + /// Only works for top-level fields. + fn try_normalize_schema(events: &[Event], schema: Schema) -> Schema { + let mut ts_seen: HashSet = HashSet::new(); + let mut non_ts_seen: HashSet = HashSet::new(); + + for event in events.iter().filter_map(Event::maybe_as_log) { + if let Some(object_map) = event.as_map() { + for (path, value) in object_map { + if value.is_timestamp() { + ts_seen.insert(path.to_string()); + } else if !value.is_null() { + non_ts_seen.insert(path.to_string()); + } + } + } + } + + let new_fields: Vec = schema + .fields() + .iter() + .map(|f| { + if ts_seen.contains(f.name()) && !non_ts_seen.contains(f.name()) { + Field::new( + f.name(), + DataType::Timestamp( + arrow::datatypes::TimeUnit::Microsecond, + Some("UTC".into()), + ), + f.is_nullable(), + ) + } else { + f.as_ref().clone() + } + }) + .collect(); + + Schema::new_with_metadata(new_fields, schema.metadata().clone()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use bytes::Bytes; + use parquet::file::reader::{FileReader, SerializedFileReader}; + use parquet::record::reader::RowIter; + use tokio_util::codec::Encoder; + use vector_core::event::LogEvent; + + fn create_event(fields: Vec<(&str, V)>) -> Event + where + V: Into, + { + let mut log = LogEvent::default(); + for (key, value) in fields { + log.insert(key, value.into()); + } + Event::Log(log) + } + + fn assert_parquet_magic(data: &[u8]) { + assert!(data.len() >= 4, "Output too short to be valid Parquet"); + assert_eq!(&data[..4], b"PAR1", "Missing Parquet magic bytes"); + } + + fn parquet_row_count(data: &[u8]) -> usize { + let reader = + SerializedFileReader::new(Bytes::copy_from_slice(data)).expect("Invalid Parquet file"); + let iter = RowIter::from_file_into(Box::new(reader)); + iter.count() + } + + fn parquet_column_names(data: &[u8]) -> Vec { + let reader = + SerializedFileReader::new(Bytes::copy_from_slice(data)).expect("Invalid Parquet file"); + let schema = reader.metadata().file_metadata().schema_descr(); + schema + .columns() + .iter() + .map(|c| c.name().to_string()) + .collect() + } + + fn parse_timestamp(s: &str) -> chrono::DateTime { + chrono::DateTime::parse_from_rfc3339(s) + .expect("invalid test timestamp") + .with_timezone(&chrono::Utc) + } + + fn demo_log_event( + message: &str, + timestamp: chrono::DateTime, + status_code: i64, + response_time_secs: f64, + ) -> Event { + use vector_core::event::Value; + let mut log = LogEvent::default(); + log.insert("host", "localhost"); + log.insert("message", message); + log.insert("service", "vector"); + log.insert("source_type", "demo_logs"); + log.insert("timestamp", Value::Timestamp(timestamp)); + log.insert("random_time", Value::Timestamp(timestamp)); + log.insert("status_code", Value::Integer(status_code)); + log.insert("response_time_secs", response_time_secs); + Event::Log(log) + } + + fn sample_events() -> Vec { + const EVENTS: [(&str, &str, i64, f64); 5] = [ + ( + "GET /api/v1/health HTTP/1.1", + "2026-03-05T20:49:08.037194Z", + 200, + 0.037, + ), + ( + "POST /api/v1/ingest HTTP/1.1", + "2026-03-05T20:49:09.038051Z", + 201, + 0.013, + ), + ( + "GET /metrics HTTP/1.1", + "2026-03-05T20:49:10.036612Z", + 200, + 0.022, + ), + ( + "DELETE /api/v1/resource HTTP/1.1", + "2026-03-05T20:49:11.537131Z", + 404, + 0.005, + ), + ( + "PATCH /api/v1/config HTTP/1.1", + "2026-03-05T20:49:12.037491Z", + 500, + 0.091, + ), + ]; + EVENTS + .iter() + .map(|(msg, ts, status, rt)| demo_log_event(msg, parse_timestamp(ts), *status, *rt)) + .collect() + } + + fn encode_autoinfer_and_read_schema( + events: Vec, + ) -> (arrow::datatypes::SchemaRef, usize) { + use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; + + let mut serializer = ParquetSerializer::new(ParquetSerializerConfig { + schema_mode: ParquetSchemaMode::AutoInfer, + ..Default::default() + }) + .expect("AutoInfer serializer should be created without a static schema"); + + let mut buffer = BytesMut::new(); + serializer + .encode(events, &mut buffer) + .expect("encoding should succeed"); + + let data = buffer.freeze(); + assert_parquet_magic(&data); + + let builder = ParquetRecordBatchReaderBuilder::try_new(data) + .expect("should build ParquetRecordBatchReaderBuilder"); + let schema = builder.schema().clone(); + let num_rows: usize = builder + .build() + .expect("should build reader") + .map(|b| b.expect("batch read error").num_rows()) + .sum(); + (schema, num_rows) + } + + /// Write a temporary Parquet schema file and return its path. + /// + /// `name` must be unique per test to avoid parallel-test races on the same file. + fn write_temp_schema(name: &str, content: &str) -> std::path::PathBuf { + use std::io::Write; + let path = std::env::temp_dir().join(format!( + "vector_parquet_test_{}_{}.schema", + std::process::id(), + name, + )); + let mut f = std::fs::File::create(&path).expect("Failed to create schema file"); + write!(f, "{content}").expect("Failed to write schema"); + path + } + + // ── AutoInfer mode ─────────────────────────────────────────────────────── + + #[test] + fn encode_input_produces_parquet_output() { + let events = sample_events(); + let n_events = events.len(); + let (schema, num_rows) = encode_autoinfer_and_read_schema(events); + + assert_eq!(num_rows, n_events, "row count should match event count"); + + for field_name in &["timestamp", "random_time"] { + let field = schema + .field_with_name(field_name) + .unwrap_or_else(|_| panic!("field '{field_name}' should exist in schema")); + assert!( + matches!( + field.data_type(), + DataType::Timestamp(arrow::datatypes::TimeUnit::Microsecond, Some(tz)) if tz.as_ref() == "UTC" + ), + "'{field_name}' should be Timestamp(Microsecond, UTC), got {:?}", + field.data_type() + ); + } + + let status_field = schema + .field_with_name("status_code") + .expect("status_code field should exist"); + assert_eq!(status_field.data_type(), &DataType::Int64); + + let rt_field = schema + .field_with_name("response_time_secs") + .expect("response_time_secs field should exist"); + assert_eq!(rt_field.data_type(), &DataType::Float64); + + for field_name in &["host", "message", "service", "source_type"] { + let field = schema + .field_with_name(field_name) + .unwrap_or_else(|_| panic!("field '{field_name}' should exist in schema")); + assert_eq!(field.data_type(), &DataType::Utf8); + } + } + + #[test] + fn test_parquet_empty_events() { + let mut serializer = ParquetSerializer::new(ParquetSerializerConfig { + schema_mode: ParquetSchemaMode::AutoInfer, + ..Default::default() + }) + .expect("AutoInfer serializer should succeed"); + + let events: Vec = vec![]; + let mut buffer = BytesMut::new(); + serializer + .encode(events, &mut buffer) + .expect("Empty events should succeed"); + + assert!(buffer.is_empty(), "Buffer should be empty for empty events"); + } + + #[test] + fn test_parquet_compression_variants() { + let events = vec![create_event(vec![("msg", "hello world")])]; + + let compressions = vec![ + ParquetCompression::None, + ParquetCompression::Snappy, + ParquetCompression::Zstd { level: 1 }, + ParquetCompression::Gzip { level: 1 }, + ParquetCompression::Lz4, + ]; + + for compression in compressions { + let mut serializer = ParquetSerializer::new(ParquetSerializerConfig { + schema_mode: ParquetSchemaMode::AutoInfer, + compression, + ..Default::default() + }) + .expect("Failed to create serializer"); + + let mut buffer = BytesMut::new(); + serializer + .encode(events.clone(), &mut buffer) + .unwrap_or_else(|e| panic!("Encoding with {:?} failed: {}", compression, e)); + + let data = buffer.freeze(); + assert_parquet_magic(&data); + assert_eq!( + parquet_row_count(&data), + 1, + "Wrong row count for {:?}", + compression + ); + } + } + + #[test] + fn test_parquet_output_has_footer() { + let mut serializer = ParquetSerializer::new(ParquetSerializerConfig { + schema_mode: ParquetSchemaMode::AutoInfer, + ..Default::default() + }) + .expect("AutoInfer serializer should succeed"); + + let events = vec![create_event(vec![("msg", "test")])]; + let mut buffer = BytesMut::new(); + serializer.encode(events, &mut buffer).unwrap(); + + let data = buffer.freeze(); + let len = data.len(); + assert!(len >= 8, "Parquet output too short"); + assert_eq!( + &data[len - 4..], + b"PAR1", + "Parquet footer magic bytes missing" + ); + } + + #[test] + fn test_writer_props_arc_shared() { + let serializer = ParquetSerializer::new(ParquetSerializerConfig { + schema_mode: ParquetSchemaMode::AutoInfer, + ..Default::default() + }) + .expect("AutoInfer serializer should succeed"); + let cloned = serializer.clone(); + + assert_eq!(Arc::strong_count(&serializer.writer_props), 2); + drop(cloned); + assert_eq!(Arc::strong_count(&serializer.writer_props), 1); + } + + #[test] + fn test_mixed_log_and_non_log_events() { + use vector_core::event::{Metric, MetricKind, MetricValue}; + + let mut serializer = ParquetSerializer::new(ParquetSerializerConfig { + schema_mode: ParquetSchemaMode::AutoInfer, + ..Default::default() + }) + .expect("AutoInfer serializer should succeed"); + + let metric = Metric::new( + "cpu.usage", + MetricKind::Absolute, + MetricValue::Gauge { value: 42.0 }, + ); + let events = vec![ + create_event(vec![("msg", "hello")]), + Event::Metric(metric), + create_event(vec![("msg", "world")]), + ]; + + let mut buffer = BytesMut::new(); + serializer + .encode(events, &mut buffer) + .expect("Mixed batch should succeed (non-log events dropped)"); + + assert_parquet_magic(&buffer); + assert_eq!(parquet_row_count(&buffer), 2); + } + + // ── Schema file mode ───────────────────────────────────────────────────── + + #[test] + fn test_parquet_schema_file() { + let schema_path = write_temp_schema( + "schema_file", + "message logs {\n required binary name (STRING);\n optional int64 age;\n}", + ); + + let config: ParquetSerializerConfig = serde_json::from_value(serde_json::json!({ + "schema_file": schema_path.to_str().unwrap() + })) + .expect("Config should deserialize"); + + let mut serializer = + ParquetSerializer::new(config).expect("Should create serializer from schema file"); + + let mut log = LogEvent::default(); + log.insert("name", "alice"); + + let mut buffer = BytesMut::new(); + serializer + .encode(vec![Event::Log(log)], &mut buffer) + .expect("Encoding with schema file should succeed"); + + let data = buffer.freeze(); + assert_parquet_magic(&data); + assert_eq!(parquet_row_count(&data), 1); + + let columns = parquet_column_names(&data); + assert_eq!(columns, vec!["name", "age"]); + } + + #[test] + fn test_parquet_schema_file_not_found_error() { + let config: ParquetSerializerConfig = serde_json::from_value(serde_json::json!({ + "schema_file": "/nonexistent/path/schema.parquet" + })) + .expect("Config should deserialize"); + + let result = ParquetSerializer::new(config); + assert!(result.is_err(), "Missing schema file should error"); + assert!( + result.unwrap_err().to_string().contains("Failed to read"), + "Error should mention file read failure" + ); + } + + #[test] + fn test_parquet_schema_file_invalid_syntax_error() { + let schema_path = write_temp_schema( + "invalid_syntax", + "this is not valid parquet schema syntax !!!", + ); + + let config: ParquetSerializerConfig = serde_json::from_value(serde_json::json!({ + "schema_file": schema_path.to_str().unwrap() + })) + .expect("Config should deserialize"); + + let result = ParquetSerializer::new(config); + assert!(result.is_err(), "Invalid Parquet schema should error"); + assert!( + result + .unwrap_err() + .to_string() + .contains("Failed to parse Parquet schema"), + "Error should mention parsing failure" + ); + } + + #[test] + fn test_parquet_no_schema_error() { + let config = ParquetSerializerConfig::default(); + let result = ParquetSerializer::new(config); + assert!( + result.is_err(), + "Should fail without schema_file or auto_infer" + ); + } + + // ── Schema mode: strict / relaxed ──────────────────────────────────────── + + #[test] + fn test_parquet_strict_mode_rejects_extra_fields() { + let schema_path = write_temp_schema( + "strict_rejects", + "message logs {\n required binary name (STRING);\n}", + ); + + let mut serializer = ParquetSerializer::new(ParquetSerializerConfig { + schema_file: Some(schema_path), + schema_mode: ParquetSchemaMode::Strict, + ..Default::default() + }) + .expect("Failed to create strict serializer"); + + let events = vec![create_event(vec![("name", "alice"), ("city", "paris")])]; + let mut buffer = BytesMut::new(); + let result = serializer.encode(events, &mut buffer); + assert!(result.is_err(), "Strict mode should reject extra fields"); + assert!(result.unwrap_err().to_string().contains("city")); + } + + #[test] + fn test_parquet_strict_mode_allows_schema_fields() { + let schema_path = write_temp_schema( + "strict_allows", + "message logs {\n required binary name (STRING);\n required binary level (STRING);\n}", + ); + + let mut serializer = ParquetSerializer::new(ParquetSerializerConfig { + schema_file: Some(schema_path), + schema_mode: ParquetSchemaMode::Strict, + ..Default::default() + }) + .expect("Failed to create strict serializer"); + + let mut log = LogEvent::default(); + log.insert("name", "test"); + log.insert("level", "info"); + + let mut buffer = BytesMut::new(); + assert!( + serializer + .encode(vec![Event::Log(log)], &mut buffer) + .is_ok(), + "Strict mode should pass when all fields match schema" + ); + } + + #[test] + fn test_parquet_relaxed_mode_drops_extra_fields() { + let schema_path = write_temp_schema( + "relaxed_drops", + "message logs {\n required binary name (STRING);\n}", + ); + + let mut serializer = ParquetSerializer::new(ParquetSerializerConfig { + schema_file: Some(schema_path), + schema_mode: ParquetSchemaMode::Relaxed, + ..Default::default() + }) + .expect("Failed to create relaxed serializer"); + + let events = vec![create_event(vec![("name", "alice"), ("city", "paris")])]; + let mut buffer = BytesMut::new(); + serializer + .encode(events, &mut buffer) + .expect("Relaxed mode should drop extra fields silently"); + + let data = buffer.freeze(); + assert_parquet_magic(&data); + assert_eq!(parquet_row_count(&data), 1); + let columns = parquet_column_names(&data); + assert_eq!(columns, vec!["name"]); + } + + #[test] + fn test_parquet_schema_file_binary_without_string_annotation_rejected() { + // Native Parquet "binary" without (STRING) annotation resolves to Arrow Binary, + // which is rejected at config time. + let schema_path = write_temp_schema( + "binary_rejected", + "message logs {\n required binary name (STRING);\n optional binary raw_data;\n}", + ); + + let config: ParquetSerializerConfig = serde_json::from_value(serde_json::json!({ + "schema_file": schema_path.to_str().unwrap() + })) + .expect("Config should deserialize"); + + let result = ParquetSerializer::new(config); + assert!( + result.is_err(), + "Parquet binary without STRING annotation should be rejected" + ); + assert!( + result.unwrap_err().to_string().contains("raw_data"), + "Error should name the offending field" + ); + } +} diff --git a/lib/codecs/src/encoding/mod.rs b/lib/codecs/src/encoding/mod.rs index 25f88406fbae9..a9f698f79153d 100644 --- a/lib/codecs/src/encoding/mod.rs +++ b/lib/codecs/src/encoding/mod.rs @@ -27,6 +27,10 @@ pub use format::{ }; #[cfg(feature = "opentelemetry")] pub use format::{OtlpSerializer, OtlpSerializerConfig}; +#[cfg(feature = "parquet")] +pub use format::{ + ParquetCompression, ParquetSchemaMode, ParquetSerializer, ParquetSerializerConfig, +}; #[cfg(feature = "syslog")] pub use format::{SyslogSerializer, SyslogSerializerConfig}; pub use framing::{ diff --git a/lib/codecs/src/encoding/serializer.rs b/lib/codecs/src/encoding/serializer.rs index 536f836ad8163..c6081e3e2f274 100644 --- a/lib/codecs/src/encoding/serializer.rs +++ b/lib/codecs/src/encoding/serializer.rs @@ -8,6 +8,8 @@ use vector_core::{config::DataType, event::Event, schema}; use super::format::{ArrowStreamSerializer, ArrowStreamSerializerConfig}; #[cfg(feature = "opentelemetry")] use super::format::{OtlpSerializer, OtlpSerializerConfig}; +#[cfg(feature = "parquet")] +use super::format::{ParquetSerializer, ParquetSerializerConfig}; #[cfg(feature = "syslog")] use super::format::{SyslogSerializer, SyslogSerializerConfig}; use super::{ @@ -160,17 +162,29 @@ pub enum BatchSerializerConfig { #[cfg(feature = "arrow")] #[serde(rename = "arrow_stream")] ArrowStream(ArrowStreamSerializerConfig), + /// Encodes events in [Apache Parquet][apache_parquet] columnar format. + /// + /// [apache_parquet]: https://parquet.apache.org/ + #[cfg(feature = "parquet")] + #[serde(rename = "parquet")] + Parquet(ParquetSerializerConfig), } -#[cfg(feature = "arrow")] +#[cfg(any(feature = "arrow", feature = "parquet"))] impl BatchSerializerConfig { - /// Build the `ArrowStreamSerializer` from this configuration. - pub fn build( + /// Build the batch serializer from this configuration. + pub fn build_batch_serializer( &self, - ) -> Result> { + ) -> Result> { match self { BatchSerializerConfig::ArrowStream(arrow_config) => { - Ok(ArrowStreamSerializer::new(arrow_config.clone())?) + let serializer = ArrowStreamSerializer::new(arrow_config.clone())?; + Ok(super::BatchSerializer::Arrow(serializer)) + } + #[cfg(feature = "parquet")] + BatchSerializerConfig::Parquet(parquet_config) => { + let serializer = ParquetSerializer::new(parquet_config.clone())?; + Ok(super::BatchSerializer::Parquet(Box::new(serializer))) } } } @@ -178,14 +192,20 @@ impl BatchSerializerConfig { /// The data type of events that are accepted by this batch serializer. pub fn input_type(&self) -> DataType { match self { + #[cfg(feature = "arrow")] BatchSerializerConfig::ArrowStream(arrow_config) => arrow_config.input_type(), + #[cfg(feature = "parquet")] + BatchSerializerConfig::Parquet(parquet_config) => parquet_config.input_type(), } } /// The schema required by the batch serializer. pub fn schema_requirement(&self) -> schema::Requirement { match self { + #[cfg(feature = "arrow")] BatchSerializerConfig::ArrowStream(arrow_config) => arrow_config.schema_requirement(), + #[cfg(feature = "parquet")] + BatchSerializerConfig::Parquet(parquet_config) => parquet_config.schema_requirement(), } } } diff --git a/lib/codecs/src/internal_events.rs b/lib/codecs/src/internal_events.rs index 134fee16ecf8b..0ed24719dd8bc 100644 --- a/lib/codecs/src/internal_events.rs +++ b/lib/codecs/src/internal_events.rs @@ -157,6 +157,8 @@ impl InternalEvent for EncoderWriteError<'_, E> { pub struct EncoderNullConstraintError<'a> { /// The schema constraint error that occurred. pub error: &'a vector_common::Error, + /// The number of events dropped due to the constraint violation. + pub count: usize, } #[cfg(feature = "arrow")] @@ -178,7 +180,104 @@ impl InternalEvent for EncoderNullConstraintError<'_> { ) .increment(1); emit(ComponentEventsDropped:: { - count: 1, + count: self.count, + reason: CONSTRAINT_REASON, + }); + } +} + +#[cfg(feature = "parquet")] +#[derive(NamedInternalEvent)] +pub(crate) struct SchemaGenerationError<'a> { + pub error: &'a arrow::error::ArrowError, + pub batch_count: usize, +} + +#[cfg(feature = "parquet")] +impl InternalEvent for SchemaGenerationError<'_> { + fn emit(self) { + const REASON: &str = "Could not generate schema for batched events"; + error!( + message = REASON, + error = %self.error, + error_code = "parquet_schema_generation_failed", + error_type = error_type::ENCODER_FAILED, + stage = error_stage::SENDING, + internal_log_rate_limit = false, + ); + counter!( + "component_errors_total", + "error_code" => "parquet_schema_generation_failed", + "error_type" => error_type::ENCODER_FAILED, + "stage" => error_stage::SENDING, + ) + .increment(1); + emit(ComponentEventsDropped:: { + count: self.batch_count, + reason: REASON, + }); + } +} + +#[cfg(feature = "parquet")] +#[derive(NamedInternalEvent)] +pub(crate) struct ArrowWriterError<'a> { + pub error: &'a parquet::errors::ParquetError, + pub batch_count: usize, +} + +#[cfg(feature = "parquet")] +impl InternalEvent for ArrowWriterError<'_> { + fn emit(self) { + const REASON: &str = "Failed to write record batch with ArrowWriter."; + error!( + message = REASON, + error = %self.error, + error_code = "parquet_arrow_writer_failed", + error_type = error_type::ENCODER_FAILED, + stage = error_stage::SENDING, + internal_log_rate_limit = false, + ); + counter!( + "component_errors_total", + "error_code" => "parquet_arrow_writer_failed", + "error_type" => error_type::ENCODER_FAILED, + "stage" => error_stage::SENDING, + ) + .increment(1); + emit(ComponentEventsDropped:: { + count: self.batch_count, + reason: REASON, + }); + } +} + +#[cfg(feature = "parquet")] +#[derive(NamedInternalEvent)] +pub(crate) struct JsonSerializationError<'a> { + pub error: &'a serde_json::Error, + pub batch_count: usize, +} + +#[cfg(feature = "parquet")] +impl InternalEvent for JsonSerializationError<'_> { + fn emit(self) { + const CONSTRAINT_REASON: &str = "Could not serialize event to JSON."; + error!( + message = CONSTRAINT_REASON, + error = %self.error, + error_type = error_type::ENCODER_FAILED, + stage = error_stage::SENDING, + internal_log_rate_limit = true, + ); + counter!( + "component_errors_total", + "error_type" => error_type::ENCODER_FAILED, + "stage" => error_stage::SENDING, + ) + .increment(1); + emit(ComponentEventsDropped:: { + count: self.batch_count, reason: CONSTRAINT_REASON, }); } diff --git a/lib/vector-lib/Cargo.toml b/lib/vector-lib/Cargo.toml index 6539c82dfaa80..6f42af4dbb627 100644 --- a/lib/vector-lib/Cargo.toml +++ b/lib/vector-lib/Cargo.toml @@ -27,6 +27,7 @@ vrl = { workspace = true, optional = true } allocation-tracing = ["vector-top?/allocation-tracing"] api-client = ["dep:vector-api-client"] arrow = ["codecs/arrow"] +parquet = ["codecs/parquet"] api = ["vector-tap/api"] file-source = ["dep:file-source", "dep:file-source-common"] lua = ["vector-core/lua"] diff --git a/src/sinks/aws_s3/config.rs b/src/sinks/aws_s3/config.rs index 08ba92245d65c..61b82105a1211 100644 --- a/src/sinks/aws_s3/config.rs +++ b/src/sinks/aws_s3/config.rs @@ -1,9 +1,13 @@ use aws_sdk_s3::Client as S3Client; use tower::ServiceBuilder; +#[cfg(feature = "codecs-parquet")] +use vector_lib::codecs::BatchEncoder; +#[cfg(feature = "codecs-parquet")] +use vector_lib::codecs::encoding::BatchSerializerConfig; use vector_lib::{ TimeZone, codecs::{ - TextSerializerConfig, + EncoderKind, TextSerializerConfig, encoding::{Framer, FramingConfig}, }, configurable::configurable_component, @@ -105,6 +109,16 @@ pub struct S3SinkConfig { #[serde(flatten)] pub encoding: EncodingConfigWithFraming, + /// Batch encoding configuration for columnar formats. + /// + /// When set, events are encoded together as a batch in a columnar format (e.g., Parquet) + /// instead of the standard per-event framing-based encoding. The columnar format handles + /// its own internal compression, so the top-level `compression` setting is bypassed. + #[cfg(feature = "codecs-parquet")] + #[configurable(derived)] + #[serde(default)] + pub batch_encoding: Option, + /// Compression configuration. /// /// All compression algorithms use the default compression level unless otherwise specified. @@ -176,6 +190,8 @@ impl GenerateConfig for S3SinkConfig { options: S3Options::default(), region: RegionOrEndpoint::default(), encoding: (None::, TextSerializerConfig::default()).into(), + #[cfg(feature = "codecs-parquet")] + batch_encoding: None, compression: Compression::gzip_default(), batch: BatchConfig::default(), request: TowerRequestConfig::default(), @@ -201,6 +217,10 @@ impl SinkConfig for S3SinkConfig { } fn input(&self) -> Input { + #[cfg(feature = "codecs-parquet")] + if let Some(batch_config) = &self.batch_encoding { + return Input::new(batch_config.input_type()); + } Input::new(self.encoding.config().1.input_type()) } @@ -246,8 +266,62 @@ impl S3SinkConfig { let partitioner = S3KeyPartitioner::new(key_prefix, ssekms_key_id, None); let transformer = self.encoding.transformer(); + + // When batch_encoding is configured (e.g., Parquet), use batch mode + // with internal compression and appropriate file extension. + #[cfg(feature = "codecs-parquet")] + if let Some(batch_config) = &self.batch_encoding { + if !matches!(batch_config, BatchSerializerConfig::Parquet(_)) { + return Err( + "batch_encoding only supports encoding with parquet format for amazon s3 sink" + .into(), + ); + } + + let batch_serializer = batch_config.build_batch_serializer()?; + let batch_encoder = BatchEncoder::new(batch_serializer); + + // Auto-detect Content-Type from batch format. Users can still + // override via `options.content_type`; we only set it when unset. + let mut api_options = self.options.clone(); + if api_options.content_type.is_none() { + api_options.content_type = Some(batch_encoder.content_type().to_string()); + } + + let encoder = EncoderKind::Batch(batch_encoder); + + // Auto-detect file extension from batch format + let filename_extension = + self.filename_extension + .clone() + .or_else(|| match batch_config { + BatchSerializerConfig::Parquet(_) => Some("parquet".to_string()), + #[allow(unreachable_patterns)] + _ => None, + }); + + if self.compression != Compression::None { + warn!("Top level compression setting ignored when batch_encoding set to parquet.") + } + + let request_options = S3RequestOptions { + bucket: self.bucket.clone(), + api_options, + filename_extension, + filename_time_format: self.filename_time_format.clone(), + filename_append_uuid: self.filename_append_uuid, + encoder: (transformer, encoder), + // Batch formats handle their own compression internally + compression: Compression::None, + filename_tz_offset: offset, + }; + + let sink = S3Sink::new(service, request_options, partitioner, batch_settings); + return Ok(VectorSink::from_event_streamsink(sink)); + } + let (framer, serializer) = self.encoding.build(SinkType::MessageBased)?; - let encoder = Encoder::::new(framer, serializer); + let encoder = EncoderKind::Framed(Box::new(Encoder::::new(framer, serializer))); let request_options = S3RequestOptions { bucket: self.bucket.clone(), @@ -289,4 +363,260 @@ mod tests { fn generate_config() { crate::test_util::test_generate_config::(); } + + /// Correct TOML shape: `batch_encoding.codec = "parquet"` with `schema_mode = "auto_infer"`. + #[cfg(feature = "codecs-parquet")] + #[test] + fn parquet_batch_encoding_correct_toml_shape() { + let config: S3SinkConfig = toml::from_str( + r#" + bucket = "test-bucket" + compression = "none" + + [encoding] + codec = "text" + + [batch_encoding] + schema_mode = "auto_infer" + codec = "parquet" + + [batch_encoding.compression] + algorithm = "snappy" + + "#, + ) + .expect("correct batch_encoding shape should parse"); + + let batch_enc = config + .batch_encoding + .expect("batch_encoding should be Some"); + match batch_enc { + vector_lib::codecs::encoding::BatchSerializerConfig::Parquet(ref p) => { + use vector_lib::codecs::encoding::format::{ParquetCompression, ParquetSchemaMode}; + assert_eq!(p.schema_mode, ParquetSchemaMode::AutoInfer); + assert_eq!(p.compression, ParquetCompression::Snappy); + } + #[allow(unreachable_patterns)] + _ => panic!("expected Parquet variant"), + } + } + + /// Content-Type must be auto-detected as `application/vnd.apache.parquet` + /// when `batch_encoding` is set and `content_type` is not explicitly provided. + #[cfg(feature = "codecs-parquet")] + #[test] + fn parquet_content_type_auto_detected() { + use vector_lib::codecs::encoding::format::{ + ParquetCompression, ParquetSchemaMode, ParquetSerializerConfig, + }; + + use crate::sinks::s3_common::config::S3Options; + use crate::sinks::util::{BatchConfig, BulkSizeBasedDefaultBatchSettings, Compression}; + use vector_lib::codecs::TextSerializerConfig; + use vector_lib::codecs::encoding::{BatchSerializerConfig, FramingConfig}; + + let parquet_config = ParquetSerializerConfig { + schema_mode: ParquetSchemaMode::AutoInfer, + compression: ParquetCompression::Snappy, + ..Default::default() + }; + + let config = S3SinkConfig { + bucket: "test".to_string(), + key_prefix: super::default_key_prefix(), + filename_time_format: super::default_filename_time_format(), + filename_append_uuid: true, + filename_extension: None, + options: S3Options::default(), + region: crate::aws::RegionOrEndpoint::with_both("us-east-1", "http://localhost:4566"), + encoding: (None::, TextSerializerConfig::default()).into(), + batch_encoding: Some(BatchSerializerConfig::Parquet(parquet_config)), + compression: Compression::None, + batch: BatchConfig::::default(), + request: Default::default(), + tls: Default::default(), + auth: Default::default(), + acknowledgements: Default::default(), + timezone: Default::default(), + force_path_style: true, + retry_strategy: Default::default(), + }; + + let batch_config = config.batch_encoding.as_ref().unwrap(); + let batch_serializer = batch_config.build_batch_serializer().unwrap(); + let batch_encoder = vector_lib::codecs::BatchEncoder::new(batch_serializer); + + let mut api_options = config.options.clone(); + if api_options.content_type.is_none() { + api_options.content_type = Some(batch_encoder.content_type().to_string()); + } + + assert_eq!( + api_options.content_type.as_deref(), + Some("application/vnd.apache.parquet"), + "Content-Type must be auto-detected for Parquet" + ); + } + + /// When user explicitly sets `content_type`, the auto-detection must not override it. + #[cfg(feature = "codecs-parquet")] + #[test] + fn parquet_content_type_user_override_preserved() { + let config: S3SinkConfig = toml::from_str( + r#" + bucket = "test-bucket" + compression = "none" + content_type = "application/octet-stream" + + [encoding] + codec = "text" + + [batch_encoding] + codec = "parquet" + schema_mode = "auto_infer" + + [batch_encoding.compression] + algorithm = "gzip" + level = 9 + "#, + ) + .unwrap(); + + let batch_config = config.batch_encoding.as_ref().unwrap(); + let batch_serializer = batch_config.build_batch_serializer().unwrap(); + let batch_encoder = vector_lib::codecs::BatchEncoder::new(batch_serializer); + + let mut api_options = config.options.clone(); + if api_options.content_type.is_none() { + api_options.content_type = Some(batch_encoder.content_type().to_string()); + } + + assert_eq!( + api_options.content_type.as_deref(), + Some("application/octet-stream"), + "User-specified Content-Type must not be overridden" + ); + } + + /// Parquet filename extension defaults to `.parquet` when not explicitly set. + #[cfg(feature = "codecs-parquet")] + #[test] + fn parquet_filename_extension_defaults_to_parquet() { + let config: S3SinkConfig = toml::from_str( + r#" + bucket = "test-bucket" + compression = "none" + + [encoding] + codec = "text" + + [batch_encoding] + codec = "parquet" + schema_mode = "auto_infer" + "#, + ) + .unwrap(); + + assert!( + config.filename_extension.is_none(), + "fixture must not set filename_extension" + ); + + let batch_config = config.batch_encoding.as_ref().unwrap(); + let extension = config + .filename_extension + .clone() + .or_else(|| match batch_config { + vector_lib::codecs::encoding::BatchSerializerConfig::Parquet(_) => { + Some("parquet".to_string()) + } + #[allow(unreachable_patterns)] + _ => None, + }); + + assert_eq!(extension.as_deref(), Some("parquet")); + } + + /// Explicit filename_extension overrides the `.parquet` default. + #[cfg(feature = "codecs-parquet")] + #[test] + fn parquet_filename_extension_user_override() { + let config: S3SinkConfig = toml::from_str( + r#" + bucket = "test-bucket" + compression = "none" + filename_extension = "pq" + + [encoding] + codec = "text" + + [batch_encoding] + codec = "parquet" + schema_mode = "auto_infer" + "#, + ) + .unwrap(); + + assert_eq!(config.filename_extension.as_deref(), Some("pq")); + } + + /// `schema_mode` defaults to `relaxed` when not specified. + #[cfg(feature = "codecs-parquet")] + #[test] + fn parquet_schema_mode_defaults_to_relaxed() { + use vector_lib::codecs::encoding::format::ParquetSchemaMode; + + let config: S3SinkConfig = toml::from_str( + r#" + bucket = "test-bucket" + compression = "none" + + [encoding] + codec = "text" + + [batch_encoding] + codec = "parquet" + "#, + ) + .unwrap(); + + match config.batch_encoding.unwrap() { + vector_lib::codecs::encoding::BatchSerializerConfig::Parquet(p) => { + assert_eq!(p.schema_mode, ParquetSchemaMode::Relaxed); + } + #[allow(unreachable_patterns)] + _ => panic!("expected Parquet variant"), + } + } + + /// Explicit `schema_mode = "strict"` is correctly parsed. + #[cfg(feature = "codecs-parquet")] + #[test] + fn parquet_schema_mode_strict_parsed() { + use vector_lib::codecs::encoding::format::ParquetSchemaMode; + + let config: S3SinkConfig = toml::from_str( + r#" + bucket = "test-bucket" + compression = "none" + + [encoding] + codec = "text" + + [batch_encoding] + codec = "parquet" + schema_mode = "strict" + schema_file = "tmp/something.schema" + "#, + ) + .unwrap(); + + match config.batch_encoding.unwrap() { + vector_lib::codecs::encoding::BatchSerializerConfig::Parquet(p) => { + assert_eq!(p.schema_mode, ParquetSchemaMode::Strict); + } + #[allow(unreachable_patterns)] + _ => panic!("expected Parquet variant"), + } + } } diff --git a/src/sinks/aws_s3/integration_tests.rs b/src/sinks/aws_s3/integration_tests.rs index 31907d70b5c0f..ff2e0e5a5146b 100644 --- a/src/sinks/aws_s3/integration_tests.rs +++ b/src/sinks/aws_s3/integration_tests.rs @@ -19,6 +19,8 @@ use flate2::read::MultiGzDecoder; use futures::{Stream, stream}; use similar_asserts::assert_eq; use tokio_stream::StreamExt; +#[cfg(feature = "codecs-parquet")] +use vector_lib::codecs::encoding::BatchSerializerConfig; use vector_lib::{ codecs::{TextSerializerConfig, encoding::FramingConfig}, config::proxy::ProxyConfig, @@ -433,6 +435,8 @@ async fn s3_flush_on_exhaustion() { options: S3Options::default(), region: RegionOrEndpoint::with_both("us-east-1", s3_address()), encoding: (None::, TextSerializerConfig::default()).into(), + #[cfg(feature = "codecs-parquet")] + batch_encoding: None, compression: Compression::None, batch, request: TowerRequestConfig::default(), @@ -490,6 +494,106 @@ async fn s3_flush_on_exhaustion() { assert_eq!(lines, response_lines); // if all events are received, and lines.len() < batch size, then a flush was performed. } +#[cfg(feature = "codecs-parquet")] +#[tokio::test] +async fn s3_parquet_insert_message() { + use vector_lib::codecs::encoding::format::{ + ParquetCompression, ParquetSchemaMode, ParquetSerializerConfig, + }; + + let cx = SinkContext::default(); + let bucket = uuid::Uuid::new_v4().to_string(); + create_bucket(&bucket, false).await; + + let parquet_config = ParquetSerializerConfig { + schema_mode: ParquetSchemaMode::AutoInfer, + compression: ParquetCompression::Snappy, + ..Default::default() + }; + + let mut batch = BatchConfig::default(); + batch.max_events = Some(100); + batch.timeout_secs = Some(5.0); + + let config = S3SinkConfig { + bucket: bucket.to_string(), + key_prefix: random_string(10) + "/date=%F", + filename_time_format: default_filename_time_format(), + filename_append_uuid: true, + filename_extension: None, + options: S3Options::default(), + region: RegionOrEndpoint::with_both("us-east-1", s3_address()), + encoding: (None::, TextSerializerConfig::default()).into(), + batch_encoding: Some(BatchSerializerConfig::Parquet(parquet_config)), + compression: Compression::None, + batch, + request: TowerRequestConfig::default(), + tls: Default::default(), + auth: Default::default(), + acknowledgements: Default::default(), + timezone: Default::default(), + force_path_style: true, + retry_strategy: Default::default(), + }; + + let prefix = config.key_prefix.clone(); + let service = config.create_service(&cx.globals.proxy).await.unwrap(); + let sink = config.build_processor(service, cx).unwrap(); + + let (batch_notifier, receiver) = BatchNotifier::new_with_receiver(); + let events: Vec = (0..10) + .map(|i| { + let mut log = LogEvent::from(format!("message_{}", i)); + log.insert("host", format!("host_{}", i % 3)); + Event::from(log).with_batch_notifier(&batch_notifier) + }) + .collect(); + + drop(batch_notifier); + run_and_assert_sink_compliance(sink, stream::iter(events), &AWS_SINK_TAGS).await; + assert_eq!(receiver.await, BatchStatus::Delivered); + + let keys = get_keys(&bucket, prefix).await; + assert_eq!(keys.len(), 1); + + let key = keys[0].clone(); + assert!( + key.ends_with(".parquet"), + "Expected .parquet extension, got: {}", + key + ); + + // Download and validate Parquet file + let obj = get_object(&bucket, key).await; + assert_eq!(obj.content_encoding, None); + + let body = obj.body.collect().await.unwrap().into_bytes(); + assert!(body.len() >= 4, "Output too short to be valid Parquet"); + assert_eq!(&body[..4], b"PAR1", "Missing Parquet magic bytes"); + + // Verify we can read rows from the Parquet file + use bytes::Bytes; + use parquet::file::reader::{FileReader, SerializedFileReader}; + use parquet::record::reader::RowIter; + + let reader = + SerializedFileReader::new(Bytes::copy_from_slice(&body)).expect("Invalid Parquet file"); + let row_count = RowIter::from_file_into(Box::new(reader)).count(); + assert_eq!(row_count, 10, "Expected 10 rows in Parquet file"); + + // Verify schema has our columns + let reader = + SerializedFileReader::new(Bytes::copy_from_slice(&body)).expect("Invalid Parquet file"); + let schema = reader.metadata().file_metadata().schema_descr(); + let columns: Vec = schema + .columns() + .iter() + .map(|c| c.name().to_string()) + .collect(); + assert!(columns.contains(&"message".to_string())); + assert!(columns.contains(&"host".to_string())); +} + async fn client() -> S3Client { let auth = AwsAuthentication::test_auth(); let region = RegionOrEndpoint::with_both("us-east-1", s3_address()); @@ -526,6 +630,8 @@ fn config(bucket: &str, batch_size: usize) -> S3SinkConfig { options: S3Options::default(), region: RegionOrEndpoint::with_both("us-east-1", s3_address()), encoding: (None::, TextSerializerConfig::default()).into(), + #[cfg(feature = "codecs-parquet")] + batch_encoding: None, compression: Compression::None, batch, request: TowerRequestConfig::default(), diff --git a/src/sinks/aws_s3/sink.rs b/src/sinks/aws_s3/sink.rs index 26d47cdb7039c..442e0beb544b6 100644 --- a/src/sinks/aws_s3/sink.rs +++ b/src/sinks/aws_s3/sink.rs @@ -3,10 +3,10 @@ use std::io; use bytes::Bytes; use chrono::{FixedOffset, Utc}; use uuid::Uuid; -use vector_lib::{codecs::encoding::Framer, event::Finalizable, request_metadata::RequestMetadata}; +use vector_lib::{codecs::EncoderKind, event::Finalizable, request_metadata::RequestMetadata}; use crate::{ - codecs::{Encoder, Transformer}, + codecs::Transformer, event::Event, sinks::{ s3_common::{ @@ -28,7 +28,7 @@ pub struct S3RequestOptions { pub filename_append_uuid: bool, pub filename_extension: Option, pub api_options: S3Options, - pub encoder: (Transformer, Encoder), + pub encoder: (Transformer, EncoderKind), pub compression: Compression, pub filename_tz_offset: Option, } @@ -36,7 +36,7 @@ pub struct S3RequestOptions { impl RequestBuilder<(S3PartitionKey, Vec)> for S3RequestOptions { type Metadata = S3Metadata; type Events = Vec; - type Encoder = (Transformer, Encoder); + type Encoder = (Transformer, EncoderKind); type Payload = Bytes; type Request = S3Request; type Error = io::Error; // TODO: this is ugly. diff --git a/src/sinks/clickhouse/config.rs b/src/sinks/clickhouse/config.rs index 58ed0a0ac3445..d7af3413c6b97 100644 --- a/src/sinks/clickhouse/config.rs +++ b/src/sinks/clickhouse/config.rs @@ -279,7 +279,7 @@ impl ClickhouseConfig { }; if let Some(batch_encoding) = &self.batch_encoding { - use vector_lib::codecs::{BatchEncoder, BatchSerializer}; + use vector_lib::codecs::BatchEncoder; // Validate that batch_encoding is only compatible with ArrowStream format if self.format != Format::ArrowStream { @@ -292,6 +292,13 @@ impl ClickhouseConfig { let mut arrow_config = match batch_encoding { BatchSerializerConfig::ArrowStream(config) => config.clone(), + #[cfg(feature = "codecs-parquet")] + BatchSerializerConfig::Parquet(_) => { + return Err( + "ClickHouse sink does not support Parquet batch encoding. Use 'arrow_stream' instead." + .into(), + ); + } }; self.resolve_arrow_schema( @@ -304,8 +311,7 @@ impl ClickhouseConfig { .await?; let resolved_batch_config = BatchSerializerConfig::ArrowStream(arrow_config); - let arrow_serializer = resolved_batch_config.build()?; - let batch_serializer = BatchSerializer::Arrow(arrow_serializer); + let batch_serializer = resolved_batch_config.build_batch_serializer()?; let encoder = EncoderKind::Batch(BatchEncoder::new(batch_serializer)); return Ok((Format::ArrowStream, encoder)); diff --git a/website/cue/reference/components/sinks/aws_s3.cue b/website/cue/reference/components/sinks/aws_s3.cue index cec5a50e47f82..6f42d082959ba 100644 --- a/website/cue/reference/components/sinks/aws_s3.cue +++ b/website/cue/reference/components/sinks/aws_s3.cue @@ -214,6 +214,135 @@ components: sinks: aws_s3: components._aws & { `storage_class` option. """ } + + parquet_encoding: { + title: "Parquet Batch Encoding" + body: """ + The S3 sink supports Apache Parquet batch encoding with the `batch_encoding` + option. When configured, events are encoded together as Parquet columnar files + instead of the default per-event JSON or text encoding. Parquet files are + optimized for analytical queries using Athena, Trino, Spark, and other columnar + query engines. + + Parquet handles compression internally at the column page level, so the + top-level `compression` setting must be set to `"none"`. + + Output files automatically use the `.parquet` extension. + + This feature requires the `codecs-parquet` feature flag at compile time. + + There are two ways to provide a schema: supply a `schema_file`, or set + `schema_mode` to `auto_infer` and let Vector derive the schema from each + incoming batch. + + #### Option 1: Schema file + + Load the schema from a native Parquet `.schema` file. The file must + contain a valid Parquet message type definition. + + ```toml + [sinks.s3_parquet] + type = "aws_s3" + inputs = ["my-source"] + bucket = "my-analytics-bucket" + key_prefix = "logs/date=%F" + compression = "none" + + [sinks.s3_parquet.encoding] + codec = "text" + + [sinks.s3_parquet.batch_encoding] + codec = "parquet" + schema_file = "/etc/vector/schemas/logs.schema" + schema_mode = "relaxed" + + [sinks.s3_parquet.batch_encoding.compression] + algorithm = "snappy" + ``` + + #### Option 2: Auto-infer schema + + Vector infers the Arrow schema from the fields present in each batch. + `Value::Timestamp` fields are automatically promoted to + `Timestamp(Microsecond, UTC)`. No schema file is required. + + ```toml + [sinks.s3_parquet] + type = "aws_s3" + inputs = ["my-source"] + bucket = "my-analytics-bucket" + key_prefix = "logs/date=%F" + compression = "none" + + [sinks.s3_parquet.encoding] + codec = "text" + + [sinks.s3_parquet.batch_encoding] + codec = "parquet" + schema_mode = "auto_infer" + + [sinks.s3_parquet.batch_encoding.compression] + algorithm = "snappy" + ``` + + #### YAML example + + ```yaml + sinks: + s3_parquet: + type: aws_s3 + inputs: + - my-source + bucket: my-analytics-bucket + key_prefix: "logs/date=%F" + compression: none + encoding: + codec: text + batch_encoding: + codec: parquet + schema_mode: auto_infer + compression: + algorithm: gzip + level: 9 + ``` + + #### Configuration reference + + | Field | Type | Required | Description | + |---|---|---|---| + | `codec` | string | yes | Must be `"parquet"` | + | `schema_file` | path | no | Path to a native Parquet `.schema` file. Required when `schema_mode` is `relaxed` or `strict`. | + | `schema_mode` | string | no | `relaxed` (default), `strict`, or `auto_infer`. See the section on schema_mode values. | + | `compression` | object | no | Column-level compression. See the section on compression options. | + + #### `schema_mode` values + + | Value | Description | + |---|---| + | `relaxed` (default) | Missing schema fields become null. Extra event fields are silently dropped. | + | `strict` | Missing schema fields become null. Extra event fields cause an encoding error. | + | `auto_infer` | Schema is inferred from each batch. No `schema_file` needed. `Value::Timestamp` fields are promoted to `Timestamp(Microsecond, UTC)`. | + + #### Compression options + + Compression is configured as a nested object with an `algorithm` key. + Algorithms that support levels accept an additional `level` key. + + | Algorithm | Level range | Default | + |---|---|---| + | `snappy` | — | yes | + | `zstd` | 1–21 | — | + | `gzip` | 1–9 | — | + | `lz4` | — | — | + | `none` | — | — | + + #### Unsupported types + + Binary fields are rejected at config time because the internal Arrow JSON + encoder cannot materialize them. Use `utf8` with base64 or hex encoding + for binary data instead. + """ + } } permissions: iam: [ From 51121047d3201394c2958e4a66696f3aef2d3f11 Mon Sep 17 00:00:00 2001 From: Thomas Date: Wed, 15 Apr 2026 13:25:02 -0400 Subject: [PATCH 127/326] chore(deps): bump rsa from 0.9.3 to 0.9.10 (#25198) * chore(deps): bump rsa from 0.9.3 to 0.9.10 * Update licenses --- Cargo.lock | 17 +++++++++-------- LICENSE-3rdparty.csv | 2 +- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ae4c753529775..d751a3e296d54 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3590,15 +3590,16 @@ dependencies = [ [[package]] name = "ed25519-dalek" -version = "2.0.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7277392b266383ef8396db7fdeb1e77b6c52fed775f5df15bb24f35b72156980" +checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" dependencies = [ "curve25519-dalek", "ed25519", "serde", "sha2", "signature", + "subtle", "zeroize", ] @@ -9323,9 +9324,9 @@ dependencies = [ [[package]] name = "rsa" -version = "0.9.3" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86ef35bf3e7fe15a53c4ab08a998e42271eab13eb0db224126bc7bc4c4bad96d" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" dependencies = [ "const-oid", "digest", @@ -10159,9 +10160,9 @@ dependencies = [ [[package]] name = "signature" -version = "2.0.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fe458c98333f9c8152221191a77e2a44e8325d0193484af2e9421a53019e57d" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ "digest", "rand_core 0.6.4", @@ -10398,9 +10399,9 @@ dependencies = [ [[package]] name = "spki" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d1e996ef02c474957d681f1b05213dfb0abab947b446a62d37770b23500184a" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" dependencies = [ "base64ct", "der", diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index ac6f8bc2acb4a..ed18e3c3c172b 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -242,7 +242,7 @@ dotenvy,https://github.com/allan2/dotenvy,MIT,"Noemi Lapresta ecdsa,https://github.com/RustCrypto/signatures/tree/master/ecdsa,Apache-2.0 OR MIT,RustCrypto Developers ed25519,https://github.com/RustCrypto/signatures/tree/master/ed25519,Apache-2.0 OR MIT,RustCrypto Developers -ed25519-dalek,https://github.com/dalek-cryptography/ed25519-dalek,BSD-3-Clause,"isis lovecruft , Tony Arcieri , Michael Rosenberg " +ed25519-dalek,https://github.com/dalek-cryptography/curve25519-dalek/tree/main/ed25519-dalek,BSD-3-Clause,"isis lovecruft , Tony Arcieri , Michael Rosenberg " educe,https://github.com/magiclen/educe,MIT,Magic Len either,https://github.com/bluss/either,MIT OR Apache-2.0,bluss elliptic-curve,https://github.com/RustCrypto/traits/tree/master/elliptic-curve,Apache-2.0 OR MIT,RustCrypto Developers From dac7c31501defdf451f9422b6a569358fb5ec19e Mon Sep 17 00:00:00 2001 From: Thomas Date: Wed, 15 Apr 2026 13:53:42 -0400 Subject: [PATCH 128/326] chore(deps): bump rustls-webpki 0.103.10 to 0.103.12 (RUSTSEC-2026-0099) (#25200) * chore(deps): bump rustls-webpki 0.103.10 to 0.103.12 (RUSTSEC-2026-0099) * Add RUSTSEC-2026-0099 to deny.toml * Add RUSTSEC-2026-0098 to deny.toml --- Cargo.lock | 6 +++--- deny.toml | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d751a3e296d54..dc20d39632c7c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9492,7 +9492,7 @@ dependencies = [ "once_cell", "ring", "rustls-pki-types", - "rustls-webpki 0.103.10", + "rustls-webpki 0.103.12", "subtle", "zeroize", ] @@ -9574,9 +9574,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.10" +version = "0.103.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef" +checksum = "8279bb85272c9f10811ae6a6c547ff594d6a7f3c6c6b02ee9726d1d0dcfcdd06" dependencies = [ "ring", "rustls-pki-types", diff --git a/deny.toml b/deny.toml index c19b0a23f5ee6..ffc2f8b68f240 100644 --- a/deny.toml +++ b/deny.toml @@ -44,5 +44,7 @@ ignore = [ { id = "RUSTSEC-2024-0388", reason = "derivative is unmaintained (https://github.com/vectordotdev/vector/issues/24940)" }, { id = "RUSTSEC-2025-0134", reason = "rustls-pemfile is unmaintained - unpatched crate (https://github.com/bytebeamio/rumqtt/issues/1010) & tonic/reqwest upgrade (https://github.com/vectordotdev/vector/issues/19179)" }, { id = "RUSTSEC-2026-0049", reason = "rustls-webpki 0.102 is vulnerable - tonic upgrade (https://github.com/vectordotdev/vector/issues/19179)" }, + { id = "RUSTSEC-2026-0098", reason = "rustls-webpki 0.102/0.101 is vulnerable - tonic upgrade (https://github.com/vectordotdev/vector/issues/19179)" }, + { id = "RUSTSEC-2026-0099", reason = "rustls-webpki 0.102/0.101 is vulnerable - tonic upgrade (https://github.com/vectordotdev/vector/issues/19179)" }, { id = "RUSTSEC-2024-0436", reason = "paste crate is unmaintained - transitive dependency via parquet v56.2.0, no safe upgrade available" }, ] From 6818b9c348cd4c3edd39f9f303f66362676e47a3 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Wed, 15 Apr 2026 14:22:40 -0400 Subject: [PATCH 129/326] feat(ci): add work-in-progress label workflow for docs PRs (#24950) * feat(ci): add workflow to manage work-in-progress label on PRs Automatically adds a "work in progress" label when a PR is opened or reopened, and removes it when a member or owner approves the PR. Co-Authored-By: Claude Sonnet 4.6 * feat(ci): replace work-in-progress with under_review label for docs PRs Target only PRs that touch docs-owned paths (matching CODEOWNERS) so the docs team isn't pinged until a Vector team member approves. The label is auto-added on open/reopen and auto-removed on first member approval. Co-Authored-By: Claude Opus 4.6 (1M context) * chore(ci): fix prettier formatting Co-Authored-By: Claude Opus 4.6 (1M context) * feat(ci): use work-in-progress label instead of under_review Co-Authored-By: Claude Opus 4.6 (1M context) * feat(ci): split wip label into add/remove workflows with tested approach - Split single workflow into add_wip_label.yml and remove_wip_label.yml to avoid paths filter affecting pull_request_review events - Move association and label checks into github-script step instead of job if condition (GitHub Actions silently fails author_association checks in if expressions) - Tested end-to-end in vectordotdev/ci-sandbox Co-Authored-By: Claude Opus 4.6 (1M context) * feat(ci): trigger WIP label on synchronize to catch docs paths added later Co-Authored-By: Claude Opus 4.6 (1M context) --------- Co-authored-by: Claude Sonnet 4.6 --- .github/CODEOWNERS | 1 + .github/workflows/add_wip_label.yml | 22 +++++++++++++ .github/workflows/remove_wip_label.yml | 45 ++++++++++++++++++++++++++ 3 files changed, 68 insertions(+) create mode 100644 .github/workflows/add_wip_label.yml create mode 100644 .github/workflows/remove_wip_label.yml diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ced3f0fb7bc07..ecd6fb374af9c 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -3,6 +3,7 @@ .github/workflows/regression.yml @vectordotdev/vector @vectordotdev/single-machine-performance regression/config.yaml @vectordotdev/vector @vectordotdev/single-machine-performance +# Keep documentation team paths in sync with .github/workflows/add_wip_label.yml docs/ @vectordotdev/vector @vectordotdev/documentation website/ @vectordotdev/vector website/content @vectordotdev/vector @vectordotdev/documentation diff --git a/.github/workflows/add_wip_label.yml b/.github/workflows/add_wip_label.yml new file mode 100644 index 0000000000000..a07f9ca0921fa --- /dev/null +++ b/.github/workflows/add_wip_label.yml @@ -0,0 +1,22 @@ +name: Add Work In Progress Label + +permissions: + pull-requests: write + +on: + pull_request_target: + types: [opened, reopened, synchronize] + # Keep these paths in sync with the documentation team entries in .github/CODEOWNERS + paths: + - "docs/**" + - "website/content/**" + - "website/cue/reference/**" + +jobs: + add_wip_label: + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - uses: actions-ecosystem/action-add-labels@18f1af5e3544586314bbe15c0273249c770b2daf # v1.1.3 + with: + labels: "work in progress" diff --git a/.github/workflows/remove_wip_label.yml b/.github/workflows/remove_wip_label.yml new file mode 100644 index 0000000000000..46d6e79cf1ad8 --- /dev/null +++ b/.github/workflows/remove_wip_label.yml @@ -0,0 +1,45 @@ +name: Remove Work In Progress Label + +permissions: + pull-requests: write + +on: + pull_request_review: + types: [submitted] + +jobs: + remove_wip_label: + if: github.event.review.state == 'approved' + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - name: Check association, label, and remove + uses: actions/github-script@v7 + with: + script: | + const association = context.payload.review.author_association; + core.info(`Author association: ${association}`); + + const allowed = ['MEMBER', 'OWNER']; + if (!allowed.includes(association)) { + core.info(`Association "${association}" not in allowed list, skipping`); + return; + } + + const { data: labels } = await github.rest.issues.listLabelsOnIssue({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.payload.pull_request.number, + }); + const hasLabel = labels.some(l => l.name === 'work in progress'); + if (!hasLabel) { + core.info('Label "work in progress" not found, skipping'); + return; + } + await github.rest.issues.removeLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.payload.pull_request.number, + name: 'work in progress', + }); + core.info('Removed "work in progress" label'); From 2d14034bf8115764c52636a59ec298d7444a4d28 Mon Sep 17 00:00:00 2001 From: Thomas Date: Wed, 15 Apr 2026 14:52:21 -0400 Subject: [PATCH 130/326] fix(deps): override smol-toml to 1.6.1 for markdownlint-cli2 (#25202) fix(deps): override smol-toml to 1.6.1 for markdownlint-cli2 (GHSA-v3rj-xjv7-4jmq) --- scripts/environment/npm-tools/package-lock.json | 6 +++--- scripts/environment/npm-tools/package.json | 5 +++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/scripts/environment/npm-tools/package-lock.json b/scripts/environment/npm-tools/package-lock.json index d9e098652b6ad..ac93d1c99990a 100644 --- a/scripts/environment/npm-tools/package-lock.json +++ b/scripts/environment/npm-tools/package-lock.json @@ -1232,9 +1232,9 @@ } }, "node_modules/smol-toml": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.6.0.tgz", - "integrity": "sha512-4zemZi0HvTnYwLfrpk/CF9LOd9Lt87kAt50GnqhMpyF9U3poDAP2+iukq2bZsO/ufegbYehBkqINbsWxj4l4cw==", + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.6.1.tgz", + "integrity": "sha512-dWUG8F5sIIARXih1DTaQAX4SsiTXhInKf1buxdY9DIg4ZYPZK5nGM1VRIYmEbDbsHt7USo99xSLFu5Q1IqTmsg==", "license": "BSD-3-Clause", "engines": { "node": ">= 18" diff --git a/scripts/environment/npm-tools/package.json b/scripts/environment/npm-tools/package.json index a40c7f7c3e2ae..fdffa6f8a2ac0 100644 --- a/scripts/environment/npm-tools/package.json +++ b/scripts/environment/npm-tools/package.json @@ -1,6 +1,11 @@ { "private": true, "description": "Pinned npm tools for CI. Run 'npm ci' to install with exact versions from package-lock.json.", + "overrides": { + "markdownlint-cli2": { + "smol-toml": "1.6.1" + } + }, "dependencies": { "@datadog/datadog-ci": "5.13.0", "markdownlint-cli2": "0.22.0", From 08c9f1e6b54cf1b30e88d536f3545e74ff193c0f Mon Sep 17 00:00:00 2001 From: Thomas Date: Wed, 15 Apr 2026 15:48:46 -0400 Subject: [PATCH 131/326] chore(deps): bump VRL and add feature for enable_crypto_functions (#25205) --- Cargo.toml | 1 + Makefile | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 6b233580143c8..84ded822b5790 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -533,6 +533,7 @@ tokio-console = ["dep:console-subscriber", "tokio/tracing"] vrl-functions-env = ["vrl/enable_env_functions"] vrl-functions-system = ["vrl/enable_system_functions"] vrl-functions-network = ["vrl/enable_network_functions"] +vrl-functions-crypto = ["vrl/enable_crypto_functions"] # Enables the binary secret-backend-example secret-backend-example = ["transforms"] diff --git a/Makefile b/Makefile index 34da1302487d9..821ff961de4ac 100644 --- a/Makefile +++ b/Makefile @@ -373,7 +373,7 @@ test-behavior-config: ## Runs configuration related behavioral tests .PHONY: test-behavior-% test-behavior-%: ## Runs behavioral test for a given category - ${MAYBE_ENVIRONMENT_EXEC} cargo run --no-default-features --features transforms,vrl-functions-env,vrl-functions-system,vrl-functions-network -- test tests/behavior/$*/* + ${MAYBE_ENVIRONMENT_EXEC} cargo run --no-default-features --features transforms,vrl-functions-env,vrl-functions-system,vrl-functions-network,vrl-functions-crypto -- test tests/behavior/$*/* .PHONY: test-behavior test-behavior: ## Runs all behavioral tests From a1469590bed6996b1eb7001248232b0aa48cd777 Mon Sep 17 00:00:00 2001 From: Thomas Date: Wed, 15 Apr 2026 16:06:08 -0400 Subject: [PATCH 132/326] chore(deps): bump rand 0.10.0 to 0.10.1 and 0.9.2 to 0.9.4 (#25204) * chore(deps): bump rand 0.10.0 to 0.10.1 (RUSTSEC-2026-0097) * chore(deps): bump rand 0.9.2 to 0.9.4 (RUSTSEC-2026-0097) --- Cargo.lock | 64 +++++++++++++++++++++++++++--------------------------- deny.toml | 1 + 2 files changed, 33 insertions(+), 32 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dc20d39632c7c..aadbb2dd580a9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -271,7 +271,7 @@ dependencies = [ "miniz_oxide", "num-bigint", "quad-rand", - "rand 0.9.2", + "rand 0.9.4", "regex-lite", "serde", "serde_bytes", @@ -2022,7 +2022,7 @@ dependencies = [ "indexmap 2.12.0", "js-sys", "once_cell", - "rand 0.9.2", + "rand 0.9.4", "serde", "serde_bytes", "serde_json", @@ -2446,7 +2446,7 @@ dependencies = [ "pin-project", "prost 0.12.6", "prost-reflect", - "rand 0.9.2", + "rand 0.9.4", "regex", "rstest", "rust_decimal", @@ -3910,7 +3910,7 @@ version = "0.1.0" dependencies = [ "chrono", "fakedata_generator", - "rand 0.9.2", + "rand 0.9.4", ] [[package]] @@ -3920,7 +3920,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b42ec18312c068f0d6008ad66ad2fa289ab40a16d07a06406c200baa09eb1e7c" dependencies = [ "passt", - "rand 0.9.2", + "rand 0.9.4", "serde", "serde_json", ] @@ -4412,7 +4412,7 @@ dependencies = [ "nonzero_ext", "parking_lot", "portable-atomic", - "rand 0.9.2", + "rand 0.9.4", "smallvec", "spinning_top", "web-time", @@ -4445,7 +4445,7 @@ dependencies = [ "greptime-proto", "parking_lot", "prost 0.12.6", - "rand 0.9.2", + "rand 0.9.4", "snafu 0.8.9", "tokio", "tokio-stream", @@ -4819,7 +4819,7 @@ dependencies = [ "idna", "ipnet", "once_cell", - "rand 0.9.2", + "rand 0.9.4", "ring", "rustls-pki-types", "thiserror 2.0.17", @@ -4864,7 +4864,7 @@ dependencies = [ "moka", "once_cell", "parking_lot", - "rand 0.9.2", + "rand 0.9.4", "resolv-conf", "smallvec", "thiserror 2.0.17", @@ -5794,7 +5794,7 @@ dependencies = [ "indoc", "k8s-openapi", "k8s-test-framework", - "rand 0.9.2", + "rand 0.9.4", "regex", "reqwest 0.11.26", "serde_json", @@ -7942,7 +7942,7 @@ dependencies = [ "hmac", "md-5", "memchr", - "rand 0.9.2", + "rand 0.9.4", "sha2", "stringprep", ] @@ -8172,7 +8172,7 @@ dependencies = [ "bit-vec", "bitflags 2.10.0", "num-traits", - "rand 0.9.2", + "rand 0.9.4", "rand_chacha 0.9.0", "rand_xorshift", "regex-syntax", @@ -8251,7 +8251,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4" dependencies = [ "bytes", - "heck 0.5.0", + "heck 0.4.1", "itertools 0.12.1", "log", "multimap", @@ -8271,7 +8271,7 @@ version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ - "heck 0.5.0", + "heck 0.4.1", "itertools 0.14.0", "log", "multimap", @@ -8528,7 +8528,7 @@ checksum = "95c589f335db0f6aaa168a7cd27b1fc6920f5e1470c804f814d9cd6e62a0f70b" dependencies = [ "env_logger", "log", - "rand 0.10.0", + "rand 0.10.1", ] [[package]] @@ -8569,7 +8569,7 @@ dependencies = [ "bytes", "getrandom 0.3.4", "lru-slab", - "rand 0.9.2", + "rand 0.9.4", "ring", "rustc-hash", "rustls 0.23.37", @@ -8660,9 +8660,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.2" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.0", @@ -8670,9 +8670,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc266eb313df6c5c09c1c7b1fbe2510961e5bcd3add930c1e31f7ed9da0feff8" +checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" dependencies = [ "chacha20 0.10.0", "getrandom 0.4.2", @@ -8731,7 +8731,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" dependencies = [ "num-traits", - "rand 0.9.2", + "rand 0.9.4", ] [[package]] @@ -8900,7 +8900,7 @@ dependencies = [ "num-bigint", "percent-encoding", "pin-project-lite", - "rand 0.9.2", + "rand 0.9.4", "ryu", "socket2 0.6.0", "tokio", @@ -9211,7 +9211,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46a4bd6027df676bcb752d3724db0ea3c0c5fc1dd0376fec51ac7dcaf9cc69be" dependencies = [ - "rand 0.9.2", + "rand 0.9.4", ] [[package]] @@ -10323,7 +10323,7 @@ version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" dependencies = [ - "heck 0.5.0", + "heck 0.4.1", "proc-macro2 1.0.106", "quote 1.0.44", "syn 2.0.117", @@ -10335,7 +10335,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54254b8531cafa275c5e096f62d48c81435d1015405a91198ddb11e967301d40" dependencies = [ - "heck 0.5.0", + "heck 0.4.1", "proc-macro2 1.0.106", "quote 1.0.44", "syn 2.0.117", @@ -11262,7 +11262,7 @@ dependencies = [ "pin-project-lite", "postgres-protocol", "postgres-types", - "rand 0.9.2", + "rand 0.9.4", "socket2 0.6.0", "tokio", "tokio-util", @@ -11987,7 +11987,7 @@ dependencies = [ "futures", "getrandom 0.3.4", "pin-project", - "rand 0.9.2", + "rand 0.9.4", "reqwest 0.12.28", "serde", "serde_json", @@ -12231,7 +12231,7 @@ checksum = "a68d3c8f01c0cfa54a75291d83601161799e4a89a39e0929f4b0354d88757a37" dependencies = [ "getrandom 0.4.2", "js-sys", - "rand 0.10.0", + "rand 0.10.1", "serde_core", "wasm-bindgen", ] @@ -12428,7 +12428,7 @@ dependencies = [ "quick-junit", "quick-xml 0.31.0", "quickcheck", - "rand 0.9.2", + "rand 0.9.4", "rand_distr", "rdkafka", "redis", @@ -12549,7 +12549,7 @@ dependencies = [ "pastey", "proptest", "quickcheck", - "rand 0.9.2", + "rand 0.9.4", "rkyv", "serde", "serde_yaml", @@ -12698,7 +12698,7 @@ dependencies = [ "quanta", "quickcheck", "quickcheck_macros", - "rand 0.9.2", + "rand 0.9.4", "rand_distr", "regex", "ryu", @@ -12775,7 +12775,7 @@ dependencies = [ "futures-util", "pin-project", "proptest", - "rand 0.9.2", + "rand 0.9.4", "rand_distr", "tokio", "tokio-util", diff --git a/deny.toml b/deny.toml index ffc2f8b68f240..ac8e588fa1cbe 100644 --- a/deny.toml +++ b/deny.toml @@ -47,4 +47,5 @@ ignore = [ { id = "RUSTSEC-2026-0098", reason = "rustls-webpki 0.102/0.101 is vulnerable - tonic upgrade (https://github.com/vectordotdev/vector/issues/19179)" }, { id = "RUSTSEC-2026-0099", reason = "rustls-webpki 0.102/0.101 is vulnerable - tonic upgrade (https://github.com/vectordotdev/vector/issues/19179)" }, { id = "RUSTSEC-2024-0436", reason = "paste crate is unmaintained - transitive dependency via parquet v56.2.0, no safe upgrade available" }, + { id = "RUSTSEC-2026-0097", reason = "rand 0.8.5 unsound with custom logger - transitive dependency, upstream crates have not updated to rand 0.9+" }, ] From 5ba8405bf48cd64509bf02388c7901451bbe67f4 Mon Sep 17 00:00:00 2001 From: Thomas Date: Wed, 15 Apr 2026 16:20:05 -0400 Subject: [PATCH 133/326] chore(vdev): bump version to 0.3.1 (#25206) --- Cargo.lock | 2 +- vdev/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index aadbb2dd580a9..2d641cc2a2934 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12260,7 +12260,7 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] name = "vdev" -version = "0.3.0" +version = "0.3.1" dependencies = [ "anyhow", "cfg-if", diff --git a/vdev/Cargo.toml b/vdev/Cargo.toml index 7de3fc06d5a59..2a0f6169a160e 100644 --- a/vdev/Cargo.toml +++ b/vdev/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "vdev" -version = "0.3.0" +version = "0.3.1" edition = "2024" authors = ["Vector Contributors "] license = "MPL-2.0" From ce651754d4e2eb134ee9ab2fe85a8a3a5ed923a0 Mon Sep 17 00:00:00 2001 From: Yoenn Burban <62966490+gwenaskell@users.noreply.github.com> Date: Thu, 16 Apr 2026 09:16:53 +0200 Subject: [PATCH 134/326] feat(sources): add source latency metric and fix source lag time on large batches (#24987) * add send latency distribution to source sender * add changelog * use outputMetrics in the builder * fix clippy * apply suggestion * only record the await point duration * make sure metrics are updated if an error is returned * add documentation for the added metrics * fix source_lag_time_seconds measurement for large batches --- ...fix_source_lag_time_chunked_batches.fix.md | 3 + ...urce_sender_latency_metrics.enhancement.md | 5 ++ lib/vector-core/src/source_sender/builder.rs | 20 +++-- lib/vector-core/src/source_sender/mod.rs | 4 +- lib/vector-core/src/source_sender/output.rs | 87 +++++++++++++++---- lib/vector-core/src/source_sender/sender.rs | 19 +++- .../components/sources/internal_metrics.cue | 12 +++ 7 files changed, 122 insertions(+), 28 deletions(-) create mode 100644 changelog.d/fix_source_lag_time_chunked_batches.fix.md create mode 100644 changelog.d/source_sender_latency_metrics.enhancement.md diff --git a/changelog.d/fix_source_lag_time_chunked_batches.fix.md b/changelog.d/fix_source_lag_time_chunked_batches.fix.md new file mode 100644 index 0000000000000..6303ba10edd33 --- /dev/null +++ b/changelog.d/fix_source_lag_time_chunked_batches.fix.md @@ -0,0 +1,3 @@ +Fixed an incorrect source_lag_time_seconds measurement in sources that use `send_batch` with large event batches. When a batch was split into multiple chunks, the reference timestamp used to compute lag time was re-captured on each chunk send, causing the lag time for later chunks to be overstated by the amount of time spent waiting for the channel to accept earlier chunks. The reference timestamp is now captured once before iteration and shared across all chunks. + +authors: gwenaskell diff --git a/changelog.d/source_sender_latency_metrics.enhancement.md b/changelog.d/source_sender_latency_metrics.enhancement.md new file mode 100644 index 0000000000000..6876641a37764 --- /dev/null +++ b/changelog.d/source_sender_latency_metrics.enhancement.md @@ -0,0 +1,5 @@ +Sources now record the distribution metrics `source_send_latency_seconds` (measuring the time spent +blocking on a single events chunk send operation on the output) and `source_send_batch_latency_seconds` +(encompassing all chunks within a received events batch). + +authors: gwenaskell diff --git a/lib/vector-core/src/source_sender/builder.rs b/lib/vector-core/src/source_sender/builder.rs index ec16d54d482a9..f3051726a7f56 100644 --- a/lib/vector-core/src/source_sender/builder.rs +++ b/lib/vector-core/src/source_sender/builder.rs @@ -1,17 +1,20 @@ use std::{collections::HashMap, time::Duration}; -use metrics::{Histogram, histogram}; +use metrics::histogram; use vector_buffers::topology::channel::LimitedReceiver; use vector_common::internal_event::DEFAULT_OUTPUT; -use super::{CHUNK_SIZE, LAG_TIME_NAME, Output, SourceSender, SourceSenderItem}; +use super::{ + CHUNK_SIZE, LAG_TIME_NAME, Output, OutputMetrics, SEND_BATCH_LATENCY_NAME, SEND_LATENCY_NAME, + SourceSender, SourceSenderItem, +}; use crate::config::{ComponentKey, OutputId, SourceOutput}; pub struct Builder { buf_size: usize, default_output: Option, named_outputs: HashMap, - lag_time: Option, + output_metrics: OutputMetrics, timeout: Option, ewma_half_life_seconds: Option, } @@ -22,7 +25,11 @@ impl Default for Builder { buf_size: CHUNK_SIZE, default_output: None, named_outputs: Default::default(), - lag_time: Some(histogram!(LAG_TIME_NAME)), + output_metrics: OutputMetrics::new( + Some(histogram!(LAG_TIME_NAME)), + Some(histogram!(SEND_LATENCY_NAME)), + Some(histogram!(SEND_BATCH_LATENCY_NAME)), + ), timeout: None, ewma_half_life_seconds: None, } @@ -53,7 +60,6 @@ impl Builder { output: SourceOutput, component_key: ComponentKey, ) -> LimitedReceiver { - let lag_time = self.lag_time.clone(); let log_definition = output.schema_definition.clone(); let output_id = OutputId { component: component_key, @@ -64,7 +70,7 @@ impl Builder { let (output, rx) = Output::new_with_buffer( self.buf_size, DEFAULT_OUTPUT.to_owned(), - lag_time, + self.output_metrics.clone(), log_definition, output_id, self.timeout, @@ -77,7 +83,7 @@ impl Builder { let (output, rx) = Output::new_with_buffer( self.buf_size, name.clone(), - lag_time, + self.output_metrics.clone(), log_definition, output_id, self.timeout, diff --git a/lib/vector-core/src/source_sender/mod.rs b/lib/vector-core/src/source_sender/mod.rs index c8af2db8bbf87..fc0cb417890a0 100644 --- a/lib/vector-core/src/source_sender/mod.rs +++ b/lib/vector-core/src/source_sender/mod.rs @@ -14,7 +14,7 @@ mod tests; pub use builder::Builder; pub use errors::SendError; -use output::Output; +use output::{Output, OutputMetrics}; pub use sender::{SourceSender, SourceSenderItem}; pub const CHUNK_SIZE: usize = 1000; @@ -23,3 +23,5 @@ pub const CHUNK_SIZE: usize = 1000; const TEST_BUFFER_SIZE: usize = 100; const LAG_TIME_NAME: &str = "source_lag_time_seconds"; +const SEND_LATENCY_NAME: &str = "source_send_latency_seconds"; +const SEND_BATCH_LATENCY_NAME: &str = "source_send_batch_latency_seconds"; diff --git a/lib/vector-core/src/source_sender/output.rs b/lib/vector-core/src/source_sender/output.rs index df0bee2a43da0..4a194cc3e1c1f 100644 --- a/lib/vector-core/src/source_sender/output.rs +++ b/lib/vector-core/src/source_sender/output.rs @@ -85,7 +85,7 @@ impl Drop for UnsentEventCount { #[derive(Clone)] pub(super) struct Output { sender: LimitedSender, - lag_time: Option, + metrics: OutputMetrics, events_sent: Registered, /// The schema definition that will be attached to Log events sent through here log_definition: Option>, @@ -95,6 +95,27 @@ pub(super) struct Output { timeout: Option, } +#[derive(Clone, Default)] +pub(super) struct OutputMetrics { + lag_time: Option, + send_latency: Option, + send_batch_latency: Option, +} + +impl OutputMetrics { + pub(super) fn new( + lag_time: Option, + send_latency: Option, + send_batch_latency: Option, + ) -> Self { + Self { + lag_time, + send_latency, + send_batch_latency, + } + } +} + #[expect(clippy::missing_fields_in_debug)] impl fmt::Debug for Output { fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { @@ -111,19 +132,20 @@ impl Output { pub(super) fn new_with_buffer( n: usize, output: String, - lag_time: Option, + metrics: OutputMetrics, log_definition: Option>, output_id: OutputId, timeout: Option, ewma_half_life_seconds: Option, ) -> (Self, LimitedReceiver) { let limit = MemoryBufferSize::MaxEvents(NonZeroUsize::new(n).unwrap()); - let metrics = ChannelMetricMetadata::new(UTILIZATION_METRIC_PREFIX, Some(output.clone())); - let (tx, rx) = channel::limited(limit, Some(metrics), ewma_half_life_seconds); + let channel_metrics = + ChannelMetricMetadata::new(UTILIZATION_METRIC_PREFIX, Some(output.clone())); + let (tx, rx) = channel::limited(limit, Some(channel_metrics), ewma_half_life_seconds); ( Self { sender: tx, - lag_time, + metrics, events_sent: internal_event::register(EventsSent::from(internal_event::Output( Some(output.into()), ))), @@ -136,12 +158,21 @@ impl Output { } pub(super) async fn send( + &mut self, + events: EventArray, + unsent_event_count: &mut UnsentEventCount, + ) -> Result<(), SendError> { + let reference = Utc::now().timestamp_millis(); + self.send_inner(events, unsent_event_count, reference).await + } + + async fn send_inner( &mut self, mut events: EventArray, unsent_event_count: &mut UnsentEventCount, + reference: i64, ) -> Result<(), SendError> { let send_reference = Instant::now(); - let reference = Utc::now().timestamp_millis(); events .iter_events() .for_each(|event| self.emit_lag_time(event, reference)); @@ -156,7 +187,17 @@ impl Output { let byte_size = events.estimated_json_encoded_size_of(); let count = events.len(); - self.send_with_timeout(events, send_reference).await?; + + let send_start = Instant::now(); + + let send_result = self.send_with_timeout(events, send_reference).await; + + if let Some(send_latency) = &self.metrics.send_latency { + send_latency.record(send_start.elapsed().as_secs_f64()); + } + + send_result?; + self.events_sent.emit(CountByteSize(count, byte_size)); unsent_event_count.decr(count); Ok(()) @@ -218,25 +259,39 @@ impl Output { I: IntoIterator, ::IntoIter: ExactSizeIterator, { + // Capture a single reference timestamp for the entire batch so that lag time + // measurements are not inflated by channel-send latency for later chunks. + let reference = Utc::now().timestamp_millis(); + // It's possible that the caller stops polling this future while it is blocked waiting // on `self.send()`. When that happens, we use `UnsentEventCount` to correctly emit // `ComponentEventsDropped` events. let events = events.into_iter().map(Into::into); let mut unsent_event_count = UnsentEventCount::new(events.len()); + let send_batch_start = Instant::now(); + for events in array::events_into_arrays(events, Some(CHUNK_SIZE)) { - self.send(events, &mut unsent_event_count) + self.send_inner(events, &mut unsent_event_count, reference) .await - .inspect_err(|error| match error { - SendError::Timeout => { - unsent_event_count.timed_out(); + .inspect_err(|error| { + match error { + SendError::Timeout => { + unsent_event_count.timed_out(); + } + SendError::Closed => { + // The unsent event count is discarded here because the callee emits the + // `StreamClosedError`. + unsent_event_count.discard(); + } } - SendError::Closed => { - // The unsent event count is discarded here because the callee emits the - // `StreamClosedError`. - unsent_event_count.discard(); + if let Some(send_batch_latency) = &self.metrics.send_batch_latency { + send_batch_latency.record(send_batch_start.elapsed().as_secs_f64()); } })?; } + if let Some(send_batch_latency) = &self.metrics.send_batch_latency { + send_batch_latency.record(send_batch_start.elapsed().as_secs_f64()); + } Ok(()) } @@ -244,7 +299,7 @@ impl Output { /// timestamp stored in the given event reference, and emit the /// different, as expressed in milliseconds, as a histogram. pub(super) fn emit_lag_time(&self, event: EventRef<'_>, reference: i64) { - if let Some(lag_time_metric) = &self.lag_time { + if let Some(lag_time_metric) = &self.metrics.lag_time { let timestamp = match event { EventRef::Log(log) => { log_schema() diff --git a/lib/vector-core/src/source_sender/sender.rs b/lib/vector-core/src/source_sender/sender.rs index 88cb50172c70a..6041dad3dad9f 100644 --- a/lib/vector-core/src/source_sender/sender.rs +++ b/lib/vector-core/src/source_sender/sender.rs @@ -22,7 +22,9 @@ use vector_common::{ use super::{Builder, Output, SendError}; #[cfg(any(test, feature = "test"))] -use super::{LAG_TIME_NAME, TEST_BUFFER_SIZE}; +use super::{ + LAG_TIME_NAME, OutputMetrics, SEND_BATCH_LATENCY_NAME, SEND_LATENCY_NAME, TEST_BUFFER_SIZE, +}; use crate::{ EstimatedJsonEncodedSizeOf, event::{Event, EventArray, EventContainer, array::EventArrayIntoIter}, @@ -108,6 +110,8 @@ impl SourceSender { timeout: Option, ) -> (Self, LimitedReceiver) { let lag_time = Some(histogram!(LAG_TIME_NAME)); + let send_latency = Some(histogram!(SEND_LATENCY_NAME)); + let send_batch_latency = Some(histogram!(SEND_BATCH_LATENCY_NAME)); let output_id = OutputId { component: "test".to_string().into(), port: None, @@ -115,7 +119,7 @@ impl SourceSender { let (default_output, rx) = Output::new_with_buffer( n, DEFAULT_OUTPUT.to_owned(), - lag_time, + OutputMetrics::new(lag_time, send_latency, send_batch_latency), None, output_id, timeout, @@ -192,8 +196,15 @@ impl SourceSender { component: "test".to_string().into(), port: Some(name.clone()), }; - let (output, recv) = - Output::new_with_buffer(100, name.clone(), None, None, output_id, None, None); + let (output, recv) = Output::new_with_buffer( + 100, + name.clone(), + OutputMetrics::default(), + None, + output_id, + None, + None, + ); let recv = recv.into_stream().map(move |mut item| { item.events.iter_events_mut().for_each(|mut event| { let metadata = event.metadata_mut(); diff --git a/website/cue/reference/components/sources/internal_metrics.cue b/website/cue/reference/components/sources/internal_metrics.cue index 98ed1ffe025d7..1e990965638c6 100644 --- a/website/cue/reference/components/sources/internal_metrics.cue +++ b/website/cue/reference/components/sources/internal_metrics.cue @@ -774,6 +774,18 @@ components: sources: internal_metrics: { default_namespace: "vector" tags: _component_tags } + source_send_batch_latency_seconds: { + description: "The time elapsed blocking on the downstream channel to accept an entire batch of events received at the source" + type: "histogram" + default_namespace: "vector" + tags: _component_tags + } + source_send_latency_seconds: { + description: "The time elapsed blocking on the downstream channel to accept a single chunk from a batch of events received at the source" + type: "histogram" + default_namespace: "vector" + tags: _component_tags + } source_buffer_max_byte_size: { description: "The maximum number of bytes the source buffer can hold. The outputs of the source send data to this buffer." type: "gauge" From 8138c4593455dd284475ed4ed61fe55ee176bd34 Mon Sep 17 00:00:00 2001 From: Thomas Date: Thu, 16 Apr 2026 11:27:11 -0400 Subject: [PATCH 135/326] chore(website): update DD RUM/Logs config to cose site (#25213) --- website/assets/js/dd-browser-logs-rum.js | 6 ++++++ website/config.toml | 5 +++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/website/assets/js/dd-browser-logs-rum.js b/website/assets/js/dd-browser-logs-rum.js index fa9414b4099d2..3ec4e2a2fb085 100644 --- a/website/assets/js/dd-browser-logs-rum.js +++ b/website/assets/js/dd-browser-logs-rum.js @@ -12,10 +12,15 @@ if (datadogRum) { datadogRum.init({ applicationId: '{{ $ddConfig.application_id }}', clientToken: '{{ $ddConfig.client_token }}', + site: '{{ $ddConfig.site }}', env, service: '{{ $ddConfig.service_name }}', version: '{{ $latest }}', + sessionSampleRate: 100, + sessionReplaySampleRate: 20, + trackResources: true, trackInteractions: true, + trackLongTasks: true, internalAnalyticsSubdomain: '8b61d74c' }); } @@ -25,6 +30,7 @@ if (datadogLogs) { if (env === 'preview' || env === 'production') { datadogLogs.init({ clientToken: '{{ $ddConfig.client_token }}', + site: '{{ $ddConfig.site }}', forwardErrorsToLogs: true, env, service: '{{ $ddConfig.service_name }}', diff --git a/website/config.toml b/website/config.toml index f107186e280da..414451d38b57d 100644 --- a/website/config.toml +++ b/website/config.toml @@ -93,8 +93,9 @@ discord = "https://chat.vector.dev" # Datadog configuration [params.datadog_config] -application_id = "0b95923b-b06d-445b-893f-a861e93d6ea3" -client_token = "puba5f23a97d613091ae2ca8c0f4a135af4" +application_id = "894af722-5919-4a62-97fc-33f49bc52803" +client_token = "pubcee1dc56e7f5bd03397cc3624eb7961c" +site = "cose.datadoghq.com" service_name = "vector" ## Menus From 61b4563fc60211feba19dcbf9296872b8640ec94 Mon Sep 17 00:00:00 2001 From: Thomas Date: Thu, 16 Apr 2026 12:19:11 -0400 Subject: [PATCH 136/326] fix(ci): fix secrets in static analysis workflow (#25208) fix(ci): use env context instead of secrets in workflow expression --- .github/workflows/static-analysis.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml index 04ed6b19d9f25..9411f2844e264 100644 --- a/.github/workflows/static-analysis.yml +++ b/.github/workflows/static-analysis.yml @@ -13,14 +13,17 @@ permissions: jobs: static-analysis: runs-on: ubuntu-latest + env: + DD_API_KEY: ${{ secrets.DD_API_KEY }} + DD_APP_KEY: ${{ secrets.DD_APP_KEY }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Datadog Static Analyzer - if: ${{ secrets.DD_API_KEY != '' }} + if: ${{ env.DD_API_KEY != '' }} uses: DataDog/datadog-static-analyzer-github-action@8340f18875fcefca86844b5f947ce2431387e552 # v3.0.0 with: - dd_api_key: ${{ secrets.DD_API_KEY }} - dd_app_key: ${{ secrets.DD_APP_KEY }} + dd_api_key: ${{ env.DD_API_KEY }} + dd_app_key: ${{ env.DD_APP_KEY }} dd_site: datadoghq.com secrets_enabled: true From 467ef876a286df316ba273e4ad1779f1b08ba392 Mon Sep 17 00:00:00 2001 From: Vladimir Zhuk Date: Thu, 16 Apr 2026 19:18:13 +0200 Subject: [PATCH 137/326] fix(datadog_common sink): preserve site prefix when computing API endpoint (#25211) * fix(common): preserve site prefix when computing API endpoint from intake endpoints The `compute_api_endpoint` regex was stripping site prefixes (e.g. `us3.`, `us5.`, `ap1.`) from Datadog domain names when deriving the API endpoint from intake endpoints. This caused the sink healthcheck to call `https://api.datadoghq.com` instead of the site-specific endpoint (e.g. `https://api.us3.datadoghq.com`), resulting in unintended cross-site egress traffic. The fix moves the optional site prefix inside the regex capture group so it is preserved in the rewritten API URL. Co-Authored-By: Claude Opus 4.6 (1M context) * add changelog fragment and fix formatting Co-Authored-By: Claude Opus 4.6 (1M context) * fix changelog * refactor test --------- Co-authored-by: Claude Opus 4.6 (1M context) Co-authored-by: Pavlos Rontidis --- changelog.d/fix_site_prefix_stripping.fix.md | 3 ++ src/common/datadog.rs | 30 +++++++++++++++++++- 2 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 changelog.d/fix_site_prefix_stripping.fix.md diff --git a/changelog.d/fix_site_prefix_stripping.fix.md b/changelog.d/fix_site_prefix_stripping.fix.md new file mode 100644 index 0000000000000..8136664b5b2fd --- /dev/null +++ b/changelog.d/fix_site_prefix_stripping.fix.md @@ -0,0 +1,3 @@ +Fixed the Datadog sink healthcheck endpoint computation to preserve site prefixes (e.g. `us3.`, `us5.`, `ap1.`) when deriving the API URL from intake endpoints. Previously, the healthcheck for site-specific endpoints like `https://http-intake.logs.us3.datadoghq.com` would incorrectly call `https://api.datadoghq.com` instead of `https://api.us3.datadoghq.com`, causing unintended cross-site egress traffic. + +authors: vladimir-dd diff --git a/src/common/datadog.rs b/src/common/datadog.rs index 9b84a2f8351dc..05638b474e940 100644 --- a/src/common/datadog.rs +++ b/src/common/datadog.rs @@ -105,7 +105,7 @@ fn compute_api_endpoint(endpoint: &str) -> String { // https://github.com/DataDog/datadog-agent/blob/cdcf0fc809b9ac1cd6e08057b4971c7dbb8dbe30/comp/forwarder/defaultforwarder/forwarder_health.go#L45-L47 // https://github.com/DataDog/datadog-agent/blob/cdcf0fc809b9ac1cd6e08057b4971c7dbb8dbe30/comp/forwarder/defaultforwarder/forwarder_health.go#L188-L190 static DOMAIN_REGEX: LazyLock = LazyLock::new(|| { - Regex::new(r"(?:[a-z]{2}\d\.)?(datadoghq\.[a-z]+|ddog-gov\.com)/*$") + Regex::new(r"((?:[a-z]{2}\d\.)?(?:datadoghq\.[a-z]+|ddog-gov\.com))/*$") .expect("Could not build Datadog domain regex") }); @@ -175,6 +175,27 @@ mod tests { ); } + #[test] + fn preserves_site_prefix_in_api_endpoint() { + for (prefix, tld) in [ + ("us3", "com"), + ("us5", "com"), + ("ap1", "com"), + ("eu1", "eu"), + ] { + assert_eq!( + compute_api_endpoint(&format!( + "https://http-intake.logs.{prefix}.datadoghq.{tld}" + )), + format!("https://api.{prefix}.datadoghq.{tld}") + ); + } + assert_eq!( + compute_api_endpoint("https://1-2-3-observability-pipelines.agent.us3.datadoghq.com"), + "https://api.us3.datadoghq.com" + ); + } + #[test] fn gets_correct_api_base_endpoint() { assert_eq!( @@ -189,5 +210,12 @@ mod tests { get_api_base_endpoint(Some("https://logs.datadoghq.eu"), DD_US_SITE), "https://api.datadoghq.eu" ); + assert_eq!( + get_api_base_endpoint( + Some("https://http-intake.logs.us3.datadoghq.com"), + DD_US_SITE + ), + "https://api.us3.datadoghq.com" + ); } } From 9efe09d254fdad82f5d1239e3eae28348390ddfa Mon Sep 17 00:00:00 2001 From: Thomas Date: Thu, 16 Apr 2026 20:34:41 -0400 Subject: [PATCH 138/326] fix(ci): modernize cross ubuntu bootstrap script (#25215) * chore(cross): modernize ubuntu bootstrap script for cross-compilation * Pin cross edge images * Install libcurl dev headers for librdkafka build * Configure cmake --- Makefile | 14 +++++++++++ scripts/cross/Dockerfile | 14 +++++++++-- scripts/cross/bootstrap-ubuntu.sh | 39 ++++++++++++++----------------- 3 files changed, 43 insertions(+), 24 deletions(-) diff --git a/Makefile b/Makefile index 821ff961de4ac..847a64069f61a 100644 --- a/Makefile +++ b/Makefile @@ -267,13 +267,27 @@ cross-enable: cargo-install-cross CARGO_HANDLES_FRESHNESS: ${EMPTY} +# Pinned digests for ghcr.io/cross-rs/:edge. +# Source: cross-rs/cross @ f86fd03bb70b4c6802847c18087e21391498b0b4, built 2026-04-10 (Ubuntu 20.04 focal). +# Refresh with: crane digest ghcr.io/cross-rs/:edge +CROSS_DIGEST_x86_64-unknown-linux-gnu := sha256:13f7a68e55cb05a19e840bce65834fc785dc069e0c2218d12b8fdb8f8a1519d5 +CROSS_DIGEST_aarch64-unknown-linux-gnu := sha256:3bf094d22fc4f73c9bdce45ddd7a8bbae349efdbd51b4d4b5ee1bedd8454466b +CROSS_DIGEST_x86_64-unknown-linux-musl := sha256:c59deede3efcd7cb6f6a57641241ba1c63cfe35b7965be09a851242b4209639d +CROSS_DIGEST_aarch64-unknown-linux-musl := sha256:dad492e0f040c6e712d4be9b970c9de5f3b8ef9cde6b9a2b437d56d1dabeb808 +CROSS_DIGEST_armv7-unknown-linux-gnueabihf := sha256:73294ebb06e077e49bbbecfe8f17507e9e0b733a2a1ba23056abcd9c0ba617c9 +CROSS_DIGEST_armv7-unknown-linux-musleabihf := sha256:49bdc9a4cf2f1bcb385389c85be8f43c4399fa6d6fe22883702ef13eb921e443 +CROSS_DIGEST_arm-unknown-linux-gnueabi := sha256:0c70b0e54724bd599dff00a2888f8ea176a5b6c85af47aad9ad25296f63e2967 +CROSS_DIGEST_arm-unknown-linux-musleabi := sha256:0ca8f4afcc29fb5964aa63e482452e8869311a610c5868f22ded400c4e483328 + # GNU Make < 3.82 pattern matching priority depends on the definition order # so cross-image-% must be defined before cross-% .PHONY: cross-image-% cross-image-%: export TRIPLE =$($(strip @):cross-image-%=%) cross-image-%: + $(if $(CROSS_DIGEST_$*),,$(error No CROSS_DIGEST pinned for $*. Add it to the digest table in Makefile.)) $(CONTAINER_TOOL) build \ --build-arg TARGET=${TRIPLE} \ + --build-arg CROSS_DIGEST=$(CROSS_DIGEST_$*) \ --file scripts/cross/Dockerfile \ --tag vector-cross-env:${TRIPLE} \ . diff --git a/scripts/cross/Dockerfile b/scripts/cross/Dockerfile index 80a75caba683c..3abe8cd8b2d11 100644 --- a/scripts/cross/Dockerfile +++ b/scripts/cross/Dockerfile @@ -1,13 +1,23 @@ -ARG CROSS_VERSION=0.2.5 ARG TARGET=x86_64-unknown-linux-musl +ARG CROSS_DIGEST -FROM ghcr.io/cross-rs/${TARGET}:${CROSS_VERSION} +FROM ghcr.io/cross-rs/${TARGET}@${CROSS_DIGEST} # Common steps for all targets COPY scripts/cross/bootstrap-ubuntu.sh / COPY scripts/environment/install-protoc.sh / RUN /bootstrap-ubuntu.sh && bash /install-protoc.sh +# Allow cmake to find pre-built dependencies (e.g. curl from curl-sys) that +# land in CMAKE_PREFIX_PATH rather than the cross sysroot. The cross-rs +# toolchain.cmake sets CMAKE_FIND_ROOT_PATH_MODE_{LIBRARY,INCLUDE} to ONLY, +# which makes cmake ignore CMAKE_PREFIX_PATH entries outside the sysroot. +# Switching to BOTH lets FindCURL (and similar modules) locate libraries that +# Cargo crates like curl-sys build from source and expose via CMAKE_PREFIX_PATH. +RUN if [ -f /opt/toolchain.cmake ]; then \ + printf '\n# Allow finding pre-built cargo dependencies via CMAKE_PREFIX_PATH\nset(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY BOTH)\nset(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE BOTH)\n' >> /opt/toolchain.cmake; \ + fi + # Relocate libstdc++ for musl targets that need it (TODO: investigate if still required) RUN if [ "$TARGET" = "arm-unknown-linux-musleabi" ]; then \ LIBSTDC=/usr/local/arm-linux-musleabi/lib/libstdc++.a; \ diff --git a/scripts/cross/bootstrap-ubuntu.sh b/scripts/cross/bootstrap-ubuntu.sh index 8068d1bcd3eb5..d89ea85f37ca5 100755 --- a/scripts/cross/bootstrap-ubuntu.sh +++ b/scripts/cross/bootstrap-ubuntu.sh @@ -1,31 +1,26 @@ #!/bin/sh set -o errexit -echo 'Acquire::Retries "5";' > /etc/apt/apt.conf.d/80-retries - -apt-get update -apt-get install -y \ - apt-transport-https \ - gnupg \ - wget - -# we need LLVM >= 3.9 for onig_sys/bindgen - -cat <<-EOF > /etc/apt/sources.list.d/llvm.list -deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-9 main -deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-9 main +export DEBIAN_FRONTEND=noninteractive +export ACCEPT_EULA=Y + +# Configure apt for speed and efficiency +cat > /etc/apt/apt.conf.d/90-vector-optimizations < Date: Fri, 17 Apr 2026 13:32:34 -0400 Subject: [PATCH 139/326] chore(dev): split VERSION-dependent packaging targets into Makefile.packaging (#25101) * chore(build): split VERSION-dependent packaging targets into Makefile.packaging * fix(build): unset TRIPLE before delegating to main Makefile in packaging targets --- Makefile | 113 ++------------------------------------- Makefile.packaging | 130 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 135 insertions(+), 108 deletions(-) create mode 100644 Makefile.packaging diff --git a/Makefile b/Makefile index 847a64069f61a..5421a62eed83e 100644 --- a/Makefile +++ b/Makefile @@ -77,9 +77,6 @@ endif export AWS_ACCESS_KEY_ID ?= "dummy" export AWS_SECRET_ACCESS_KEY ?= "dummy" -# Set version -export VERSION ?= $(shell command -v cargo >/dev/null && $(VDEV) version || echo unknown) - # Set if you are on the CI and actually want the things to happen. (Non-CI users should never set this.) export CI ?= false @@ -571,117 +568,17 @@ build-rustdoc: ## Build Vector's Rustdocs # This command is mostly intended for use by the build process in vectordotdev/vector-rustdoc ${MAYBE_ENVIRONMENT_EXEC} cargo doc --no-deps --workspace -##@ Packaging +##@ Packaging (forwarded to Makefile.packaging) -# archives -target/artifacts/vector-${VERSION}-%.tar.gz: export TRIPLE :=$(@:target/artifacts/vector-${VERSION}-%.tar.gz=%) -target/artifacts/vector-${VERSION}-%.tar.gz: override PROFILE =release -target/artifacts/vector-${VERSION}-%.tar.gz: target/%/release/vector.tar.gz - @echo "Built to ${<}, relocating to ${@}" - @mkdir -p target/artifacts/ - @cp -v \ - ${<} \ - ${@} +# Packaging targets that depend on VERSION live in Makefile.packaging to avoid +# running `cargo vdev version` when invoking non-packaging targets. .PHONY: package package: build ## Build the Vector archive ${MAYBE_ENVIRONMENT_EXEC} $(VDEV) package archive -.PHONY: package-x86_64-unknown-linux-gnu-all -package-x86_64-unknown-linux-gnu-all: package-x86_64-unknown-linux-gnu package-deb-x86_64-unknown-linux-gnu package-rpm-x86_64-unknown-linux-gnu # .tar.gz, .deb, .rpm - -.PHONY: package-x86_64-unknown-linux-musl-all -package-x86_64-unknown-linux-musl-all: package-x86_64-unknown-linux-musl # .tar.gz - -.PHONY: package-aarch64-unknown-linux-musl-all -package-aarch64-unknown-linux-musl-all: package-aarch64-unknown-linux-musl # .tar.gz - -.PHONY: package-aarch64-unknown-linux-gnu-all -package-aarch64-unknown-linux-gnu-all: package-aarch64-unknown-linux-gnu package-deb-aarch64 package-rpm-aarch64 # .tar.gz, .deb, .rpm - -.PHONY: package-armv7-unknown-linux-gnueabihf-all -package-armv7-unknown-linux-gnueabihf-all: package-armv7-unknown-linux-gnueabihf package-deb-armv7-gnu package-rpm-armv7hl-gnu # .tar.gz, .deb, .rpm - -.PHONY: package-armv7-unknown-linux-musleabihf-all -package-armv7-unknown-linux-musleabihf-all: package-armv7-unknown-linux-musleabihf # .tar.gz - -.PHONY: package-arm-unknown-linux-gnueabi-all -package-arm-unknown-linux-gnueabi-all: package-arm-unknown-linux-gnueabi package-deb-arm-gnu # .tar.gz, .deb - -.PHONY: package-arm-unknown-linux-musleabi-all -package-arm-unknown-linux-musleabi-all: package-arm-unknown-linux-musleabi # .tar.gz - -.PHONY: package-x86_64-unknown-linux-gnu -package-x86_64-unknown-linux-gnu: target/artifacts/vector-${VERSION}-x86_64-unknown-linux-gnu.tar.gz ## Build an archive suitable for the `x86_64-unknown-linux-gnu` triple. - @echo "Output to ${<}." - -.PHONY: package-x86_64-unknown-linux-musl -package-x86_64-unknown-linux-musl: target/artifacts/vector-${VERSION}-x86_64-unknown-linux-musl.tar.gz ## Build an archive suitable for the `x86_64-unknown-linux-musl` triple. - @echo "Output to ${<}." - -.PHONY: package-aarch64-unknown-linux-musl -package-aarch64-unknown-linux-musl: target/artifacts/vector-${VERSION}-aarch64-unknown-linux-musl.tar.gz ## Build an archive suitable for the `aarch64-unknown-linux-musl` triple. - @echo "Output to ${<}." - -.PHONY: package-aarch64-unknown-linux-gnu -package-aarch64-unknown-linux-gnu: target/artifacts/vector-${VERSION}-aarch64-unknown-linux-gnu.tar.gz ## Build an archive suitable for the `aarch64-unknown-linux-gnu` triple. - @echo "Output to ${<}." - -.PHONY: package-armv7-unknown-linux-gnueabihf -package-armv7-unknown-linux-gnueabihf: target/artifacts/vector-${VERSION}-armv7-unknown-linux-gnueabihf.tar.gz ## Build an archive suitable for the `armv7-unknown-linux-gnueabihf` triple. - @echo "Output to ${<}." - -.PHONY: package-armv7-unknown-linux-musleabihf -package-armv7-unknown-linux-musleabihf: target/artifacts/vector-${VERSION}-armv7-unknown-linux-musleabihf.tar.gz ## Build an archive suitable for the `armv7-unknown-linux-musleabihf triple. - @echo "Output to ${<}." - -.PHONY: package-arm-unknown-linux-gnueabi -package-arm-unknown-linux-gnueabi: target/artifacts/vector-${VERSION}-arm-unknown-linux-gnueabi.tar.gz ## Build an archive suitable for the `arm-unknown-linux-gnueabi` triple. - @echo "Output to ${<}." - -.PHONY: package-arm-unknown-linux-musleabi -package-arm-unknown-linux-musleabi: target/artifacts/vector-${VERSION}-arm-unknown-linux-musleabi.tar.gz ## Build an archive suitable for the `arm-unknown-linux-musleabi` triple. - @echo "Output to ${<}." - -# debs - -.PHONY: package-deb-x86_64-unknown-linux-gnu -package-deb-x86_64-unknown-linux-gnu: package-x86_64-unknown-linux-gnu ## Build the x86_64 GNU deb package - TARGET=x86_64-unknown-linux-gnu $(VDEV) package deb - -.PHONY: package-deb-x86_64-unknown-linux-musl -package-deb-x86_64-unknown-linux-musl: package-x86_64-unknown-linux-musl ## Build the x86_64 GNU deb package - TARGET=x86_64-unknown-linux-musl $(VDEV) package deb - -.PHONY: package-deb-aarch64 -package-deb-aarch64: package-aarch64-unknown-linux-gnu ## Build the aarch64 deb package - TARGET=aarch64-unknown-linux-gnu $(VDEV) package deb - -.PHONY: package-deb-armv7-gnu -package-deb-armv7-gnu: package-armv7-unknown-linux-gnueabihf ## Build the armv7-unknown-linux-gnueabihf deb package - TARGET=armv7-unknown-linux-gnueabihf $(VDEV) package deb - -.PHONY: package-deb-arm-gnu -package-deb-arm-gnu: package-arm-unknown-linux-gnueabi ## Build the arm-unknown-linux-gnueabi deb package - TARGET=arm-unknown-linux-gnueabi $(VDEV) package deb - -# rpms - -.PHONY: package-rpm-x86_64-unknown-linux-gnu -package-rpm-x86_64-unknown-linux-gnu: package-x86_64-unknown-linux-gnu ## Build the x86_64 rpm package - TARGET=x86_64-unknown-linux-gnu $(VDEV) package rpm - -.PHONY: package-rpm-x86_64-unknown-linux-musl -package-rpm-x86_64-unknown-linux-musl: package-x86_64-unknown-linux-musl ## Build the x86_64 musl rpm package - TARGET=x86_64-unknown-linux-musl $(VDEV) package rpm - -.PHONY: package-rpm-aarch64 -package-rpm-aarch64: package-aarch64-unknown-linux-gnu ## Build the aarch64 rpm package - TARGET=aarch64-unknown-linux-gnu $(VDEV) package rpm - -.PHONY: package-rpm-armv7hl-gnu -package-rpm-armv7hl-gnu: package-armv7-unknown-linux-gnueabihf ## Build the armv7hl-unknown-linux-gnueabihf rpm package - TARGET=armv7-unknown-linux-gnueabihf ARCH=armv7hl $(VDEV) package rpm +package-%: + $(MAKE) -f Makefile.packaging $@ ##@ Releasing diff --git a/Makefile.packaging b/Makefile.packaging new file mode 100644 index 0000000000000..032e334094d19 --- /dev/null +++ b/Makefile.packaging @@ -0,0 +1,130 @@ +# Makefile.packaging +# +# Packaging and release targets for Vector. +# Separated from the main Makefile so that `cargo vdev version` (needed for +# VERSION) is only executed when packaging/release targets are invoked. +# +# Usage: +# make -f Makefile.packaging +# +# These targets are also accessible via the main Makefile, which forwards +# packaging and release targets here automatically. + +VDEV := cargo vdev + +# Set version — only evaluated when this Makefile is loaded +export VERSION ?= $(shell command -v cargo >/dev/null && $(VDEV) version || echo unknown) + +# Delegate cross-compilation prerequisites to the main Makefile +target/%/release/vector.tar.gz: + unset TRIPLE && $(MAKE) $@ + +##@ Packaging + +# archives +target/artifacts/vector-${VERSION}-%.tar.gz: export TRIPLE :=$(@:target/artifacts/vector-${VERSION}-%.tar.gz=%) +target/artifacts/vector-${VERSION}-%.tar.gz: override PROFILE =release +target/artifacts/vector-${VERSION}-%.tar.gz: target/%/release/vector.tar.gz + @echo "Built to ${<}, relocating to ${@}" + @mkdir -p target/artifacts/ + @cp -v \ + ${<} \ + ${@} + +.PHONY: package-x86_64-unknown-linux-gnu-all +package-x86_64-unknown-linux-gnu-all: package-x86_64-unknown-linux-gnu package-deb-x86_64-unknown-linux-gnu package-rpm-x86_64-unknown-linux-gnu # .tar.gz, .deb, .rpm + +.PHONY: package-x86_64-unknown-linux-musl-all +package-x86_64-unknown-linux-musl-all: package-x86_64-unknown-linux-musl # .tar.gz + +.PHONY: package-aarch64-unknown-linux-musl-all +package-aarch64-unknown-linux-musl-all: package-aarch64-unknown-linux-musl # .tar.gz + +.PHONY: package-aarch64-unknown-linux-gnu-all +package-aarch64-unknown-linux-gnu-all: package-aarch64-unknown-linux-gnu package-deb-aarch64 package-rpm-aarch64 # .tar.gz, .deb, .rpm + +.PHONY: package-armv7-unknown-linux-gnueabihf-all +package-armv7-unknown-linux-gnueabihf-all: package-armv7-unknown-linux-gnueabihf package-deb-armv7-gnu package-rpm-armv7hl-gnu # .tar.gz, .deb, .rpm + +.PHONY: package-armv7-unknown-linux-musleabihf-all +package-armv7-unknown-linux-musleabihf-all: package-armv7-unknown-linux-musleabihf # .tar.gz + +.PHONY: package-arm-unknown-linux-gnueabi-all +package-arm-unknown-linux-gnueabi-all: package-arm-unknown-linux-gnueabi package-deb-arm-gnu # .tar.gz, .deb + +.PHONY: package-arm-unknown-linux-musleabi-all +package-arm-unknown-linux-musleabi-all: package-arm-unknown-linux-musleabi # .tar.gz + +.PHONY: package-x86_64-unknown-linux-gnu +package-x86_64-unknown-linux-gnu: target/artifacts/vector-${VERSION}-x86_64-unknown-linux-gnu.tar.gz ## Build an archive suitable for the `x86_64-unknown-linux-gnu` triple. + @echo "Output to ${<}." + +.PHONY: package-x86_64-unknown-linux-musl +package-x86_64-unknown-linux-musl: target/artifacts/vector-${VERSION}-x86_64-unknown-linux-musl.tar.gz ## Build an archive suitable for the `x86_64-unknown-linux-musl` triple. + @echo "Output to ${<}." + +.PHONY: package-aarch64-unknown-linux-musl +package-aarch64-unknown-linux-musl: target/artifacts/vector-${VERSION}-aarch64-unknown-linux-musl.tar.gz ## Build an archive suitable for the `aarch64-unknown-linux-musl` triple. + @echo "Output to ${<}." + +.PHONY: package-aarch64-unknown-linux-gnu +package-aarch64-unknown-linux-gnu: target/artifacts/vector-${VERSION}-aarch64-unknown-linux-gnu.tar.gz ## Build an archive suitable for the `aarch64-unknown-linux-gnu` triple. + @echo "Output to ${<}." + +.PHONY: package-armv7-unknown-linux-gnueabihf +package-armv7-unknown-linux-gnueabihf: target/artifacts/vector-${VERSION}-armv7-unknown-linux-gnueabihf.tar.gz ## Build an archive suitable for the `armv7-unknown-linux-gnueabihf` triple. + @echo "Output to ${<}." + +.PHONY: package-armv7-unknown-linux-musleabihf +package-armv7-unknown-linux-musleabihf: target/artifacts/vector-${VERSION}-armv7-unknown-linux-musleabihf.tar.gz ## Build an archive suitable for the `armv7-unknown-linux-musleabihf triple. + @echo "Output to ${<}." + +.PHONY: package-arm-unknown-linux-gnueabi +package-arm-unknown-linux-gnueabi: target/artifacts/vector-${VERSION}-arm-unknown-linux-gnueabi.tar.gz ## Build an archive suitable for the `arm-unknown-linux-gnueabi` triple. + @echo "Output to ${<}." + +.PHONY: package-arm-unknown-linux-musleabi +package-arm-unknown-linux-musleabi: target/artifacts/vector-${VERSION}-arm-unknown-linux-musleabi.tar.gz ## Build an archive suitable for the `arm-unknown-linux-musleabi` triple. + @echo "Output to ${<}." + +# debs + +.PHONY: package-deb-x86_64-unknown-linux-gnu +package-deb-x86_64-unknown-linux-gnu: package-x86_64-unknown-linux-gnu ## Build the x86_64 GNU deb package + TARGET=x86_64-unknown-linux-gnu $(VDEV) package deb + +.PHONY: package-deb-x86_64-unknown-linux-musl +package-deb-x86_64-unknown-linux-musl: package-x86_64-unknown-linux-musl ## Build the x86_64 GNU deb package + TARGET=x86_64-unknown-linux-musl $(VDEV) package deb + +.PHONY: package-deb-aarch64 +package-deb-aarch64: package-aarch64-unknown-linux-gnu ## Build the aarch64 deb package + TARGET=aarch64-unknown-linux-gnu $(VDEV) package deb + +.PHONY: package-deb-armv7-gnu +package-deb-armv7-gnu: package-armv7-unknown-linux-gnueabihf ## Build the armv7-unknown-linux-gnueabihf deb package + TARGET=armv7-unknown-linux-gnueabihf $(VDEV) package deb + +.PHONY: package-deb-arm-gnu +package-deb-arm-gnu: package-arm-unknown-linux-gnueabi ## Build the arm-unknown-linux-gnueabi deb package + TARGET=arm-unknown-linux-gnueabi $(VDEV) package deb + +# rpms + +.PHONY: package-rpm-x86_64-unknown-linux-gnu +package-rpm-x86_64-unknown-linux-gnu: package-x86_64-unknown-linux-gnu ## Build the x86_64 rpm package + TARGET=x86_64-unknown-linux-gnu $(VDEV) package rpm + +.PHONY: package-rpm-x86_64-unknown-linux-musl +package-rpm-x86_64-unknown-linux-musl: package-x86_64-unknown-linux-musl ## Build the x86_64 musl rpm package + TARGET=x86_64-unknown-linux-musl $(VDEV) package rpm + +.PHONY: package-rpm-aarch64 +package-rpm-aarch64: package-aarch64-unknown-linux-gnu ## Build the aarch64 rpm package + TARGET=aarch64-unknown-linux-gnu $(VDEV) package rpm + +.PHONY: package-rpm-armv7hl-gnu +package-rpm-armv7hl-gnu: package-armv7-unknown-linux-gnueabihf ## Build the armv7hl-unknown-linux-gnueabihf rpm package + TARGET=armv7-unknown-linux-gnueabihf ARCH=armv7hl $(VDEV) package rpm + + From 35351b9cc9ae7dffdf1a461203137905f987a8b5 Mon Sep 17 00:00:00 2001 From: Thomas Date: Fri, 17 Apr 2026 17:22:12 -0400 Subject: [PATCH 140/326] chore(codecs): replace native encoding fixture patch files with cfg-gated code (#24971) * cfg-gate fixture generating code * Use generate-fixtures vrl branch * Move fixture generation into its own binary and use a set SEED * Gate from_size_and_seed behind generate-fixtures * Remove vrl_generate_fixtures.patch * Format * Fix clippy warnings * Fix markdown * Format again * fix(codecs): normalize -0.0 after rounding in fixture generation * chore(deps): update vrl dependency to main branch * Update VRL * fix(ci): enable vrl-functions-crypto for http-client integration tests --------- Co-authored-by: Pavlos Rontidis --- Cargo.lock | 19 +- Cargo.toml | 2 +- .../tests/data/native_encoding/README.md | 31 ++-- .../vector_generate_fixtures.patch | 163 ------------------ .../vrl_generate_fixtures.patch | 69 -------- lib/vector-core/Cargo.toml | 7 + lib/vector-core/src/bin/generate_fixtures.rs | 38 ++++ .../{test/common.rs => arbitrary_impl.rs} | 52 ++++-- lib/vector-core/src/event/metric/tags.rs | 4 +- lib/vector-core/src/event/mod.rs | 3 + lib/vector-core/src/event/test/mod.rs | 1 - .../src/event/test/serialization.rs | 5 +- lib/vector-core/src/event/test/size_of.rs | 3 +- lib/vector-core/src/metrics/ddsketch.rs | 11 ++ 14 files changed, 130 insertions(+), 278 deletions(-) delete mode 100644 lib/codecs/tests/data/native_encoding/vector_generate_fixtures.patch delete mode 100644 lib/codecs/tests/data/native_encoding/vrl_generate_fixtures.patch create mode 100644 lib/vector-core/src/bin/generate_fixtures.rs rename lib/vector-core/src/event/{test/common.rs => arbitrary_impl.rs} (91%) diff --git a/Cargo.lock b/Cargo.lock index 2d641cc2a2934..3215e97f0fd44 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1781,7 +1781,7 @@ dependencies = [ "bitflags 2.10.0", "cexpr", "clang-sys", - "itertools 0.13.0", + "itertools 0.10.5", "proc-macro2 1.0.106", "quote 1.0.44", "regex", @@ -8251,7 +8251,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4" dependencies = [ "bytes", - "heck 0.4.1", + "heck 0.5.0", "itertools 0.12.1", "log", "multimap", @@ -8271,8 +8271,8 @@ version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ - "heck 0.4.1", - "itertools 0.14.0", + "heck 0.5.0", + "itertools 0.10.5", "log", "multimap", "once_cell", @@ -8318,7 +8318,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.10.5", "proc-macro2 1.0.106", "quote 1.0.44", "syn 2.0.117", @@ -10323,7 +10323,7 @@ version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "proc-macro2 1.0.106", "quote 1.0.44", "syn 2.0.117", @@ -10335,7 +10335,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54254b8531cafa275c5e096f62d48c81435d1015405a91198ddb11e967301d40" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "proc-macro2 1.0.106", "quote 1.0.44", "syn 2.0.117", @@ -12932,7 +12932,7 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "vrl" version = "0.31.0" -source = "git+https://github.com/vectordotdev/vrl.git?branch=main#d21e53192e6a3ed3d38ef6f32886f1cbf30d308c" +source = "git+https://github.com/vectordotdev/vrl.git?branch=main#1357941d4481e84fb0323bd89c69cedf35c323f8" dependencies = [ "aes", "aes-siv", @@ -12966,6 +12966,7 @@ dependencies = [ "exitcode", "fancy-regex", "flate2", + "getrandom 0.3.4", "grok", "hex", "hmac", @@ -13005,7 +13006,7 @@ dependencies = [ "publicsuffix", "quickcheck", "quoted_printable", - "rand 0.8.5", + "rand 0.9.4", "regex", "relative-path 2.0.1", "reqwest 0.12.28", diff --git a/Cargo.toml b/Cargo.toml index 84ded822b5790..03aff35ec8c13 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1031,7 +1031,7 @@ gcp-integration-tests = ["sinks-gcp"] gcp-pubsub-integration-tests = ["sinks-gcp", "sources-gcp_pubsub"] greptimedb-integration-tests = ["sinks-greptimedb_metrics", "sinks-greptimedb_logs"] humio-integration-tests = ["sinks-humio"] -http-client-integration-tests = ["sources-http_client"] +http-client-integration-tests = ["sources-http_client", "vrl-functions-crypto"] influxdb-integration-tests = ["sinks-influxdb"] kafka-integration-tests = ["sinks-kafka", "sources-kafka"] logstash-integration-tests = ["docker", "sources-logstash"] diff --git a/lib/codecs/tests/data/native_encoding/README.md b/lib/codecs/tests/data/native_encoding/README.md index 0a84c98043c41..2cf2aaba974be 100644 --- a/lib/codecs/tests/data/native_encoding/README.md +++ b/lib/codecs/tests/data/native_encoding/README.md @@ -6,28 +6,23 @@ and we test that all the examples can be successfully parsed, parse the same across both formats, and match the current serialized format. In order to avoid small inherent serialization differences between JSON and -protobuf (e.g. float handling), some changes were made to the `Arbitrary` -implementation for `Event` to give simpler values. These are not changes we want -in most property testing scenarios, but they are appropriate in this case where -we only care about the overall structure of the events. +protobuf (e.g. float handling), the `generate-fixtures` feature flag in +`vector-core` activates a stricter `Arbitrary` implementation for `Event` that +produces simpler, round-trip-safe f64 values and non-empty field names. These +changes are intentionally scoped to fixture generation and not used in regular +property testing. -There is currently a multi-step procedure to re-generate the data files. -There are two diffs committed to this directory: - - `vector_generate_fixtures.patch` - - `vrl_generate_fixtures.patch` +## Re-generating fixtures -The `vrl_` one must be applied to the vectordotdev/vrl repo. -The `vector_` one must be applied to the vector repo (you are here). +Both this repo and the VRL repo have a `generate-fixtures` feature flag that +activates fixture-stable `Arbitrary` implementations. The vector-core +`generate-fixtures` feature automatically enables `vrl/generate-fixtures`. -Part of the vector patch file is a `roundtrip` unit test definition that needs -to be evoked from `lib/vector-core`. Before invoking it, the `_json` and `_proto` -directories need to be created. +### Run the generator ```bash - $ cd lib/vector-core - $ mkdir _json/ proto/ - $ cargo test event::test::serialization::roundtrip +cargo run -p vector-core --features generate-fixtures --bin generate-fixtures ``` -That test case writes out the appropriate files into the dirs, which then need to be -moved to their location here. +The binary writes files directly into this directory's `json/` and `proto/` +subdirectories, replacing the existing fixtures. diff --git a/lib/codecs/tests/data/native_encoding/vector_generate_fixtures.patch b/lib/codecs/tests/data/native_encoding/vector_generate_fixtures.patch deleted file mode 100644 index a0262928b3b3b..0000000000000 --- a/lib/codecs/tests/data/native_encoding/vector_generate_fixtures.patch +++ /dev/null @@ -1,163 +0,0 @@ -diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml -index 0f51fc830..aadf99841 100644 ---- a/lib/vector-core/Cargo.toml -+++ b/lib/vector-core/Cargo.toml -@@ -95,7 +95,7 @@ rand = "0.8.5" - rand_distr = "0.4.3" - tracing-subscriber = { version = "0.3.17", default-features = false, features = ["env-filter", "fmt", "ansi", "registry"] } - vector-common = { path = "../vector-common", default-features = false, features = ["test"] } --vrl = { version = "0.9.0", features = ["cli", "test", "test_framework", "arbitrary"] } -+vrl = { path = "../../../vrl", features = ["cli", "test", "test_framework", "arbitrary"] } - - [features] - api = ["dep:async-graphql"] -diff --git a/lib/vector-core/src/event/test/common.rs b/lib/vector-core/src/event/test/common.rs -index c7ccca952..f851288ce 100644 ---- a/lib/vector-core/src/event/test/common.rs -+++ b/lib/vector-core/src/event/test/common.rs -@@ -26,6 +26,15 @@ const ALPHABET: [&str; 27] = [ - "t", "u", "v", "w", "x", "y", "z", "_", - ]; - -+fn make_simple_f64(g: &mut Gen) -> f64 { -+ let mut value = f64::arbitrary(g) % MAX_F64_SIZE; -+ while value.is_nan() || value == -0.0 { -+ value = f64::arbitrary(g) % MAX_F64_SIZE; -+ } -+ value = (value * 10_000.0).round() / 10_000.0; -+ value -+} -+ - #[derive(Debug, Clone)] - pub struct Name { - inner: String, -@@ -34,7 +43,7 @@ pub struct Name { - impl Arbitrary for Name { - fn arbitrary(g: &mut Gen) -> Self { - let mut name = String::with_capacity(MAX_STR_SIZE); -- for _ in 0..(g.size() % MAX_STR_SIZE) { -+ for _ in 0..(usize::max(1, g.size() % MAX_STR_SIZE)) { - let idx: usize = usize::arbitrary(g) % ALPHABET.len(); - name.push_str(ALPHABET[idx]); - } -@@ -182,10 +191,10 @@ impl Arbitrary for MetricValue { - // here toward `MetricValue::Counter` and `MetricValue::Gauge`. - match u8::arbitrary(g) % 7 { - 0 => MetricValue::Counter { -- value: f64::arbitrary(g) % MAX_F64_SIZE, -+ value: make_simple_f64(g), - }, - 1 => MetricValue::Gauge { -- value: f64::arbitrary(g) % MAX_F64_SIZE, -+ value: make_simple_f64(g), - }, - 2 => MetricValue::Set { - values: BTreeSet::arbitrary(g), -@@ -197,19 +206,20 @@ impl Arbitrary for MetricValue { - 4 => MetricValue::AggregatedHistogram { - buckets: Vec::arbitrary(g), - count: u64::arbitrary(g), -- sum: f64::arbitrary(g) % MAX_F64_SIZE, -+ sum: make_simple_f64(g), - }, - 5 => MetricValue::AggregatedSummary { - quantiles: Vec::arbitrary(g), - count: u64::arbitrary(g), -- sum: f64::arbitrary(g) % MAX_F64_SIZE, -+ sum: make_simple_f64(g), - }, - 6 => { - // We're working around quickcheck's limitations here, and - // should really migrate the tests in question to use proptest - let num_samples = u8::arbitrary(g); - let samples = std::iter::repeat_with(|| loop { -- let f = f64::arbitrary(g); -+ // let f = f64::arbitrary(g); -+ let f = make_simple_f64(g); - if f.is_normal() { - return f; - } -@@ -219,6 +229,8 @@ impl Arbitrary for MetricValue { - - let mut sketch = AgentDDSketch::with_agent_defaults(); - sketch.insert_many(&samples); -+ sketch.sum = make_simple_f64(g); -+ sketch.avg = make_simple_f64(g); - - MetricValue::Sketch { - sketch: MetricSketch::AgentDDSketch(sketch), -@@ -368,7 +380,7 @@ impl Arbitrary for MetricValue { - impl Arbitrary for Sample { - fn arbitrary(g: &mut Gen) -> Self { - Sample { -- value: f64::arbitrary(g) % MAX_F64_SIZE, -+ value: make_simple_f64(g), - rate: u32::arbitrary(g), - } - } -@@ -398,8 +410,8 @@ impl Arbitrary for Sample { - impl Arbitrary for Quantile { - fn arbitrary(g: &mut Gen) -> Self { - Quantile { -- quantile: f64::arbitrary(g) % MAX_F64_SIZE, -- value: f64::arbitrary(g) % MAX_F64_SIZE, -+ quantile: make_simple_f64(g), -+ value: make_simple_f64(g), - } - } - -@@ -428,7 +440,7 @@ impl Arbitrary for Quantile { - impl Arbitrary for Bucket { - fn arbitrary(g: &mut Gen) -> Self { - Bucket { -- upper_limit: f64::arbitrary(g) % MAX_F64_SIZE, -+ upper_limit: make_simple_f64(g), - count: u64::arbitrary(g), - } - } -diff --git a/lib/vector-core/src/event/test/serialization.rs b/lib/vector-core/src/event/test/serialization.rs -index aaab559da..5db6c0613 100644 ---- a/lib/vector-core/src/event/test/serialization.rs -+++ b/lib/vector-core/src/event/test/serialization.rs -@@ -96,3 +96,24 @@ fn type_serialization() { - assert_eq!(map["bool"], json!(true)); - assert_eq!(map["string"], json!("thisisastring")); - } -+ -+#[test] -+fn roundtrip() { -+ use prost::Message; -+ use quickcheck::{Arbitrary, Gen}; -+ use std::{fs::File, io::Write}; -+ -+ let mut gen = Gen::new(128); -+ for n in 0..1024 { -+ let mut json_out = File::create(format!("_json/{n:04}.json")).unwrap(); -+ let mut proto_out = File::create(format!("_proto/{n:04}.pb")).unwrap(); -+ let event = Event::arbitrary(&mut gen); -+ serde_json::to_writer(&mut json_out, &event).unwrap(); -+ -+ let array = EventArray::from(event); -+ let proto = proto::EventArray::from(array); -+ let mut buf = BytesMut::new(); -+ proto.encode(&mut buf).unwrap(); -+ proto_out.write_all(&buf).unwrap(); -+ } -+} -diff --git a/lib/vector-core/src/metrics/ddsketch.rs b/lib/vector-core/src/metrics/ddsketch.rs -index 3d0f80bb5..fcc6bcc97 100644 ---- a/lib/vector-core/src/metrics/ddsketch.rs -+++ b/lib/vector-core/src/metrics/ddsketch.rs -@@ -229,10 +229,10 @@ pub struct AgentDDSketch { - max: f64, - - /// The sum of all observations within the sketch. -- sum: f64, -+ pub sum: f64, - - /// The average value of all observations within the sketch. -- avg: f64, -+ pub avg: f64, - } - - impl AgentDDSketch { diff --git a/lib/codecs/tests/data/native_encoding/vrl_generate_fixtures.patch b/lib/codecs/tests/data/native_encoding/vrl_generate_fixtures.patch deleted file mode 100644 index 430b74abb8b66..0000000000000 --- a/lib/codecs/tests/data/native_encoding/vrl_generate_fixtures.patch +++ /dev/null @@ -1,69 +0,0 @@ -diff --git a/src/lib.rs b/src/lib.rs -index 29cad8dfc..0b166beb2 100644 ---- a/src/lib.rs -+++ b/src/lib.rs -@@ -1,4 +1,4 @@ --#![deny(warnings)] -+// #![deny(warnings)] - #![deny(clippy::all)] - #![deny(unused_allocation)] - #![deny(unused_extern_crates)] -diff --git a/src/value/value/arbitrary.rs b/src/value/value/arbitrary.rs -index dde213281..33b2f2518 100644 ---- a/src/value/value/arbitrary.rs -+++ b/src/value/value/arbitrary.rs -@@ -1,7 +1,6 @@ - use std::collections::BTreeMap; - - use bytes::Bytes; --use chrono::{DateTime, NaiveDateTime, Utc}; - use ordered_float::NotNan; - use quickcheck::{Arbitrary, Gen}; - -@@ -11,16 +10,13 @@ const MAX_ARRAY_SIZE: usize = 4; - const MAX_MAP_SIZE: usize = 4; - const MAX_F64_SIZE: f64 = 1_000_000.0; - --fn datetime(g: &mut Gen) -> DateTime { -- // `chrono` documents that there is an out-of-range for both second and -- // nanosecond values but doesn't actually document what the valid ranges -- // are. We just sort of arbitrarily restrict things. -- let secs = i64::arbitrary(g) % 32_000; -- let nanosecs = u32::arbitrary(g) % 32_000; -- DateTime::::from_utc( -- NaiveDateTime::from_timestamp_opt(secs, nanosecs).expect("invalid timestamp"), -- Utc, -- ) -+fn make_simple_f64(g: &mut Gen) -> f64 { -+ let mut value = f64::arbitrary(g) % MAX_F64_SIZE; -+ while value.is_nan() || value == -0.0 { -+ value = f64::arbitrary(g) % MAX_F64_SIZE; -+ } -+ value = (value * 10_000.0).round() / 10_000.0; -+ value - } - - impl Arbitrary for Value { -@@ -32,18 +28,18 @@ impl Arbitrary for Value { - // field picking. - match u8::arbitrary(g) % 8 { - 0 => { -- let bytes: Vec = Vec::arbitrary(g); -+ let bytes = String::arbitrary(g); - Self::Bytes(Bytes::from(bytes)) - } - 1 => Self::Integer(i64::arbitrary(g)), - 2 => { -- let f = f64::arbitrary(g) % MAX_F64_SIZE; -+ //let f = f64::arbitrary(g) % MAX_F64_SIZE; -+ let f = make_simple_f64(g); - let not_nan = NotNan::new(f).unwrap_or_else(|_| NotNan::new(0.0).unwrap()); - Self::from(not_nan) - } - 3 => Self::Boolean(bool::arbitrary(g)), -- 4 => Self::Timestamp(datetime(g)), -- 5 => { -+ 4 | 5 => { - let mut gen = Gen::new(MAX_MAP_SIZE); - Self::Object(BTreeMap::arbitrary(&mut gen)) - } diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml index 8a512884b2c32..7d2d8f8979266 100644 --- a/lib/vector-core/Cargo.toml +++ b/lib/vector-core/Cargo.toml @@ -65,6 +65,7 @@ vector-config = { path = "../vector-config" } vector-config-common = { path = "../vector-config-common" } vrl.workspace = true cfg-if.workspace = true +quickcheck = { workspace = true, optional = true } [target.'cfg(target_os = "macos")'.dependencies] security-framework = "3.6.0" @@ -100,6 +101,12 @@ default = [] lua = ["dep:mlua", "dep:tokio-stream", "vrl/lua"] vrl = [] test = ["vector-common/test", "proptest"] +generate-fixtures = ["vrl/generate-fixtures", "dep:quickcheck"] + +[[bin]] +name = "generate-fixtures" +path = "src/bin/generate_fixtures.rs" +required-features = ["generate-fixtures"] [[bench]] name = "event" diff --git a/lib/vector-core/src/bin/generate_fixtures.rs b/lib/vector-core/src/bin/generate_fixtures.rs new file mode 100644 index 0000000000000..9ab133757bbf8 --- /dev/null +++ b/lib/vector-core/src/bin/generate_fixtures.rs @@ -0,0 +1,38 @@ +use std::{fs::File, io::Write, path::PathBuf}; + +use bytes::BytesMut; +use prost::Message; +use quickcheck::{Arbitrary as _, Gen}; +use vector_core::event::{Event, EventArray, proto}; + +const SEED: u64 = 0; +const GEN_SIZE: usize = 128; + +fn main() { + let fixture_dir = + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../codecs/tests/data/native_encoding"); + let json_dir = fixture_dir.join("json"); + let proto_dir = fixture_dir.join("proto"); + std::fs::create_dir_all(&json_dir).unwrap(); + std::fs::create_dir_all(&proto_dir).unwrap(); + + let mut rng = Gen::from_size_and_seed(GEN_SIZE, SEED); + for n in 0..1024_usize { + let event = Event::arbitrary(&mut rng); + + let mut json_out = File::create(json_dir.join(format!("{n:04}.json"))).unwrap(); + serde_json::to_writer(&mut json_out, &event).unwrap(); + + let mut proto_out = File::create(proto_dir.join(format!("{n:04}.pb"))).unwrap(); + let mut buf = BytesMut::new(); + proto::EventArray::from(EventArray::from(event)) + .encode(&mut buf) + .unwrap(); + proto_out.write_all(&buf).unwrap(); + } + + #[allow(clippy::print_stdout)] + { + println!("Written 1024 fixtures to {}", fixture_dir.display()); + } +} diff --git a/lib/vector-core/src/event/test/common.rs b/lib/vector-core/src/event/arbitrary_impl.rs similarity index 91% rename from lib/vector-core/src/event/test/common.rs rename to lib/vector-core/src/event/arbitrary_impl.rs index ab9460a7fa90f..fe02b846aa9b2 100644 --- a/lib/vector-core/src/event/test/common.rs +++ b/lib/vector-core/src/event/arbitrary_impl.rs @@ -4,7 +4,7 @@ use chrono::{DateTime, Utc}; use quickcheck::{Arbitrary, Gen, empty_shrinker}; use vrl::value::{ObjectMap, Value}; -use super::super::{ +use super::{ Event, EventMetadata, LogEvent, Metric, MetricKind, MetricValue, StatisticKind, TraceEvent, metric::{ Bucket, MetricData, MetricName, MetricSeries, MetricSketch, MetricTags, MetricTime, @@ -21,6 +21,27 @@ const ALPHABET: [&str; 27] = [ "t", "u", "v", "w", "x", "y", "z", "_", ]; +// When generating fixtures we need f64 values that survive a JSON round-trip +// without any loss of precision or serialization ambiguity (NaN, -0.0). +// Under the `generate-fixtures` feature the helper produces clean values; +// otherwise it falls back to the standard quickcheck approach. +fn f64_for_arbitrary(g: &mut Gen) -> f64 { + #[cfg(feature = "generate-fixtures")] + { + let mut value = f64::arbitrary(g) % MAX_F64_SIZE; + while value.is_nan() || value == -0.0 { + value = f64::arbitrary(g) % MAX_F64_SIZE; + } + let rounded = (value * 10_000.0).round() / 10_000.0; + // Rounding can produce -0.0 from small negatives; normalize to +0.0. + if rounded == -0.0_f64 { 0.0 } else { rounded } + } + #[cfg(not(feature = "generate-fixtures"))] + { + f64::arbitrary(g) % MAX_F64_SIZE + } +} + #[derive(Debug, Clone)] pub struct Name { inner: String, @@ -29,7 +50,11 @@ pub struct Name { impl Arbitrary for Name { fn arbitrary(g: &mut Gen) -> Self { let mut name = String::with_capacity(MAX_STR_SIZE); - for _ in 0..(g.size() % MAX_STR_SIZE) { + #[cfg(feature = "generate-fixtures")] + let len = usize::max(1, g.size() % MAX_STR_SIZE); + #[cfg(not(feature = "generate-fixtures"))] + let len = g.size() % MAX_STR_SIZE; + for _ in 0..len { let idx: usize = usize::arbitrary(g) % ALPHABET.len(); name.push_str(ALPHABET[idx]); } @@ -76,6 +101,9 @@ impl Arbitrary for Event { impl Arbitrary for LogEvent { fn arbitrary(g: &mut Gen) -> Self { + #[cfg(feature = "generate-fixtures")] + let mut generator = Gen::from_size_and_seed(MAX_MAP_SIZE, u64::arbitrary(g)); + #[cfg(not(feature = "generate-fixtures"))] let mut generator = Gen::new(MAX_MAP_SIZE); let map: ObjectMap = ObjectMap::arbitrary(&mut generator); let metadata: EventMetadata = EventMetadata::arbitrary(g); @@ -175,10 +203,10 @@ impl Arbitrary for MetricValue { // here toward `MetricValue::Counter` and `MetricValue::Gauge`. match u8::arbitrary(g) % 7 { 0 => MetricValue::Counter { - value: f64::arbitrary(g) % MAX_F64_SIZE, + value: f64_for_arbitrary(g), }, 1 => MetricValue::Gauge { - value: f64::arbitrary(g) % MAX_F64_SIZE, + value: f64_for_arbitrary(g), }, 2 => MetricValue::Set { values: BTreeSet::arbitrary(g), @@ -190,12 +218,12 @@ impl Arbitrary for MetricValue { 4 => MetricValue::AggregatedHistogram { buckets: Vec::arbitrary(g), count: u64::arbitrary(g), - sum: f64::arbitrary(g) % MAX_F64_SIZE, + sum: f64_for_arbitrary(g), }, 5 => MetricValue::AggregatedSummary { quantiles: Vec::arbitrary(g), count: u64::arbitrary(g), - sum: f64::arbitrary(g) % MAX_F64_SIZE, + sum: f64_for_arbitrary(g), }, 6 => { // We're working around quickcheck's limitations here, and @@ -203,7 +231,7 @@ impl Arbitrary for MetricValue { let num_samples = u8::arbitrary(g); let samples = std::iter::repeat_with(|| { loop { - let f = f64::arbitrary(g); + let f = f64_for_arbitrary(g); if f.is_normal() { return f; } @@ -214,6 +242,8 @@ impl Arbitrary for MetricValue { let mut sketch = AgentDDSketch::with_agent_defaults(); sketch.insert_many(&samples); + #[cfg(feature = "generate-fixtures")] + sketch.set_sum_avg(f64_for_arbitrary(g), f64_for_arbitrary(g)); MetricValue::Sketch { sketch: MetricSketch::AgentDDSketch(sketch), @@ -363,7 +393,7 @@ impl Arbitrary for MetricValue { impl Arbitrary for Sample { fn arbitrary(g: &mut Gen) -> Self { Sample { - value: f64::arbitrary(g) % MAX_F64_SIZE, + value: f64_for_arbitrary(g), rate: u32::arbitrary(g), } } @@ -393,8 +423,8 @@ impl Arbitrary for Sample { impl Arbitrary for Quantile { fn arbitrary(g: &mut Gen) -> Self { Quantile { - quantile: f64::arbitrary(g) % MAX_F64_SIZE, - value: f64::arbitrary(g) % MAX_F64_SIZE, + quantile: f64_for_arbitrary(g), + value: f64_for_arbitrary(g), } } @@ -423,7 +453,7 @@ impl Arbitrary for Quantile { impl Arbitrary for Bucket { fn arbitrary(g: &mut Gen) -> Self { Bucket { - upper_limit: f64::arbitrary(g) % MAX_F64_SIZE, + upper_limit: f64_for_arbitrary(g), count: u64::arbitrary(g), } } diff --git a/lib/vector-core/src/event/metric/tags.rs b/lib/vector-core/src/event/metric/tags.rs index 416f239af5ab5..52c9a365d2534 100644 --- a/lib/vector-core/src/event/metric/tags.rs +++ b/lib/vector-core/src/event/metric/tags.rs @@ -603,13 +603,13 @@ impl ByteSizeOf for MetricTags { } } -#[cfg(test)] +#[cfg(any(test, feature = "generate-fixtures"))] mod test_support { use std::collections::HashSet; use quickcheck::{Arbitrary, Gen}; - use super::*; + use super::{BTreeMap, MetricTags, TagValue, TagValueSet}; impl Arbitrary for TagValue { fn arbitrary(g: &mut Gen) -> Self { diff --git a/lib/vector-core/src/event/mod.rs b/lib/vector-core/src/event/mod.rs index 9d72301e85804..e70ebc4335c9d 100644 --- a/lib/vector-core/src/event/mod.rs +++ b/lib/vector-core/src/event/mod.rs @@ -23,6 +23,8 @@ pub use vrl_target::{TargetEvents, VrlTarget}; use crate::config::{LogNamespace, OutputId}; +#[cfg(any(test, feature = "generate-fixtures"))] +pub(crate) mod arbitrary_impl; pub mod array; pub mod discriminant; mod estimated_json_encoded_size_of; @@ -35,6 +37,7 @@ pub mod metric; pub mod proto; mod r#ref; mod ser; + #[cfg(test)] mod test; mod trace; diff --git a/lib/vector-core/src/event/test/mod.rs b/lib/vector-core/src/event/test/mod.rs index 3762a53363602..8f9cbad4da2c4 100644 --- a/lib/vector-core/src/event/test/mod.rs +++ b/lib/vector-core/src/event/test/mod.rs @@ -1,4 +1,3 @@ -mod common; mod serialization; mod size_of; diff --git a/lib/vector-core/src/event/test/serialization.rs b/lib/vector-core/src/event/test/serialization.rs index 75304e3960d54..4bd7ac369e5c8 100644 --- a/lib/vector-core/src/event/test/serialization.rs +++ b/lib/vector-core/src/event/test/serialization.rs @@ -1,12 +1,11 @@ +use super::*; +use crate::config::log_schema; use bytes::{Buf, BufMut, BytesMut}; use quickcheck::{QuickCheck, TestResult}; use regex::Regex; use similar_asserts::assert_eq; use vector_buffers::encoding::Encodable; -use super::*; -use crate::config::log_schema; - fn encode_value(value: T, buffer: &mut B) { value.encode(buffer).expect("encoding should not fail"); } diff --git a/lib/vector-core/src/event/test/size_of.rs b/lib/vector-core/src/event/test/size_of.rs index 2428bc5ac6491..36bb1a0e23c4a 100644 --- a/lib/vector-core/src/event/test/size_of.rs +++ b/lib/vector-core/src/event/test/size_of.rs @@ -4,7 +4,8 @@ use lookup::{PathPrefix, path}; use quickcheck::{Arbitrary, Gen, QuickCheck, TestResult}; use vector_common::byte_size_of::ByteSizeOf; -use super::{common::Name, *}; +use super::*; +use crate::event::arbitrary_impl::Name; #[test] fn at_least_wrapper_size() { diff --git a/lib/vector-core/src/metrics/ddsketch.rs b/lib/vector-core/src/metrics/ddsketch.rs index b66f25ceb150b..886a5c190bf32 100644 --- a/lib/vector-core/src/metrics/ddsketch.rs +++ b/lib/vector-core/src/metrics/ddsketch.rs @@ -284,6 +284,17 @@ impl AgentDDSketch { }) } + /// Overrides `sum` and `avg` with arbitrary values. + /// + /// Only available under the `generate-fixtures` feature, where we need to + /// produce sketches with independently-randomized summary statistics to + /// exercise round-trip serialization of those fields. + #[cfg(feature = "generate-fixtures")] + pub fn set_sum_avg(&mut self, sum: f64, avg: f64) { + self.sum = sum; + self.avg = avg; + } + pub fn gamma(&self) -> f64 { self.config.gamma_v } From 604866d560deea7a14cbfa767a8ca99eee07089a Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 20 Apr 2026 14:58:40 -0400 Subject: [PATCH 141/326] fix(website): use parent site for RUM to resolve intake host (#25224) --- website/config.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/config.toml b/website/config.toml index 414451d38b57d..15cab204233af 100644 --- a/website/config.toml +++ b/website/config.toml @@ -95,7 +95,7 @@ discord = "https://chat.vector.dev" [params.datadog_config] application_id = "894af722-5919-4a62-97fc-33f49bc52803" client_token = "pubcee1dc56e7f5bd03397cc3624eb7961c" -site = "cose.datadoghq.com" +site = "datadoghq.com" service_name = "vector" ## Menus From bdcf7476c55141ecf535cec6caafea7b2cc3d16d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Apr 2026 15:43:03 -0400 Subject: [PATCH 142/326] chore(ci): bump dorny/paths-filter from 3.0.2 to 4.0.1 (#25105) Bumps [dorny/paths-filter](https://github.com/dorny/paths-filter) from 3.0.2 to 4.0.1. - [Release notes](https://github.com/dorny/paths-filter/releases) - [Changelog](https://github.com/dorny/paths-filter/blob/master/CHANGELOG.md) - [Commits](https://github.com/dorny/paths-filter/compare/de90cc6fb38fc0963ad72b210f1f284cd68cea36...fbd0ab8f3e69293af611ebaee6363fc25e6d187d) --- updated-dependencies: - dependency-name: dorny/paths-filter dependency-version: 4.0.1 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/changes.yml | 6 +++--- .github/workflows/check_generated_vrl_docs.yml | 2 +- .github/workflows/integration_windows.yml | 2 +- .github/workflows/regression.yml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/changes.yml b/.github/workflows/changes.yml index 999548111e9a7..f9cd9ce4ba5b9 100644 --- a/.github/workflows/changes.yml +++ b/.github/workflows/changes.yml @@ -189,7 +189,7 @@ jobs: steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1 id: filter with: base: ${{ env.BASE_SHA }} @@ -360,7 +360,7 @@ jobs: - name: Create filter rules for integrations run: vdev int ci-paths > int_test_filters.yaml - - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1 id: filter with: base: ${{ env.BASE_SHA }} @@ -454,7 +454,7 @@ jobs: - name: Create filter rules for e2e tests run: vdev e2e ci-paths > int_test_filters.yaml - - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1 id: filter with: base: ${{ env.BASE_SHA }} diff --git a/.github/workflows/check_generated_vrl_docs.yml b/.github/workflows/check_generated_vrl_docs.yml index c2a9852474461..867ffd7619379 100644 --- a/.github/workflows/check_generated_vrl_docs.yml +++ b/.github/workflows/check_generated_vrl_docs.yml @@ -28,7 +28,7 @@ jobs: docs: ${{ steps.filter.outputs.docs }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1 id: filter with: filters: | diff --git a/.github/workflows/integration_windows.yml b/.github/workflows/integration_windows.yml index 39289d786fd24..1074ba4a3b97b 100644 --- a/.github/workflows/integration_windows.yml +++ b/.github/workflows/integration_windows.yml @@ -16,7 +16,7 @@ jobs: windows: ${{ steps.filter.outputs.windows }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1 id: filter with: filters: | diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index cc1170a8aaa21..aaf71eb9687a4 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -124,7 +124,7 @@ jobs: - name: Collect file changes id: changes - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1 with: base: ${{ needs.resolve-inputs.outputs.baseline-sha }} ref: ${{ needs.resolve-inputs.outputs.comparison-sha }} From c2022541ee792988c465ca9a76b93a0f54ea4e00 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 20 Apr 2026 15:43:44 -0400 Subject: [PATCH 143/326] docs(deprecations): add greptimedb v0 support deprecation entry (#25226) --- docs/DEPRECATIONS.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/DEPRECATIONS.md b/docs/DEPRECATIONS.md index cf3a9377bfe24..b57db75f736af 100644 --- a/docs/DEPRECATIONS.md +++ b/docs/DEPRECATIONS.md @@ -21,3 +21,5 @@ For example: ## To be migrated ## To be removed + +- `v0.56.0` | `greptimedb-v0-support` | The `greptimedb_metrics` and `greptimedb_logs` sinks drop support for GreptimeDB v0.x. Users must upgrade their GreptimeDB instance to v1.x before upgrading Vector. From 0f7574ad33ce07243ab8f9d6399c5e6c2f451af1 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Mon, 20 Apr 2026 16:49:34 -0400 Subject: [PATCH 144/326] revert(observability): fix panic in allocation tracing dealloc path (#25222) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Revert "fix(observability): fix panic in allocation tracing dealloc path (#25…" This reverts commit 43f620435a353b048c40661f59a2d2d18f2419ed. --- ...ix_allocation_tracing_dealloc_panic.fix.md | 3 - .../allocator/tracing_allocator.rs | 202 ++---------------- 2 files changed, 13 insertions(+), 192 deletions(-) delete mode 100644 changelog.d/fix_allocation_tracing_dealloc_panic.fix.md diff --git a/changelog.d/fix_allocation_tracing_dealloc_panic.fix.md b/changelog.d/fix_allocation_tracing_dealloc_panic.fix.md deleted file mode 100644 index 732c836eac0e8..0000000000000 --- a/changelog.d/fix_allocation_tracing_dealloc_panic.fix.md +++ /dev/null @@ -1,3 +0,0 @@ -Fixed a panic (abort) when running with `--allocation-tracing` in debug builds, caused by deallocating memory that was allocated before tracking was enabled. Also fixed per-group memory accounting skew for reentrant allocations whose tracing closure was skipped, which left the group ID header uninitialized and caused deallocations to be attributed to wrong groups. - -authors: pront diff --git a/src/internal_telemetry/allocations/allocator/tracing_allocator.rs b/src/internal_telemetry/allocations/allocator/tracing_allocator.rs index d0a2027eeb2e4..8d1eff3e36d4e 100644 --- a/src/internal_telemetry/allocations/allocator/tracing_allocator.rs +++ b/src/internal_telemetry/allocations/allocator/tracing_allocator.rs @@ -9,24 +9,8 @@ use super::{ }; use crate::internal_telemetry::allocations::TRACK_ALLOCATIONS; -/// Header value for allocations made while tracking is disabled. -/// On deallocation we free with the wrapped layout but skip tracing. -const UNTRACKED: u8 = 0; - -/// Header value for allocations whose tracing closure was skipped due to -/// reentrancy. `register()` never hands out `u8::MAX`, so this cannot -/// collide with a real group ID. On deallocation we free with the wrapped -/// layout but skip `trace_deallocation` to keep accounting balanced. -const UNTRACED: u8 = u8::MAX; - /// A tracing allocator that groups allocation events by groups. /// -/// Every allocation made through this allocator uses the "wrapped" layout: -/// the requested layout extended by one byte to store an allocation group -/// ID. This byte is always present regardless of whether tracking is -/// enabled, which guarantees that `dealloc` can always read a valid header -/// and free with the correct (wrapped) layout. -/// /// This allocator can only be used when specified via `#[global_allocator]`. pub struct GroupedTraceableAllocator { allocator: A, @@ -45,6 +29,11 @@ unsafe impl GlobalAlloc for GroupedTraceableAllocator #[inline] unsafe fn alloc(&self, object_layout: Layout) -> *mut u8 { unsafe { + if !TRACK_ALLOCATIONS.load(Ordering::Relaxed) { + return self.allocator.alloc(object_layout); + } + + // Allocate our wrapped layout and make sure the allocation succeeded. let (actual_layout, offset_to_group_id) = get_wrapped_layout(object_layout); let actual_ptr = self.allocator.alloc(actual_layout); if actual_ptr.is_null() { @@ -53,16 +42,6 @@ unsafe impl GlobalAlloc for GroupedTraceableAllocator let group_id_ptr = actual_ptr.add(offset_to_group_id).cast::(); - if !TRACK_ALLOCATIONS.load(Ordering::Relaxed) { - group_id_ptr.write(UNTRACKED); - return actual_ptr; - } - - // Write the untraced sentinel so that `dealloc` always finds a - // valid header, even when the tracing closure below is skipped - // due to reentrancy. - group_id_ptr.write(UNTRACED); - let object_size = object_layout.size(); try_with_suspended_allocation_group( @@ -79,19 +58,19 @@ unsafe impl GlobalAlloc for GroupedTraceableAllocator #[inline] unsafe fn dealloc(&self, object_ptr: *mut u8, object_layout: Layout) { unsafe { + if !TRACK_ALLOCATIONS.load(Ordering::Relaxed) { + self.allocator.dealloc(object_ptr, object_layout); + return; + } + // Regenerate the wrapped layout so we know where we have to look, as the pointer we've given relates to the + // requested layout, not the wrapped layout that was actually allocated. let (wrapped_layout, offset_to_group_id) = get_wrapped_layout(object_layout); + let raw_group_id = object_ptr.add(offset_to_group_id).cast::().read(); - // Always free with the wrapped layout since all allocations - // (tracked or not) use it. + // Deallocate before tracking, just to make sure we're reclaiming memory as soon as possible. self.allocator.dealloc(object_ptr, wrapped_layout); - // Skip tracing for untracked (tracking was off) and untraced - // (reentrant, closure skipped) allocations. - if raw_group_id == UNTRACKED || raw_group_id == UNTRACED { - return; - } - let object_size = object_layout.size(); let source_group_id = AllocationGroupId::from_raw(raw_group_id); @@ -117,158 +96,3 @@ fn get_wrapped_layout(object_layout: Layout) -> (Layout, usize) { (actual_layout.pad_to_align(), offset_to_group_id) } - -#[cfg(test)] -mod tests { - use std::{ - alloc::{GlobalAlloc, Layout, System}, - sync::atomic::{AtomicUsize, Ordering}, - }; - - use serial_test::serial; - - use super::*; - use crate::internal_telemetry::allocations::allocator::{ - token::AllocationGroupId, tracer::Tracer, - }; - - /// RAII guard that enables `TRACK_ALLOCATIONS` on creation and - /// restores it to `false` on drop, ensuring cleanup even if the - /// test panics. - struct TrackingGuard; - - impl TrackingGuard { - fn enable() -> Self { - TRACK_ALLOCATIONS.store(true, Ordering::Release); - Self - } - } - - impl Drop for TrackingGuard { - fn drop(&mut self) { - TRACK_ALLOCATIONS.store(false, Ordering::Release); - } - } - - struct CountingTracer { - allocs: AtomicUsize, - deallocs: AtomicUsize, - } - - impl CountingTracer { - const fn new() -> Self { - Self { - allocs: AtomicUsize::new(0), - deallocs: AtomicUsize::new(0), - } - } - } - - impl Tracer for CountingTracer { - fn trace_allocation(&self, _size: usize, _group_id: AllocationGroupId) { - self.allocs.fetch_add(1, Ordering::Relaxed); - } - - fn trace_deallocation(&self, _size: usize, _source_group_id: AllocationGroupId) { - self.deallocs.fetch_add(1, Ordering::Relaxed); - } - } - - #[test] - fn sentinels_do_not_collide_with_root_id() { - assert_eq!(UNTRACED, u8::MAX); - assert_ne!(UNTRACED, AllocationGroupId::ROOT.as_raw()); - assert_ne!(UNTRACKED, AllocationGroupId::ROOT.as_raw()); - } - - /// Allocations made while tracking is disabled get UNTRACKED (0) in the - /// header. Deallocating them (whether tracking is on or off) must use - /// the wrapped layout and skip tracing. - #[test] - #[serial] - fn untracked_alloc_dealloc_skips_tracing() { - let allocator = GroupedTraceableAllocator::new(System, CountingTracer::new()); - let layout = Layout::from_size_align(64, 8).unwrap(); - - // Tracking starts off (default state). Allocate while disabled. - let ptr = unsafe { allocator.alloc(layout) }; - assert!(!ptr.is_null()); - - // Header must be UNTRACKED. - let (_, offset) = get_wrapped_layout(layout); - let raw_id = unsafe { ptr.add(offset).cast::().read() }; - assert_eq!(raw_id, UNTRACKED); - - // Enable tracking, then dealloc -- must not panic, no trace events. - let _guard = TrackingGuard::enable(); - unsafe { allocator.dealloc(ptr, layout) }; - - assert_eq!(allocator.tracer.allocs.load(Ordering::Relaxed), 0); - assert_eq!(allocator.tracer.deallocs.load(Ordering::Relaxed), 0); - } - - /// Tracked allocation: header is a valid non-zero, non-sentinel group - /// ID, tracing fires for both alloc and dealloc, and dealloc completes - /// without panic. - #[test] - #[serial] - fn tracked_alloc_dealloc_does_not_panic() { - let allocator = GroupedTraceableAllocator::new(System, CountingTracer::new()); - let layout = Layout::from_size_align(64, 8).unwrap(); - - let _guard = TrackingGuard::enable(); - let ptr = unsafe { allocator.alloc(layout) }; - assert!(!ptr.is_null()); - - let (_, offset) = get_wrapped_layout(layout); - let raw_id = unsafe { ptr.add(offset).cast::().read() }; - assert_eq!( - raw_id, - AllocationGroupId::ROOT.as_raw(), - "header must be the ROOT group ID" - ); - assert_eq!(allocator.tracer.allocs.load(Ordering::Relaxed), 1); - - unsafe { allocator.dealloc(ptr, layout) }; - - assert_eq!(allocator.tracer.deallocs.load(Ordering::Relaxed), 1); - } - - /// End-to-end reentrant allocation: hold a mutable borrow on the - /// thread-local group stack so `try_with_suspended_allocation_group` - /// skips the tracing closure. The header must be UNTRACED and both - /// trace counters must stay at zero through alloc + dealloc. - #[test] - #[serial] - fn reentrant_alloc_writes_untraced_and_skips_tracing() { - use crate::internal_telemetry::allocations::allocator::token::LOCAL_ALLOCATION_GROUP_STACK; - - let allocator = GroupedTraceableAllocator::new(System, CountingTracer::new()); - let layout = Layout::from_size_align(64, 8).unwrap(); - - let _guard = TrackingGuard::enable(); - - // Hold a mutable borrow to simulate reentrancy -- any nested - // `try_borrow_mut` inside `try_with_suspended_allocation_group` - // will fail, causing the tracing closure to be skipped. - LOCAL_ALLOCATION_GROUP_STACK.with(|group_stack| { - let _borrow = group_stack.borrow_mut(); - - let ptr = unsafe { allocator.alloc(layout) }; - assert!(!ptr.is_null()); - - let (_, offset) = get_wrapped_layout(layout); - let raw_id = unsafe { ptr.add(offset).cast::().read() }; - assert_eq!( - raw_id, UNTRACED, - "reentrant alloc must write UNTRACED sentinel" - ); - - assert_eq!(allocator.tracer.allocs.load(Ordering::Relaxed), 0); - - unsafe { allocator.dealloc(ptr, layout) }; - - assert_eq!(allocator.tracer.deallocs.load(Ordering::Relaxed), 0); - }); - } -} From 3ca3e61ec8b8cd89be836b47346343069b229398 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Mon, 20 Apr 2026 17:11:31 -0400 Subject: [PATCH 145/326] fix(ci): use CUE raw multi-line strings in release generator (#25228) Fragment descriptions are embedded into CUE via `"""..."""`, where backslashes are still interpreted as escape sequences. A fragment containing e.g. a shell line continuation (`\`) produces invalid CUE and breaks `cue export`, which is then used by the next `cargo vdev release prepare` invocation. Switch to raw multi-line strings (`#"""..."""#`) so backslashes are literal and no escaping is needed on the description content. Co-authored-by: Claude Opus 4.7 (1M context) --- scripts/generate-release-cue.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/generate-release-cue.rb b/scripts/generate-release-cue.rb index 80a35c0683015..d11d9f8f02c8a 100755 --- a/scripts/generate-release-cue.rb +++ b/scripts/generate-release-cue.rb @@ -217,11 +217,14 @@ def generate_changelog!(new_version) # Note: `pr_numbers`, `scopes` and `breaking` are being omitted from the entries. # These are currently not required for rendering in the website. + # Use CUE raw multi-line strings (#"""..."""#) so backslashes in the + # fragment (e.g. shell line continuations) are not interpreted as + # escape sequences. entry = "{\n" + "type: #{type.to_json}\n" + - "description: \"\"\"\n" + + "description: #\"\"\"\n" + "#{description}\n" + - "\"\"\"\n" + "\"\"\"#\n" if contributors.length() > 0 entry += "contributors: #{contributors.to_json}\n" From 1c70988b54156abf8d031538f0f81f28e7c0a0e4 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 21 Apr 2026 13:00:33 -0400 Subject: [PATCH 146/326] fix(api): restore HTTP GET /health endpoint (#25234) * refactor(api): serve gRPC via hyper + axum router Convert tonic's Server to an axum Router via into_router(), then serve over the same TcpListener via hyper::Server. Enables HTTP/1.1 acceptance so additional HTTP routes can be added alongside gRPC on the same port. Behavior-preserving. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(api): restore HTTP GET /health endpoint Re-expose the HTTP health endpoint that was removed as part of the GraphQL-to-gRPC migration (#24364). The endpoint matches the pre-0.55 response shape: 200 with body {"ok": true} while serving and 503 with body {"ok": false} after set_not_serving() is called during drain. HEAD is also handled. gRPC clients continue to use grpc.health.v1.Health/Check; both probes now share the same serving state so they agree during shutdown. Co-Authored-By: Claude Opus 4.7 (1M context) * test(api): cover HTTP GET/HEAD /health endpoint Adds two integration tests hitting the restored HTTP health endpoint via reqwest: - GET returns 200 with body {"ok":true} - HEAD returns 200 Exposes harness.api_port() so tests can reach the API port directly. Co-Authored-By: Claude Opus 4.7 (1M context) * docs(api): document HTTP /health endpoint Document the HTTP GET/HEAD /health endpoint served alongside the gRPC API, framed as compatibility with Vector 0.54.0 and earlier. Updates the reference endpoints schema to allow HEAD, adds HEAD/GET entries for /health in api.cue with 200/503 responses, and adds a curl example to the API reference page. Co-Authored-By: Claude Opus 4.7 (1M context) * style(api): apply cargo fmt Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) --- src/api/grpc_server.rs | 85 +++++++++++++++++++++--- tests/vector_api/harness.rs | 5 ++ tests/vector_api/health.rs | 44 +++++++++++- website/content/en/docs/reference/api.md | 13 ++++ website/cue/reference.cue | 1 + website/cue/reference/api.cue | 25 ++++++- 6 files changed, 162 insertions(+), 11 deletions(-) diff --git a/src/api/grpc_server.rs b/src/api/grpc_server.rs index ce33b0da3f7d5..715aeecad2b8e 100644 --- a/src/api/grpc_server.rs +++ b/src/api/grpc_server.rs @@ -1,6 +1,20 @@ -use std::{error::Error as StdError, net::SocketAddr}; +use std::{ + error::Error as StdError, + net::SocketAddr, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, +}; + +use axum::{ + Router, + extract::State, + http::{StatusCode, header}, + response::IntoResponse, + routing::get, +}; use tokio::sync::oneshot; -use tokio_stream::wrappers::TcpListenerStream; use tonic::transport::Server as TonicServer; use tonic_health::server::{HealthReporter, health_reporter}; use vector_lib::tap::topology::WatchRx; @@ -8,10 +22,15 @@ use vector_lib::tap::topology::WatchRx; use super::grpc::ObservabilityService; use crate::{config::Config, proto::observability::Server as ObservabilityServer}; +/// Shared flag backing the HTTP `/health` endpoint. Mirrors the gRPC +/// `HealthReporter` serving status so HTTP and gRPC probes agree. +type ServingState = Arc; + /// gRPC API server for Vector observability. pub struct GrpcServer { _shutdown: oneshot::Sender<()>, health_reporter: HealthReporter, + serving: ServingState, addr: SocketAddr, } @@ -46,12 +65,22 @@ impl GrpcServer { // The empty service ("") is registered as SERVING by default. let (health_reporter, health_service) = health_reporter(); + let serving: ServingState = Arc::new(AtomicBool::new(true)); + let (_shutdown, rx) = oneshot::channel(); + // Convert the tokio TcpListener into a std listener for hyper's Server. + let std_listener = listener + .into_std() + .map_err(|e| crate::Error::from(format!("Failed to convert TCP listener: {}", e)))?; + std_listener.set_nonblocking(true).map_err(|e| { + crate::Error::from(format!("Failed to set TCP listener non-blocking: {}", e)) + })?; + + let router_serving = Arc::clone(&serving); + // Spawn the server with the already-bound listener tokio::spawn(async move { - let incoming = TcpListenerStream::new(listener); - // Build reflection service for tools like grpcurl let reflection_service = tonic_reflection::server::Builder::configure() .register_encoded_file_descriptor_set( @@ -61,11 +90,21 @@ impl GrpcServer { .build() .expect("Failed to build reflection service"); - let result = TonicServer::builder() + // Build the tonic router (gRPC services) and merge with the HTTP router + // so both protocols share the same port. `accept_http1(true)` lets plain + // HTTP/1.1 requests reach the merged axum routes. + let router = TonicServer::builder() + .accept_http1(true) .add_service(health_service) .add_service(ObservabilityServer::new(service)) .add_service(reflection_service) - .serve_with_incoming_shutdown(incoming, async { + .into_router() + .merge(http_router(router_serving)); + + let result = hyper::Server::from_tcp(std_listener) + .expect("Failed to build HTTP server from TCP listener") + .serve(router.into_make_service()) + .with_graceful_shutdown(async { rx.await.ok(); info!("GRPC API server shutting down."); }) @@ -86,6 +125,7 @@ impl GrpcServer { Ok(Self { _shutdown, health_reporter, + serving, addr: actual_addr, }) } @@ -93,9 +133,10 @@ impl GrpcServer { /// Signal that the server is no longer serving. /// /// Call this **before** draining the topology so that Kubernetes gRPC - /// readiness probes fail early and the pod is removed from endpoints - /// before the process exits. + /// readiness probes and HTTP `/health` probes fail early and the pod is + /// removed from endpoints before the process exits. pub async fn set_not_serving(&mut self) { + self.serving.store(false, Ordering::Relaxed); self.health_reporter .set_service_status("", tonic_health::ServingStatus::NotServing) .await; @@ -106,3 +147,31 @@ impl GrpcServer { self.addr } } + +/// Axum router exposing `GET`/`HEAD /health`. +/// +/// Returns `200 {"ok":true}` while the server is serving and +/// `503 {"ok":false}` once [`GrpcServer::set_not_serving`] has been called. +/// Matches the response shape of the pre-gRPC GraphQL-era endpoint so +/// existing HTTP health probes (Kubernetes, load balancers) keep working. +fn http_router(state: ServingState) -> Router { + Router::new() + .route("/health", get(health_handler).head(health_handler)) + .with_state(state) +} + +async fn health_handler(State(state): State) -> impl IntoResponse { + if state.load(Ordering::Relaxed) { + ( + StatusCode::OK, + [(header::CONTENT_TYPE, "application/json")], + r#"{"ok":true}"#, + ) + } else { + ( + StatusCode::SERVICE_UNAVAILABLE, + [(header::CONTENT_TYPE, "application/json")], + r#"{"ok":false}"#, + ) + } +} diff --git a/tests/vector_api/harness.rs b/tests/vector_api/harness.rs index 274592fac5dae..ba79b521bba22 100644 --- a/tests/vector_api/harness.rs +++ b/tests/vector_api/harness.rs @@ -133,6 +133,11 @@ impl TestHarness { &mut self.api_client } + /// Returns the TCP port the API server is bound to + pub fn api_port(&self) -> u16 { + self.api_port + } + /// Reloads Vector configuration by sending SIGHUP or using watch mode /// /// Polls Vector to detect crashes early and succeed fast when reload completes. diff --git a/tests/vector_api/health.rs b/tests/vector_api/health.rs index a17d16eecf913..1e88c5aa34ed0 100644 --- a/tests/vector_api/health.rs +++ b/tests/vector_api/health.rs @@ -1,4 +1,7 @@ -//! Integration tests for the standard gRPC health check on the observability API. +//! Integration tests for the health endpoints on the observability API: +//! +//! * the standard gRPC health check (`grpc.health.v1.Health/Check`) +//! * the HTTP `/health` endpoint served on the same port use super::{common::*, harness::*}; @@ -23,3 +26,42 @@ async fn health_check_reports_serving() { assert!(harness.check_running(), "Vector should still be running"); } + +/// Verifies the HTTP `GET /health` endpoint returns 200 with `{"ok":true}` on a +/// running Vector instance. This endpoint is load-balancer friendly and is +/// shared with gRPC clients on the same API port. +#[tokio::test] +async fn http_health_endpoint_returns_200_when_serving() { + let config = single_source_config("demo", 1.0, None); + let harness = TestHarness::new(&config) + .await + .expect("Vector should start"); + + let url = format!("http://127.0.0.1:{}/health", harness.api_port()); + let response = reqwest::get(&url) + .await + .expect("GET /health should succeed"); + + assert_eq!(response.status(), reqwest::StatusCode::OK); + let body = response.text().await.expect("body should be readable"); + assert_eq!(body, r#"{"ok":true}"#); +} + +/// Verifies HTTP `HEAD /health` returns 200 without a body. ALB/ELB style +/// probes that prefer HEAD should work the same as GET. +#[tokio::test] +async fn http_health_endpoint_supports_head() { + let config = single_source_config("demo", 1.0, None); + let harness = TestHarness::new(&config) + .await + .expect("Vector should start"); + + let url = format!("http://127.0.0.1:{}/health", harness.api_port()); + let response = reqwest::Client::new() + .head(&url) + .send() + .await + .expect("HEAD /health should succeed"); + + assert_eq!(response.status(), reqwest::StatusCode::OK); +} diff --git a/website/content/en/docs/reference/api.md b/website/content/en/docs/reference/api.md index 949353241929b..9d49b863eb8ab 100644 --- a/website/content/en/docs/reference/api.md +++ b/website/content/en/docs/reference/api.md @@ -21,6 +21,12 @@ instance. This page covers how to configure and enable Vector's API. The API exposes a gRPC service defined in [`proto/vector/observability.proto`](https://github.com/vectordotdev/vector/blob/master/proto/vector/observability.proto). You can interact with it using any standard gRPC tooling. +For compatibility with Vector 0.54.0 and earlier, the HTTP `GET /health` +endpoint continues to be served on the same port as the gRPC API, so +existing HTTP probes (for example AWS ALB health checks and Kubernetes +HTTP liveness/readiness probes) keep working without changes. See the +[Endpoints](#endpoints) section above for details. + ### Example using grpcurl ```bash @@ -35,3 +41,10 @@ grpcurl -plaintext \ -d '{"outputs_patterns": ["*"], "limit": 100, "interval_ms": 500}' \ localhost:8686 vector.observability.v1.ObservabilityService/StreamOutputEvents ``` + +### Example using curl (HTTP health) + +```bash +# 200 with body {"ok":true} while serving, 503 {"ok":false} during drain +curl -i http://localhost:8686/health +``` diff --git a/website/cue/reference.cue b/website/cue/reference.cue index 05d22f27e1f1a..789e2e66bce82 100644 --- a/website/cue/reference.cue +++ b/website/cue/reference.cue @@ -68,6 +68,7 @@ _values: { #Endpoints: [Path=string]: { DELETE?: #Endpoint GET?: #Endpoint + HEAD?: #Endpoint POST?: #Endpoint PUT?: #Endpoint } diff --git a/website/cue/reference/api.cue b/website/cue/reference/api.cue index 38feeff331893..bb1e3baf129ba 100644 --- a/website/cue/reference/api.cue +++ b/website/cue/reference/api.cue @@ -24,13 +24,34 @@ api: { "/health": { GET: { description: """ - Healthcheck endpoint. Useful to verify that - Vector is up and running. + HTTP healthcheck endpoint served on the same port as the + gRPC API, preserved for compatibility with Vector 0.54.0 + and earlier so existing HTTP probes (for example AWS ALB + and Kubernetes HTTP probes) keep working unchanged. + The response body is `{"ok": true}` while Vector is + serving and `{"ok": false}` once Vector begins draining. """ responses: { "200": { description: "Vector is initialized and running." } + "503": { + description: "Vector is draining or shutting down and should be removed from the load balancer." + } + } + } + HEAD: { + description: """ + Same semantics as `GET /health` but returns no body. + Intended for load balancer probes that prefer `HEAD`. + """ + responses: { + "200": { + description: "Vector is initialized and running." + } + "503": { + description: "Vector is draining or shutting down and should be removed from the load balancer." + } } } } From aafd4cb44f5649e692722b97d82973fea5509a41 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 21 Apr 2026 14:21:39 -0400 Subject: [PATCH 147/326] fix(demo_logs source): drop fakedata_generator, fix broken fake domains (#25236) * fix(demo_logs source): drop fakedata_generator, fix broken fake domains The fakedata_generator crate (v0.7.1) has an upstream bug: its get_dataset() match table is missing a "tlds" arm, so gen_domain() falls through to an empty JSON string. That fails to parse and the crate prints "Failed getting dataset for tlds. EOF while parsing a value at line 1 column 0" to stderr on every call, while gen_domain() returns ".Error: dataset not found" as the fake domain. Drop the dependency (and its transitive passt dep) and inline the three functions demo_logs actually used: gen_domain, gen_ipv4, gen_username. Domains now look like "random.io" instead of "random.Error: dataset not found", and stderr is quiet. Co-Authored-By: Claude Opus 4.7 (1M context) * chore(releasing): changelog fragment for demo_logs fakedata fix Co-Authored-By: Claude Opus 4.7 (1M context) * chore(demo_logs source): drop stale comment, allow TLDs/fakedata in spelling Co-Authored-By: Claude Opus 4.7 (1M context) * chore(demo_logs source): swap fake domain name list to unique entries Replace the 8-word domain-name pool with fresh placeholder-style entries so nothing in lib/fakedata derives from the fakedata_generator crate's data. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) --- .github/actions/spelling/allow.txt | 2 + Cargo.lock | 19 -------- LICENSE-3rdparty.csv | 2 - changelog.d/drop_fakedata_generator.fix.md | 9 ++++ lib/fakedata/Cargo.toml | 1 - lib/fakedata/src/logs.rs | 54 ++++++++++++++++++++-- 6 files changed, 61 insertions(+), 26 deletions(-) create mode 100644 changelog.d/drop_fakedata_generator.fix.md diff --git a/.github/actions/spelling/allow.txt b/.github/actions/spelling/allow.txt index b43abdd6fba66..db6d30301dcd7 100644 --- a/.github/actions/spelling/allow.txt +++ b/.github/actions/spelling/allow.txt @@ -154,6 +154,7 @@ Evercoss exactlyonce Explay Fabro +fakedata fakeintake FAQs fargate @@ -509,6 +510,7 @@ timeseries timespan timestamped Tizen +TLDs Tmobile tobz Tomtec diff --git a/Cargo.lock b/Cargo.lock index 3215e97f0fd44..2a7c05bd91410 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3909,22 +3909,9 @@ name = "fakedata" version = "0.1.0" dependencies = [ "chrono", - "fakedata_generator", "rand 0.9.4", ] -[[package]] -name = "fakedata_generator" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b42ec18312c068f0d6008ad66ad2fa289ab40a16d07a06406c200baa09eb1e7c" -dependencies = [ - "passt", - "rand 0.9.4", - "serde", - "serde_json", -] - [[package]] name = "fallible-iterator" version = "0.2.0" @@ -7615,12 +7602,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "487f2ccd1e17ce8c1bfab3a65c89525af41cfad4c8659021a1e9a2aacd73b89b" -[[package]] -name = "passt" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13242a5ce97f39a8095d03c8b273e91d09f2690c0b7d69a2af844941115bab24" - [[package]] name = "paste" version = "1.0.15" diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index ed18e3c3c172b..eb9e179e57c2b 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -270,7 +270,6 @@ event-listener-strategy,https://github.com/smol-rs/event-listener-strategy,Apach evmap,https://github.com/jonhoo/rust-evmap,MIT OR Apache-2.0,Jon Gjengset evmap-derive,https://github.com/jonhoo/rust-evmap,MIT OR Apache-2.0,Jon Gjengset exitcode,https://github.com/benwilber/exitcode,Apache-2.0,Ben Wilber -fakedata_generator,https://github.com/KevinGimbel/fakedata_generator,MIT,Kevin Gimbel fallible-iterator,https://github.com/sfackler/rust-fallible-iterator,MIT OR Apache-2.0,Steven Fackler fancy-regex,https://github.com/fancy-regex/fancy-regex,MIT,"Raph Levien , Robin Stocker , Keith Hall " fastrand,https://github.com/smol-rs/fastrand,Apache-2.0 OR MIT,Stjepan Glavina @@ -536,7 +535,6 @@ parking_lot,https://github.com/Amanieu/parking_lot,MIT OR Apache-2.0,Amanieu d'A parking_lot_core,https://github.com/Amanieu/parking_lot,MIT OR Apache-2.0,Amanieu d'Antras parquet,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow parse-size,https://github.com/kennytm/parse-size,MIT,kennytm -passt,https://github.com/kevingimbel/passt,MIT OR Apache-2.0,Kevin Gimbel paste,https://github.com/dtolnay/paste,MIT OR Apache-2.0,David Tolnay pastey,https://github.com/as1100k/pastey,MIT OR Apache-2.0,"Aditya Kumar , David Tolnay " pbkdf2,https://github.com/RustCrypto/password-hashes/tree/master/pbkdf2,MIT OR Apache-2.0,RustCrypto Developers diff --git a/changelog.d/drop_fakedata_generator.fix.md b/changelog.d/drop_fakedata_generator.fix.md new file mode 100644 index 0000000000000..500e826bc3239 --- /dev/null +++ b/changelog.d/drop_fakedata_generator.fix.md @@ -0,0 +1,9 @@ +The text content generated by the `demo_logs` source has changed: the +pool of fake usernames and the pool of fake domain TLDs are now both +defined inside Vector rather than pulled from an external crate. The +line formats (`apache_common`, `apache_error`, `json`, `syslog`, +`bsd_syslog`) are unchanged. If any of your tests or downstream +pipelines assert on specific generated usernames or TLDs, please +update those expectations. + +authors: pront diff --git a/lib/fakedata/Cargo.toml b/lib/fakedata/Cargo.toml index 576ad22e96cad..52ba22387ef89 100644 --- a/lib/fakedata/Cargo.toml +++ b/lib/fakedata/Cargo.toml @@ -8,5 +8,4 @@ license = "MPL-2.0" [dependencies] chrono.workspace = true -fakedata_generator = "0.7.1" rand.workspace = true diff --git a/lib/fakedata/src/logs.rs b/lib/fakedata/src/logs.rs index 1c3aa7683d2c4..768d67b06890d 100644 --- a/lib/fakedata/src/logs.rs +++ b/lib/fakedata/src/logs.rs @@ -3,9 +3,44 @@ use chrono::{ format::{DelayedFormat, StrftimeItems}, prelude::Local, }; -use fakedata_generator::{gen_domain, gen_ipv4, gen_username}; use rand::{Rng, rng}; +static FAKE_USERNAMES: [&str; 20] = [ + "log_whisperer", + "commit_conductor", + "cache_cowboy", + "compile_captain", + "latency_llama", + "yaml_yoda", + "regex_rider", + "semver_sage", + "kernel_keith", + "pixel_pilgrim", + "kubectl_kev", + "pipeline_pat", + "telemetry_tina", + "merge_maria", + "parser_pete", + "debug_duchess", + "nullable_nate", + "grep_greg", + "stderr_stan", + "segfault_sue", +]; + +static FAKE_DOMAIN_NAMES: [&str; 8] = [ + "acme", + "contoso", + "widgets", + "example", + "placeholder", + "sample", + "foobar", + "testbench", +]; + +static FAKE_DOMAIN_TLDS: [&str; 8] = ["com", "net", "org", "io", "dev", "co", "app", "biz"]; + static APPLICATION_NAMES: [&str; 10] = [ "auth", "data", "deploy", "etl", "scraper", "cron", "ingress", "egress", "alerter", "fwd", ]; @@ -156,7 +191,11 @@ fn application() -> &'static str { } fn domain() -> String { - gen_domain() + format!( + "{}.{}", + random_from_array(&FAKE_DOMAIN_NAMES), + random_from_array(&FAKE_DOMAIN_TLDS), + ) } fn error_level() -> &'static str { @@ -188,7 +227,14 @@ fn http_version() -> &'static str { } fn ipv4_address() -> String { - gen_ipv4() + let mut r = rng(); + format!( + "{}.{}.{}.{}", + r.random_range(1..255), + r.random_range(1..255), + r.random_range(1..255), + r.random_range(1..255), + ) } fn pid() -> usize { @@ -208,7 +254,7 @@ fn referer() -> String { } fn username() -> String { - gen_username() + (*random_from_array(&FAKE_USERNAMES)).to_string() } fn syslog_version() -> usize { From 8cf773df0e3b219e40537119aa2b95b916342df2 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Wed, 22 Apr 2026 14:31:21 -0400 Subject: [PATCH 148/326] chore(releasing): start 0.56.0 development cycle (#25242) * chore(releasing): prepare v0.55.0 release (#25229) * chore(releasing): Pinned VRL version to 0.32.0 * chore(releasing): Generated release CUE file * chore(releasing): Updated website/cue/reference/administration/interfaces/kubectl.cue vector version to 0.55.0 * chore(releasing): Updated distribution/install.sh vector version to 0.55.0 * chore(releasing): Add 0.55.0 to versions.cue * chore(releasing): Created release md file * fix(api): restore HTTP GET /health endpoint (#25234) * refactor(api): serve gRPC via hyper + axum router Convert tonic's Server to an axum Router via into_router(), then serve over the same TcpListener via hyper::Server. Enables HTTP/1.1 acceptance so additional HTTP routes can be added alongside gRPC on the same port. Behavior-preserving. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(api): restore HTTP GET /health endpoint Re-expose the HTTP health endpoint that was removed as part of the GraphQL-to-gRPC migration (#24364). The endpoint matches the pre-0.55 response shape: 200 with body {"ok": true} while serving and 503 with body {"ok": false} after set_not_serving() is called during drain. HEAD is also handled. gRPC clients continue to use grpc.health.v1.Health/Check; both probes now share the same serving state so they agree during shutdown. Co-Authored-By: Claude Opus 4.7 (1M context) * test(api): cover HTTP GET/HEAD /health endpoint Adds two integration tests hitting the restored HTTP health endpoint via reqwest: - GET returns 200 with body {"ok":true} - HEAD returns 200 Exposes harness.api_port() so tests can reach the API port directly. Co-Authored-By: Claude Opus 4.7 (1M context) * docs(api): document HTTP /health endpoint Document the HTTP GET/HEAD /health endpoint served alongside the gRPC API, framed as compatibility with Vector 0.54.0 and earlier. Updates the reference endpoints schema to allow HEAD, adds HEAD/GET entries for /health in api.cue with 200/503 responses, and adds a curl example to the API reference page. Co-Authored-By: Claude Opus 4.7 (1M context) * style(api): apply cargo fmt Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) * fix(demo_logs source): drop fakedata_generator, fix broken fake domains (#25236) * fix(demo_logs source): drop fakedata_generator, fix broken fake domains The fakedata_generator crate (v0.7.1) has an upstream bug: its get_dataset() match table is missing a "tlds" arm, so gen_domain() falls through to an empty JSON string. That fails to parse and the crate prints "Failed getting dataset for tlds. EOF while parsing a value at line 1 column 0" to stderr on every call, while gen_domain() returns ".Error: dataset not found" as the fake domain. Drop the dependency (and its transitive passt dep) and inline the three functions demo_logs actually used: gen_domain, gen_ipv4, gen_username. Domains now look like "random.io" instead of "random.Error: dataset not found", and stderr is quiet. Co-Authored-By: Claude Opus 4.7 (1M context) * chore(releasing): changelog fragment for demo_logs fakedata fix Co-Authored-By: Claude Opus 4.7 (1M context) * chore(demo_logs source): drop stale comment, allow TLDs/fakedata in spelling Co-Authored-By: Claude Opus 4.7 (1M context) * chore(demo_logs source): swap fake domain name list to unique entries Replace the 8-word domain-name pool with fresh placeholder-style entries so nothing in lib/fakedata derives from the fakedata_generator crate's data. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) * chore(releasing): address Codex review feedback on 0.55 release cue - Strip trailing whitespace on blank lines in `0.55.0.cue` so `make check-fmt` passes. - Clarify that `azure_credential_kind` must live under the `azure_logs_ingestion` sink's `auth` block, not at the sink root, and link the 0.55 upgrade guide for a concrete example. Co-Authored-By: Claude Opus 4.7 (1M context) * update date --------- Co-authored-by: Claude Opus 4.7 (1M context) * chore(releasing): start 0.56.0 development cycle Bump version to 0.56.0 and switch VRL back to tracking the vectordotdev/vrl main branch after the 0.55.0 pin. Co-Authored-By: Claude Opus 4.7 (1M context) * chore(releasing): regenerate Kubernetes manifests Output of cargo vdev build manifests for the 0.56.0 development cycle; picks up helm chart version 0.52.0. Co-Authored-By: Claude Opus 4.7 (1M context) * chore(deps): address cargo deny advisories for 0.56.0 - Upgrade rustls-webpki 0.103.12 -> 0.103.13 to pick up the fix for RUSTSEC-2026-0104 (reachable panic in CRL parsing). - Ignore RUSTSEC-2026-0104 for the still-reachable rustls-webpki 0.101 and 0.102, which are blocked on the tonic upgrade tracked in #19179. - Ignore RUSTSEC-2026-0105 (core2 unmaintained and yanked), reached transitively via libflate -> apache-avro; no safe upgrade available. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) --- Cargo.lock | 18 +- Cargo.toml | 2 +- .../23657-grpc-health-protocol.enhancement.md | 7 - .../24291_async_nats_websockets.feature.md | 4 - .../24411_geoip_network_field.enhancement.md | 3 - ..._source_per_signal_decoding.enhancement.md | 21 - .../24532_vector_stopped_log_ordering.fix.md | 5 - ..._dnstap_parser_message_size.enhancement.md | 3 - .../24593_graph_edge_attributes.feature.md | 3 - ...telemetry-header-enrichment.enhancement.md | 7 - ...9_kafka_sink_add_traces_support.feature.md | 3 - .../24641-task-transform-utilization.fix.md | 5 - changelog.d/24708_log_error.fix.md | 5 - ...24729_azure_blob_authentication.feature.md | 3 - .../24729_azure_logs_ingestion.breaking.md | 3 - ...-buffer-utilization-metric-tracking.fix.md | 3 - ...3_aggregate_transform_reduce_memory.fix.md | 4 - .../24951_top_events_out_column.fix.md | 3 - ...adog_metrics_zstd_series_v2.enhancement.md | 3 - ...fixed_high_cpu_usage_in_file_source.fix.md | 3 - .../aws_s3_parquet_encoding.feature.md | 10 - .../bump_kube_k8s_openapi.enhancement.md | 2 - ...ickhouse_arrow_uuid_support.enhancement.md | 3 - .../datadog_agent_source_device_tag_v2.fix.md | 5 - ...g_metrics_series_v2_default.enhancement.md | 5 - changelog.d/fix_site_prefix_stripping.fix.md | 3 - ...fix_source_lag_time_chunked_batches.fix.md | 3 - changelog.d/graphql_to_grpc_api.breaking.md | 42 -- ...deprecated_http_headers_option.breaking.md | 4 - ...urce_sender_latency_metrics.enhancement.md | 5 - ...or_top_disabled_memory_used.enhancement.md | 3 - .../windows_event_log_source.feature.md | 3 - .../windows_service_stop_state_wait.fix.md | 3 - deny.toml | 2 + distribution/install.sh | 2 +- .../kubernetes/vector-agent/README.md | 2 +- .../kubernetes/vector-agent/configmap.yaml | 5 +- .../kubernetes/vector-agent/daemonset.yaml | 8 +- .../kubernetes/vector-agent/rbac.yaml | 4 +- .../vector-agent/service-headless.yaml | 2 +- .../vector-agent/serviceaccount.yaml | 2 +- .../kubernetes/vector-aggregator/README.md | 2 +- .../vector-aggregator/configmap.yaml | 5 +- .../vector-aggregator/service-headless.yaml | 2 +- .../kubernetes/vector-aggregator/service.yaml | 2 +- .../vector-aggregator/serviceaccount.yaml | 2 +- .../vector-aggregator/statefulset.yaml | 8 +- .../vector-stateless-aggregator/README.md | 2 +- .../configmap.yaml | 5 +- .../deployment.yaml | 8 +- .../service-headless.yaml | 2 +- .../vector-stateless-aggregator/service.yaml | 2 +- .../serviceaccount.yaml | 2 +- .../2026-04-20-0-55-0-upgrade-guide.md | 139 +++++ website/content/en/releases/0.55.0.md | 4 + .../administration/interfaces/kubectl.cue | 2 +- website/cue/reference/releases/0.55.0.cue | 583 ++++++++++++++++++ website/cue/reference/versions.cue | 1 + website/layouts/releases/single.html | 22 +- 59 files changed, 786 insertions(+), 233 deletions(-) delete mode 100644 changelog.d/23657-grpc-health-protocol.enhancement.md delete mode 100644 changelog.d/24291_async_nats_websockets.feature.md delete mode 100644 changelog.d/24411_geoip_network_field.enhancement.md delete mode 100644 changelog.d/24455_otel_source_per_signal_decoding.enhancement.md delete mode 100644 changelog.d/24532_vector_stopped_log_ordering.fix.md delete mode 100644 changelog.d/24552_dnstap_parser_message_size.enhancement.md delete mode 100644 changelog.d/24593_graph_edge_attributes.feature.md delete mode 100644 changelog.d/24619-opentelemetry-header-enrichment.enhancement.md delete mode 100644 changelog.d/24639_kafka_sink_add_traces_support.feature.md delete mode 100644 changelog.d/24641-task-transform-utilization.fix.md delete mode 100644 changelog.d/24708_log_error.fix.md delete mode 100644 changelog.d/24729_azure_blob_authentication.feature.md delete mode 100644 changelog.d/24729_azure_logs_ingestion.breaking.md delete mode 100644 changelog.d/24911-buffer-utilization-metric-tracking.fix.md delete mode 100644 changelog.d/24943_aggregate_transform_reduce_memory.fix.md delete mode 100644 changelog.d/24951_top_events_out_column.fix.md delete mode 100644 changelog.d/24956_datadog_metrics_zstd_series_v2.enhancement.md delete mode 100644 changelog.d/25064_fixed_high_cpu_usage_in_file_source.fix.md delete mode 100644 changelog.d/aws_s3_parquet_encoding.feature.md delete mode 100644 changelog.d/bump_kube_k8s_openapi.enhancement.md delete mode 100644 changelog.d/clickhouse_arrow_uuid_support.enhancement.md delete mode 100644 changelog.d/datadog_agent_source_device_tag_v2.fix.md delete mode 100644 changelog.d/datadog_metrics_series_v2_default.enhancement.md delete mode 100644 changelog.d/fix_site_prefix_stripping.fix.md delete mode 100644 changelog.d/fix_source_lag_time_chunked_batches.fix.md delete mode 100644 changelog.d/graphql_to_grpc_api.breaking.md delete mode 100644 changelog.d/remove_deprecated_http_headers_option.breaking.md delete mode 100644 changelog.d/source_sender_latency_metrics.enhancement.md delete mode 100644 changelog.d/vector_top_disabled_memory_used.enhancement.md delete mode 100644 changelog.d/windows_event_log_source.feature.md delete mode 100644 changelog.d/windows_service_stop_state_wait.fix.md create mode 100644 website/content/en/highlights/2026-04-20-0-55-0-upgrade-guide.md create mode 100644 website/content/en/releases/0.55.0.md create mode 100644 website/cue/reference/releases/0.55.0.cue diff --git a/Cargo.lock b/Cargo.lock index 2a7c05bd91410..a39a96a6500b1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1781,7 +1781,7 @@ dependencies = [ "bitflags 2.10.0", "cexpr", "clang-sys", - "itertools 0.10.5", + "itertools 0.13.0", "proc-macro2 1.0.106", "quote 1.0.44", "regex", @@ -8253,7 +8253,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ "heck 0.5.0", - "itertools 0.10.5", + "itertools 0.14.0", "log", "multimap", "once_cell", @@ -8299,7 +8299,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.14.0", "proc-macro2 1.0.106", "quote 1.0.44", "syn 2.0.117", @@ -9473,7 +9473,7 @@ dependencies = [ "once_cell", "ring", "rustls-pki-types", - "rustls-webpki 0.103.12", + "rustls-webpki 0.103.13", "subtle", "zeroize", ] @@ -9555,9 +9555,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.12" +version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8279bb85272c9f10811ae6a6c547ff594d6a7f3c6c6b02ee9726d1d0dcfcdd06" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ "ring", "rustls-pki-types", @@ -12275,7 +12275,7 @@ dependencies = [ [[package]] name = "vector" -version = "0.55.0" +version = "0.56.0" dependencies = [ "apache-avro 0.16.0", "approx", @@ -12912,8 +12912,8 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "vrl" -version = "0.31.0" -source = "git+https://github.com/vectordotdev/vrl.git?branch=main#1357941d4481e84fb0323bd89c69cedf35c323f8" +version = "0.32.0" +source = "git+https://github.com/vectordotdev/vrl.git?branch=main#8a5b4711f37d0b3bd4d0d2ea8bbb500942bd8e98" dependencies = [ "aes", "aes-siv", diff --git a/Cargo.toml b/Cargo.toml index 03aff35ec8c13..1045e07aae1a1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "vector" -version = "0.55.0" +version = "0.56.0" authors = ["Vector Contributors "] edition = "2024" description = "A lightweight and ultra-fast tool for building observability pipelines" diff --git a/changelog.d/23657-grpc-health-protocol.enhancement.md b/changelog.d/23657-grpc-health-protocol.enhancement.md deleted file mode 100644 index a26e9ebf42b2c..0000000000000 --- a/changelog.d/23657-grpc-health-protocol.enhancement.md +++ /dev/null @@ -1,7 +0,0 @@ -`vector` source: Implement standard gRPC health checking protocol (`grpc.health.v1.Health`) -alongside the existing custom health check endpoint. This enables compatibility with standard -tools like `grpc-health-probe` for Kubernetes and other orchestration systems. - -Issue: https://github.com/vectordotdev/vector/issues/23657 - -authors: jpds diff --git a/changelog.d/24291_async_nats_websockets.feature.md b/changelog.d/24291_async_nats_websockets.feature.md deleted file mode 100644 index c11099ce6daf4..0000000000000 --- a/changelog.d/24291_async_nats_websockets.feature.md +++ /dev/null @@ -1,4 +0,0 @@ -Added websocket support to the `nats` source and sink. The `url` field now supports both `ws://` and -`wss://` protocols. - -authors: gedemagt diff --git a/changelog.d/24411_geoip_network_field.enhancement.md b/changelog.d/24411_geoip_network_field.enhancement.md deleted file mode 100644 index 65d33f50f6b1b..0000000000000 --- a/changelog.d/24411_geoip_network_field.enhancement.md +++ /dev/null @@ -1,3 +0,0 @@ -The `geoip` enrichment table now includes a `network` field containing the CIDR network associated with the lookup result, available for all database types (City, ISP/ASN, Connection-Type, Anonymous-IP). - -authors: naa0yama diff --git a/changelog.d/24455_otel_source_per_signal_decoding.enhancement.md b/changelog.d/24455_otel_source_per_signal_decoding.enhancement.md deleted file mode 100644 index 79db8934df463..0000000000000 --- a/changelog.d/24455_otel_source_per_signal_decoding.enhancement.md +++ /dev/null @@ -1,21 +0,0 @@ -The `opentelemetry` source now supports independent configuration of OTLP decoding for logs, metrics, and traces. This allows more granular -control over which signal types are decoded, while maintaining backward compatibility with the existing boolean configuration. - -## Simple boolean form (applies to all signals) - -```yaml -use_otlp_decoding: true # All signals preserve OTLP format -# or -use_otlp_decoding: false # All signals use Vector native format (default) -``` - -## Per-signal configuration - -```yaml -use_otlp_decoding: - logs: false # Convert to Vector native format - metrics: false # Convert to Vector native format - traces: true # Preserve OTLP format -``` - -authors: pront diff --git a/changelog.d/24532_vector_stopped_log_ordering.fix.md b/changelog.d/24532_vector_stopped_log_ordering.fix.md deleted file mode 100644 index c5e395c37ab6f..0000000000000 --- a/changelog.d/24532_vector_stopped_log_ordering.fix.md +++ /dev/null @@ -1,5 +0,0 @@ -Fixed log message ordering on shutdown where `Vector has stopped.` was logged before components had finished draining, causing confusing output interleaved with `Waiting on running components` messages. - -A new `VectorStopping` event was added in the place of the `VectorStopped` event. - -authors: tronboto diff --git a/changelog.d/24552_dnstap_parser_message_size.enhancement.md b/changelog.d/24552_dnstap_parser_message_size.enhancement.md deleted file mode 100644 index 9472bde7f3346..0000000000000 --- a/changelog.d/24552_dnstap_parser_message_size.enhancement.md +++ /dev/null @@ -1,3 +0,0 @@ -Adds new fields to parsed dnstap data: `requestMessageSize` and `responseMessageSize`. It represents the size of the DNS message. - -authors: esensar Quad9DNS diff --git a/changelog.d/24593_graph_edge_attributes.feature.md b/changelog.d/24593_graph_edge_attributes.feature.md deleted file mode 100644 index 719cbc6b4635a..0000000000000 --- a/changelog.d/24593_graph_edge_attributes.feature.md +++ /dev/null @@ -1,3 +0,0 @@ -`graph.edge_attributes` can now be added to transforms and sinks to add attributes to edges in graphs generated using `vector graph`. Memory enrichment tables are also considered for graphs, because they can have inputs and outputs. - -authors: esensar Quad9DNS diff --git a/changelog.d/24619-opentelemetry-header-enrichment.enhancement.md b/changelog.d/24619-opentelemetry-header-enrichment.enhancement.md deleted file mode 100644 index e0b0b33a5caa3..0000000000000 --- a/changelog.d/24619-opentelemetry-header-enrichment.enhancement.md +++ /dev/null @@ -1,7 +0,0 @@ -`opentelemetry` source: Implemented header enrichment for OTLP metrics and traces. Unlike logs, which support enriching -the event itself or its metadata, depending on `log_namespace` settings, for metrics and traces this setting is ignored -and header values are added to the event metadata. - -Issue: https://github.com/vectordotdev/vector/issues/24619 - -authors: ozanichkovsky diff --git a/changelog.d/24639_kafka_sink_add_traces_support.feature.md b/changelog.d/24639_kafka_sink_add_traces_support.feature.md deleted file mode 100644 index b89dff7437ff3..0000000000000 --- a/changelog.d/24639_kafka_sink_add_traces_support.feature.md +++ /dev/null @@ -1,3 +0,0 @@ -The `kafka` sink now supports trace events. - -authors: pstalmach diff --git a/changelog.d/24641-task-transform-utilization.fix.md b/changelog.d/24641-task-transform-utilization.fix.md deleted file mode 100644 index f59a9c8a8eddf..0000000000000 --- a/changelog.d/24641-task-transform-utilization.fix.md +++ /dev/null @@ -1,5 +0,0 @@ -Fixed utilization for task transforms to not account for time spent when downstream -is not polling. If the transform is frequently blocked on downstream components, -the reported utilization should be lower. - -authors: gwenaskell diff --git a/changelog.d/24708_log_error.fix.md b/changelog.d/24708_log_error.fix.md deleted file mode 100644 index d53626b09af05..0000000000000 --- a/changelog.d/24708_log_error.fix.md +++ /dev/null @@ -1,5 +0,0 @@ -The `opentelemetry` source now logs an error if it fails to start up or during runtime. -This can happen when the configuration is invalid, for example trying to bind to the wrong -IP or when hitting the open file limit. - -authors: fbs diff --git a/changelog.d/24729_azure_blob_authentication.feature.md b/changelog.d/24729_azure_blob_authentication.feature.md deleted file mode 100644 index 7b224172563b7..0000000000000 --- a/changelog.d/24729_azure_blob_authentication.feature.md +++ /dev/null @@ -1,3 +0,0 @@ -Re-introduced Azure authentication support to `azure_blob`, including Azure CLI, Managed Identity, Workload Identity, and Managed Identity-based Client Assertion authentication types. - -authors: jlaundry diff --git a/changelog.d/24729_azure_logs_ingestion.breaking.md b/changelog.d/24729_azure_logs_ingestion.breaking.md deleted file mode 100644 index ce0230a352f38..0000000000000 --- a/changelog.d/24729_azure_logs_ingestion.breaking.md +++ /dev/null @@ -1,3 +0,0 @@ -If using the `azure_logs_ingestion` sink (added in Vector 0.54.0) with Client Secret credentials, add `azure_credential_kind = "client_secret_credential"` to your sink config (this was previously the default, and now must be explicitly configured). - -authors: jlaundry diff --git a/changelog.d/24911-buffer-utilization-metric-tracking.fix.md b/changelog.d/24911-buffer-utilization-metric-tracking.fix.md deleted file mode 100644 index fd1c23e5414b6..0000000000000 --- a/changelog.d/24911-buffer-utilization-metric-tracking.fix.md +++ /dev/null @@ -1,3 +0,0 @@ -Fixed regression in buffer utilization metric tracking around underflow. - -authors: bruceg diff --git a/changelog.d/24943_aggregate_transform_reduce_memory.fix.md b/changelog.d/24943_aggregate_transform_reduce_memory.fix.md deleted file mode 100644 index 4f3be38504bcd..0000000000000 --- a/changelog.d/24943_aggregate_transform_reduce_memory.fix.md +++ /dev/null @@ -1,4 +0,0 @@ -Reduced the memory usage in the `aggregate` transform where previous values were being held -even if `mode` was not set to `Diff`. - -authors: thomasqueirozb diff --git a/changelog.d/24951_top_events_out_column.fix.md b/changelog.d/24951_top_events_out_column.fix.md deleted file mode 100644 index e4191569cc205..0000000000000 --- a/changelog.d/24951_top_events_out_column.fix.md +++ /dev/null @@ -1,3 +0,0 @@ -Fixed `vector top` displaying per-output sent events in the wrong column (Bytes In instead of Events Out) for components with multiple output ports. - -authors: pront diff --git a/changelog.d/24956_datadog_metrics_zstd_series_v2.enhancement.md b/changelog.d/24956_datadog_metrics_zstd_series_v2.enhancement.md deleted file mode 100644 index 1ecaef6b46b8c..0000000000000 --- a/changelog.d/24956_datadog_metrics_zstd_series_v2.enhancement.md +++ /dev/null @@ -1,3 +0,0 @@ -The `datadog_metrics` sink now uses zstd compression when submitting metrics to the Series v2 (`/api/v2/series`) and Sketches endpoints. Series v1 continues to use zlib (deflate). - -authors: vladimir-dd diff --git a/changelog.d/25064_fixed_high_cpu_usage_in_file_source.fix.md b/changelog.d/25064_fixed_high_cpu_usage_in_file_source.fix.md deleted file mode 100644 index 8d64593e51b5c..0000000000000 --- a/changelog.d/25064_fixed_high_cpu_usage_in_file_source.fix.md +++ /dev/null @@ -1,3 +0,0 @@ -Fixed an issue in the `file`/`kubernetes_logs` source that could cause unexpectedly high CPU usage after the async file server migration. - -authors: fcfangcc diff --git a/changelog.d/aws_s3_parquet_encoding.feature.md b/changelog.d/aws_s3_parquet_encoding.feature.md deleted file mode 100644 index afaa01cc70c26..0000000000000 --- a/changelog.d/aws_s3_parquet_encoding.feature.md +++ /dev/null @@ -1,10 +0,0 @@ -Add Apache Parquet batch encoding support for the `aws_s3` sink with flexible schema definitions. - -Events can now be encoded as Parquet columnar files with multiple schema input options: - -- **Native Parquet schema** — automatically generate a schema or supply `.schema` file -- **Configurable compression** - (Snappy, ZSTD, GZIP, LZ4, None). - -Enable the `codecs-parquet` feature and configure `batch_encoding` with `codec = "parquet"` in the S3 sink configuration. - -authors: szibis petere-datadog diff --git a/changelog.d/bump_kube_k8s_openapi.enhancement.md b/changelog.d/bump_kube_k8s_openapi.enhancement.md deleted file mode 100644 index 92b8fd24de684..0000000000000 --- a/changelog.d/bump_kube_k8s_openapi.enhancement.md +++ /dev/null @@ -1,2 +0,0 @@ -Bumped `kube` dependency from 0.93.0 to 3.0.1 and `k8s-openapi` from 0.22.0 to 0.27.0, adding support for Kubernetes API versions up to v1.35. -authors: hligit diff --git a/changelog.d/clickhouse_arrow_uuid_support.enhancement.md b/changelog.d/clickhouse_arrow_uuid_support.enhancement.md deleted file mode 100644 index e6dfc806b6775..0000000000000 --- a/changelog.d/clickhouse_arrow_uuid_support.enhancement.md +++ /dev/null @@ -1,3 +0,0 @@ -Added support for the ClickHouse `UUID` type in the ArrowStream format for the `clickhouse` sink. UUID columns are now automatically mapped to Arrow `Utf8` and cast by ClickHouse on insert. - -authors: benjamin-awd diff --git a/changelog.d/datadog_agent_source_device_tag_v2.fix.md b/changelog.d/datadog_agent_source_device_tag_v2.fix.md deleted file mode 100644 index 669d594f8c89f..0000000000000 --- a/changelog.d/datadog_agent_source_device_tag_v2.fix.md +++ /dev/null @@ -1,5 +0,0 @@ -`datadog_agent` source: Preserve `device` as a plain tag when decoding v2 series metrics, -instead of incorrectly prefixing it as `resource.device`. This matches the v1 series behavior -and fixes tag remapping for disk, SNMP, and other integrations that use the `device` resource type. - -authors: lisaqvu diff --git a/changelog.d/datadog_metrics_series_v2_default.enhancement.md b/changelog.d/datadog_metrics_series_v2_default.enhancement.md deleted file mode 100644 index 39e857b73f99d..0000000000000 --- a/changelog.d/datadog_metrics_series_v2_default.enhancement.md +++ /dev/null @@ -1,5 +0,0 @@ -The `datadog_metrics` sink now defaults to the Datadog series v2 endpoint (`/api/v2/series`) and -exposes a new `series_api_version` configuration option (`v1` or `v2`) to control which endpoint is -used. Set `series_api_version: v1` to fall back to the legacy v1 endpoint if needed. - -authors: vladimir-dd diff --git a/changelog.d/fix_site_prefix_stripping.fix.md b/changelog.d/fix_site_prefix_stripping.fix.md deleted file mode 100644 index 8136664b5b2fd..0000000000000 --- a/changelog.d/fix_site_prefix_stripping.fix.md +++ /dev/null @@ -1,3 +0,0 @@ -Fixed the Datadog sink healthcheck endpoint computation to preserve site prefixes (e.g. `us3.`, `us5.`, `ap1.`) when deriving the API URL from intake endpoints. Previously, the healthcheck for site-specific endpoints like `https://http-intake.logs.us3.datadoghq.com` would incorrectly call `https://api.datadoghq.com` instead of `https://api.us3.datadoghq.com`, causing unintended cross-site egress traffic. - -authors: vladimir-dd diff --git a/changelog.d/fix_source_lag_time_chunked_batches.fix.md b/changelog.d/fix_source_lag_time_chunked_batches.fix.md deleted file mode 100644 index 6303ba10edd33..0000000000000 --- a/changelog.d/fix_source_lag_time_chunked_batches.fix.md +++ /dev/null @@ -1,3 +0,0 @@ -Fixed an incorrect source_lag_time_seconds measurement in sources that use `send_batch` with large event batches. When a batch was split into multiple chunks, the reference timestamp used to compute lag time was re-captured on each chunk send, causing the lag time for later chunks to be overstated by the amount of time spent waiting for the channel to accept earlier chunks. The reference timestamp is now captured once before iteration and shared across all chunks. - -authors: gwenaskell diff --git a/changelog.d/graphql_to_grpc_api.breaking.md b/changelog.d/graphql_to_grpc_api.breaking.md deleted file mode 100644 index 2c9756ee16a89..0000000000000 --- a/changelog.d/graphql_to_grpc_api.breaking.md +++ /dev/null @@ -1,42 +0,0 @@ -The Vector observability API has been migrated from GraphQL to gRPC for improved -performance, efficiency and maintainability. The `vector top` and `vector tap` -commands continue to work as before, as they have been updated to use the new -gRPC API internally. The gRPC service definition is available in -[`proto/vector/observability.proto`](https://github.com/vectordotdev/vector/blob/master/proto/vector/observability.proto). - -Note: `vector top` and `vector tap` from version 0.55.0 or later are not -compatible with Vector instances running earlier versions. - -- Remove the `api.graphql` and `api.playground` fields from your config. Vector - now rejects configs that contain them. - -- If you use `vector top` or `vector tap` with an explicit `--url`, remove the - `/graphql` path suffix: - -```bash -# Old -vector top --url http://localhost:8686/graphql - -# New (the gRPC API listens at the root) -vector top --url http://localhost:8686 -``` - -- The GraphQL API (HTTP endpoint `/graphql`, WebSocket subscriptions, and the - GraphQL Playground at `/playground`) has been removed. You can interact with - the new gRPC API using tools like - [grpcurl](https://github.com/fullstorydev/grpcurl): - -```bash -# Check health (standard gRPC health check, compatible with Kubernetes gRPC probes) -grpcurl -plaintext localhost:8686 grpc.health.v1.Health/Check - -# List components -grpcurl -plaintext localhost:8686 vector.observability.v1.ObservabilityService/GetComponents - -# Stream events (tap) — limit and interval_ms are required and must be >= 1 -grpcurl -plaintext \ - -d '{"outputs_patterns": ["*"], "limit": 100, "interval_ms": 500}' \ - localhost:8686 vector.observability.v1.ObservabilityService/StreamOutputEvents -``` - -authors: pront diff --git a/changelog.d/remove_deprecated_http_headers_option.breaking.md b/changelog.d/remove_deprecated_http_headers_option.breaking.md deleted file mode 100644 index b2b1fd28b869e..0000000000000 --- a/changelog.d/remove_deprecated_http_headers_option.breaking.md +++ /dev/null @@ -1,4 +0,0 @@ -The `headers` option has been removed from the `http` and `opentelemetry` sinks. -Use `request.headers` instead. This option has been deprecated since v0.33.0. - -authors: thomasqueirozb diff --git a/changelog.d/source_sender_latency_metrics.enhancement.md b/changelog.d/source_sender_latency_metrics.enhancement.md deleted file mode 100644 index 6876641a37764..0000000000000 --- a/changelog.d/source_sender_latency_metrics.enhancement.md +++ /dev/null @@ -1,5 +0,0 @@ -Sources now record the distribution metrics `source_send_latency_seconds` (measuring the time spent -blocking on a single events chunk send operation on the output) and `source_send_batch_latency_seconds` -(encompassing all chunks within a received events batch). - -authors: gwenaskell diff --git a/changelog.d/vector_top_disabled_memory_used.enhancement.md b/changelog.d/vector_top_disabled_memory_used.enhancement.md deleted file mode 100644 index e929a065019ef..0000000000000 --- a/changelog.d/vector_top_disabled_memory_used.enhancement.md +++ /dev/null @@ -1,3 +0,0 @@ -`vector top` terminal UI now shows `disabled` in the Memory Used column when the connected Vector instance was not started with `--allocation-tracing`, instead of displaying misleading zeros. A new `GetAllocationTracingStatus` gRPC endpoint is queried on connect to determine the status. - -authors: pront diff --git a/changelog.d/windows_event_log_source.feature.md b/changelog.d/windows_event_log_source.feature.md deleted file mode 100644 index 7f64f115a3391..0000000000000 --- a/changelog.d/windows_event_log_source.feature.md +++ /dev/null @@ -1,3 +0,0 @@ -Added a new `windows_event_log` source that collects logs from Windows Event Log channels using the native Windows Event Log API with pull-mode subscriptions, bookmark-based checkpointing, and configurable field filtering. - -authors: tot19 diff --git a/changelog.d/windows_service_stop_state_wait.fix.md b/changelog.d/windows_service_stop_state_wait.fix.md deleted file mode 100644 index 45b60f01ca5fa..0000000000000 --- a/changelog.d/windows_service_stop_state_wait.fix.md +++ /dev/null @@ -1,3 +0,0 @@ -Fixed Windows service state checks in `vector service start`/`stop`, and made `vector service stop` wait until the service reaches `Stopped`. Added `--stop-timeout` to `vector service stop` and `vector service uninstall`. - -authors: iMithrellas diff --git a/deny.toml b/deny.toml index ac8e588fa1cbe..0d563622a509e 100644 --- a/deny.toml +++ b/deny.toml @@ -48,4 +48,6 @@ ignore = [ { id = "RUSTSEC-2026-0099", reason = "rustls-webpki 0.102/0.101 is vulnerable - tonic upgrade (https://github.com/vectordotdev/vector/issues/19179)" }, { id = "RUSTSEC-2024-0436", reason = "paste crate is unmaintained - transitive dependency via parquet v56.2.0, no safe upgrade available" }, { id = "RUSTSEC-2026-0097", reason = "rand 0.8.5 unsound with custom logger - transitive dependency, upstream crates have not updated to rand 0.9+" }, + { id = "RUSTSEC-2026-0104", reason = "rustls-webpki 0.102/0.101 CRL parsing panic - tonic upgrade (https://github.com/vectordotdev/vector/issues/19179); 0.103 already patched to 0.103.13" }, + { id = "RUSTSEC-2026-0105", reason = "core2 is unmaintained and all versions yanked - transitive dependency via libflate -> apache-avro, no safe upgrade available" }, ] diff --git a/distribution/install.sh b/distribution/install.sh index ceeb50c253862..f2ab269e2fd01 100755 --- a/distribution/install.sh +++ b/distribution/install.sh @@ -13,7 +13,7 @@ set -u # If PACKAGE_ROOT is unset or empty, default it. PACKAGE_ROOT="${PACKAGE_ROOT:-"https://packages.timber.io/vector"}" # If VECTOR_VERSION is unset or empty, default it. -VECTOR_VERSION="${VECTOR_VERSION:-"0.54.0"}" +VECTOR_VERSION="${VECTOR_VERSION:-"0.55.0"}" _divider="--------------------------------------------------------------------------------" _prompt=">>>" _indent=" " diff --git a/distribution/kubernetes/vector-agent/README.md b/distribution/kubernetes/vector-agent/README.md index dfe0f05deb137..15dfbf6207643 100644 --- a/distribution/kubernetes/vector-agent/README.md +++ b/distribution/kubernetes/vector-agent/README.md @@ -1,6 +1,6 @@ The kubernetes manifests found in this directory have been automatically generated from the [helm chart `vector/vector`](https://github.com/vectordotdev/helm-charts/tree/master/charts/vector) -version 0.51.0 with the following `values.yaml`: +version 0.52.0 with the following `values.yaml`: ```yaml role: Agent diff --git a/distribution/kubernetes/vector-agent/configmap.yaml b/distribution/kubernetes/vector-agent/configmap.yaml index e3d7499257236..99cb8d91f4345 100644 --- a/distribution/kubernetes/vector-agent/configmap.yaml +++ b/distribution/kubernetes/vector-agent/configmap.yaml @@ -9,14 +9,13 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Agent - app.kubernetes.io/version: "0.54.0-distroless-libc" + app.kubernetes.io/version: "0.55.0-distroless-libc" data: agent.yaml: | data_dir: /vector-data-dir api: enabled: true - address: 127.0.0.1:8686 - playground: false + address: 0.0.0.0:8686 sources: kubernetes_logs: type: kubernetes_logs diff --git a/distribution/kubernetes/vector-agent/daemonset.yaml b/distribution/kubernetes/vector-agent/daemonset.yaml index 4c468b88989a7..8c74a70107dec 100644 --- a/distribution/kubernetes/vector-agent/daemonset.yaml +++ b/distribution/kubernetes/vector-agent/daemonset.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Agent - app.kubernetes.io/version: "0.54.0-distroless-libc" + app.kubernetes.io/version: "0.55.0-distroless-libc" spec: selector: matchLabels: @@ -30,7 +30,7 @@ spec: dnsPolicy: ClusterFirst containers: - name: vector - image: "docker.io/timberio/vector:0.54.0-distroless-libc" + image: "docker.io/timberio/vector:0.55.0-distroless-libc" imagePullPolicy: IfNotPresent args: - --config-dir @@ -58,6 +58,10 @@ spec: - name: prom-exporter containerPort: 9090 protocol: TCP + readinessProbe: + httpGet: + path: /health + port: 8686 volumeMounts: - name: data mountPath: "/vector-data-dir" diff --git a/distribution/kubernetes/vector-agent/rbac.yaml b/distribution/kubernetes/vector-agent/rbac.yaml index 93b151826e238..5cd8e99594981 100644 --- a/distribution/kubernetes/vector-agent/rbac.yaml +++ b/distribution/kubernetes/vector-agent/rbac.yaml @@ -10,7 +10,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Agent - app.kubernetes.io/version: "0.54.0-distroless-libc" + app.kubernetes.io/version: "0.55.0-distroless-libc" rules: - apiGroups: - "" @@ -31,7 +31,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Agent - app.kubernetes.io/version: "0.54.0-distroless-libc" + app.kubernetes.io/version: "0.55.0-distroless-libc" roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole diff --git a/distribution/kubernetes/vector-agent/service-headless.yaml b/distribution/kubernetes/vector-agent/service-headless.yaml index 2f3a72b2619fd..573c2e67310c4 100644 --- a/distribution/kubernetes/vector-agent/service-headless.yaml +++ b/distribution/kubernetes/vector-agent/service-headless.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Agent - app.kubernetes.io/version: "0.54.0-distroless-libc" + app.kubernetes.io/version: "0.55.0-distroless-libc" annotations: spec: clusterIP: None diff --git a/distribution/kubernetes/vector-agent/serviceaccount.yaml b/distribution/kubernetes/vector-agent/serviceaccount.yaml index 9db1dca796f94..c1b65be5aea95 100644 --- a/distribution/kubernetes/vector-agent/serviceaccount.yaml +++ b/distribution/kubernetes/vector-agent/serviceaccount.yaml @@ -9,5 +9,5 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Agent - app.kubernetes.io/version: "0.54.0-distroless-libc" + app.kubernetes.io/version: "0.55.0-distroless-libc" automountServiceAccountToken: true diff --git a/distribution/kubernetes/vector-aggregator/README.md b/distribution/kubernetes/vector-aggregator/README.md index 213cd06790f2e..ab45515cdaf94 100644 --- a/distribution/kubernetes/vector-aggregator/README.md +++ b/distribution/kubernetes/vector-aggregator/README.md @@ -1,6 +1,6 @@ The kubernetes manifests found in this directory have been automatically generated from the [helm chart `vector/vector`](https://github.com/vectordotdev/helm-charts/tree/master/charts/vector) -version 0.51.0 with the following `values.yaml`: +version 0.52.0 with the following `values.yaml`: ```yaml diff --git a/distribution/kubernetes/vector-aggregator/configmap.yaml b/distribution/kubernetes/vector-aggregator/configmap.yaml index 39fa1093bffa6..464539b0afd4d 100644 --- a/distribution/kubernetes/vector-aggregator/configmap.yaml +++ b/distribution/kubernetes/vector-aggregator/configmap.yaml @@ -9,14 +9,13 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Aggregator - app.kubernetes.io/version: "0.54.0-distroless-libc" + app.kubernetes.io/version: "0.55.0-distroless-libc" data: aggregator.yaml: | data_dir: /vector-data-dir api: enabled: true - address: 127.0.0.1:8686 - playground: false + address: 0.0.0.0:8686 sources: datadog_agent: address: 0.0.0.0:8282 diff --git a/distribution/kubernetes/vector-aggregator/service-headless.yaml b/distribution/kubernetes/vector-aggregator/service-headless.yaml index 3170f1c3751cd..bd5a2c5cc8432 100644 --- a/distribution/kubernetes/vector-aggregator/service-headless.yaml +++ b/distribution/kubernetes/vector-aggregator/service-headless.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Aggregator - app.kubernetes.io/version: "0.54.0-distroless-libc" + app.kubernetes.io/version: "0.55.0-distroless-libc" annotations: spec: clusterIP: None diff --git a/distribution/kubernetes/vector-aggregator/service.yaml b/distribution/kubernetes/vector-aggregator/service.yaml index 3c423d4590371..5277bea614bdf 100644 --- a/distribution/kubernetes/vector-aggregator/service.yaml +++ b/distribution/kubernetes/vector-aggregator/service.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Aggregator - app.kubernetes.io/version: "0.54.0-distroless-libc" + app.kubernetes.io/version: "0.55.0-distroless-libc" annotations: spec: ports: diff --git a/distribution/kubernetes/vector-aggregator/serviceaccount.yaml b/distribution/kubernetes/vector-aggregator/serviceaccount.yaml index 4f97844c7fee1..8bc7b591147b7 100644 --- a/distribution/kubernetes/vector-aggregator/serviceaccount.yaml +++ b/distribution/kubernetes/vector-aggregator/serviceaccount.yaml @@ -9,5 +9,5 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Aggregator - app.kubernetes.io/version: "0.54.0-distroless-libc" + app.kubernetes.io/version: "0.55.0-distroless-libc" automountServiceAccountToken: true diff --git a/distribution/kubernetes/vector-aggregator/statefulset.yaml b/distribution/kubernetes/vector-aggregator/statefulset.yaml index d107f7d55cd91..e5ec9f1fcd1aa 100644 --- a/distribution/kubernetes/vector-aggregator/statefulset.yaml +++ b/distribution/kubernetes/vector-aggregator/statefulset.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Aggregator - app.kubernetes.io/version: "0.54.0-distroless-libc" + app.kubernetes.io/version: "0.55.0-distroless-libc" annotations: {} spec: replicas: 1 @@ -34,7 +34,7 @@ spec: dnsPolicy: ClusterFirst containers: - name: vector - image: "docker.io/timberio/vector:0.54.0-distroless-libc" + image: "docker.io/timberio/vector:0.55.0-distroless-libc" imagePullPolicy: IfNotPresent args: - --config-dir @@ -67,6 +67,10 @@ spec: - name: prom-exporter containerPort: 9090 protocol: TCP + readinessProbe: + httpGet: + path: /health + port: 8686 volumeMounts: - name: data mountPath: "/vector-data-dir" diff --git a/distribution/kubernetes/vector-stateless-aggregator/README.md b/distribution/kubernetes/vector-stateless-aggregator/README.md index d03fb7eb5250b..d3a701364d611 100644 --- a/distribution/kubernetes/vector-stateless-aggregator/README.md +++ b/distribution/kubernetes/vector-stateless-aggregator/README.md @@ -1,6 +1,6 @@ The kubernetes manifests found in this directory have been automatically generated from the [helm chart `vector/vector`](https://github.com/vectordotdev/helm-charts/tree/master/charts/vector) -version 0.51.0 with the following `values.yaml`: +version 0.52.0 with the following `values.yaml`: ```yaml role: Stateless-Aggregator diff --git a/distribution/kubernetes/vector-stateless-aggregator/configmap.yaml b/distribution/kubernetes/vector-stateless-aggregator/configmap.yaml index e13b42240886c..5fe8fa6d1a4ca 100644 --- a/distribution/kubernetes/vector-stateless-aggregator/configmap.yaml +++ b/distribution/kubernetes/vector-stateless-aggregator/configmap.yaml @@ -9,14 +9,13 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Stateless-Aggregator - app.kubernetes.io/version: "0.54.0-distroless-libc" + app.kubernetes.io/version: "0.55.0-distroless-libc" data: aggregator.yaml: | data_dir: /vector-data-dir api: enabled: true - address: 127.0.0.1:8686 - playground: false + address: 0.0.0.0:8686 sources: datadog_agent: address: 0.0.0.0:8282 diff --git a/distribution/kubernetes/vector-stateless-aggregator/deployment.yaml b/distribution/kubernetes/vector-stateless-aggregator/deployment.yaml index 60e585952669b..d7b31fd50a07c 100644 --- a/distribution/kubernetes/vector-stateless-aggregator/deployment.yaml +++ b/distribution/kubernetes/vector-stateless-aggregator/deployment.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Stateless-Aggregator - app.kubernetes.io/version: "0.54.0-distroless-libc" + app.kubernetes.io/version: "0.55.0-distroless-libc" annotations: {} spec: replicas: 1 @@ -32,7 +32,7 @@ spec: dnsPolicy: ClusterFirst containers: - name: vector - image: "docker.io/timberio/vector:0.54.0-distroless-libc" + image: "docker.io/timberio/vector:0.55.0-distroless-libc" imagePullPolicy: IfNotPresent args: - --config-dir @@ -65,6 +65,10 @@ spec: - name: prom-exporter containerPort: 9090 protocol: TCP + readinessProbe: + httpGet: + path: /health + port: 8686 volumeMounts: - name: data mountPath: "/vector-data-dir" diff --git a/distribution/kubernetes/vector-stateless-aggregator/service-headless.yaml b/distribution/kubernetes/vector-stateless-aggregator/service-headless.yaml index bb2c35189c395..0c6143ee4074c 100644 --- a/distribution/kubernetes/vector-stateless-aggregator/service-headless.yaml +++ b/distribution/kubernetes/vector-stateless-aggregator/service-headless.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Stateless-Aggregator - app.kubernetes.io/version: "0.54.0-distroless-libc" + app.kubernetes.io/version: "0.55.0-distroless-libc" annotations: spec: clusterIP: None diff --git a/distribution/kubernetes/vector-stateless-aggregator/service.yaml b/distribution/kubernetes/vector-stateless-aggregator/service.yaml index b96e58ef1ea31..2f325ba8df63a 100644 --- a/distribution/kubernetes/vector-stateless-aggregator/service.yaml +++ b/distribution/kubernetes/vector-stateless-aggregator/service.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Stateless-Aggregator - app.kubernetes.io/version: "0.54.0-distroless-libc" + app.kubernetes.io/version: "0.55.0-distroless-libc" annotations: spec: ports: diff --git a/distribution/kubernetes/vector-stateless-aggregator/serviceaccount.yaml b/distribution/kubernetes/vector-stateless-aggregator/serviceaccount.yaml index 2636f02bed059..0a282a591d5e7 100644 --- a/distribution/kubernetes/vector-stateless-aggregator/serviceaccount.yaml +++ b/distribution/kubernetes/vector-stateless-aggregator/serviceaccount.yaml @@ -9,5 +9,5 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Stateless-Aggregator - app.kubernetes.io/version: "0.54.0-distroless-libc" + app.kubernetes.io/version: "0.55.0-distroless-libc" automountServiceAccountToken: true diff --git a/website/content/en/highlights/2026-04-20-0-55-0-upgrade-guide.md b/website/content/en/highlights/2026-04-20-0-55-0-upgrade-guide.md new file mode 100644 index 0000000000000..ee06ab61b83fd --- /dev/null +++ b/website/content/en/highlights/2026-04-20-0-55-0-upgrade-guide.md @@ -0,0 +1,139 @@ +--- +date: "2026-04-20" +title: "0.55 Upgrade Guide" +description: "An upgrade guide that addresses breaking changes in 0.55.0" +authors: ["pront"] +release: "0.55.0" +hide_on_release_notes: false +badges: + type: breaking change +--- + + +## Vector breaking changes + +1. [Observability API migrated from GraphQL to gRPC](#graphql-to-grpc) +2. [`headers` option removed from `http` and `opentelemetry` sinks](#http-headers-removed) +3. [`azure_logs_ingestion` sink requires explicit `azure_credential_kind`](#azure-logs-ingestion-credential) + +## Vector upgrade guide + +### Observability API migrated from GraphQL to gRPC {#graphql-to-grpc} + +Vector's observability API has been rewritten from GraphQL to gRPC. The following HTTP endpoints have been removed: + +- `/graphql` (HTTP POST queries and WebSocket subscriptions) +- `/playground` (the GraphQL Playground) + +The HTTP `GET /health` endpoint is unchanged and continues to serve Kubernetes HTTP liveness/readiness probes as before. + +Vector will now reject configs that contain `api.graphql` or `api.playground`. + +The gRPC service definition is available at [`proto/vector/observability.proto`](https://github.com/vectordotdev/vector/blob/master/proto/vector/observability.proto). A standard [gRPC Health Checking Protocol](https://grpc.io/docs/guides/health-checking/) service (`grpc.health.v1.Health`) is also exposed for tooling that prefers gRPC probes, such as [`grpc-health-probe`](https://github.com/grpc-ecosystem/grpc-health-probe) or native Kubernetes gRPC probes. + +Note: `vector top` and `vector tap` from version 0.55.0 or later are not compatible with Vector instances running earlier versions. + +#### Action needed + +1. Please remove the `api.graphql` and `api.playground` fields from your Vector configuration. + +2. If you invoke `vector top` or `vector tap` with an explicit `--url`, please drop the `/graphql` suffix: + + ```bash + # Old + vector top --url http://localhost:8686/graphql + + # New (the gRPC API listens at the root) + vector top --url http://localhost:8686 + ``` + +3. If you interact with the API from external tooling, please migrate to the gRPC service. For example, using [`grpcurl`](https://github.com/fullstorydev/grpcurl): + + ```bash + # Check health (standard gRPC health check) + grpcurl -plaintext localhost:8686 grpc.health.v1.Health/Check + + # List components + grpcurl -plaintext localhost:8686 vector.observability.v1.ObservabilityService/GetComponents + + # Stream events (tap). Note: limit and interval_ms are required and must be >= 1 + grpcurl -plaintext \ + -d '{"outputs_patterns": ["*"], "limit": 100, "interval_ms": 500}' \ + localhost:8686 vector.observability.v1.ObservabilityService/StreamOutputEvents + ``` + +### `headers` option removed from `http` and `opentelemetry` sinks {#http-headers-removed} + +The top-level `headers` option on the `http` and `opentelemetry` sinks has been removed. It was deprecated in v0.33.0 in favor of `request.headers`. + +#### Action needed + +Please move any `headers` values under `request.headers`. The exact location differs between the two sinks. + +For the `http` sink, `request` lives at the sink root: + +```yaml +# Old +sinks: + my_http: + type: http + uri: https://example.com + headers: + X-Custom-Header: value + +# New +sinks: + my_http: + type: http + uri: https://example.com + request: + headers: + X-Custom-Header: value +``` + +For the `opentelemetry` sink, `request` lives under the `protocol` block (because the sink wraps its transport under `protocol:`): + +```yaml +# Old +sinks: + my_otel: + type: opentelemetry + protocol: + type: http + uri: http://localhost:5318/v1/logs + headers: + X-Custom-Header: value + +# New +sinks: + my_otel: + type: opentelemetry + protocol: + type: http + uri: http://localhost:5318/v1/logs + request: + headers: + X-Custom-Header: value +``` + +Note: placing `request` at the `opentelemetry` sink root will not error, but the headers will be silently ignored. Please place `request` under `protocol`. + +### `azure_logs_ingestion` sink requires explicit `azure_credential_kind` {#azure-logs-ingestion-credential} + +The `azure_logs_ingestion` sink (introduced in 0.54.0) previously used `client_secret_credential` as the implicit default when Client Secret credentials were provided. That implicit default has been removed; `azure_credential_kind` must now be set explicitly. + +#### Action needed + +If you are using `azure_logs_ingestion` with Client Secret credentials, please add `azure_credential_kind: client_secret_credential` under the sink's `auth` block (alongside `azure_tenant_id`, `azure_client_id`, and `azure_client_secret`): + +```yaml +sinks: + my_azure_logs: + type: azure_logs_ingestion + # ... remaining config + auth: + azure_credential_kind: client_secret_credential + azure_tenant_id: "" + azure_client_id: "" + azure_client_secret: "" +``` diff --git a/website/content/en/releases/0.55.0.md b/website/content/en/releases/0.55.0.md new file mode 100644 index 0000000000000..55389520154a6 --- /dev/null +++ b/website/content/en/releases/0.55.0.md @@ -0,0 +1,4 @@ +--- +title: Vector v0.55.0 release notes +weight: 34 +--- diff --git a/website/cue/reference/administration/interfaces/kubectl.cue b/website/cue/reference/administration/interfaces/kubectl.cue index 32258dfb538b9..94d30ab6bedc2 100644 --- a/website/cue/reference/administration/interfaces/kubectl.cue +++ b/website/cue/reference/administration/interfaces/kubectl.cue @@ -19,7 +19,7 @@ administration: interfaces: kubectl: { role_implementations: [Name=string]: { commands: { _deployment_variant: string - _vector_version: "0.54" + _vector_version: "0.55" _namespace: string | *"vector" _controller_resource_type: string _controller_resource_name: string | *_deployment_variant diff --git a/website/cue/reference/releases/0.55.0.cue b/website/cue/reference/releases/0.55.0.cue new file mode 100644 index 0000000000000..1e022d2151a93 --- /dev/null +++ b/website/cue/reference/releases/0.55.0.cue @@ -0,0 +1,583 @@ +package metadata + +releases: "0.55.0": { + date: "2026-04-22" + codename: "" + + whats_next: [] + + description: """ + The [COSE team](https://opensource.datadoghq.com/about/#the-community-open-source-engineering-team) is excited to announce version `0.55.0`! + + ## Release highlights + + - New `windows_event_log` source that collects logs from Windows Event Log channels using the + native Windows Event Log API, with pull-mode subscriptions, bookmark-based checkpointing, and + configurable field filtering. + - The `aws_s3` sink now supports Apache Parquet batch encoding. Events can be written as + Parquet columnar files with either an auto-generated native schema or a supplied `.schema` + file, and configurable compression (Snappy, ZSTD, GZIP, LZ4, or none). + - The `azure_blob` sink re-gains first-class [Azure authentication](https://learn.microsoft.com/en-us/azure/storage/blobs/authorize-access-azure-active-directory): + Azure CLI, Managed Identity, Workload Identity, and Managed Identity-based Client Assertion + credential kinds are all supported again. + - The `datadog_metrics` sink now defaults to the Series v2 endpoint (`/api/v2/series`) and + uses zstd compression for Series v2 and Sketches, which should yield smaller payloads + and more efficient batching and intake. A new `series_api_version` option (`v1` or `v2`) + is available to opt back to the legacy v1 endpoint; Series v1 continues to use zlib. + - `vector top` is more trustworthy: per-output events for components with multiple output + ports are now shown in the correct `Events Out` column, and the `Memory Used` column + now reports `disabled` when the target Vector instance was started without + `--allocation-tracing` instead of a misleading `0`. + - Better internal metrics for capacity planning and alerting: + - New source-send latency distributions (`source_send_latency_seconds`, + `source_send_batch_latency_seconds`) surface backpressure close to the source. + - Task-transform `utilization` no longer counts time spent waiting on downstream + components, giving a more representative view of transform saturation. + - Fixed a regression in buffer utilization metric tracking around underflow. + - Fixed a performance regression in the `file` and `kubernetes_logs` sources that could + cause unexpectedly high CPU usage, introduced in 0.50.0. + + ## Breaking Changes + + See the [0.55 upgrade guide](/highlights/2026-04-20-0-55-0-upgrade-guide/) for full details + and migration steps. At a glance, you are affected if you: + + - query or tail the Vector observability API in any way: the API has moved from + GraphQL to gRPC. This includes `vector top`, `vector tap`, and anything that talked to + `/graphql` or the `/playground`. The HTTP `GET /health` endpoint is unchanged and continues + to serve Kubernetes HTTP probes as before. + - set the top-level `headers` option on the `http` or `opentelemetry` sinks: it has been + removed. + - use the `azure_logs_ingestion` sink with Client Secret credentials: `azure_credential_kind` + must now be set explicitly. + """ + + changelog: [ + { + type: "enhancement" + description: #""" + `vector` source: Implement standard gRPC health checking protocol (`grpc.health.v1.Health`) + alongside the existing custom health check endpoint. This enables compatibility with standard + tools like `grpc-health-probe` for Kubernetes and other orchestration systems. + + Issue: https://github.com/vectordotdev/vector/issues/23657 + """# + contributors: ["jpds"] + }, + { + type: "feat" + description: #""" + Added websocket support to the `nats` source and sink. The `url` field now supports both `ws://` and + `wss://` protocols. + """# + contributors: ["gedemagt"] + }, + { + type: "enhancement" + description: #""" + The `geoip` enrichment table now includes a `network` field containing the CIDR network associated with the lookup result, available for all database types (City, ISP/ASN, Connection-Type, Anonymous-IP). + """# + contributors: ["naa0yama"] + }, + { + type: "enhancement" + description: #""" + The `opentelemetry` source now supports independent configuration of OTLP decoding for logs, metrics, and traces. This allows more granular + control over which signal types are decoded, while maintaining backward compatibility with the existing boolean configuration. + + ## Simple boolean form (applies to all signals) + + ```yaml + use_otlp_decoding: true # All signals preserve OTLP format + # or + use_otlp_decoding: false # All signals use Vector native format (default) + ``` + + ## Per-signal configuration + + ```yaml + use_otlp_decoding: + logs: false # Convert to Vector native format + metrics: false # Convert to Vector native format + traces: true # Preserve OTLP format + ``` + """# + contributors: ["pront"] + }, + { + type: "fix" + description: #""" + Fixed log message ordering on shutdown where `Vector has stopped.` was logged before components had finished draining, causing confusing output interleaved with `Waiting on running components` messages. + + A new `VectorStopping` event was added in the place of the `VectorStopped` event. + """# + contributors: ["tronboto"] + }, + { + type: "enhancement" + description: #""" + Adds new fields to parsed dnstap data: `requestMessageSize` and `responseMessageSize`. It represents the size of the DNS message. + """# + contributors: ["esensar", "Quad9DNS"] + }, + { + type: "feat" + description: #""" + `graph.edge_attributes` can now be added to transforms and sinks to add attributes to edges in graphs generated using `vector graph`. Memory enrichment tables are also considered for graphs, because they can have inputs and outputs. + """# + contributors: ["esensar", "Quad9DNS"] + }, + { + type: "enhancement" + description: #""" + `opentelemetry` source: Implemented header enrichment for OTLP metrics and traces. Unlike logs, which support enriching + the event itself or its metadata, depending on `log_namespace` settings, for metrics and traces this setting is ignored + and header values are added to the event metadata. + + Issue: https://github.com/vectordotdev/vector/issues/24619 + """# + contributors: ["ozanichkovsky"] + }, + { + type: "feat" + description: #""" + The `kafka` sink now supports trace events. + """# + contributors: ["pstalmach"] + }, + { + type: "fix" + description: #""" + Fixed utilization for task transforms to not account for time spent when downstream + is not polling. If the transform is frequently blocked on downstream components, + the reported utilization should be lower. + """# + contributors: ["gwenaskell"] + }, + { + type: "fix" + description: #""" + The `opentelemetry` source now logs an error if it fails to start up or during runtime. + This can happen when the configuration is invalid, for example trying to bind to the wrong + IP or when hitting the open file limit. + """# + contributors: ["fbs"] + }, + { + type: "feat" + description: #""" + Re-introduced Azure authentication support to `azure_blob`, including Azure CLI, Managed Identity, Workload Identity, and Managed Identity-based Client Assertion authentication types. + """# + contributors: ["jlaundry"] + }, + { + type: "chore" + description: #""" + If using the `azure_logs_ingestion` sink (added in Vector 0.54.0) with Client Secret credentials, add `azure_credential_kind = "client_secret_credential"` under the sink's `auth` block (alongside `azure_tenant_id`, `azure_client_id`, and `azure_client_secret`). This was previously the default, and now must be explicitly configured. See the [0.55 upgrade guide](/highlights/2026-04-20-0-55-0-upgrade-guide/) for an example. + """# + contributors: ["jlaundry"] + }, + { + type: "fix" + description: #""" + Fixed regression in buffer utilization metric tracking around underflow. + """# + contributors: ["bruceg"] + }, + { + type: "fix" + description: #""" + Reduced the memory usage in the `aggregate` transform where previous values were being held + even if `mode` was not set to `Diff`. + """# + contributors: ["thomasqueirozb"] + }, + { + type: "fix" + description: #""" + Fixed `vector top` displaying per-output sent events in the wrong column (Bytes In instead of Events Out) for components with multiple output ports. + """# + contributors: ["pront"] + }, + { + type: "enhancement" + description: #""" + The `datadog_metrics` sink now uses zstd compression when submitting metrics to the Series v2 (`/api/v2/series`) and Sketches endpoints. Series v1 continues to use zlib (deflate). + """# + contributors: ["vladimir-dd"] + }, + { + type: "fix" + description: #""" + Fixed an issue in the `file`/`kubernetes_logs` source that could cause unexpectedly high CPU usage after the async file server migration. + """# + contributors: ["fcfangcc"] + }, + { + type: "feat" + description: #""" + Add Apache Parquet batch encoding support for the `aws_s3` sink with flexible schema definitions. + + Events can now be encoded as Parquet columnar files with multiple schema input options: + + - **Native Parquet schema** — automatically generate a schema or supply `.schema` file + - **Configurable compression** - (Snappy, ZSTD, GZIP, LZ4, None). + + Enable the `codecs-parquet` feature and configure `batch_encoding` with `codec = "parquet"` in the S3 sink configuration. + """# + contributors: ["szibis", "petere-datadog"] + }, + { + type: "enhancement" + description: #""" + Bumped `kube` dependency from 0.93.0 to 3.0.1 and `k8s-openapi` from 0.22.0 to 0.27.0, adding support for Kubernetes API versions up to v1.35. + """# + contributors: ["hligit"] + }, + { + type: "enhancement" + description: #""" + Added support for the ClickHouse `UUID` type in the ArrowStream format for the `clickhouse` sink. UUID columns are now automatically mapped to Arrow `Utf8` and cast by ClickHouse on insert. + """# + contributors: ["benjamin-awd"] + }, + { + type: "fix" + description: #""" + `datadog_agent` source: Preserve `device` as a plain tag when decoding v2 series metrics, + instead of incorrectly prefixing it as `resource.device`. This matches the v1 series behavior + and fixes tag remapping for disk, SNMP, and other integrations that use the `device` resource type. + """# + contributors: ["lisaqvu"] + }, + { + type: "enhancement" + description: #""" + The `datadog_metrics` sink now defaults to the Datadog series v2 endpoint (`/api/v2/series`) and + exposes a new `series_api_version` configuration option (`v1` or `v2`) to control which endpoint is + used. Set `series_api_version: v1` to fall back to the legacy v1 endpoint if needed. + """# + contributors: ["vladimir-dd"] + }, + { + type: "fix" + description: #""" + Fixed the Datadog sink healthcheck endpoint computation to preserve site prefixes (e.g. `us3.`, `us5.`, `ap1.`) when deriving the API URL from intake endpoints. Previously, the healthcheck for site-specific endpoints like `https://http-intake.logs.us3.datadoghq.com` would incorrectly call `https://api.datadoghq.com` instead of `https://api.us3.datadoghq.com`, causing unintended cross-site egress traffic. + """# + contributors: ["vladimir-dd"] + }, + { + type: "fix" + description: #""" + Fixed an incorrect source_lag_time_seconds measurement in sources that use `send_batch` with large event batches. When a batch was split into multiple chunks, the reference timestamp used to compute lag time was re-captured on each chunk send, causing the lag time for later chunks to be overstated by the amount of time spent waiting for the channel to accept earlier chunks. The reference timestamp is now captured once before iteration and shared across all chunks. + """# + contributors: ["gwenaskell"] + }, + { + type: "chore" + description: #""" + The Vector observability API has been migrated from GraphQL to gRPC for improved + performance, efficiency and maintainability. The `vector top` and `vector tap` + commands continue to work as before, as they have been updated to use the new + gRPC API internally. The gRPC service definition is available in + [`proto/vector/observability.proto`](https://github.com/vectordotdev/vector/blob/master/proto/vector/observability.proto). + + Note: `vector top` and `vector tap` from version 0.55.0 or later are not + compatible with Vector instances running earlier versions. + + - Remove the `api.graphql` and `api.playground` fields from your config. Vector + now rejects configs that contain them. + + - If you use `vector top` or `vector tap` with an explicit `--url`, remove the + `/graphql` path suffix: + + ```bash + # Old + vector top --url http://localhost:8686/graphql + + # New (the gRPC API listens at the root) + vector top --url http://localhost:8686 + ``` + + - The GraphQL API (HTTP endpoint `/graphql`, WebSocket subscriptions, and the + GraphQL Playground at `/playground`) has been removed. You can interact with + the new gRPC API using tools like + [grpcurl](https://github.com/fullstorydev/grpcurl): + + ```bash + # Check health (standard gRPC health check, compatible with Kubernetes gRPC probes) + grpcurl -plaintext localhost:8686 grpc.health.v1.Health/Check + + # List components + grpcurl -plaintext localhost:8686 vector.observability.v1.ObservabilityService/GetComponents + + # Stream events (tap) — limit and interval_ms are required and must be >= 1 + grpcurl -plaintext \ + -d '{"outputs_patterns": ["*"], "limit": 100, "interval_ms": 500}' \ + localhost:8686 vector.observability.v1.ObservabilityService/StreamOutputEvents + ``` + """# + contributors: ["pront"] + }, + { + type: "chore" + description: #""" + The `headers` option has been removed from the `http` and `opentelemetry` sinks. + Use `request.headers` instead. On the `opentelemetry` sink, `request` is nested under + `protocol`; see the [0.55 upgrade guide](/highlights/2026-04-20-0-55-0-upgrade-guide/) + for examples. This option has been deprecated since v0.33.0. + """# + contributors: ["thomasqueirozb"] + }, + { + type: "enhancement" + description: #""" + Sources now record the distribution metrics `source_send_latency_seconds` (measuring the time spent + blocking on a single events chunk send operation on the output) and `source_send_batch_latency_seconds` + (encompassing all chunks within a received events batch). + """# + contributors: ["gwenaskell"] + }, + { + type: "enhancement" + description: #""" + `vector top` terminal UI now shows `disabled` in the Memory Used column when the connected Vector instance was not started with `--allocation-tracing`, instead of displaying misleading zeros. A new `GetAllocationTracingStatus` gRPC endpoint is queried on connect to determine the status. + """# + contributors: ["pront"] + }, + { + type: "feat" + description: #""" + Added a new `windows_event_log` source that collects logs from Windows Event Log channels using the native Windows Event Log API with pull-mode subscriptions, bookmark-based checkpointing, and configurable field filtering. + """# + contributors: ["tot19"] + }, + { + type: "fix" + description: #""" + Fixed Windows service state checks in `vector service start`/`stop`, and made `vector service stop` wait until the service reaches `Stopped`. Added `--stop-timeout` to `vector service stop` and `vector service uninstall`. + """# + contributors: ["iMithrellas"] + }, + { + type: "fix" + description: #""" + The text content generated by the `demo_logs` source has changed: the + pool of fake usernames and the pool of fake domain TLDs are now both + defined inside Vector rather than pulled from an external crate. The + line formats (`apache_common`, `apache_error`, `json`, `syslog`, + `bsd_syslog`) are unchanged. If any of your tests or downstream + pipelines assert on specific generated usernames or TLDs, please + update those expectations. + """# + contributors: ["pront"] + }, + ] + + vrl_changelog: """ + ### [0.32.0 (2026-04-16)] + + #### New Features + + - Added a new `encode_csv` function that encodes an array of values into a CSV-formatted string. This is the inverse of the existing `parse_csv` function and supports an optional single-byte delimiter (defaults to `,`). + + authors: armleth (https://github.com/vectordotdev/vrl/pull/1649) + - Added `to_entries` and `from_entries` with jq-compatible behavior: `to_entries` supports both objects and arrays, and `from_entries` accepts `key`/`Key`/`name`/`Name` and `value`/`Value` aliases. + + authors: close2code-palm (https://github.com/vectordotdev/vrl/pull/1653) + + #### Enhancements + + - Added `except` parameter to `flatten` function to exclude specific keys from being flattened. + + authors: benjamin-awd (https://github.com/vectordotdev/vrl/pull/1682) + + #### Fixes + + - Fixed a bug where the REPL input validator was executing programs instead of only compiling them, causing functions with side effects (e.g. `http_request`) to run twice per submission. + + authors: prontidis (https://github.com/vectordotdev/vrl/pull/1701) + + + ### [0.31.0 (2026-03-05)] + """ + + commits: [ + {sha: "bc30368b01670480b4e83bebdd9eb09ec78bd2e3", date: "2026-03-10 01:08:29 UTC", description: "make build vrl-docs work with released VRL version", pr_number: 24877, scopes: ["vdev"], type: "fix", breaking_change: false, author: "Thomas", files_count: 2, insertions_count: 21, deletions_count: 19}, + {sha: "3f98053f5105d0bedbd9b1cb0f15917b22819e36", date: "2026-03-10 01:42:09 UTC", description: "use cross-strip tools when packaging RPMs for non-x86_64 targets", pr_number: 24873, scopes: ["ci"], type: "fix", breaking_change: false, author: "Thomas", files_count: 2, insertions_count: 29, deletions_count: 4}, + {sha: "0d462e06d4fd1088cd7054cd9ee7c49f91fe9b65", date: "2026-03-10 19:46:12 UTC", description: "increase timeouts for vdev compilation", pr_number: 24881, scopes: ["ci"], type: "fix", breaking_change: false, author: "Thomas", files_count: 2, insertions_count: 4, deletions_count: 4}, + {sha: "e66270f937998848a439252d46a160ad085b17c6", date: "2026-03-10 19:52:15 UTC", description: "add retries to address choco intermittent failures", pr_number: 24880, scopes: ["ci"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 26, deletions_count: 5}, + {sha: "145d333fc0fc8a53a087c64c56775ff55612d4a8", date: "2026-03-10 20:36:38 UTC", description: "remove unused compilation timings workflow", pr_number: 24882, scopes: ["ci"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 0, deletions_count: 77}, + {sha: "685c31fff163575019ce315b4b88991b92e3a61f", date: "2026-03-10 18:09:30 UTC", description: "pin actions to sha", pr_number: 24884, scopes: ["ci"], type: "chore", breaking_change: false, author: "StepSecurity Bot", files_count: 1, insertions_count: 1, deletions_count: 1}, + {sha: "40143b32740e253be37b214cf9f860851f65e6e0", date: "2026-03-10 18:31:15 UTC", description: "pin image tags in Dockerfiles", pr_number: 24885, scopes: ["ci"], type: "chore", breaking_change: false, author: "StepSecurity Bot", files_count: 7, insertions_count: 11, deletions_count: 11}, + {sha: "b1c7d7bf7133c43e472d15b4e4345e571043d63a", date: "2026-03-10 22:59:33 UTC", description: "fix broken links", pr_number: 24886, scopes: ["website"], type: "chore", breaking_change: false, author: "Thomas", files_count: 15, insertions_count: 18, deletions_count: 18}, + {sha: "1cc1c25147bab5ead963ff8065086d3b5daa1b2f", date: "2026-03-11 00:13:05 UTC", description: "minor fixes to release template", pr_number: 24887, scopes: ["releasing"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 13, deletions_count: 13}, + {sha: "e1ecad3d67361dd38f074e71da45d4d90b3578bb", date: "2026-03-11 00:37:56 UTC", description: "add docs::warnings macro support and warn about no auth", pr_number: 24866, scopes: ["website"], type: "feat", breaking_change: false, author: "Thomas", files_count: 3, insertions_count: 15, deletions_count: 1}, + {sha: "909b08318dd309c8a59e1c40241be42f46e67db8", date: "2026-03-11 00:53:15 UTC", description: "0.54.0 post release steps", pr_number: 24888, scopes: ["releasing"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 51, insertions_count: 512, deletions_count: 144}, + {sha: "df396d18b6101f1af5a75bbc918accb9291a7a74", date: "2026-03-11 18:01:29 UTC", description: "minor fixes to release changelog", pr_number: 24893, scopes: ["external docs"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 2, deletions_count: 4}, + {sha: "356409dba6cc78cdaa0d070941efd9c0d52b1579", date: "2026-03-11 23:53:27 UTC", description: "test_udp_syslog can overflow default size receive buffer", pr_number: 24878, scopes: ["unit tests"], type: "fix", breaking_change: false, author: "strophy", files_count: 1, insertions_count: 1, deletions_count: 1}, + {sha: "901a571712d7714f5ad38defb60b01dbabe43617", date: "2026-03-11 19:11:14 UTC", description: "show top level api configuration in API reference page", pr_number: 24894, scopes: ["website"], type: "chore", breaking_change: false, author: "Thomas", files_count: 2, insertions_count: 1, deletions_count: 3}, + {sha: "202b65612ee4cc27a338fc16a94d880c01b8ace6", date: "2026-03-11 19:34:07 UTC", description: "fix docker warning when publishing", pr_number: 24895, scopes: ["ci"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 0, deletions_count: 1}, + {sha: "2958f982b221060b68dcb63837823837d4c0600a", date: "2026-03-11 22:57:01 UTC", description: "fetch full history in changelog workflow so origin/master is available", pr_number: 24898, scopes: ["ci"], type: "fix", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 2, deletions_count: 0}, + {sha: "d112a1b6826d0a35ff27be40d949d09ee66b6a7e", date: "2026-03-11 23:17:02 UTC", description: "Support per-signal OTLP decoding config", pr_number: 24822, scopes: ["opentelemetry source"], type: "feat", breaking_change: false, author: "Pavlos Rontidis", files_count: 5, insertions_count: 472, deletions_count: 24}, + {sha: "ff7164a43992728f3490e38dbf72e5125b162f50", date: "2026-03-12 00:55:04 UTC", description: "bump protoc version", pr_number: 24902, scopes: ["ci"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 1, deletions_count: 1}, + {sha: "71d597f894f59ff6190913b1091a1c568fedba63", date: "2026-03-12 17:15:23 UTC", description: "remove unused Hugo shortcodes", pr_number: 24903, scopes: ["website"], type: "chore", breaking_change: false, author: "Thomas", files_count: 8, insertions_count: 0, deletions_count: 296}, + {sha: "40f94ee88a453a6a900a00606fc940d990c5143e", date: "2026-03-12 19:20:22 UTC", description: "bump VRL and resolve RUSTSEC-2021-0139", pr_number: 24908, scopes: ["deps"], type: "chore", breaking_change: false, author: "Thomas", files_count: 11, insertions_count: 59, deletions_count: 94}, + {sha: "478d7c4561a1a8b2390e97973ec9315963dd0dfb", date: "2026-03-13 10:54:17 UTC", description: "bump kube from 0.93.0 to 3.0.1 and k8s-openapi from 0.22.0 to 0.27.0", pr_number: 24787, scopes: ["deps"], type: "chore", breaking_change: false, author: "Haitao Li", files_count: 8, insertions_count: 146, deletions_count: 140}, + {sha: "958ff6357a6ad700d592aabf53c6b161f5fbb9a0", date: "2026-03-12 21:54:25 UTC", description: "replace netlink-* crates with procfs", pr_number: 24765, scopes: ["deps"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 4, insertions_count: 148, deletions_count: 369}, + {sha: "d141e60bfcc923a6d2df34c16f49db5fc0734e3c", date: "2026-03-12 23:40:12 UTC", description: "deprecate azure_monitor_logs sink", pr_number: 24910, scopes: ["external docs"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 1, deletions_count: 0}, + {sha: "01c86e826a83175002b609b9a404325931f11cc1", date: "2026-03-13 21:24:47 UTC", description: "bump async-executor to 1.14.0", pr_number: 24919, scopes: ["deps"], type: "chore", breaking_change: false, author: "Thomas", files_count: 2, insertions_count: 8, deletions_count: 5}, + {sha: "891643edcb177b9cc1e8db8fb753b36ad07f9182", date: "2026-03-14 10:54:10 UTC", description: "Add Windows Event Log source", pr_number: 24305, scopes: ["new source"], type: "feat", breaking_change: false, author: "tot19", files_count: 26, insertions_count: 10298, deletions_count: 14}, + {sha: "4e958028e836a62080d70ac1099d7db635d5b739", date: "2026-03-14 02:38:46 UTC", description: "implement standard gRPC health checking protocol", pr_number: 24916, scopes: ["vector source"], type: "fix", breaking_change: false, author: "Jonathan Davies", files_count: 6, insertions_count: 139, deletions_count: 7}, + {sha: "2064a15aa1029aec49dd2b4c8b3a09a4b76ff9e7", date: "2026-03-13 22:55:11 UTC", description: "update dd-rust-license-tool from 1.0.5 to 1.0.6", pr_number: 24920, scopes: ["deps"], type: "chore", breaking_change: false, author: "Thomas", files_count: 2, insertions_count: 6, deletions_count: 1}, + {sha: "60cd0e1347d4ae680da806401a6c89be1a2c0536", date: "2026-03-14 00:09:55 UTC", description: "Bump rmp from 0.8.14 to 0.8.15 and resolve RUSTSEC-2024-0436", pr_number: 24922, scopes: ["deps"], type: "chore", breaking_change: false, author: "Thomas", files_count: 3, insertions_count: 3, deletions_count: 13}, + {sha: "da85874bd56ee88699778fb3ed3f4f847915581c", date: "2026-03-14 00:46:55 UTC", description: "bump aws crates and resolve RUSTSEC-2026-0002", pr_number: 24909, scopes: ["deps"], type: "chore", breaking_change: false, author: "Thomas", files_count: 5, insertions_count: 277, deletions_count: 178}, + {sha: "59f53e2bc72caae239970b355b4edab14176cd17", date: "2026-03-14 01:11:33 UTC", description: "fix typos in rustdoc comments", pr_number: 24923, scopes: ["external docs"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 4, insertions_count: 4, deletions_count: 4}, + {sha: "89bc15fc1815572000d1fd2e9e20c4b98116d1b2", date: "2026-03-14 05:17:33 UTC", description: "Bump lapin from 2.5.3 to 4.3.0", pr_number: 23316, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 8, insertions_count: 183, deletions_count: 156}, + {sha: "51aebc8445fece5e0e7daf62603bf004b9dfb8ce", date: "2026-03-16 22:18:15 UTC", description: "bump undici from 7.18.2 to 7.24.1 in /website", pr_number: 24925, scopes: ["website deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 1, insertions_count: 3, deletions_count: 3}, + {sha: "e3225205fa88d04c4d76d0e485f27dc771900693", date: "2026-03-17 06:31:01 UTC", description: "add UUID type support for ArrowStream", pr_number: 24856, scopes: ["clickhouse sink"], type: "enhancement", breaking_change: false, author: "Benjamin Dornel", files_count: 4, insertions_count: 43, deletions_count: 4}, + {sha: "7390fe0bd4922b6f804b7c59e56dac36747af86d", date: "2026-03-16 19:08:19 UTC", description: "document Cargo feature placement rules", pr_number: 24933, scopes: ["external docs"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 25, deletions_count: 0}, + {sha: "c27f6d6dd8acc01f57e93c53af4fad0138a07aa1", date: "2026-03-16 20:05:44 UTC", description: "update heim fork to remove unmaintained crates", pr_number: 24924, scopes: ["deps"], type: "chore", breaking_change: false, author: "Thomas", files_count: 4, insertions_count: 71, deletions_count: 214}, + {sha: "0d8bb9bea3418bd6993e08b456fc495050a42930", date: "2026-03-17 01:07:42 UTC", description: "bump pulsar from 6.3.1 to 6.7.0", pr_number: 24746, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 4, insertions_count: 24, deletions_count: 23}, + {sha: "fd766a0738802e9dee2eae1e5d6f243ed9c69229", date: "2026-03-16 21:10:58 UTC", description: "make rdkafka use gssapi-vendored when publishing/testing", pr_number: 24912, scopes: ["deps"], type: "chore", breaking_change: false, author: "Thomas", files_count: 4, insertions_count: 18, deletions_count: 10}, + {sha: "d38425ad1302489d7420a9d24447224559a5a446", date: "2026-03-17 01:16:13 UTC", description: "bump regex from 1.11.2 to 1.12.3", pr_number: 24796, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 4, insertions_count: 13, deletions_count: 13}, + {sha: "c2fc606306e10e02263d2982f20c8c17f69a143b", date: "2026-03-16 22:05:29 UTC", description: "check Cargo.lock for changes after cargo check/clippy", pr_number: 24935, scopes: ["vdev"], type: "feat", breaking_change: false, author: "Thomas", files_count: 6, insertions_count: 57, deletions_count: 12}, + {sha: "0573297e8ffcc1ad7a096f50f012b2efd0518c27", date: "2026-03-17 03:54:36 UTC", description: "Added feature flag for websockets in async-nats", pr_number: 24291, scopes: ["nats source", "nats sink"], type: "feat", breaking_change: false, author: "Jesper", files_count: 3, insertions_count: 6, deletions_count: 1}, + {sha: "3905b4f12067fb2508f0bcb031607da84623f17f", date: "2026-03-16 23:07:53 UTC", description: "Update README.md and cleanup unused commands", pr_number: 24936, scopes: ["vdev"], type: "chore", breaking_change: false, author: "Thomas", files_count: 7, insertions_count: 43, deletions_count: 95}, + {sha: "1eca246d5bf297afcc153b555c19ac5f9e9ff372", date: "2026-03-17 00:49:29 UTC", description: "replace derivative macro with standard Default derives on enums", pr_number: 24938, scopes: ["dev"], type: "chore", breaking_change: false, author: "Thomas", files_count: 31, insertions_count: 74, deletions_count: 119}, + {sha: "a8730894c366cd5232353530fef4a4970cd6c307", date: "2026-03-17 19:00:09 UTC", description: "bump lz4_flex from 0.11.5 to 0.11.6", pr_number: 24939, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 1, insertions_count: 2, deletions_count: 2}, + {sha: "9dc3d3e9f7d486ce363975f21b1e1b3b537cb46d", date: "2026-03-17 19:11:36 UTC", description: "reduce memory when not in `Diff` mode", pr_number: 24943, scopes: ["aggregate transform"], type: "fix", breaking_change: false, author: "Thomas", files_count: 2, insertions_count: 89, deletions_count: 24}, + {sha: "42b844e76882fa5cbf1fd5cfafcf653101dc9ad0", date: "2026-03-17 19:13:06 UTC", description: "remove unused fs package", pr_number: 24945, scopes: ["website deps"], type: "chore", breaking_change: false, author: "Thomas", files_count: 2, insertions_count: 0, deletions_count: 6}, + {sha: "218aa4d29b7b0922f604b7f6d8a3ebd0cc8e0931", date: "2026-03-17 20:01:18 UTC", description: "Add osv-scanner.toml and ignore MAL-2025-3174", pr_number: 24946, scopes: ["website"], type: "chore", breaking_change: false, author: "Thomas", files_count: 1, insertions_count: 3, deletions_count: 0}, + {sha: "83bcf4cb297f85525f88a3cff1870ab8fbddd47b", date: "2026-03-17 20:23:49 UTC", description: "bump quinn-proto from 0.11.9 to 0.11.14", pr_number: 24926, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 2, insertions_count: 12, deletions_count: 4}, + {sha: "dc9d8de0c8f2ff7a04a7373820f1ef652ed30520", date: "2026-03-18 03:29:20 UTC", description: "switch to v2 endpoint", pr_number: 24842, scopes: ["datadog_metrics sink"], type: "chore", breaking_change: false, author: "Vladimir Zhuk", files_count: 32, insertions_count: 382, deletions_count: 98}, + {sha: "1444b2bac009aa51343820dba7a36ce119dce56f", date: "2026-03-17 22:44:09 UTC", description: "Bump rdkafka from 0.38.0 to 0.39.0", pr_number: 24602, scopes: ["deps"], type: "chore", breaking_change: false, author: "zapdos26", files_count: 4, insertions_count: 22, deletions_count: 5}, + {sha: "089db6775500eb6cbdfa5f4b7e6069185a8d61c5", date: "2026-03-18 11:45:28 UTC", description: "add `network` CIDR field to lookup results", pr_number: 24576, scopes: ["enrichment tables"], type: "feat", breaking_change: false, author: "Naoki Aoyama", files_count: 5, insertions_count: 40, deletions_count: 8}, + {sha: "5011c5dfd20a0200c37a42a6c3001ae3df94ef4d", date: "2026-03-17 23:16:34 UTC", description: "bump docker/build-push-action from 6.18.0 to 6.19.2", pr_number: 24734, scopes: ["ci"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 2, insertions_count: 4, deletions_count: 4}, + {sha: "6527346294203cec7fb0d8b86c7fc98b26c14ff0", date: "2026-03-18 03:17:07 UTC", description: "bump aws-actions/configure-aws-credentials from 5.1.1 to 6.0.0", pr_number: 24733, scopes: ["ci"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 1, insertions_count: 6, deletions_count: 6}, + {sha: "b1a43a4ba24b0e76e57736953a0f6f9d0ee58328", date: "2026-03-18 04:52:26 UTC", description: "bump sysinfo from 0.37.2 to 0.38.2", pr_number: 24816, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 2, insertions_count: 61, deletions_count: 39}, + {sha: "cfee4fb9d1853a5ae7db85e0bd5647ca7a7b57b0", date: "2026-03-18 00:54:19 UTC", description: "upgrade mold linker from 1.2.1/2.0.0 to 2.40.4", pr_number: 24947, scopes: ["deps"], type: "chore", breaking_change: false, author: "Thomas", files_count: 3, insertions_count: 3, deletions_count: 3}, + {sha: "fbb1e4b501d6081d72f864da891a98464e60d097", date: "2026-03-18 02:05:08 UTC", description: "Improve buffer utilization metric tracking", pr_number: 24911, scopes: ["buffers"], type: "fix", breaking_change: false, author: "Bruce Guenter", files_count: 2, insertions_count: 66, deletions_count: 95}, + {sha: "95f439420ae1dea0df7fbd295e7f354b144360c8", date: "2026-03-18 18:25:48 UTC", description: "place per-output sent events in Events Out column", pr_number: 24951, scopes: ["api"], type: "fix", breaking_change: false, author: "Pavlos Rontidis", files_count: 3, insertions_count: 8, deletions_count: 3}, + {sha: "30936678f92a03c827f9fd8506fa02693332f59b", date: "2026-03-19 03:34:43 UTC", description: "add support for edge_attributes in graph configuration", pr_number: 24593, scopes: ["cli"], type: "feat", breaking_change: false, author: "Ensar Sarajčić", files_count: 7, insertions_count: 505, deletions_count: 125}, + {sha: "a8f3a9b45e6d2f6a897995bce85e3ad9a9a13442", date: "2026-03-19 03:39:25 UTC", description: "expose message size when parsing dnstap data", pr_number: 24552, scopes: ["dnstap source"], type: "enhancement", breaking_change: false, author: "Ensar Sarajčić", files_count: 4, insertions_count: 51, deletions_count: 0}, + {sha: "f34e8d32140ab25024ff51863ed9b5038defdc80", date: "2026-03-19 17:13:57 UTC", description: "regenerate VRL docs with message size fields", pr_number: 24960, scopes: ["dnstap source"], type: "docs", breaking_change: false, author: "Thomas", files_count: 1, insertions_count: 2, deletions_count: 0}, + {sha: "483db702e3745f4b823edac33e1d76bf7060f964", date: "2026-03-19 22:16:35 UTC", description: "add libsasl2-2 runtime dep to Debian-based images", pr_number: 24962, scopes: ["releasing"], type: "fix", breaking_change: false, author: "Vladimir Zhuk", files_count: 2, insertions_count: 2, deletions_count: 2}, + {sha: "5f0f03e181a031ee51a71e2bd9fcf0698f073a5b", date: "2026-03-19 23:13:14 UTC", description: "bump fakedata_generator from 0.5.0 to 0.7.1", pr_number: 24800, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 3, insertions_count: 5, deletions_count: 5}, + {sha: "3c69a8b8a668c6d20ea57d6883a3771803893648", date: "2026-03-19 19:43:24 UTC", description: "instruct agents to use PR template when creating PRs", pr_number: 24965, scopes: ["external docs"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 4, deletions_count: 0}, + {sha: "f02424389cfa4abf8e1168b5b63fb227499c9bd4", date: "2026-03-19 19:54:48 UTC", description: "add Datadog static analysis workflow", pr_number: 24966, scopes: ["ci"], type: "feat", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 26, deletions_count: 0}, + {sha: "a7fae1f7ad17b51e3960850300b45955ac0952e0", date: "2026-03-19 20:27:12 UTC", description: "add code coverage workflow with Datadog upload", pr_number: 24964, scopes: ["ci"], type: "feat", breaking_change: false, author: "Pavlos Rontidis", files_count: 3, insertions_count: 54, deletions_count: 2}, + {sha: "37a48ae53eb0e5930f4f4506903eb5b10c613789", date: "2026-03-20 05:44:04 UTC", description: "bump cargo-lock from 10.1.0 to 11.0.1", pr_number: 24748, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 2, insertions_count: 11, deletions_count: 37}, + {sha: "97dfb6c6a870a162d07827706e85f5ee2fe3b3a9", date: "2026-03-20 17:53:12 UTC", description: "bump const-str from 1.0.0 to 1.1.0", pr_number: 24815, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 2, insertions_count: 3, deletions_count: 3}, + {sha: "18898e0149f43924a07554303cce3d0a4595eb5b", date: "2026-03-20 21:22:10 UTC", description: "bump quickcheck to 1.1.0 and use workspace dep", pr_number: 24970, scopes: ["deps"], type: "chore", breaking_change: false, author: "Thomas", files_count: 8, insertions_count: 218, deletions_count: 31}, + {sha: "2097fc35d7dc8d3caee9607d563165374e312e21", date: "2026-03-21 03:12:17 UTC", description: "bump criterion from 0.7.0 to 0.8.2", pr_number: 24803, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 6, insertions_count: 31, deletions_count: 9}, + {sha: "2c334d223578ea17d7ca5bbbf305f21c9b68a7e2", date: "2026-03-21 05:07:16 UTC", description: "Add support for traces in kafka sink", pr_number: 24639, scopes: ["kafka sink"], type: "enhancement", breaking_change: false, author: "Piotr Stalmach", files_count: 4, insertions_count: 160, deletions_count: 12}, + {sha: "c748d6f9e19632cb64f666058794aa2d9004926f", date: "2026-03-21 02:28:58 UTC", description: "use cargo nextest in coverage workflow to prevent test pollution", pr_number: 24973, scopes: ["ci"], type: "fix", breaking_change: false, author: "Thomas", files_count: 1, insertions_count: 4, deletions_count: 1}, + {sha: "4d8ca64dbbbc2434bd4a69a3d46705c3fad6a56c", date: "2026-03-23 21:53:30 UTC", description: "pin localstack image to SHA256 digest", pr_number: 24988, scopes: ["dev"], type: "chore", breaking_change: false, author: "Thomas", files_count: 1, insertions_count: 1, deletions_count: 1}, + {sha: "6ef3752d12be1e8de38449df30f953f75301f64f", date: "2026-03-23 20:52:03 UTC", description: "remove futures features and resolve RUSTSEC-2026-0058", pr_number: 24975, scopes: ["deps"], type: "chore", breaking_change: false, author: "Thomas", files_count: 3, insertions_count: 124, deletions_count: 166}, + {sha: "415059f727929adee4d88801ca760be2ac0b967a", date: "2026-03-23 23:19:24 UTC", description: "bump async-nats from 0.42.0 to 0.46.0", pr_number: 24974, scopes: ["deps"], type: "chore", breaking_change: false, author: "Thomas", files_count: 3, insertions_count: 6, deletions_count: 16}, + {sha: "05f7b4338b06ef7c8731cd8f391ca4e62f875b43", date: "2026-03-23 20:45:40 UTC", description: "add missing Windows source files to integration_windows filter", pr_number: 24992, scopes: ["ci"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 3, deletions_count: 0}, + {sha: "f62a9ffa8729fb1e7e6d074d4d115c96f7f2740a", date: "2026-03-23 21:25:16 UTC", description: "ignore RUSTSEC-2026-0049 (rustls-webpki) until rustls can be upgraded", pr_number: 24986, scopes: ["deps"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 3, deletions_count: 0}, + {sha: "aedd2a66cb27b26621fff2c6d7dbd53a199fe0fa", date: "2026-03-23 21:26:02 UTC", description: "add security warning to path template field", pr_number: 24983, scopes: ["file sink"], type: "docs", breaking_change: false, author: "Pavlos Rontidis", files_count: 2, insertions_count: 4, deletions_count: 0}, + {sha: "93a9771b1824c848c8efdf7166638226f01f42d2", date: "2026-03-24 14:22:00 UTC", description: "replace GraphQL observability API with gRPC, remove async-graphql dependencies", pr_number: 24364, scopes: ["api"], type: "chore", breaking_change: true, author: "Pavlos Rontidis", files_count: 128, insertions_count: 3020, deletions_count: 16042}, + {sha: "f1d8a93402168a2c5daf1eab2f4584aa5762bac4", date: "2026-03-25 00:52:15 UTC", description: "correct stop/start state checks and add configurable stop timeout", pr_number: 24772, scopes: ["windows platform"], type: "fix", breaking_change: false, author: "iMithrellas", files_count: 3, insertions_count: 100, deletions_count: 32}, + {sha: "0122cf6ec8fe7cce20a2bb694db137e3d5a9038a", date: "2026-03-24 20:04:14 UTC", description: "remove deprecated `headers` option from http and opentelemetry sinks", pr_number: 24994, scopes: ["sinks"], type: "chore", breaking_change: true, author: "Thomas", files_count: 9, insertions_count: 5, deletions_count: 43}, + {sha: "8bac1dbdcd27550444ea3b73e9dd4ef36172b2d4", date: "2026-03-24 23:00:40 UTC", description: "switch series v2 and sketches to zstd compression", pr_number: 24956, scopes: ["datadog_metrics sink"], type: "chore", breaking_change: false, author: "Vladimir Zhuk", files_count: 8, insertions_count: 629, deletions_count: 277}, + {sha: "753cf9a7c03a5cef85a29f3561666d94e9084d8b", date: "2026-03-24 23:57:50 UTC", description: "consolidate reference documentation into detailed docs table in AGENTS.md", pr_number: 25036, scopes: ["internal docs"], type: "docs", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 15, deletions_count: 22}, + {sha: "47b5b02b9fdd13b67294df65175e9eca99c4cd8b", date: "2026-03-25 00:36:30 UTC", description: "split deny check into optional (all) and required (licenses)", pr_number: 24990, scopes: ["ci"], type: "chore", breaking_change: false, author: "Thomas", files_count: 3, insertions_count: 38, deletions_count: 6}, + {sha: "ca9b29bc52b7c7ba857c337b25278edbad1c5e66", date: "2026-03-25 21:17:07 UTC", description: "bump serial_test from 3.2.0 to 3.4.0", pr_number: 25022, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 2, insertions_count: 7, deletions_count: 6}, + {sha: "a2918e6ac4b874dc7c67b558a1e2bc9fd2b4ae81", date: "2026-03-25 21:58:38 UTC", description: "bump quick-junit from 0.5.2 to 0.6.0", pr_number: 25032, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 2, insertions_count: 3, deletions_count: 3}, + {sha: "209b252deed214144fd2a6d4ed6c5ea3a3e08821", date: "2026-03-25 23:11:04 UTC", description: "enforce and fix markdownlint rules", pr_number: 25038, scopes: ["dev"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 82, insertions_count: 415, deletions_count: 407}, + {sha: "7dc3ce3f34de6b79c0f09b4e44fe0e9d87cc09c2", date: "2026-03-26 02:32:34 UTC", description: "add links to design issues", pr_number: 25046, scopes: ["dev"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 2, insertions_count: 3, deletions_count: 0}, + {sha: "4fba57e9c4d6561c126851df8c4065221b44e992", date: "2026-03-26 07:51:02 UTC", description: "allow to add headers to metadata for OpenTelemetry metrics and traces", pr_number: 24942, scopes: ["opentelemetry source"], type: "feat", breaking_change: false, author: "Oleksandr Zanichkovskyi", files_count: 6, insertions_count: 463, deletions_count: 47}, + {sha: "498d1daa38432a329d168a2eec97a212922e71bc", date: "2026-03-26 04:51:47 UTC", description: "improve CLA instructions", pr_number: 25039, scopes: ["ci"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 12, deletions_count: 13}, + {sha: "bc1e6bdde6b07f51efd4764c65d75c32d056499c", date: "2026-03-26 23:10:51 UTC", description: "bump uuid from 1.18.1 to 1.22.0", pr_number: 25025, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 3, insertions_count: 37, deletions_count: 15}, + {sha: "072f6284c83279cd920177b32114f10f8fd5164c", date: "2026-03-26 23:15:29 UTC", description: "bump snafu from 0.8.9 to 0.9.0", pr_number: 25029, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 2, insertions_count: 34, deletions_count: 13}, + {sha: "041769359ba0a9d68017e10e6e84cb7a83b3df27", date: "2026-03-26 23:22:32 UTC", description: "bump picomatch from 2.3.1 to 2.3.2 in /website", pr_number: 25047, scopes: ["website deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 1, insertions_count: 3, deletions_count: 3}, + {sha: "d0c6cea89661577079d3eec790e4c054772de7d6", date: "2026-03-26 23:25:49 UTC", description: "bump yaml from 1.10.2 to 1.10.3 in /website", pr_number: 25043, scopes: ["website deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 1, insertions_count: 3, deletions_count: 3}, + {sha: "5fcf1516151f659b5832fc433a4e10a219e31c7b", date: "2026-03-26 23:26:25 UTC", description: "bump ipnet from 2.11.0 to 2.12.0", pr_number: 25030, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 1, insertions_count: 2, deletions_count: 2}, + {sha: "09629f02d60614a38b20b49cb10acbce47a9f6cc", date: "2026-03-26 23:49:33 UTC", description: "bump docker/setup-buildx-action from 3.12.0 to 4.0.0", pr_number: 25003, scopes: ["ci"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 3, insertions_count: 4, deletions_count: 4}, + {sha: "3255b66b2e8a2fb7ec3412904b856a8349a236ab", date: "2026-03-26 23:49:57 UTC", description: "bump actions/cache from 5.0.3 to 5.0.4", pr_number: 25002, scopes: ["ci"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 1, insertions_count: 1, deletions_count: 1}, + {sha: "9337c80bb9d2491a8d617df408c20a88385d719a", date: "2026-03-26 23:50:14 UTC", description: "bump docker/login-action from 3.7.0 to 4.0.0", pr_number: 25001, scopes: ["ci"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 4, insertions_count: 6, deletions_count: 6}, + {sha: "84abc68df859d58b2a7681757e52aca7a8094ad2", date: "2026-03-27 04:24:22 UTC", description: "bump bollard from 0.19.2 to 0.20.2", pr_number: 25026, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 4, insertions_count: 9, deletions_count: 13}, + {sha: "6267fcc78d8cd347f6405ab03ca100b97944a5ef", date: "2026-03-27 06:31:32 UTC", description: "log error", pr_number: 24708, scopes: ["opentelemetry source"], type: "fix", breaking_change: false, author: "bas smit", files_count: 2, insertions_count: 10, deletions_count: 2}, + {sha: "e8526b2a3ef7ac257d0fb816f611cddbc494a478", date: "2026-03-27 06:32:43 UTC", description: "bump deadpool from 0.12.2 to 0.13.0", pr_number: 25014, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 3, insertions_count: 23, deletions_count: 6}, + {sha: "af15a5a595f342038fd3368854fa75c74c31e932", date: "2026-03-27 05:09:12 UTC", description: "vanila `cargo build` should run on windows", pr_number: 24991, scopes: ["dev"], type: "fix", breaking_change: false, author: "Pavlos Rontidis", files_count: 4, insertions_count: 13, deletions_count: 7}, + {sha: "4e6b49d6951b1f6ed9e2851aec674577ccc4c2b2", date: "2026-03-27 20:47:55 UTC", description: "deduct downstream utilization on task transforms", pr_number: 24731, scopes: ["metrics"], type: "fix", breaking_change: false, author: "Yoenn Burban", files_count: 3, insertions_count: 221, deletions_count: 18}, + {sha: "a7450cdbba3eb86782d67bea39b81e5b641067d9", date: "2026-03-27 18:50:50 UTC", description: "bump tempfile from 3.23.0 to 3.27.0", pr_number: 25016, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 2, insertions_count: 14, deletions_count: 14}, + {sha: "7066adccecc24fe005f1fb7c934bbda4b557ea6c", date: "2026-03-27 18:51:39 UTC", description: "bump proptest from 1.10.0 to 1.11.0", pr_number: 25021, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 5, insertions_count: 7, deletions_count: 7}, + {sha: "3ca3374c5b0359a3839f6c8091e2c3632faec719", date: "2026-03-27 23:02:57 UTC", description: "bump rust_decimal from 1.39.0 to 1.40.0", pr_number: 24799, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 2, insertions_count: 3, deletions_count: 3}, + {sha: "c1f34ef803fe5365a5be54f46877590bc8c91ba6", date: "2026-03-28 00:44:05 UTC", description: "bump brace-expansion from 1.1.12 to 1.1.13 in /website", pr_number: 25056, scopes: ["website deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 1, insertions_count: 3, deletions_count: 3}, + {sha: "d29a524ea05dcbb6f823121ffdff449ecd960673", date: "2026-03-28 00:49:17 UTC", description: "bump security-framework from 3.5.1 to 3.6.0", pr_number: 24763, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 2, insertions_count: 7, deletions_count: 7}, + {sha: "89e5c343c631e36090d2543106b88685d7a1c908", date: "2026-03-28 00:55:57 UTC", description: "bump juliangruber/read-file-action from 1.1.7 to 1.1.8", pr_number: 25000, scopes: ["ci"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 1, insertions_count: 1, deletions_count: 1}, + {sha: "55be10e5860afd47656a67a08335bb17a1d641c8", date: "2026-03-30 20:14:44 UTC", description: "add missing `contents: read` permission to integration review jobs", pr_number: 25067, scopes: ["ci"], type: "fix", breaking_change: false, author: "Thomas", files_count: 1, insertions_count: 3, deletions_count: 0}, + {sha: "d906ba19487c0552be7557d49a3f1dbfcd2405d9", date: "2026-03-31 01:12:33 UTC", description: "upgrade rustls from 0.23.23 to 0.23.37", pr_number: 25075, scopes: ["deps"], type: "chore", breaking_change: false, author: "Thomas", files_count: 1, insertions_count: 25, deletions_count: 13}, + {sha: "6a50bd50c9d482511f5a2a21466a3427dd752263", date: "2026-03-31 01:27:02 UTC", description: "update security policy structure and content", pr_number: 25074, scopes: ["internal docs"], type: "chore", breaking_change: false, author: "Thomas", files_count: 1, insertions_count: 43, deletions_count: 59}, + {sha: "122cca4678172bbc0c067c65fcc909c6b7aa9ce6", date: "2026-03-31 01:51:55 UTC", description: "update GHCR cleanup workflow", pr_number: 25037, scopes: ["ci"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 18, deletions_count: 14}, + {sha: "87ed519cc40011224208914f1dd79b7240f7e1d7", date: "2026-03-31 17:36:52 UTC", description: "consolidate advisory ignores with issue links", pr_number: 25076, scopes: ["dev"], type: "chore", breaking_change: false, author: "Thomas", files_count: 1, insertions_count: 4, deletions_count: 12}, + {sha: "84c7af2264deacb1afc0529671d53c8c6792bf88", date: "2026-03-31 17:41:49 UTC", description: "install vdev via setup action in ci-integration-review workflow", pr_number: 25077, scopes: ["ci"], type: "fix", breaking_change: false, author: "Thomas", files_count: 1, insertions_count: 10, deletions_count: 4}, + {sha: "d732259aaa6b5e156b001672ccf0e59be5e84802", date: "2026-03-31 18:08:30 UTC", description: "pin GitHub actions to full sha", pr_number: 25080, scopes: ["ci"], type: "chore", breaking_change: false, author: "StepSecurity Bot", files_count: 1, insertions_count: 1, deletions_count: 1}, + {sha: "0943dd938f247b0a1963e6bcb1a464ae49392c5f", date: "2026-03-31 22:54:02 UTC", description: "migrate from markdownlint-cli to markdownlint-cli2", pr_number: 25081, scopes: ["ci"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 6, insertions_count: 13, deletions_count: 19}, + {sha: "50345d5929ecb45d74e1355e612f044c24ff4a93", date: "2026-03-31 22:27:15 UTC", description: "skip test runner pull, datadog-ci install, and checkout when int tests won't run", pr_number: 25068, scopes: ["ci"], type: "chore", breaking_change: false, author: "Thomas", files_count: 2, insertions_count: 80, deletions_count: 74}, + {sha: "24d8db42e01e28b590e5484fede9279908805901", date: "2026-04-01 02:04:57 UTC", description: "remove unused publish-homebrew workflow", pr_number: 25085, scopes: ["ci"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 0, deletions_count: 36}, + {sha: "df00c7eec91ecb35eb0a8db0249701562de7dc87", date: "2026-04-01 23:37:50 UTC", description: "pin Pulsar integration test image to 4.1.3", pr_number: 25097, scopes: ["ci"], type: "fix", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 3, deletions_count: 1}, + {sha: "639c924c733442bcc7443817d16ea0dd737ded54", date: "2026-04-02 04:24:01 UTC", description: "bump debian from `1d3c811` to `26f98cc` in /distribution/docker/debian in the docker-images group across 1 directory", pr_number: 24996, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 1, insertions_count: 2, deletions_count: 2}, + {sha: "9dbdb0662b668867ac8db6e444ac710d4346a037", date: "2026-04-02 04:25:08 UTC", description: "bump distroless/static from `28efbe9` to `47b2d72` in /distribution/docker/distroless-static in the docker-images group across 1 directory", pr_number: 24998, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 1, insertions_count: 1, deletions_count: 1}, + {sha: "3c879836d586695701e9cf60fed3e7ef4b408f2e", date: "2026-04-02 04:25:30 UTC", description: "bump debian from `1d3c811` to `26f98cc` in /distribution/docker/distroless-libc in the docker-images group across 1 directory", pr_number: 24997, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 1, insertions_count: 1, deletions_count: 1}, + {sha: "9f484c2b9c1c219f44c2fd5289c40e3a4b63f6e3", date: "2026-04-02 03:26:12 UTC", description: "prevent script injection in integration review workflow", pr_number: 25106, scopes: ["ci"], type: "fix", breaking_change: false, author: "Thomas", files_count: 1, insertions_count: 18, deletions_count: 2}, + {sha: "9055388bc0c89a56177f6df60de7a7e09fe74de1", date: "2026-04-02 17:48:09 UTC", description: "add repo-wide prettier config for YAML, JS, TS, and JSON", pr_number: 25082, scopes: ["dev"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 226, insertions_count: 4028, deletions_count: 2779}, + {sha: "b59c9a25b899223b6633776eeafe2e67fe418640", date: "2026-04-02 19:32:25 UTC", description: "Add http_server source to semantic PR scope list", pr_number: 25078, scopes: ["ci"], type: "chore", breaking_change: false, author: "steveduan-IDME", files_count: 1, insertions_count: 1, deletions_count: 0}, + {sha: "1a78c93520ab25377b7162c9aa5756f29492f433", date: "2026-04-02 21:57:21 UTC", description: "fix Pulsar TLS integration tests with latest image", pr_number: 25099, scopes: ["ci"], type: "fix", breaking_change: false, author: "Thomas", files_count: 2, insertions_count: 5, deletions_count: 3}, + {sha: "dedb9667511691731c30c05235c319867d96580c", date: "2026-04-03 19:19:53 UTC", description: "pin npm transitive deps via package-lock.json and npm ci", pr_number: 25115, scopes: ["ci"], type: "fix", breaking_change: false, author: "Pavlos Rontidis", files_count: 5, insertions_count: 3449, deletions_count: 24}, + {sha: "04f3b782e95f044fc7307fe4f6230dc48793800b", date: "2026-04-03 18:23:45 UTC", description: "Reverse performance regression in buffer metrics", pr_number: 24995, scopes: ["buffers"], type: "fix", breaking_change: false, author: "Bruce Guenter", files_count: 1, insertions_count: 215, deletions_count: 114}, + {sha: "51030d2aade15deae23297a1ed1994cb36c530e9", date: "2026-04-03 20:36:36 UTC", description: "make file source tests not flaky", pr_number: 24957, scopes: ["dev"], type: "fix", breaking_change: false, author: "Thomas", files_count: 1, insertions_count: 17, deletions_count: 6}, + {sha: "2fe515ccd2dfc23d98b84300e4fd87bee27b35fa", date: "2026-04-03 22:13:12 UTC", description: "use helm-charts develop branch for K8s E2E tests", pr_number: 25118, scopes: ["ci"], type: "fix", breaking_change: false, author: "Pavlos Rontidis", files_count: 7, insertions_count: 44, deletions_count: 24}, + {sha: "b428aba5dd4f26285ce907212acae648bdb6d8df", date: "2026-04-03 23:39:14 UTC", description: "resolve build.rs HEAD path in worktrees", pr_number: 25120, scopes: ["dev"], type: "fix", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 18, deletions_count: 1}, + {sha: "738c1d7758024a074f4fbe9ba9fa6d6110b316d7", date: "2026-04-04 05:14:48 UTC", description: "run K8s E2E suite on a weekly schedule", pr_number: 25119, scopes: ["ci"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 2, deletions_count: 2}, + {sha: "703590bff3c2e49aa722c493329c753b1943c4ff", date: "2026-04-06 17:25:55 UTC", description: "Migrate smp regression workflow auth from static secrets to oidc", pr_number: 25112, scopes: ["ci"], type: "chore", breaking_change: false, author: "Caleb Metz", files_count: 1, insertions_count: 27, deletions_count: 12}, + {sha: "2d6fea2dfe7536202c83305cff5021bfadc90442", date: "2026-04-07 09:29:11 UTC", description: "update check-spelling", pr_number: 25124, scopes: ["ci"], type: "chore", breaking_change: false, author: "Jed Laundry", files_count: 1, insertions_count: 1, deletions_count: 1}, + {sha: "1152614bc73e74df27986d1bff7864a376025544", date: "2026-04-06 17:32:37 UTC", description: "collect K8s diagnostics on E2E test failure", pr_number: 25114, scopes: ["ci"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 32, deletions_count: 1}, + {sha: "cab8fab16b2da5478d495cd37091b525f527d8ba", date: "2026-04-06 23:37:04 UTC", description: "bump the artifact group across 1 directory with 2 updates", pr_number: 24999, scopes: ["ci"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 6, insertions_count: 17, deletions_count: 17}, + {sha: "4b2353a35dd54b888cdee7d1ca70c8723e12873c", date: "2026-04-06 23:46:52 UTC", description: "bump nick-fields/retry from 3.0.2 to 4.0.0", pr_number: 25102, scopes: ["ci"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 5, insertions_count: 7, deletions_count: 7}, + {sha: "9e897388fbf772a23c48df93de923a05b2d3ecbe", date: "2026-04-07 04:05:11 UTC", description: "bump fast-xml-parser from 5.3.7 to 5.5.9 in /scripts/environment/npm-tools in the npm_and_yarn group across 1 directory", pr_number: 25127, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 2, insertions_count: 113, deletions_count: 81}, + {sha: "0c5fd58228bfab0dd9f6e81a1dc651da2ea8e670", date: "2026-04-07 20:14:58 UTC", description: "improve dashboard column layout", pr_number: 25135, scopes: ["api top"], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 8, deletions_count: 8}, + {sha: "02f281d6ad2786f8bbfad9ece820b8d9f4925571", date: "2026-04-07 23:38:29 UTC", description: "show \"disabled\" in Memory Used when allocation tracing is off", pr_number: 25138, scopes: ["api top"], type: "enhancement", breaking_change: false, author: "Pavlos Rontidis", files_count: 7, insertions_count: 65, deletions_count: 2}, + {sha: "256db9c12951987ea235a0c5a4a88898d5187642", date: "2026-04-08 19:32:50 UTC", description: "avoid unused-mut warning on Windows builds", pr_number: 25142, scopes: ["api top"], type: "fix", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 4, deletions_count: 3}, + {sha: "711b039fe7a8ddac2296dfe1e78685bb901795e6", date: "2026-04-08 21:30:51 UTC", description: "bump lodash from 4.17.23 to 4.18.1 in /website", pr_number: 25113, scopes: ["website deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 1, insertions_count: 3, deletions_count: 3}, + {sha: "dcf98a7ccf54080a74626e9277ac5b2f8340ded8", date: "2026-04-08 21:37:57 UTC", description: "fix spell check CI failures", pr_number: 25144, scopes: [], type: "chore", breaking_change: false, author: "Pavlos Rontidis", files_count: 3, insertions_count: 3, deletions_count: 3}, + {sha: "30d9a58669556a9bc014ba0b2ea4f33f8e372d98", date: "2026-04-09 13:43:50 UTC", description: "Expand support for Azure authentication types", pr_number: 24729, scopes: ["azure_blob sink"], type: "feat", breaking_change: false, author: "Jed Laundry", files_count: 28, insertions_count: 1542, deletions_count: 342}, + {sha: "985f669ffae07c1104ae9cec1b097e74130e0341", date: "2026-04-09 09:50:38 UTC", description: "high CPU usage after async file server migration", pr_number: 25064, scopes: ["file source"], type: "fix", breaking_change: false, author: "fcfangcc", files_count: 5, insertions_count: 212, deletions_count: 11}, + {sha: "9c4abf31a91d888d0b4ca204643b0e6e1a559466", date: "2026-04-09 05:26:19 UTC", description: "emit VectorStopped after topology drains", pr_number: 25083, scopes: ["shutdown"], type: "fix", breaking_change: false, author: "tronboto", files_count: 3, insertions_count: 32, deletions_count: 10}, + {sha: "26eef13087e1a463efa4bdcc9d3919b13936e06c", date: "2026-04-09 00:26:27 UTC", description: "preserve `device` tag from v2 series resources", pr_number: 25146, scopes: ["datadog_agent source"], type: "fix", breaking_change: false, author: "Lisa Vu", files_count: 3, insertions_count: 69, deletions_count: 0}, + {sha: "b57d8b0ef0ec12a1645b556a6b1bf080dc66e50b", date: "2026-04-09 04:53:19 UTC", description: "bump basic-ftp from 5.2.0 to 5.2.1 in /scripts/environment/npm-tools in the npm_and_yarn group across 1 directory", pr_number: 25147, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 1, insertions_count: 3, deletions_count: 3}, + {sha: "51f6fce6d850dc4c6833d910013a189119068734", date: "2026-04-09 21:35:17 UTC", description: "fix typo in 24532 changelog fragment", pr_number: 25155, scopes: ["shutdown"], type: "chore", breaking_change: false, author: "Jonathan Davies", files_count: 1, insertions_count: 1, deletions_count: 1}, + {sha: "a7f6487c94b98766bdff802aac0cd1372b0311f6", date: "2026-04-13 21:07:40 UTC", description: "bump basic-ftp from 5.2.1 to 5.2.2 in /scripts/environment/npm-tools in the npm_and_yarn group across 1 directory", pr_number: 25170, scopes: ["deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 1, insertions_count: 3, deletions_count: 3}, + {sha: "5ab0ba0d27392c70995a3d5e182a8307e65c641f", date: "2026-04-13 18:37:29 UTC", description: "bump axios from 1.13.5 to 1.15.0 in /website", pr_number: 25172, scopes: ["website deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 1, insertions_count: 8, deletions_count: 8}, + {sha: "480ca58f52a019454a36e9c9b754791224fcee57", date: "2026-04-13 19:22:13 UTC", description: "extract vector-vrl-doc-builder into unpublished crate", pr_number: 25034, scopes: ["vdev"], type: "chore", breaking_change: false, author: "Thomas", files_count: 10, insertions_count: 114, deletions_count: 58}, + {sha: "3112033384d47b0eae9fa36a1297dab32cac1f88", date: "2026-04-13 20:08:25 UTC", description: "convert TOML config examples to YAML", pr_number: 25163, scopes: ["website"], type: "docs", breaking_change: false, author: "Pavlos Rontidis", files_count: 68, insertions_count: 2096, deletions_count: 1829}, + {sha: "43f620435a353b048c40661f59a2d2d18f2419ed", date: "2026-04-13 20:24:37 UTC", description: "fix panic in allocation tracing dealloc path", pr_number: 25136, scopes: ["observability"], type: "fix", breaking_change: false, author: "Pavlos Rontidis", files_count: 2, insertions_count: 192, deletions_count: 13}, + {sha: "c6574fd66c328ef57fba27d2ad5422666c3e18eb", date: "2026-04-13 20:26:51 UTC", description: "replace custom Health RPC with standard gRPC health service", pr_number: 25139, scopes: ["api"], type: "enhancement", breaking_change: true, author: "Pavlos Rontidis", files_count: 16, insertions_count: 119, deletions_count: 77}, + {sha: "44637c3056c8850752e6f1c6aef8ab5ead810b0f", date: "2026-04-14 19:04:29 UTC", description: "bump datadog-ci from 5.12.0 to 5.13.0", pr_number: 25187, scopes: ["deps"], type: "chore", breaking_change: false, author: "Thomas", files_count: 2, insertions_count: 145, deletions_count: 2278}, + {sha: "16458a84ba60660bbdc4258e3245981102da2921", date: "2026-04-14 19:48:09 UTC", description: "bump follow-redirects from 1.15.11 to 1.16.0 in /website", pr_number: 25189, scopes: ["website deps"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 1, insertions_count: 3, deletions_count: 3}, + {sha: "e32661adb9138ff721d882f059f35eaf792d355c", date: "2026-04-15 02:53:04 UTC", description: "add Parquet encoder with schema_file and auto infer schema support", pr_number: 25156, scopes: ["aws_s3 sink"], type: "feat", breaking_change: false, author: "Peter Ehikhuemen", files_count: 21, insertions_count: 1818, deletions_count: 43}, + {sha: "51121047d3201394c2958e4a66696f3aef2d3f11", date: "2026-04-15 21:25:02 UTC", description: "bump rsa from 0.9.3 to 0.9.10", pr_number: 25198, scopes: ["deps"], type: "chore", breaking_change: false, author: "Thomas", files_count: 2, insertions_count: 10, deletions_count: 9}, + {sha: "dac7c31501defdf451f9422b6a569358fb5ec19e", date: "2026-04-15 21:53:42 UTC", description: "bump rustls-webpki 0.103.10 to 0.103.12 (RUSTSEC-2026-0099)", pr_number: 25200, scopes: ["deps"], type: "chore", breaking_change: false, author: "Thomas", files_count: 2, insertions_count: 5, deletions_count: 3}, + {sha: "6818b9c348cd4c3edd39f9f303f66362676e47a3", date: "2026-04-15 22:22:40 UTC", description: "add work-in-progress label workflow for docs PRs", pr_number: 24950, scopes: ["ci"], type: "feat", breaking_change: false, author: "Pavlos Rontidis", files_count: 3, insertions_count: 68, deletions_count: 0}, + {sha: "2d14034bf8115764c52636a59ec298d7444a4d28", date: "2026-04-15 22:52:21 UTC", description: "override smol-toml to 1.6.1 for markdownlint-cli2", pr_number: 25202, scopes: ["deps"], type: "fix", breaking_change: false, author: "Thomas", files_count: 2, insertions_count: 8, deletions_count: 3}, + {sha: "08c9f1e6b54cf1b30e88d536f3545e74ff193c0f", date: "2026-04-15 23:48:46 UTC", description: "bump VRL and add feature for enable_crypto_functions", pr_number: 25205, scopes: ["deps"], type: "chore", breaking_change: false, author: "Thomas", files_count: 2, insertions_count: 2, deletions_count: 1}, + {sha: "a1469590bed6996b1eb7001248232b0aa48cd777", date: "2026-04-16 00:06:08 UTC", description: "bump rand 0.10.0 to 0.10.1 and 0.9.2 to 0.9.4", pr_number: 25204, scopes: ["deps"], type: "chore", breaking_change: false, author: "Thomas", files_count: 2, insertions_count: 33, deletions_count: 32}, + {sha: "5ba8405bf48cd64509bf02388c7901451bbe67f4", date: "2026-04-16 00:20:05 UTC", description: "bump version to 0.3.1", pr_number: 25206, scopes: ["vdev"], type: "chore", breaking_change: false, author: "Thomas", files_count: 2, insertions_count: 2, deletions_count: 2}, + {sha: "ce651754d4e2eb134ee9ab2fe85a8a3a5ed923a0", date: "2026-04-16 17:16:53 UTC", description: "add source latency metric and fix source lag time on large batches", pr_number: 24987, scopes: ["sources"], type: "feat", breaking_change: false, author: "Yoenn Burban", files_count: 7, insertions_count: 122, deletions_count: 28}, + {sha: "8138c4593455dd284475ed4ed61fe55ee176bd34", date: "2026-04-16 19:27:11 UTC", description: "update DD RUM/Logs config to cose site", pr_number: 25213, scopes: ["website"], type: "chore", breaking_change: false, author: "Thomas", files_count: 2, insertions_count: 9, deletions_count: 2}, + {sha: "61b4563fc60211feba19dcbf9296872b8640ec94", date: "2026-04-16 20:19:11 UTC", description: "fix secrets in static analysis workflow", pr_number: 25208, scopes: ["ci"], type: "fix", breaking_change: false, author: "Thomas", files_count: 1, insertions_count: 6, deletions_count: 3}, + {sha: "467ef876a286df316ba273e4ad1779f1b08ba392", date: "2026-04-17 03:18:13 UTC", description: "preserve site prefix when computing API endpoint", pr_number: 25211, scopes: ["datadog_common sink"], type: "fix", breaking_change: false, author: "Vladimir Zhuk", files_count: 2, insertions_count: 32, deletions_count: 1}, + {sha: "9efe09d254fdad82f5d1239e3eae28348390ddfa", date: "2026-04-17 04:34:41 UTC", description: "modernize cross ubuntu bootstrap script", pr_number: 25215, scopes: ["ci"], type: "fix", breaking_change: false, author: "Thomas", files_count: 3, insertions_count: 43, deletions_count: 24}, + {sha: "8790fa07177e93242395746b8328e2cd2e88b440", date: "2026-04-17 21:32:34 UTC", description: "split VERSION-dependent packaging targets into Makefile.packaging", pr_number: 25101, scopes: ["dev"], type: "chore", breaking_change: false, author: "Thomas", files_count: 2, insertions_count: 135, deletions_count: 108}, + {sha: "35351b9cc9ae7dffdf1a461203137905f987a8b5", date: "2026-04-18 01:22:12 UTC", description: "replace native encoding fixture patch files with cfg-gated code", pr_number: 24971, scopes: ["codecs"], type: "chore", breaking_change: false, author: "Thomas", files_count: 14, insertions_count: 130, deletions_count: 278}, + {sha: "604866d560deea7a14cbfa767a8ca99eee07089a", date: "2026-04-20 22:58:40 UTC", description: "use parent site for RUM to resolve intake host", pr_number: 25224, scopes: ["website"], type: "fix", breaking_change: false, author: "Thomas", files_count: 1, insertions_count: 1, deletions_count: 1}, + {sha: "bdcf7476c55141ecf535cec6caafea7b2cc3d16d", date: "2026-04-20 23:43:03 UTC", description: "bump dorny/paths-filter from 3.0.2 to 4.0.1", pr_number: 25105, scopes: ["ci"], type: "chore", breaking_change: false, author: "dependabot[bot]", files_count: 4, insertions_count: 6, deletions_count: 6}, + {sha: "c2022541ee792988c465ca9a76b93a0f54ea4e00", date: "2026-04-20 23:43:44 UTC", description: "add greptimedb v0 support deprecation entry", pr_number: 25226, scopes: ["deprecations"], type: "docs", breaking_change: false, author: "Thomas", files_count: 1, insertions_count: 2, deletions_count: 0}, + {sha: "0f7574ad33ce07243ab8f9d6399c5e6c2f451af1", date: "2026-04-21 00:49:34 UTC", description: "fix panic in allocation tracing dealloc path", pr_number: 25222, scopes: ["observability"], type: "revert", breaking_change: false, author: "Pavlos Rontidis", files_count: 2, insertions_count: 13, deletions_count: 192}, + {sha: "3ca3e61ec8b8cd89be836b47346343069b229398", date: "2026-04-21 01:11:31 UTC", description: "use CUE raw multi-line strings in release generator", pr_number: 25228, scopes: ["ci"], type: "fix", breaking_change: false, author: "Pavlos Rontidis", files_count: 1, insertions_count: 5, deletions_count: 2}, + ] +} diff --git a/website/cue/reference/versions.cue b/website/cue/reference/versions.cue index ae51453c8f728..0ca3310065d7f 100644 --- a/website/cue/reference/versions.cue +++ b/website/cue/reference/versions.cue @@ -2,6 +2,7 @@ package metadata // This has to be maintained manually because there's currently no way to sort versions programmatically versions: [string, ...string] & [ + "0.55.0", "0.54.0", "0.53.0", "0.52.0", diff --git a/website/layouts/releases/single.html b/website/layouts/releases/single.html index fe4cb606aacdf..f83cb9e349122 100644 --- a/website/layouts/releases/single.html +++ b/website/layouts/releases/single.html @@ -31,6 +31,14 @@

{{ end }} + {{ with $highlights }} +
+ {{ range . }} + {{ .Render "li" }} + {{ end }} +
+ {{ end }} + {{ with $release.description }}
{{ . | markdownify }} @@ -60,20 +68,6 @@
- {{ with $highlights }} -
-
- {{ partial "heading.html" (dict "text" "Highlights" "level" 2) }} -
- -
- {{ range . }} - {{ .Render "li" }} - {{ end }} -
-
- {{ end }} - {{ with $release.known_issues }} {{ if gt (len $release.known_issues) 0 }}
From f36ed0e2cebf36efdc7392aed0ed52ae28a29101 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Wed, 22 Apr 2026 15:54:48 -0400 Subject: [PATCH 149/326] chore(website): render release date on per-version release page (#25244) Mirror the pattern used by the highlights/guides content header and render \$release.date below the hero on /releases// pages. The date already exists in each version's CUE (e.g. date: "2026-04-22" in website/cue/reference/releases/0.55.0.cue) but was only surfaced on the releases index (releases/li.html). Co-authored-by: Claude Opus 4.7 (1M context) --- website/layouts/releases/single.html | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/website/layouts/releases/single.html b/website/layouts/releases/single.html index f83cb9e349122..c1708ea955a9c 100644 --- a/website/layouts/releases/single.html +++ b/website/layouts/releases/single.html @@ -18,6 +18,15 @@ {{ partial "hero.html" . }}
+ {{ with $release.date }} + {{ $date := . | dateFormat "January 2, 2006" }} +
+ +
+ {{ end }} +
{{ with $release.codename }}

From 29b17aadc63fdd9a9bdc3b64bbd8f9c647844cdb Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Wed, 22 Apr 2026 16:30:30 -0400 Subject: [PATCH 150/326] fix(ci): skip re-adding WIP label on synchronize if already approved (#25246) The add_wip_label workflow fires on synchronize to catch docs paths added in later pushes. However, it also re-added the label after a reviewer had already approved and remove_wip_label removed it. Guard against that by checking the latest review state per member: skip only when at least one member approved and none have a standing changes-requested review. This preserves the "catch docs added later" behavior while avoiding label churn after approval. Co-authored-by: Claude Opus 4.7 (1M context) --- .github/workflows/add_wip_label.yml | 30 ++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/.github/workflows/add_wip_label.yml b/.github/workflows/add_wip_label.yml index a07f9ca0921fa..9a1260b693e81 100644 --- a/.github/workflows/add_wip_label.yml +++ b/.github/workflows/add_wip_label.yml @@ -17,6 +17,34 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 5 steps: - - uses: actions-ecosystem/action-add-labels@18f1af5e3544586314bbe15c0273249c770b2daf # v1.1.3 + - name: Check member review state + id: check + uses: actions/github-script@v7 + with: + script: | + const { data: reviews } = await github.rest.pulls.listReviews({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.payload.pull_request.number, + }); + + const allowed = ['MEMBER', 'OWNER']; + const latestByUser = new Map(); + for (const r of reviews) { + if (!allowed.includes(r.author_association)) continue; + if (r.state === 'COMMENTED') continue; + latestByUser.set(r.user.id, r.state); + } + + const states = [...latestByUser.values()]; + const hasApproved = states.includes('APPROVED'); + const hasChangesRequested = states.includes('CHANGES_REQUESTED'); + const skip = hasApproved && !hasChangesRequested; + + core.info(`Latest member review states: ${states.join(', ') || '(none)'}`); + core.info(`skip=${skip}`); + core.setOutput('skip', skip); + - if: steps.check.outputs.skip != 'true' + uses: actions-ecosystem/action-add-labels@18f1af5e3544586314bbe15c0273249c770b2daf # v1.1.3 with: labels: "work in progress" From d99899e2f9cb94c8d421270ada8688243402b5a1 Mon Sep 17 00:00:00 2001 From: "shalk(xiao kun)" Date: Thu, 23 Apr 2026 04:41:08 +0800 Subject: [PATCH 151/326] fix(website): fix docs sidebar expand/collapse navigation (#25238) - Use strings.HasPrefix to keep sections expanded when on any child page (the previous eq check only matched the section index, not child pages) - Add @click="open = true" to section title links so clicking the title expands the group in addition to navigating, without requiring the chevron Co-authored-by: Claude Sonnet 4.6 Co-authored-by: Pavlos Rontidis --- website/layouts/partials/docs/sidebar.html | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/website/layouts/partials/docs/sidebar.html b/website/layouts/partials/docs/sidebar.html index f9c81ed763cd8..e9b9af98de037 100644 --- a/website/layouts/partials/docs/sidebar.html +++ b/website/layouts/partials/docs/sidebar.html @@ -47,10 +47,13 @@ {{ define "subsection-group" }} {{ $here := .here }} {{ $section := .section }} -{{ $open := or (.ctx.IsAncestor $section) (eq .ctx.RelPermalink $here) }} +{{ $open := or (.ctx.IsAncestor $section) (strings.HasPrefix $here .ctx.RelPermalink) }} +{{ $isActive := eq .ctx.RelPermalink $here }}

- {{ template "link" (dict "here" $here "url" .ctx.RelPermalink "title" (.ctx.Params.short | default .ctx.Title)) }} + + {{ .ctx.Params.short | default .ctx.Title }} + {{ template "chevron-icon" }} From ecdaa50ac15e6a26e811bf21e3a6e33729bebb33 Mon Sep 17 00:00:00 2001 From: Thomas Date: Wed, 22 Apr 2026 19:23:12 -0400 Subject: [PATCH 152/326] fix(ci): verify choco package install after 5xx from feed (#25116) * fix(ci): verify choco package install after 5xx from feed * fix(ci): match package name in choco list output to verify install --- scripts/environment/bootstrap-windows-2025.ps1 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/environment/bootstrap-windows-2025.ps1 b/scripts/environment/bootstrap-windows-2025.ps1 index 862efc554a187..9686811a97238 100644 --- a/scripts/environment/bootstrap-windows-2025.ps1 +++ b/scripts/environment/bootstrap-windows-2025.ps1 @@ -11,7 +11,10 @@ function Install-ChocoPackage { for ($attempt = 1; $attempt -le $MaxRetries; $attempt++) { choco install $Package --execution-timeout=7200 -y - if ($LASTEXITCODE -eq 0) { + # Both `choco install` and `choco list` can exit 0 even on 5xx errors + # from the feed, so verify install by matching a "name|version" line + # in the list output. --limit-output strips headers/warnings. + if ((choco list --limit-output -e $Package) -match "^$([regex]::Escape($Package))\|") { return } From f1557d68c8e7f36feb8e2d0c7a04286817c6bf9f Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Thu, 23 Apr 2026 10:16:39 -0400 Subject: [PATCH 153/326] feat(unit tests): add `expected_event_count` field to test outputs (#25186) * feat(unit tests): add `expected_event_count` field to test outputs Add an optional `expected_event_count` field to the `TestOutput` config struct, allowing users to assert on the number of events emitted by a transform in unit tests. The count check is independent of conditions and is verified first. - Reject `expected_event_count: 0` with conditions at build time - Error on conflicting counts when multiple outputs share extract_from - Allow `expected_event_count: 0` to suppress the "no events received" error when zero events are genuinely expected Closes #20278 Co-Authored-By: Claude Opus 4.6 (1M context) * fix(unit tests): add missing authors line to changelog fragment Co-Authored-By: Claude Opus 4.6 (1M context) * fix(unit tests): reject zero-count outputs with conditions after merge A config could bypass the zero-count/conditions validation by splitting expected_event_count: 0 and conditions across two output entries sharing the same extract_from. After merge the combined entry had count 0 with non-empty conditions, causing conditions to pass vacuously against zero events. Add a post-merge check to catch this. Co-Authored-By: Claude Opus 4.6 (1M context) * fix author * chore(unit tests): replace output tuple with BuiltOutput struct Addresses review feedback on PR #25186: the `(Option, Vec>)` tuple in `build_outputs` was too opaque, so this introduces a `BuiltOutput` struct with named fields. Co-Authored-By: Claude Opus 4.7 (1M context) * chore(unit tests): fix indoc indentation in expected_event_count tests Addresses review nit on PR #25186: the YAML in `expected_event_count_correct` and `expected_event_count_wrong` was over-indented; align it with the other tests in the file. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.6 (1M context) --- .../20278_expected_event_count.feature.md | 3 + src/config/mod.rs | 16 +- src/config/unit_test/mod.rs | 58 ++++- src/config/unit_test/tests.rs | 235 ++++++++++++++++++ src/config/unit_test/unit_test_components.rs | 22 +- 5 files changed, 323 insertions(+), 11 deletions(-) create mode 100644 changelog.d/20278_expected_event_count.feature.md diff --git a/changelog.d/20278_expected_event_count.feature.md b/changelog.d/20278_expected_event_count.feature.md new file mode 100644 index 0000000000000..23dac198ecdea --- /dev/null +++ b/changelog.d/20278_expected_event_count.feature.md @@ -0,0 +1,3 @@ +Unit tests now support an optional `expected_event_count` field on test outputs, allowing assertions on the number of events emitted by a transform. + +authors: pront diff --git a/src/config/mod.rs b/src/config/mod.rs index dceade72d6eeb..4a24954926c13 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -446,11 +446,12 @@ impl TestDefinition { let TestOutput { extract_from, conditions, + expected_event_count, } = old; - (extract_from.to_vec(), conditions) + (extract_from.to_vec(), conditions, expected_event_count) }) - .filter_map(|(extract_from, conditions)| { + .filter_map(|(extract_from, conditions, expected_event_count)| { let mut outputs = Vec::new(); for from in extract_from { if no_outputs_from.contains(&from) { @@ -471,6 +472,7 @@ impl TestDefinition { Some(TestOutput { extract_from: outputs.into(), conditions, + expected_event_count, }) } }) @@ -525,6 +527,7 @@ impl TestDefinition { .collect::>() .into(), conditions: old.conditions, + expected_event_count: old.expected_event_count, }) .collect(); @@ -596,6 +599,15 @@ pub struct TestOutput { /// The conditions to run against the output to validate that they were transformed as expected. pub conditions: Option>, + + /// The expected number of events to be produced by the transform. + /// + /// If specified, the test will fail if the number of events emitted by the + /// transform does not match this value. This check is independent of + /// `conditions` -- the count is verified first, then each condition is + /// evaluated against the output events separately. This is useful for + /// transforms that may emit multiple events. + pub expected_event_count: Option, } #[cfg(all(test, feature = "sources-file", feature = "sinks-console"))] diff --git a/src/config/unit_test/mod.rs b/src/config/unit_test/mod.rs index 4d8c1fa97ad65..5020c0b506629 100644 --- a/src/config/unit_test/mod.rs +++ b/src/config/unit_test/mod.rs @@ -293,14 +293,17 @@ impl UnitTestBuildMetadata { let mut template_sinks = IndexMap::new(); let mut test_result_rxs = Vec::new(); // Add sinks with checks - for (ids, checks) in outputs { + for (ids, built) in outputs { let (tx, rx) = oneshot::channel(); let sink_ids = ids.clone(); let sink_config = UnitTestSinkConfig { test_name: test_name.to_string(), transform_ids: ids.iter().map(|id| id.to_string()).collect(), result_tx: Arc::new(Mutex::new(Some(tx))), - check: UnitTestSinkCheck::Checks(checks), + check: UnitTestSinkCheck::Checks { + conditions: built.conditions, + expected_event_count: built.expected_event_count, + }, }; test_result_rxs.push(rx); @@ -567,10 +570,16 @@ fn build_and_validate_inputs( } } +#[derive(Default)] +pub(super) struct BuiltOutput { + pub(super) expected_event_count: Option, + pub(super) conditions: Vec>, +} + fn build_outputs( test_outputs: &[TestOutput], -) -> Result, Vec>>, Vec> { - let mut outputs: IndexMap, Vec>> = IndexMap::new(); +) -> Result, BuiltOutput>, Vec> { + let mut outputs: IndexMap, BuiltOutput> = IndexMap::new(); let mut errors = Vec::new(); for output in test_outputs { @@ -590,10 +599,47 @@ fn build_outputs( } } + let expected_event_count = output.expected_event_count; + if expected_event_count == Some(0) && !conditions.is_empty() { + errors.push(format!( + "output for {:?} has expected_event_count of 0 but also defines conditions; \ + conditions cannot be evaluated when no events are expected", + output.extract_from + )); + } outputs .entry(output.extract_from.clone().to_vec()) - .and_modify(|existing_conditions| existing_conditions.push(conditions.clone())) - .or_insert(vec![conditions.clone()]); + .and_modify(|existing| { + if let (Some(prev), Some(new)) = + (existing.expected_event_count, expected_event_count) + { + if prev != new { + errors.push(format!( + "conflicting expected_event_count for extract_from {:?}: {} vs {}", + output.extract_from, prev, new + )); + } + } else if existing.expected_event_count.is_none() { + existing.expected_event_count = expected_event_count; + } + existing.conditions.push(conditions.clone()); + }) + .or_insert_with(|| BuiltOutput { + expected_event_count, + conditions: vec![conditions.clone()], + }); + } + + // Post-merge validation: after merging entries that share the same + // extract_from, reject any that ended up with expected_event_count of 0 and + // non-empty conditions (which would pass vacuously against zero events). + for (extract_from, built) in &outputs { + if built.expected_event_count == Some(0) && built.conditions.iter().any(|c| !c.is_empty()) { + errors.push(format!( + "output for {extract_from:?} has expected_event_count of 0 but also defines conditions; \ + conditions cannot be evaluated when no events are expected", + )); + } } if errors.is_empty() { diff --git a/src/config/unit_test/tests.rs b/src/config/unit_test/tests.rs index 938286ac9ba9f..6881ba6f75901 100644 --- a/src/config/unit_test/tests.rs +++ b/src/config/unit_test/tests.rs @@ -1298,3 +1298,238 @@ async fn test_glob_input() { let mut tests = build_unit_tests(config).await.unwrap(); assert!(tests.remove(0).run().await.errors.is_empty()); } + +#[tokio::test] +async fn expected_event_count_correct() { + crate::test_util::trace_init(); + + let config: ConfigBuilder = crate::config::format::deserialize( + indoc! {r#" + transforms: + foo: + inputs: + - ignored + type: remap + source: | + . = [{"message": "one"}, {"message": "two"}, {"message": "three"}] + tests: + - name: event count check + inputs: + - insert_at: foo + value: doesnt matter + outputs: + - extract_from: foo + expected_event_count: 3 + conditions: + - type: vrl + source: | + assert!(exists(.message), "message field must exist") + "#}, + crate::config::Format::Yaml, + ) + .unwrap(); + + let mut tests = build_unit_tests(config).await.unwrap(); + assert!(tests.remove(0).run().await.errors.is_empty()); +} + +#[tokio::test] +async fn expected_event_count_wrong() { + crate::test_util::trace_init(); + + let config: ConfigBuilder = crate::config::format::deserialize( + indoc! {r#" + transforms: + foo: + inputs: + - ignored + type: remap + source: | + . = [{"message": "one"}, {"message": "two"}, {"message": "three"}] + tests: + - name: wrong event count + inputs: + - insert_at: foo + value: doesnt matter + outputs: + - extract_from: foo + expected_event_count: 2 + conditions: + - type: vrl + source: | + assert!(exists(.message), "message field must exist") + "#}, + crate::config::Format::Yaml, + ) + .unwrap(); + + let mut tests = build_unit_tests(config).await.unwrap(); + let errors = tests.remove(0).run().await.errors; + assert!(!errors.is_empty()); + assert!( + errors + .iter() + .any(|e| e.contains("expected 2 events") && e.contains("but received 3")), + "expected event count error, got: {errors:?}" + ); +} + +#[tokio::test] +async fn expected_event_count_zero_no_conditions() { + crate::test_util::trace_init(); + + let config: ConfigBuilder = crate::config::format::deserialize( + indoc! {r#" + transforms: + foo: + inputs: + - ignored + type: remap + source: | + if .message == "drop me" { + abort + } + tests: + - name: zero events expected + inputs: + - insert_at: foo + value: "drop me" + outputs: + - extract_from: foo + expected_event_count: 0 + "#}, + crate::config::Format::Yaml, + ) + .unwrap(); + + let mut tests = build_unit_tests(config).await.unwrap(); + assert!(tests.remove(0).run().await.errors.is_empty()); +} + +#[tokio::test] +async fn expected_event_count_zero_with_conditions_rejected() { + crate::test_util::trace_init(); + + let config: ConfigBuilder = crate::config::format::deserialize( + indoc! {r#" + transforms: + foo: + inputs: + - ignored + type: remap + source: | + if .message == "drop me" { + abort + } + tests: + - name: zero with conditions + inputs: + - insert_at: foo + value: "drop me" + outputs: + - extract_from: foo + expected_event_count: 0 + conditions: + - type: vrl + source: | + assert!(exists(.message), "unreachable") + "#}, + crate::config::Format::Yaml, + ) + .unwrap(); + + let errs = build_unit_tests(config).await.err().unwrap(); + assert!( + errs.iter() + .any(|e| e.contains("expected_event_count of 0") && e.contains("conditions")), + "expected config error about zero count with conditions, got: {errs:?}" + ); +} + +#[tokio::test] +async fn expected_event_count_conflicting_values() { + crate::test_util::trace_init(); + + let config: ConfigBuilder = crate::config::format::deserialize( + indoc! {r#" + transforms: + foo: + inputs: + - ignored + type: remap + source: | + . = [{"message": "one"}, {"message": "two"}] + tests: + - name: conflicting counts + inputs: + - insert_at: foo + value: doesnt matter + outputs: + - extract_from: foo + expected_event_count: 2 + conditions: + - type: vrl + source: | + assert!(exists(.message), "ok") + - extract_from: foo + expected_event_count: 5 + conditions: + - type: vrl + source: | + assert!(exists(.message), "ok") + "#}, + crate::config::Format::Yaml, + ) + .unwrap(); + + let errs = build_unit_tests(config).await.err().unwrap(); + assert!( + errs.iter() + .any(|e| e.contains("conflicting expected_event_count")), + "expected conflicting count error, got: {errs:?}" + ); +} + +#[tokio::test] +async fn expected_event_count_zero_split_with_conditions_rejected() { + crate::test_util::trace_init(); + + // Splitting expected_event_count: 0 and conditions across two output + // entries that share the same extract_from must still be rejected after + // merge, otherwise the conditions would pass vacuously against zero events. + let config: ConfigBuilder = crate::config::format::deserialize( + indoc! {r#" + transforms: + foo: + inputs: + - ignored + type: remap + source: | + if .message == "drop me" { + abort + } + tests: + - name: split zero with conditions + inputs: + - insert_at: foo + value: "drop me" + outputs: + - extract_from: foo + expected_event_count: 0 + - extract_from: foo + conditions: + - type: vrl + source: | + assert!(false, "should never be evaluated") + "#}, + crate::config::Format::Yaml, + ) + .unwrap(); + + let errs = build_unit_tests(config).await.err().unwrap(); + assert!( + errs.iter() + .any(|e| e.contains("expected_event_count of 0") && e.contains("conditions")), + "expected config error about zero count with conditions after merge, got: {errs:?}" + ); +} diff --git a/src/config/unit_test/unit_test_components.rs b/src/config/unit_test/unit_test_components.rs index 02808fb083ee7..b3f09098237da 100644 --- a/src/config/unit_test/unit_test_components.rs +++ b/src/config/unit_test/unit_test_components.rs @@ -117,7 +117,10 @@ impl SourceConfig for UnitTestStreamSourceConfig { #[derive(Clone, Default)] pub enum UnitTestSinkCheck { /// Check all events that are received against the list of conditions. - Checks(Vec>), + Checks { + conditions: Vec>, + expected_event_count: Option, + }, /// Check that no events were received. NoOutputs, @@ -203,8 +206,21 @@ impl StreamSink for UnitTestSink { } match self.check { - UnitTestSinkCheck::Checks(checks) => { - if output_events.is_empty() { + UnitTestSinkCheck::Checks { + conditions: checks, + expected_event_count, + } => { + if let Some(expected) = expected_event_count { + let actual = output_events.len(); + if actual != expected { + result.test_errors.push(format!( + "expected {} events from transforms {:?}, but received {}", + expected, self.transform_ids, actual + )); + } + } + + if output_events.is_empty() && expected_event_count != Some(0) { result .test_errors .push(format!("checks for transforms {:?} failed: no events received. Topology may be disconnected or transform is missing inputs.", self.transform_ids)); From cfb942bf7d1983b9eeaeb93ba2f843a816d1db5d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 23 Apr 2026 13:39:22 -0400 Subject: [PATCH 154/326] chore(deps): bump openssl from 0.10.75 to 0.10.78 (#25250) Bumps [openssl](https://github.com/rust-openssl/rust-openssl) from 0.10.75 to 0.10.78. - [Release notes](https://github.com/rust-openssl/rust-openssl/releases) - [Commits](https://github.com/rust-openssl/rust-openssl/compare/openssl-v0.10.75...openssl-v0.10.78) --- updated-dependencies: - dependency-name: openssl dependency-version: 0.10.78 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 8 ++++---- lib/vector-core/Cargo.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a39a96a6500b1..57f9a916e8812 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7373,9 +7373,9 @@ dependencies = [ [[package]] name = "openssl" -version = "0.10.75" +version = "0.10.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" +checksum = "f38c4372413cdaaf3cc79dd92d29d7d9f5ab09b51b10dded508fb90bb70b9222" dependencies = [ "bitflags 2.10.0", "cfg-if", @@ -7414,9 +7414,9 @@ dependencies = [ [[package]] name = "openssl-sys" -version = "0.9.111" +version = "0.9.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82cab2d520aa75e3c58898289429321eb788c3106963d0dc886ec7a5f4adc321" +checksum = "13ce1245cd07fcc4cfdb438f7507b0c7e4f3849a69fd84d52374c66d83741bb6" dependencies = [ "cc", "libc", diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml index 7d2d8f8979266..c26953cb3590a 100644 --- a/lib/vector-core/Cargo.toml +++ b/lib/vector-core/Cargo.toml @@ -34,7 +34,7 @@ metrics-util.workspace = true mlua = { version = "0.10.5", default-features = false, features = ["lua54", "send", "vendored"], optional = true } no-proxy = { version = "0.3.6", default-features = false, features = ["serialize"] } ordered-float.workspace = true -openssl = { version = "0.10.75", default-features = false, features = ["vendored"] } +openssl = { version = "0.10.78", default-features = false, features = ["vendored"] } parking_lot = { version = "0.12.5", default-features = false } pin-project.workspace = true proptest = { workspace = true, optional = true } From c0cdf2068f79176fd8af4dca9b27cabae5be3091 Mon Sep 17 00:00:00 2001 From: Thomas Date: Thu, 23 Apr 2026 13:52:07 -0400 Subject: [PATCH 155/326] chore(ci): extract unit tests into reusable workflow (#25247) * chore(ci): extract unit tests into reusable workflow * chore(ci): fix component-validation input and add OS-based FEATURES default * chore(ci): pass Datadog credentials into reusable unit-tests workflow --- .github/workflows/changes.yml | 1 + .github/workflows/test.yml | 30 ++------ .github/workflows/unit-tests.yml | 118 +++++++++++++++++++++++++++++++ 3 files changed, 125 insertions(+), 24 deletions(-) create mode 100644 .github/workflows/unit-tests.yml diff --git a/.github/workflows/changes.yml b/.github/workflows/changes.yml index f9cd9ce4ba5b9..b07447715d75f 100644 --- a/.github/workflows/changes.yml +++ b/.github/workflows/changes.yml @@ -280,6 +280,7 @@ jobs: - ".github/workflows/changes.yml" test-yml: - ".github/workflows/test.yml" + - ".github/workflows/unit-tests.yml" - ".github/workflows/changes.yml" integration-yml: - ".github/workflows/integration.yml" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 81ceef93ca5f1..84a4c31226b1f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -56,31 +56,13 @@ jobs: - run: make check-clippy test: - name: Unit and Component Validation tests - x86_64-unknown-linux-gnu - runs-on: ubuntu-24.04-8core - if: ${{ needs.changes.outputs.source == 'true' || needs.changes.outputs.test-yml == 'true' }} needs: changes - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: ./.github/actions/setup - with: - rust: true - cargo-nextest: true - datadog-ci: true - protoc: true - libsasl2: true - - name: Unit Test - run: make test - env: - CARGO_BUILD_JOBS: 5 - - # Validates components for adherence to the Component Specification - - name: Check Component Spec - run: make test-component-validation - - - name: Upload test results - run: scripts/upload-test-results.sh - if: always() + if: ${{ needs.changes.outputs.source == 'true' || needs.changes.outputs.test-yml == 'true' }} + uses: ./.github/workflows/unit-tests.yml + with: + default: true + component-validation: true + secrets: inherit check-scripts: name: Check scripts diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml new file mode 100644 index 0000000000000..e3b6501247858 --- /dev/null +++ b/.github/workflows/unit-tests.yml @@ -0,0 +1,118 @@ +# Reusable Workflow: Tests +# +# Runs test suites via the Makefile. By default runs unit tests and component +# validation. Callers can enable additional suites (CLI, Vector API, behavior) +# via boolean inputs. +# +# When `default` is true, all nextest-based suites run in a single `make test` +# invocation with the relevant features enabled. When `default` is false, only +# the explicitly enabled suites run, using a nextest filter expression to +# select the matching tests. + +name: Tests + +on: + workflow_call: + inputs: + ref: + description: "Git ref to checkout" + required: false + type: string + default: + description: "Run the full default unit test suite (--workspace with default features)" + required: true + type: boolean + component-validation: + description: "Include component validation tests" + required: false + type: boolean + default: true + cli: + description: "Include CLI tests" + required: false + type: boolean + default: false + vector-api: + description: "Include Vector API tests" + required: false + type: boolean + default: false + behavior: + description: "Include behavior tests" + required: false + type: boolean + default: false + +permissions: + contents: read + +env: + CARGO_INCREMENTAL: "0" + CI: true + DD_ENV: "ci" + DD_API_KEY: ${{ secrets.DD_API_KEY }} + +jobs: + tests: + name: Tests + runs-on: ubuntu-24.04-8core + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ inputs.ref }} + + - uses: ./.github/actions/setup + with: + rust: true + cargo-nextest: true + datadog-ci: true + protoc: true + libsasl2: true + + - name: Run tests + run: | + add_suite() { + local enabled="$1" feature="$2" filter="$3" + [[ "${enabled}" != "true" ]] && return + FEATURES="${FEATURES:+$FEATURES,}${feature}" + if [[ "${{ inputs.default }}" != "true" ]]; then + SCOPE="${SCOPE:+$SCOPE | }${filter}" + fi + } + + # Mirror the Makefile's OS-based default: `default-msvc` on Windows, + # `default` elsewhere. Keep in sync with the FEATURES defaults in Makefile. + if [[ "${RUNNER_OS}" == "Windows" ]]; then + DEFAULT_FEATURES="default-msvc" + else + DEFAULT_FEATURES="default" + fi + + if [[ "${{ inputs.default }}" == "true" ]]; then + FEATURES="${DEFAULT_FEATURES}" + else + FEATURES="" + fi + + add_suite "${{ inputs.component-validation }}" "component-validation-tests" "test(components::validation::tests)" + add_suite "${{ inputs.cli }}" "cli-tests" "package(vector) & binary(=integration)" + add_suite "${{ inputs.vector-api }}" "vector-api-tests" "binary(=vector_api)" + + # Skip `make test` entirely if no nextest-based suite was selected + # (e.g. behavior-only runs). Running `make test` with empty FEATURES + # would otherwise execute the workspace-wide suite. + if [[ "${{ inputs.default }}" == "true" ]]; then + make test FEATURES="${FEATURES}" + elif [[ -n "${SCOPE}" ]]; then + make test FEATURES="${FEATURES}" SCOPE="-E '${SCOPE}'" + else + echo "No nextest suites selected; skipping 'make test'" + fi + + - name: Behavior tests + if: ${{ inputs.behavior }} + run: make test-behavior + + - name: Upload test results to Datadog + if: ${{ always() }} + run: scripts/upload-test-results.sh From f56f4fa7234440345d790a2b81c7aa880be831d8 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Thu, 23 Apr 2026 16:29:50 -0400 Subject: [PATCH 156/326] chore(ci): remove Chocolatey from Windows bootstrap (#25254) * chore(ci): remove Chocolatey from Windows bootstrap Install protoc directly from the upstream GitHub release (matching the pinned version used on Linux/macOS in scripts/environment/install-protoc.sh) instead of going through Chocolatey. GNU make is already on the default PATH of the windows-2025 runner image via C:\mingw64\bin, so no extra install step is needed for it. This eliminates the recurring "Chocolatey installed 0/0 packages" failures caused by Chocolatey CDN blips, and pins the Windows build to the same protoc version used on Linux/macOS. Verified on the ci-sandbox repo: - protoc resolves to $RUNNER_TEMP\protoc\bin\protoc.exe -> libprotoc 3.21.12 - make resolves to C:\mingw64\bin\make.exe -> GNU Make 4.4.1 Co-Authored-By: Claude Opus 4.7 (1M context) * chore(ci): share install-protoc.sh across platforms Extend the existing Linux/macOS protoc installer with MINGW/MSYS platform detection and .exe handling, and have bootstrap-windows-2025.ps1 delegate to it via Git Bash. This replaces the inline PowerShell download introduced earlier and makes all three platforms pin the same protoc version from the same upstream source. Also adds curl --retry 5 --retry-delay 10 --retry-all-errors so transient download blips are retried without extra scaffolding. Co-Authored-By: Claude Opus 4.7 (1M context) * chore(ci): drop curl --retry-all-errors for focal compatibility The ghcr.io/cross-rs/* base images used by scripts/cross/Dockerfile are Ubuntu 20.04 focal, which ships curl 7.68.x. --retry-all-errors was added in curl 7.71.0, so it would be rejected as an unknown option and break cross-target builds that run install-protoc.sh inside those images. Plain --retry 5 --retry-delay 10 still handles the transport-level retries that cover the CDN blips we care about. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) --- .../environment/bootstrap-windows-2025.ps1 | 42 ++++++------------- scripts/environment/install-protoc.sh | 37 +++++++++++----- 2 files changed, 39 insertions(+), 40 deletions(-) diff --git a/scripts/environment/bootstrap-windows-2025.ps1 b/scripts/environment/bootstrap-windows-2025.ps1 index 9686811a97238..7d73eeccbc32d 100644 --- a/scripts/environment/bootstrap-windows-2025.ps1 +++ b/scripts/environment/bootstrap-windows-2025.ps1 @@ -2,32 +2,6 @@ $ErrorActionPreference = "Stop" Set-StrictMode -Version Latest -# Helper function to install choco packages with exponential backoff retry -function Install-ChocoPackage { - param( - [string]$Package, - [int]$MaxRetries = 5 - ) - - for ($attempt = 1; $attempt -le $MaxRetries; $attempt++) { - choco install $Package --execution-timeout=7200 -y - # Both `choco install` and `choco list` can exit 0 even on 5xx errors - # from the feed, so verify install by matching a "name|version" line - # in the list output. --limit-output strips headers/warnings. - if ((choco list --limit-output -e $Package) -match "^$([regex]::Escape($Package))\|") { - return - } - - if ($attempt -lt $MaxRetries) { - $delay = 5 * [math]::Pow(2, $attempt) # Exponential: 10, 20, 40, 80 seconds - Write-Host "choco install $Package failed (attempt $attempt of $MaxRetries). Retrying in $delay seconds..." - Start-Sleep -Seconds $delay - } else { - throw "choco install $Package failed after $MaxRetries attempts" - } - } -} - # Set up our Cargo path so we can do Rust-y things. echo "$HOME\.cargo\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append @@ -37,9 +11,19 @@ if ($env:RELEASE_BUILDER -ne "true") { bash scripts/environment/prepare.sh --modules=rustup } -# Install Chocolatey packages with exponential backoff retry -Install-ChocoPackage "make" -Install-ChocoPackage "protoc" +# Install protoc via the shared cross-platform script. It pins the same version +# used on Linux/macOS and downloads directly from the upstream GitHub release, +# so we avoid the recurring Chocolatey CDN failures. +$ProtocInstallDir = Join-Path $env:RUNNER_TEMP "protoc-bin" +bash scripts/environment/install-protoc.sh "$ProtocInstallDir" +if ($LASTEXITCODE -ne 0) { + throw "install-protoc.sh failed with exit code $LASTEXITCODE" +} +echo "$ProtocInstallDir" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + +# GNU make is already on PATH on the windows-2025 runner image via the +# pre-installed MinGW toolchain at C:\mingw64\bin, so no extra install is +# needed here. # Set a specific override path for libclang. echo "LIBCLANG_PATH=$( (gcm clang).source -replace "clang.exe" )" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append diff --git a/scripts/environment/install-protoc.sh b/scripts/environment/install-protoc.sh index 4d1debe6c3251..1e75279a3aec2 100755 --- a/scripts/environment/install-protoc.sh +++ b/scripts/environment/install-protoc.sh @@ -23,13 +23,12 @@ trap 'rm -rf "${TMP_DIR?}"' EXIT get_platform() { local os os=$(uname) - if [[ "${os}" == "Darwin" ]]; then - echo "osx" - elif [[ "${os}" == "Linux" ]]; then - echo "linux" - else - >&2 echo "unsupported os: ${os}" && exit 1 - fi + case "${os}" in + Darwin) echo "osx" ;; + Linux) echo "linux" ;; + MINGW*|MSYS*|CYGWIN*) echo "win64" ;; + *) >&2 echo "unsupported os: ${os}" && exit 1 ;; + esac } get_arch() { @@ -47,20 +46,36 @@ get_arch() { fi } +get_bin_name() { + if [[ "$(get_platform)" == "win64" ]]; then + echo "protoc.exe" + else + echo "protoc" + fi +} + install_protoc() { local version=$1 local install_path=$2 local base_url="https://github.com/protocolbuffers/protobuf/releases/download" local url - url="${base_url}/v${version}/protoc-${version}-$(get_platform)-$(get_arch).zip" + if [[ "$(get_platform)" == "win64" ]]; then + # Windows release assets are named without an explicit arch suffix. + url="${base_url}/v${version}/protoc-${version}-win64.zip" + else + url="${base_url}/v${version}/protoc-${version}-$(get_platform)-$(get_arch).zip" + fi local download_path="${TMP_DIR}/protoc.zip" echo "Downloading ${url}" - curl -fsSL "${url}" -o "${download_path}" + # Stay compatible with the curl shipped by Ubuntu 20.04 focal (7.68.x) used + # in the ghcr.io/cross-rs/* base images: --retry-all-errors was only added + # in curl 7.71.0, so rely on the transport-level retry that --retry covers. + curl --retry 5 --retry-delay 10 -fsSL "${url}" -o "${download_path}" unzip -qq "${download_path}" -d "${TMP_DIR}" - mv -f -v "${TMP_DIR}/bin/protoc" "${install_path}" + mv -f -v "${TMP_DIR}/bin/$(get_bin_name)" "${install_path}" } -install_protoc "21.12" "${INSTALL_PATH}/protoc" +install_protoc "21.12" "${INSTALL_PATH}/$(get_bin_name)" From 7428dc3f4b23c56f1f2eecf44b3f6b2532c28dac Mon Sep 17 00:00:00 2001 From: Thomas Date: Fri, 24 Apr 2026 14:40:53 -0400 Subject: [PATCH 157/326] chore(ci): use dd-sts instead of DD_API_KEY (#25235) * chore(ci): use dd-octo-sts instead of DD_API_KEY * chore(ci): rename policy to public-vectordotdev-vector * chore(ci): grant id-token: write to reusable workflow callers * Format * chore(ci): introduce dd-token composite action to federate Datadog credentials * Fix policy name --- .github/actions/dd-token/action.yml | 28 +++++++++++++++++++++ .github/workflows/ci-integration-review.yml | 9 ++++++- .github/workflows/ci-review-trigger.yml | 1 + .github/workflows/coverage.yml | 6 ++++- .github/workflows/integration-test.yml | 6 ++++- .github/workflows/integration.yml | 13 +++++++++- .github/workflows/master_merge_queue.yml | 2 +- .github/workflows/static-analysis.yml | 12 ++++++--- .github/workflows/test-make-command.yml | 7 +++++- .github/workflows/test.yml | 4 ++- .github/workflows/unit-tests.yml | 6 ++++- 11 files changed, 82 insertions(+), 12 deletions(-) create mode 100644 .github/actions/dd-token/action.yml diff --git a/.github/actions/dd-token/action.yml b/.github/actions/dd-token/action.yml new file mode 100644 index 0000000000000..8ea92dc1c2dd5 --- /dev/null +++ b/.github/actions/dd-token/action.yml @@ -0,0 +1,28 @@ +name: "Datadog Short-Lived Token" +description: "Federate a short-lived Datadog token via dd-sts-action and export DD_API_KEY (and DD_APP_KEY when available) to the job environment." + +inputs: + policy: + required: false + default: public-vectordotdev-vector + description: "dd-sts policy to federate against." + +runs: + using: "composite" + steps: + - name: Federate Datadog token + id: dd-sts + uses: DataDog/dd-sts-action@2e8187910199bd93129520183c093e19aa585c75 + with: + policy: ${{ inputs.policy }} + + - name: Export Datadog credentials to environment + shell: bash + env: + DD_STS_API_KEY: ${{ steps.dd-sts.outputs.api_key }} + DD_STS_APP_KEY: ${{ steps.dd-sts.outputs.app_key }} + run: | + echo "DD_API_KEY=${DD_STS_API_KEY}" >> "$GITHUB_ENV" + if [ -n "${DD_STS_APP_KEY}" ]; then + echo "DD_APP_KEY=${DD_STS_APP_KEY}" >> "$GITHUB_ENV" + fi diff --git a/.github/workflows/ci-integration-review.yml b/.github/workflows/ci-integration-review.yml index 77b20b347e318..b042a76eb8ba9 100644 --- a/.github/workflows/ci-integration-review.yml +++ b/.github/workflows/ci-integration-review.yml @@ -41,7 +41,6 @@ env: TEST_DATADOG_API_KEY: ${{ secrets.CI_TEST_DATADOG_API_KEY }} CONTAINER_TOOL: "docker" DD_ENV: "ci" - DD_API_KEY: ${{ secrets.DD_API_KEY }} RUST_BACKTRACE: full VECTOR_LOG: vector=debug VERBOSE: true @@ -100,6 +99,7 @@ jobs: timeout-minutes: 90 permissions: contents: read + id-token: write packages: read # Required to pull test runner image from GHCR strategy: fail-fast: false @@ -160,6 +160,9 @@ jobs: submodules: "recursive" ref: ${{ github.event.review.commit_id }} + - uses: ./.github/actions/dd-token + if: steps.run_condition.outputs.should_run == 'true' + - uses: ./.github/actions/setup with: vdev: true @@ -191,6 +194,7 @@ jobs: permissions: contents: read packages: read # Required to pull test runner image from GHCR + id-token: write strategy: fail-fast: false matrix: @@ -214,6 +218,9 @@ jobs: submodules: "recursive" ref: ${{ github.event.review.commit_id }} + - uses: ./.github/actions/dd-token + if: steps.run_condition.outputs.should_run == 'true' + - uses: ./.github/actions/setup with: vdev: true diff --git a/.github/workflows/ci-review-trigger.yml b/.github/workflows/ci-review-trigger.yml index 8a2eb206efdcd..9cd2a1d67e3f5 100644 --- a/.github/workflows/ci-review-trigger.yml +++ b/.github/workflows/ci-review-trigger.yml @@ -31,6 +31,7 @@ on: permissions: contents: read + id-token: write env: DD_ENV: "ci" diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index db19d27ffd582..8ca10e1127a0d 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -16,6 +16,9 @@ env: jobs: coverage: runs-on: ubuntu-24.04 + permissions: + contents: read + id-token: write steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -31,9 +34,10 @@ jobs: - name: "Generate code coverage" run: cargo llvm-cov nextest --workspace --lcov --output-path lcov.info + - uses: ./.github/actions/dd-token + - name: "Upload coverage to Datadog" env: - DD_API_KEY: ${{ secrets.DD_API_KEY }} DD_SITE: datadoghq.com DD_ENV: ci run: datadog-ci coverage upload lcov.info diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 74b782aa7d336..641f9ceb96d4b 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -23,7 +23,6 @@ env: TEST_APPSIGNAL_PUSH_API_KEY: ${{ secrets.TEST_APPSIGNAL_PUSH_API_KEY }} CONTAINER_TOOL: "docker" DD_ENV: "ci" - DD_API_KEY: ${{ secrets.DD_API_KEY }} RUST_BACKTRACE: full VECTOR_LOG: vector=debug VERBOSE: true @@ -34,6 +33,9 @@ jobs: test-integration: runs-on: ubuntu-24.04 timeout-minutes: 40 + permissions: + contents: read + id-token: write if: inputs.if || github.event_name == 'workflow_dispatch' steps: - name: (PR comment) Get PR branch @@ -51,6 +53,8 @@ jobs: if: ${{ github.event_name != 'issue_comment' }} uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: ./.github/actions/dd-token + - run: bash scripts/environment/prepare.sh --modules=rustup,datadog-ci - run: make test-integration-${{ inputs.test_name }} diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 2c1e91c57a8e7..833a7ae13d5de 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -23,7 +23,6 @@ permissions: env: CONTAINER_TOOL: "docker" DD_ENV: "ci" - DD_API_KEY: ${{ secrets.DD_API_KEY }} TEST_DATADOG_API_KEY: ${{ secrets.CI_TEST_DATADOG_API_KEY }} TEST_APPSIGNAL_PUSH_API_KEY: ${{ secrets.TEST_APPSIGNAL_PUSH_API_KEY }} AXIOM_TOKEN: ${{ secrets.AXIOM_TOKEN }} @@ -65,6 +64,9 @@ jobs: integration-tests: runs-on: ubuntu-24.04-8core + permissions: + contents: read + id-token: write needs: - changes - build-test-runner @@ -141,6 +143,9 @@ jobs: with: submodules: "recursive" + - uses: ./.github/actions/dd-token + if: steps.check.outputs.should_run == 'true' + - uses: ./.github/actions/setup if: steps.check.outputs.should_run == 'true' with: @@ -167,6 +172,9 @@ jobs: e2e-tests: runs-on: ubuntu-24.04-8core + permissions: + contents: read + id-token: write needs: - changes - build-test-runner @@ -205,6 +213,9 @@ jobs: with: submodules: "recursive" + - uses: ./.github/actions/dd-token + if: steps.check.outputs.should_run == 'true' + - uses: ./.github/actions/setup if: steps.check.outputs.should_run == 'true' with: diff --git a/.github/workflows/master_merge_queue.yml b/.github/workflows/master_merge_queue.yml index 8f6866f42dfe2..3eaffecfdb2cd 100644 --- a/.github/workflows/master_merge_queue.yml +++ b/.github/workflows/master_merge_queue.yml @@ -21,6 +21,7 @@ on: permissions: contents: read + id-token: write concurrency: # `github.ref` is unique for MQ runs and PRs @@ -30,7 +31,6 @@ concurrency: env: CONTAINER_TOOL: "docker" DD_ENV: "ci" - DD_API_KEY: ${{ secrets.DD_API_KEY }} RUST_BACKTRACE: full VECTOR_LOG: vector=debug VERBOSE: true diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml index 9411f2844e264..f74c6681d8ea5 100644 --- a/.github/workflows/static-analysis.yml +++ b/.github/workflows/static-analysis.yml @@ -13,14 +13,18 @@ permissions: jobs: static-analysis: runs-on: ubuntu-latest - env: - DD_API_KEY: ${{ secrets.DD_API_KEY }} - DD_APP_KEY: ${{ secrets.DD_APP_KEY }} + permissions: + contents: read + id-token: write steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - id: dd-token + uses: ./.github/actions/dd-token + with: + policy: public-vectordotdev-vector-static-analysis + - name: Datadog Static Analyzer - if: ${{ env.DD_API_KEY != '' }} uses: DataDog/datadog-static-analyzer-github-action@8340f18875fcefca86844b5f947ce2431387e552 # v3.0.0 with: dd_api_key: ${{ env.DD_API_KEY }} diff --git a/.github/workflows/test-make-command.yml b/.github/workflows/test-make-command.yml index 79c27d54792f4..2a93f9016fcd1 100644 --- a/.github/workflows/test-make-command.yml +++ b/.github/workflows/test-make-command.yml @@ -32,17 +32,22 @@ permissions: jobs: run-make-command: runs-on: ubuntu-24.04 + permissions: + contents: read + id-token: write timeout-minutes: 90 env: CARGO_INCREMENTAL: 0 DD_ENV: "ci" - DD_API_KEY: ${{ secrets.DD_API_KEY }} steps: - name: Checkout branch uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ inputs.ref }} + - uses: ./.github/actions/dd-token + if: ${{ inputs.upload_test_results }} + - uses: ./.github/actions/setup with: rust: true diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 84a4c31226b1f..7e5c28aec476c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,7 +17,6 @@ concurrency: env: CONTAINER_TOOL: "docker" DD_ENV: "ci" - DD_API_KEY: ${{ secrets.DD_API_KEY }} VECTOR_LOG: vector=debug VERBOSE: true CI: true @@ -57,6 +56,9 @@ jobs: test: needs: changes + permissions: + contents: read + id-token: write if: ${{ needs.changes.outputs.source == 'true' || needs.changes.outputs.test-yml == 'true' }} uses: ./.github/workflows/unit-tests.yml with: diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index e3b6501247858..63c9de7d19351 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -50,17 +50,21 @@ env: CARGO_INCREMENTAL: "0" CI: true DD_ENV: "ci" - DD_API_KEY: ${{ secrets.DD_API_KEY }} jobs: tests: name: Tests runs-on: ubuntu-24.04-8core + permissions: + contents: read + id-token: write steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ inputs.ref }} + - uses: ./.github/actions/dd-token + - uses: ./.github/actions/setup with: rust: true From 7815f26ef4f357c7e3e90048e4287728c9506423 Mon Sep 17 00:00:00 2001 From: "shalk(xiao kun)" Date: Sat, 25 Apr 2026 02:43:56 +0800 Subject: [PATCH 158/326] chore(website): add Docker support for local development (#25237) * feat(website): add Docker support for local development Adds Dockerfile, docker-compose.yml, and README instructions so the Vector website can be previewed locally without installing Hugo, CUE, or Node.js. The image detects host architecture via uname -m to support both amd64 and arm64. VRL function docs still require a one-time `make generate-vrl-docs` on the host (needs Rust/vdev). Co-Authored-By: Claude Sonnet 4.6 * Update website/Dockerfile Co-authored-by: Thomas * docs(website): note Docker setup is experimental and not CI-enforced Co-Authored-By: Claude Sonnet 4.6 --------- Co-authored-by: Claude Sonnet 4.6 Co-authored-by: Thomas --- website/Dockerfile | 44 ++++++++++++++++++++++++++++++++++++++ website/README.md | 24 +++++++++++++++++++++ website/docker-compose.yml | 15 +++++++++++++ 3 files changed, 83 insertions(+) create mode 100644 website/Dockerfile create mode 100644 website/docker-compose.yml diff --git a/website/Dockerfile b/website/Dockerfile new file mode 100644 index 0000000000000..e9946941bb459 --- /dev/null +++ b/website/Dockerfile @@ -0,0 +1,44 @@ +FROM debian:bookworm-slim + +ARG HUGO_VERSION=0.154.5 +ARG CUE_VERSION=0.10.0 +ARG NODE_VERSION=24 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + curl ca-certificates git make \ + && rm -rf /var/lib/apt/lists/* + +# Hugo extended (required for Sass/PostCSS processing) +# uname -m: x86_64 -> amd64, aarch64 -> arm64 +RUN ARCH=$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/') && \ + curl -fsSL https://github.com/gohugoio/hugo/releases/download/v${HUGO_VERSION}/hugo_extended_${HUGO_VERSION}_linux-${ARCH}.tar.gz \ + | tar -xz -C /usr/local/bin hugo + +# CUE CLI +RUN ARCH=$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/') && \ + curl -fsSL https://github.com/cue-lang/cue/releases/download/v${CUE_VERSION}/cue_v${CUE_VERSION}_linux_${ARCH}.tar.gz \ + | tar -xz -C /usr/local/bin cue + +# Node.js + Yarn +RUN curl -fsSL https://deb.nodesource.com/setup_${NODE_VERSION}.x | bash - \ + && apt-get install -y --no-install-recommends nodejs \ + && npm install -g yarn \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app/website + +EXPOSE 1313 + +# generate-vrl-docs (requires Rust/vdev) must be run on the host first. +# Call cue directly instead of scripts/cue.sh to avoid its `git rev-parse` dependency. +ENTRYPOINT ["/bin/sh", "-c", \ + "if [ -z \"$(find cue/reference/remap/functions -name '*.cue' 2>/dev/null | head -1)\" ]; then \ + echo 'ERROR: cue/reference/remap/functions/ is empty.'; \ + echo 'Run on the host first: make generate-vrl-docs'; \ + exit 1; \ + fi && \ + yarn && \ + cp -f /app/Cargo.lock data/cargo-lock.toml && \ + rm -f data/docs.json && find cue -name '*.cue' | xargs cue export --all-errors --outfile data/docs.json && \ + yarn config-examples && \ + hugo server --bind 0.0.0.0 --port 1313 --buildDrafts --buildFuture --environment development"] diff --git a/website/README.md b/website/README.md index 47475945dbf9d..79a565b89fb7b 100644 --- a/website/README.md +++ b/website/README.md @@ -136,6 +136,30 @@ This builds all the necessary [prereqs](#prerequisites) for the site and starts When you make changes to the Markdown sources, Sass/CSS, or JavaScript, the site re-builds and Hugo automatically reloads the page that you're on. If you make changes to the [structured data](#structured-data) sources, however, you need to stop the server and run `make serve` again. +### Run the site with Docker + +If you don't want to install Hugo, CUE, or Node.js locally, you can use Docker instead. You still need Rust and [vdev] on the host for one step. + +> **Note:** This Docker setup is experimental and not currently enforced by CI. It may break as dependencies or the build process evolve. + +**Step 1:** Generate VRL function docs (required once; only re-run when VRL function signatures change): + +```shell +make generate-vrl-docs +``` + +This generates CUE files into `cue/reference/remap/functions/` from the Rust source. These files are gitignored and must exist before the site can build. + +**Step 2:** Build and start the site: + +```shell +cd website +docker compose build +docker compose up +``` + +Navigate to http://localhost:1313. The site source is mounted as a volume so Hugo's live-reload works for Markdown, CSS, and JavaScript changes. If you modify [structured data](#structured-data) sources, restart the container. + ### Add a new version of Vector 1. Add the new version to the `versions` list in [`cue/reference/versions.cue`](./cue/reference/versions.cue). Make sure to preserve reverse ordering. diff --git a/website/docker-compose.yml b/website/docker-compose.yml new file mode 100644 index 0000000000000..2f58c9a12c815 --- /dev/null +++ b/website/docker-compose.yml @@ -0,0 +1,15 @@ +services: + website: + build: . + ports: + - "1313:1313" + volumes: + - .:/app/website + - ../Cargo.lock:/app/Cargo.lock:ro + - ../scripts:/app/scripts:ro + - website_node_modules:/app/website/node_modules + environment: + - CI=false + +volumes: + website_node_modules: From 74574a360112a4bb212915b896471320fb157b96 Mon Sep 17 00:00:00 2001 From: Thomas Date: Fri, 24 Apr 2026 14:50:53 -0400 Subject: [PATCH 159/326] chore(vdev): add linux arm64 to publish matrix and bump to 0.3.2 (#25260) --- .github/workflows/vdev_publish.yml | 2 ++ Cargo.lock | 2 +- scripts/environment/prepare.sh | 2 +- vdev/Cargo.toml | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/vdev_publish.yml b/.github/workflows/vdev_publish.yml index 3a46822098beb..80200d395f6ca 100644 --- a/.github/workflows/vdev_publish.yml +++ b/.github/workflows/vdev_publish.yml @@ -18,6 +18,8 @@ jobs: target: aarch64-apple-darwin - os: ubuntu-24.04 target: x86_64-unknown-linux-gnu + - os: ubuntu-24.04-arm + target: aarch64-unknown-linux-gnu steps: - name: Checkout Vector diff --git a/Cargo.lock b/Cargo.lock index 57f9a916e8812..16d4bb553fbce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12241,7 +12241,7 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] name = "vdev" -version = "0.3.1" +version = "0.3.2" dependencies = [ "anyhow", "cfg-if", diff --git a/scripts/environment/prepare.sh b/scripts/environment/prepare.sh index 02e100999c8de..b5c63c02f9d69 100755 --- a/scripts/environment/prepare.sh +++ b/scripts/environment/prepare.sh @@ -39,7 +39,7 @@ CARGO_LLVM_COV_VERSION="0.8.4" WASM_PACK_VERSION="0.13.1" # npm tool versions are defined in scripts/environment/npm-tools/package.json # and pinned (including transitive deps) in npm-tools/package-lock.json. -VDEV_VERSION="0.3.0" +VDEV_VERSION="0.3.2" ALL_MODULES=( rustup diff --git a/vdev/Cargo.toml b/vdev/Cargo.toml index 2a0f6169a160e..ff0265fddfc3e 100644 --- a/vdev/Cargo.toml +++ b/vdev/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "vdev" -version = "0.3.1" +version = "0.3.2" edition = "2024" authors = ["Vector Contributors "] license = "MPL-2.0" From 3b043d33b95670ef1cbd2f5af274fa25ebb1db84 Mon Sep 17 00:00:00 2001 From: Thomas Date: Fri, 24 Apr 2026 14:55:51 -0400 Subject: [PATCH 160/326] feat(greptimedb sink)!: add support for GreptimeDB v1.0.0 (#25178) * feat(greptimedb sink)!: upgrade greptimedb-ingester to published crate v0.16.0 * test(greptimedb): add latest to integration test version matrix * fix(greptimedb sink): reject unsupported gRPC compression types * refactor(greptimedb sink): make grpc_compression a typed enum * Format * Update licenses * Update generated docs * fix(greptimedb sink): fail early when TLS paths are missing * fix(greptimedb sink): handle partial TLS configs gracefully and document gzip removal * Update greptimedb-ingester and add back gzip support * Reject incomplete TLS configs rather than falling back to non-TLS * Match on all TlsConfig entries to emit warning log * Update generated docs --- Cargo.lock | 700 +++++++++++++----- Cargo.toml | 2 +- LICENSE-3rdparty.csv | 4 + .../greptimedb_ingester_crate.breaking.md | 3 + src/sinks/greptimedb/metrics/config.rs | 8 +- src/sinks/greptimedb/metrics/request.rs | 3 +- .../greptimedb/metrics/request_builder.rs | 6 +- src/sinks/greptimedb/metrics/service.rs | 116 +-- src/sinks/greptimedb/mod.rs | 14 + tests/integration/greptimedb/config/test.yaml | 4 +- .../components/sinks/generated/greptimedb.cue | 18 +- .../sinks/generated/greptimedb_metrics.cue | 18 +- 12 files changed, 623 insertions(+), 273 deletions(-) create mode 100644 changelog.d/greptimedb_ingester_crate.breaking.md diff --git a/Cargo.lock b/Cargo.lock index 16d4bb553fbce..0ad2d1635d855 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -343,18 +343,39 @@ version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e833808ff2d94ed40d9379848a950d995043c7fb3e81a30b383f4c6033821cc" dependencies = [ - "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-ipc", - "arrow-json", - "arrow-ord", - "arrow-row", - "arrow-schema", - "arrow-select", - "arrow-string", + "arrow-arith 56.2.0", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-cast 56.2.0", + "arrow-data 56.2.0", + "arrow-ipc 56.2.0", + "arrow-json 56.2.0", + "arrow-ord 56.2.0", + "arrow-row 56.2.0", + "arrow-schema 56.2.0", + "arrow-select 56.2.0", + "arrow-string 56.2.0", +] + +[[package]] +name = "arrow" +version = "58.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d441fdda254b65f3e9025910eb2c2066b6295d9c8ed409522b8d2ace1ff8574c" +dependencies = [ + "arrow-arith 58.1.0", + "arrow-array 58.1.0", + "arrow-buffer 58.1.0", + "arrow-cast 58.1.0", + "arrow-csv", + "arrow-data 58.1.0", + "arrow-ipc 58.1.0", + "arrow-json 58.1.0", + "arrow-ord 58.1.0", + "arrow-row 58.1.0", + "arrow-schema 58.1.0", + "arrow-select 58.1.0", + "arrow-string 58.1.0", ] [[package]] @@ -363,14 +384,28 @@ version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad08897b81588f60ba983e3ca39bda2b179bdd84dced378e7df81a5313802ef8" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", "chrono", "num", ] +[[package]] +name = "arrow-arith" +version = "58.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced5406f8b720cc0bc3aa9cf5758f93e8593cda5490677aa194e4b4b383f9a59" +dependencies = [ + "arrow-array 58.1.0", + "arrow-buffer 58.1.0", + "arrow-data 58.1.0", + "arrow-schema 58.1.0", + "chrono", + "num-traits", +] + [[package]] name = "arrow-array" version = "56.2.0" @@ -378,9 +413,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8548ca7c070d8db9ce7aa43f37393e4bfcf3f2d3681df278490772fd1673d08d" dependencies = [ "ahash 0.8.11", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", "chrono", "chrono-tz", "half", @@ -388,6 +423,25 @@ dependencies = [ "num", ] +[[package]] +name = "arrow-array" +version = "58.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "772bd34cacdda8baec9418d80d23d0fb4d50ef0735685bd45158b83dfeb6e62d" +dependencies = [ + "ahash 0.8.11", + "arrow-buffer 58.1.0", + "arrow-data 58.1.0", + "arrow-schema 58.1.0", + "chrono", + "chrono-tz", + "half", + "hashbrown 0.16.0", + "num-complex", + "num-integer", + "num-traits", +] + [[package]] name = "arrow-buffer" version = "56.2.0" @@ -399,17 +453,29 @@ dependencies = [ "num", ] +[[package]] +name = "arrow-buffer" +version = "58.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "898f4cf1e9598fdb77f356fdf2134feedfd0ee8d5a4e0a5f573e7d0aec16baa4" +dependencies = [ + "bytes", + "half", + "num-bigint", + "num-traits", +] + [[package]] name = "arrow-cast" version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "919418a0681298d3a77d1a315f625916cb5678ad0d74b9c60108eb15fd083023" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", + "arrow-select 56.2.0", "atoi", "base64 0.22.1", "chrono", @@ -419,30 +485,116 @@ dependencies = [ "ryu", ] +[[package]] +name = "arrow-cast" +version = "58.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0127816c96533d20fc938729f48c52d3e48f99717e7a0b5ade77d742510736d" +dependencies = [ + "arrow-array 58.1.0", + "arrow-buffer 58.1.0", + "arrow-data 58.1.0", + "arrow-ord 58.1.0", + "arrow-schema 58.1.0", + "arrow-select 58.1.0", + "atoi", + "base64 0.22.1", + "chrono", + "comfy-table", + "half", + "lexical-core", + "num-traits", + "ryu", +] + +[[package]] +name = "arrow-csv" +version = "58.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca025bd0f38eeecb57c2153c0123b960494138e6a957bbda10da2b25415209fe" +dependencies = [ + "arrow-array 58.1.0", + "arrow-cast 58.1.0", + "arrow-schema 58.1.0", + "chrono", + "csv", + "csv-core", + "regex", +] + [[package]] name = "arrow-data" version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a5c64fff1d142f833d78897a772f2e5b55b36cb3e6320376f0961ab0db7bd6d0" dependencies = [ - "arrow-buffer", - "arrow-schema", + "arrow-buffer 56.2.0", + "arrow-schema 56.2.0", "half", "num", ] +[[package]] +name = "arrow-data" +version = "58.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42d10beeab2b1c3bb0b53a00f7c944a178b622173a5c7bcabc3cb45d90238df4" +dependencies = [ + "arrow-buffer 58.1.0", + "arrow-schema 58.1.0", + "half", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-flight" +version = "58.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "302b2e036335f3f04d65dad3f74ff1f2aae6dc671d6aa04dc6b61193761e16fb" +dependencies = [ + "arrow-array 58.1.0", + "arrow-buffer 58.1.0", + "arrow-cast 58.1.0", + "arrow-ipc 58.1.0", + "arrow-schema 58.1.0", + "base64 0.22.1", + "bytes", + "futures", + "prost 0.14.3", + "prost-types 0.14.3", + "tonic 0.14.5", + "tonic-prost", +] + [[package]] name = "arrow-ipc" version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d3594dcddccc7f20fd069bc8e9828ce37220372680ff638c5e00dea427d88f5" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", - "flatbuffers", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", + "arrow-select 56.2.0", + "flatbuffers 25.9.23", +] + +[[package]] +name = "arrow-ipc" +version = "58.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "609a441080e338147a84e8e6904b6da482cefb957c5cdc0f3398872f69a315d0" +dependencies = [ + "arrow-array 58.1.0", + "arrow-buffer 58.1.0", + "arrow-data 58.1.0", + "arrow-schema 58.1.0", + "arrow-select 58.1.0", + "flatbuffers 25.9.23", + "lz4_flex 0.13.0", + "zstd", ] [[package]] @@ -451,11 +603,11 @@ version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "88cf36502b64a127dc659e3b305f1d993a544eab0d48cce704424e62074dc04b" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-schema", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-cast 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", "chrono", "half", "indexmap 2.12.0", @@ -467,17 +619,54 @@ dependencies = [ "simdutf8", ] +[[package]] +name = "arrow-json" +version = "58.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ead0914e4861a531be48fe05858265cf854a4880b9ed12618b1d08cba9bebc8" +dependencies = [ + "arrow-array 58.1.0", + "arrow-buffer 58.1.0", + "arrow-cast 58.1.0", + "arrow-data 58.1.0", + "arrow-schema 58.1.0", + "chrono", + "half", + "indexmap 2.12.0", + "itoa", + "lexical-core", + "memchr", + "num-traits", + "ryu", + "serde_core", + "serde_json", + "simdutf8", +] + [[package]] name = "arrow-ord" version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c8f82583eb4f8d84d4ee55fd1cb306720cddead7596edce95b50ee418edf66f" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", + "arrow-select 56.2.0", +] + +[[package]] +name = "arrow-ord" +version = "58.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "763a7ba279b20b52dad300e68cfc37c17efa65e68623169076855b3a9e941ca5" +dependencies = [ + "arrow-array 58.1.0", + "arrow-buffer 58.1.0", + "arrow-data 58.1.0", + "arrow-schema 58.1.0", + "arrow-select 58.1.0", ] [[package]] @@ -486,10 +675,23 @@ version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d07ba24522229d9085031df6b94605e0f4b26e099fb7cdeec37abd941a73753" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", + "half", +] + +[[package]] +name = "arrow-row" +version = "58.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14fe367802f16d7668163ff647830258e6e0aeea9a4d79aaedf273af3bdcd3e" +dependencies = [ + "arrow-array 58.1.0", + "arrow-buffer 58.1.0", + "arrow-data 58.1.0", + "arrow-schema 58.1.0", "half", ] @@ -499,6 +701,16 @@ version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3aa9e59c611ebc291c28582077ef25c97f1975383f1479b12f3b9ffee2ffabe" +[[package]] +name = "arrow-schema" +version = "58.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c30a1365d7a7dc50cc847e54154e6af49e4c4b0fddc9f607b687f29212082743" +dependencies = [ + "serde", + "serde_core", +] + [[package]] name = "arrow-select" version = "56.2.0" @@ -506,30 +718,61 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c41dbbd1e97bfcaee4fcb30e29105fb2c75e4d82ae4de70b792a5d3f66b2e7a" dependencies = [ "ahash 0.8.11", - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", "num", ] +[[package]] +name = "arrow-select" +version = "58.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78694888660a9e8ac949853db393af2a8b8fc82c19ce333132dfa2e72cc1a7fe" +dependencies = [ + "ahash 0.8.11", + "arrow-array 58.1.0", + "arrow-buffer 58.1.0", + "arrow-data 58.1.0", + "arrow-schema 58.1.0", + "num-traits", +] + [[package]] name = "arrow-string" version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53f5183c150fbc619eede22b861ea7c0eebed8eaac0333eaa7f6da5205fd504d" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", + "arrow-select 56.2.0", "memchr", "num", "regex", "regex-syntax", ] +[[package]] +name = "arrow-string" +version = "58.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61e04a01f8bb73ce54437514c5fd3ee2aa3e8abe4c777ee5cc55853b1652f79e" +dependencies = [ + "arrow-array 58.1.0", + "arrow-buffer 58.1.0", + "arrow-data 58.1.0", + "arrow-schema 58.1.0", + "arrow-select 58.1.0", + "memchr", + "num-traits", + "regex", + "regex-syntax", +] + [[package]] name = "ascii-canvas" version = "4.0.0" @@ -1548,7 +1791,7 @@ dependencies = [ "http-body 0.4.6", "hyper 0.14.32", "itoa", - "matchit", + "matchit 0.7.3", "memchr", "mime", "percent-encoding", @@ -1576,19 +1819,44 @@ dependencies = [ "http-body 1.0.1", "http-body-util", "itoa", - "matchit", + "matchit 0.7.3", "memchr", "mime", "percent-encoding", "pin-project-lite", "rustversion", "serde", - "sync_wrapper 1.0.1", + "sync_wrapper 1.0.2", "tower 0.4.13", "tower-layer", "tower-service", ] +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core 0.5.6", + "bytes", + "futures-util", + "http 1.3.1", + "http-body 1.0.1", + "http-body-util", + "itoa", + "matchit 0.8.4", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "sync_wrapper 1.0.2", + "tower 0.5.3", + "tower-layer", + "tower-service", +] + [[package]] name = "axum-core" version = "0.3.4" @@ -1621,7 +1889,25 @@ dependencies = [ "mime", "pin-project-lite", "rustversion", - "sync_wrapper 1.0.1", + "sync_wrapper 1.0.2", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http 1.3.1", + "http-body 1.0.1", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper 1.0.2", "tower-layer", "tower-service", ] @@ -1950,7 +2236,7 @@ checksum = "77e9d642a7e3a318e37c2c9427b5a6a48aa1ad55dcd986f3034ab2239045a645" dependencies = [ "darling 0.21.3", "ident_case", - "prettyplease 0.2.37", + "prettyplease", "proc-macro2 1.0.106", "quote 1.0.44", "rustversion", @@ -2425,7 +2711,7 @@ name = "codecs" version = "0.1.0" dependencies = [ "apache-avro 0.20.0", - "arrow", + "arrow 56.2.0", "async-trait", "bytes", "chrono", @@ -2515,6 +2801,16 @@ dependencies = [ "tokio-util", ] +[[package]] +name = "comfy-table" +version = "7.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "958c5d6ecf1f214b4c2bbbbf6ab9523a864bd136dcf71a7e8904799acfe1ad47" +dependencies = [ + "unicode-segmentation", + "unicode-width 0.2.0", +] + [[package]] name = "community-id" version = "0.2.3" @@ -2553,8 +2849,8 @@ dependencies = [ "compression-core", "flate2", "memchr", - "zstd 0.13.2", - "zstd-safe 7.2.1", + "zstd", + "zstd-safe", ] [[package]] @@ -4008,6 +4304,16 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" +[[package]] +name = "flatbuffers" +version = "24.12.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f1baf0dbf96932ec9a3038d57900329c015b0bfb7b63d904f3bc27e2b02a096" +dependencies = [ + "bitflags 1.3.2", + "rustc_version", +] + [[package]] name = "flatbuffers" version = "25.9.23" @@ -4408,37 +4714,54 @@ dependencies = [ [[package]] name = "greptime-proto" version = "0.1.0" -source = "git+https://github.com/GreptimeTeam/greptime-proto.git?tag=v0.9.0#396206c2801b5a3ec51bfe8984c66b686da910e6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25fc58f8ba94c488192d5928f24caf16c503fb83f4784f617e83febe08647b08" dependencies = [ - "prost 0.12.6", + "prost 0.14.3", + "prost-types 0.14.3", "serde", "serde_json", "strum 0.25.0", "strum_macros 0.25.3", - "tonic 0.11.0", - "tonic-build 0.11.0", + "tonic 0.14.5", + "tonic-prost", ] [[package]] name = "greptimedb-ingester" -version = "0.1.0" -source = "git+https://github.com/GreptimeTeam/greptimedb-ingester-rust?rev=f7243393808640f5123b0d5b7b798da591a4df6e#f7243393808640f5123b0d5b7b798da591a4df6e" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5ee8f8f967c2b8183136920bdb0a36d06618cc9402f901495c0da66cb2cc74b" dependencies = [ + "arrow 58.1.0", + "arrow-array 58.1.0", + "arrow-flight", + "arrow-ipc 58.1.0", + "arrow-schema 58.1.0", + "async-stream", + "async-trait", + "base64 0.22.1", "dashmap", "derive_builder", "enum_dispatch", + "flatbuffers 24.12.23", "futures", "futures-util", "greptime-proto", + "hyper 1.7.0", + "lazy_static", "parking_lot", - "prost 0.12.6", + "prost 0.14.3", "rand 0.9.4", + "serde", + "serde_json", "snafu 0.8.9", "tokio", "tokio-stream", - "tonic 0.11.0", - "tonic-build 0.9.2", - "tower 0.4.13", + "tokio-util", + "toml", + "tonic 0.14.5", + "tower 0.5.3", ] [[package]] @@ -6309,7 +6632,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "823ec7bedb1819b11633bd583ae981b0082db08492b0c3396412b85dd329ffee" dependencies = [ "cc", - "which 5.0.0", + "which", ] [[package]] @@ -6340,6 +6663,15 @@ dependencies = [ "twox-hash", ] +[[package]] +name = "lz4_flex" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db9a0d582c2874f68138a16ce1867e0ffde6c0bb0a0df85e1f36d04146db488a" +dependencies = [ + "twox-hash", +] + [[package]] name = "macaddr" version = "1.0.1" @@ -6433,6 +6765,12 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + [[package]] name = "matrixmultiply" version = "0.3.8" @@ -7065,9 +7403,9 @@ checksum = "63335b2e2c34fae2fb0aa2cecfd9f0832a1e24b3b32ecec612c3426d46dc8aaa" [[package]] name = "num-complex" -version = "0.4.4" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ba157ca0885411de85d6ca030ba7e2a83a28636056c7c699b07c8b6f7383214" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" dependencies = [ "num-traits", ] @@ -7437,7 +7775,7 @@ dependencies = [ "prost 0.12.6", "prost-build 0.12.6", "tonic 0.11.0", - "tonic-build 0.11.0", + "tonic-build", "vector-core", "vector-lookup", "vrl", @@ -7572,20 +7910,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0dbd48ad52d7dccf8ea1b90a3ddbfaea4f69878dd7683e51c507d4bc52b5b27" dependencies = [ "ahash 0.8.11", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-ipc", - "arrow-schema", - "arrow-select", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-cast 56.2.0", + "arrow-data 56.2.0", + "arrow-ipc 56.2.0", + "arrow-schema 56.2.0", + "arrow-select 56.2.0", "base64 0.22.1", "bytes", "chrono", "flate2", "half", "hashbrown 0.16.0", - "lz4_flex", + "lz4_flex 0.11.6", "num", "num-bigint", "paste", @@ -7593,7 +7931,7 @@ dependencies = [ "snap", "thrift", "twox-hash", - "zstd 0.13.2", + "zstd", ] [[package]] @@ -7996,16 +8334,6 @@ dependencies = [ "pad", ] -[[package]] -name = "prettyplease" -version = "0.1.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86" -dependencies = [ - "proc-macro2 1.0.106", - "syn 1.0.109", -] - [[package]] name = "prettyplease" version = "0.2.37" @@ -8173,16 +8501,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "prost" -version = "0.11.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" -dependencies = [ - "bytes", - "prost-derive 0.11.9", -] - [[package]] name = "prost" version = "0.12.6" @@ -8204,25 +8522,13 @@ dependencies = [ ] [[package]] -name = "prost-build" -version = "0.11.9" +name = "prost" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" +checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" dependencies = [ "bytes", - "heck 0.4.1", - "itertools 0.10.5", - "lazy_static", - "log", - "multimap", - "petgraph", - "prettyplease 0.1.25", - "prost 0.11.9", - "prost-types 0.11.9", - "regex", - "syn 1.0.109", - "tempfile", - "which 4.4.2", + "prost-derive 0.14.3", ] [[package]] @@ -8238,7 +8544,7 @@ dependencies = [ "multimap", "once_cell", "petgraph", - "prettyplease 0.2.37", + "prettyplease", "prost 0.12.6", "prost-types 0.12.6", "regex", @@ -8258,7 +8564,7 @@ dependencies = [ "multimap", "once_cell", "petgraph", - "prettyplease 0.2.37", + "prettyplease", "prost 0.13.5", "prost-types 0.13.5", "regex", @@ -8268,25 +8574,25 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.11.9" +version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" +checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.12.1", "proc-macro2 1.0.106", "quote 1.0.44", - "syn 1.0.109", + "syn 2.0.117", ] [[package]] name = "prost-derive" -version = "0.12.6" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.12.1", + "itertools 0.14.0", "proc-macro2 1.0.106", "quote 1.0.44", "syn 2.0.117", @@ -8294,9 +8600,9 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.13.5" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" dependencies = [ "anyhow", "itertools 0.14.0", @@ -8319,15 +8625,6 @@ dependencies = [ "serde-value", ] -[[package]] -name = "prost-types" -version = "0.11.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" -dependencies = [ - "prost 0.11.9", -] - [[package]] name = "prost-types" version = "0.12.6" @@ -8346,6 +8643,15 @@ dependencies = [ "prost 0.13.5", ] +[[package]] +name = "prost-types" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" +dependencies = [ + "prost 0.14.3", +] + [[package]] name = "psl" version = "2.1.22" @@ -8426,7 +8732,7 @@ dependencies = [ "tokio-util", "url", "uuid", - "zstd 0.13.2", + "zstd", ] [[package]] @@ -9129,7 +9435,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "sync_wrapper 1.0.1", + "sync_wrapper 1.0.2", "tokio", "tokio-native-tls", "tokio-rustls 0.26.2", @@ -10790,9 +11096,9 @@ checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" [[package]] name = "sync_wrapper" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" dependencies = [ "futures-core", ] @@ -11480,7 +11786,6 @@ dependencies = [ "tower-layer", "tower-service", "tracing 0.1.44", - "zstd 0.12.4", ] [[package]] @@ -11514,16 +11819,35 @@ dependencies = [ ] [[package]] -name = "tonic-build" -version = "0.9.2" +name = "tonic" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6fdaae4c2c638bb70fe42803a26fbd6fc6ac8c72f5c59f67ecc2a2dcabf4b07" +checksum = "fec7c61a0695dc1887c1b53952990f3ad2e3a31453e1f49f10e75424943a93ec" dependencies = [ - "prettyplease 0.1.25", - "proc-macro2 1.0.106", - "prost-build 0.11.9", - "quote 1.0.44", - "syn 1.0.109", + "async-trait", + "axum 0.8.9", + "base64 0.22.1", + "bytes", + "flate2", + "h2 0.4.13", + "http 1.3.1", + "http-body 1.0.1", + "http-body-util", + "hyper 1.7.0", + "hyper-timeout 0.5.1", + "hyper-util", + "percent-encoding", + "pin-project", + "socket2 0.6.0", + "sync_wrapper 1.0.2", + "tokio", + "tokio-rustls 0.26.2", + "tokio-stream", + "tower 0.5.3", + "tower-layer", + "tower-service", + "tracing 0.1.44", + "zstd", ] [[package]] @@ -11532,7 +11856,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be4ef6dd70a610078cb4e338a0f79d06bc759ff1b22d2120c2ff02ae264ba9c2" dependencies = [ - "prettyplease 0.2.37", + "prettyplease", "proc-macro2 1.0.106", "prost-build 0.12.6", "quote 1.0.44", @@ -11552,6 +11876,17 @@ dependencies = [ "tonic 0.11.0", ] +[[package]] +name = "tonic-prost" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a55376a0bbaa4975a3f10d009ad763d8f4108f067c7c2e74f3001fb49778d309" +dependencies = [ + "bytes", + "prost 0.14.3", + "tonic 0.14.5", +] + [[package]] name = "tonic-reflection" version = "0.11.0" @@ -11596,7 +11931,7 @@ dependencies = [ "indexmap 2.12.0", "pin-project-lite", "slab", - "sync_wrapper 1.0.1", + "sync_wrapper 1.0.2", "tokio", "tokio-util", "tower-layer", @@ -12281,8 +12616,8 @@ dependencies = [ "approx", "arc-swap", "arr_macro", - "arrow", - "arrow-schema", + "arrow 56.2.0", + "arrow-schema 56.2.0", "assert_cmd", "async-compression", "async-nats", @@ -12455,7 +12790,7 @@ dependencies = [ "tokio-util", "toml", "tonic 0.11.0", - "tonic-build 0.11.0", + "tonic-build", "tonic-health", "tonic-reflection", "tower 0.5.3", @@ -12482,7 +12817,7 @@ dependencies = [ "windows 0.58.0", "windows-service", "wiremock", - "zstd 0.13.2", + "zstd", ] [[package]] @@ -12498,7 +12833,7 @@ dependencies = [ "tokio", "tokio-stream", "tonic 0.11.0", - "tonic-build 0.11.0", + "tonic-build", "tonic-health", ] @@ -12962,7 +13297,7 @@ dependencies = [ "jsonschema", "lalrpop", "lalrpop-util", - "lz4_flex", + "lz4_flex 0.11.6", "md-5", "mlua", "nom 8.0.0", @@ -13023,7 +13358,7 @@ dependencies = [ "webbrowser", "woothee", "xxhash-rust", - "zstd 0.13.2", + "zstd", ] [[package]] @@ -13299,18 +13634,6 @@ dependencies = [ "rustls-pki-types", ] -[[package]] -name = "which" -version = "4.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" -dependencies = [ - "either", - "home", - "once_cell", - "rustix 0.38.40", -] - [[package]] name = "which" version = "5.0.0" @@ -13981,7 +14304,7 @@ dependencies = [ "anyhow", "heck 0.5.0", "indexmap 2.12.0", - "prettyplease 0.2.37", + "prettyplease", "syn 2.0.117", "wasm-metadata", "wit-bindgen-core", @@ -13995,7 +14318,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" dependencies = [ "anyhow", - "prettyplease 0.2.37", + "prettyplease", "proc-macro2 1.0.106", "quote 1.0.44", "syn 2.0.117", @@ -14202,32 +14525,13 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7948af682ccbc3342b6e9420e8c51c1fe5d7bf7756002b4a3c6cabfe96a7e3c" -[[package]] -name = "zstd" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a27595e173641171fc74a1232b7b1c7a7cb6e18222c11e9dfb9888fa424c53c" -dependencies = [ - "zstd-safe 6.0.6", -] - [[package]] name = "zstd" version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9" dependencies = [ - "zstd-safe 7.2.1", -] - -[[package]] -name = "zstd-safe" -version = "6.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee98ffd0b48ee95e6c5168188e44a54550b1564d9d530ee21d5f0eaed1069581" -dependencies = [ - "libc", - "zstd-sys", + "zstd-safe", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 1045e07aae1a1..70f9fc12a8acc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -345,7 +345,7 @@ deadpool = { version = "0.13.0", default-features = false, features = ["managed" hex = { version = "0.4.3", default-features = false, optional = true } # GreptimeDB -greptimedb-ingester = { git = "https://github.com/GreptimeTeam/greptimedb-ingester-rust", rev = "f7243393808640f5123b0d5b7b798da591a4df6e", optional = true } +greptimedb-ingester = { version = "0.17.0", default-features = false, optional = true } # External libs arc-swap = { workspace = true, default-features = false, optional = true } diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index eb9e179e57c2b..ea2ff8e85d12e 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -32,7 +32,9 @@ arrow-arith,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow arrow-buffer,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow arrow-cast,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow +arrow-csv,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow arrow-data,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow +arrow-flight,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow arrow-ipc,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow arrow-json,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow arrow-ord,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow @@ -165,6 +167,7 @@ codespan-reporting,https://github.com/brendanzab/codespan,Apache-2.0,Brendan Zab colorchoice,https://github.com/rust-cli/anstyle,MIT OR Apache-2.0,The colorchoice Authors colored,https://github.com/mackwic/colored,MPL-2.0,Thomas Wickham combine,https://github.com/Marwes/combine,MIT,Markus Westerlind +comfy-table,https://github.com/nukesor/comfy-table,MIT,Arne Beer community-id,https://github.com/traceflight/rs-community-id,MIT OR Apache-2.0,Julian Wang compact_str,https://github.com/ParkMyCar/compact_str,MIT,Parker Timmerman compression-codecs,https://github.com/Nullus157/async-compression,MIT OR Apache-2.0,"Wim Looman , Allen Bui " @@ -789,6 +792,7 @@ toml_write,https://github.com/toml-rs/toml,MIT OR Apache-2.0,The toml_write Auth toml_writer,https://github.com/toml-rs/toml,MIT OR Apache-2.0,The toml_writer Authors tonic,https://github.com/hyperium/tonic,MIT,Lucio Franco tonic-health,https://github.com/hyperium/tonic,MIT,James Nugent +tonic-prost,https://github.com/hyperium/tonic,MIT,Lucio Franco tonic-reflection,https://github.com/hyperium/tonic,MIT,"James Nugent , Samani G. Gikandi " tower,https://github.com/tower-rs/tower,MIT,Tower Maintainers tower-http,https://github.com/tower-rs/tower-http,MIT,Tower Maintainers diff --git a/changelog.d/greptimedb_ingester_crate.breaking.md b/changelog.d/greptimedb_ingester_crate.breaking.md new file mode 100644 index 0000000000000..79984693ef8e1 --- /dev/null +++ b/changelog.d/greptimedb_ingester_crate.breaking.md @@ -0,0 +1,3 @@ +The `greptimedb_metrics` and `greptimedb_logs` sinks now require GreptimeDB v1.x. Users running GreptimeDB v0.x must upgrade their GreptimeDB instance before upgrading Vector. + +authors: thomasqueirozb diff --git a/src/sinks/greptimedb/metrics/config.rs b/src/sinks/greptimedb/metrics/config.rs index 4261486a05706..8f91392325698 100644 --- a/src/sinks/greptimedb/metrics/config.rs +++ b/src/sinks/greptimedb/metrics/config.rs @@ -2,7 +2,7 @@ use vector_lib::{configurable::configurable_component, sensitive_string::Sensiti use crate::sinks::{ greptimedb::{ - GreptimeDBDefaultBatchSettings, default_dbname, + GreptimeDBDefaultBatchSettings, GrpcCompression, default_dbname, metrics::{ request::GreptimeDBGrpcRetryLogic, request_builder::RequestBuilderOptions, @@ -85,11 +85,9 @@ pub struct GreptimeDBMetricsConfig { #[configurable(metadata(docs::examples = "password"))] #[serde(default)] pub password: Option, - /// Set gRPC compression encoding for the request - /// Default to none, `gzip` or `zstd` is supported. - #[configurable(metadata(docs::examples = "gzip"))] + /// Set gRPC compression encoding for the request. #[serde(default)] - pub grpc_compression: Option, + pub grpc_compression: GrpcCompression, #[configurable(derived)] #[serde(default)] diff --git a/src/sinks/greptimedb/metrics/request.rs b/src/sinks/greptimedb/metrics/request.rs index eedfc6f26f2d8..68767844412dc 100644 --- a/src/sinks/greptimedb/metrics/request.rs +++ b/src/sinks/greptimedb/metrics/request.rs @@ -1,6 +1,7 @@ use std::num::NonZeroUsize; -use greptimedb_ingester::{Error as GreptimeError, api::v1::*}; +use greptimedb_ingester::Error as GreptimeError; +use greptimedb_ingester::api::v1::RowInsertRequests; use vector_lib::event::Metric; use crate::sinks::{ diff --git a/src/sinks/greptimedb/metrics/request_builder.rs b/src/sinks/greptimedb/metrics/request_builder.rs index 1980d67e6a78c..e596ae06e0cee 100644 --- a/src/sinks/greptimedb/metrics/request_builder.rs +++ b/src/sinks/greptimedb/metrics/request_builder.rs @@ -1,5 +1,8 @@ use chrono::Utc; -use greptimedb_ingester::{api::v1::*, helpers::values::*}; +use greptimedb_ingester::api::v1::{ + ColumnDataType, ColumnSchema, Row, RowInsertRequest, Rows, SemanticType, Value, +}; +use greptimedb_ingester::helpers::values::{f64_value, string_value, timestamp_millisecond_value}; use vector_lib::{ event::{ Metric, MetricValue, @@ -238,6 +241,7 @@ fn tag_column(name: &str) -> ColumnSchema { #[cfg(test)] mod tests { + use greptimedb_ingester::api::v1::value; use similar_asserts::assert_eq; use super::*; diff --git a/src/sinks/greptimedb/metrics/service.rs b/src/sinks/greptimedb/metrics/service.rs index 77ac61593302a..e3bdc4a07b1b6 100644 --- a/src/sinks/greptimedb/metrics/service.rs +++ b/src/sinks/greptimedb/metrics/service.rs @@ -1,16 +1,22 @@ use std::{sync::Arc, task::Poll}; use greptimedb_ingester::{ - Client, ClientBuilder, Compression, Database, Error as GreptimeError, - api::v1::{auth_header::AuthScheme, *}, - channel_manager::*, + Error as GreptimeError, GrpcCompression as IngesterGrpcCompression, + api::v1::Basic, + api::v1::auth_header::AuthScheme, + channel_manager::{ChannelConfig, ChannelManager, ClientTlsOption}, + client::Client, + database::Database, }; use vector_lib::sensitive_string::SensitiveString; use crate::sinks::{ - greptimedb::metrics::{ - config::GreptimeDBMetricsConfig, - request::{GreptimeDBGrpcBatchOutput, GreptimeDBGrpcRequest}, + greptimedb::{ + GrpcCompression, + metrics::{ + config::GreptimeDBMetricsConfig, + request::{GreptimeDBGrpcBatchOutput, GreptimeDBGrpcRequest}, + }, }, prelude::*, }; @@ -22,49 +28,63 @@ pub struct GreptimeDBGrpcService { } fn new_client_from_config(config: &GreptimeDBGrpcServiceConfig) -> crate::Result { - let mut builder = ClientBuilder::default().peers(vec![&config.endpoint]); + let send_compression_encoding = match config.compression { + GrpcCompression::None => None, + GrpcCompression::Gzip => Some(IngesterGrpcCompression::Gzip), + GrpcCompression::Zstd => Some(IngesterGrpcCompression::Zstd), + }; + + let mut channel_config = ChannelConfig { + send_compression_encoding, + ..Default::default() + }; + + let channel_manager = if let Some(tls_config) = &config.tls { + let TlsConfig { + verify_certificate, + verify_hostname, + alpn_protocols, + ca_file, + crt_file, + key_file, + key_pass, + server_name, + } = tls_config; + + if verify_certificate.is_some() + || verify_hostname.is_some() + || alpn_protocols.is_some() + || key_pass.is_some() + || server_name.is_some() + { + warn!( + message = "TlsConfig: verify_certificate, verify_hostname, alpn_protocols, key_pass and server_name are not supported by greptimedb client at the moment." + ); + } - if let Some(compression) = config.compression.as_ref() { - let compression = match compression.as_str() { - "gzip" => Compression::Gzip, - "zstd" => Compression::Zstd, + // The greptimedb ingester requires all three TLS paths (ca_file, crt_file, + // key_file) to be set. Refuse a partial config rather than downgrading to plaintext. + match (ca_file, crt_file, key_file) { + (Some(ca), Some(crt), Some(key)) => { + channel_config.client_tls = Some(ClientTlsOption { + server_ca_cert_path: ca.to_string_lossy().into_owned(), + client_cert_path: crt.to_string_lossy().into_owned(), + client_key_path: key.to_string_lossy().into_owned(), + }); + ChannelManager::with_tls_config(channel_config).map_err(Box::new)? + } _ => { - warn!(message = "Unknown gRPC compression type: {compression}, disabled."); - Compression::None + return Err( + "GreptimeDB TLS requires ca_file, crt_file, and key_file to all be set.".into(), + ); } - }; - builder = builder.compression(compression); - } - - if let Some(tls_config) = &config.tls { - let channel_config = ChannelConfig { - client_tls: Some(try_from_tls_config(tls_config)?), - ..Default::default() - }; - - builder = builder - .channel_manager(ChannelManager::with_tls_config(channel_config).map_err(Box::new)?); - } - - Ok(builder.build()) -} - -fn try_from_tls_config(tls_config: &TlsConfig) -> crate::Result { - if tls_config.key_pass.is_some() - || tls_config.alpn_protocols.is_some() - || tls_config.verify_certificate.is_some() - || tls_config.verify_hostname.is_some() - { - warn!( - message = "TlsConfig: key_pass, alpn_protocols, verify_certificate and verify_hostname are not supported by greptimedb client at the moment." - ); - } + } + } else { + ChannelManager::with_config(channel_config) + }; + let client = Client::with_manager_and_urls(channel_manager, vec![&config.endpoint]); - Ok(ClientTlsOption { - server_ca_cert_path: tls_config.ca_file.clone(), - client_cert_path: tls_config.crt_file.clone(), - client_key_path: tls_config.key_file.clone(), - }) + Ok(client) } impl GreptimeDBGrpcService { @@ -103,7 +123,7 @@ impl Service for GreptimeDBGrpcService { Box::pin(async move { let metadata = req.metadata; - let result = client.row_insert(req.items).await?; + let result = client.insert(req.items).await?; Ok(GreptimeDBGrpcBatchOutput { _item_count: result, @@ -119,7 +139,7 @@ pub(super) struct GreptimeDBGrpcServiceConfig { dbname: String, username: Option, password: Option, - compression: Option, + compression: GrpcCompression, tls: Option, } @@ -130,7 +150,7 @@ impl From<&GreptimeDBMetricsConfig> for GreptimeDBGrpcServiceConfig { dbname: val.dbname.clone(), username: val.username.clone(), password: val.password.clone(), - compression: val.grpc_compression.clone(), + compression: val.grpc_compression, tls: val.tls.clone(), } } diff --git a/src/sinks/greptimedb/mod.rs b/src/sinks/greptimedb/mod.rs index 8116681652dc2..5369992e21c46 100644 --- a/src/sinks/greptimedb/mod.rs +++ b/src/sinks/greptimedb/mod.rs @@ -4,6 +4,20 @@ use crate::sinks::prelude::*; mod logs; mod metrics; +/// Compression algorithm for gRPC requests to GreptimeDB. +#[configurable_component] +#[derive(Clone, Copy, Debug, Default)] +#[serde(rename_all = "lowercase")] +enum GrpcCompression { + /// No compression. + #[default] + None, + /// Gzip compression. + Gzip, + /// Zstandard compression. + Zstd, +} + fn default_dbname() -> String { greptimedb_ingester::DEFAULT_SCHEMA_NAME.to_string() } diff --git a/tests/integration/greptimedb/config/test.yaml b/tests/integration/greptimedb/config/test.yaml index 2ec3984938617..8a79569ac06fa 100644 --- a/tests/integration/greptimedb/config/test.yaml +++ b/tests/integration/greptimedb/config/test.yaml @@ -10,9 +10,7 @@ runner: no_proxy: greptimedb matrix: - # Temporarily pegging to the latest known stable release - # since using `latest` is failing consistently. - version: [v0.13.2] + version: [v1.0.0, latest] # changes to these files/paths will invoke the integration test in CI # expressions are evaluated using https://github.com/micromatch/picomatch diff --git a/website/cue/reference/components/sinks/generated/greptimedb.cue b/website/cue/reference/components/sinks/generated/greptimedb.cue index 90cbf9c459d62..27abfd61e2960 100644 --- a/website/cue/reference/components/sinks/generated/greptimedb.cue +++ b/website/cue/reference/components/sinks/generated/greptimedb.cue @@ -92,14 +92,16 @@ generated: components: sinks: greptimedb: configuration: { type: string: examples: ["example.com:4001"] } grpc_compression: { - description: """ - Set gRPC compression encoding for the request - Default to none, `gzip` or `zstd` is supported. - """ - required: false - type: string: examples: [ - "gzip", - ] + description: "Set gRPC compression encoding for the request." + required: false + type: string: { + default: "none" + enum: { + gzip: "Gzip compression." + none: "No compression." + zstd: "Zstandard compression." + } + } } new_naming: { description: """ diff --git a/website/cue/reference/components/sinks/generated/greptimedb_metrics.cue b/website/cue/reference/components/sinks/generated/greptimedb_metrics.cue index aea1d76a0eb46..eb262d81fbea5 100644 --- a/website/cue/reference/components/sinks/generated/greptimedb_metrics.cue +++ b/website/cue/reference/components/sinks/generated/greptimedb_metrics.cue @@ -92,14 +92,16 @@ generated: components: sinks: greptimedb_metrics: configuration: { type: string: examples: ["example.com:4001"] } grpc_compression: { - description: """ - Set gRPC compression encoding for the request - Default to none, `gzip` or `zstd` is supported. - """ - required: false - type: string: examples: [ - "gzip", - ] + description: "Set gRPC compression encoding for the request." + required: false + type: string: { + default: "none" + enum: { + gzip: "Gzip compression." + none: "No compression." + zstd: "Zstandard compression." + } + } } new_naming: { description: """ From 61cc4d84d2966e7552a308d5df5a0f03305d7eed Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 27 Apr 2026 09:40:18 -0400 Subject: [PATCH 161/326] fix(ci): make publish-s3 wait for generate-sha256sum (#25265) --- .github/workflows/publish.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index d5093f229017c..5ccb5ea82c968 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -382,6 +382,7 @@ jobs: - deb-verify - rpm-verify - macos-verify + - generate-sha256sum env: VECTOR_VERSION: ${{ needs.generate-publish-metadata.outputs.vector_version }} CHANNEL: ${{ needs.generate-publish-metadata.outputs.vector_release_channel }} From 454b096abcfd609b4e317b83fa4ec1ec79af5da8 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Mon, 27 Apr 2026 10:14:28 -0400 Subject: [PATCH 162/326] docs(deprecations): add datadog_metrics series v1 deprecation entry (#25271) Co-authored-by: Claude Opus 4.7 (1M context) --- docs/DEPRECATIONS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/DEPRECATIONS.md b/docs/DEPRECATIONS.md index b57db75f736af..9c36ff3eb7859 100644 --- a/docs/DEPRECATIONS.md +++ b/docs/DEPRECATIONS.md @@ -17,6 +17,7 @@ For example: - `v0.50.0` | `http-server-encoding` | The `encoding` field will be removed. Use `decoding` and `framing` instead. - `v0.53.0` | `buffer-bytes-events-metrics` | The `buffer_byte_size` and `buffer_events` gauges are deprecated in favor of the `buffer_size_bytes`/`buffer_size_events` metrics described in `docs/specs/buffer.md`. - `v0.58.0` | `azure-monitor-logs-sink` | The `azure_monitor_logs` sink is deprecated in favor of the new `azure_logs_ingestion` sink, which uses the Azure Monitor Logs Ingestion API. Users should migrate before Microsoft ends support for the old Data Collector API (scheduled for September 2026). +- `v0.58.0` | `datadog-metrics-series-v1` | The `series_api_version: v1` option on the `datadog_metrics` sink is deprecated in favor of `v2` (the default). The v1 series endpoint (`/api/v1/series`) is a legacy endpoint; users should remove `series_api_version: v1` from their configuration or set it to `v2`. ## To be migrated From aa9265d6026f7d5998cada6a50e9c66790fbf054 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 27 Apr 2026 10:47:04 -0400 Subject: [PATCH 163/326] chore(deps): add dependabot config for scripts/environment/npm-tools (#25175) * chore(deps): add dependabot config for scripts/environment/npm-tools * Set open-pull-requests-limit: 0 --- .github/dependabot.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 609da972baa98..ade867f95145f 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -176,3 +176,19 @@ updates: npm_and_yarn: patterns: - "*" + - package-ecosystem: "npm" + directory: "/scripts/environment/npm-tools" + schedule: + interval: "monthly" + time: "05:00" # UTC + labels: + - "domain: deps" + - "no-changelog" + commit-message: + prefix: "chore(deps)" + # Disable version updates; security updates bypass this limit + open-pull-requests-limit: 0 + groups: + npm_and_yarn: + patterns: + - "*" From 69fd3b6f7e544a3f4bbba1d6d7e882a67387b081 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Apr 2026 10:56:51 -0400 Subject: [PATCH 164/326] chore(ci): bump github/codeql-action from 4.32.4 to 4.35.2 (#25280) Bumps [github/codeql-action](https://github.com/github/codeql-action) from 4.32.4 to 4.35.2. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/89a39a4e59826350b863aa6b6252a07ad50cf83e...95e58e9a2cdfd71adc6e0353d5c52f41a045d225) --- updated-dependencies: - dependency-name: github/codeql-action dependency-version: 4.35.2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/scorecard.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 7a8fd8c9a7d47..391406f824e27 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -68,6 +68,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard (optional). # Commenting out will disable upload of results to your repo's Code Scanning dashboard - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@89a39a4e59826350b863aa6b6252a07ad50cf83e # v4.32.4 + uses: github/codeql-action/upload-sarif@95e58e9a2cdfd71adc6e0353d5c52f41a045d225 # v4.35.2 with: sarif_file: results.sarif From 413caac10f64475b509687000317e06e55de55e0 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 27 Apr 2026 11:18:47 -0400 Subject: [PATCH 165/326] fix(ci): allow dd-token federation to fail on fork PRs (#25284) --- .github/actions/dd-token/action.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/actions/dd-token/action.yml b/.github/actions/dd-token/action.yml index 8ea92dc1c2dd5..f2b6b32eb0a16 100644 --- a/.github/actions/dd-token/action.yml +++ b/.github/actions/dd-token/action.yml @@ -10,8 +10,11 @@ inputs: runs: using: "composite" steps: + # Fork PRs don't get `id-token: write`, so federation cannot succeed. + # Allow the step to fail and let downstream steps run without DD credentials. - name: Federate Datadog token id: dd-sts + continue-on-error: true uses: DataDog/dd-sts-action@2e8187910199bd93129520183c093e19aa585c75 with: policy: ${{ inputs.policy }} @@ -22,7 +25,9 @@ runs: DD_STS_API_KEY: ${{ steps.dd-sts.outputs.api_key }} DD_STS_APP_KEY: ${{ steps.dd-sts.outputs.app_key }} run: | - echo "DD_API_KEY=${DD_STS_API_KEY}" >> "$GITHUB_ENV" + if [ -n "${DD_STS_API_KEY}" ]; then + echo "DD_API_KEY=${DD_STS_API_KEY}" >> "$GITHUB_ENV" + fi if [ -n "${DD_STS_APP_KEY}" ]; then echo "DD_APP_KEY=${DD_STS_APP_KEY}" >> "$GITHUB_ENV" fi From ba84da5d8965aef1d5eb5455dc9f4d723177e914 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Apr 2026 11:01:24 -0400 Subject: [PATCH 166/326] chore(ci): bump actions/github-script from 7 to 9 (#25278) * chore(ci): bump actions/github-script from 7 to 9 Bumps [actions/github-script](https://github.com/actions/github-script) from 7 to 9. - [Release notes](https://github.com/actions/github-script/releases) - [Commits](https://github.com/actions/github-script/compare/v7...v9) --- updated-dependencies: - dependency-name: actions/github-script dependency-version: '9' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] * Use pinned github-script sha --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Thomas --- .github/workflows/add_wip_label.yml | 2 +- .github/workflows/check_generated_vrl_docs_comment.yml | 2 +- .github/workflows/create_preview_sites.yml | 10 +++++----- .github/workflows/k8s_e2e.yml | 2 +- .github/workflows/preview_site_trigger.yml | 2 +- .github/workflows/remove_wip_label.yml | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/add_wip_label.yml b/.github/workflows/add_wip_label.yml index 9a1260b693e81..7924d4fade926 100644 --- a/.github/workflows/add_wip_label.yml +++ b/.github/workflows/add_wip_label.yml @@ -19,7 +19,7 @@ jobs: steps: - name: Check member review state id: check - uses: actions/github-script@v7 + uses: actions/github-script@v9 with: script: | const { data: reviews } = await github.rest.pulls.listReviews({ diff --git a/.github/workflows/check_generated_vrl_docs_comment.yml b/.github/workflows/check_generated_vrl_docs_comment.yml index e455203f75a6b..faa429af4de9b 100644 --- a/.github/workflows/check_generated_vrl_docs_comment.yml +++ b/.github/workflows/check_generated_vrl_docs_comment.yml @@ -26,7 +26,7 @@ jobs: github-token: ${{ github.token }} - name: Comment on PR - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | const fs = require('fs'); diff --git a/.github/workflows/create_preview_sites.yml b/.github/workflows/create_preview_sites.yml index 6cd7edf89bdb9..282a012192400 100644 --- a/.github/workflows/create_preview_sites.yml +++ b/.github/workflows/create_preview_sites.yml @@ -38,7 +38,7 @@ jobs: steps: # Get the artifacts with the PR number and branch name - name: Download artifact - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | const fs = require('fs'); @@ -58,7 +58,7 @@ jobs: # Extract the info from the artifact and set variables - name: Extract PR info from artifact - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | const fs = require('fs'); @@ -85,7 +85,7 @@ jobs: # Validate the integrity of the artifact - name: Validate Artifact Integrity - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | const crypto = require('crypto'); @@ -104,7 +104,7 @@ jobs: # Kick off the job in amplify - name: Deploy Site - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: APP_ID: ${{ inputs.APP_ID }} APP_NAME: ${{ inputs.APP_NAME }} @@ -171,7 +171,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} APP_ID: ${{ inputs.APP_ID }} APP_NAME: ${{ inputs.APP_NAME }} - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | const fs = require('fs'); diff --git a/.github/workflows/k8s_e2e.yml b/.github/workflows/k8s_e2e.yml index 3b35ec0b7b9dc..08aea2e9c3c09 100644 --- a/.github/workflows/k8s_e2e.yml +++ b/.github/workflows/k8s_e2e.yml @@ -116,7 +116,7 @@ jobs: outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} steps: - - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 id: set-matrix with: script: | diff --git a/.github/workflows/preview_site_trigger.yml b/.github/workflows/preview_site_trigger.yml index 98c5a6a2df720..fb506c89ead29 100644 --- a/.github/workflows/preview_site_trigger.yml +++ b/.github/workflows/preview_site_trigger.yml @@ -32,7 +32,7 @@ jobs: # Save PR information (only if branch is valid) - name: Validate and save PR information if: steps.validate.outputs.valid == 'true' - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | const fs = require('fs').promises; diff --git a/.github/workflows/remove_wip_label.yml b/.github/workflows/remove_wip_label.yml index 46d6e79cf1ad8..7c30ee892a949 100644 --- a/.github/workflows/remove_wip_label.yml +++ b/.github/workflows/remove_wip_label.yml @@ -14,7 +14,7 @@ jobs: timeout-minutes: 5 steps: - name: Check association, label, and remove - uses: actions/github-script@v7 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | const association = context.payload.review.author_association; From a4ea6de53b829e6ec66fadaddd78317f099207b9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Apr 2026 11:34:31 -0400 Subject: [PATCH 167/326] chore(ci): bump actions/upload-artifact from 7.0.0 to 7.0.1 in the artifact group across 1 directory (#25276) chore(ci): bump actions/upload-artifact Bumps the artifact group with 1 update in the / directory: [actions/upload-artifact](https://github.com/actions/upload-artifact). Updates `actions/upload-artifact` from 7.0.0 to 7.0.1 - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/bbbca2ddaa5d8feaa63e36b76fdaad77386f024f...043fb46d1a93c77aae656e7c1c64a875d1fc6a0a) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-version: 7.0.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: artifact ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/changes.yml | 4 ++-- .github/workflows/check_generated_vrl_docs.yml | 2 +- .github/workflows/cross.yml | 2 +- .github/workflows/k8s_e2e.yml | 2 +- .github/workflows/preview_site_trigger.yml | 2 +- .github/workflows/publish.yml | 8 ++++---- .github/workflows/regression.yml | 8 ++++---- .github/workflows/scorecard.yml | 2 +- 8 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/changes.yml b/.github/workflows/changes.yml index b07447715d75f..710932c694bc2 100644 --- a/.github/workflows/changes.yml +++ b/.github/workflows/changes.yml @@ -422,7 +422,7 @@ jobs: echo "any=$any_changed" >> $GITHUB_OUTPUT - name: Upload JSON artifact - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: int_tests_changes path: int_tests_changes.json @@ -484,7 +484,7 @@ jobs: echo "any=$any_changed" >> $GITHUB_OUTPUT - name: Upload JSON artifact - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: e2e_tests_changes path: e2e_tests_changes.json diff --git a/.github/workflows/check_generated_vrl_docs.yml b/.github/workflows/check_generated_vrl_docs.yml index 867ffd7619379..4d30b4f8d5bd2 100644 --- a/.github/workflows/check_generated_vrl_docs.yml +++ b/.github/workflows/check_generated_vrl_docs.yml @@ -114,7 +114,7 @@ jobs: && steps.last-commit.outputs.is-auto != 'true' && github.event_name == 'pull_request' && steps.push.outcome != 'success' - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: vrl-docs-check-pr path: /tmp/docs-check/pr-number diff --git a/.github/workflows/cross.yml b/.github/workflows/cross.yml index b554a45d45c09..bc2b2b559503b 100644 --- a/.github/workflows/cross.yml +++ b/.github/workflows/cross.yml @@ -55,7 +55,7 @@ jobs: # aarch64 and musl in particular are notoriously hard to link. # While it may be tempting to slot a `check` in here for quickness, please don't. - run: make cross-build-${{ matrix.target }} - - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: "vector-debug-${{ matrix.target }}" path: "./target/${{ matrix.target }}/debug/vector" diff --git a/.github/workflows/k8s_e2e.yml b/.github/workflows/k8s_e2e.yml index 08aea2e9c3c09..6b65985cc6eb6 100644 --- a/.github/workflows/k8s_e2e.yml +++ b/.github/workflows/k8s_e2e.yml @@ -95,7 +95,7 @@ jobs: - run: VECTOR_VERSION="$(vdev version)" make package-deb-x86_64-unknown-linux-gnu - - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: e2e-test-deb-package path: target/artifacts/* diff --git a/.github/workflows/preview_site_trigger.yml b/.github/workflows/preview_site_trigger.yml index fb506c89ead29..db74832c36d60 100644 --- a/.github/workflows/preview_site_trigger.yml +++ b/.github/workflows/preview_site_trigger.yml @@ -54,7 +54,7 @@ jobs: # Upload the artifact using latest version (only if branch is valid) - name: Upload PR information artifact if: steps.validate.outputs.valid == 'true' - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: pr path: pr/ diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 5ccb5ea82c968..4f82bd06e3894 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -81,7 +81,7 @@ jobs: - name: Build Vector run: make package-${{ matrix.target }}-all - name: Stage package artifacts for publish - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: vector-${{ env.VECTOR_VERSION }}-${{ matrix.target }} path: target/artifacts/vector* @@ -129,7 +129,7 @@ jobs: export PATH="$HOME/.cargo/bin:$PATH" make package - name: Stage package artifacts for publish - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: vector-${{ env.VECTOR_VERSION }}-${{ matrix.architecture }}-apple-darwin path: target/artifacts/vector* @@ -176,7 +176,7 @@ jobs: export PATH="/c/wix:$PATH" ./scripts/package-msi.sh - name: Stage package artifacts for publish - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: vector-${{ env.VECTOR_VERSION }}-x86_64-pc-windows-msvc path: target/artifacts/vector* @@ -470,7 +470,7 @@ jobs: - name: Generate SHA256 checksums for artifacts run: make sha256sum - name: Stage checksum for publish - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: vector-${{ env.VECTOR_VERSION }}-SHA256SUMS path: target/artifacts/vector-${{ env.VECTOR_VERSION }}-SHA256SUMS diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index aaf71eb9687a4..94b47333a8ed8 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -223,7 +223,7 @@ jobs: vector:${{ needs.resolve-inputs.outputs.baseline-tag }} - name: Upload image as artifact - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: baseline-image path: "${{ runner.temp }}/baseline-image.tar" @@ -266,7 +266,7 @@ jobs: vector:${{ needs.resolve-inputs.outputs.comparison-tag }} - name: Upload image as artifact - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: comparison-image path: "${{ runner.temp }}/comparison-image.tar" @@ -433,7 +433,7 @@ jobs: --submission-metadata ${{ runner.temp }}/submission-metadata \ --replicas ${{ env.SMP_REPLICAS }} - - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: vector-submission-metadata path: ${{ runner.temp }}/submission-metadata @@ -557,7 +557,7 @@ jobs: path: ${{ runner.temp }}/outputs/report.md - name: Upload regression report to artifacts - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: capture-artifacts path: ${{ runner.temp }}/outputs/* diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 391406f824e27..9e64b64ccfabd 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -59,7 +59,7 @@ jobs: # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF # format to the repository Actions tab. - name: "Upload artifact" - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: SARIF file path: results.sarif From 1d4b6c2122b82997b808bba0527ec60dbb37a3ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josu=C3=A9=20AGBEKODO?= Date: Mon, 27 Apr 2026 17:36:38 +0200 Subject: [PATCH 168/326] fix(splunk_hec source): Emit warn on unauthenticated request (#25230) * fix(splunk_hec source): Emit warn on unauthenticated request * add changelog * Address comments * fix(splunk_hec source): use authentication_failed error type for unauthenticated requests Add a new `authentication_failed` error_type constant so consumers can filter auth failures from other request errors in `component_errors_total`. * Update changelog.md --------- Co-authored-by: Thomas --- ...plunk_hec_source_specify_wrong_auth.fix.md | 3 ++ .../src/internal_event/prelude.rs | 2 + src/internal_events/splunk_hec.rs | 42 +++++++++++++------ 3 files changed, 35 insertions(+), 12 deletions(-) create mode 100644 changelog.d/25230_splunk_hec_source_specify_wrong_auth.fix.md diff --git a/changelog.d/25230_splunk_hec_source_specify_wrong_auth.fix.md b/changelog.d/25230_splunk_hec_source_specify_wrong_auth.fix.md new file mode 100644 index 0000000000000..8c2af5dd212e4 --- /dev/null +++ b/changelog.d/25230_splunk_hec_source_specify_wrong_auth.fix.md @@ -0,0 +1,3 @@ +The error log + metric that `splunk_hec` source emit on missing/invalid auth header now specifies "authentication_failed" as error_type. + +authors: 20agbekodo diff --git a/lib/vector-common/src/internal_event/prelude.rs b/lib/vector-common/src/internal_event/prelude.rs index 92aa160fe509e..bcc68aef26c6b 100644 --- a/lib/vector-common/src/internal_event/prelude.rs +++ b/lib/vector-common/src/internal_event/prelude.rs @@ -15,6 +15,8 @@ pub mod error_type { // When a condition for the event to be valid failed. // This is used for example when a field is missing or should be a string. pub const CONDITION_FAILED: &str = "condition_failed"; + // When the component received a request with missing or invalid authentication credentials. + pub const AUTHENTICATION_FAILED: &str = "authentication_failed"; // When the component or the service on which it depends is not configured properly. pub const CONFIGURATION_FAILED: &str = "configuration_failed"; // When the component failed to connect to an external service. diff --git a/src/internal_events/splunk_hec.rs b/src/internal_events/splunk_hec.rs index ea090684ed85d..6dcd63084d8e6 100644 --- a/src/internal_events/splunk_hec.rs +++ b/src/internal_events/splunk_hec.rs @@ -234,18 +234,36 @@ mod source { impl InternalEvent for SplunkHecRequestError { fn emit(self) { - error!( - message = "Error processing request.", - error = ?self.error, - error_type = error_type::REQUEST_FAILED, - stage = error_stage::RECEIVING - ); - counter!( - "component_errors_total", - "error_type" => error_type::REQUEST_FAILED, - "stage" => error_stage::RECEIVING, - ) - .increment(1); + match self.error { + ApiError::InvalidAuthorization | ApiError::MissingAuthorization => { + error!( + message = "Unauthenticated request.", + error = ?self.error, + error_type = error_type::AUTHENTICATION_FAILED, + stage = error_stage::RECEIVING + ); + counter!( + "component_errors_total", + "error_type" => error_type::AUTHENTICATION_FAILED, + "stage" => error_stage::RECEIVING, + ) + .increment(1); + } + _ => { + error!( + message = "Error processing request.", + error = ?self.error, + error_type = error_type::REQUEST_FAILED, + stage = error_stage::RECEIVING + ); + counter!( + "component_errors_total", + "error_type" => error_type::REQUEST_FAILED, + "stage" => error_stage::RECEIVING, + ) + .increment(1); + } + } } } } From 156b832637c2f18b4d32fbb347e0e5f0919407d3 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 27 Apr 2026 14:33:20 -0400 Subject: [PATCH 169/326] fix(website): include page title in docs search query fields (#25255) Co-authored-by: Pavlos Rontidis --- website/typesense.config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/typesense.config.json b/website/typesense.config.json index 249a303dc895e..9ea772c4c5c12 100644 --- a/website/typesense.config.json +++ b/website/typesense.config.json @@ -143,7 +143,7 @@ "name": "vector_docs_search", "search_parameters": { "sort_by": "level:desc,ranking:desc,_text_match:desc", - "query_by": "content,hierarchy,title,tags" + "query_by": "pageTitle,title,hierarchy,content,tags" } } ] From a843435f9677fe45f482ff4604bfe92b8a3d57c7 Mon Sep 17 00:00:00 2001 From: Rob Blafford Date: Mon, 27 Apr 2026 16:16:32 -0400 Subject: [PATCH 170/326] fix(topology): Fix for issue causing stalling on shutdown for sinks configured w/ disk buffers (#24949) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(buffers): prevent reload stall when disk buffer config changes - Changing a disk buffer's configuration (e.g. `max_size`) during a live config reload would stall indefinitely or fail with a `buffer.lock` error. This was caused by two issues: 1. The sink's detach trigger was only cancelled for buffer-reuse cases, so sinks with changed disk buffer configs never had their input stream terminated — disk buffer readers do not return `None` when the writer disconnects, causing the old sink task to hang forever. 2. The source output pump only processed fanout control messages (Remove/Pause) during active sends, so idle sources would never drop their `BufferSender` clone, keeping the `Arc` file lock alive even after the sink task completed. - Cancel the detach trigger for changed disk buffer sinks so the old sink task can complete. - Make the source output pump `select!` on both events and fanout control messages, so Remove/Pause is processed promptly even when the source is idle. - Add a retry loop (30s timeout) when acquiring the disk buffer lock to handle the small race window between the sink task completing and the fanout releasing the writer. - Add `BufferConfig::has_disk_stage()` helper for identifying disk buffer configurations. * Add unit tests for reload when disk buffer config modified * Add changelog fragment * Integration test exercising full disk buffer path w/ s3 sink - This test sets up a pipeline that sends using the s3 sinks with disk buffers enabled. - Data is sent through the disk buffer before and after configuration reload. * Clean-up changelog fragment * Fix spelling issues * Fix merge error and cleanup use of config() - Make consistent use of the config() method in all tests within this file * Simplify set of control_channel_open status * Remove unnecessary mutable modifiers * Prefer tempDir to just writing to /tmp --------- Co-authored-by: Pavlos Rontidis --- .../24929_fix_stall_on_disk_shutdown.fix.md | 6 + lib/vector-buffers/src/config.rs | 7 + lib/vector-core/src/fanout.rs | 14 ++ src/sinks/aws_s3/integration_tests.rs | 235 ++++++++++++------ src/topology/builder.rs | 62 +++-- src/topology/running.rs | 46 ++-- src/topology/test/reload.rs | 137 +++++++++- 7 files changed, 396 insertions(+), 111 deletions(-) create mode 100644 changelog.d/24929_fix_stall_on_disk_shutdown.fix.md diff --git a/changelog.d/24929_fix_stall_on_disk_shutdown.fix.md b/changelog.d/24929_fix_stall_on_disk_shutdown.fix.md new file mode 100644 index 0000000000000..2c7633caef857 --- /dev/null +++ b/changelog.d/24929_fix_stall_on_disk_shutdown.fix.md @@ -0,0 +1,6 @@ +Fixed issue during in place reload of a sink with a disk buffer configured, where +the component would stall for batch.timeout_sec before gracefully reloading. +This fix also resolves issues Vector had where it would ignore SIGINT during +cases where the pipeline stall had occurred. + +authors: graphcareful diff --git a/lib/vector-buffers/src/config.rs b/lib/vector-buffers/src/config.rs index ea51bfce3cb57..0cc9d77cd29d1 100644 --- a/lib/vector-buffers/src/config.rs +++ b/lib/vector-buffers/src/config.rs @@ -371,6 +371,13 @@ impl Default for BufferConfig { } impl BufferConfig { + /// Returns true if any stage in this buffer configuration uses disk-based storage. + pub fn has_disk_stage(&self) -> bool { + self.stages() + .iter() + .any(|stage| matches!(stage, BufferType::DiskV2 { .. })) + } + /// Gets all of the configured stages for this buffer. pub fn stages(&self) -> &[BufferType] { match self { diff --git a/lib/vector-core/src/fanout.rs b/lib/vector-core/src/fanout.rs index 0fd5cb034b703..97e21bcc8db17 100644 --- a/lib/vector-core/src/fanout.rs +++ b/lib/vector-core/src/fanout.rs @@ -108,6 +108,20 @@ impl Fanout { } } + /// Waits for the next control message and applies it. + /// + /// Returns `true` if a message was processed, `false` if the control + /// channel was closed. + pub async fn recv_control_message(&mut self) -> bool { + match self.control_channel.recv().await { + Some(msg) => { + self.apply_control_message(msg); + true + } + None => false, + } + } + /// Apply a control message directly against this instance. /// /// This method should not be used if there is an active `SendGroup` being processed. diff --git a/src/sinks/aws_s3/integration_tests.rs b/src/sinks/aws_s3/integration_tests.rs index ff2e0e5a5146b..a1a2c10af2201 100644 --- a/src/sinks/aws_s3/integration_tests.rs +++ b/src/sinks/aws_s3/integration_tests.rs @@ -1,7 +1,9 @@ #![cfg(all(test, feature = "aws-s3-integration-tests"))] use std::{ + collections::HashSet, io::{BufRead, BufReader}, + num::NonZeroU64, time::Duration, }; @@ -18,12 +20,14 @@ use bytes::Buf; use flate2::read::MultiGzDecoder; use futures::{Stream, stream}; use similar_asserts::assert_eq; +use tempfile::TempDir; use tokio_stream::StreamExt; #[cfg(feature = "codecs-parquet")] use vector_lib::codecs::encoding::BatchSerializerConfig; use vector_lib::{ + buffers::{BufferConfig, BufferType, WhenFull}, codecs::{TextSerializerConfig, encoding::FramingConfig}, - config::proxy::ProxyConfig, + config::{ComponentKey, proxy::ProxyConfig}, event::{BatchNotifier, BatchStatus, BatchStatusReceiver, Event, EventArray, LogEvent}, }; @@ -31,18 +35,20 @@ use super::S3SinkConfig; use crate::{ aws::{AwsAuthentication, RegionOrEndpoint, create_client}, common::s3::S3ClientBuilder, - config::SinkContext, + config::{Config, SinkContext}, sinks::{ aws_s3::config::default_filename_time_format, s3_common::config::{S3Options, S3ServerSideEncryption}, util::{BatchConfig, Compression, TowerRequestConfig}, }, test_util::{ + self, components::{ AWS_SINK_TAGS, COMPONENT_ERROR_TAGS, run_and_assert_sink_compliance, run_and_assert_sink_error, }, - random_lines_with_stream, random_string, + mock::basic_source, + random_lines_with_stream, random_string, start_topology, }, }; @@ -58,8 +64,10 @@ async fn s3_insert_message_into_with_flat_key_prefix() { create_bucket(&bucket, false).await; - let mut config = config(&bucket, 1000000); - config.key_prefix = "test-prefix".to_string(); + let config = S3SinkConfig { + key_prefix: "test-prefix".to_string(), + ..config(&bucket, 1000000, 5.0) + }; let prefix = config.key_prefix.clone(); let service = config.create_service(&cx.globals.proxy).await.unwrap(); let sink = config.build_processor(service, cx).unwrap(); @@ -92,8 +100,10 @@ async fn s3_insert_message_into_with_folder_key_prefix() { create_bucket(&bucket, false).await; - let mut config = config(&bucket, 1000000); - config.key_prefix = "test-prefix/".to_string(); + let config = S3SinkConfig { + key_prefix: "test-prefix/".to_string(), + ..config(&bucket, 1000000, 5.0) + }; let prefix = config.key_prefix.clone(); let service = config.create_service(&cx.globals.proxy).await.unwrap(); let sink = config.build_processor(service, cx).unwrap(); @@ -126,11 +136,16 @@ async fn s3_insert_message_into_with_ssekms_key_id() { create_bucket(&bucket, false).await; - let mut config = config(&bucket, 1000000); - config.key_prefix = "test-prefix".to_string(); + let config = S3SinkConfig { + key_prefix: "test-prefix".to_string(), + options: S3Options { + server_side_encryption: Some(S3ServerSideEncryption::AwsKms), + ssekms_key_id: Some("alias/aws/s3".to_string()), + ..S3Options::default() + }, + ..config(&bucket, 1000000, 5.0) + }; let prefix = config.key_prefix.clone(); - config.options.server_side_encryption = Some(S3ServerSideEncryption::AwsKms); - config.options.ssekms_key_id = Some("alias/aws/s3".to_string()); let service = config.create_service(&cx.globals.proxy).await.unwrap(); let sink = config.build_processor(service, cx).unwrap(); @@ -167,7 +182,7 @@ async fn s3_rotate_files_after_the_buffer_size_is_reached() { key_prefix: format!("{}/{}", random_string(10), "{{i}}"), filename_time_format: "waitsforfullbatch".into(), filename_append_uuid: false, - ..config(&bucket, 10) + ..config(&bucket, 10, 5.0) }; let prefix = config.key_prefix.clone(); let service = config.create_service(&cx.globals.proxy).await.unwrap(); @@ -225,7 +240,7 @@ async fn s3_gzip() { let config = S3SinkConfig { compression: Compression::gzip_default(), filename_time_format: "%s%f".into(), - ..config(&bucket, batch_size) + ..config(&bucket, batch_size, 5.0) }; let prefix = config.key_prefix.clone(); @@ -270,7 +285,7 @@ async fn s3_zstd() { let config = S3SinkConfig { compression: Compression::zstd_default(), filename_time_format: "%s%f".into(), - ..config(&bucket, batch_size) + ..config(&bucket, batch_size, 5.0) }; let prefix = config.key_prefix.clone(); @@ -334,7 +349,7 @@ async fn s3_insert_message_into_object_lock() { .await .unwrap(); - let config = config(&bucket, 1000000); + let config = config(&bucket, 1000000, 5.0); let prefix = config.key_prefix.clone(); let service = config.create_service(&cx.globals.proxy).await.unwrap(); let sink = config.build_processor(service, cx).unwrap(); @@ -364,9 +379,10 @@ async fn acknowledges_failures() { create_bucket(&bucket, false).await; - let mut config = config(&bucket, 1); - // Break the bucket name - config.bucket = format!("BREAK{}IT", config.bucket); + let config = S3SinkConfig { + bucket: format!("BREAK{}IT", &bucket), // Break the bucket name + ..config(&bucket, 1, 5.0) + }; let prefix = config.key_prefix.clone(); let service = config.create_service(&cx.globals.proxy).await.unwrap(); let sink = config.build_processor(service, cx).unwrap(); @@ -385,7 +401,7 @@ async fn s3_healthchecks() { create_bucket(&bucket, false).await; - let config = config(&bucket, 1); + let config = config(&bucket, 1, 5.0); let service = config .create_service(&ProxyConfig::from_env()) .await @@ -399,7 +415,7 @@ async fn s3_healthchecks() { #[tokio::test] async fn s3_healthchecks_invalid_bucket() { - let config = config("s3_healthchecks_invalid_bucket", 1); + let config = config("s3_healthchecks_invalid_bucket", 1, 5.0); let service = config .create_service(&ProxyConfig::from_env()) .await @@ -421,33 +437,7 @@ async fn s3_flush_on_exhaustion() { create_bucket(&bucket, false).await; // batch size of ten events, timeout of ten seconds - let config = { - let mut batch = BatchConfig::default(); - batch.max_events = Some(10); - batch.timeout_secs = Some(10.0); - - S3SinkConfig { - bucket: bucket.to_string(), - key_prefix: random_string(10) + "/date=%F", - filename_time_format: default_filename_time_format(), - filename_append_uuid: true, - filename_extension: None, - options: S3Options::default(), - region: RegionOrEndpoint::with_both("us-east-1", s3_address()), - encoding: (None::, TextSerializerConfig::default()).into(), - #[cfg(feature = "codecs-parquet")] - batch_encoding: None, - compression: Compression::None, - batch, - request: TowerRequestConfig::default(), - tls: Default::default(), - auth: Default::default(), - acknowledgements: Default::default(), - timezone: Default::default(), - force_path_style: true, - retry_strategy: Default::default(), - } - }; + let config = config(&bucket, 10, 10.0); let prefix = config.key_prefix.clone(); let service = config.create_service(&cx.globals.proxy).await.unwrap(); let sink = config.build_processor(service, cx).unwrap(); @@ -511,29 +501,9 @@ async fn s3_parquet_insert_message() { ..Default::default() }; - let mut batch = BatchConfig::default(); - batch.max_events = Some(100); - batch.timeout_secs = Some(5.0); - let config = S3SinkConfig { - bucket: bucket.to_string(), - key_prefix: random_string(10) + "/date=%F", - filename_time_format: default_filename_time_format(), - filename_append_uuid: true, - filename_extension: None, - options: S3Options::default(), - region: RegionOrEndpoint::with_both("us-east-1", s3_address()), - encoding: (None::, TextSerializerConfig::default()).into(), batch_encoding: Some(BatchSerializerConfig::Parquet(parquet_config)), - compression: Compression::None, - batch, - request: TowerRequestConfig::default(), - tls: Default::default(), - auth: Default::default(), - acknowledgements: Default::default(), - timezone: Default::default(), - force_path_style: true, - retry_strategy: Default::default(), + ..config(&bucket, 100, 5.0) }; let prefix = config.key_prefix.clone(); @@ -594,6 +564,131 @@ async fn s3_parquet_insert_message() { assert!(columns.contains(&"host".to_string())); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn s3_disk_buffer_reload_delivers_all_events() { + test_util::trace_init(); + + let bucket = uuid::Uuid::new_v4().to_string(); + create_bucket(&bucket, false).await; + + let data_dir = TempDir::new().expect("Could not create tempdir"); + + // batch.timeout_secs is deliberately large (300 s) so that the test would + // hang if the reload waited for the batch timer instead of cancelling it. + let s3_config = config(&bucket, 10, 300.0); + let prefix = s3_config.key_prefix.clone(); + + // Build topology + let (mut source_tx, source_config) = basic_source(); + + let mut old_config = Config::builder(); + old_config.global.data_dir = Some(data_dir.path().to_path_buf()); + old_config.add_source("in", source_config); + old_config.add_sink("out", &["in"], s3_config); + + let sink_key = ComponentKey::from("out"); + old_config.sinks[&sink_key].buffer = BufferConfig::Single(BufferType::DiskV2 { + max_size: NonZeroU64::new(268435488).unwrap(), + when_full: WhenFull::Block, + }); + + // Clone config before building so we can create the reload config. + let mut new_config = old_config.clone(); + new_config.sinks[&sink_key].buffer = BufferConfig::Single(BufferType::DiskV2 { + max_size: NonZeroU64::new(536870912).unwrap(), + when_full: WhenFull::Block, + }); + + // 1. Start topology with initial disk buffer config. + let (mut topology, _crash) = start_topology(old_config.build().unwrap(), false).await; + + // 2. Send first batch of events (enough to trigger batch.max_events flush). + let (pre_lines, pre_events, pre_receiver) = make_events_batch(100, 10); + for event in pre_events.collect::>().await { + source_tx.send_event(event).await.unwrap(); + } + + // 3. Wait for events to appear in S3. + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + loop { + let keys = get_keys(&bucket, prefix.clone()).await; + let count: usize = futures::future::join_all( + keys.into_iter() + .map(|key| async { get_lines(get_object(&bucket, key).await).await.len() }), + ) + .await + .into_iter() + .sum(); + if count >= pre_lines.len() { + break; + } + assert!( + tokio::time::Instant::now() < deadline, + "Timed out waiting for pre-reload events in S3 (found {count}/{})", + pre_lines.len() + ); + tokio::time::sleep(Duration::from_millis(500)).await; + } + assert_eq!(pre_receiver.await, BatchStatus::Delivered); + + // 4. Reload config with a different disk buffer max_size. + // Simulate what the `-w` file watcher does: mark the changed sink so its + // buffer is rebuilt rather than reused. + topology.extend_reload_set(HashSet::from_iter(vec![sink_key])); + + let reload_result = tokio::time::timeout( + Duration::from_secs(5), + topology.reload_config_and_respawn(new_config.build().unwrap(), Default::default()), + ) + .await; + + assert!( + reload_result.is_ok(), + "Reload timed out: disk buffer config change should not stall the reload" + ); + reload_result.unwrap().unwrap(); + + // Give the new sink a moment to initialise. + tokio::time::sleep(Duration::from_secs(1)).await; + + // 5. Send more events post-reload. + let (post_lines, post_events, post_receiver) = make_events_batch(100, 10); + for event in post_events.collect::>().await { + source_tx.send_event(event).await.unwrap(); + } + + // 6. Assert all events (pre- and post-reload) are present in S3. + let mut all_expected: Vec = pre_lines; + all_expected.extend(post_lines); + + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + let mut all_actual: Vec; + loop { + all_actual = Vec::new(); + let keys = get_keys(&bucket, prefix.clone()).await; + for key in keys { + all_actual.extend(get_lines(get_object(&bucket, key).await).await); + } + if all_actual.len() >= all_expected.len() { + break; + } + assert!( + tokio::time::Instant::now() < deadline, + "Timed out waiting for all events in S3 (found {}/{})", + all_actual.len(), + all_expected.len() + ); + tokio::time::sleep(Duration::from_millis(500)).await; + } + assert_eq!(post_receiver.await, BatchStatus::Delivered); + + all_expected.sort(); + all_actual.sort(); + assert_eq!(all_expected, all_actual); + + topology.stop().await; +} + async fn client() -> S3Client { let auth = AwsAuthentication::test_auth(); let region = RegionOrEndpoint::with_both("us-east-1", s3_address()); @@ -616,10 +711,10 @@ async fn client() -> S3Client { .unwrap() } -fn config(bucket: &str, batch_size: usize) -> S3SinkConfig { +fn config(bucket: &str, batch_size: usize, timeout_secs: f64) -> S3SinkConfig { let mut batch = BatchConfig::default(); batch.max_events = Some(batch_size); - batch.timeout_secs = Some(5.0); + batch.timeout_secs = Some(timeout_secs); S3SinkConfig { bucket: bucket.to_string(), diff --git a/src/topology/builder.rs b/src/topology/builder.rs index 8048b7a6073ed..8948109fa7fb7 100644 --- a/src/topology/builder.rs +++ b/src/topology/builder.rs @@ -884,30 +884,44 @@ async fn run_source_output_pump( ) -> TaskResult { debug!("Source pump starting."); - while let Some(SourceSenderItem { - events: mut array, - send_reference, - }) = rx.next().await - { - // Even though we have a `send_reference` timestamp above, that reference time is when - // the events were enqueued in the `SourceSender`, not when they were pulled out of the - // `rx` stream on this end. Since those times can be quite different (due to blocking - // inherent to the fanout send operation), we set the `last_transform_timestamp` to the - // current time instead to get an accurate reference for when the events started waiting - // for the first transform. - let now = Instant::now(); - array.for_each_metadata_mut(|metadata| { - metadata.set_source_id(Arc::clone(&source)); - metadata.set_source_type(source_type); - metadata.set_last_transform_timestamp(now); - }); - fanout - .send(array, Some(send_reference)) - .await - .map_err(|e| { - debug!("Source pump finished with an error."); - TaskError::wrapped(e) - })?; + let mut control_channel_open = true; + loop { + tokio::select! { + biased; + // Process control messages (e.g. Remove/Pause) even when the source + // is idle, so that config reloads can proceed without waiting for the + // next event. + alive = fanout.recv_control_message(), if control_channel_open => { + control_channel_open = alive; + } + item = rx.next() => { + match item { + Some(SourceSenderItem { events: mut array, send_reference }) => { + // Even though we have a `send_reference` timestamp above, that reference + // time is when the events were enqueued in the `SourceSender`, not when + // they were pulled out of the `rx` stream on this end. Since those times + // can be quite different (due to blocking inherent to the fanout send + // operation), we set the `last_transform_timestamp` to the current time + // instead to get an accurate reference for when the events started + // waiting for the first transform. + let now = Instant::now(); + array.for_each_metadata_mut(|metadata| { + metadata.set_source_id(Arc::clone(&source)); + metadata.set_source_type(source_type); + metadata.set_last_transform_timestamp(now); + }); + fanout + .send(array, Some(send_reference)) + .await + .map_err(|e| { + debug!("Source pump finished with an error."); + TaskError::wrapped(e) + })?; + } + None => break, + } + } + } } debug!("Source pump finished normally."); diff --git a/src/topology/running.rs b/src/topology/running.rs index a1ecf7278ae89..97e309ec0d0b8 100644 --- a/src/topology/running.rs +++ b/src/topology/running.rs @@ -583,10 +583,26 @@ impl RunningTopology { .collect::>(); // For any existing sink that has a conflicting resource dependency with a changed/added - // sink, or for any sink that we want to reuse their buffer, we need to explicit wait for - // them to finish processing so we can reclaim ownership of those resources/buffers. + // sink, for any sink that we want to reuse their buffer, or for any changed sink with + // a disk buffer that is not being reused, we need to explicitly wait for them to finish + // processing so we can reclaim ownership of those resources/buffers. + let changed_disk_buffer_sinks = diff + .sinks + .to_change + .iter() + .filter(|key| { + !reuse_buffers.contains(*key) + && self + .config + .sink(key) + .is_some_and(|s| s.buffer.has_disk_stage()) + }) + .cloned() + .collect::>(); + let wait_for_sinks = conflicting_sinks .chain(reuse_buffers.iter().cloned()) + .chain(changed_disk_buffer_sinks.iter().cloned()) .collect::>(); // First, we remove any inputs to removed sinks so they can naturally shut down. @@ -628,24 +644,26 @@ impl RunningTopology { for key in &sinks_to_change { debug!(component_id = %key, "Changing sink."); - if reuse_buffers.contains(key) { + if reuse_buffers.contains(key) || changed_disk_buffer_sinks.contains(key) { self.detach_triggers .remove(key) .unwrap() .into_inner() .cancel(); - // We explicitly clone the input side of the buffer and store it so we don't lose - // it when we remove the inputs below. - // - // We clone instead of removing here because otherwise the input will be missing for - // the rest of the reload process, which violates the assumption that all previous - // inputs for components not being removed are still available. It's simpler to - // allow the "old" input to stick around and be replaced (even though that's - // basically a no-op since we're reusing the same buffer) than it is to pass around - // info about which sinks are having their buffers reused and treat them differently - // at other stages. - buffer_tx.insert((*key).clone(), self.inputs.get(key).unwrap().clone()); + if reuse_buffers.contains(key) { + // We explicitly clone the input side of the buffer and store it so we don't lose + // it when we remove the inputs below. + // + // We clone instead of removing here because otherwise the input will be missing for + // the rest of the reload process, which violates the assumption that all previous + // inputs for components not being removed are still available. It's simpler to + // allow the "old" input to stick around and be replaced (even though that's + // basically a no-op since we're reusing the same buffer) than it is to pass around + // info about which sinks are having their buffers reused and treat them differently + // at other stages. + buffer_tx.insert((*key).clone(), self.inputs.get(key).unwrap().clone()); + } } self.remove_inputs(key, diff, new_config).await; } diff --git a/src/topology/test/reload.rs b/src/topology/test/reload.rs index 907be19f6ec0f..7b2db20d4f337 100644 --- a/src/topology/test/reload.rs +++ b/src/topology/test/reload.rs @@ -1,7 +1,7 @@ use std::{ collections::HashSet, net::{SocketAddr, TcpListener}, - num::NonZeroU64, + num::{NonZeroU64, NonZeroUsize}, time::Duration, }; @@ -9,7 +9,7 @@ use futures::StreamExt; use tokio::time::sleep; use tokio_stream::wrappers::UnboundedReceiverStream; use vector_lib::{ - buffers::{BufferConfig, BufferType, WhenFull}, + buffers::{BufferConfig, BufferType, MemoryBufferSize, WhenFull}, config::ComponentKey, }; @@ -293,7 +293,6 @@ async fn topology_readd_input() { #[tokio::test] async fn topology_reload_component() { test_util::trace_init(); - let (_guard, address_0) = next_addr(); let mut old_config = Config::builder(); @@ -320,6 +319,138 @@ async fn topology_reload_component() { } } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn topology_disk_buffer_config_change_does_not_stall() { + // Changing a disk buffer's configuration on a running sink (e.g. via in-situ + // config edit) must not stall the reload. Previously, the detach trigger was + // only cancelled for sinks whose buffers were being reused, so sinks with + // changed disk buffer configs would never have their input stream terminated, + // causing the reload to hang indefinitely. + test_util::trace_init(); + + let (_guard, address) = next_addr(); + + let data_dir = temp_dir(); + std::fs::create_dir(&data_dir).unwrap(); + + let mut old_config = Config::builder(); + old_config.global.data_dir = Some(data_dir); + old_config.add_source("in", internal_metrics_source()); + old_config.add_sink("out", &["in"], prom_exporter_sink(address, 1)); + + let sink_key = ComponentKey::from("out"); + old_config.sinks[&sink_key].buffer = BufferConfig::Single(BufferType::DiskV2 { + max_size: NonZeroU64::new(268435488).unwrap(), + when_full: WhenFull::Block, + }); + + // Change only the disk buffer's max_size. + let mut new_config = old_config.clone(); + new_config.sinks[&sink_key].buffer = BufferConfig::Single(BufferType::DiskV2 { + max_size: NonZeroU64::new(536870912).unwrap(), + when_full: WhenFull::Block, + }); + + let (mut topology, crash) = start_topology(old_config.build().unwrap(), true).await; + let mut crash_stream = UnboundedReceiverStream::new(crash); + + tokio::select! { + _ = wait_for_tcp(address) => {}, + _ = crash_stream.next() => panic!("topology crashed before reload"), + } + + // Simulate an in-situ config edit: the config watcher would put the changed + // sink into components_to_reload, which excludes it from reuse_buffers. + topology.extend_reload_set(HashSet::from_iter(vec![sink_key])); + + let reload_result = tokio::time::timeout( + Duration::from_secs(5), + topology.reload_config_and_respawn(new_config.build().unwrap(), Default::default()), + ) + .await; + + assert!( + reload_result.is_ok(), + "Reload stalled: changing a disk buffer config should not cause the reload to hang" + ); + reload_result.unwrap().unwrap(); + + // Verify the new sink is running. + tokio::select! { + _ = wait_for_tcp(address) => {}, + _ = crash_stream.next() => panic!("topology crashed after reload"), + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn topology_disk_buffer_config_change_chained_does_not_stall() { + // Same as above but with a chained memory → disk overflow buffer to verify + // that the writer-drop notification is collected from overflow stages too. + test_util::trace_init(); + + let (_guard, address) = next_addr(); + + let data_dir = temp_dir(); + std::fs::create_dir(&data_dir).unwrap(); + + let memory_stage = BufferType::Memory { + size: MemoryBufferSize::MaxEvents(NonZeroUsize::new(100).unwrap()), + when_full: WhenFull::Overflow, + }; + + let mut old_config = Config::builder(); + old_config.global.data_dir = Some(data_dir); + old_config.add_source("in", internal_metrics_source()); + old_config.add_sink("out", &["in"], prom_exporter_sink(address, 1)); + + let sink_key = ComponentKey::from("out"); + old_config.sinks[&sink_key].buffer = BufferConfig::Chained(vec![ + memory_stage, + BufferType::DiskV2 { + max_size: NonZeroU64::new(268435488).unwrap(), + when_full: WhenFull::Block, + }, + ]); + + // Change only the disk overflow stage's max_size. + let mut new_config = old_config.clone(); + new_config.sinks[&sink_key].buffer = BufferConfig::Chained(vec![ + memory_stage, + BufferType::DiskV2 { + max_size: NonZeroU64::new(536870912).unwrap(), + when_full: WhenFull::Block, + }, + ]); + + let (mut topology, crash) = start_topology(old_config.build().unwrap(), true).await; + let mut crash_stream = UnboundedReceiverStream::new(crash); + + tokio::select! { + _ = wait_for_tcp(address) => {}, + _ = crash_stream.next() => panic!("topology crashed before reload"), + } + + topology.extend_reload_set(HashSet::from_iter(vec![sink_key])); + + let reload_result = tokio::time::timeout( + Duration::from_secs(5), + topology.reload_config_and_respawn(new_config.build().unwrap(), Default::default()), + ) + .await; + + assert!( + reload_result.is_ok(), + "Reload stalled: changing a chained disk buffer config should not cause the reload to hang" + ); + reload_result.unwrap().unwrap(); + + // Verify the new sink is running. + tokio::select! { + _ = wait_for_tcp(address) => {}, + _ = crash_stream.next() => panic!("topology crashed after reload"), + } +} + async fn reload_sink_test( old_config: Config, new_config: Config, From ff4754ab10da5838f68cc143e507087090518f91 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 28 Apr 2026 10:04:28 -0400 Subject: [PATCH 171/326] chore(ci): make nightly S3 verify resilient to CDN staleness (#25259) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(ci): remove dead nightly artifact-redirect loop The loop referenced an undefined $i (copy-paste from the release branch where $i iterates over version tags). Under set -u the subshell errored, the for loop received empty input, and the body never ran — so this has been a no-op since it was added. Co-Authored-By: Claude Opus 4.7 (1M context) * chore(ci): retry verify_artifact on CDN cache staleness packages.timber.io is fronted by a CDN; after `aws s3 rm --recursive` + `cp --recursive` on nightly/latest, the edge can keep serving stale bytes for longer than the existing 30s VERIFY_TIMEOUT, failing `cmp` and the whole job. wget's --retry-on-http-error=404 only retries 404s, not a 200 with stale content. Wrap the compare in an exponential-backoff retry loop (1, 2, 4, 8, 16, 32s; 7 attempts, ~63s of total sleep) so a transient stale-cache hit no longer fails the release. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) --- scripts/release-s3.sh | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/scripts/release-s3.sh b/scripts/release-s3.sh index 9889fa556e4eb..8f6c51de08669 100755 --- a/scripts/release-s3.sh +++ b/scripts/release-s3.sh @@ -36,11 +36,27 @@ ls "$td_nightly" # # A helper function for verifying a published artifact. # +# Retries a content mismatch as well as a 404, since packages.timber.io is +# fronted by a CDN and an object we just overwrote via `aws s3 rm` + `cp` can +# serve stale bytes at the edge for a while. verify_artifact() { local URL="$1" local FILENAME="$2" + local attempts=7 + local delay=1 echo "Verifying $URL" - cmp <(wget -qO- --retry-on-http-error=404 --wait 10 --tries "$VERIFY_RETRIES" "$URL") "$FILENAME" + for ((attempt = 1; attempt <= attempts; attempt++)); do + if cmp <(wget -qO- --retry-on-http-error=404 --wait 10 --tries "$VERIFY_RETRIES" "$URL") "$FILENAME"; then + return 0 + fi + if (( attempt < attempts )); then + echo "Attempt $attempt/$attempts did not match (likely stale CDN cache); retrying in ${delay}s" + sleep "$delay" + delay=$((delay * 2)) + fi + done + echo "Verification of $URL failed after $attempts attempts" + return 1 } # @@ -59,15 +75,6 @@ if [[ "$CHANNEL" == "nightly" ]]; then aws s3 cp "$td_nightly" "s3://packages.timber.io/vector/nightly/latest" --recursive --sse --acl public-read echo "Uploaded archives" - echo "Redirecting old artifact names" - for file in $(aws s3api list-objects-v2 --bucket packages.timber.io --prefix "vector/$i/" --query 'Contents[*].Key' --output text | tr "\t" "\n" | grep '\-nightly'); do - file=$(basename "$file") - # vector-nightly-amd64.deb -> vector-amd64.deb - echo -n "" | aws s3 cp - "s3://packages.timber.io/vector/nightly/$DATE/${file/-nightly/}" --website-redirect "/vector/nightly/$DATE/$file" --acl public-read - echo -n "" | aws s3 cp - "s3://packages.timber.io/vector/nightly/latest/${file/-nightly/}" --website-redirect "/vector/nightly/latest/$file" --acl public-read - done - echo "Redirected old artifact names" - # Verify that the files exist and can be downloaded echo "Waiting for $VERIFY_TIMEOUT seconds before running the verifications" sleep "$VERIFY_TIMEOUT" From 6f3857906d3a54d9c05aa5d3aaa8b132ac8f5e6e Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 28 Apr 2026 11:12:07 -0400 Subject: [PATCH 172/326] fix(vrl): restore stdlib functions in CLI and playground (#25310) --- changelog.d/25267_vrl_get_env_var_playground.fix.md | 2 ++ lib/vector-vrl/cli/Cargo.toml | 4 ++++ lib/vector-vrl/web-playground/Cargo.toml | 4 ++++ 3 files changed, 10 insertions(+) create mode 100644 changelog.d/25267_vrl_get_env_var_playground.fix.md diff --git a/changelog.d/25267_vrl_get_env_var_playground.fix.md b/changelog.d/25267_vrl_get_env_var_playground.fix.md new file mode 100644 index 0000000000000..29de5d36fe6a5 --- /dev/null +++ b/changelog.d/25267_vrl_get_env_var_playground.fix.md @@ -0,0 +1,2 @@ +Restored the full VRL stdlib, including `get_env_var`, in the standalone VRL CLI and web playground by default. +authors: pront diff --git a/lib/vector-vrl/cli/Cargo.toml b/lib/vector-vrl/cli/Cargo.toml index 32f0b8ccce166..8e3c78597de8b 100644 --- a/lib/vector-vrl/cli/Cargo.toml +++ b/lib/vector-vrl/cli/Cargo.toml @@ -6,6 +6,10 @@ edition = "2024" publish = false license = "MPL-2.0" +[features] +# Enable the full VRL stdlib, which includes all available VRL functions. +default = ["vrl/stdlib"] + [dependencies] clap.workspace = true vector-vrl-functions.workspace = true diff --git a/lib/vector-vrl/web-playground/Cargo.toml b/lib/vector-vrl/web-playground/Cargo.toml index 9b3fbd6392576..59db0427b3698 100644 --- a/lib/vector-vrl/web-playground/Cargo.toml +++ b/lib/vector-vrl/web-playground/Cargo.toml @@ -11,6 +11,10 @@ license = "MPL-2.0" [lib] crate-type = ["cdylib"] +[features] +# Enable the full VRL stdlib, which includes all available VRL functions. +default = ["vrl/stdlib"] + [dependencies] wasm-bindgen = "0.2" vrl.workspace = true From 23016adf7d87f41a3d887d587b21d3409e71e3fe Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 28 Apr 2026 11:52:16 -0400 Subject: [PATCH 173/326] chore(internal docs): fix release issue templates (#25318) - Fetch and fast-forward the release branch before inspecting HEAD so `git show --stat HEAD` reflects origin, not a stale local tip. - Use `git push --force-with-lease` when resetting the `website` branch to the release branch's HEAD; a plain `git push` is rejected as non-fast-forward, which is the expected outcome of `reset --hard` to a different branch. Co-authored-by: Claude Opus 4.7 (1M context) --- .github/ISSUE_TEMPLATE/minor-release.md | 5 +++-- .github/ISSUE_TEMPLATE/patch-release.md | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/minor-release.md b/.github/ISSUE_TEMPLATE/minor-release.md index 7ab406601e4cb..f295a03555512 100644 --- a/.github/ISSUE_TEMPLATE/minor-release.md +++ b/.github/ISSUE_TEMPLATE/minor-release.md @@ -71,7 +71,8 @@ Automated steps include: # On the day of release - [ ] Make sure the release branch is in sync with origin/master and has only one squashed commit with all commits from the prepare branch. If you made a PR from the prepare branch into the release branch this should already be the case. - - [ ] `git checkout "${RELEASE_BRANCH}"` + - [ ] `git fetch origin` + - [ ] `git checkout "${RELEASE_BRANCH}" && git pull --ff-only origin "${RELEASE_BRANCH}"` - [ ] `git show --stat HEAD` - This should show the squashed prepare commit. - [ ] Ensure release date in `website/cue/reference/releases/0.XX.Y.cue` matches current date. - If this needs to be updated commit and squash it in the release branch. @@ -88,7 +89,7 @@ Automated steps include: - [ ] Wait for release workflow to complete. - Discoverable via [release.yml](https://github.com/vectordotdev/vector/actions/workflows/release.yml) - [ ] Reset the `website` branch to the `HEAD` of the release branch to update https://vector.dev - - [ ] `git switch website && git reset --hard origin/"${RELEASE_BRANCH}" && git push` + - [ ] `git fetch origin && git switch website && git reset --hard origin/"${RELEASE_BRANCH}" && git push --force-with-lease` - [ ] Confirm that the release changelog was published to https://vector.dev/releases/ - Refer to the internal releasing doc to monitor the deployment. - [ ] Release Linux packages. Refer to the internal releasing doc. diff --git a/.github/ISSUE_TEMPLATE/patch-release.md b/.github/ISSUE_TEMPLATE/patch-release.md index afc5d8d826423..4a36f7255e36b 100644 --- a/.github/ISSUE_TEMPLATE/patch-release.md +++ b/.github/ISSUE_TEMPLATE/patch-release.md @@ -62,5 +62,5 @@ export PREP_BRANCH=prepare-v-0-"${CURRENT_MINOR_VERSION}"-"${NEW_PATCH_VERSION}" - Follow the [instructions at the top of the mirror.yaml file](https://github.com/DataDog/images/blob/fbf12868e90d52e513ebca0389610dea8a3c7e1a/mirror.yaml#L33-L49). - [ ] Cherry-pick any release commits from the release branch that are not on `master`, to `master` - [ ] Reset the `website` branch to the `HEAD` of the release branch to update https://vector.dev - - [ ] `git checkout website && git reset --hard origin/"${RELEASE_BRANCH}" && git push` + - [ ] `git fetch origin && git checkout website && git reset --hard origin/"${RELEASE_BRANCH}" && git push --force-with-lease` - [ ] Kick-off post-mortems for any regressions resolved by the release From 96ad9edc5bd894029af95961ea205e0d89b17bf0 Mon Sep 17 00:00:00 2001 From: Thomas Date: Tue, 28 Apr 2026 14:17:38 -0400 Subject: [PATCH 174/326] fix(website): improve docs search ranking for component pages (#25319) --- website/typesense.config.json | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/website/typesense.config.json b/website/typesense.config.json index 9ea772c4c5c12..f0543b9133c8b 100644 --- a/website/typesense.config.json +++ b/website/typesense.config.json @@ -5,6 +5,7 @@ "file_path": "./public/search.json", "synonyms": [], "schema": { + "token_separators": ["_", "-"], "fields": [ { "facet": false, @@ -142,8 +143,10 @@ { "name": "vector_docs_search", "search_parameters": { - "sort_by": "level:desc,ranking:desc,_text_match:desc", - "query_by": "pageTitle,title,hierarchy,content,tags" + "sort_by": "_text_match:desc,ranking:desc,level:desc", + "query_by": "pageTitle,title,hierarchy,content,tags", + "query_by_weights": "4,3,2,1,2", + "text_match_type": "sum_score" } } ] From ce6ca439caf80bd2af10864a84e949ef244e2163 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 28 Apr 2026 15:40:29 -0400 Subject: [PATCH 175/326] chore(codecs): centralize `events_dropped` emission for batch encoding errors (#25199) * fix(codecs): centralize events_dropped emission for batch encoding errors Move events_dropped emission from individual internal events inside serializers to a single wrapper in (Transformer, BatchEncoder)::encode_input. This ensures all batch encoding error paths (Arrow IPC and Parquet) consistently emit events_dropped without requiring each new error path to remember to add it. Co-Authored-By: Claude Opus 4.6 (1M context) * chore: fix formatting Co-Authored-By: Claude Opus 4.6 (1M context) * test(codecs): add unit test for type mismatch in Parquet encoding Covers the build_record_batch ArrowJsonDecode error path where a schema expects int64 but the event contains a string value. Co-Authored-By: Claude Opus 4.6 (1M context) * fix(codecs): emit ComponentEventsDropped directly to avoid double-counting Replace EncoderWriteError with a direct ComponentEventsDropped emission in the batch encode wrapper. EncoderWriteError was incrementing component_errors_total and logging "Failed writing bytes." which double-counted errors (codec-specific events already increment component_errors_total) and was misleading (the failure was encoding, not writing). Co-Authored-By: Claude Opus 4.6 (1M context) * docs: clarify comment about error counting and overcount edge case Co-Authored-By: Claude Opus 4.6 (1M context) * fix(codecs): guard batch-drop imports behind codecs-arrow feature Move ComponentEventsDropped and UNINTENTIONAL imports inside the cfg(feature = "codecs-arrow") impl block to avoid unused import errors when the feature is disabled. Co-Authored-By: Claude Opus 4.6 (1M context) * docs: clarify overcount edge case is a misconfiguration scenario Co-Authored-By: Claude Opus 4.6 (1M context) * chore(codecs): emit codec event for build_record_batch failures The new EncoderRecordBatchError fires from build_record_batch's RecordBatchCreation and ArrowJsonDecode paths, so type-mismatch and decoder-build failures emit a granular component_errors_total counter at stage="sending" with a specific error_code, instead of relying solely on the downstream SinkRequestBuildError at stage="processing". Co-Authored-By: Claude Opus 4.7 (1M context) * chore(codecs): add changelog fragment for batch encoding event coverage Co-Authored-By: Claude Opus 4.7 (1M context) * test(codecs): assert EncoderRecordBatchError fires on Arrow type mismatch Drives (Transformer, BatchEncoder)::encode_input through ArrowStreamSerializer with an Int64 schema field and a string-valued event to trigger the ArrowJsonDecode path in build_record_batch. Asserts both EncoderRecordBatchError and ComponentEventsDropped are recorded so a regression in either emission fails the test. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.6 (1M context) --- ...25199_batch_encoding_events_dropped.fix.md | 3 + lib/codecs/src/encoding/format/arrow.rs | 23 ++++++- lib/codecs/src/encoding/format/parquet.rs | 63 +++++++++--------- lib/codecs/src/internal_events.rs | 64 ++++++++++--------- src/sinks/util/encoding.rs | 61 +++++++++++++++++- 5 files changed, 153 insertions(+), 61 deletions(-) create mode 100644 changelog.d/25199_batch_encoding_events_dropped.fix.md diff --git a/changelog.d/25199_batch_encoding_events_dropped.fix.md b/changelog.d/25199_batch_encoding_events_dropped.fix.md new file mode 100644 index 0000000000000..214aad79a528c --- /dev/null +++ b/changelog.d/25199_batch_encoding_events_dropped.fix.md @@ -0,0 +1,3 @@ +Sinks using batch encoding (Parquet, Arrow IPC) now consistently emit `ComponentEventsDropped` for every encode failure path. Previously some `build_record_batch` failures (notably type mismatches) dropped events silently. A new `EncoderRecordBatchError` internal event also reports `component_errors_total` with `error_code="arrow_json_decode"` or `"arrow_record_batch_creation"` at `stage="sending"` for granular alerting. + +authors: pront diff --git a/lib/codecs/src/encoding/format/arrow.rs b/lib/codecs/src/encoding/format/arrow.rs index b8c86239ddb6a..6c600b3777d18 100644 --- a/lib/codecs/src/encoding/format/arrow.rs +++ b/lib/codecs/src/encoding/format/arrow.rs @@ -322,7 +322,6 @@ pub(crate) fn build_record_batch( }); vector_common::internal_event::emit(crate::internal_events::EncoderNullConstraintError { error: &error, - count: values.len(), }); return Err(ArrowEncodingError::NullConstraint { field_name: missing.join(", "), @@ -331,12 +330,32 @@ pub(crate) fn build_record_batch( let mut decoder = ReaderBuilder::new(schema) .build_decoder() + .inspect_err(|e| { + vector_common::internal_event::emit(crate::internal_events::EncoderRecordBatchError { + error: e, + error_code: "arrow_record_batch_creation", + }); + }) .context(RecordBatchCreationSnafu)?; - decoder.serialize(values).context(ArrowJsonDecodeSnafu)?; + decoder + .serialize(values) + .inspect_err(|e| { + vector_common::internal_event::emit(crate::internal_events::EncoderRecordBatchError { + error: e, + error_code: "arrow_json_decode", + }); + }) + .context(ArrowJsonDecodeSnafu)?; decoder .flush() + .inspect_err(|e| { + vector_common::internal_event::emit(crate::internal_events::EncoderRecordBatchError { + error: e, + error_code: "arrow_json_decode", + }); + }) .context(ArrowJsonDecodeSnafu)? .ok_or(ArrowEncodingError::NoEvents) } diff --git a/lib/codecs/src/encoding/format/parquet.rs b/lib/codecs/src/encoding/format/parquet.rs index 7349119371a5b..adb5b4082749b 100644 --- a/lib/codecs/src/encoding/format/parquet.rs +++ b/lib/codecs/src/encoding/format/parquet.rs @@ -276,39 +276,29 @@ impl ParquetSerializer { /// Writes `record_batch` into `buffer` as a complete Parquet file. /// - /// On failure, emits an [`ArrowWriterError`] internal event (which also - /// increments `component_errors_total` and emits the events-dropped metric) - /// before returning the error. + /// On failure, emits an [`ArrowWriterError`] internal event (which + /// increments `component_errors_total`) before returning the error. + /// The caller is responsible for emitting `events_dropped`. fn write_record_batch( - &self, record_batch: &RecordBatch, buffer: &mut BytesMut, - event_count: usize, + writer_props: &WriterProperties, ) -> Result<(), parquet::errors::ParquetError> { let mut writer = ArrowWriter::try_new( buffer.writer(), Arc::clone(record_batch.schema_ref()), - Some((*self.writer_props).clone()), + Some(writer_props.clone()), ) .inspect_err(|e| { - emit(ArrowWriterError { - error: e, - batch_count: event_count, - }); + emit(ArrowWriterError { error: e }); })?; writer.write(record_batch).inspect_err(|e| { - emit(ArrowWriterError { - error: e, - batch_count: event_count, - }); + emit(ArrowWriterError { error: e }); })?; writer.close().inspect_err(|e| { - emit(ArrowWriterError { - error: e, - batch_count: event_count, - }); + emit(ArrowWriterError { error: e }); })?; Ok(()) @@ -326,10 +316,7 @@ impl tokio_util::codec::Encoder> for ParquetSerializer { let json_values = match vector_log_events_to_json_values(&events) { Ok(values) => values, Err(e) => { - emit(JsonSerializationError { - error: &e, - batch_count: events.len(), - }); + emit(JsonSerializationError { error: &e }); return Err(Box::new(e)); } }; @@ -358,7 +345,6 @@ impl tokio_util::codec::Encoder> for ParquetSerializer { { for top_level in object_map.keys() { if !self.schema_field_names.contains(top_level.as_str()) { - self.events_dropped_handle.emit(Count(events.len())); return Err(Box::new(ArrowEncodingError::SchemaFetchError { message: format!( "Strict schema mode: event contains field '{top_level}' not in schema", @@ -381,8 +367,7 @@ impl tokio_util::codec::Encoder> for ParquetSerializer { let record_batch = build_record_batch(Arc::clone(&self.schema), &json_values).map_err(Box::new)?; - self.write_record_batch(&record_batch, buffer, json_values.len()) - .map_err(Box::new)?; + Self::write_record_batch(&record_batch, buffer, &self.writer_props).map_err(Box::new)?; Ok(()) } @@ -394,10 +379,7 @@ impl ParquetSchemaGenerator { pub fn infer_schema(events: &[serde_json::Value]) -> Result { let schema = infer_json_schema_from_iterator(events.iter().map(Ok::<_, ArrowError>)) .map_err(|e| { - emit(SchemaGenerationError { - error: &e, - batch_count: events.len(), - }); + emit(SchemaGenerationError { error: &e }); Error::new(ErrorKind::InvalidData, e.to_string()) })?; @@ -916,6 +898,29 @@ mod tests { assert_eq!(columns, vec!["name"]); } + #[test] + fn test_parquet_type_mismatch_returns_error() { + let schema_path = + write_temp_schema("type_mismatch", "message logs {\n required int64 name;\n}"); + + let mut serializer = ParquetSerializer::new(ParquetSerializerConfig { + schema_file: Some(schema_path), + schema_mode: ParquetSchemaMode::Relaxed, + ..Default::default() + }) + .expect("Failed to create serializer"); + + let events = vec![create_event(vec![("name", "not_an_integer")])]; + let mut buffer = BytesMut::new(); + let result = serializer.encode(events, &mut buffer); + assert!(result.is_err(), "Type mismatch should return an error"); + let err = result.unwrap_err().to_string(); + assert!( + err.contains("Int64"), + "Error should mention the expected type, got: {err}" + ); + } + #[test] fn test_parquet_schema_file_binary_without_string_annotation_rejected() { // Native Parquet "binary" without (STRING) annotation resolves to Arrow Binary, diff --git a/lib/codecs/src/internal_events.rs b/lib/codecs/src/internal_events.rs index 0ed24719dd8bc..d21119419887d 100644 --- a/lib/codecs/src/internal_events.rs +++ b/lib/codecs/src/internal_events.rs @@ -157,16 +157,13 @@ impl InternalEvent for EncoderWriteError<'_, E> { pub struct EncoderNullConstraintError<'a> { /// The schema constraint error that occurred. pub error: &'a vector_common::Error, - /// The number of events dropped due to the constraint violation. - pub count: usize, } #[cfg(feature = "arrow")] impl InternalEvent for EncoderNullConstraintError<'_> { fn emit(self) { - const CONSTRAINT_REASON: &str = "Schema constraint violation."; error!( - message = CONSTRAINT_REASON, + message = "Schema constraint violation.", error = %self.error, error_code = "encoding_null_constraint", error_type = error_type::ENCODER_FAILED, @@ -179,10 +176,37 @@ impl InternalEvent for EncoderNullConstraintError<'_> { "stage" => error_stage::SENDING, ) .increment(1); - emit(ComponentEventsDropped:: { - count: self.count, - reason: CONSTRAINT_REASON, - }); + } +} + +#[cfg(feature = "arrow")] +#[derive(Debug, NamedInternalEvent)] +/// Emitted when Arrow record batch construction fails (e.g. schema decoder build, +/// JSON-to-Arrow decoding such as type mismatches). +pub struct EncoderRecordBatchError<'a, E> { + /// The encoding error that occurred. + pub error: &'a E, + /// Stable error code identifying the failure mode. + pub error_code: &'static str, +} + +#[cfg(feature = "arrow")] +impl InternalEvent for EncoderRecordBatchError<'_, E> { + fn emit(self) { + error!( + message = "Failed to build Arrow record batch.", + error = %self.error, + error_code = self.error_code, + error_type = error_type::ENCODER_FAILED, + stage = error_stage::SENDING, + ); + counter!( + "component_errors_total", + "error_code" => self.error_code, + "error_type" => error_type::ENCODER_FAILED, + "stage" => error_stage::SENDING, + ) + .increment(1); } } @@ -190,15 +214,13 @@ impl InternalEvent for EncoderNullConstraintError<'_> { #[derive(NamedInternalEvent)] pub(crate) struct SchemaGenerationError<'a> { pub error: &'a arrow::error::ArrowError, - pub batch_count: usize, } #[cfg(feature = "parquet")] impl InternalEvent for SchemaGenerationError<'_> { fn emit(self) { - const REASON: &str = "Could not generate schema for batched events"; error!( - message = REASON, + message = "Could not generate schema for batched events", error = %self.error, error_code = "parquet_schema_generation_failed", error_type = error_type::ENCODER_FAILED, @@ -212,10 +234,6 @@ impl InternalEvent for SchemaGenerationError<'_> { "stage" => error_stage::SENDING, ) .increment(1); - emit(ComponentEventsDropped:: { - count: self.batch_count, - reason: REASON, - }); } } @@ -223,15 +241,13 @@ impl InternalEvent for SchemaGenerationError<'_> { #[derive(NamedInternalEvent)] pub(crate) struct ArrowWriterError<'a> { pub error: &'a parquet::errors::ParquetError, - pub batch_count: usize, } #[cfg(feature = "parquet")] impl InternalEvent for ArrowWriterError<'_> { fn emit(self) { - const REASON: &str = "Failed to write record batch with ArrowWriter."; error!( - message = REASON, + message = "Failed to write record batch with ArrowWriter.", error = %self.error, error_code = "parquet_arrow_writer_failed", error_type = error_type::ENCODER_FAILED, @@ -245,10 +261,6 @@ impl InternalEvent for ArrowWriterError<'_> { "stage" => error_stage::SENDING, ) .increment(1); - emit(ComponentEventsDropped:: { - count: self.batch_count, - reason: REASON, - }); } } @@ -256,15 +268,13 @@ impl InternalEvent for ArrowWriterError<'_> { #[derive(NamedInternalEvent)] pub(crate) struct JsonSerializationError<'a> { pub error: &'a serde_json::Error, - pub batch_count: usize, } #[cfg(feature = "parquet")] impl InternalEvent for JsonSerializationError<'_> { fn emit(self) { - const CONSTRAINT_REASON: &str = "Could not serialize event to JSON."; error!( - message = CONSTRAINT_REASON, + message = "Could not serialize event to JSON.", error = %self.error, error_type = error_type::ENCODER_FAILED, stage = error_stage::SENDING, @@ -276,9 +286,5 @@ impl InternalEvent for JsonSerializationError<'_> { "stage" => error_stage::SENDING, ) .increment(1); - emit(ComponentEventsDropped:: { - count: self.batch_count, - reason: CONSTRAINT_REASON, - }); } } diff --git a/src/sinks/util/encoding.rs b/src/sinks/util/encoding.rs index 4cc49b00f358b..63f8407cef53b 100644 --- a/src/sinks/util/encoding.rs +++ b/src/sinks/util/encoding.rs @@ -107,6 +107,7 @@ impl Encoder> for (Transformer, vector_lib::codecs::BatchEncoder) { writer: &mut dyn io::Write, ) -> io::Result<(usize, GroupedCountByteSize)> { use tokio_util::codec::Encoder as _; + use vector_lib::internal_event::{ComponentEventsDropped, UNINTENTIONAL}; let mut encoder = self.1.clone(); let mut byte_size = telemetry().create_request_count_byte_size(); @@ -122,7 +123,22 @@ impl Encoder> for (Transformer, vector_lib::codecs::BatchEncoder) { let mut bytes = BytesMut::new(); encoder .encode(transformed_events, &mut bytes) - .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?; + .map_err(|error| { + // Codec error paths emit their own internal event + // (e.g. SchemaGenerationError, EncoderNullConstraintError, + // EncoderRecordBatchError) which logs the error and increments + // component_errors_total. We only emit the drop count here to + // avoid double-counting. + // n_events is the pre-filter count; Parquet filters non-log + // events before encoding, but that only happens if a sink is + // misconfigured to send non-log events into a log-only encoder, + // so the overcount is not a practical concern. + emit!(ComponentEventsDropped:: { + count: n_events, + reason: "Failed to batch encode events.", + }); + io::Error::new(io::ErrorKind::InvalidData, error) + })?; write_all(writer, n_events, &bytes)?; Ok((bytes.len(), byte_size)) @@ -202,6 +218,7 @@ mod tests { use std::{collections::BTreeMap, env, path::PathBuf}; use bytes::{BufMut, Bytes}; + use cfg_if::cfg_if; use vector_lib::{ codecs::{ CharacterDelimitedEncoder, JsonSerializerConfig, LengthDelimitedEncoder, @@ -214,6 +231,17 @@ mod tests { }; use vrl::value::{KeyString, Value}; + cfg_if! { + if #[cfg(feature = "codecs-arrow")] { + use arrow::datatypes::{DataType, Field, Schema as ArrowSchema}; + use vector_lib::codecs::{ + BatchEncoder, + encoding::{ArrowStreamSerializer, ArrowStreamSerializerConfig, BatchSerializer}, + }; + use vector_lib::event_test_util::{clear_recorded_events, contains_name_once}; + } + } + use super::*; #[test] @@ -558,4 +586,35 @@ mod tests { assert_eq!(CountByteSize(2, input_json_size), size.size().unwrap()); assert_eq!(Bytes::copy_from_slice(&writer), expected_bytes); } + + #[cfg(feature = "codecs-arrow")] + #[test] + fn test_encode_batch_arrow_emits_record_batch_error_on_type_mismatch() { + clear_recorded_events(); + + // Schema declares `message` as Int64, but the event below carries a string, + // so `build_record_batch` returns `ArrowEncodingError::ArrowJsonDecode`. + let schema = ArrowSchema::new(vec![Field::new("message", DataType::Int64, false)]); + let serializer = ArrowStreamSerializer::new(ArrowStreamSerializerConfig::new(schema)) + .expect("failed to build ArrowStreamSerializer"); + let encoder = BatchEncoder::new(BatchSerializer::Arrow(serializer)); + let encoding = (Transformer::default(), encoder); + + let event = Event::Log(LogEvent::from(BTreeMap::from([( + KeyString::from("message"), + Value::from("not_an_integer"), + )]))); + + let mut writer = Vec::new(); + let result = encoding.encode_input(vec![event], &mut writer); + assert!( + result.is_err(), + "type mismatch should fail batch encoding, got {result:?}" + ); + + contains_name_once("EncoderRecordBatchError") + .expect("EncoderRecordBatchError should be emitted on ArrowJsonDecode failure"); + contains_name_once("ComponentEventsDropped") + .expect("ComponentEventsDropped should be emitted by the wrapper"); + } } From d6cdf031d16a382a38046127fbc7ff30c2457709 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 28 Apr 2026 17:40:45 -0400 Subject: [PATCH 176/326] fix(releasing): enable codecs-parquet in all release feature sets (#25321) * fix(releasing): enable codecs-parquet in all release feature sets The aws_s3 sink's `batch_encoding` field is gated behind the `codecs-parquet` Cargo feature, but the v0.55.0 release artifacts did not include that feature. Users running the precompiled binaries hit `unknown field batch_encoding` even though the feature was advertised in the v0.55.0 release notes. Enable `codecs-parquet` in `target-base` (covers every Linux release triple), `default` (macOS), and `default-msvc` (Windows), plus the related aggregator features for consistency. Closes #25313 Co-Authored-By: Claude Opus 4.7 (1M context) * fix(releasing): correct changelog author handle Co-Authored-By: Claude Opus 4.7 (1M context) * chore(releasing): trim changelog wording Co-Authored-By: Claude Opus 4.7 (1M context) * chore(external docs): regenerate component docs with parquet codec Adds the missing `docs::enum_tag_description` metadata on `ParquetCompression` (required by the doc generator for internally tagged enums) and regenerates the aws_s3 and clickhouse component Cue files now that `codecs-parquet` and `codecs-arrow` are part of the default release feature set. Co-Authored-By: Claude Opus 4.7 (1M context) * docs: address review wording on batch encoding fields Apply @maycmlee's suggestions in the source rustdocs (and regenerate the affected Cue): - aws_s3 `batch_encoding`: `e.g., Parquet` -> `for example, Parquet` - arrow `allow_nullable_fields`: tighter wording for both enabled and disabled cases - parquet `Zstd`/`Gzip` `level`: drop "currently" from "Vector currently supports" Co-Authored-By: Claude Opus 4.7 (1M context) * refactor(releasing): consolidate codecs-parquet into base feature Every release feature aggregator (default, default-cmake, default-msvc, default-musl, default-no-api-client, target-base) transitively pulls `base` either directly or via `enable-api-client`/`enable-unix`. Drop the redundant per-aggregator listings and add `codecs-parquet` to `base` once so the feature can't be silently lost when a future aggregator gets added. Co-Authored-By: Claude Opus 4.7 (1M context) * docs(aws_s3, clickhouse): note supported batch_encoding codec per sink The `batch_encoding` field on both sinks types as `Option`, which advertises every codec variant in the generated reference. Each sink only accepts a subset at runtime: aws_s3 rejects everything except `parquet`, clickhouse rejects `parquet`. Call out the supported codec in the field's rustdoc so the published reference matches what the sink will actually accept. A follow-up should replace the shared enum with sink-specific config types so the type system enforces the constraint. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) --- Cargo.toml | 2 +- ...3_enable_codecs_parquet_in_releases.fix.md | 3 + lib/codecs/src/encoding/format/arrow.rs | 6 +- lib/codecs/src/encoding/format/parquet.rs | 7 +- src/sinks/aws_s3/config.rs | 4 +- src/sinks/clickhouse/config.rs | 2 + .../components/sinks/generated/aws_s3.cue | 99 +++++++++++++++++++ .../components/sinks/generated/clickhouse.cue | 86 +++++++++++++--- 8 files changed, 186 insertions(+), 23 deletions(-) create mode 100644 changelog.d/25313_enable_codecs_parquet_in_releases.fix.md diff --git a/Cargo.toml b/Cargo.toml index 70f9fc12a8acc..52161db4b2905 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -519,7 +519,7 @@ vendored = ["rdkafka?/gssapi-vendored"] # Default features for *-pc-windows-msvc # TODO: Enable SASL https://github.com/vectordotdev/vector/pull/3081#issuecomment-659298042 -base = ["api", "enrichment-tables", "sinks", "sources", "transforms", "secrets", "vrl/stdlib"] +base = ["api", "enrichment-tables", "sinks", "sources", "transforms", "secrets", "vrl/stdlib", "codecs-parquet"] enable-api-client = ["base", "api-client"] enable-unix = ["enable-api-client", "sources-dnstap", "unix"] diff --git a/changelog.d/25313_enable_codecs_parquet_in_releases.fix.md b/changelog.d/25313_enable_codecs_parquet_in_releases.fix.md new file mode 100644 index 0000000000000..fe3baaf110d5e --- /dev/null +++ b/changelog.d/25313_enable_codecs_parquet_in_releases.fix.md @@ -0,0 +1,3 @@ +Parquet encoding in the `aws_s3` sink (`batch_encoding`) now works out of the box in the official release binaries. Previously it required compiling Vector from source with the `codecs-parquet` feature. + +authors: pront diff --git a/lib/codecs/src/encoding/format/arrow.rs b/lib/codecs/src/encoding/format/arrow.rs index 6c600b3777d18..e4b3ca565887f 100644 --- a/lib/codecs/src/encoding/format/arrow.rs +++ b/lib/codecs/src/encoding/format/arrow.rs @@ -40,12 +40,12 @@ pub struct ArrowStreamSerializerConfig { /// Allow null values for non-nullable fields in the schema. /// - /// When enabled, missing or incompatible values will be encoded as null even for fields + /// When enabled, missing or incompatible values are encoded as null, even for fields /// marked as non-nullable in the Arrow schema. This is useful when working with downstream /// systems that can handle null values through defaults, computed columns, or other mechanisms. /// - /// When disabled (default), missing values for non-nullable fields will cause encoding errors, - /// ensuring all required data is present before sending to the sink. + /// When disabled (default), missing values for non-nullable fields results in encoding errors. This is to + /// help ensure all required data is present before sending it to the sink. #[serde(default)] #[configurable(derived)] pub allow_nullable_fields: bool, diff --git a/lib/codecs/src/encoding/format/parquet.rs b/lib/codecs/src/encoding/format/parquet.rs index adb5b4082749b..b8874d9b1de90 100644 --- a/lib/codecs/src/encoding/format/parquet.rs +++ b/lib/codecs/src/encoding/format/parquet.rs @@ -35,17 +35,20 @@ type EventsDroppedError = ComponentEventsDropped<'static, UNINTENTIONAL>; /// Compression algorithm and optional level for archive objects. #[configurable_component] #[derive(Default, Copy, Clone, Debug, PartialEq)] +#[configurable(metadata( + docs::enum_tag_description = "Compression codec applied per column page inside the Parquet file." +))] #[serde(tag = "algorithm", rename_all = "snake_case")] pub enum ParquetCompression { /// Zstd compression. Level must be between 1 and 21. Zstd { - /// Compression level (1–21). This is the range Vector currently supports; higher values compress more but are slower. + /// Compression level (1–21). This is the range Vector supports; higher values compress more but are slower. #[configurable(validation(range(min = 1, max = 21)))] level: u8, }, /// Gzip compression. Level must be between 1 and 9. Gzip { - /// Compression level (1–9). This is the range Vector currently supports; higher values compress more but are slower. + /// Compression level (1–9). This is the range Vector supports; higher values compress more but are slower. #[configurable(validation(range(min = 1, max = 9)))] level: u8, }, diff --git a/src/sinks/aws_s3/config.rs b/src/sinks/aws_s3/config.rs index 61b82105a1211..6e8d2e248c518 100644 --- a/src/sinks/aws_s3/config.rs +++ b/src/sinks/aws_s3/config.rs @@ -111,9 +111,11 @@ pub struct S3SinkConfig { /// Batch encoding configuration for columnar formats. /// - /// When set, events are encoded together as a batch in a columnar format (e.g., Parquet) + /// When set, events are encoded together as a batch in a columnar format (for example, Parquet) /// instead of the standard per-event framing-based encoding. The columnar format handles /// its own internal compression, so the top-level `compression` setting is bypassed. + /// + /// Only the `parquet` codec is supported by the AWS S3 sink. #[cfg(feature = "codecs-parquet")] #[configurable(derived)] #[serde(default)] diff --git a/src/sinks/clickhouse/config.rs b/src/sinks/clickhouse/config.rs index d7af3413c6b97..435b2504e6a9e 100644 --- a/src/sinks/clickhouse/config.rs +++ b/src/sinks/clickhouse/config.rs @@ -104,6 +104,8 @@ pub struct ClickhouseConfig { /// /// When specified, events are encoded together as a single batch. /// This is mutually exclusive with per-event encoding based on the `format` field. + /// + /// Only the `arrow_stream` codec is supported by the ClickHouse sink. #[configurable(derived)] #[serde(default)] pub batch_encoding: Option, diff --git a/website/cue/reference/components/sinks/generated/aws_s3.cue b/website/cue/reference/components/sinks/generated/aws_s3.cue index 6c6dffb4fb3cb..1a4713e2897dd 100644 --- a/website/cue/reference/components/sinks/generated/aws_s3.cue +++ b/website/cue/reference/components/sinks/generated/aws_s3.cue @@ -256,6 +256,105 @@ generated: components: sinks: aws_s3: configuration: { } } } + batch_encoding: { + description: """ + Batch encoding configuration for columnar formats. + + When set, events are encoded together as a batch in a columnar format (for example, Parquet) + instead of the standard per-event framing-based encoding. The columnar format handles + its own internal compression, so the top-level `compression` setting is bypassed. + + Only the `parquet` codec is supported by the AWS S3 sink. + """ + required: false + type: object: options: { + allow_nullable_fields: { + description: """ + Allow null values for non-nullable fields in the schema. + + When enabled, missing or incompatible values are encoded as null, even for fields + marked as non-nullable in the Arrow schema. This is useful when working with downstream + systems that can handle null values through defaults, computed columns, or other mechanisms. + + When disabled (default), missing values for non-nullable fields results in encoding errors. This is to + help ensure all required data is present before sending it to the sink. + """ + relevant_when: "codec = \"arrow_stream\"" + required: false + type: bool: default: false + } + codec: { + description: "The codec to use for batch encoding events." + required: true + type: string: enum: { + arrow_stream: """ + Encodes events in [Apache Arrow][apache_arrow] IPC streaming format. + + This is the streaming variant of the Arrow IPC format, which writes + a continuous stream of record batches. + + [apache_arrow]: https://arrow.apache.org/ + """ + parquet: """ + Encodes events in [Apache Parquet][apache_parquet] columnar format. + + [apache_parquet]: https://parquet.apache.org/ + """ + } + } + compression: { + description: "Compression codec applied per column page inside the Parquet file." + relevant_when: "codec = \"parquet\"" + required: false + type: object: options: { + algorithm: { + description: "Compression codec applied per column page inside the Parquet file." + required: false + type: string: { + default: "snappy" + enum: { + gzip: "Gzip compression. Level must be between 1 and 9." + lz4: "LZ4 raw compression" + none: "No compression" + snappy: "Snappy compression (no level)." + zstd: "Zstd compression. Level must be between 1 and 21." + } + } + } + level: { + description: "Compression level (1–21). This is the range Vector supports; higher values compress more but are slower." + relevant_when: "algorithm = \"zstd\" or algorithm = \"gzip\"" + required: true + type: uint: {} + } + } + } + schema_file: { + description: """ + Path to a native Parquet schema file (`.schema`). + + Required unless `schema_mode` is `auto_infer`. The file must contain a valid + Parquet message type definition. + """ + relevant_when: "codec = \"parquet\"" + required: false + type: string: {} + } + schema_mode: { + description: "Controls how events with fields not present in the schema are handled." + relevant_when: "codec = \"parquet\"" + required: false + type: string: { + default: "relaxed" + enum: { + auto_infer: "Auto infer schema based on the batch. No schema file needed." + relaxed: "Missing fields become null. Extra fields are silently dropped." + strict: "Missing fields become null. Extra fields cause an error." + } + } + } + } + } bucket: { description: """ The S3 bucket name. diff --git a/website/cue/reference/components/sinks/generated/clickhouse.cue b/website/cue/reference/components/sinks/generated/clickhouse.cue index a0f4399bdc112..547fbc943e9d9 100644 --- a/website/cue/reference/components/sinks/generated/clickhouse.cue +++ b/website/cue/reference/components/sinks/generated/clickhouse.cue @@ -249,6 +249,8 @@ generated: components: sinks: clickhouse: configuration: { When specified, events are encoded together as a single batch. This is mutually exclusive with per-event encoding based on the `format` field. + + Only the `arrow_stream` codec is supported by the ClickHouse sink. """ required: false type: object: options: { @@ -256,34 +258,86 @@ generated: components: sinks: clickhouse: configuration: { description: """ Allow null values for non-nullable fields in the schema. - When enabled, missing or incompatible values will be encoded as null even for fields + When enabled, missing or incompatible values are encoded as null, even for fields marked as non-nullable in the Arrow schema. This is useful when working with downstream systems that can handle null values through defaults, computed columns, or other mechanisms. - When disabled (default), missing values for non-nullable fields will cause encoding errors, - ensuring all required data is present before sending to the sink. + When disabled (default), missing values for non-nullable fields results in encoding errors. This is to + help ensure all required data is present before sending it to the sink. """ - required: false + relevant_when: "codec = \"arrow_stream\"" + required: false type: bool: default: false } codec: { - description: """ - Encodes events in [Apache Arrow][apache_arrow] IPC streaming format. + description: "The codec to use for batch encoding events." + required: true + type: string: enum: { + arrow_stream: """ + Encodes events in [Apache Arrow][apache_arrow] IPC streaming format. - This is the streaming variant of the Arrow IPC format, which writes - a continuous stream of record batches. + This is the streaming variant of the Arrow IPC format, which writes + a continuous stream of record batches. - [apache_arrow]: https://arrow.apache.org/ - """ - required: true - type: string: enum: arrow_stream: """ - Encodes events in [Apache Arrow][apache_arrow] IPC streaming format. + [apache_arrow]: https://arrow.apache.org/ + """ + parquet: """ + Encodes events in [Apache Parquet][apache_parquet] columnar format. - This is the streaming variant of the Arrow IPC format, which writes - a continuous stream of record batches. + [apache_parquet]: https://parquet.apache.org/ + """ + } + } + compression: { + description: "Compression codec applied per column page inside the Parquet file." + relevant_when: "codec = \"parquet\"" + required: false + type: object: options: { + algorithm: { + description: "Compression codec applied per column page inside the Parquet file." + required: false + type: string: { + default: "snappy" + enum: { + gzip: "Gzip compression. Level must be between 1 and 9." + lz4: "LZ4 raw compression" + none: "No compression" + snappy: "Snappy compression (no level)." + zstd: "Zstd compression. Level must be between 1 and 21." + } + } + } + level: { + description: "Compression level (1–21). This is the range Vector supports; higher values compress more but are slower." + relevant_when: "algorithm = \"zstd\" or algorithm = \"gzip\"" + required: true + type: uint: {} + } + } + } + schema_file: { + description: """ + Path to a native Parquet schema file (`.schema`). - [apache_arrow]: https://arrow.apache.org/ + Required unless `schema_mode` is `auto_infer`. The file must contain a valid + Parquet message type definition. """ + relevant_when: "codec = \"parquet\"" + required: false + type: string: {} + } + schema_mode: { + description: "Controls how events with fields not present in the schema are handled." + relevant_when: "codec = \"parquet\"" + required: false + type: string: { + default: "relaxed" + enum: { + auto_infer: "Auto infer schema based on the batch. No schema file needed." + relaxed: "Missing fields become null. Extra fields are silently dropped." + strict: "Missing fields become null. Extra fields cause an error." + } + } } } } From 233a35c47eab1a0691b39e4af06991dfe4b0f571 Mon Sep 17 00:00:00 2001 From: Thomas Date: Tue, 28 Apr 2026 19:33:23 -0400 Subject: [PATCH 177/326] fix(ci): correct cross-build artifact name and path (#25282) --- .github/workflows/cross.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cross.yml b/.github/workflows/cross.yml index bc2b2b559503b..ec0a7f33f7076 100644 --- a/.github/workflows/cross.yml +++ b/.github/workflows/cross.yml @@ -57,5 +57,5 @@ jobs: - run: make cross-build-${{ matrix.target }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: "vector-debug-${{ matrix.target }}" - path: "./target/${{ matrix.target }}/debug/vector" + name: "vector-${{ matrix.target }}" + path: "./target/${{ matrix.target }}/release/vector" From 8b465f6406fd088302d42f7e879297937bb8783f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 29 Apr 2026 17:44:41 +0000 Subject: [PATCH 178/326] chore(deps): bump the patches group across 1 directory with 13 updates (#25283) * chore(deps): bump the patches group across 1 directory with 13 updates Bumps the patches group with 13 updates in the / directory: | Package | From | To | | --- | --- | --- | | [async-rs](https://github.com/amqp-rs/async-rs) | `0.8.1` | `0.8.4` | | [async-compression](https://github.com/Nullus157/async-compression) | `0.4.41` | `0.4.42` | | [inventory](https://github.com/dtolnay/inventory) | `0.3.22` | `0.3.24` | | [postgres-openssl](https://github.com/rust-postgres/rust-postgres) | `0.5.2` | `0.5.3` | | [pulsar](https://github.com/streamnative/pulsar-rs) | `6.7.1` | `6.7.2` | | [roaring](https://github.com/RoaringBitmap/roaring-rs) | `0.11.3` | `0.11.4` | | [libc](https://github.com/rust-lang/libc) | `0.2.182` | `0.2.186` | | [pastey](https://github.com/as1100k/pastey) | `0.2.1` | `0.2.2` | | [semver](https://github.com/dtolnay/semver) | `1.0.27` | `1.0.28` | | [env_logger](https://github.com/rust-cli/env_logger) | `0.11.6` | `0.11.9` | | [quote](https://github.com/dtolnay/quote) | `1.0.44` | `1.0.45` | | [schannel](https://github.com/steffengy/schannel-rs) | `0.1.28` | `0.1.29` | | [web-sys](https://github.com/wasm-bindgen/wasm-bindgen) | `0.3.91` | `0.3.97` | Updates `async-rs` from 0.8.1 to 0.8.4 - [Commits](https://github.com/amqp-rs/async-rs/compare/v0.8.1...v0.8.4) Updates `async-compression` from 0.4.41 to 0.4.42 - [Release notes](https://github.com/Nullus157/async-compression/releases) - [Commits](https://github.com/Nullus157/async-compression/compare/async-compression-v0.4.41...async-compression-v0.4.42) Updates `inventory` from 0.3.22 to 0.3.24 - [Release notes](https://github.com/dtolnay/inventory/releases) - [Commits](https://github.com/dtolnay/inventory/compare/0.3.22...0.3.24) Updates `postgres-openssl` from 0.5.2 to 0.5.3 - [Release notes](https://github.com/rust-postgres/rust-postgres/releases) - [Commits](https://github.com/rust-postgres/rust-postgres/compare/postgres-openssl-v0.5.2...postgres-openssl-v0.5.3) Updates `pulsar` from 6.7.1 to 6.7.2 - [Release notes](https://github.com/streamnative/pulsar-rs/releases) - [Commits](https://github.com/streamnative/pulsar-rs/compare/v6.7.1...v6.7.2) Updates `roaring` from 0.11.3 to 0.11.4 - [Release notes](https://github.com/RoaringBitmap/roaring-rs/releases) - [Commits](https://github.com/RoaringBitmap/roaring-rs/compare/v0.11.3...v0.11.4) Updates `libc` from 0.2.182 to 0.2.186 - [Release notes](https://github.com/rust-lang/libc/releases) - [Changelog](https://github.com/rust-lang/libc/blob/0.2.186/CHANGELOG.md) - [Commits](https://github.com/rust-lang/libc/compare/0.2.182...0.2.186) Updates `pastey` from 0.2.1 to 0.2.2 - [Release notes](https://github.com/as1100k/pastey/releases) - [Changelog](https://github.com/AS1100K/pastey/blob/master/CHANGELOG.md) - [Commits](https://github.com/as1100k/pastey/compare/v0.2.1...v0.2.2) Updates `semver` from 1.0.27 to 1.0.28 - [Release notes](https://github.com/dtolnay/semver/releases) - [Commits](https://github.com/dtolnay/semver/compare/1.0.27...1.0.28) Updates `env_logger` from 0.11.6 to 0.11.9 - [Release notes](https://github.com/rust-cli/env_logger/releases) - [Changelog](https://github.com/rust-cli/env_logger/blob/main/CHANGELOG.md) - [Commits](https://github.com/rust-cli/env_logger/compare/v0.11.6...v0.11.9) Updates `quote` from 1.0.44 to 1.0.45 - [Release notes](https://github.com/dtolnay/quote/releases) - [Commits](https://github.com/dtolnay/quote/compare/1.0.44...1.0.45) Updates `schannel` from 0.1.28 to 0.1.29 - [Release notes](https://github.com/steffengy/schannel-rs/releases) - [Commits](https://github.com/steffengy/schannel-rs/compare/v0.1.28...v0.1.29) Updates `web-sys` from 0.3.91 to 0.3.97 - [Release notes](https://github.com/wasm-bindgen/wasm-bindgen/releases) - [Changelog](https://github.com/wasm-bindgen/wasm-bindgen/blob/main/CHANGELOG.md) - [Commits](https://github.com/wasm-bindgen/wasm-bindgen/commits) --- updated-dependencies: - dependency-name: async-compression dependency-version: 0.4.42 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: async-rs dependency-version: 0.8.4 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: env_logger dependency-version: 0.11.9 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: inventory dependency-version: 0.3.24 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: libc dependency-version: 0.2.186 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: pastey dependency-version: 0.2.2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: postgres-openssl dependency-version: 0.5.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: pulsar dependency-version: 6.7.2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: quote dependency-version: 1.0.45 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: roaring dependency-version: 0.11.4 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: schannel dependency-version: 0.1.29 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: semver dependency-version: 1.0.28 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches - dependency-name: web-sys dependency-version: 0.3.95 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: patches ... Signed-off-by: dependabot[bot] * update licenses --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Pavlos Rontidis --- Cargo.lock | 433 +++++++++++++++++++----------- Cargo.toml | 2 +- LICENSE-3rdparty.csv | 7 + lib/file-source-common/Cargo.toml | 2 +- lib/file-source/Cargo.toml | 2 +- lib/vector-core/Cargo.toml | 2 +- 6 files changed, 293 insertions(+), 155 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0ad2d1635d855..52ed5cd06c593 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -327,7 +327,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c6368f9ae5c6ec403ca910327ae0c9437b0a85255b6950c90d497e6177f6e5e" dependencies = [ "proc-macro-hack", - "quote 1.0.44", + "quote 1.0.45", "syn 1.0.109", ] @@ -847,9 +847,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.41" +version = "0.4.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0f9ee0f6e02ffd7ad5816e9464499fba7b3effd01123b515c41d1697c43dad1" +checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac" dependencies = [ "compression-codecs", "compression-core", @@ -1019,15 +1019,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] [[package]] name = "async-rs" -version = "0.8.1" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d47e5e0df9bce1f132f67a64409a973b031481366b858330013060ca76b3a80" +checksum = "31ebdd144e4199c67b5134fe2280e40d9656071f11df525d3322b43b6534c714" dependencies = [ "async-compat", "async-global-executor", @@ -1035,7 +1035,7 @@ dependencies = [ "cfg-if", "futures-core", "futures-io", - "hickory-resolver 0.25.2", + "hickory-resolver 0.26.0", "tokio", "tokio-stream", ] @@ -1076,7 +1076,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -1093,7 +1093,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -1940,7 +1940,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94f7cc1bbae04cfe11de9e39e2c6dc755947901e0f4e76180ab542b6deb5e15e" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", "tracing 0.1.44", ] @@ -2069,7 +2069,7 @@ dependencies = [ "clang-sys", "itertools 0.13.0", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "regex", "rustc-hash", "shlex", @@ -2112,7 +2112,7 @@ version = "2.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6cbbb8f56245b5a479b30a62cdc86d26e2f35c2b9f594bc4671654b03851380" dependencies = [ - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -2238,7 +2238,7 @@ dependencies = [ "ident_case", "prettyplease", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "rustversion", "syn 2.0.117", ] @@ -2268,7 +2268,7 @@ dependencies = [ "once_cell", "proc-macro-crate 3.2.0", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -2362,7 +2362,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7ec4c6f261935ad534c0c22dbef2201b45918860eb1c574b972bd213a76af61" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 1.0.109", ] @@ -2667,7 +2667,7 @@ checksum = "a92793da1a46a5f2a02a6f4c46c6496b28c43638adea8306fcb0caa1634f24e5" dependencies = [ "heck 0.5.0", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -2841,9 +2841,9 @@ dependencies = [ [[package]] name = "compression-codecs" -version = "0.4.37" +version = "0.4.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb7b51a7d9c967fc26773061ba86150f19c50c0d65c887cb1fbe295fd16619b7" +checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" dependencies = [ "brotli", "compression-core", @@ -2855,9 +2855,9 @@ dependencies = [ [[package]] name = "compression-core" -version = "0.4.31" +version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75984efb6ed102a0d42db99afb6c1948f0380d1d91808d5529916e6c08b49d8d" +checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" [[package]] name = "concurrent-queue" @@ -3352,7 +3352,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -3385,7 +3385,7 @@ dependencies = [ "fnv", "ident_case", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "strsim", "syn 2.0.117", ] @@ -3399,7 +3399,7 @@ dependencies = [ "fnv", "ident_case", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "strsim", "syn 2.0.117", ] @@ -3411,7 +3411,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core 0.20.11", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -3422,7 +3422,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" dependencies = [ "darling_core 0.21.3", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -3555,7 +3555,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 1.0.109", ] @@ -3566,7 +3566,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d65d7ce8132b7c0e54497a4d9a55a1c2a0912a0d786cf894472ba818fba45762" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -3577,7 +3577,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef941ded77d15ca19b40374869ac6000af1c9f2a4c0f3d4c70926287e6364a8f" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -3588,7 +3588,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -3609,7 +3609,7 @@ checksum = "d48cda787f839151732d396ac69e3473923d54312c070ee21e9effcaa8ca0b1d" dependencies = [ "darling 0.20.11", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -3631,7 +3631,7 @@ checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" dependencies = [ "convert_case 0.4.0", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "rustc_version", "syn 1.0.109", ] @@ -3653,7 +3653,7 @@ checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" dependencies = [ "convert_case 0.10.0", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "rustc_version", "syn 2.0.117", "unicode-xid 0.2.4", @@ -3726,7 +3726,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -3832,7 +3832,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e197fdfd2cdb5fdeb7f8ddcf3aed5d5d04ecde2890d448b14ffb716f7376b70" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -3907,7 +3907,7 @@ checksum = "1d7bc049e1bd8cdeb31b68bbd586a9464ecf9f3944af3958a7a9d0f8b9799417" dependencies = [ "enum-ordinalize", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -4009,7 +4009,7 @@ checksum = "5ffccbb6966c05b32ef8fbac435df276c4ae4d3dc55a8cd0eb9745e6c12f546a" dependencies = [ "heck 0.4.1", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -4029,7 +4029,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ca9601fb2d62598ee17836250842873a413586e5d7ed88b356e38ddbb0ec631" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -4041,7 +4041,7 @@ checksum = "aa18ce2bc66555b3218614519ac839ddb759a7d6720732f979ef8d13be147ecd" dependencies = [ "once_cell", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -4061,7 +4061,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67c78a4d8fdf9953a5c9d458f9efe940fd97a0cab0941c075a813ac594733827" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -4073,9 +4073,9 @@ checksum = "62a61b2faff777e62dbccd7f82541d873f96264d050c5dd7e95194f79fc4de29" [[package]] name = "env_filter" -version = "0.1.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a009aa4810eb158359dda09d0c87378e4bbb89b5a801f016885a4707ba24f7ea" +checksum = "32e90c2accc4b07a8456ea0debdc2e7587bdd890680d71173a15d4ae604f6eef" dependencies = [ "log", "regex", @@ -4083,14 +4083,14 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.11.6" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcaee3d8e3cfc3fd92428d477bc97fc29ec8716d180c0d74c643bb26166660e0" +checksum = "b2daee4ea451f429a58296525ddf28b45a3b64f1acf6587e2067437bb11e218d" dependencies = [ "anstream", "anstyle", "env_filter", - "humantime", + "jiff", "log", ] @@ -4190,7 +4190,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "332b1937705b7ed2fce76837024e9ae6f41cd2ad18a32c052de081f89982561b" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 1.0.109", ] @@ -4530,7 +4530,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -5088,6 +5088,30 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hickory-net" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c61c8db47fae51ba9f8f2a2748bd87542acfbe22f2ec9cf9c8ec72d1ee6e9a6" +dependencies = [ + "async-trait", + "cfg-if", + "data-encoding", + "futures-channel", + "futures-io", + "futures-util", + "hickory-proto 0.26.0", + "idna", + "ipnet", + "jni 0.22.4", + "rand 0.10.1", + "thiserror 2.0.17", + "tinyvec", + "tokio", + "tracing 0.1.44", + "url", +] + [[package]] name = "hickory-proto" version = "0.24.4" @@ -5135,7 +5159,26 @@ dependencies = [ "thiserror 2.0.17", "time", "tinyvec", - "tokio", + "tracing 0.1.44", + "url", +] + +[[package]] +name = "hickory-proto" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a916d0494600d99ecb15aadfab677ad97c4de559e8f1af0c129353a733ac1fcc" +dependencies = [ + "data-encoding", + "idna", + "ipnet", + "jni 0.22.4", + "once_cell", + "prefix-trie", + "rand 0.10.1", + "ring", + "thiserror 2.0.17", + "tinyvec", "tracing 0.1.44", "url", ] @@ -5163,20 +5206,25 @@ dependencies = [ [[package]] name = "hickory-resolver" -version = "0.25.2" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc62a9a99b0bfb44d2ab95a7208ac952d31060efc16241c87eaf36406fecf87a" +checksum = "a10bd64d950b4d38ca21e25c8ae230712e4955fb8290cfcb29a5e5dc6017e544" dependencies = [ "cfg-if", "futures-util", - "hickory-proto 0.25.2", + "hickory-net", + "hickory-proto 0.26.0", "ipconfig", + "ipnet", + "jni 0.22.4", "moka", + "ndk-context", "once_cell", "parking_lot", - "rand 0.9.4", + "rand 0.10.1", "resolv-conf", "smallvec", + "system-configuration 0.7.0", "thiserror 2.0.17", "tokio", "tracing 0.1.44", @@ -5703,7 +5751,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -5835,7 +5883,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b23a0c8dfe501baac4adf6ebbfa6eddf8f0c07f56b058cc1288017e32397846c" dependencies = [ - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -5847,9 +5895,9 @@ checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" [[package]] name = "inventory" -version = "0.3.22" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "009ae045c87e7082cb72dab0ccd01ae075dd00141ddc108f43a0ea150a9e7227" +checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b" dependencies = [ "rustversion", ] @@ -5988,7 +6036,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -6001,19 +6049,68 @@ dependencies = [ "cesu8", "cfg-if", "combine", - "jni-sys", + "jni-sys 0.3.0", "log", "thiserror 1.0.68", "walkdir", "windows-sys 0.45.0", ] +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys 0.4.1", + "log", + "simd_cesu8", + "thiserror 2.0.17", + "walkdir", + "windows-link 0.2.0", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2 1.0.106", + "quote 1.0.45", + "rustc_version", + "simd_cesu8", + "syn 2.0.117", +] + [[package]] name = "jni-sys" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote 1.0.45", + "syn 2.0.117", +] + [[package]] name = "jobserver" version = "0.1.32" @@ -6025,10 +6122,12 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.91" +version = "0.3.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b49715b7073f385ba4bc528e5747d02e66cb39c6146efb66b781f131f0fb399c" +checksum = "a1840c94c045fbcf8ba2812c95db44499f7c64910a912551aaaa541decebcacf" dependencies = [ + "cfg-if", + "futures-util", "once_cell", "wasm-bindgen", ] @@ -6398,9 +6497,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.182" +version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "libflate" @@ -6695,7 +6794,7 @@ checksum = "cc33f9f0351468d26fbc53d9ce00a096c8522ecb42f19b50f34f2c422f76d21d" dependencies = [ "macro_magic_core", "macro_magic_macros", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -6709,7 +6808,7 @@ dependencies = [ "derive-syn-parse", "macro_magic_core_macros", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -6720,7 +6819,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b02abfe41815b5bd98dbd4260173db2c116dda171dc0fe7838cb206333b83308" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -6731,7 +6830,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73ea28ee64b88876bf45277ed9a5817c1817df061a74f2b988971a12570e5869" dependencies = [ "macro_magic_core", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -6916,7 +7015,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd2209fff77f705b00c737016a48e73733d7fbccb8b007194db148f03561fb70" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -6988,7 +7087,7 @@ dependencies = [ "once_cell", "proc-macro-error2", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "regex", "syn 2.0.117", ] @@ -7099,7 +7198,7 @@ checksum = "63981427a0f26b89632fd2574280e069d09fb2912a3138da15de0174d11dd077" dependencies = [ "macro_magic", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -7515,7 +7614,7 @@ checksum = "96667db765a921f7b295ffee8b60472b686a51d4f21c2ee4ffdb94c7013b65a6" dependencies = [ "proc-macro-crate 1.3.1", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -7527,7 +7626,7 @@ checksum = "af1844ef2428cc3e1cb900be36181049ef3d3193c63e43026cfe202983b27a56" dependencies = [ "proc-macro-crate 3.2.0", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -7731,7 +7830,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -7948,9 +8047,9 @@ checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" [[package]] name = "pastey" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b867cad97c0791bbd3aaa6472142568c6c9e8f71937e98379f584cfb0cf35bec" +checksum = "c5a797f0e07bdf071d15742978fc3128ec6c22891c31a3a931513263904c982a" [[package]] name = "pbkdf2" @@ -8021,7 +8120,7 @@ dependencies = [ "pest", "pest_meta", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -8107,7 +8206,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9b20ed30f105399776b9c883e68e536ef602a16ae6f596d2c473591d6ad64c6" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -8238,9 +8337,9 @@ dependencies = [ [[package]] name = "postgres-openssl" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f86f073ad570f76e9e278ce6f05775fc723eed7daa6b4f9c2aa078080a564a0" +checksum = "06743eefaa1a5c0ef2ccb6d9abf6528790a229eabd62ddcabf9b2a3aeff09fa4" dependencies = [ "openssl", "tokio", @@ -8324,6 +8423,17 @@ dependencies = [ "termtree", ] +[[package]] +name = "prefix-trie" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23370be78b7e5bcbb0cab4a02047eb040279a693c78daad04c2c5f1c24a83503" +dependencies = [ + "either", + "ipnet", + "num-traits", +] + [[package]] name = "prettydiff" version = "0.8.0" @@ -8392,7 +8502,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", ] [[package]] @@ -8403,7 +8513,7 @@ checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" dependencies = [ "proc-macro-error-attr2", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -8497,7 +8607,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "095a99f75c69734802359b682be8daaf8980296731f6470434ea2c652af1dd30" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -8581,7 +8691,7 @@ dependencies = [ "anyhow", "itertools 0.12.1", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -8594,7 +8704,7 @@ dependencies = [ "anyhow", "itertools 0.14.0", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -8607,7 +8717,7 @@ dependencies = [ "anyhow", "itertools 0.14.0", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -8683,7 +8793,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 1.0.109", ] @@ -8699,9 +8809,9 @@ dependencies = [ [[package]] name = "pulsar" -version = "6.7.1" +version = "6.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24c00856d6c64af2078a74c2de029842cdaece2f35b1223f5166809ea49cf5cd" +checksum = "e2367cb38f1b65857bc11dd13b2adf13b7a1d991ef1cd43572f1420958c56cc2" dependencies = [ "async-channel", "async-trait", @@ -8825,7 +8935,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9a28b8493dd664c8b171dd944da82d933f7d456b829bfb236738e1fe06c5ba4" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -8893,9 +9003,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.44" +version = "1.0.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" dependencies = [ "proc-macro2 1.0.106", ] @@ -9261,7 +9371,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1165225c21bff1f3bbce98f5a1f889949bc902d3575308cc7b0de30b4f6d27c7" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -9550,7 +9660,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "84d7b42d4b8d06048d3ac8db0eb31bcb942cbeb709f0b5f2b2ebde398d3038f5" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 1.0.109", ] @@ -9592,9 +9702,9 @@ dependencies = [ [[package]] name = "roaring" -version = "0.11.3" +version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ba9ce64a8f45d7fc86358410bb1a82e8c987504c0d4900e9141d69a9f26c885" +checksum = "1dedc5658c6ecb3bdb5ef5f3295bb9253f42dcf3fd1402c03f6b1f7659c3c4a9" dependencies = [ "bytemuck", "byteorder", @@ -9650,7 +9760,7 @@ dependencies = [ "glob", "proc-macro-crate 3.2.0", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "regex", "relative-path 1.9.3", "rustc_version", @@ -9955,9 +10065,9 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.28" +version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" dependencies = [ "windows-sys 0.61.0", ] @@ -10081,9 +10191,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.27" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" dependencies = [ "serde", "serde_core", @@ -10162,7 +10272,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -10173,7 +10283,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -10226,7 +10336,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3081f5ffbb02284dda55132aa26daecedd7372a42417bbbab6f14ab7d6bb9145" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -10279,7 +10389,7 @@ checksum = "de90945e6565ce0d9a25098082ed4ee4002e047cb59892c318d66821e14bb30f" dependencies = [ "darling 0.20.11", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -10331,7 +10441,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0a7d91949b85b0d2fb687445e448b40d322b6b3e4af6b44a29b21d9a5f33e6d9" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -10461,6 +10571,16 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" +[[package]] +name = "simd_cesu8" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94f90157bb87cddf702797c5dadfa0be7d266cdf49e22da2fcaa32eff75b2c33" +dependencies = [ + "rustc_version", + "simdutf8", +] + [[package]] name = "simdutf8" version = "0.1.5" @@ -10600,7 +10720,7 @@ checksum = "990079665f075b699031e9c08fd3ab99be5029b96f3b78dc0709e8f77e4efebf" dependencies = [ "heck 0.4.1", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 1.0.109", ] @@ -10612,7 +10732,7 @@ checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" dependencies = [ "heck 0.5.0", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -10624,7 +10744,7 @@ checksum = "54254b8531cafa275c5e096f62d48c81435d1015405a91198ddb11e967301d40" dependencies = [ "heck 0.5.0", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -10751,7 +10871,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2d452988ccaacfbf5e0bdbc348fb91d7c8af5bee192173ac3636b5fb6e6715d" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "sqlx-core", "sqlx-macros-core", "syn 2.0.117", @@ -10769,7 +10889,7 @@ dependencies = [ "hex", "once_cell", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "serde", "serde_json", "sha2", @@ -10988,7 +11108,7 @@ checksum = "23dc1fa9ac9c169a78ba62f0b841814b7abae11bdd047b9c58f893439e309ea0" dependencies = [ "heck 0.4.1", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "rustversion", "syn 2.0.117", ] @@ -11001,7 +11121,7 @@ checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" dependencies = [ "heck 0.5.0", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "rustversion", "syn 2.0.117", ] @@ -11014,7 +11134,7 @@ checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" dependencies = [ "heck 0.5.0", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -11026,7 +11146,7 @@ checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664" dependencies = [ "heck 0.5.0", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -11073,7 +11193,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "unicode-ident", ] @@ -11084,7 +11204,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "unicode-ident", ] @@ -11110,7 +11230,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -11183,6 +11303,17 @@ dependencies = [ "system-configuration-sys 0.6.0", ] +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags 2.10.0", + "core-foundation 0.9.3", + "system-configuration-sys 0.6.0", +] + [[package]] name = "system-configuration-sys" version = "0.5.0" @@ -11337,7 +11468,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7c61ec9a6f64d2793d8a45faba21efbe3ced62a886d44c36a009b2b519b4c7e" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -11348,7 +11479,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -11505,7 +11636,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -11859,7 +11990,7 @@ dependencies = [ "prettyplease", "proc-macro2 1.0.106", "prost-build 0.12.6", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -12042,7 +12173,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -12168,7 +12299,7 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad06847b7afb65c7866a36664b75c40b895e318cea4f71299f013fb22965329d" dependencies = [ - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -12255,7 +12386,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f03ca4cb38206e2bef0700092660bb74d696f808514dae47fa1467cbfe26e96e" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -12266,7 +12397,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c36781cc0e46a83726d9879608e4cf6c2505237e263a8eb8c24502989cfdb28" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -12323,7 +12454,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ecee5b05c459ea4cd97df7685db58699c32e070465f88dc806d7c98a5088edc" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "rustc_version", "syn 2.0.117", ] @@ -12348,7 +12479,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27a7a9b72ba121f6f1f6c3632b85604cac41aedb5ddc70accbebb6cac83de846" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -12912,7 +13043,7 @@ name = "vector-common-macros" version = "0.1.0" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -12948,7 +13079,7 @@ dependencies = [ "convert_case 0.8.0", "darling 0.20.11", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "serde", "serde_json", "syn 2.0.117", @@ -12961,7 +13092,7 @@ version = "0.1.0" dependencies = [ "darling 0.20.11", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "serde", "serde_derive_internals", "syn 2.0.117", @@ -13463,9 +13594,9 @@ checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" [[package]] name = "wasm-bindgen" -version = "0.2.114" +version = "0.2.120" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e" +checksum = "df52b6d9b87e0c74c9edfa1eb2d9bf85e5d63515474513aa50fa181b3c4f5db1" dependencies = [ "cfg-if", "once_cell", @@ -13488,32 +13619,32 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.114" +version = "0.2.120" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18a2d50fcf105fb33bb15f00e7a77b772945a2ee45dcf454961fd843e74c18e6" +checksum = "78b1041f495fb322e64aca85f5756b2172e35cd459376e67f2a6c9dffcedb103" dependencies = [ - "quote 1.0.44", + "quote 1.0.45", "wasm-bindgen-macro-support", ] [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.114" +version = "0.2.120" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03ce4caeaac547cdf713d280eda22a730824dd11e6b8c3ca9e42247b25c631e3" +checksum = "9dcd0ff20416988a18ac686d4d4d0f6aae9ebf08a389ff5d29012b05af2a1b41" dependencies = [ "bumpalo", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.114" +version = "0.2.120" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75a326b8c223ee17883a4251907455a2431acc2791c98c26279376490c378c16" +checksum = "49757b3c82ebf16c57d69365a142940b384176c24df52a087fb748e2085359ea" dependencies = [ "unicode-ident", ] @@ -13581,9 +13712,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.91" +version = "0.3.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "854ba17bb104abfb26ba36da9729addc7ce7f06f5c0f90f3c391f8461cca21f9" +checksum = "2eadbac71025cd7b0834f20d1fe8472e8495821b4e9801eb0a60bd1f19827602" dependencies = [ "js-sys", "wasm-bindgen", @@ -13607,7 +13738,7 @@ checksum = "60b6f804e41d0852e16d2eaee61c7e4f7d3e8ffdb7b8ed85886aeb0791fe9fcd" dependencies = [ "core-foundation 0.9.3", "home", - "jni", + "jni 0.21.1", "log", "ndk-context", "objc", @@ -13786,7 +13917,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2bbd5b46c938e506ecbce286b6628a02171d56153ba733b6c741fc627ec9579b" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -13797,7 +13928,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -13808,7 +13939,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "053c4c462dc91d3b1504c6fe5a726dd15e216ba718e84a0e46a88fbe5ded3515" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -13819,7 +13950,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -14320,7 +14451,7 @@ dependencies = [ "anyhow", "prettyplease", "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", "wit-bindgen-core", "wit-bindgen-rust", @@ -14425,7 +14556,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", "synstructure", ] @@ -14455,7 +14586,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3c129550b3e6de3fd0ba67ba5c81818f9805e58b8d7fee80a3a59d2c9fc601a" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -14466,7 +14597,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5226bc9a9a9836e7428936cde76bb6b22feea1a8bfdbc0d241136e4d13417e25" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] @@ -14486,7 +14617,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", "synstructure", ] @@ -14515,7 +14646,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2 1.0.106", - "quote 1.0.44", + "quote 1.0.45", "syn 2.0.117", ] diff --git a/Cargo.toml b/Cargo.toml index 52161db4b2905..db582e2930491 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -187,7 +187,7 @@ rdkafka = { version = "0.39.0", default-features = false } regex = { version = "1.12.3", default-features = false, features = ["std", "perf"] } reqwest = { version = "0.11", features = ["json"] } rust_decimal = { version = "1.40.0", default-features = false, features = ["std"] } -semver = { version = "1.0.27", default-features = false, features = ["serde", "std"] } +semver = { version = "1.0.28", default-features = false, features = ["serde", "std"] } serde = { version = "1.0.219", default-features = false, features = ["alloc", "derive", "rc"] } serde_json = { version = "1.0.143", default-features = false, features = ["preserve_order", "raw_value", "std"] } serde_yaml = { version = "0.9.34", default-features = false } diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index ea2ff8e85d12e..31c7d69405a20 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -336,6 +336,7 @@ heim-net,https://github.com/heim-rs/heim,Apache-2.0 OR MIT,svartalf hermit-abi,https://github.com/hermit-os/hermit-rs,MIT OR Apache-2.0,Stefan Lankes hex,https://github.com/KokaKiwi/rust-hex,MIT OR Apache-2.0,KokaKiwi +hickory-net,https://github.com/hickory-dns/hickory-dns,MIT OR Apache-2.0,The contributors to Hickory DNS hickory-proto,https://github.com/hickory-dns/hickory-dns,MIT OR Apache-2.0,The contributors to Hickory DNS hickory-resolver,https://github.com/hickory-dns/hickory-dns,MIT OR Apache-2.0,The contributors to Hickory DNS hkdf,https://github.com/RustCrypto/KDFs,MIT OR Apache-2.0,RustCrypto Developers @@ -398,7 +399,11 @@ itoa,https://github.com/dtolnay/itoa,MIT OR Apache-2.0,David Tolnay jiff-static,https://github.com/BurntSushi/jiff,Unlicense OR MIT,Andrew Gallant jni,https://github.com/jni-rs/jni-rs,MIT OR Apache-2.0,Josh Chase +jni,https://github.com/jni-rs/jni-rs,MIT OR Apache-2.0,jni team +jni-macros,https://github.com/jni-rs/jni-rs,MIT OR Apache-2.0,The jni-macros Authors +jni-sys,https://github.com/jni-rs/jni-sys,MIT OR Apache-2.0,"Steven Fackler , Robert Bragg " jni-sys,https://github.com/sfackler/rust-jni-sys,MIT OR Apache-2.0,Steven Fackler +jni-sys-macros,https://github.com/jni-rs/jni-sys,MIT OR Apache-2.0,Robert Bragg js-sys,https://github.com/wasm-bindgen/wasm-bindgen/tree/master/crates/js-sys,MIT OR Apache-2.0,The wasm-bindgen Developers json-patch,https://github.com/idubrov/json-patch,MIT OR Apache-2.0,Ivan Dubrov jsonpath-rust,https://github.com/besok/jsonpath-rust,MIT,BorisZhguchev @@ -568,6 +573,7 @@ postgres-protocol,https://github.com/rust-postgres/rust-postgres,MIT OR Apache-2 postgres-types,https://github.com/rust-postgres/rust-postgres,MIT OR Apache-2.0,Steven Fackler powerfmt,https://github.com/jhpratt/powerfmt,MIT OR Apache-2.0,Jacob Pratt ppv-lite86,https://github.com/cryptocorrosion/cryptocorrosion,MIT OR Apache-2.0,The CryptoCorrosion Contributors +prefix-trie,https://github.com/tiborschneider/prefix-trie,MIT OR Apache-2.0,The prefix-trie Authors prettydiff,https://github.com/romankoblov/prettydiff,MIT,Roman Koblov prettyplease,https://github.com/dtolnay/prettyplease,MIT OR Apache-2.0,David Tolnay prettytable-rs,https://github.com/phsym/prettytable-rs,BSD-3-Clause,Pierre-Henri Symoneaux @@ -709,6 +715,7 @@ signal-hook-registry,https://github.com/vorner/signal-hook,Apache-2.0 OR MIT,"Mi signatory,https://github.com/iqlusioninc/crates/tree/main/signatory,Apache-2.0 OR MIT,Tony Arcieri signature,https://github.com/RustCrypto/traits/tree/master/signature,Apache-2.0 OR MIT,RustCrypto Developers simd-adler32,https://github.com/mcountryman/simd-adler32,MIT,Marvin Countryman +simd_cesu8,https://github.com/seancroach/simd_cesu8,Apache-2.0 OR MIT,Sean C. Roach simdutf8,https://github.com/rusticstuff/simdutf8,MIT OR Apache-2.0,Hans Kratz simpl,https://github.com/durch/simplerr,MIT,Drazen Urch siphasher,https://github.com/jedisct1/rust-siphash,MIT OR Apache-2.0,Frank Denis diff --git a/lib/file-source-common/Cargo.toml b/lib/file-source-common/Cargo.toml index e115f3a74bf52..6d3a885bc9a7a 100644 --- a/lib/file-source-common/Cargo.toml +++ b/lib/file-source-common/Cargo.toml @@ -20,7 +20,7 @@ serde_json = { version = "1.0.143", default-features = false } bstr = { version = "1.12", default-features = false } bytes = { version = "1.11.1", default-features = false, features = ["serde"] } dashmap = { version = "6.1", default-features = false } -async-compression = { version = "0.4.41", features = ["tokio", "gzip"] } +async-compression = { version = "0.4.42", features = ["tokio", "gzip"] } vector-common = { path = "../vector-common", default-features = false } vector-config = { path = "../vector-config", default-features = false } tokio = { workspace = true, features = ["full"] } diff --git a/lib/file-source/Cargo.toml b/lib/file-source/Cargo.toml index 7e8e6b6cd1153..a6b6e94c3b27f 100644 --- a/lib/file-source/Cargo.toml +++ b/lib/file-source/Cargo.toml @@ -21,7 +21,7 @@ futures = { version = "0.3.31", default-features = false, features = ["executor" futures-util.workspace = true vector-common = { path = "../vector-common", default-features = false } file-source-common = { path = "../file-source-common" } -async-compression = { version = "0.4.41", features = ["tokio", "gzip"] } +async-compression = { version = "0.4.42", features = ["tokio", "gzip"] } [dev-dependencies] tokio = { workspace = true, features = ["full"] } diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml index c26953cb3590a..8836c2f70beaf 100644 --- a/lib/vector-core/Cargo.toml +++ b/lib/vector-core/Cargo.toml @@ -71,7 +71,7 @@ quickcheck = { workspace = true, optional = true } security-framework = "3.6.0" [target.'cfg(windows)'.dependencies] -schannel = "0.1.28" +schannel = "0.1.29" [build-dependencies] prost-build.workspace = true From 75b9d07a8231e7c321652e4ff031d8ce0757d9ab Mon Sep 17 00:00:00 2001 From: Thomas Date: Wed, 29 Apr 2026 15:57:15 -0400 Subject: [PATCH 179/326] feat(website): improve search ranking for component reference pages (#25327) * feat(website): improve search ranking for component reference pages * Improve regex match --- website/scripts/typesense-index.ts | 20 ++++++++++++++++---- website/typesense.config.json | 30 ++++++++++++++++++++++++++++-- 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/website/scripts/typesense-index.ts b/website/scripts/typesense-index.ts index f800e710d1754..81c241b331276 100644 --- a/website/scripts/typesense-index.ts +++ b/website/scripts/typesense-index.ts @@ -33,6 +33,8 @@ type AlgoliaRecord = { ranking: number; section: string; content: string; + componentWithType: string; + component: string; }; type Section = { @@ -94,6 +96,11 @@ async function indexHTMLFiles( const pageTitle = $("meta[name='algolia:title']").attr("content") || ""; const pageTagsString = $("meta[name='keywords']").attr("content") || ""; const pageTags: string[] = pageTagsString === "" ? [] : pageTagsString.split(","); + const pageUrl = getPageUrl(file); + const componentMatch = pageUrl.match(/\/docs\/reference\/configuration\/(sources|sinks|transforms)\/([^/]+)/); + const componentKindMap: Record = { sources: "source", sinks: "sink", transforms: "transform" }; + const pageComponent = componentMatch ? componentMatch[2] : ""; + const pageComponentWithType = componentMatch ? `${componentMatch[2]} ${componentKindMap[componentMatch[1]]}` : ""; // @ts-ignore $(".algolia-no-index").each((_, d) => $(d).remove()); @@ -130,7 +137,6 @@ async function indexHTMLFiles( let activeRecord: AlgoliaRecord | null = null; for (const item of payload) { - const pageUrl = getPageUrl(file); const itemUrl = getItemUrl(file, item); const hashedId = hashString(itemUrl); @@ -146,7 +152,9 @@ async function indexHTMLFiles( ranking, hierarchy: [], tags: pageTags, - content: "" + content: "", + componentWithType: pageComponentWithType, + component: pageComponent }; } else if (item.level === 1) { // h1 logic @@ -165,7 +173,9 @@ async function indexHTMLFiles( ranking, hierarchy: [...activeRecord.hierarchy, activeRecord.title.trim()], tags: pageTags, - content: "" + content: "", + componentWithType: pageComponentWithType, + component: pageComponent }; } else { // h2-h6 logic @@ -186,7 +196,9 @@ async function indexHTMLFiles( ranking, hierarchy: [...activeRecord.hierarchy.slice(0, lastIndex)], tags: pageTags, - content: "" + content: "", + componentWithType: pageComponentWithType, + component: pageComponent }; } diff --git a/website/typesense.config.json b/website/typesense.config.json index f0543b9133c8b..5e09a5ee14694 100644 --- a/website/typesense.config.json +++ b/website/typesense.config.json @@ -136,6 +136,32 @@ "type": "string[]", "stem_dictionary": "", "store": true + }, + { + "facet": false, + "index": true, + "infix": false, + "locale": "", + "name": "componentWithType", + "optional": true, + "sort": false, + "stem": false, + "type": "string", + "stem_dictionary": "", + "store": true + }, + { + "facet": false, + "index": true, + "infix": false, + "locale": "", + "name": "component", + "optional": true, + "sort": false, + "stem": false, + "type": "string", + "stem_dictionary": "", + "store": true } ] }, @@ -144,8 +170,8 @@ "name": "vector_docs_search", "search_parameters": { "sort_by": "_text_match:desc,ranking:desc,level:desc", - "query_by": "pageTitle,title,hierarchy,content,tags", - "query_by_weights": "4,3,2,1,2", + "query_by": "componentWithType,component,pageTitle,title,hierarchy,content,tags", + "query_by_weights": "5,4,4,3,2,1,2", "text_match_type": "sum_score" } } From 625c1d3a57a59784da7b1e42b2318d4c372649d8 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Wed, 29 Apr 2026 16:06:31 -0400 Subject: [PATCH 180/326] chore(deps): bump the serde group across 1 directory with 2 updates (#25227) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(deps): bump the serde group across 1 directory with 2 updates Bumps the serde group with 2 updates in the / directory: [serde_with](https://github.com/jonasbb/serde_with) and [serde_json](https://github.com/serde-rs/json). Updates `serde_with` from 3.14.0 to 3.17.0 - [Release notes](https://github.com/jonasbb/serde_with/releases) - [Commits](https://github.com/jonasbb/serde_with/compare/v3.14.0...v3.17.0) Updates `serde_json` from 1.0.145 to 1.0.149 - [Release notes](https://github.com/serde-rs/json/releases) - [Commits](https://github.com/serde-rs/json/compare/v1.0.145...v1.0.149) --- updated-dependencies: - dependency-name: serde_with dependency-version: 3.17.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: serde - dependency-name: serde_json dependency-version: 1.0.149 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: serde ... Signed-off-by: dependabot[bot] * fix(core): update float size estimator to match serde_json's new formatter serde_json 1.0.147 switched from Ryū to Żmij for float-to-string formatting. The EstimatedJsonEncodedSizeOf impls for f32/f64 were using ryu directly to predict serde_json's output size, which silently drifts after the bump (for example ryu emits `1e16` while serde_json now emits `1e+16`). Swap to zmij so the estimate tracks what serde_json actually produces. * chore(codecs): update native_json fixtures for new serde_json float formatter Round-tripped the affected fixtures through serde_json 1.0.149 so positive exponents render as `e+N` (e.g. `1.797...e+308`) to match the new Żmij formatter introduced in 1.0.147. Event content is unchanged. Co-Authored-By: Claude Opus 4.7 (1M context) * chore(deps): regenerate 3rd-party license file for zmij Co-Authored-By: Claude Opus 4.7 (1M context) * chore(ci): allow `mij` in spell check The changelog mentions the Żmij formatter; check-spelling tokenizes around `Ż` and flags the residual `mij`. Co-Authored-By: Claude Opus 4.7 (1M context) * chore(deps): centralize serde_with as a workspace dependency Move the serde_with version + features into [workspace.dependencies], mirroring how serde and serde_json are already declared. The workspace entry carries the union of features previously enabled across the three dependents (std, macros, chrono_0_4); since Cargo unions features across the workspace anyway, the resulting workspace build is identical. Co-Authored-By: Claude Opus 4.7 (1M context) * chore(deps): use workspace serde_json in file-source-common Addresses review feedback from @thomasqueirozb on #25227. Cargo unions features across the workspace, so the union for any binary that pulls file-source-common alongside other serde_json dependents is unchanged. Co-Authored-By: Claude Opus 4.7 (1M context) * Update Cargo.lock --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Claude Opus 4.7 (1M context) Co-authored-by: Thomas --- .github/actions/spelling/expect.txt | 1 + Cargo.lock | 61 +++++++++++++++---- Cargo.toml | 3 +- LICENSE-3rdparty.csv | 1 + changelog.d/serde_group_bump.enhancement.md | 3 + lib/codecs/Cargo.toml | 2 +- .../tests/data/native_encoding/json/0605.json | 2 +- .../tests/data/native_encoding/json/0638.json | 2 +- .../tests/data/native_encoding/json/0639.json | 2 +- .../tests/data/native_encoding/json/0647.json | 2 +- .../tests/data/native_encoding/json/0700.json | 2 +- .../tests/data/native_encoding/json/0788.json | 2 +- lib/file-source-common/Cargo.toml | 2 +- lib/vector-config/Cargo.toml | 4 +- lib/vector-core/Cargo.toml | 4 +- .../event/estimated_json_encoded_size_of.rs | 4 +- 16 files changed, 71 insertions(+), 26 deletions(-) create mode 100644 changelog.d/serde_group_bump.enhancement.md diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt index 3656732c51bba..353de8bdf6f91 100644 --- a/.github/actions/spelling/expect.txt +++ b/.github/actions/spelling/expect.txt @@ -350,6 +350,7 @@ mezmo mfoo mgmt MIDP +mij mingrammer mkto mlua diff --git a/Cargo.lock b/Cargo.lock index 52ed5cd06c593..9bab182e05373 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3376,6 +3376,16 @@ dependencies = [ "darling_macro 0.21.3", ] +[[package]] +name = "darling" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core 0.23.0", + "darling_macro 0.23.0", +] + [[package]] name = "darling_core" version = "0.20.11" @@ -3404,6 +3414,19 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "darling_core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +dependencies = [ + "ident_case", + "proc-macro2 1.0.106", + "quote 1.0.45", + "strsim", + "syn 2.0.117", +] + [[package]] name = "darling_macro" version = "0.20.11" @@ -3426,6 +3449,17 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core 0.23.0", + "quote 1.0.45", + "syn 2.0.117", +] + [[package]] name = "dary_heap" version = "0.3.8" @@ -10289,16 +10323,16 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.145" +version = "1.0.149" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" dependencies = [ "indexmap 2.12.0", "itoa", "memchr", - "ryu", "serde", "serde_core", + "zmij", ] [[package]] @@ -10363,9 +10397,9 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.14.0" +version = "3.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2c45cd61fefa9db6f254525d46e392b852e0e61d9a1fd36e5bd183450a556d5" +checksum = "dd5414fad8e6907dbdd5bc441a50ae8d6e26151a03b1de04d89a5576de61d01f" dependencies = [ "base64 0.22.1", "chrono", @@ -10374,8 +10408,7 @@ dependencies = [ "indexmap 2.12.0", "schemars 0.9.0", "schemars 1.0.3", - "serde", - "serde_derive", + "serde_core", "serde_json", "serde_with_macros", "time", @@ -10383,11 +10416,11 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.14.0" +version = "3.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de90945e6565ce0d9a25098082ed4ee4002e047cb59892c318d66821e14bb30f" +checksum = "d3db8978e608f1fe7357e211969fd9abdcae80bac1ba7a3369bb7eb6b404eb65" dependencies = [ - "darling 0.20.11", + "darling 0.23.0", "proc-macro2 1.0.106", "quote 1.0.45", "syn 2.0.117", @@ -13148,7 +13181,6 @@ dependencies = [ "rand 0.9.4", "rand_distr", "regex", - "ryu", "schannel", "security-framework 3.6.0", "serde", @@ -13177,6 +13209,7 @@ dependencies = [ "vector-config-common", "vector-lookup", "vrl", + "zmij", ] [[package]] @@ -14656,6 +14689,12 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7948af682ccbc3342b6e9420e8c51c1fe5d7bf7756002b4a3c6cabfe96a7e3c" +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + [[package]] name = "zstd" version = "0.13.2" diff --git a/Cargo.toml b/Cargo.toml index db582e2930491..904f540cd26a5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -189,7 +189,8 @@ reqwest = { version = "0.11", features = ["json"] } rust_decimal = { version = "1.40.0", default-features = false, features = ["std"] } semver = { version = "1.0.28", default-features = false, features = ["serde", "std"] } serde = { version = "1.0.219", default-features = false, features = ["alloc", "derive", "rc"] } -serde_json = { version = "1.0.143", default-features = false, features = ["preserve_order", "raw_value", "std"] } +serde_json = { version = "1.0.149", default-features = false, features = ["preserve_order", "raw_value", "std"] } +serde_with = { version = "3.18.0", default-features = false, features = ["std", "macros", "chrono_0_4"] } serde_yaml = { version = "0.9.34", default-features = false } snafu = { version = "0.9.0", default-features = false, features = ["futures", "std"] } socket2 = { version = "0.5.10", default-features = false } diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 31c7d69405a20..311b35452684c 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -937,6 +937,7 @@ zeroize,https://github.com/RustCrypto/utils/tree/master/zeroize,Apache-2.0 OR MI zerovec,https://github.com/unicode-org/icu4x,Unicode-3.0,The ICU4X Project Developers zerovec-derive,https://github.com/unicode-org/icu4x,Unicode-3.0,Manish Goregaokar zlib-rs,https://github.com/trifectatechfoundation/zlib-rs,Zlib,The zlib-rs Authors +zmij,https://github.com/dtolnay/zmij,MIT,David Tolnay zstd,https://github.com/gyscos/zstd-rs,MIT,Alexandre Bury zstd-safe,https://github.com/gyscos/zstd-rs,MIT OR Apache-2.0,Alexandre Bury zstd-sys,https://github.com/gyscos/zstd-rs,MIT OR Apache-2.0,Alexandre Bury diff --git a/changelog.d/serde_group_bump.enhancement.md b/changelog.d/serde_group_bump.enhancement.md new file mode 100644 index 0000000000000..3c9ea910c6601 --- /dev/null +++ b/changelog.d/serde_group_bump.enhancement.md @@ -0,0 +1,3 @@ +Bumped `serde_json` to `1.0.149` and `serde_with` to `3.18.0`. `serde_json` switched its float-to-string formatter from Ryū to Żmij in `1.0.147`, so floats serialized via the `native_json` codec may render with slightly different textual form (for example `1e+16` instead of `1e16`). The change is purely cosmetic: parsed `f32`/`f64` values round-trip identically, and Vector-to-Vector communication between old and new versions is unaffected. + +authors: pront diff --git a/lib/codecs/Cargo.toml b/lib/codecs/Cargo.toml index b9043b7223d8a..6a5f8424a61fd 100644 --- a/lib/codecs/Cargo.toml +++ b/lib/codecs/Cargo.toml @@ -45,7 +45,7 @@ prost-reflect.workspace = true rand.workspace = true regex.workspace = true serde.workspace = true -serde_with = { version = "3.14.0", default-features = false, features = ["std", "macros", "chrono_0_4"] } +serde_with.workspace = true serde_json.workspace = true serde-aux = { version = "4.5", optional = true } smallvec = { version = "1", default-features = false, features = ["union"] } diff --git a/lib/codecs/tests/data/native_encoding/json/0605.json b/lib/codecs/tests/data/native_encoding/json/0605.json index bcd39f82ae418..1c46a13cf2fb7 100644 --- a/lib/codecs/tests/data/native_encoding/json/0605.json +++ b/lib/codecs/tests/data/native_encoding/json/0605.json @@ -1 +1 @@ -{"metric":{"name":"l","tags":{"a":"l","e":"l","q":"n"},"timestamp":"1970-01-01T07:31:02.000011250Z","kind":"absolute","sketch":{"sketch":{"AgentDDSketch":{"bins":{"k":[],"n":[]},"count":0,"min":1.7976931348623157e308,"max":-1.7976931348623157e308,"sum":-209216.0,"avg":-113920.0}}}}} \ No newline at end of file +{"metric":{"name":"l","tags":{"a":"l","e":"l","q":"n"},"timestamp":"1970-01-01T07:31:02.000011250Z","kind":"absolute","sketch":{"sketch":{"AgentDDSketch":{"bins":{"k":[],"n":[]},"count":0,"min":1.7976931348623157e+308,"max":-1.7976931348623157e+308,"sum":-209216.0,"avg":-113920.0}}}}} \ No newline at end of file diff --git a/lib/codecs/tests/data/native_encoding/json/0638.json b/lib/codecs/tests/data/native_encoding/json/0638.json index 61f4ec83d0fb4..85fb25087875f 100644 --- a/lib/codecs/tests/data/native_encoding/json/0638.json +++ b/lib/codecs/tests/data/native_encoding/json/0638.json @@ -1 +1 @@ -{"metric":{"name":"t","namespace":"k","tags":{"s":"b","x":"y"},"timestamp":"1969-12-31T19:41:49.000002928Z","interval_ms":2984613283,"kind":"absolute","sketch":{"sketch":{"AgentDDSketch":{"bins":{"k":[],"n":[]},"count":0,"min":1.7976931348623157e308,"max":-1.7976931348623157e308,"sum":-667968.0,"avg":-404160.0}}}}} \ No newline at end of file +{"metric":{"name":"t","namespace":"k","tags":{"s":"b","x":"y"},"timestamp":"1969-12-31T19:41:49.000002928Z","interval_ms":2984613283,"kind":"absolute","sketch":{"sketch":{"AgentDDSketch":{"bins":{"k":[],"n":[]},"count":0,"min":1.7976931348623157e+308,"max":-1.7976931348623157e+308,"sum":-667968.0,"avg":-404160.0}}}}} \ No newline at end of file diff --git a/lib/codecs/tests/data/native_encoding/json/0639.json b/lib/codecs/tests/data/native_encoding/json/0639.json index 3f8fd591f1128..9bbe86b1aa695 100644 --- a/lib/codecs/tests/data/native_encoding/json/0639.json +++ b/lib/codecs/tests/data/native_encoding/json/0639.json @@ -1 +1 @@ -{"metric":{"name":"y","namespace":"d","kind":"incremental","sketch":{"sketch":{"AgentDDSketch":{"bins":{"k":[],"n":[]},"count":0,"min":1.7976931348623157e308,"max":-1.7976931348623157e308,"sum":-422464.0,"avg":906752.0}}}}} \ No newline at end of file +{"metric":{"name":"y","namespace":"d","kind":"incremental","sketch":{"sketch":{"AgentDDSketch":{"bins":{"k":[],"n":[]},"count":0,"min":1.7976931348623157e+308,"max":-1.7976931348623157e+308,"sum":-422464.0,"avg":906752.0}}}}} \ No newline at end of file diff --git a/lib/codecs/tests/data/native_encoding/json/0647.json b/lib/codecs/tests/data/native_encoding/json/0647.json index 8dbe86fb16fdd..c6e6cdb1dfbab 100644 --- a/lib/codecs/tests/data/native_encoding/json/0647.json +++ b/lib/codecs/tests/data/native_encoding/json/0647.json @@ -1 +1 @@ -{"metric":{"name":"b","interval_ms":586459638,"kind":"incremental","sketch":{"sketch":{"AgentDDSketch":{"bins":{"k":[],"n":[]},"count":0,"min":1.7976931348623157e308,"max":-1.7976931348623157e308,"sum":999808.0,"avg":-148288.0}}}}} \ No newline at end of file +{"metric":{"name":"b","interval_ms":586459638,"kind":"incremental","sketch":{"sketch":{"AgentDDSketch":{"bins":{"k":[],"n":[]},"count":0,"min":1.7976931348623157e+308,"max":-1.7976931348623157e+308,"sum":999808.0,"avg":-148288.0}}}}} \ No newline at end of file diff --git a/lib/codecs/tests/data/native_encoding/json/0700.json b/lib/codecs/tests/data/native_encoding/json/0700.json index 42fdfec5980f1..da5055a337409 100644 --- a/lib/codecs/tests/data/native_encoding/json/0700.json +++ b/lib/codecs/tests/data/native_encoding/json/0700.json @@ -1 +1 @@ -{"metric":{"name":"j","tags":{"f":"k"},"timestamp":"1969-12-31T18:05:17.000026769Z","kind":"absolute","sketch":{"sketch":{"AgentDDSketch":{"bins":{"k":[],"n":[]},"count":0,"min":1.7976931348623157e308,"max":-1.7976931348623157e308,"sum":-481728.0,"avg":590528.0}}}}} \ No newline at end of file +{"metric":{"name":"j","tags":{"f":"k"},"timestamp":"1969-12-31T18:05:17.000026769Z","kind":"absolute","sketch":{"sketch":{"AgentDDSketch":{"bins":{"k":[],"n":[]},"count":0,"min":1.7976931348623157e+308,"max":-1.7976931348623157e+308,"sum":-481728.0,"avg":590528.0}}}}} \ No newline at end of file diff --git a/lib/codecs/tests/data/native_encoding/json/0788.json b/lib/codecs/tests/data/native_encoding/json/0788.json index f458d97be6718..13f2d0913ec85 100644 --- a/lib/codecs/tests/data/native_encoding/json/0788.json +++ b/lib/codecs/tests/data/native_encoding/json/0788.json @@ -1 +1 @@ -{"metric":{"name":"w","interval_ms":870813348,"kind":"incremental","sketch":{"sketch":{"AgentDDSketch":{"bins":{"k":[],"n":[]},"count":0,"min":1.7976931348623157e308,"max":-1.7976931348623157e308,"sum":-356992.0,"avg":-572800.0}}}}} \ No newline at end of file +{"metric":{"name":"w","interval_ms":870813348,"kind":"incremental","sketch":{"sketch":{"AgentDDSketch":{"bins":{"k":[],"n":[]},"count":0,"min":1.7976931348623157e+308,"max":-1.7976931348623157e+308,"sum":-356992.0,"avg":-572800.0}}}}} \ No newline at end of file diff --git a/lib/file-source-common/Cargo.toml b/lib/file-source-common/Cargo.toml index 6d3a885bc9a7a..f7a191d9cae7d 100644 --- a/lib/file-source-common/Cargo.toml +++ b/lib/file-source-common/Cargo.toml @@ -16,7 +16,7 @@ chrono.workspace = true tracing.workspace = true crc = "3.3.0" serde = { version = "1.0", default-features = false, features = ["derive"] } -serde_json = { version = "1.0.143", default-features = false } +serde_json.workspace = true bstr = { version = "1.12", default-features = false } bytes = { version = "1.11.1", default-features = false, features = ["serde"] } dashmap = { version = "6.1", default-features = false } diff --git a/lib/vector-config/Cargo.toml b/lib/vector-config/Cargo.toml index 2283df3d133e1..5f4cfdbdcf1b2 100644 --- a/lib/vector-config/Cargo.toml +++ b/lib/vector-config/Cargo.toml @@ -20,7 +20,7 @@ no-proxy = { version = "0.3.6", default-features = false, features = ["serialize num-traits = { version = "0.2.19", default-features = false } serde.workspace = true serde_json.workspace = true -serde_with = { version = "3.14.0", default-features = false, features = ["std"] } +serde_with.workspace = true snafu.workspace = true toml.workspace = true tracing.workspace = true @@ -32,4 +32,4 @@ vector-config-macros = { path = "../vector-config-macros" } [dev-dependencies] assert-json-diff = { version = "2", default-features = false } -serde_with = { version = "3.14.0", default-features = false, features = ["std", "macros"] } +serde_with.workspace = true diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml index 8836c2f70beaf..5c3370ac1a526 100644 --- a/lib/vector-core/Cargo.toml +++ b/lib/vector-core/Cargo.toml @@ -42,10 +42,10 @@ prost-types.workspace = true prost.workspace = true quanta = { version = "0.12.6", default-features = false } regex = { version = "1.12.3", default-features = false, features = ["std", "perf"] } -ryu = { version = "1", default-features = false } +zmij = { version = "1", default-features = false } serde.workspace = true serde_json.workspace = true -serde_with = { version = "3.14.0", default-features = false, features = ["std", "macros"] } +serde_with.workspace = true smallvec = { version = "1", default-features = false, features = ["serde", "const_generics"] } snafu.workspace = true socket2.workspace = true diff --git a/lib/vector-core/src/event/estimated_json_encoded_size_of.rs b/lib/vector-core/src/event/estimated_json_encoded_size_of.rs index 832dc70cfd343..c6e8c17bc95b2 100644 --- a/lib/vector-core/src/event/estimated_json_encoded_size_of.rs +++ b/lib/vector-core/src/event/estimated_json_encoded_size_of.rs @@ -120,13 +120,13 @@ impl EstimatedJsonEncodedSizeOf for bool { impl EstimatedJsonEncodedSizeOf for f64 { fn estimated_json_encoded_size_of(&self) -> JsonSize { - ryu::Buffer::new().format_finite(*self).len().into() + zmij::Buffer::new().format_finite(*self).len().into() } } impl EstimatedJsonEncodedSizeOf for f32 { fn estimated_json_encoded_size_of(&self) -> JsonSize { - ryu::Buffer::new().format_finite(*self).len().into() + zmij::Buffer::new().format_finite(*self).len().into() } } From b5fb618eb24bade72d5277be6d23836810d24fa8 Mon Sep 17 00:00:00 2001 From: Thomas Date: Thu, 30 Apr 2026 17:53:00 -0400 Subject: [PATCH 181/326] chore(dev): remove unused update_counter macro (#25333) --- lib/vector-core/src/metrics/mod.rs | 44 ------------------------------ 1 file changed, 44 deletions(-) diff --git a/lib/vector-core/src/metrics/mod.rs b/lib/vector-core/src/metrics/mod.rs index 660f29c000db3..cd900d392f004 100644 --- a/lib/vector-core/src/metrics/mod.rs +++ b/lib/vector-core/src/metrics/mod.rs @@ -200,50 +200,6 @@ impl Controller { } } -#[macro_export] -/// This macro is used to emit metrics as a `counter` while simultaneously -/// converting from absolute values to incremental values. -/// -/// Values that do not arrive in strictly monotonically increasing order are -/// ignored and will not be emitted. -macro_rules! update_counter { - ($label:literal, $value:expr) => {{ - use ::std::sync::atomic::{AtomicU64, Ordering}; - - static PREVIOUS_VALUE: AtomicU64 = AtomicU64::new(0); - - let new_value = $value; - let mut previous_value = PREVIOUS_VALUE.load(Ordering::Relaxed); - - loop { - // Either a new greater value has been emitted before this thread updated the counter - // or values were provided that are not in strictly monotonically increasing order. - // Ignore. - if new_value <= previous_value { - break; - } - - match PREVIOUS_VALUE.compare_exchange_weak( - previous_value, - new_value, - Ordering::SeqCst, - Ordering::Relaxed, - ) { - // Another thread has written a new value before us. Re-enter loop. - Err(value) => previous_value = value, - // Calculate delta to last emitted value and emit it. - Ok(_) => { - let delta = new_value - previous_value; - // Albeit very unlikely, note that this sequence of deltas might be emitted in - // a different order than they were calculated. - counter!($label).increment(delta); - break; - } - } - } - }}; -} - #[cfg(test)] mod tests { use super::*; From bd8ab1a245a439134b2bd9f2cb540ca550db860c Mon Sep 17 00:00:00 2001 From: Andy <26137827+ndrsg@users.noreply.github.com> Date: Fri, 1 May 2026 15:30:24 +0200 Subject: [PATCH 182/326] feat(sinks): introduce RetryStrategy / Config for http based sinks (#25057) * feat(sinks): introduce a configurable retry strategy for http * feat(sinks): wire the RetryStrategy with the HttpRetryLogic * feat(sinks): wire RetryStrategy with all HttpStatusRetryLogic and all affected sinks * docs(sinks): http example config with custom retry strategy * feat(http sink): add integration test for custom retry strategy * docs(sinks): add changelog fragment * fix(http sink): update is_retriable_error logic to respect retry strategy * fix(http sink): add handling for NOT_IMPLEMENTED status in retry strategy * chore(sinks): format code * feat(sinks): implement timeout retry logic and add tests for non-retriable timeouts * docs(sinks): clarify that success status codes are not retried on RetryAll strategy * feat(sinks): improve typesafety of RetryStrategy StatusCode * docs(sinks http): enhance transport error classification for HTTP sinks with shared retry logic * feat(sinks http): add debug logging for unsuccessful HTTP responses in retry logic * refactor(http sink): avoid heap allocation on reasoning happy path * fix(http sink): format example config * chore: add custom attributes for numeric type in StatusCode metadata * docs(sinks http): update docs for http sinks supporting retry_strategy --------- Co-authored-by: Pavlos Rontidis --- benches/http.rs | 1 + .../10870_http_retry_strategy.feature.md | 7 + ...http_retry_transport_errors.enhancement.md | 9 + config/examples/http_sink_custom_retry.yaml | 42 +++ lib/vector-config/src/http.rs | 6 + src/sinks/appsignal/config.rs | 11 +- src/sinks/axiom/config.rs | 8 +- src/sinks/azure_logs_ingestion/config.rs | 16 +- src/sinks/azure_monitor_logs/config.rs | 16 +- src/sinks/datadog/events/config.rs | 14 +- src/sinks/gcp/stackdriver/logs/config.rs | 11 +- src/sinks/gcp/stackdriver/metrics/config.rs | 12 +- src/sinks/honeycomb/config.rs | 11 +- src/sinks/http/config.rs | 15 +- src/sinks/http/tests.rs | 100 +++++++ src/sinks/keep/config.rs | 11 +- src/sinks/opentelemetry/mod.rs | 1 + src/sinks/prometheus/remote_write/config.rs | 11 +- src/sinks/util/http.rs | 282 ++++++++++++++++-- src/sinks/util/retries.rs | 60 +++- .../components/sinks/generated/appsignal.cue | 31 ++ .../components/sinks/generated/axiom.cue | 31 ++ .../sinks/generated/azure_logs_ingestion.cue | 31 ++ .../sinks/generated/azure_monitor_logs.cue | 31 ++ .../sinks/generated/datadog_events.cue | 31 ++ .../sinks/generated/gcp_stackdriver_logs.cue | 31 ++ .../generated/gcp_stackdriver_metrics.cue | 31 ++ .../components/sinks/generated/honeycomb.cue | 31 ++ .../components/sinks/generated/http.cue | 31 ++ .../components/sinks/generated/keep.cue | 31 ++ .../sinks/generated/opentelemetry.cue | 31 ++ .../generated/prometheus_remote_write.cue | 31 ++ 32 files changed, 957 insertions(+), 59 deletions(-) create mode 100644 changelog.d/10870_http_retry_strategy.feature.md create mode 100644 changelog.d/10870_http_retry_transport_errors.enhancement.md create mode 100644 config/examples/http_sink_custom_retry.yaml diff --git a/benches/http.rs b/benches/http.rs index 30b35b8b257f7..5c8b5eaac7efe 100644 --- a/benches/http.rs +++ b/benches/http.rs @@ -63,6 +63,7 @@ fn benchmark_http(c: &mut Criterion) { request: Default::default(), tls: Default::default(), acknowledgements: Default::default(), + retry_strategy: Default::default(), }, ); diff --git a/changelog.d/10870_http_retry_strategy.feature.md b/changelog.d/10870_http_retry_strategy.feature.md new file mode 100644 index 0000000000000..09afba2eddd40 --- /dev/null +++ b/changelog.d/10870_http_retry_strategy.feature.md @@ -0,0 +1,7 @@ +HTTP-based sinks that use the shared retry helpers now support a `retry_strategy` configuration +option to control which HTTP response codes are retried. The `http` sink also includes a new +example showing how to retry only specific transient status codes. + +Issue: https://github.com/vectordotdev/vector/issues/10870 + +authors: ndrsg diff --git a/changelog.d/10870_http_retry_transport_errors.enhancement.md b/changelog.d/10870_http_retry_transport_errors.enhancement.md new file mode 100644 index 0000000000000..f06f63b2f0e07 --- /dev/null +++ b/changelog.d/10870_http_retry_transport_errors.enhancement.md @@ -0,0 +1,9 @@ +HTTP-based sinks using the shared retry logic now classify transport-layer failures with +`HttpError::is_retriable`: connection and TLS connector issues may be retried, while failures +such as invalid HTTP request construction or an invalid proxy URI are not. Setting +`retry_strategy` to `none` disables retries for these transport errors and for request +timeouts, in addition to status-code-based retries. + +Issue: https://github.com/vectordotdev/vector/issues/10870 + +authors: ndrsg diff --git a/config/examples/http_sink_custom_retry.yaml b/config/examples/http_sink_custom_retry.yaml new file mode 100644 index 0000000000000..fd973c890e991 --- /dev/null +++ b/config/examples/http_sink_custom_retry.yaml @@ -0,0 +1,42 @@ +# HTTP sink example with a custom retry strategy +# ---------------------------------------------------- +# Sends demo logs to an HTTP endpoint and only retries the response codes +# that the upstream API documents as transient. + +data_dir: "/var/lib/vector" + +sources: + demo_logs: + type: "demo_logs" + format: "json" + interval: 1 + +sinks: + http_out: + type: "http" + inputs: ["demo_logs"] + uri: "https://example.com/ingest" + method: "post" + + # Skip the startup probe so the example can be adapted locally. + healthcheck: + enabled: false + + # Send newline-delimited JSON in the request body. + framing: + method: "newline_delimited" + encoding: + codec: "json" + + # Control how many retries are made and how quickly backoff grows. + request: + timeout_secs: 60 + retry_attempts: 8 + retry_initial_backoff_secs: 2 + retry_max_duration_secs: 30 + + # Retry only on the exact HTTP status codes that this destination + # treats as temporary failures. + retry_strategy: + type: "custom" + status_codes: [408, 425, 429, 503] diff --git a/lib/vector-config/src/http.rs b/lib/vector-config/src/http.rs index 94029610f0441..da51ea1cc156e 100644 --- a/lib/vector-config/src/http.rs +++ b/lib/vector-config/src/http.rs @@ -2,9 +2,11 @@ use std::cell::RefCell; use http::StatusCode; use serde_json::Value; +use vector_config_common::{attributes::CustomAttribute, constants}; use crate::{ Configurable, GenerateError, Metadata, ToValue, + num::NumberClass, schema::{SchemaGenerator, SchemaObject, generate_number_schema}, }; @@ -27,6 +29,10 @@ impl Configurable for StatusCode { let mut metadata = Metadata::default(); metadata.set_description("HTTP response status code"); metadata.set_default_value(StatusCode::OK); + metadata.add_custom_attribute(CustomAttribute::kv( + constants::DOCS_META_NUMERIC_TYPE, + NumberClass::Unsigned, + )); metadata } diff --git a/src/sinks/appsignal/config.rs b/src/sinks/appsignal/config.rs index 9e8c000f995a3..fea94af05b132 100644 --- a/src/sinks/appsignal/config.rs +++ b/src/sinks/appsignal/config.rs @@ -21,7 +21,7 @@ use crate::{ prelude::{SinkConfig, SinkContext}, util::{ BatchConfig, Compression, ServiceBuilderExt, SinkBatchSettings, TowerRequestConfig, - http::HttpStatusRetryLogic, + http::{HttpStatusRetryLogic, RetryStrategy}, }, }, }; @@ -67,6 +67,10 @@ pub(super) struct AppsignalConfig { skip_serializing_if = "crate::serde::is_default" )] acknowledgements: AcknowledgementsConfig, + + #[configurable(derived)] + #[serde(default)] + retry_strategy: RetryStrategy, } pub(super) fn default_endpoint() -> String { @@ -99,7 +103,10 @@ impl AppsignalConfig { let request_opts = self.request; let request_settings = request_opts.into_settings(); - let retry_logic = HttpStatusRetryLogic::new(|req: &AppsignalResponse| req.http_status); + let retry_logic = HttpStatusRetryLogic::new( + |req: &AppsignalResponse| req.http_status, + self.retry_strategy.clone(), + ); let service = ServiceBuilder::new() .settings(request_settings, retry_logic) diff --git a/src/sinks/axiom/config.rs b/src/sinks/axiom/config.rs index 61421973d5475..0930a997abc12 100644 --- a/src/sinks/axiom/config.rs +++ b/src/sinks/axiom/config.rs @@ -15,7 +15,8 @@ use crate::{ Healthcheck, VectorSink, http::config::{HttpMethod, HttpSinkConfig}, util::{ - BatchConfig, Compression, RealtimeSizeBasedDefaultBatchSettings, http::RequestConfig, + BatchConfig, Compression, RealtimeSizeBasedDefaultBatchSettings, + http::{RequestConfig, RetryStrategy}, }, }, tls::TlsConfig, @@ -124,6 +125,10 @@ pub struct AxiomConfig { skip_serializing_if = "crate::serde::is_default" )] pub acknowledgements: AcknowledgementsConfig, + + #[configurable(derived)] + #[serde(default)] + pub retry_strategy: RetryStrategy, } impl GenerateConfig for AxiomConfig { @@ -180,6 +185,7 @@ impl SinkConfig for AxiomConfig { ), payload_prefix: "".into(), // Always newline delimited JSON payload_suffix: "".into(), // Always newline delimited JSON + retry_strategy: self.retry_strategy.clone(), }; http_sink_config.build(cx).await diff --git a/src/sinks/azure_logs_ingestion/config.rs b/src/sinks/azure_logs_ingestion/config.rs index 3b105d549d0c1..89da496045d9a 100644 --- a/src/sinks/azure_logs_ingestion/config.rs +++ b/src/sinks/azure_logs_ingestion/config.rs @@ -10,7 +10,10 @@ use crate::{ sinks::{ azure_common::config::AzureAuthentication, prelude::*, - util::{RealtimeSizeBasedDefaultBatchSettings, UriSerde, http::HttpStatusRetryLogic}, + util::{ + RealtimeSizeBasedDefaultBatchSettings, UriSerde, + http::{HttpStatusRetryLogic, RetryStrategy}, + }, }, }; @@ -102,6 +105,10 @@ pub struct AzureLogsIngestionConfig { skip_serializing_if = "crate::serde::is_default" )] pub acknowledgements: AcknowledgementsConfig, + + #[configurable(derived)] + #[serde(default)] + pub retry_strategy: RetryStrategy, } impl Default for AzureLogsIngestionConfig { @@ -118,6 +125,7 @@ impl Default for AzureLogsIngestionConfig { request: Default::default(), tls: None, acknowledgements: Default::default(), + retry_strategy: Default::default(), } } } @@ -156,8 +164,10 @@ impl AzureLogsIngestionConfig { )?; let healthcheck = service.healthcheck(); - let retry_logic = - HttpStatusRetryLogic::new(|res: &AzureLogsIngestionResponse| res.http_status); + let retry_logic = HttpStatusRetryLogic::new( + |res: &AzureLogsIngestionResponse| res.http_status, + self.retry_strategy.clone(), + ); let request_settings = self.request.into_settings(); let service = ServiceBuilder::new() .settings(request_settings, retry_logic) diff --git a/src/sinks/azure_monitor_logs/config.rs b/src/sinks/azure_monitor_logs/config.rs index dc60d179a2ff5..6d68aed7efd75 100644 --- a/src/sinks/azure_monitor_logs/config.rs +++ b/src/sinks/azure_monitor_logs/config.rs @@ -16,7 +16,10 @@ use crate::{ http::{HttpClient, get_http_scheme_from_uri}, sinks::{ prelude::*, - util::{RealtimeSizeBasedDefaultBatchSettings, UriSerde, http::HttpStatusRetryLogic}, + util::{ + RealtimeSizeBasedDefaultBatchSettings, UriSerde, + http::{HttpStatusRetryLogic, RetryStrategy}, + }, }, }; @@ -113,6 +116,10 @@ pub struct AzureMonitorLogsConfig { skip_serializing_if = "crate::serde::is_default" )] pub acknowledgements: AcknowledgementsConfig, + + #[configurable(derived)] + #[serde(default)] + pub retry_strategy: RetryStrategy, } impl Default for AzureMonitorLogsConfig { @@ -129,6 +136,7 @@ impl Default for AzureMonitorLogsConfig { time_generated_key: None, tls: None, acknowledgements: Default::default(), + retry_strategy: Default::default(), } } } @@ -181,8 +189,10 @@ impl AzureMonitorLogsConfig { )?; let healthcheck = service.healthcheck(); - let retry_logic = - HttpStatusRetryLogic::new(|res: &AzureMonitorLogsResponse| res.http_status); + let retry_logic = HttpStatusRetryLogic::new( + |res: &AzureMonitorLogsResponse| res.http_status, + self.retry_strategy.clone(), + ); let request_settings = self.request.into_settings(); let service = ServiceBuilder::new() .settings(request_settings, retry_logic) diff --git a/src/sinks/datadog/events/config.rs b/src/sinks/datadog/events/config.rs index 2bfdc66315c3e..f9b0f9361bba8 100644 --- a/src/sinks/datadog/events/config.rs +++ b/src/sinks/datadog/events/config.rs @@ -14,7 +14,10 @@ use crate::{ sinks::{ Healthcheck, VectorSink, datadog::{DatadogCommonConfig, LocalDatadogCommonConfig}, - util::{ServiceBuilderExt, TowerRequestConfig, http::HttpStatusRetryLogic}, + util::{ + ServiceBuilderExt, TowerRequestConfig, + http::{HttpStatusRetryLogic, RetryStrategy}, + }, }, tls::MaybeTlsSettings, }; @@ -33,6 +36,10 @@ pub struct DatadogEventsConfig { #[configurable(derived)] #[serde(default)] pub request: TowerRequestConfig, + + #[configurable(derived)] + #[serde(default)] + pub retry_strategy: RetryStrategy, } impl GenerateConfig for DatadogEventsConfig { @@ -64,7 +71,10 @@ impl DatadogEventsConfig { let request_opts = self.request; let request_settings = request_opts.into_settings(); - let retry_logic = HttpStatusRetryLogic::new(|req: &DatadogEventsResponse| req.http_status); + let retry_logic = HttpStatusRetryLogic::new( + |req: &DatadogEventsResponse| req.http_status, + self.retry_strategy.clone(), + ); let service = ServiceBuilder::new() .settings(request_settings, retry_logic) diff --git a/src/sinks/gcp/stackdriver/logs/config.rs b/src/sinks/gcp/stackdriver/logs/config.rs index 9b66eea4ac214..addb45163df76 100644 --- a/src/sinks/gcp/stackdriver/logs/config.rs +++ b/src/sinks/gcp/stackdriver/logs/config.rs @@ -21,7 +21,7 @@ use crate::{ prelude::*, util::{ BoxedRawValue, RealtimeSizeBasedDefaultBatchSettings, - http::{HttpService, http_response_retry_logic}, + http::{HttpService, RetryStrategy, http_response_retry_logic}, service::TowerRequestConfigDefaults, }, }, @@ -106,6 +106,10 @@ pub(super) struct StackdriverConfig { skip_serializing_if = "crate::serde::is_default" )] acknowledgements: AcknowledgementsConfig, + + #[configurable(derived)] + #[serde(default)] + pub retry_strategy: RetryStrategy, } pub(super) fn default_endpoint() -> String { @@ -269,7 +273,10 @@ impl SinkConfig for StackdriverConfig { let service = HttpService::new(client.clone(), stackdriver_logs_service_request_builder); let service = ServiceBuilder::new() - .settings(request_limits, http_response_retry_logic()) + .settings( + request_limits, + http_response_retry_logic(self.retry_strategy.clone()), + ) .service(service); let sink = StackdriverLogsSink::new(service, batch_settings, request_builder); diff --git a/src/sinks/gcp/stackdriver/metrics/config.rs b/src/sinks/gcp/stackdriver/metrics/config.rs index 2398a18e9beb9..ea2cc9d6b5293 100644 --- a/src/sinks/gcp/stackdriver/metrics/config.rs +++ b/src/sinks/gcp/stackdriver/metrics/config.rs @@ -15,7 +15,8 @@ use crate::{ prelude::*, util::{ http::{ - HttpRequest, HttpService, HttpServiceRequestBuilder, http_response_retry_logic, + HttpRequest, HttpService, HttpServiceRequestBuilder, RetryStrategy, + http_response_retry_logic, }, service::TowerRequestConfigDefaults, }, @@ -77,6 +78,10 @@ pub struct StackdriverConfig { skip_serializing_if = "crate::serde::is_default" )] pub(super) acknowledgements: AcknowledgementsConfig, + + #[configurable(derived)] + #[serde(default)] + pub retry_strategy: RetryStrategy, } fn default_metric_namespace_value() -> String { @@ -126,7 +131,10 @@ impl SinkConfig for StackdriverConfig { let service = HttpService::new(client, stackdriver_metrics_service_request_builder); let service = ServiceBuilder::new() - .settings(request_limits, http_response_retry_logic()) + .settings( + request_limits, + http_response_retry_logic(self.retry_strategy.clone()), + ) .service(service); let sink = StackdriverMetricsSink::new(service, batch_settings, request_builder); diff --git a/src/sinks/honeycomb/config.rs b/src/sinks/honeycomb/config.rs index 05a56cd9df368..f179248e1697b 100644 --- a/src/sinks/honeycomb/config.rs +++ b/src/sinks/honeycomb/config.rs @@ -16,7 +16,7 @@ use crate::{ prelude::*, util::{ BatchConfig, BoxedRawValue, - http::{HttpService, http_response_retry_logic}, + http::{HttpService, RetryStrategy, http_response_retry_logic}, }, }, }; @@ -71,6 +71,10 @@ pub struct HoneycombConfig { skip_serializing_if = "crate::serde::is_default" )] acknowledgements: AcknowledgementsConfig, + + #[configurable(derived)] + #[serde(default)] + pub retry_strategy: RetryStrategy, } fn default_endpoint() -> String { @@ -124,7 +128,10 @@ impl SinkConfig for HoneycombConfig { let request_limits = self.request.into_settings(); let service = ServiceBuilder::new() - .settings(request_limits, http_response_retry_logic()) + .settings( + request_limits, + http_response_retry_logic(self.retry_strategy.clone()), + ) .service(service); let sink = HoneycombSink::new(service, batch_settings, request_builder); diff --git a/src/sinks/http/config.rs b/src/sinks/http/config.rs index a3a105fa5c608..2b6abebe0d402 100644 --- a/src/sinks/http/config.rs +++ b/src/sinks/http/config.rs @@ -30,7 +30,10 @@ use crate::{ prelude::*, util::{ RealtimeSizeBasedDefaultBatchSettings, UriSerde, - http::{HttpService, OrderedHeaderName, RequestConfig, http_response_retry_logic}, + http::{ + HttpService, OrderedHeaderName, RequestConfig, RetryStrategy, + http_response_retry_logic, + }, }, }, }; @@ -100,6 +103,10 @@ pub struct HttpSinkConfig { skip_serializing_if = "crate::serde::is_default" )] pub acknowledgements: AcknowledgementsConfig, + + #[configurable(derived)] + #[serde(default)] + pub retry_strategy: RetryStrategy, } /// HTTP method. @@ -330,7 +337,10 @@ impl SinkConfig for HttpSinkConfig { let request_limits = self.request.tower.into_settings(); let service = ServiceBuilder::new() - .settings(request_limits, http_response_retry_logic()) + .settings( + request_limits, + http_response_retry_logic(self.retry_strategy.clone()), + ) .service(service); let sink = HttpSink::new( @@ -405,6 +415,7 @@ mod tests { acknowledgements: AcknowledgementsConfig::default(), payload_prefix: String::new(), payload_suffix: String::new(), + retry_strategy: RetryStrategy::default(), }; let external_resource = ExternalResource::new( diff --git a/src/sinks/http/tests.rs b/src/sinks/http/tests.rs index 12ff994239f5d..2b4b6942c5c7b 100644 --- a/src/sinks/http/tests.rs +++ b/src/sinks/http/tests.rs @@ -67,6 +67,7 @@ fn default_cfg(encoding: EncodingConfigWithFraming) -> HttpSinkConfig { request: Default::default(), tls: Default::default(), acknowledgements: Default::default(), + retry_strategy: Default::default(), } } @@ -445,6 +446,105 @@ async fn retries_on_temporary_error() { .await; } +#[tokio::test] +async fn custom_retry_retries_only_configured_status_code() { + components::assert_sink_compliance(&HTTP_SINK_TAGS, async { + const NUM_LINES: usize = 1; + const NUM_FAILURES: usize = 2; + const CUSTOM_RETRY_CONFIG: &str = r#" + request.retry_attempts = 2 + request.retry_initial_backoff_secs = 1 + request.retry_max_duration_secs = 1 + retry_strategy.type = "custom" + retry_strategy.status_codes = [408, 425, 429, 503] + "#; + + let (in_addr, sink) = build_sink(CUSTOM_RETRY_CONFIG).await; + + let counter = Arc::new(atomic::AtomicUsize::new(0)); + let in_counter = Arc::clone(&counter); + let (rx, trigger, server) = build_test_server_generic(in_addr, move || { + let count = in_counter.fetch_add(1, atomic::Ordering::Relaxed); + if count < NUM_FAILURES { + Response::builder() + .status(StatusCode::SERVICE_UNAVAILABLE) + .body(Body::empty()) + .unwrap_or_else(|_| unreachable!()) + } else { + Response::new(Body::empty()) + } + }); + + let (batch, mut receiver) = BatchNotifier::new_with_receiver(); + let (input_lines, events) = random_lines_with_stream(100, NUM_LINES, Some(batch)); + let pump = sink.run(events); + + tokio::spawn(server); + + pump.await.unwrap(); + drop(trigger); + + assert_eq!(receiver.try_recv(), Ok(BatchStatus::Delivered)); + + let output_lines = get_received_gzip(rx, |parts| { + assert_eq!(Method::POST, parts.method); + assert_eq!("/frames", parts.uri.path()); + }) + .await; + + let tries = counter.load(atomic::Ordering::Relaxed); + assert_eq!(tries, NUM_FAILURES + 1); + assert_eq!(NUM_LINES, output_lines.len()); + assert_eq!(input_lines, output_lines); + }) + .await; +} + +#[tokio::test] +async fn custom_retry_does_not_retry_unconfigured_status_code() { + components::assert_sink_error(&COMPONENT_ERROR_TAGS, async { + const NUM_LINES: usize = 1; + const CUSTOM_RETRY_CONFIG: &str = r#" + request.retry_attempts = 2 + request.retry_initial_backoff_secs = 1 + request.retry_max_duration_secs = 1 + retry_strategy.type = "custom" + retry_strategy.status_codes = [408, 425, 429, 503] + "#; + + let (in_addr, sink) = build_sink(CUSTOM_RETRY_CONFIG).await; + + let counter = Arc::new(atomic::AtomicUsize::new(0)); + let in_counter = Arc::clone(&counter); + let (rx, trigger, server) = build_test_server_generic(in_addr, move || { + in_counter.fetch_add(1, atomic::Ordering::Relaxed); + Response::builder() + .status(StatusCode::BAD_REQUEST) + .body(Body::empty()) + .unwrap_or_else(|_| unreachable!()) + }); + + let (batch, mut receiver) = BatchNotifier::new_with_receiver(); + let (_input_lines, events) = random_lines_with_stream(100, NUM_LINES, Some(batch)); + let pump = sink.run(events); + + tokio::spawn(server); + + pump.await.unwrap(); + drop(trigger); + + assert_eq!(receiver.try_recv(), Ok(BatchStatus::Rejected)); + assert_eq!(counter.load(atomic::Ordering::Relaxed), 1); + + let output_lines = get_received_gzip(rx, |_| { + unreachable!("There should be no successful requests") + }) + .await; + assert!(output_lines.is_empty()); + }) + .await; +} + #[tokio::test] async fn fails_on_permanent_error() { components::assert_sink_error(&COMPONENT_ERROR_TAGS, async { diff --git a/src/sinks/keep/config.rs b/src/sinks/keep/config.rs index 7b81281253672..d3810976aa3c1 100644 --- a/src/sinks/keep/config.rs +++ b/src/sinks/keep/config.rs @@ -16,7 +16,7 @@ use crate::{ prelude::*, util::{ BatchConfig, BoxedRawValue, - http::{HttpService, http_response_retry_logic}, + http::{HttpService, RetryStrategy, http_response_retry_logic}, }, }, }; @@ -59,6 +59,10 @@ pub struct KeepConfig { skip_serializing_if = "crate::serde::is_default" )] acknowledgements: AcknowledgementsConfig, + + #[configurable(derived)] + #[serde(default)] + pub retry_strategy: RetryStrategy, } fn default_endpoint() -> String { @@ -111,7 +115,10 @@ impl SinkConfig for KeepConfig { let request_limits = self.request.into_settings(); let service = ServiceBuilder::new() - .settings(request_limits, http_response_retry_logic()) + .settings( + request_limits, + http_response_retry_logic(self.retry_strategy.clone()), + ) .service(service); let sink = KeepSink::new(service, batch_settings, request_builder); diff --git a/src/sinks/opentelemetry/mod.rs b/src/sinks/opentelemetry/mod.rs index 7d73e25c4030d..dcc13624372d5 100644 --- a/src/sinks/opentelemetry/mod.rs +++ b/src/sinks/opentelemetry/mod.rs @@ -56,6 +56,7 @@ impl Default for Protocol { request: Default::default(), tls: Default::default(), acknowledgements: Default::default(), + retry_strategy: Default::default(), }) } } diff --git a/src/sinks/prometheus/remote_write/config.rs b/src/sinks/prometheus/remote_write/config.rs index c1b420d6b216d..c9f214953003c 100644 --- a/src/sinks/prometheus/remote_write/config.rs +++ b/src/sinks/prometheus/remote_write/config.rs @@ -17,7 +17,7 @@ use crate::{ prometheus::PrometheusRemoteWriteAuth, util::{ auth::Auth, - http::{OrderedHeaderName, http_response_retry_logic}, + http::{OrderedHeaderName, RetryStrategy, http_response_retry_logic}, service::TowerRequestConfig, }, }, @@ -129,6 +129,10 @@ pub struct RemoteWriteConfig { #[serde(default = "default_compression")] #[derivative(Default(value = "default_compression()"))] pub compression: Compression, + + #[configurable(derived)] + #[serde(default)] + pub retry_strategy: RetryStrategy, } const fn default_compression() -> Compression { @@ -251,7 +255,10 @@ impl SinkConfig for RemoteWriteConfig { headers: validated_headers, }; let service = ServiceBuilder::new() - .settings(request_settings, http_response_retry_logic()) + .settings( + request_settings, + http_response_retry_logic(self.retry_strategy.clone()), + ) .service(service); let sink = RemoteWriteSink { diff --git a/src/sinks/util/http.rs b/src/sinks/util/http.rs index e19025e3c6ee3..89225f4077741 100644 --- a/src/sinks/util/http.rs +++ b/src/sinks/util/http.rs @@ -7,6 +7,7 @@ use futures::{Sink, future::BoxFuture}; use headers::HeaderName; use http::{HeaderValue, Request, Response, StatusCode, header}; use http_body::Body as _; +use tracing::debug; #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct OrderedHeaderName(HeaderName); @@ -550,14 +551,137 @@ impl sink::Response for http::Response { } } +/// Serializes and deserializes a [`Vec`] +mod status_code_vec { + use http::StatusCode; + use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error}; + + /// Deserializes a [`Vec`] + pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + Vec::::deserialize(deserializer)? + .into_iter() + .map(StatusCode::from_u16) + .collect::, _>>() + .map_err(Error::custom) + } + + /// Serializes a [`Vec`] + pub fn serialize(status_codes: &[StatusCode], serializer: S) -> Result + where + S: Serializer, + { + status_codes + .iter() + .map(StatusCode::as_u16) + .collect::>() + .serialize(serializer) + } +} + +/// Configurable retry strategy for `http` based sinks. +/// +/// For more information about error responses, see [Client Error Responses][error_responses]. +/// +/// [error_responses]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Status#client_error_responses +#[configurable_component] +#[derive(Debug, Clone, Default, PartialEq)] +#[serde(tag = "type", rename_all = "snake_case")] +#[configurable(metadata(docs::enum_tag_description = "The retry strategy enum."))] +pub enum RetryStrategy { + /// Don't retry any errors, including request timeouts. + None, + + /// Default strategy. See [`RetryStrategy::retry_action`] for more details. + #[default] + Default, + + /// Retry on *all* HTTP status codes except for success codes (2xx) + All, + + /// Custom retry strategy + Custom { + /// Retry on these specific HTTP status codes + #[serde(with = "status_code_vec")] + status_codes: Vec, + }, +} + +impl RetryStrategy { + /// Returns the name of the retry strategy. + #[must_use] + const fn name(&self) -> &str { + match self { + Self::None => "Never retry strategy", + Self::Default => "Default retry strategy", + Self::All => "Retry all strategy", + Self::Custom { .. } => "Custom retry strategy", + } + } + + /// Determines if the given status code should be retried. + /// + /// For the `Default` strategy, the following status codes will be retried: + /// - 429 (Too Many Requests) + /// - 408 (Request Timeout) + /// - 5xx (Server Error) + /// + /// For the `Custom` strategy, the status codes specified in the `status_codes` field will be retried. + /// + /// For the `All` strategy, all non-success status codes will be retried. + #[must_use] + pub fn retry_action(&self, status: http::StatusCode) -> RetryAction { + if status.is_success() { + return RetryAction::Successful; + } + + let reason = format!( + "{}: {}", + self.name(), + status.canonical_reason().unwrap_or_else(|| status.as_str()) + ) + .into(); + + match self { + Self::None => RetryAction::DontRetry(reason), + Self::Default => match status { + StatusCode::TOO_MANY_REQUESTS | StatusCode::REQUEST_TIMEOUT => { + RetryAction::Retry(reason) + } + StatusCode::NOT_IMPLEMENTED => RetryAction::DontRetry(reason), + _ => { + if status.is_server_error() { + RetryAction::Retry(reason) + } else { + RetryAction::DontRetry(reason) + } + } + }, + Self::All => RetryAction::Retry(reason), + Self::Custom { status_codes } => { + if status_codes.contains(&status) { + RetryAction::Retry(reason) + } else { + RetryAction::DontRetry(reason) + } + } + } + } +} + #[derive(Debug, Clone)] pub struct HttpRetryLogic { request: PhantomData, + retry_strategy: RetryStrategy, } + impl Default for HttpRetryLogic { fn default() -> Self { Self { request: PhantomData, + retry_strategy: RetryStrategy::Default, } } } @@ -567,25 +691,28 @@ impl RetryLogic for HttpRetryLogic { type Request = Req; type Response = hyper::Response; - fn is_retriable_error(&self, _error: &Self::Error) -> bool { - true + fn is_retriable_error(&self, error: &Self::Error) -> bool { + if self.retry_strategy == RetryStrategy::None { + false + } else { + error.is_retriable() + } + } + + fn is_retriable_timeout(&self) -> bool { + self.retry_strategy != RetryStrategy::None } fn should_retry_response(&self, response: &Self::Response) -> RetryAction { let status = response.status(); - - match status { - StatusCode::TOO_MANY_REQUESTS => RetryAction::Retry("too many requests".into()), - StatusCode::REQUEST_TIMEOUT => RetryAction::Retry("request timeout".into()), - StatusCode::NOT_IMPLEMENTED => { - RetryAction::DontRetry("endpoint not implemented".into()) - } - _ if status.is_server_error() => RetryAction::Retry( - format!("{}: {}", status, String::from_utf8_lossy(response.body())).into(), - ), - _ if status.is_success() => RetryAction::Successful, - _ => RetryAction::DontRetry(format!("response status: {status}").into()), + if !status.is_success() { + debug!( + message = "HTTP response.", + %status, + body = %String::from_utf8_lossy(response.body()), + ); } + self.retry_strategy.retry_action(status) } } @@ -596,6 +723,7 @@ pub struct HttpStatusRetryLogic { func: F, request: PhantomData, response: PhantomData, + retry_strategy: RetryStrategy, } impl HttpStatusRetryLogic @@ -604,11 +732,12 @@ where Req: Send + Sync + 'static, Res: Send + Sync + 'static, { - pub const fn new(func: F) -> HttpStatusRetryLogic { + pub const fn new(func: F, retry_strategy: RetryStrategy) -> HttpStatusRetryLogic { HttpStatusRetryLogic { func, request: PhantomData, response: PhantomData, + retry_strategy, } } } @@ -623,25 +752,21 @@ where type Request = Req; type Response = Res; - fn is_retriable_error(&self, _error: &Self::Error) -> bool { - true + fn is_retriable_error(&self, error: &Self::Error) -> bool { + if self.retry_strategy == RetryStrategy::None { + false + } else { + error.is_retriable() + } + } + + fn is_retriable_timeout(&self) -> bool { + self.retry_strategy != RetryStrategy::None } fn should_retry_response(&self, response: &Res) -> RetryAction { let status = (self.func)(response); - - match status { - StatusCode::TOO_MANY_REQUESTS => RetryAction::Retry("too many requests".into()), - StatusCode::REQUEST_TIMEOUT => RetryAction::Retry("request timeout".into()), - StatusCode::NOT_IMPLEMENTED => { - RetryAction::DontRetry("endpoint not implemented".into()) - } - _ if status.is_server_error() => { - RetryAction::Retry(format!("Http Status: {status}").into()) - } - _ if status.is_success() => RetryAction::Successful, - _ => RetryAction::DontRetry(format!("Http status: {status}").into()), - } + self.retry_strategy.retry_action(status) } } @@ -654,6 +779,7 @@ where func: self.func.clone(), request: PhantomData, response: PhantomData, + retry_strategy: self.retry_strategy.clone(), } } } @@ -820,12 +946,17 @@ impl DriverResponse for HttpResponse { } /// Creates a `RetryLogic` for use with `HttpResponse`. -pub fn http_response_retry_logic() -> HttpStatusRetryLogic< +pub fn http_response_retry_logic( + retry_strategy: RetryStrategy, +) -> HttpStatusRetryLogic< impl Fn(&HttpResponse) -> StatusCode + Clone + Send + Sync + 'static, Request, HttpResponse, > { - HttpStatusRetryLogic::new(|req: &HttpResponse| req.http_response.status()) + HttpStatusRetryLogic::new( + |req: &HttpResponse| req.http_response.status(), + retry_strategy, + ) } /// Uses the estimated json encoded size to determine batch sizing. @@ -969,6 +1100,93 @@ mod test { ); } + #[test] + fn retry_strategy_none_preserves_success_and_rejects_failures() { + let strategy = RetryStrategy::None; + + assert!(strategy.retry_action::<()>(StatusCode::OK).is_successful()); + assert!( + strategy + .retry_action::<()>(StatusCode::INTERNAL_SERVER_ERROR) + .is_not_retryable() + ); + } + + #[test] + fn retry_strategy_none_disables_timeout_retries() { + let logic = HttpRetryLogic::<()> { + request: PhantomData, + retry_strategy: RetryStrategy::None, + }; + let status_logic = + HttpStatusRetryLogic::<_, (), ()>::new(|_: &()| StatusCode::OK, RetryStrategy::None); + + assert!(!logic.is_retriable_timeout()); + assert!(!status_logic.is_retriable_timeout()); + } + + #[test] + fn retry_strategy_all_preserves_success_and_retries_failures() { + let strategy = RetryStrategy::All; + + assert!(strategy.retry_action::<()>(StatusCode::OK).is_successful()); + assert!( + strategy + .retry_action::<()>(StatusCode::BAD_REQUEST) + .is_retryable() + ); + assert!( + strategy + .retry_action::<()>(StatusCode::INTERNAL_SERVER_ERROR) + .is_retryable() + ); + } + + #[test] + fn retry_strategy_custom_only_retries_configured_statuses() { + let strategy = RetryStrategy::Custom { + status_codes: vec![StatusCode::BAD_REQUEST], + }; + + assert!(strategy.retry_action::<()>(StatusCode::OK).is_successful()); + assert!( + strategy + .retry_action::<()>(StatusCode::BAD_REQUEST) + .is_retryable() + ); + assert!( + strategy + .retry_action::<()>(StatusCode::INTERNAL_SERVER_ERROR) + .is_not_retryable() + ); + } + + #[test] + fn retry_strategy_custom_serde_roundtrips_status_codes() { + let json = r#"{"type":"custom","status_codes":[400,503]}"#; + let strategy: RetryStrategy = serde_json::from_str(json).unwrap(); + assert_eq!( + strategy, + RetryStrategy::Custom { + status_codes: vec![StatusCode::BAD_REQUEST, StatusCode::SERVICE_UNAVAILABLE], + } + ); + let encoded = serde_json::to_string(&strategy).unwrap(); + let roundtrip: RetryStrategy = serde_json::from_str(&encoded).unwrap(); + assert_eq!(roundtrip, strategy); + } + + #[test] + fn retry_strategy_custom_serde_rejects_invalid_status_codes() { + // `http::StatusCode::from_u16` only accepts 100–999; 1000 is out of range. + let json = r#"{"type":"custom","status_codes":[1000]}"#; + let result = serde_json::from_str::(json); + assert!( + result.is_err(), + "expected invalid status code to fail deserialization" + ); + } + #[tokio::test] async fn util_http_it_makes_http_requests() { let (_guard, addr) = next_addr(); diff --git a/src/sinks/util/retries.rs b/src/sinks/util/retries.rs index be05d559bb18c..2a38a2b4ee900 100644 --- a/src/sinks/util/retries.rs +++ b/src/sinks/util/retries.rs @@ -34,6 +34,12 @@ pub trait RetryLogic: Clone + Send + Sync + 'static { /// implementors to specify what kinds of errors can be retried. fn is_retriable_error(&self, error: &Self::Error) -> bool; + /// When the Service call times out, this function allows implementors to + /// specify if the timeout should be retried. + fn is_retriable_timeout(&self) -> bool { + true + } + /// When the Service call returns an `Ok` response, this function allows /// implementors to specify additional logic to determine if the success response /// is actually an error. This is particularly useful when the downstream service @@ -197,10 +203,18 @@ where None } } else if error.downcast_ref::().is_some() { - warn!( - "Request timed out. If this happens often while the events are actually reaching their destination, try decreasing `batch.max_bytes` and/or using `compression` if applicable. Alternatively `request.timeout_secs` can be increased." - ); - Some(self.build_retry()) + if self.logic.is_retriable_timeout() { + warn!( + "Request timed out. If this happens often while the events are actually reaching their destination, try decreasing `batch.max_bytes` and/or using `compression` if applicable. Alternatively `request.timeout_secs` can be increased." + ); + Some(self.build_retry()) + } else { + error!( + message = + "Request timed out and is not retriable; dropping the request." + ); + None + } } else { error!( message = "Unexpected error type; dropping the request.", @@ -338,6 +352,27 @@ mod tests { assert_eq!(fut.await.unwrap(), "world"); } + #[tokio::test] + async fn timeout_error_no_retry() { + trace_init(); + + let policy = FibonacciRetryPolicy::new( + 5, + Duration::from_secs(1), + Duration::from_secs(10), + NoTimeoutRetryLogic, + JitterMode::None, + ); + + let (mut svc, mut handle) = mock::spawn_layer(RetryLayer::new(policy)); + + assert_ready_ok!(svc.poll_ready()); + + let mut fut = task::spawn(svc.call("hello")); + assert_request_eq!(handle, "hello").send_error(Elapsed::new()); + assert_ready_err!(fut.poll()); + } + #[test] fn backoff_grows_to_max() { let mut policy = FibonacciRetryPolicy::new( @@ -425,6 +460,23 @@ mod tests { } } + #[derive(Debug, Clone)] + struct NoTimeoutRetryLogic; + + impl RetryLogic for NoTimeoutRetryLogic { + type Error = Error; + type Request = &'static str; + type Response = &'static str; + + fn is_retriable_error(&self, error: &Self::Error) -> bool { + error.0 + } + + fn is_retriable_timeout(&self) -> bool { + false + } + } + #[derive(Debug)] struct Error(bool); diff --git a/website/cue/reference/components/sinks/generated/appsignal.cue b/website/cue/reference/components/sinks/generated/appsignal.cue index eb3710974335f..47943be3c1f93 100644 --- a/website/cue/reference/components/sinks/generated/appsignal.cue +++ b/website/cue/reference/components/sinks/generated/appsignal.cue @@ -323,6 +323,37 @@ generated: components: sinks: appsignal: configuration: { } } } + retry_strategy: { + description: """ + Configurable retry strategy for `http` based sinks. + + For more information about error responses, see [Client Error Responses][error_responses]. + + [error_responses]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Status#client_error_responses + """ + required: false + type: object: options: { + status_codes: { + description: "Retry on these specific HTTP status codes" + relevant_when: "type = \"custom\"" + required: true + type: array: items: type: uint: default: 200 + } + type: { + description: "The retry strategy enum." + required: false + type: string: { + default: "default" + enum: { + all: "Retry on *all* HTTP status codes except for success codes (2xx)" + custom: "Custom retry strategy" + default: "Default strategy. See [`RetryStrategy::retry_action`] for more details." + none: "Don't retry any errors, including request timeouts." + } + } + } + } + } tls: { description: "Configures the TLS options for incoming/outgoing connections." required: false diff --git a/website/cue/reference/components/sinks/generated/axiom.cue b/website/cue/reference/components/sinks/generated/axiom.cue index d4dc62e6294fc..a1f42404ac0c1 100644 --- a/website/cue/reference/components/sinks/generated/axiom.cue +++ b/website/cue/reference/components/sinks/generated/axiom.cue @@ -305,6 +305,37 @@ generated: components: sinks: axiom: configuration: { } } } + retry_strategy: { + description: """ + Configurable retry strategy for `http` based sinks. + + For more information about error responses, see [Client Error Responses][error_responses]. + + [error_responses]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Status#client_error_responses + """ + required: false + type: object: options: { + status_codes: { + description: "Retry on these specific HTTP status codes" + relevant_when: "type = \"custom\"" + required: true + type: array: items: type: uint: default: 200 + } + type: { + description: "The retry strategy enum." + required: false + type: string: { + default: "default" + enum: { + all: "Retry on *all* HTTP status codes except for success codes (2xx)" + custom: "Custom retry strategy" + default: "Default strategy. See [`RetryStrategy::retry_action`] for more details." + none: "Don't retry any errors, including request timeouts." + } + } + } + } + } tls: { description: """ The TLS settings for the connection. diff --git a/website/cue/reference/components/sinks/generated/azure_logs_ingestion.cue b/website/cue/reference/components/sinks/generated/azure_logs_ingestion.cue index 2aaf9b628e3c6..68944f5a6e7d5 100644 --- a/website/cue/reference/components/sinks/generated/azure_logs_ingestion.cue +++ b/website/cue/reference/components/sinks/generated/azure_logs_ingestion.cue @@ -408,6 +408,37 @@ generated: components: sinks: azure_logs_ingestion: configuration: { } } } + retry_strategy: { + description: """ + Configurable retry strategy for `http` based sinks. + + For more information about error responses, see [Client Error Responses][error_responses]. + + [error_responses]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Status#client_error_responses + """ + required: false + type: object: options: { + status_codes: { + description: "Retry on these specific HTTP status codes" + relevant_when: "type = \"custom\"" + required: true + type: array: items: type: uint: default: 200 + } + type: { + description: "The retry strategy enum." + required: false + type: string: { + default: "default" + enum: { + all: "Retry on *all* HTTP status codes except for success codes (2xx)" + custom: "Custom retry strategy" + default: "Default strategy. See [`RetryStrategy::retry_action`] for more details." + none: "Don't retry any errors, including request timeouts." + } + } + } + } + } stream_name: { description: """ The [Stream name][stream_name] for the Data collection rule. diff --git a/website/cue/reference/components/sinks/generated/azure_monitor_logs.cue b/website/cue/reference/components/sinks/generated/azure_monitor_logs.cue index 98b14f8b4faad..c7039b70097e1 100644 --- a/website/cue/reference/components/sinks/generated/azure_monitor_logs.cue +++ b/website/cue/reference/components/sinks/generated/azure_monitor_logs.cue @@ -314,6 +314,37 @@ generated: components: sinks: azure_monitor_logs: configuration: { } } } + retry_strategy: { + description: """ + Configurable retry strategy for `http` based sinks. + + For more information about error responses, see [Client Error Responses][error_responses]. + + [error_responses]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Status#client_error_responses + """ + required: false + type: object: options: { + status_codes: { + description: "Retry on these specific HTTP status codes" + relevant_when: "type = \"custom\"" + required: true + type: array: items: type: uint: default: 200 + } + type: { + description: "The retry strategy enum." + required: false + type: string: { + default: "default" + enum: { + all: "Retry on *all* HTTP status codes except for success codes (2xx)" + custom: "Custom retry strategy" + default: "Default strategy. See [`RetryStrategy::retry_action`] for more details." + none: "Don't retry any errors, including request timeouts." + } + } + } + } + } shared_key: { description: """ The [primary or the secondary key][shared_key] for the Log Analytics workspace. diff --git a/website/cue/reference/components/sinks/generated/datadog_events.cue b/website/cue/reference/components/sinks/generated/datadog_events.cue index 7316613acef4e..23b9fb43d6413 100644 --- a/website/cue/reference/components/sinks/generated/datadog_events.cue +++ b/website/cue/reference/components/sinks/generated/datadog_events.cue @@ -242,6 +242,37 @@ generated: components: sinks: datadog_events: configuration: { } } } + retry_strategy: { + description: """ + Configurable retry strategy for `http` based sinks. + + For more information about error responses, see [Client Error Responses][error_responses]. + + [error_responses]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Status#client_error_responses + """ + required: false + type: object: options: { + status_codes: { + description: "Retry on these specific HTTP status codes" + relevant_when: "type = \"custom\"" + required: true + type: array: items: type: uint: default: 200 + } + type: { + description: "The retry strategy enum." + required: false + type: string: { + default: "default" + enum: { + all: "Retry on *all* HTTP status codes except for success codes (2xx)" + custom: "Custom retry strategy" + default: "Default strategy. See [`RetryStrategy::retry_action`] for more details." + none: "Don't retry any errors, including request timeouts." + } + } + } + } + } site: { description: """ The Datadog [site][dd_site] to send observability data to. diff --git a/website/cue/reference/components/sinks/generated/gcp_stackdriver_logs.cue b/website/cue/reference/components/sinks/generated/gcp_stackdriver_logs.cue index 2909192f14d3f..9d5e76262bfd2 100644 --- a/website/cue/reference/components/sinks/generated/gcp_stackdriver_logs.cue +++ b/website/cue/reference/components/sinks/generated/gcp_stackdriver_logs.cue @@ -420,6 +420,37 @@ generated: components: sinks: gcp_stackdriver_logs: configuration: { } } } + retry_strategy: { + description: """ + Configurable retry strategy for `http` based sinks. + + For more information about error responses, see [Client Error Responses][error_responses]. + + [error_responses]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Status#client_error_responses + """ + required: false + type: object: options: { + status_codes: { + description: "Retry on these specific HTTP status codes" + relevant_when: "type = \"custom\"" + required: true + type: array: items: type: uint: default: 200 + } + type: { + description: "The retry strategy enum." + required: false + type: string: { + default: "default" + enum: { + all: "Retry on *all* HTTP status codes except for success codes (2xx)" + custom: "Custom retry strategy" + default: "Default strategy. See [`RetryStrategy::retry_action`] for more details." + none: "Don't retry any errors, including request timeouts." + } + } + } + } + } severity_key: { description: """ The field of the log event from which to take the outgoing log’s `severity` field. diff --git a/website/cue/reference/components/sinks/generated/gcp_stackdriver_metrics.cue b/website/cue/reference/components/sinks/generated/gcp_stackdriver_metrics.cue index 23ef271177f3a..52c344e278250 100644 --- a/website/cue/reference/components/sinks/generated/gcp_stackdriver_metrics.cue +++ b/website/cue/reference/components/sinks/generated/gcp_stackdriver_metrics.cue @@ -334,6 +334,37 @@ generated: components: sinks: gcp_stackdriver_metrics: configuration: { } } } + retry_strategy: { + description: """ + Configurable retry strategy for `http` based sinks. + + For more information about error responses, see [Client Error Responses][error_responses]. + + [error_responses]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Status#client_error_responses + """ + required: false + type: object: options: { + status_codes: { + description: "Retry on these specific HTTP status codes" + relevant_when: "type = \"custom\"" + required: true + type: array: items: type: uint: default: 200 + } + type: { + description: "The retry strategy enum." + required: false + type: string: { + default: "default" + enum: { + all: "Retry on *all* HTTP status codes except for success codes (2xx)" + custom: "Custom retry strategy" + default: "Default strategy. See [`RetryStrategy::retry_action`] for more details." + none: "Don't retry any errors, including request timeouts." + } + } + } + } + } tls: { description: "TLS configuration." required: false diff --git a/website/cue/reference/components/sinks/generated/honeycomb.cue b/website/cue/reference/components/sinks/generated/honeycomb.cue index 66ba9ed795a27..c65064a135a00 100644 --- a/website/cue/reference/components/sinks/generated/honeycomb.cue +++ b/website/cue/reference/components/sinks/generated/honeycomb.cue @@ -321,4 +321,35 @@ generated: components: sinks: honeycomb: configuration: { } } } + retry_strategy: { + description: """ + Configurable retry strategy for `http` based sinks. + + For more information about error responses, see [Client Error Responses][error_responses]. + + [error_responses]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Status#client_error_responses + """ + required: false + type: object: options: { + status_codes: { + description: "Retry on these specific HTTP status codes" + relevant_when: "type = \"custom\"" + required: true + type: array: items: type: uint: default: 200 + } + type: { + description: "The retry strategy enum." + required: false + type: string: { + default: "default" + enum: { + all: "Retry on *all* HTTP status codes except for success codes (2xx)" + custom: "Custom retry strategy" + default: "Default strategy. See [`RetryStrategy::retry_action`] for more details." + none: "Don't retry any errors, including request timeouts." + } + } + } + } + } } diff --git a/website/cue/reference/components/sinks/generated/http.cue b/website/cue/reference/components/sinks/generated/http.cue index 0fcc53fab8092..8523a27dd0d9b 100644 --- a/website/cue/reference/components/sinks/generated/http.cue +++ b/website/cue/reference/components/sinks/generated/http.cue @@ -1024,6 +1024,37 @@ generated: components: sinks: http: configuration: { } } } + retry_strategy: { + description: """ + Configurable retry strategy for `http` based sinks. + + For more information about error responses, see [Client Error Responses][error_responses]. + + [error_responses]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Status#client_error_responses + """ + required: false + type: object: options: { + status_codes: { + description: "Retry on these specific HTTP status codes" + relevant_when: "type = \"custom\"" + required: true + type: array: items: type: uint: default: 200 + } + type: { + description: "The retry strategy enum." + required: false + type: string: { + default: "default" + enum: { + all: "Retry on *all* HTTP status codes except for success codes (2xx)" + custom: "Custom retry strategy" + default: "Default strategy. See [`RetryStrategy::retry_action`] for more details." + none: "Don't retry any errors, including request timeouts." + } + } + } + } + } tls: { description: "TLS configuration." required: false diff --git a/website/cue/reference/components/sinks/generated/keep.cue b/website/cue/reference/components/sinks/generated/keep.cue index 29b1d7536611b..d2ed08114a2a9 100644 --- a/website/cue/reference/components/sinks/generated/keep.cue +++ b/website/cue/reference/components/sinks/generated/keep.cue @@ -286,4 +286,35 @@ generated: components: sinks: keep: configuration: { } } } + retry_strategy: { + description: """ + Configurable retry strategy for `http` based sinks. + + For more information about error responses, see [Client Error Responses][error_responses]. + + [error_responses]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Status#client_error_responses + """ + required: false + type: object: options: { + status_codes: { + description: "Retry on these specific HTTP status codes" + relevant_when: "type = \"custom\"" + required: true + type: array: items: type: uint: default: 200 + } + type: { + description: "The retry strategy enum." + required: false + type: string: { + default: "default" + enum: { + all: "Retry on *all* HTTP status codes except for success codes (2xx)" + custom: "Custom retry strategy" + default: "Default strategy. See [`RetryStrategy::retry_action`] for more details." + none: "Don't retry any errors, including request timeouts." + } + } + } + } + } } diff --git a/website/cue/reference/components/sinks/generated/opentelemetry.cue b/website/cue/reference/components/sinks/generated/opentelemetry.cue index 4505d8a9bf512..4e8e9d9ff7210 100644 --- a/website/cue/reference/components/sinks/generated/opentelemetry.cue +++ b/website/cue/reference/components/sinks/generated/opentelemetry.cue @@ -1025,6 +1025,37 @@ generated: components: sinks: opentelemetry: configuration: protocol: { } } } + retry_strategy: { + description: """ + Configurable retry strategy for `http` based sinks. + + For more information about error responses, see [Client Error Responses][error_responses]. + + [error_responses]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Status#client_error_responses + """ + required: false + type: object: options: { + status_codes: { + description: "Retry on these specific HTTP status codes" + relevant_when: "type = \"custom\"" + required: true + type: array: items: type: uint: default: 200 + } + type: { + description: "The retry strategy enum." + required: false + type: string: { + default: "default" + enum: { + all: "Retry on *all* HTTP status codes except for success codes (2xx)" + custom: "Custom retry strategy" + default: "Default strategy. See [`RetryStrategy::retry_action`] for more details." + none: "Don't retry any errors, including request timeouts." + } + } + } + } + } tls: { description: "TLS configuration." required: false diff --git a/website/cue/reference/components/sinks/generated/prometheus_remote_write.cue b/website/cue/reference/components/sinks/generated/prometheus_remote_write.cue index c13fa424999ec..5d74d52359bdd 100644 --- a/website/cue/reference/components/sinks/generated/prometheus_remote_write.cue +++ b/website/cue/reference/components/sinks/generated/prometheus_remote_write.cue @@ -539,6 +539,37 @@ generated: components: sinks: prometheus_remote_write: configuration: { } } } + retry_strategy: { + description: """ + Configurable retry strategy for `http` based sinks. + + For more information about error responses, see [Client Error Responses][error_responses]. + + [error_responses]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Status#client_error_responses + """ + required: false + type: object: options: { + status_codes: { + description: "Retry on these specific HTTP status codes" + relevant_when: "type = \"custom\"" + required: true + type: array: items: type: uint: default: 200 + } + type: { + description: "The retry strategy enum." + required: false + type: string: { + default: "default" + enum: { + all: "Retry on *all* HTTP status codes except for success codes (2xx)" + custom: "Custom retry strategy" + default: "Default strategy. See [`RetryStrategy::retry_action`] for more details." + none: "Don't retry any errors, including request timeouts." + } + } + } + } + } tenant_id: { description: """ The tenant ID to send. From 9c617a7b766dc95ea919384ff16da2654595f6a4 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Fri, 1 May 2026 10:04:29 -0400 Subject: [PATCH 183/326] fix(ci): grant issues:write to remove_wip_label workflow (#25339) The github.rest.issues.removeLabel call (DELETE /repos/{owner}/{repo}/issues/{issue_number}/labels/{name}) requires the issues:write permission per the GitHub REST API. The workflow only declared pull-requests:write, so the call returned 403 the first time all preconditions actually held simultaneously (MEMBER reviewer + label present + approval). Co-authored-by: Claude Opus 4.7 (1M context) --- .github/workflows/remove_wip_label.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/remove_wip_label.yml b/.github/workflows/remove_wip_label.yml index 7c30ee892a949..b4750b0f2b0d6 100644 --- a/.github/workflows/remove_wip_label.yml +++ b/.github/workflows/remove_wip_label.yml @@ -1,6 +1,7 @@ name: Remove Work In Progress Label permissions: + issues: write pull-requests: write on: From 308d2469b2e70153d3eafd20a647bf33a9d69d73 Mon Sep 17 00:00:00 2001 From: tot19 <31141271+tot19@users.noreply.github.com> Date: Fri, 1 May 2026 23:43:39 +0900 Subject: [PATCH 184/326] fix(sources): prevent windows_event_log permanent freeze from signal-event lost wakeup (#25195) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(windows_event_log): fix pre-drain ResetEvent race and add lost-wakeup regression tests The Windows Event Log service signals the pull-mode wait handle via SetEvent each time a new matching event is recorded. Because the handle is manual-reset, any SetEvent that fires between the last EvtNext call and the post-drain ResetEvent is silently lost — the subscription then hangs until the next OS event arrives (vectordotdev/vector#25194). Fix: reset the handle *before* entering the drain loop. Signals raised during the drain are preserved because SetEvent on an already-signaled handle is a no-op. Re-arm (SetEvent) on early exits so the next pull_events revisits the channel without waiting for a fresh OS notification: - budget exhaustion - bookmark failure mid-batch - transient EvtNext error Regression tests: - test_pull_events_preserves_setevent_during_drain: installs DRAIN_STEP_HOOK to fire SetEvent mid-drain and asserts wait_for_events_blocking returns EventsAvailable, not Timeout. - test_speculative_pull_recovers_without_signal: manually clears the channel signal via ResetEvent, confirms wait times out, then asserts pull_events still returns events — proving the speculative timeout pull in mod.rs self-heals independently of signal state. Also: comment re-subscription break paths (ERROR_EVT_QUERY_RESULT_STALE and INVALID_POSITION) noting the speculative pull as a safety net if the re-subbed channel does not immediately re-signal; add serialization note to DRAIN_STEP_HOOK. * fix(windows_event_log): add speculative timeout pull, deduplicate processing, fix error handling Four related improvements to mod.rs: 1. Speculative pull on WaitResult::Timeout: call pull_events on every timeout cycle as a belt-and-suspenders self-heal. EvtNext returns ERROR_NO_MORE_ITEMS immediately on an empty channel (near-zero cost). If events are recovered a warning is emitted. Guarantees recovery within one timeout period regardless of the root cause of the lost wakeup. 2. Extract with_subscription_blocking helper: wraps the spawn_blocking ownership-transfer pattern (move subscription in, run blocking fn, return subscription + result). All three blocking calls (wait, normal pull, speculative pull) now use this helper instead of inlining spawn_blocking. 3. Extract process_event_batch helper: the parse/emit/send_batch/finalize sequence was duplicated verbatim between the EventsAvailable arm and the speculative-timeout arm. Extracted into a shared free async function. Rate limiting is applied consistently in both paths via the helper. 4. Fix error-handling asymmetry: the speculative pull Err branch previously only logged warn! and continued, so a non-recoverable error (access denied, channel not found) would spam warnings indefinitely. Now mirrors the EventsAvailable path: emit WindowsEventLogQueryError, break on non-recoverable errors, apply exponential backoff on recoverable ones. * test(windows_event_log source): harden lost-wakeup regression tests against flakiness Address two independent flakiness sources in the #25194 regression tests so the suite is stable on real Windows CI runners. test_pull_events_preserves_setevent_during_drain: - Replaced a 1000ms blocking wait with an immediate 0ms poll after pull_events returns, so the check measures only the reset/preserve behavior of pull_events and is not contaminated by unrelated Windows system events signaling the handle during a nonzero wait window. - Keyed the DRAIN_STEP_HOOK fire-once to the subscription's own signal handle so concurrent pull_events calls from other tests can't flip the hook first and SetEvent the wrong handle. test_speculative_pull_recovers_without_signal: - Same 500ms→0ms poll change, opposite direction: real events arriving during the wait would have re-signaled the manually-cleared handle and flipped the expected Timeout into a real signal result. - Seed a deterministic record via 'eventcreate' before subscription creation so the non-empty-events assertion is independent of whatever backlog the runner happens to have. Freshly provisioned images can have an empty Application log, which would otherwise cause pull_events(100) to legitimately return empty and false-fail the test. * fix(windows_event_log): implement Sync for subscription types to satisfy Send bound on source future EventLogSubscription and ChannelSubscription had unsafe impl Send but no Sync impl. Since &T: Send requires T: Sync, process_event_batch holding &EventLogSubscription across an .await made the entire source future !Send, causing a compile error (ICE + future-not-Send) in release builds. All mutation on these types requires &mut self; &self methods are read-only or delegate to already-Sync types (RateLimiter). The underlying Windows kernel handles are safe for concurrent access. Co-Authored-By: Claude Sonnet 4.6 * chore(changelog): simplify windows_event_log fix fragment to user-focused one-liner Co-Authored-By: Claude Sonnet 4.6 * fix(windows_event_log): prioritize shutdown signal * fix(windows_event_log): lighten speculative timeout pulls --------- Co-authored-by: Claude Sonnet 4.6 Co-authored-by: Pavlos Rontidis --- ...25194_windows_event_log_lost_wakeup.fix.md | 3 + src/sources/windows_event_log/mod.rs | 281 ++++++++----- src/sources/windows_event_log/subscription.rs | 374 ++++++++++++++++-- 3 files changed, 546 insertions(+), 112 deletions(-) create mode 100644 changelog.d/25194_windows_event_log_lost_wakeup.fix.md diff --git a/changelog.d/25194_windows_event_log_lost_wakeup.fix.md b/changelog.d/25194_windows_event_log_lost_wakeup.fix.md new file mode 100644 index 0000000000000..cb6f67987d67c --- /dev/null +++ b/changelog.d/25194_windows_event_log_lost_wakeup.fix.md @@ -0,0 +1,3 @@ +The `windows_event_log` source no longer freezes after periods of inactivity. + +authors: tot19 diff --git a/src/sources/windows_event_log/mod.rs b/src/sources/windows_event_log/mod.rs index 17d900cc07622..28dfc1db4cb79 100644 --- a/src/sources/windows_event_log/mod.rs +++ b/src/sources/windows_event_log/mod.rs @@ -29,7 +29,7 @@ cfg_if::cfg_if! { use vector_lib::EstimatedJsonEncodedSizeOf; use vector_lib::finalizer::OrderedFinalizer; use vector_lib::internal_event::{ - ByteSize, BytesReceived, CountByteSize, InternalEventHandle as _, Protocol, + ByteSize, BytesReceived, CountByteSize, InternalEventHandle, Protocol, }; use windows::Win32::Foundation::{DUPLICATE_SAME_ACCESS, DuplicateHandle, HANDLE}; use windows::Win32::System::Threading::GetCurrentProcess; @@ -48,6 +48,7 @@ cfg_if::cfg_if! { error::WindowsEventLogError, parser::EventLogParser, subscription::{EventLogSubscription, WaitResult}, + xml_parser::WindowsEvent, }; } } @@ -157,6 +158,107 @@ impl Finalizer { } } +/// Parse, emit metrics for, send, and finalize a non-empty batch of pulled Windows events. +/// +/// Both the `EventsAvailable` path and the speculative-timeout path share this +/// logic. Returns `true` if the downstream pipeline closed and the caller +/// should break out of the main event loop. +async fn process_event_batch( + events: Vec, + parser: &EventLogParser, + acknowledgements: bool, + subscription: &EventLogSubscription, + out: &mut SourceSender, + finalizer: &Finalizer, + events_received: &impl InternalEventHandle, + bytes_received: &impl InternalEventHandle, +) -> bool { + // Rate limiting between batches (async-compatible). + if let Some(limiter) = subscription.rate_limiter() { + limiter.until_ready().await; + } + + let (batch, receiver) = BatchNotifier::maybe_new_with_receiver(acknowledgements); + let mut log_events = Vec::new(); + let mut total_byte_size = 0usize; + let mut channels_in_batch = std::collections::HashSet::new(); + + for event in events { + let channel = event.channel.clone(); + channels_in_batch.insert(channel.clone()); + let event_id = event.event_id; + match parser.parse_event(event) { + Ok(mut log_event) => { + let byte_size = log_event.estimated_json_encoded_size_of(); + total_byte_size += byte_size.get(); + if let Some(ref batch) = batch { + log_event = log_event.with_batch_notifier(batch); + } + log_events.push(log_event); + } + Err(e) => { + emit!(WindowsEventLogParseError { + error: e.to_string(), + channel, + event_id: Some(event_id), + }); + } + } + } + + if !log_events.is_empty() { + let count = log_events.len(); + events_received.emit(CountByteSize(count, total_byte_size.into())); + bytes_received.emit(ByteSize(total_byte_size)); + + // BACK PRESSURE: block until the pipeline accepts the batch. + // We don't call EvtNext again until this completes. + if let Err(_error) = out.send_batch(log_events).await { + emit!(StreamClosedError { count }); + return true; // signal: break the main loop + } + + // Register checkpoint entry with the finalizer. + let bookmarks: Vec<(String, String)> = channels_in_batch + .into_iter() + .filter_map(|channel| { + subscription + .get_bookmark_xml(&channel) + .map(|xml| (channel, xml)) + }) + .collect(); + + if !bookmarks.is_empty() { + let entry = FinalizerEntry { bookmarks }; + finalizer.finalize(entry, receiver).await; + } + } + + false // pipeline still open +} + +/// Transfer ownership of `subscription` into a `spawn_blocking` task, run `f` +/// on it, then return both the subscription and the result. +/// +/// All blocking Windows APIs (`WaitForMultipleObjects`, `EvtNext`, `EvtRender`) +/// must run in `spawn_blocking` to avoid stalling the async runtime. The +/// ownership-transfer pattern ensures only one thread holds the subscription +/// at a time, preventing data races without requiring locks. +async fn with_subscription_blocking( + subscription: EventLogSubscription, + f: F, +) -> Result<(EventLogSubscription, R), WindowsEventLogError> +where + F: FnOnce(EventLogSubscription) -> (EventLogSubscription, R) + Send + 'static, + R: Send + 'static, +{ + tokio::task::spawn_blocking(move || f(subscription)) + .await + .map_err(|e| WindowsEventLogError::ConfigError { + message: format!("Blocking subscription task panicked: {e}"), + }) +} + /// Windows Event Log source implementation pub struct WindowsEventLogSource { config: WindowsEventLogConfig, @@ -281,42 +383,25 @@ impl WindowsEventLogSource { // Ownership transfer ensures no data races between the blocking thread // and async code. The shutdown watcher uses a raw HANDLE value (just an // integer) to signal shutdown without needing access to the subscription. - let (returned_sub, wait_result) = tokio::task::spawn_blocking({ - let sub = subscription; - move || { + let (returned_sub, wait_result) = + with_subscription_blocking(subscription, move |sub| { let result = sub.wait_for_events_blocking(timeout_ms); (sub, result) - } - }) - .await - .map_err(|e| WindowsEventLogError::ConfigError { - message: format!("Wait task panicked: {e}"), - })?; - + }) + .await?; subscription = returned_sub; match wait_result { WaitResult::EventsAvailable => { // Pull events via spawn_blocking (EvtNext/EvtRender are blocking APIs) - let (returned_sub, events_result) = tokio::task::spawn_blocking({ - let mut sub = subscription; - move || { + let (returned_sub, events_result) = + with_subscription_blocking(subscription, move |mut sub| { let result = sub.pull_events(batch_size); (sub, result) - } - }) - .await - .map_err(|e| WindowsEventLogError::ConfigError { - message: format!("Pull task panicked: {e}"), - })?; - + }) + .await?; subscription = returned_sub; - // Rate limiting between batches (async-compatible) - if let Some(limiter) = subscription.rate_limiter() { - limiter.until_ready().await; - } - match events_result { Ok(events) if events.is_empty() => { error_backoff = std::time::Duration::from_millis(100); @@ -328,65 +413,19 @@ impl WindowsEventLogSource { message = "Pulled Windows Event Log events.", event_count = events.len() ); - - let (batch, receiver) = - BatchNotifier::maybe_new_with_receiver(acknowledgements); - - let mut log_events = Vec::new(); - let mut total_byte_size = 0; - let mut channels_in_batch = std::collections::HashSet::new(); - - for event in events { - let channel = event.channel.clone(); - channels_in_batch.insert(channel.clone()); - let event_id = event.event_id; - match parser.parse_event(event) { - Ok(mut log_event) => { - let byte_size = log_event.estimated_json_encoded_size_of(); - total_byte_size += byte_size.get(); - - if let Some(ref batch) = batch { - log_event = log_event.with_batch_notifier(batch); - } - - log_events.push(log_event); - } - Err(e) => { - emit!(WindowsEventLogParseError { - error: e.to_string(), - channel, - event_id: Some(event_id), - }); - } - } - } - - if !log_events.is_empty() { - let count = log_events.len(); - events_received.emit(CountByteSize(count, total_byte_size.into())); - bytes_received.emit(ByteSize(total_byte_size)); - - // BACK PRESSURE: block here until the pipeline accepts - // the batch. We don't call EvtNext again until this completes. - if let Err(_error) = out.send_batch(log_events).await { - emit!(StreamClosedError { count }); - break; - } - - // Register checkpoint entry with finalizer - let bookmarks: Vec<(String, String)> = channels_in_batch - .into_iter() - .filter_map(|channel| { - subscription - .get_bookmark_xml(&channel) - .map(|xml| (channel, xml)) - }) - .collect(); - - if !bookmarks.is_empty() { - let entry = FinalizerEntry { bookmarks }; - finalizer.finalize(entry, receiver).await; - } + if process_event_batch( + events, + &parser, + acknowledgements, + &subscription, + &mut out, + &finalizer, + &events_received, + &bytes_received, + ) + .await + { + break; } } Err(e) => { @@ -415,10 +454,6 @@ impl WindowsEventLogSource { } WaitResult::Timeout => { - // A full wait cycle without errors means the system is healthy; - // reset backoff so the next transient error starts fresh. - error_backoff = std::time::Duration::from_millis(100); - // Periodic checkpoint flush (sync mode only) if !acknowledgements && last_checkpoint.elapsed() >= checkpoint_interval { if let Err(e) = subscription.flush_bookmarks().await { @@ -448,6 +483,74 @@ impl WindowsEventLogSource { ); } } + + // Speculative pull: self-heal against any lost-wakeup scenario, + // regardless of root cause. If the OS signal was lost through any + // mechanism (not just the pre-drain race fixed in #25194), this + // ensures the source recovers within one timeout period. + // Use the speculative pull variant so idle timeout cycles don't + // refresh per-channel record-count gauges via EvtOpenLog / + // EvtGetLogInfo on every configured channel. + let (returned_sub, speculative_result) = + with_subscription_blocking(subscription, move |mut sub| { + let result = sub.pull_events_speculative(batch_size); + (sub, result) + }) + .await?; + subscription = returned_sub; + + match speculative_result { + Ok(events) if events.is_empty() => { + // Healthy cycle: reset backoff so the next transient + // error starts fresh. + error_backoff = std::time::Duration::from_millis(100); + } + Ok(events) => { + // Healthy cycle: reset backoff so the next transient + // error starts fresh. + error_backoff = std::time::Duration::from_millis(100); + warn!( + message = "Speculative timeout pull recovered events; possible lost wakeup detected.", + event_count = events.len(), + ); + if process_event_batch( + events, + &parser, + acknowledgements, + &subscription, + &mut out, + &finalizer, + &events_received, + &bytes_received, + ) + .await + { + break; + } + } + Err(e) => { + emit!(WindowsEventLogQueryError { + channel: "all".to_string(), + query: None, + error: e.to_string(), + }); + if !e.is_recoverable() { + error!( + message = "Non-recoverable speculative pull error, shutting down.", + error = %e + ); + break; + } + // Exponential backoff mirrors the EventsAvailable error path. + warn!( + message = "Recoverable speculative pull error, backing off.", + backoff_ms = error_backoff.as_millis() as u64, + error = %e + ); + tokio::time::sleep(error_backoff).await; + error_backoff = (error_backoff * 2).min(MAX_ERROR_BACKOFF); + } + } } WaitResult::Shutdown => { diff --git a/src/sources/windows_event_log/subscription.rs b/src/sources/windows_event_log/subscription.rs index dc92713f691d9..4571561a2d349 100644 --- a/src/sources/windows_event_log/subscription.rs +++ b/src/sources/windows_event_log/subscription.rs @@ -18,9 +18,9 @@ use windows::Win32::System::EventLog::{ EvtSubscribeStartAfterBookmark, EvtSubscribeStartAtOldestRecord, EvtSubscribeStrict, EvtSubscribeToFutureEvents, }; -#[cfg(test)] -use windows::Win32::System::Threading::SetEvent; -use windows::Win32::System::Threading::{CreateEventW, ResetEvent, WaitForMultipleObjects}; +use windows::Win32::System::Threading::{ + CreateEventW, ResetEvent, SetEvent, WaitForMultipleObjects, +}; use windows::core::HSTRING; use super::{ @@ -30,6 +30,19 @@ use super::{ use crate::internal_events::WindowsEventLogBookmarkError; +/// Test-only hook called inside the `pull_events` drain loop after each +/// `EvtNext` invocation. Used by the lost-wakeup regression test +/// (see `test_pull_events_preserves_setevent_during_drain`) to race a +/// `SetEvent` against the drain without relying on thread-timing. +/// No-op and zero-cost in non-test builds. +/// +/// Only one test should install a hook at a time; tests that install a hook +/// must use `#[serial_test::serial]` or equivalent serialization to prevent +/// concurrent tests from triggering each other's hook. +#[cfg(test)] +static DRAIN_STEP_HOOK: std::sync::Mutex>> = + std::sync::Mutex::new(None); + /// Maximum number of entries in the EvtFormatMessage result cache. pub const FORMAT_CACHE_CAPACITY: usize = 10_000; /// Maximum number of cached publisher metadata handles. @@ -80,6 +93,7 @@ struct ChannelSubscription { // SAFETY: Same rationale as EventLogSubscription - Windows kernel handles are thread-safe. unsafe impl Send for ChannelSubscription {} +unsafe impl Sync for ChannelSubscription {} /// Result of waiting for events across all channels. pub enum WaitResult { @@ -130,8 +144,10 @@ pub struct EventLogSubscription { // SAFETY: Windows HANDLE and EVT_HANDLE are kernel objects safe to use across // threads. In windows 0.58, HANDLE wraps *mut c_void which is !Send/!Sync, -// but the underlying kernel handles are thread-safe. +// but the underlying kernel handles are thread-safe. All mutation requires +// &mut self; &self methods are read-only or delegate to Sync types (RateLimiter). unsafe impl Send for EventLogSubscription {} +unsafe impl Sync for EventLogSubscription {} impl EventLogSubscription { /// Create a new pull-model subscription for all configured channels. @@ -415,21 +431,20 @@ impl EventLogSubscription { /// Wait for events to become available on any channel, or for shutdown. /// /// Uses `WaitForMultipleObjects` via `spawn_blocking` to avoid blocking the - /// Tokio runtime. The wait array includes all channel signal events plus the - /// shutdown event. + /// Tokio runtime. The wait array puts shutdown first so a stop request wins + /// over any channel that is already signaled. pub fn wait_for_events_blocking(&self, timeout_ms: u32) -> WaitResult { - // Build wait handle array: [channel0_signal, channel1_signal, ..., shutdown_event] - let mut handles: Vec = self.channels.iter().map(|c| c.signal_event).collect(); + // Build wait handle array: [shutdown_event, channel0_signal, channel1_signal, ...] + let mut handles = Vec::with_capacity(self.channels.len() + 1); handles.push(self.shutdown_event); + handles.extend(self.channels.iter().map(|c| c.signal_event)); let result = unsafe { WaitForMultipleObjects(&handles, false, timeout_ms) }; - let shutdown_index = (self.channels.len()) as u32; - match result { r if r == WAIT_TIMEOUT => WaitResult::Timeout, - r if r.0 < WAIT_OBJECT_0.0 + shutdown_index => WaitResult::EventsAvailable, - r if r.0 == WAIT_OBJECT_0.0 + shutdown_index => WaitResult::Shutdown, + r if r == WAIT_OBJECT_0 => WaitResult::Shutdown, + r if r.0 <= WAIT_OBJECT_0.0 + self.channels.len() as u32 => WaitResult::EventsAvailable, _ => { // WAIT_FAILED or unexpected - treat as timeout to avoid tight loop warn!( @@ -459,6 +474,28 @@ impl EventLogSubscription { pub fn pull_events( &mut self, max_events: usize, + ) -> Result, WindowsEventLogError> { + self.pull_events_inner(max_events, true) + } + + /// Pull events for timeout-based speculative recovery. + /// + /// This keeps the same event-drain behavior as `pull_events`, but avoids + /// refreshing per-channel record-count gauges for channels that were empty. + /// Timeout pulls can run repeatedly while the host is idle, so skipping + /// those metadata queries prevents steady `EvtOpenLog`/`EvtGetLogInfo` + /// churn without changing event recovery behavior. + pub fn pull_events_speculative( + &mut self, + max_events: usize, + ) -> Result, WindowsEventLogError> { + self.pull_events_inner(max_events, false) + } + + fn pull_events_inner( + &mut self, + max_events: usize, + update_records_for_empty_channels: bool, ) -> Result, WindowsEventLogError> { let mut all_events = Vec::with_capacity(max_events.min(1000)); let num_channels = self.channels.len().max(1); @@ -479,9 +516,25 @@ impl EventLogSubscription { let mut bookmark_failed = false; let mut channel_count = 0usize; - // Drain loop: keep calling EvtNext until ERROR_NO_MORE_ITEMS or channel budget. - // Only reset the signal once the channel is fully drained; if we hit the - // budget limit the signal stays set so WaitForMultipleObjects returns immediately. + // Reset the signal BEFORE draining to avoid a lost-wakeup race + // (see vectordotdev/vector#25194). The Windows Event Log service + // signals this manual-reset event via SetEvent each time a new + // matching event is recorded; SetEvent on an already-signaled + // event is a no-op, so if we reset AFTER draining, any signal + // that arrives between our last EvtNext and ResetEvent is lost + // — the subscription then hangs until the next event arrives. + // Resetting first means any signal raised during the drain is + // preserved, causing the next WaitForMultipleObjects to return + // immediately. + // + // If we exit the drain loop early (channel budget exhausted or + // bookmark update failed mid-batch), we re-SetEvent at the end + // of this iteration so the next pull_events call revisits this + // channel without waiting for a fresh OS signal. + unsafe { + let _ = ResetEvent(channel_sub.signal_event); + } + 'drain: loop { if channel_count >= channel_limit { break; @@ -501,6 +554,17 @@ impl EventLogSubscription { ) }; + // Test-only hook: lets the lost-wakeup regression test race + // a SetEvent against the drain without thread-timing. No-op + // and zero-cost in non-test builds. + #[cfg(test)] + { + let hook = DRAIN_STEP_HOOK.lock().unwrap().clone(); + if let Some(h) = hook { + h(channel_sub.signal_event); + } + } + if let Err(err) = result { let code = (err.code().0 as u32) & 0xFFFF; if code == ERROR_NO_MORE_ITEMS { @@ -513,6 +577,8 @@ impl EventLogSubscription { channel = %channel_sub.channel ); channel_drained = true; + // Speculative pull on timeout in mod.rs is a safety net if the + // re-subscribed channel does not immediately re-signal. break; } if code == ERROR_EVT_QUERY_RESULT_INVALID_POSITION { @@ -526,7 +592,9 @@ impl EventLogSubscription { message = "Re-subscription succeeded after stale query.", channel = %channel_sub.channel ); - // Retry from fresh subscription — the signal will fire again + // Retry from fresh subscription — the signal will fire again. + // Speculative pull on timeout in mod.rs is a safety net if + // the new subscription does not immediately re-signal. channel_drained = true; break; } @@ -538,10 +606,23 @@ impl EventLogSubscription { ); channel_sub.subscription_active_gauge.set(0.0); channel_drained = true; + // Speculative pull on timeout in mod.rs is a safety net if + // the failed channel does not re-signal after recovery. break; } } } + // Re-arm the signal before returning. We reset it pre-drain + // but are bailing out without confirming the drain completed, + // so if events were left un-drained the next pull_events must + // still revisit this channel without waiting for a fresh OS + // signal. This mirrors the `else` branch below that handles + // budget-exhaustion and bookmark-failure early breaks, and + // avoids the same lost-wakeup symptom (vectordotdev/vector#25194) + // on transient EvtNext failures. + unsafe { + let _ = SetEvent(channel_sub.signal_event); + } return Err(WindowsEventLogError::PullEventsError { channel: channel_sub.channel.clone(), source: err, @@ -697,15 +778,22 @@ impl EventLogSubscription { } if channel_drained && !bookmark_failed { + // Update channel record count gauge for lag detection. + if update_records_for_empty_channels || channel_count > 0 { + super::render::update_channel_records( + &channel_sub.channel, + &channel_sub.channel_records_gauge, + ); + } + } else { + // Drain exited early (budget exhausted or bookmark_failed + // mid-batch). Re-arm the signal so the next pull_events + // revisits this channel immediately without waiting for a + // fresh OS notification. Pairs with the pre-drain ResetEvent + // above. unsafe { - let _ = ResetEvent(channel_sub.signal_event); + let _ = SetEvent(channel_sub.signal_event); } - - // Update channel record count gauge for lag detection. - super::render::update_channel_records( - &channel_sub.channel, - &channel_sub.channel_records_gauge, - ); } } @@ -816,6 +904,15 @@ impl EventLogSubscription { self.shutdown_event.0 } + /// Test-only accessor for the first channel's signal event handle. Used + /// by the lost-wakeup regression test to scope its drain-loop hook to + /// exactly this subscription, so it does not fire on concurrent + /// `pull_events` calls from other tests in the same process. + #[cfg(test)] + pub(super) fn first_channel_signal_raw(&self) -> isize { + self.channels[0].signal_event.0 as isize + } + /// Returns a reference to the rate limiter, if configured. pub const fn rate_limiter( &self, @@ -1005,6 +1102,7 @@ impl Drop for EventLogSubscription { #[cfg(test)] mod tests { use super::*; + use serial_test::serial; async fn create_test_checkpointer() -> (Arc, tempfile::TempDir) { let temp_dir = tempfile::TempDir::new().unwrap(); @@ -1136,6 +1234,31 @@ mod tests { drop(subscription); } + /// Test that shutdown wins when both shutdown and channel handles are signaled. + #[tokio::test] + async fn test_shutdown_signal_takes_priority_over_channel_signal() { + let mut config = WindowsEventLogConfig::default(); + config.channels = vec!["Application".to_string()]; + config.event_timeout_ms = 500; + + let (checkpointer, _temp_dir) = create_test_checkpointer().await; + + let subscription = EventLogSubscription::new(&config, checkpointer, false) + .await + .expect("Subscription creation should succeed"); + + unsafe { + let handle = HANDLE(subscription.shutdown_event_raw()); + let _ = SetEvent(handle); + } + + let result = subscription.wait_for_events_blocking(0); + assert!( + matches!(result, WaitResult::Shutdown), + "shutdown should take priority over already-signaled channels" + ); + } + /// Test pull_events with read_existing_events=true #[tokio::test] async fn test_pull_events_returns_events() { @@ -1272,4 +1395,209 @@ mod tests { // that the subscription is functional. let _events = subscription.pull_events(100).unwrap_or_default(); } + + /// Proves that `pull_events` works independently of signal state — the + /// invariant the speculative timeout pull in mod.rs relies on. + /// + /// Steps: + /// 1. Subscribe to the Application log with `read_existing_events = true`. + /// 2. Manually clear the channel signal via `ResetEvent`, simulating a lost wakeup. + /// 3. Assert `wait_for_events_blocking` times out (signal cleared, no OS wake-up). + /// 4. Assert `pull_events` still returns events — `EvtNext` fetches from the queue + /// regardless of signal state, so the speculative pull in mod.rs self-heals. + #[tokio::test] + #[serial] + async fn test_pull_events_works_with_cleared_signal() { + // Seed the Application log with a record so the "events remain + // available despite cleared signal" assertion below does not depend + // on whatever backlog the runner happens to have. Freshly provisioned + // CI images can have an empty Application log, which would otherwise + // make `pull_events` legitimately return empty and produce a spurious + // failure unrelated to the invariant under test. + let seed_output = std::process::Command::new("eventcreate") + .args([ + "/T", + "INFORMATION", + "/ID", + "100", + "/L", + "APPLICATION", + "/SO", + "VectorTestSpeculativePullSeed", + "/D", + "seed event for #25194 speculative-pull regression test", + ]) + .output() + .expect("failed to spawn eventcreate — required for deterministic seeding"); + assert!( + seed_output.status.success(), + "eventcreate failed to seed Application log (exit={:?}): stdout={:?} stderr={:?}. \ + This test requires a seeded event to be deterministic; a locked-down runner \ + without the privilege to write to Application cannot run this test reliably.", + seed_output.status.code(), + String::from_utf8_lossy(&seed_output.stdout), + String::from_utf8_lossy(&seed_output.stderr), + ); + // Give the service a moment to persist the record before we subscribe. + tokio::time::sleep(std::time::Duration::from_millis(500)).await; + + let mut config = WindowsEventLogConfig::default(); + config.channels = vec!["Application".to_string()]; + config.read_existing_events = true; + config.event_timeout_ms = 500; + + let (checkpointer, _temp_dir) = create_test_checkpointer().await; + + let mut subscription = EventLogSubscription::new(&config, checkpointer, false) + .await + .expect("Subscription creation should succeed"); + + // Manually clear the signal to simulate a lost wakeup. The seeded + // event above guarantees at least one record is queued in EvtNext + // regardless of the runner's pre-existing log state. + let signal_raw = subscription.first_channel_signal_raw(); + unsafe { + let _ = ResetEvent(HANDLE(signal_raw as *mut std::ffi::c_void)); + } + + // Signal is cleared: an immediate (0ms) poll must report Timeout. + // A 0ms wait reads only the current signal state with no grace + // window, so unrelated Windows system events arriving between the + // `ResetEvent` above and the poll cannot re-signal the handle and + // cause a spurious failure. + let wait_result = subscription.wait_for_events_blocking(0); + + assert!( + matches!(wait_result, WaitResult::Timeout), + "expected Timeout after manual ResetEvent; signal was not cleared" + ); + + // Despite the cleared signal, pull_events must still return events. + // This is the invariant the speculative timeout pull in mod.rs depends on. + let events = subscription.pull_events(100).unwrap_or_default(); + assert!( + !events.is_empty(), + "pull_events must return events independently of signal state; \ + this is the invariant the speculative timeout pull in mod.rs depends on" + ); + } + + /// Regression test for vectordotdev/vector#25194. + /// + /// The Windows Event Log service signals the pull-mode wait handle via + /// `SetEvent` each time a new matching event is recorded. Because the + /// handle is manual-reset, `SetEvent` on an already-signaled handle is + /// a no-op. If `pull_events` resets the signal *after* draining events + /// via `EvtNext`, any signal that fires between the last `EvtNext` and + /// the `ResetEvent` call is silently lost — the subscription then + /// permanently hangs until a subsequent event arrives. + /// + /// The fix is to reset the signal *before* the drain loop, so signals + /// raised during the drain are preserved and the next wait returns + /// immediately. + /// + /// This test pins that invariant by driving the real `pull_events` + /// against a real `EvtSubscribe` handle. It installs a + /// `DRAIN_STEP_HOOK` that runs inside the drain loop after each + /// `EvtNext` and fires `SetEvent` on the subscription's signal + /// handle — simulating the OS signaling a new event arrival during + /// the drain window. After `pull_events` returns, the signal must + /// still be set — observed via a 0ms `wait_for_events_blocking` + /// so the check measures only the reset/preserve behavior of + /// `pull_events` and is not contaminated by unrelated Windows + /// system events arriving during a nonzero wait. Under the old + /// post-drain `ResetEvent` order, the hook's `SetEvent` would be + /// clobbered by the reset and the immediate poll would return + /// `Timeout` — which is exactly what #25194 reports. + #[tokio::test] + #[serial] + async fn test_pull_events_preserves_setevent_during_drain() { + use std::sync::Arc as StdArc; + + let mut config = WindowsEventLogConfig::default(); + config.channels = vec!["Application".to_string()]; + config.read_existing_events = true; + config.event_timeout_ms = 1000; + + let (checkpointer, _temp_dir) = create_test_checkpointer().await; + + let mut subscription = EventLogSubscription::new(&config, checkpointer, false) + .await + .expect("Subscription creation should succeed"); + + // Capture THIS subscription's signal handle so the hook can scope + // itself to this test. DRAIN_STEP_HOOK is a process-global, and + // cargo runs tests in parallel by default; without handle-keying, + // a concurrent test's pull_events could trigger our one-shot + // hook first, flip `fired`, and SetEvent on the wrong handle. + let target_signal_raw = subscription.first_channel_signal_raw(); + + // Install the drain-loop hook: every EvtNext call inside + // pull_events fires SetEvent on the subscription's signal + // handle. This simulates the OS signaling a fresh event + // mid-drain, which is exactly the race window #25194 exposes. + // The hook only needs to fire once to prove the invariant; we + // use an AtomicBool to keep it deterministic. The hook is keyed + // to `target_signal_raw` so concurrent pull_events calls from + // other tests no-op here. + let fired = StdArc::new(std::sync::atomic::AtomicBool::new(false)); + { + let fired = StdArc::clone(&fired); + let hook: StdArc = StdArc::new(move |signal: HANDLE| { + if signal.0 as isize != target_signal_raw { + return; + } + if !fired.swap(true, std::sync::atomic::Ordering::SeqCst) { + unsafe { + let _ = SetEvent(signal); + } + } + }); + *DRAIN_STEP_HOOK.lock().unwrap() = Some(hook); + } + + // Drop-guard: clear the hook even if the test panics, so it + // doesn't contaminate other tests in the same process. + struct HookGuard; + impl Drop for HookGuard { + fn drop(&mut self) { + *DRAIN_STEP_HOOK.lock().unwrap() = None; + } + } + let _guard = HookGuard; + + // Drive pull_events with a very large budget so the drain + // exits via ERROR_NO_MORE_ITEMS (channel_drained = true), + // which is the path that ran the post-drain ResetEvent in the + // old buggy code. Exiting via budget exhaustion would skip + // that reset and cause this test to false-pass against the + // pre-fix code. + let _ = subscription.pull_events(usize::MAX).unwrap_or_default(); + + assert!( + fired.load(std::sync::atomic::Ordering::SeqCst), + "drain-loop hook never ran — pull_events must call EvtNext \ + at least once even on an empty channel" + ); + + // Observe the signal state IMMEDIATELY with a 0ms wait. We want + // to know whether pull_events's reset clobbered the hook's + // SetEvent — NOT whether new real events arrive during some + // wait window. A nonzero timeout against the live Application + // channel lets arbitrary Windows system events re-signal us + // and false-pass against the pre-fix code. 0ms = WaitForMultiple- + // Objects returns the current state with no grace period, so + // only the reset/preserve behavior of pull_events is measured. + let result = subscription.wait_for_events_blocking(0); + + match result { + WaitResult::EventsAvailable => {} + WaitResult::Timeout => panic!( + "signal set during the drain window was lost — this is the \ + lost-wakeup race from vectordotdev/vector#25194. \ + pull_events must call ResetEvent BEFORE draining, not after." + ), + WaitResult::Shutdown => panic!("unexpected shutdown"), + } + } } From 59a53b138d127fdca68d260628d1dc0035b3f711 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Fri, 1 May 2026 13:43:39 -0400 Subject: [PATCH 185/326] fix(internal docs): remove type-level default on StatusCode (#25345) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fix(vector-config): remove type-level default on StatusCode The Configurable impl for http::StatusCode set a type-level default of 200, which the CUE generator propagated into the `uint` schema for each item of `Vec` — including RetryStrategy::Custom.status_codes introduced in #25057. The CUE schema for #TypeArray hardcodes items.type._args.required = true, which forbids `default` on item primitives, so cue-build failed on every http-based sink with: components.sinks..configuration.retry_strategy.type.object: field not allowed Drop the type-level default. The only field that needs a default for StatusCode is sources::http_server::response_code, which already sets its own via #[serde(default = "default_http_response_code")]. Co-authored-by: Claude Opus 4.7 (1M context) --- lib/vector-config/src/http.rs | 1 - website/cue/reference/components/sinks/generated/appsignal.cue | 2 +- website/cue/reference/components/sinks/generated/axiom.cue | 2 +- .../components/sinks/generated/azure_logs_ingestion.cue | 2 +- .../reference/components/sinks/generated/azure_monitor_logs.cue | 2 +- .../cue/reference/components/sinks/generated/datadog_events.cue | 2 +- .../components/sinks/generated/gcp_stackdriver_logs.cue | 2 +- .../components/sinks/generated/gcp_stackdriver_metrics.cue | 2 +- website/cue/reference/components/sinks/generated/honeycomb.cue | 2 +- website/cue/reference/components/sinks/generated/http.cue | 2 +- website/cue/reference/components/sinks/generated/keep.cue | 2 +- .../cue/reference/components/sinks/generated/opentelemetry.cue | 2 +- .../components/sinks/generated/prometheus_remote_write.cue | 2 +- 13 files changed, 12 insertions(+), 13 deletions(-) diff --git a/lib/vector-config/src/http.rs b/lib/vector-config/src/http.rs index da51ea1cc156e..1f61630bf7cbd 100644 --- a/lib/vector-config/src/http.rs +++ b/lib/vector-config/src/http.rs @@ -28,7 +28,6 @@ impl Configurable for StatusCode { fn metadata() -> Metadata { let mut metadata = Metadata::default(); metadata.set_description("HTTP response status code"); - metadata.set_default_value(StatusCode::OK); metadata.add_custom_attribute(CustomAttribute::kv( constants::DOCS_META_NUMERIC_TYPE, NumberClass::Unsigned, diff --git a/website/cue/reference/components/sinks/generated/appsignal.cue b/website/cue/reference/components/sinks/generated/appsignal.cue index 47943be3c1f93..41769e4ec2a41 100644 --- a/website/cue/reference/components/sinks/generated/appsignal.cue +++ b/website/cue/reference/components/sinks/generated/appsignal.cue @@ -337,7 +337,7 @@ generated: components: sinks: appsignal: configuration: { description: "Retry on these specific HTTP status codes" relevant_when: "type = \"custom\"" required: true - type: array: items: type: uint: default: 200 + type: array: items: type: uint: {} } type: { description: "The retry strategy enum." diff --git a/website/cue/reference/components/sinks/generated/axiom.cue b/website/cue/reference/components/sinks/generated/axiom.cue index a1f42404ac0c1..e08085b79926b 100644 --- a/website/cue/reference/components/sinks/generated/axiom.cue +++ b/website/cue/reference/components/sinks/generated/axiom.cue @@ -319,7 +319,7 @@ generated: components: sinks: axiom: configuration: { description: "Retry on these specific HTTP status codes" relevant_when: "type = \"custom\"" required: true - type: array: items: type: uint: default: 200 + type: array: items: type: uint: {} } type: { description: "The retry strategy enum." diff --git a/website/cue/reference/components/sinks/generated/azure_logs_ingestion.cue b/website/cue/reference/components/sinks/generated/azure_logs_ingestion.cue index 68944f5a6e7d5..9300dd205df64 100644 --- a/website/cue/reference/components/sinks/generated/azure_logs_ingestion.cue +++ b/website/cue/reference/components/sinks/generated/azure_logs_ingestion.cue @@ -422,7 +422,7 @@ generated: components: sinks: azure_logs_ingestion: configuration: { description: "Retry on these specific HTTP status codes" relevant_when: "type = \"custom\"" required: true - type: array: items: type: uint: default: 200 + type: array: items: type: uint: {} } type: { description: "The retry strategy enum." diff --git a/website/cue/reference/components/sinks/generated/azure_monitor_logs.cue b/website/cue/reference/components/sinks/generated/azure_monitor_logs.cue index c7039b70097e1..205b671072bdb 100644 --- a/website/cue/reference/components/sinks/generated/azure_monitor_logs.cue +++ b/website/cue/reference/components/sinks/generated/azure_monitor_logs.cue @@ -328,7 +328,7 @@ generated: components: sinks: azure_monitor_logs: configuration: { description: "Retry on these specific HTTP status codes" relevant_when: "type = \"custom\"" required: true - type: array: items: type: uint: default: 200 + type: array: items: type: uint: {} } type: { description: "The retry strategy enum." diff --git a/website/cue/reference/components/sinks/generated/datadog_events.cue b/website/cue/reference/components/sinks/generated/datadog_events.cue index 23b9fb43d6413..f823c05445a2e 100644 --- a/website/cue/reference/components/sinks/generated/datadog_events.cue +++ b/website/cue/reference/components/sinks/generated/datadog_events.cue @@ -256,7 +256,7 @@ generated: components: sinks: datadog_events: configuration: { description: "Retry on these specific HTTP status codes" relevant_when: "type = \"custom\"" required: true - type: array: items: type: uint: default: 200 + type: array: items: type: uint: {} } type: { description: "The retry strategy enum." diff --git a/website/cue/reference/components/sinks/generated/gcp_stackdriver_logs.cue b/website/cue/reference/components/sinks/generated/gcp_stackdriver_logs.cue index 9d5e76262bfd2..2685eda9ffb10 100644 --- a/website/cue/reference/components/sinks/generated/gcp_stackdriver_logs.cue +++ b/website/cue/reference/components/sinks/generated/gcp_stackdriver_logs.cue @@ -434,7 +434,7 @@ generated: components: sinks: gcp_stackdriver_logs: configuration: { description: "Retry on these specific HTTP status codes" relevant_when: "type = \"custom\"" required: true - type: array: items: type: uint: default: 200 + type: array: items: type: uint: {} } type: { description: "The retry strategy enum." diff --git a/website/cue/reference/components/sinks/generated/gcp_stackdriver_metrics.cue b/website/cue/reference/components/sinks/generated/gcp_stackdriver_metrics.cue index 52c344e278250..d9897bbaf092e 100644 --- a/website/cue/reference/components/sinks/generated/gcp_stackdriver_metrics.cue +++ b/website/cue/reference/components/sinks/generated/gcp_stackdriver_metrics.cue @@ -348,7 +348,7 @@ generated: components: sinks: gcp_stackdriver_metrics: configuration: { description: "Retry on these specific HTTP status codes" relevant_when: "type = \"custom\"" required: true - type: array: items: type: uint: default: 200 + type: array: items: type: uint: {} } type: { description: "The retry strategy enum." diff --git a/website/cue/reference/components/sinks/generated/honeycomb.cue b/website/cue/reference/components/sinks/generated/honeycomb.cue index c65064a135a00..5c3acaed97ffa 100644 --- a/website/cue/reference/components/sinks/generated/honeycomb.cue +++ b/website/cue/reference/components/sinks/generated/honeycomb.cue @@ -335,7 +335,7 @@ generated: components: sinks: honeycomb: configuration: { description: "Retry on these specific HTTP status codes" relevant_when: "type = \"custom\"" required: true - type: array: items: type: uint: default: 200 + type: array: items: type: uint: {} } type: { description: "The retry strategy enum." diff --git a/website/cue/reference/components/sinks/generated/http.cue b/website/cue/reference/components/sinks/generated/http.cue index 8523a27dd0d9b..e08fd535971e6 100644 --- a/website/cue/reference/components/sinks/generated/http.cue +++ b/website/cue/reference/components/sinks/generated/http.cue @@ -1038,7 +1038,7 @@ generated: components: sinks: http: configuration: { description: "Retry on these specific HTTP status codes" relevant_when: "type = \"custom\"" required: true - type: array: items: type: uint: default: 200 + type: array: items: type: uint: {} } type: { description: "The retry strategy enum." diff --git a/website/cue/reference/components/sinks/generated/keep.cue b/website/cue/reference/components/sinks/generated/keep.cue index d2ed08114a2a9..43b8500746291 100644 --- a/website/cue/reference/components/sinks/generated/keep.cue +++ b/website/cue/reference/components/sinks/generated/keep.cue @@ -300,7 +300,7 @@ generated: components: sinks: keep: configuration: { description: "Retry on these specific HTTP status codes" relevant_when: "type = \"custom\"" required: true - type: array: items: type: uint: default: 200 + type: array: items: type: uint: {} } type: { description: "The retry strategy enum." diff --git a/website/cue/reference/components/sinks/generated/opentelemetry.cue b/website/cue/reference/components/sinks/generated/opentelemetry.cue index 4e8e9d9ff7210..3d52d15690bed 100644 --- a/website/cue/reference/components/sinks/generated/opentelemetry.cue +++ b/website/cue/reference/components/sinks/generated/opentelemetry.cue @@ -1039,7 +1039,7 @@ generated: components: sinks: opentelemetry: configuration: protocol: { description: "Retry on these specific HTTP status codes" relevant_when: "type = \"custom\"" required: true - type: array: items: type: uint: default: 200 + type: array: items: type: uint: {} } type: { description: "The retry strategy enum." diff --git a/website/cue/reference/components/sinks/generated/prometheus_remote_write.cue b/website/cue/reference/components/sinks/generated/prometheus_remote_write.cue index 5d74d52359bdd..0156777c09fb2 100644 --- a/website/cue/reference/components/sinks/generated/prometheus_remote_write.cue +++ b/website/cue/reference/components/sinks/generated/prometheus_remote_write.cue @@ -553,7 +553,7 @@ generated: components: sinks: prometheus_remote_write: configuration: { description: "Retry on these specific HTTP status codes" relevant_when: "type = \"custom\"" required: true - type: array: items: type: uint: default: 200 + type: array: items: type: uint: {} } type: { description: "The retry strategy enum." From f1b2c3a3f14f59ab9615829852da6e65a7d2c512 Mon Sep 17 00:00:00 2001 From: tot19 <31141271+tot19@users.noreply.github.com> Date: Sat, 2 May 2026 02:06:42 +0900 Subject: [PATCH 186/326] fix(sources): add windows_event_log source metadata (#25337) * fix(windows_event_log): add standard source metadata * chore(changelog): add windows_event_log metadata fix note --- ...332_windows_event_log_source_metadata.fix.md | 3 +++ src/sources/windows_event_log/mod.rs | 17 +++++++++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 changelog.d/25332_windows_event_log_source_metadata.fix.md diff --git a/changelog.d/25332_windows_event_log_source_metadata.fix.md b/changelog.d/25332_windows_event_log_source_metadata.fix.md new file mode 100644 index 0000000000000..936e773cc8588 --- /dev/null +++ b/changelog.d/25332_windows_event_log_source_metadata.fix.md @@ -0,0 +1,3 @@ +The `windows_event_log` source now adds standard source metadata, including `source_type`, to emitted log events. + +authors: tot19 diff --git a/src/sources/windows_event_log/mod.rs b/src/sources/windows_event_log/mod.rs index 28dfc1db4cb79..9fb2cff142ee6 100644 --- a/src/sources/windows_event_log/mod.rs +++ b/src/sources/windows_event_log/mod.rs @@ -25,6 +25,7 @@ cfg_if::cfg_if! { use std::path::PathBuf; use std::sync::Arc; + use chrono::Utc; use futures::StreamExt; use vector_lib::EstimatedJsonEncodedSizeOf; use vector_lib::finalizer::OrderedFinalizer; @@ -166,6 +167,7 @@ impl Finalizer { async fn process_event_batch( events: Vec, parser: &EventLogParser, + log_namespace: LogNamespace, acknowledgements: bool, subscription: &EventLogSubscription, out: &mut SourceSender, @@ -189,6 +191,12 @@ async fn process_event_batch( let event_id = event.event_id; match parser.parse_event(event) { Ok(mut log_event) => { + log_namespace.insert_standard_vector_source_metadata( + &mut log_event, + WindowsEventLogConfig::NAME, + Utc::now(), + ); + let byte_size = log_event.estimated_json_encoded_size_of(); total_byte_size += byte_size.get(); if let Some(ref batch) = batch { @@ -416,6 +424,7 @@ impl WindowsEventLogSource { if process_event_batch( events, &parser, + self.log_namespace, acknowledgements, &subscription, &mut out, @@ -516,6 +525,7 @@ impl WindowsEventLogSource { if process_event_batch( events, &parser, + self.log_namespace, acknowledgements, &subscription, &mut out, @@ -664,8 +674,11 @@ impl SourceConfig for WindowsEventLogConfig { ), ])), [LogNamespace::Vector], - ), - LogNamespace::Legacy => vector_lib::schema::Definition::any(), + ) + .with_standard_vector_source_metadata(), + LogNamespace::Legacy => { + vector_lib::schema::Definition::any().with_standard_vector_source_metadata() + } }; vec![SourceOutput::new_maybe_logs( From 9f15e23943d4347a6f2171eaa97a921a5e58d457 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Fri, 1 May 2026 13:56:58 -0400 Subject: [PATCH 187/326] fix(ci): bump cue and add cue-build step to Check Cue docs (#25346) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(vector-config): remove type-level default on StatusCode The Configurable impl for http::StatusCode set a type-level default of 200, which the CUE generator propagated into the `uint` schema for each item of `Vec` — including RetryStrategy::Custom.status_codes introduced in #25057. The CUE schema for #TypeArray hardcodes items.type._args.required = true, which forbids `default` on item primitives, so cue-build failed on every http-based sink with: components.sinks..configuration.retry_strategy.type.object: field not allowed Drop the type-level default. The only field that needs a default for StatusCode is sources::http_server::response_code, which already sets its own via #[serde(default = "default_http_response_code")]. Co-Authored-By: Claude Opus 4.7 (1M context) * chore(ci): bump cue from v0.10.0 to v0.16.1 CI's cue v0.10.0 was lax enough that schema violations like the "field not allowed" error fixed in the previous commit slipped through the existing `Check Cue docs` step. Newer cue versions enforce closed struct constraints in disjunctions strictly, matching what local developers see when running the website build. Co-Authored-By: Claude Opus 4.7 (1M context) * ci(test): also run make cue-build in Check Cue docs The Check Cue docs job previously only ran make check-docs, which runs cue vet. Adding make cue-build (which runs cue export and materializes website/data/docs.json) catches schema-unification failures that the website build would hit and that vet may miss. The new step reuses the cue + VRL docs already produced by the prior make check-docs invocation, so it adds only the cost of one cue export. Co-Authored-By: Claude Opus 4.7 (1M context) * chore(ci): bump cue to v0.16.1 in remaining installers Keep all three cue installers in lockstep with .github/actions/setup to avoid version skew that would let developer / website-Dockerfile paths accept cue sources the CI Check Cue docs job rejects. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) --- .github/actions/setup/action.yml | 6 +++--- .github/workflows/test.yml | 2 ++ scripts/environment/bootstrap-ubuntu-24.04.sh | 6 +++--- website/Dockerfile | 2 +- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 6faa185204ced..3c7e645e6831b 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -237,10 +237,10 @@ runs: echo "Installing cue" TEMP=$(mktemp -d) curl \ - -L https://github.com/cue-lang/cue/releases/download/v0.10.0/cue_v0.10.0_linux_amd64.tar.gz \ - -o "${TEMP}/cue_v0.10.0_linux_amd64.tar.gz" + -L https://github.com/cue-lang/cue/releases/download/v0.16.1/cue_v0.16.1_linux_amd64.tar.gz \ + -o "${TEMP}/cue_v0.16.1_linux_amd64.tar.gz" tar \ - -xvf "${TEMP}/cue_v0.10.0_linux_amd64.tar.gz" \ + -xvf "${TEMP}/cue_v0.16.1_linux_amd64.tar.gz" \ -C "${TEMP}" sudo cp "${TEMP}/cue" /usr/bin/cue rm -rf "$TEMP" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7e5c28aec476c..5244d8110d823 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -121,6 +121,8 @@ jobs: rust: true cue: true - run: make check-docs + - name: Check Cue website build + run: cd website && make cue-build check-markdown: name: Check Markdown diff --git a/scripts/environment/bootstrap-ubuntu-24.04.sh b/scripts/environment/bootstrap-ubuntu-24.04.sh index 9d135f779b933..4298fb4dba94b 100755 --- a/scripts/environment/bootstrap-ubuntu-24.04.sh +++ b/scripts/environment/bootstrap-ubuntu-24.04.sh @@ -56,10 +56,10 @@ apt-get install --yes \ # Cue TEMP=$(mktemp -d) curl \ - -L https://github.com/cue-lang/cue/releases/download/v0.10.0/cue_v0.10.0_linux_amd64.tar.gz \ - -o "${TEMP}/cue_v0.10.0_linux_amd64.tar.gz" + -L https://github.com/cue-lang/cue/releases/download/v0.16.1/cue_v0.16.1_linux_amd64.tar.gz \ + -o "${TEMP}/cue_v0.16.1_linux_amd64.tar.gz" tar \ - -xvf "${TEMP}/cue_v0.10.0_linux_amd64.tar.gz" \ + -xvf "${TEMP}/cue_v0.16.1_linux_amd64.tar.gz" \ -C "${TEMP}" cp "${TEMP}/cue" /usr/bin/cue rm -rf "$TEMP" diff --git a/website/Dockerfile b/website/Dockerfile index e9946941bb459..d98c0f2103ab1 100644 --- a/website/Dockerfile +++ b/website/Dockerfile @@ -1,7 +1,7 @@ FROM debian:bookworm-slim ARG HUGO_VERSION=0.154.5 -ARG CUE_VERSION=0.10.0 +ARG CUE_VERSION=0.16.1 ARG NODE_VERSION=24 RUN apt-get update && apt-get install -y --no-install-recommends \ From 7923556313d66be69e638022e10fe3fd13f468ac Mon Sep 17 00:00:00 2001 From: Flavio Cruz <87035001+flaviofcruz@users.noreply.github.com> Date: Fri, 1 May 2026 14:57:31 -0400 Subject: [PATCH 188/326] fix(clickhouse sink, aws_s3 sink): use dedicated batch_encoding types (#25340) * Fix the clickhouse sink so that it only accepts arrow * Do not allow arrow stream or parquet on AWS S3 * reformat * Add changelog fragment * Remove stale comments * chore(aws_s3 sink): exhaustive match for batch_encoding default extension So adding a future S3BatchEncoding variant is a compile error rather than silently defaulting to "parquet". Co-Authored-By: Claude Opus 4.7 (1M context) * chore(aws_s3 sink): rename shadowing parquet_config bind in tests Avoid rebinding parquet_config to a borrow of itself from config.batch_encoding; use a short p binding instead. Co-Authored-By: Claude Opus 4.7 (1M context) * docs(changelog): rewrite batch_encoding fragment for users Drop internal type names and dev jargon; lead with the user-visible behavior change. Co-Authored-By: Claude Opus 4.7 (1M context) * test(aws_s3 sink, clickhouse sink): reject unsupported batch_encoding codec at parse time Add per-sink deserialization-failure tests that pin the schema-tightening behavior introduced by the dedicated wrapper enums: - aws_s3 rejects codec: arrow_stream - clickhouse rejects codec: parquet Previously both codecs were accepted by serde and rejected later at sink-build time; the new wrapper enums move rejection up to parse time, and these tests prevent silent regression of that contract. Co-Authored-By: Claude Opus 4.7 (1M context) * chore(aws_s3 sink): re-export S3BatchEncoding from aws_s3 module Programmatic users constructing S3SinkConfig directly need to be able to set the batch_encoding field. With config kept private in src/sinks/aws_s3/mod.rs, S3BatchEncoding was unnameable outside the crate, regressing callers that previously used BatchSerializerConfig::Parquet(...) at the same field. Gated by codecs-parquet to mirror the type and field. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Pavlos Rontidis Co-authored-by: Claude Opus 4.7 (1M context) --- ...use_aws_s3_dedicated_batch_encoding.fix.md | 3 + src/sinks/aws_s3/config.rs | 139 ++++++++---------- src/sinks/aws_s3/integration_tests.rs | 6 +- src/sinks/aws_s3/mod.rs | 2 + src/sinks/clickhouse/config.rs | 61 ++++++-- src/sinks/clickhouse/integration_tests.rs | 14 +- .../components/sinks/generated/aws_s3.cue | 54 ++----- .../components/sinks/generated/clickhouse.cue | 80 ++-------- 8 files changed, 147 insertions(+), 212 deletions(-) create mode 100644 changelog.d/clickhouse_aws_s3_dedicated_batch_encoding.fix.md diff --git a/changelog.d/clickhouse_aws_s3_dedicated_batch_encoding.fix.md b/changelog.d/clickhouse_aws_s3_dedicated_batch_encoding.fix.md new file mode 100644 index 0000000000000..f39347d4c47bd --- /dev/null +++ b/changelog.d/clickhouse_aws_s3_dedicated_batch_encoding.fix.md @@ -0,0 +1,3 @@ +The `aws_s3` and `clickhouse` sinks now correctly advertise only the `batch_encoding.codec` values they actually support: `parquet` for `aws_s3` and `arrow_stream` for `clickhouse`. Previously the documentation and configuration schema listed both codecs for both sinks, even though picking the wrong one produced a startup error. + +authors: flaviofcruz diff --git a/src/sinks/aws_s3/config.rs b/src/sinks/aws_s3/config.rs index 6e8d2e248c518..05055e6ea0167 100644 --- a/src/sinks/aws_s3/config.rs +++ b/src/sinks/aws_s3/config.rs @@ -3,7 +3,7 @@ use tower::ServiceBuilder; #[cfg(feature = "codecs-parquet")] use vector_lib::codecs::BatchEncoder; #[cfg(feature = "codecs-parquet")] -use vector_lib::codecs::encoding::BatchSerializerConfig; +use vector_lib::codecs::encoding::{BatchSerializerConfig, format::ParquetSerializerConfig}; use vector_lib::{ TimeZone, codecs::{ @@ -37,6 +37,21 @@ use crate::{ tls::TlsConfig, }; +/// Batch encoding configuration for the `aws_s3` sink. +#[cfg(feature = "codecs-parquet")] +#[configurable_component] +#[derive(Clone, Debug)] +#[serde(tag = "codec", rename_all = "snake_case")] +#[configurable(metadata( + docs::enum_tag_description = "The codec to use for batch encoding events." +))] +pub enum S3BatchEncoding { + /// Encodes events in [Apache Parquet][apache_parquet] columnar format. + /// + /// [apache_parquet]: https://parquet.apache.org/ + Parquet(ParquetSerializerConfig), +} + /// Configuration for the `aws_s3` sink. #[configurable_component(sink( "aws_s3", @@ -111,15 +126,13 @@ pub struct S3SinkConfig { /// Batch encoding configuration for columnar formats. /// - /// When set, events are encoded together as a batch in a columnar format (for example, Parquet) + /// When set, events are encoded together as a batch in a columnar format (Parquet) /// instead of the standard per-event framing-based encoding. The columnar format handles /// its own internal compression, so the top-level `compression` setting is bypassed. - /// - /// Only the `parquet` codec is supported by the AWS S3 sink. #[cfg(feature = "codecs-parquet")] #[configurable(derived)] #[serde(default)] - pub batch_encoding: Option, + pub batch_encoding: Option, /// Compression configuration. /// @@ -220,8 +233,10 @@ impl SinkConfig for S3SinkConfig { fn input(&self) -> Input { #[cfg(feature = "codecs-parquet")] - if let Some(batch_config) = &self.batch_encoding { - return Input::new(batch_config.input_type()); + if let Some(batch_encoding) = &self.batch_encoding { + let S3BatchEncoding::Parquet(parquet_config) = batch_encoding; + let resolved = BatchSerializerConfig::Parquet(parquet_config.clone()); + return Input::new(resolved.input_type()); } Input::new(self.encoding.config().1.input_type()) } @@ -272,15 +287,11 @@ impl S3SinkConfig { // When batch_encoding is configured (e.g., Parquet), use batch mode // with internal compression and appropriate file extension. #[cfg(feature = "codecs-parquet")] - if let Some(batch_config) = &self.batch_encoding { - if !matches!(batch_config, BatchSerializerConfig::Parquet(_)) { - return Err( - "batch_encoding only supports encoding with parquet format for amazon s3 sink" - .into(), - ); - } + if let Some(batch_encoding) = &self.batch_encoding { + let S3BatchEncoding::Parquet(parquet_config) = batch_encoding; + let resolved_batch_config = BatchSerializerConfig::Parquet(parquet_config.clone()); - let batch_serializer = batch_config.build_batch_serializer()?; + let batch_serializer = resolved_batch_config.build_batch_serializer()?; let batch_encoder = BatchEncoder::new(batch_serializer); // Auto-detect Content-Type from batch format. Users can still @@ -292,15 +303,14 @@ impl S3SinkConfig { let encoder = EncoderKind::Batch(batch_encoder); - // Auto-detect file extension from batch format - let filename_extension = - self.filename_extension - .clone() - .or_else(|| match batch_config { - BatchSerializerConfig::Parquet(_) => Some("parquet".to_string()), - #[allow(unreachable_patterns)] - _ => None, - }); + let filename_extension = self.filename_extension.clone().or_else(|| { + Some( + match batch_encoding { + S3BatchEncoding::Parquet(_) => "parquet", + } + .to_string(), + ) + }); if self.compression != Compression::None { warn!("Top level compression setting ignored when batch_encoding set to parquet.") @@ -392,15 +402,10 @@ mod tests { let batch_enc = config .batch_encoding .expect("batch_encoding should be Some"); - match batch_enc { - vector_lib::codecs::encoding::BatchSerializerConfig::Parquet(ref p) => { - use vector_lib::codecs::encoding::format::{ParquetCompression, ParquetSchemaMode}; - assert_eq!(p.schema_mode, ParquetSchemaMode::AutoInfer); - assert_eq!(p.compression, ParquetCompression::Snappy); - } - #[allow(unreachable_patterns)] - _ => panic!("expected Parquet variant"), - } + let super::S3BatchEncoding::Parquet(ref p) = batch_enc; + use vector_lib::codecs::encoding::format::{ParquetCompression, ParquetSchemaMode}; + assert_eq!(p.schema_mode, ParquetSchemaMode::AutoInfer); + assert_eq!(p.compression, ParquetCompression::Snappy); } /// Content-Type must be auto-detected as `application/vnd.apache.parquet` @@ -432,7 +437,7 @@ mod tests { options: S3Options::default(), region: crate::aws::RegionOrEndpoint::with_both("us-east-1", "http://localhost:4566"), encoding: (None::, TextSerializerConfig::default()).into(), - batch_encoding: Some(BatchSerializerConfig::Parquet(parquet_config)), + batch_encoding: Some(super::S3BatchEncoding::Parquet(parquet_config)), compression: Compression::None, batch: BatchConfig::::default(), request: Default::default(), @@ -444,7 +449,8 @@ mod tests { retry_strategy: Default::default(), }; - let batch_config = config.batch_encoding.as_ref().unwrap(); + let super::S3BatchEncoding::Parquet(p) = config.batch_encoding.as_ref().unwrap(); + let batch_config = BatchSerializerConfig::Parquet(p.clone()); let batch_serializer = batch_config.build_batch_serializer().unwrap(); let batch_encoder = vector_lib::codecs::BatchEncoder::new(batch_serializer); @@ -484,7 +490,8 @@ mod tests { ) .unwrap(); - let batch_config = config.batch_encoding.as_ref().unwrap(); + let super::S3BatchEncoding::Parquet(p) = config.batch_encoding.as_ref().unwrap(); + let batch_config = vector_lib::codecs::encoding::BatchSerializerConfig::Parquet(p.clone()); let batch_serializer = batch_config.build_batch_serializer().unwrap(); let batch_encoder = vector_lib::codecs::BatchEncoder::new(batch_serializer); @@ -500,43 +507,27 @@ mod tests { ); } - /// Parquet filename extension defaults to `.parquet` when not explicitly set. + /// Codecs other than `parquet` must be rejected at parse time, since + /// `S3BatchEncoding` only exposes the `parquet` variant. #[cfg(feature = "codecs-parquet")] #[test] - fn parquet_filename_extension_defaults_to_parquet() { - let config: S3SinkConfig = toml::from_str( + fn parquet_batch_encoding_rejects_unsupported_codec() { + let err = serde_yaml::from_str::( r#" - bucket = "test-bucket" - compression = "none" - - [encoding] - codec = "text" - - [batch_encoding] - codec = "parquet" - schema_mode = "auto_infer" + bucket: test-bucket + compression: none + encoding: + codec: text + batch_encoding: + codec: arrow_stream "#, ) - .unwrap(); + .unwrap_err(); assert!( - config.filename_extension.is_none(), - "fixture must not set filename_extension" + err.to_string().contains("arrow_stream"), + "expected error to mention the offending codec, got: {err}" ); - - let batch_config = config.batch_encoding.as_ref().unwrap(); - let extension = config - .filename_extension - .clone() - .or_else(|| match batch_config { - vector_lib::codecs::encoding::BatchSerializerConfig::Parquet(_) => { - Some("parquet".to_string()) - } - #[allow(unreachable_patterns)] - _ => None, - }); - - assert_eq!(extension.as_deref(), Some("parquet")); } /// Explicit filename_extension overrides the `.parquet` default. @@ -582,13 +573,8 @@ mod tests { ) .unwrap(); - match config.batch_encoding.unwrap() { - vector_lib::codecs::encoding::BatchSerializerConfig::Parquet(p) => { - assert_eq!(p.schema_mode, ParquetSchemaMode::Relaxed); - } - #[allow(unreachable_patterns)] - _ => panic!("expected Parquet variant"), - } + let super::S3BatchEncoding::Parquet(p) = config.batch_encoding.unwrap(); + assert_eq!(p.schema_mode, ParquetSchemaMode::Relaxed); } /// Explicit `schema_mode = "strict"` is correctly parsed. @@ -613,12 +599,7 @@ mod tests { ) .unwrap(); - match config.batch_encoding.unwrap() { - vector_lib::codecs::encoding::BatchSerializerConfig::Parquet(p) => { - assert_eq!(p.schema_mode, ParquetSchemaMode::Strict); - } - #[allow(unreachable_patterns)] - _ => panic!("expected Parquet variant"), - } + let super::S3BatchEncoding::Parquet(p) = config.batch_encoding.unwrap(); + assert_eq!(p.schema_mode, ParquetSchemaMode::Strict); } } diff --git a/src/sinks/aws_s3/integration_tests.rs b/src/sinks/aws_s3/integration_tests.rs index a1a2c10af2201..3a7e156208387 100644 --- a/src/sinks/aws_s3/integration_tests.rs +++ b/src/sinks/aws_s3/integration_tests.rs @@ -7,6 +7,8 @@ use std::{ time::Duration, }; +#[cfg(feature = "codecs-parquet")] +use super::config::S3BatchEncoding; use aws_sdk_s3::{ Client as S3Client, operation::{create_bucket::CreateBucketError, get_object::GetObjectOutput}, @@ -22,8 +24,6 @@ use futures::{Stream, stream}; use similar_asserts::assert_eq; use tempfile::TempDir; use tokio_stream::StreamExt; -#[cfg(feature = "codecs-parquet")] -use vector_lib::codecs::encoding::BatchSerializerConfig; use vector_lib::{ buffers::{BufferConfig, BufferType, WhenFull}, codecs::{TextSerializerConfig, encoding::FramingConfig}, @@ -502,7 +502,7 @@ async fn s3_parquet_insert_message() { }; let config = S3SinkConfig { - batch_encoding: Some(BatchSerializerConfig::Parquet(parquet_config)), + batch_encoding: Some(S3BatchEncoding::Parquet(parquet_config)), ..config(&bucket, 100, 5.0) }; diff --git a/src/sinks/aws_s3/mod.rs b/src/sinks/aws_s3/mod.rs index 3027de1bb6287..4589746175558 100644 --- a/src/sinks/aws_s3/mod.rs +++ b/src/sinks/aws_s3/mod.rs @@ -3,4 +3,6 @@ mod sink; mod integration_tests; +#[cfg(feature = "codecs-parquet")] +pub use config::S3BatchEncoding; pub use config::S3SinkConfig; diff --git a/src/sinks/clickhouse/config.rs b/src/sinks/clickhouse/config.rs index 435b2504e6a9e..53050dca831ea 100644 --- a/src/sinks/clickhouse/config.rs +++ b/src/sinks/clickhouse/config.rs @@ -4,8 +4,8 @@ use std::fmt; use http::{Request, StatusCode, Uri}; use hyper::Body; +use vector_lib::codecs::encoding::ArrowStreamSerializerConfig; use vector_lib::codecs::encoding::format::SchemaProvider; -use vector_lib::codecs::encoding::{ArrowStreamSerializerConfig, BatchSerializerConfig}; use super::{ request_builder::ClickhouseRequestBuilder, @@ -45,6 +45,23 @@ pub enum Format { ArrowStream, } +/// Batch encoding configuration for the `clickhouse` sink. +#[configurable_component] +#[derive(Clone, Debug)] +#[serde(tag = "codec", rename_all = "snake_case")] +#[configurable(metadata( + docs::enum_tag_description = "The codec to use for batch encoding events." +))] +pub enum ClickhouseBatchEncoding { + /// Encodes events in [Apache Arrow][apache_arrow] IPC streaming format. + /// + /// This is the streaming variant of the Arrow IPC format, which writes + /// a continuous stream of record batches. + /// + /// [apache_arrow]: https://arrow.apache.org/ + ArrowStream(ArrowStreamSerializerConfig), +} + impl fmt::Display for Format { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { @@ -104,11 +121,9 @@ pub struct ClickhouseConfig { /// /// When specified, events are encoded together as a single batch. /// This is mutually exclusive with per-event encoding based on the `format` field. - /// - /// Only the `arrow_stream` codec is supported by the ClickHouse sink. #[configurable(derived)] #[serde(default)] - pub batch_encoding: Option, + pub batch_encoding: Option, #[configurable(derived)] #[serde(default)] @@ -282,6 +297,7 @@ impl ClickhouseConfig { if let Some(batch_encoding) = &self.batch_encoding { use vector_lib::codecs::BatchEncoder; + use vector_lib::codecs::encoding::BatchSerializerConfig; // Validate that batch_encoding is only compatible with ArrowStream format if self.format != Format::ArrowStream { @@ -292,16 +308,8 @@ impl ClickhouseConfig { .into()); } - let mut arrow_config = match batch_encoding { - BatchSerializerConfig::ArrowStream(config) => config.clone(), - #[cfg(feature = "codecs-parquet")] - BatchSerializerConfig::Parquet(_) => { - return Err( - "ClickHouse sink does not support Parquet batch encoding. Use 'arrow_stream' instead." - .into(), - ); - } - }; + let ClickhouseBatchEncoding::ArrowStream(arrow_config) = batch_encoding; + let mut arrow_config = arrow_config.clone(); self.resolve_arrow_schema( client, @@ -433,10 +441,31 @@ mod tests { ); } + /// Codecs other than `arrow_stream` must be rejected at parse time, since + /// `ClickhouseBatchEncoding` only exposes the `arrow_stream` variant. + #[cfg(feature = "codecs-parquet")] + #[test] + fn batch_encoding_rejects_unsupported_codec() { + let err = serde_yaml::from_str::( + r#" + endpoint: http://localhost:8123 + table: test_table + batch_encoding: + codec: parquet + "#, + ) + .unwrap_err(); + + assert!( + err.to_string().contains("parquet"), + "expected error to mention the offending codec, got: {err}" + ); + } + /// Helper to create a minimal ClickhouseConfig for testing fn create_test_config( format: Format, - batch_encoding: Option, + batch_encoding: Option, ) -> ClickhouseConfig { ClickhouseConfig { endpoint: "http://localhost:8123".parse::().unwrap().into(), @@ -469,7 +498,7 @@ mod tests { for (format, format_name) in incompatible_formats { let config = create_test_config( format, - Some(BatchSerializerConfig::ArrowStream( + Some(ClickhouseBatchEncoding::ArrowStream( ArrowStreamSerializerConfig::default(), )), ); diff --git a/src/sinks/clickhouse/integration_tests.rs b/src/sinks/clickhouse/integration_tests.rs index a32771bf55730..2e27a20340dfa 100644 --- a/src/sinks/clickhouse/integration_tests.rs +++ b/src/sinks/clickhouse/integration_tests.rs @@ -18,7 +18,7 @@ use serde::Deserialize; use serde_json::Value; use tokio::time::{Duration, timeout}; use vector_lib::{ - codecs::encoding::{ArrowStreamSerializerConfig, BatchSerializerConfig}, + codecs::encoding::ArrowStreamSerializerConfig, event::{BatchNotifier, BatchStatus, BatchStatusReceiver, Event, LogEvent}, lookup::PathPrefix, }; @@ -28,7 +28,7 @@ use crate::{ codecs::{TimestampFormat, Transformer}, config::{SinkConfig, SinkContext, log_schema}, sinks::{ - clickhouse::config::ClickhouseConfig, + clickhouse::config::{ClickhouseBatchEncoding, ClickhouseConfig}, util::{BatchConfig, Compression, TowerRequestConfig}, }, test_util::{ @@ -502,7 +502,7 @@ async fn insert_events_arrow_format() { table: table.clone().try_into().unwrap(), compression: Compression::None, format: crate::sinks::clickhouse::config::Format::ArrowStream, - batch_encoding: Some(BatchSerializerConfig::ArrowStream(Default::default())), + batch_encoding: Some(ClickhouseBatchEncoding::ArrowStream(Default::default())), batch, request: TowerRequestConfig { retry_attempts: 1, @@ -574,7 +574,7 @@ async fn insert_events_arrow_with_schema_fetching() { table: table.clone().try_into().unwrap(), compression: Compression::None, format: crate::sinks::clickhouse::config::Format::ArrowStream, - batch_encoding: Some(BatchSerializerConfig::ArrowStream(Default::default())), + batch_encoding: Some(ClickhouseBatchEncoding::ArrowStream(Default::default())), batch, request: TowerRequestConfig { retry_attempts: 1, @@ -657,7 +657,7 @@ async fn test_complex_types() { table: table.clone().try_into().unwrap(), compression: Compression::None, format: crate::sinks::clickhouse::config::Format::ArrowStream, - batch_encoding: Some(BatchSerializerConfig::ArrowStream(arrow_config)), + batch_encoding: Some(ClickhouseBatchEncoding::ArrowStream(arrow_config)), batch, request: TowerRequestConfig { retry_attempts: 1, @@ -1231,7 +1231,7 @@ async fn test_missing_required_field_emits_null_constraint_error() { table: table.clone().try_into().unwrap(), compression: Compression::None, format: crate::sinks::clickhouse::config::Format::ArrowStream, - batch_encoding: Some(BatchSerializerConfig::ArrowStream(Default::default())), + batch_encoding: Some(ClickhouseBatchEncoding::ArrowStream(Default::default())), batch, request: TowerRequestConfig { retry_attempts: 1, @@ -1323,7 +1323,7 @@ async fn arrow_schema_excludes_non_insertable_columns() { table: table.clone().try_into().unwrap(), compression: Compression::None, format: crate::sinks::clickhouse::config::Format::ArrowStream, - batch_encoding: Some(BatchSerializerConfig::ArrowStream( + batch_encoding: Some(ClickhouseBatchEncoding::ArrowStream( ArrowStreamSerializerConfig::default(), )), batch, diff --git a/website/cue/reference/components/sinks/generated/aws_s3.cue b/website/cue/reference/components/sinks/generated/aws_s3.cue index 1a4713e2897dd..5627d4636698a 100644 --- a/website/cue/reference/components/sinks/generated/aws_s3.cue +++ b/website/cue/reference/components/sinks/generated/aws_s3.cue @@ -260,52 +260,28 @@ generated: components: sinks: aws_s3: configuration: { description: """ Batch encoding configuration for columnar formats. - When set, events are encoded together as a batch in a columnar format (for example, Parquet) + When set, events are encoded together as a batch in a columnar format (Parquet) instead of the standard per-event framing-based encoding. The columnar format handles its own internal compression, so the top-level `compression` setting is bypassed. - - Only the `parquet` codec is supported by the AWS S3 sink. """ required: false type: object: options: { - allow_nullable_fields: { + codec: { description: """ - Allow null values for non-nullable fields in the schema. - - When enabled, missing or incompatible values are encoded as null, even for fields - marked as non-nullable in the Arrow schema. This is useful when working with downstream - systems that can handle null values through defaults, computed columns, or other mechanisms. + Encodes events in [Apache Parquet][apache_parquet] columnar format. - When disabled (default), missing values for non-nullable fields results in encoding errors. This is to - help ensure all required data is present before sending it to the sink. + [apache_parquet]: https://parquet.apache.org/ """ - relevant_when: "codec = \"arrow_stream\"" - required: false - type: bool: default: false - } - codec: { - description: "The codec to use for batch encoding events." - required: true - type: string: enum: { - arrow_stream: """ - Encodes events in [Apache Arrow][apache_arrow] IPC streaming format. - - This is the streaming variant of the Arrow IPC format, which writes - a continuous stream of record batches. - - [apache_arrow]: https://arrow.apache.org/ - """ - parquet: """ - Encodes events in [Apache Parquet][apache_parquet] columnar format. + required: true + type: string: enum: parquet: """ + Encodes events in [Apache Parquet][apache_parquet] columnar format. - [apache_parquet]: https://parquet.apache.org/ - """ - } + [apache_parquet]: https://parquet.apache.org/ + """ } compression: { - description: "Compression codec applied per column page inside the Parquet file." - relevant_when: "codec = \"parquet\"" - required: false + description: "Compression codec applied per column page inside the Parquet file." + required: false type: object: options: { algorithm: { description: "Compression codec applied per column page inside the Parquet file." @@ -336,14 +312,12 @@ generated: components: sinks: aws_s3: configuration: { Required unless `schema_mode` is `auto_infer`. The file must contain a valid Parquet message type definition. """ - relevant_when: "codec = \"parquet\"" - required: false + required: false type: string: {} } schema_mode: { - description: "Controls how events with fields not present in the schema are handled." - relevant_when: "codec = \"parquet\"" - required: false + description: "Controls how events with fields not present in the schema are handled." + required: false type: string: { default: "relaxed" enum: { diff --git a/website/cue/reference/components/sinks/generated/clickhouse.cue b/website/cue/reference/components/sinks/generated/clickhouse.cue index 547fbc943e9d9..e07014f38a308 100644 --- a/website/cue/reference/components/sinks/generated/clickhouse.cue +++ b/website/cue/reference/components/sinks/generated/clickhouse.cue @@ -249,8 +249,6 @@ generated: components: sinks: clickhouse: configuration: { When specified, events are encoded together as a single batch. This is mutually exclusive with per-event encoding based on the `format` field. - - Only the `arrow_stream` codec is supported by the ClickHouse sink. """ required: false type: object: options: { @@ -265,79 +263,27 @@ generated: components: sinks: clickhouse: configuration: { When disabled (default), missing values for non-nullable fields results in encoding errors. This is to help ensure all required data is present before sending it to the sink. """ - relevant_when: "codec = \"arrow_stream\"" - required: false + required: false type: bool: default: false } codec: { - description: "The codec to use for batch encoding events." - required: true - type: string: enum: { - arrow_stream: """ - Encodes events in [Apache Arrow][apache_arrow] IPC streaming format. + description: """ + Encodes events in [Apache Arrow][apache_arrow] IPC streaming format. - This is the streaming variant of the Arrow IPC format, which writes - a continuous stream of record batches. + This is the streaming variant of the Arrow IPC format, which writes + a continuous stream of record batches. - [apache_arrow]: https://arrow.apache.org/ - """ - parquet: """ - Encodes events in [Apache Parquet][apache_parquet] columnar format. + [apache_arrow]: https://arrow.apache.org/ + """ + required: true + type: string: enum: arrow_stream: """ + Encodes events in [Apache Arrow][apache_arrow] IPC streaming format. - [apache_parquet]: https://parquet.apache.org/ - """ - } - } - compression: { - description: "Compression codec applied per column page inside the Parquet file." - relevant_when: "codec = \"parquet\"" - required: false - type: object: options: { - algorithm: { - description: "Compression codec applied per column page inside the Parquet file." - required: false - type: string: { - default: "snappy" - enum: { - gzip: "Gzip compression. Level must be between 1 and 9." - lz4: "LZ4 raw compression" - none: "No compression" - snappy: "Snappy compression (no level)." - zstd: "Zstd compression. Level must be between 1 and 21." - } - } - } - level: { - description: "Compression level (1–21). This is the range Vector supports; higher values compress more but are slower." - relevant_when: "algorithm = \"zstd\" or algorithm = \"gzip\"" - required: true - type: uint: {} - } - } - } - schema_file: { - description: """ - Path to a native Parquet schema file (`.schema`). + This is the streaming variant of the Arrow IPC format, which writes + a continuous stream of record batches. - Required unless `schema_mode` is `auto_infer`. The file must contain a valid - Parquet message type definition. + [apache_arrow]: https://arrow.apache.org/ """ - relevant_when: "codec = \"parquet\"" - required: false - type: string: {} - } - schema_mode: { - description: "Controls how events with fields not present in the schema are handled." - relevant_when: "codec = \"parquet\"" - required: false - type: string: { - default: "relaxed" - enum: { - auto_infer: "Auto infer schema based on the batch. No schema file needed." - relaxed: "Missing fields become null. Extra fields are silently dropped." - strict: "Missing fields become null. Extra fields cause an error." - } - } } } } From e1c6139b9717f36027b0ac9fe4d20276da4da128 Mon Sep 17 00:00:00 2001 From: Thomas Date: Sat, 2 May 2026 16:51:00 -0400 Subject: [PATCH 189/326] chore(metrics): introduce enums for metric names (#25342) * Add MetricName into vector-common * Use EventName in lib/ * Use MetricName in src/internal_events * Format * Fix EventName imports * Use MetricName in k8s * Use MetricName in aws_sqs * Use EventName everywhere and make clippy pass * Separate MetricName -> CounterName/GaugeName/HistogramName * Merge imports * Finish MetricName separation * chore: update check-events script to support typed metric name enums * chore: fix stale MetricName doc references in macro wrappers * chore: remove stale doc paragraph from counter! macro * Fix clippy.toml * Use first metric names instead of made up names to remove clippy::disallowed_macros * chore: replace MetricName with CounterName in Windows internal events --- Cargo.lock | 3 + Cargo.toml | 1 + benches/metrics_snapshot.rs | 6 +- clippy.toml | 6 + lib/codecs/src/internal_events.rs | 29 +- lib/vector-buffers/src/internal_events.rs | 53 +-- .../src/topology/channel/limited_queue.rs | 1 + lib/vector-common/Cargo.toml | 1 + .../src/internal_event/bytes_received.rs | 8 +- .../src/internal_event/bytes_sent.rs | 8 +- .../src/internal_event/cached_event.rs | 48 +-- .../component_events_dropped.rs | 8 +- .../component_events_timed_out.rs | 10 +- .../src/internal_event/events_received.rs | 12 +- .../src/internal_event/events_sent.rs | 18 +- .../src/internal_event/metric_name.rs | 337 ++++++++++++++++++ lib/vector-common/src/internal_event/mod.rs | 2 + .../src/internal_event/service.rs | 8 +- lib/vector-common/src/lib.rs | 57 +++ lib/vector-core/Cargo.toml | 1 + lib/vector-core/src/latency.rs | 12 +- lib/vector-core/src/metrics/mod.rs | 76 ++-- lib/vector-core/src/source_sender/builder.rs | 2 +- lib/vector-core/src/source_sender/mod.rs | 8 +- lib/vector-core/src/source_sender/sender.rs | 4 +- lib/vector-lib/src/lib.rs | 6 +- scripts/check-events | 11 + .../memory/internal_events.rs | 32 +- src/internal_events/adaptive_concurrency.rs | 17 +- src/internal_events/aggregate.rs | 13 +- src/internal_events/amqp.rs | 15 +- src/internal_events/apache_metrics.rs | 11 +- src/internal_events/api.rs | 9 +- src/internal_events/aws.rs | 9 +- src/internal_events/aws_cloudwatch_logs.rs | 7 +- src/internal_events/aws_ec2_metadata.rs | 13 +- src/internal_events/aws_ecs_metrics.rs | 13 +- src/internal_events/aws_kinesis.rs | 8 +- src/internal_events/aws_kinesis_firehose.rs | 11 +- src/internal_events/aws_sqs.rs | 42 ++- src/internal_events/batch.rs | 7 +- src/internal_events/common.rs | 19 +- src/internal_events/conditions.rs | 9 +- src/internal_events/datadog_agent.rs | 9 +- src/internal_events/datadog_metrics.rs | 7 +- src/internal_events/datadog_traces.rs | 9 +- src/internal_events/dedupe.rs | 6 +- src/internal_events/demo_logs.rs | 3 +- src/internal_events/dnstap.rs | 9 +- src/internal_events/docker_logs.rs | 23 +- src/internal_events/doris.rs | 12 +- src/internal_events/encoding_transcode.rs | 13 +- src/internal_events/eventstoredb_metrics.rs | 11 +- src/internal_events/exec.rs | 22 +- src/internal_events/expansion.rs | 7 +- src/internal_events/file.rs | 72 ++-- src/internal_events/file_descriptor.rs | 9 +- src/internal_events/fluent.rs | 9 +- src/internal_events/gcp_pubsub.rs | 13 +- src/internal_events/grpc.rs | 17 +- src/internal_events/heartbeat.rs | 11 +- src/internal_events/host_metrics.rs | 13 +- src/internal_events/http.rs | 25 +- src/internal_events/http_client.rs | 22 +- src/internal_events/http_client_source.rs | 13 +- src/internal_events/influxdb.rs | 7 +- src/internal_events/internal_logs.rs | 14 +- src/internal_events/journald.rs | 14 +- src/internal_events/kafka.rs | 39 +- src/internal_events/kubernetes_logs.rs | 34 +- src/internal_events/log_to_metric.rs | 19 +- src/internal_events/logplex.rs | 9 +- src/internal_events/loki.rs | 13 +- src/internal_events/lua.rs | 11 +- src/internal_events/metric_to_log.rs | 7 +- src/internal_events/mongodb_metrics.rs | 13 +- src/internal_events/mqtt.rs | 9 +- src/internal_events/nginx_metrics.rs | 13 +- src/internal_events/open.rs | 11 +- src/internal_events/parser.rs | 15 +- src/internal_events/postgresql_metrics.rs | 9 +- src/internal_events/process.rs | 21 +- src/internal_events/prometheus.rs | 17 +- src/internal_events/pulsar.rs | 15 +- src/internal_events/redis.rs | 9 +- src/internal_events/reduce.rs | 11 +- src/internal_events/remap.rs | 8 +- src/internal_events/sample.rs | 6 +- src/internal_events/sematext_metrics.rs | 13 +- src/internal_events/socket.rs | 33 +- src/internal_events/splunk_hec.rs | 39 +- src/internal_events/statsd_sink.rs | 7 +- src/internal_events/tag_cardinality_limit.rs | 17 +- src/internal_events/tcp.rs | 19 +- src/internal_events/template.rs | 11 +- src/internal_events/throttle.rs | 9 +- src/internal_events/udp.rs | 17 +- src/internal_events/unix.rs | 15 +- src/internal_events/websocket.rs | 27 +- src/internal_events/websocket_server.rs | 23 +- src/internal_events/windows.rs | 19 +- src/internal_events/windows_event_log.rs | 11 +- src/internal_telemetry/allocations/mod.rs | 13 +- src/sources/internal_metrics.rs | 47 ++- src/topology/builder.rs | 8 +- src/utilization.rs | 7 +- 106 files changed, 1254 insertions(+), 711 deletions(-) create mode 100644 lib/vector-common/src/internal_event/metric_name.rs diff --git a/Cargo.lock b/Cargo.lock index 9bab182e05373..2e217d2d24c4f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12939,6 +12939,7 @@ dependencies = [ "sqlx", "stream-cancel", "strip-ansi-escapes", + "strum 0.28.0", "sysinfo", "syslog", "tempfile", @@ -13064,6 +13065,7 @@ dependencies = [ "serde_json", "smallvec", "stream-cancel", + "strum 0.28.0", "tokio", "tracing 0.1.44", "vector-common-macros", @@ -13191,6 +13193,7 @@ dependencies = [ "smallvec", "snafu 0.9.0", "socket2 0.5.10", + "strum 0.28.0", "tokio", "tokio-openssl", "tokio-stream", diff --git a/Cargo.toml b/Cargo.toml index 904f540cd26a5..e9cf7da0f400b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -479,6 +479,7 @@ openssl-src = { version = "300", default-features = false, features = ["force-en [dev-dependencies] approx = "0.5.1" +strum.workspace = true assert_cmd = { version = "2.0.17", default-features = false } aws-smithy-runtime = { version = "1.8.3", default-features = false, features = ["tls-rustls"] } base64 = "0.22.1" diff --git a/benches/metrics_snapshot.rs b/benches/metrics_snapshot.rs index 279911e5e65a4..4df220256f7b9 100644 --- a/benches/metrics_snapshot.rs +++ b/benches/metrics_snapshot.rs @@ -1,4 +1,7 @@ use criterion::{BenchmarkId, Criterion, criterion_group}; +use strum::IntoEnumIterator; +use vector_lib::counter; +use vector_lib::internal_event::CounterName; fn benchmark(c: &mut Criterion) { let mut group = c.benchmark_group("metrics_snapshot"); @@ -22,8 +25,9 @@ fn prepare_metrics(cardinality: usize) -> &'static vector::metrics::Controller { let controller = vector::metrics::Controller::get().unwrap(); controller.reset(); + let name = CounterName::iter().next().unwrap(); for idx in 0..cardinality { - metrics::counter!("test", "idx" => idx.to_string()).increment(1); + counter!(name, "idx" => idx.to_string()).increment(1); } controller diff --git a/clippy.toml b/clippy.toml index cbd8c640fdb71..4aaa1eb15b360 100644 --- a/clippy.toml +++ b/clippy.toml @@ -8,6 +8,12 @@ disallowed-methods = [ { path = "vrl::stdlib::all", reason = "Use `vector_vrl_functions::all()` instead for consistency across all Vector VRL functions." }, ] +disallowed-macros = [ + { path = "metrics::counter", reason = "Use the `counter!` macro from `vector_common` with a `CounterName` variant instead of a raw string." }, + { path = "metrics::histogram", reason = "Use the `histogram!` macro from `vector_common` with a `HistogramName` variant instead of a raw string." }, + { path = "metrics::gauge", reason = "Use the `gauge!` macro from `vector_common` with a `GaugeName` variant instead of a raw string." }, +] + disallowed-types = [ { path = "once_cell::sync::OnceCell", reason = "Use `std::sync::OnceLock` instead." }, { path = "once_cell::unsync::OnceCell", reason = "Use `std::cell::OnceCell` instead." }, diff --git a/lib/codecs/src/internal_events.rs b/lib/codecs/src/internal_events.rs index d21119419887d..0518247e523a3 100644 --- a/lib/codecs/src/internal_events.rs +++ b/lib/codecs/src/internal_events.rs @@ -1,9 +1,12 @@ //! Internal events for codecs. -use metrics::counter; use tracing::error; -use vector_common::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, emit, error_stage, error_type, +use vector_common::{ + counter, + internal_event::{ + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, emit, error_stage, + error_type, + }, }; use vector_common_macros::NamedInternalEvent; @@ -24,7 +27,7 @@ impl InternalEvent for DecoderFramingError { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "decoder_frame", "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, @@ -50,7 +53,7 @@ impl InternalEvent for DecoderDeserializeError<'_> { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "decoder_deserialize", "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, @@ -77,7 +80,7 @@ impl InternalEvent for EncoderFramingError<'_> { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "encoder_frame", "error_type" => error_type::ENCODER_FAILED, "stage" => error_stage::SENDING, @@ -105,7 +108,7 @@ impl InternalEvent for EncoderSerializeError<'_> { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "encoder_serialize", "error_type" => error_type::ENCODER_FAILED, "stage" => error_stage::SENDING, @@ -137,7 +140,7 @@ impl InternalEvent for EncoderWriteError<'_, E> { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::ENCODER_FAILED, "stage" => error_stage::SENDING, ) @@ -170,7 +173,7 @@ impl InternalEvent for EncoderNullConstraintError<'_> { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "encoding_null_constraint", "error_type" => error_type::ENCODER_FAILED, "stage" => error_stage::SENDING, @@ -201,7 +204,7 @@ impl InternalEvent for EncoderRecordBatchError<'_, E> { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => self.error_code, "error_type" => error_type::ENCODER_FAILED, "stage" => error_stage::SENDING, @@ -228,7 +231,7 @@ impl InternalEvent for SchemaGenerationError<'_> { internal_log_rate_limit = false, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "parquet_schema_generation_failed", "error_type" => error_type::ENCODER_FAILED, "stage" => error_stage::SENDING, @@ -255,7 +258,7 @@ impl InternalEvent for ArrowWriterError<'_> { internal_log_rate_limit = false, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "parquet_arrow_writer_failed", "error_type" => error_type::ENCODER_FAILED, "stage" => error_stage::SENDING, @@ -281,7 +284,7 @@ impl InternalEvent for JsonSerializationError<'_> { internal_log_rate_limit = true, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::ENCODER_FAILED, "stage" => error_stage::SENDING, ) diff --git a/lib/vector-buffers/src/internal_events.rs b/lib/vector-buffers/src/internal_events.rs index 6d2735c7bb75f..31a6089322d0c 100644 --- a/lib/vector-buffers/src/internal_events.rs +++ b/lib/vector-buffers/src/internal_events.rs @@ -1,9 +1,10 @@ use std::time::Duration; -use metrics::{Histogram, counter, gauge, histogram}; +use metrics::Histogram; use vector_common::NamedInternalEvent; use vector_common::{ - internal_event::{InternalEvent, error_type}, + counter, gauge, histogram, + internal_event::{CounterName, GaugeName, HistogramName, InternalEvent, error_type}, registered_event, }; @@ -21,14 +22,14 @@ impl InternalEvent for BufferCreated { let stage = self.idx.to_string(); if self.max_size_events != 0 { gauge!( - "buffer_max_size_events", + GaugeName::BufferMaxSizeEvents, "buffer_id" => self.buffer_id.clone(), "stage" => stage.clone(), ) .set(self.max_size_events as f64); // DEPRECATED: buffer-bytes-events-metrics gauge!( - "buffer_max_event_size", + GaugeName::BufferMaxEventSize, "buffer_id" => self.buffer_id.clone(), "stage" => stage.clone(), ) @@ -36,14 +37,14 @@ impl InternalEvent for BufferCreated { } if self.max_size_bytes != 0 { gauge!( - "buffer_max_size_bytes", + GaugeName::BufferMaxSizeBytes, "buffer_id" => self.buffer_id.clone(), "stage" => stage.clone(), ) .set(self.max_size_bytes as f64); // DEPRECATED: buffer-bytes-events-metrics gauge!( - "buffer_max_byte_size", + GaugeName::BufferMaxByteSize, "buffer_id" => self.buffer_id, "stage" => stage, ) @@ -66,40 +67,40 @@ impl InternalEvent for BufferEventsReceived { #[expect(clippy::cast_precision_loss)] fn emit(self) { counter!( - "buffer_received_events_total", + CounterName::BufferReceivedEventsTotal, "buffer_id" => self.buffer_id.clone(), "stage" => self.idx.to_string() ) .increment(self.count); counter!( - "buffer_received_bytes_total", + CounterName::BufferReceivedBytesTotal, "buffer_id" => self.buffer_id.clone(), "stage" => self.idx.to_string() ) .increment(self.byte_size); // DEPRECATED: buffer-bytes-events-metrics gauge!( - "buffer_events", + GaugeName::BufferEvents, "buffer_id" => self.buffer_id.clone(), "stage" => self.idx.to_string() ) .set(self.total_count as f64); gauge!( - "buffer_size_events", + GaugeName::BufferSizeEvents, "buffer_id" => self.buffer_id.clone(), "stage" => self.idx.to_string() ) .set(self.total_count as f64); gauge!( - "buffer_size_bytes", + GaugeName::BufferSizeBytes, "buffer_id" => self.buffer_id.clone(), "stage" => self.idx.to_string() ) .set(self.total_byte_size as f64); // DEPRECATED: buffer-bytes-events-metrics gauge!( - "buffer_byte_size", + GaugeName::BufferByteSize, "buffer_id" => self.buffer_id, "stage" => self.idx.to_string() ) @@ -121,39 +122,39 @@ impl InternalEvent for BufferEventsSent { #[expect(clippy::cast_precision_loss)] fn emit(self) { counter!( - "buffer_sent_events_total", + CounterName::BufferSentEventsTotal, "buffer_id" => self.buffer_id.clone(), "stage" => self.idx.to_string() ) .increment(self.count); counter!( - "buffer_sent_bytes_total", + CounterName::BufferSentBytesTotal, "buffer_id" => self.buffer_id.clone(), "stage" => self.idx.to_string() ) .increment(self.byte_size); // DEPRECATED: buffer-bytes-events-metrics gauge!( - "buffer_events", + GaugeName::BufferEvents, "buffer_id" => self.buffer_id.clone(), "stage" => self.idx.to_string() ) .set(self.total_count as f64); gauge!( - "buffer_size_events", + GaugeName::BufferSizeEvents, "buffer_id" => self.buffer_id.clone(), "stage" => self.idx.to_string() ) .set(self.total_count as f64); gauge!( - "buffer_size_bytes", + GaugeName::BufferSizeBytes, "buffer_id" => self.buffer_id.clone(), "stage" => self.idx.to_string() ) .set(self.total_byte_size as f64); // DEPRECATED: buffer-bytes-events-metrics gauge!( - "buffer_byte_size", + GaugeName::BufferByteSize, "buffer_id" => self.buffer_id, "stage" => self.idx.to_string() ) @@ -200,14 +201,14 @@ impl InternalEvent for BufferEventsDropped { } counter!( - "buffer_discarded_events_total", + CounterName::BufferDiscardedEventsTotal, "buffer_id" => self.buffer_id.clone(), "stage" => self.idx.to_string(), "intentional" => intentional_str, ) .increment(self.count); counter!( - "buffer_discarded_bytes_total", + CounterName::BufferDiscardedBytesTotal, "buffer_id" => self.buffer_id.clone(), "stage" => self.idx.to_string(), "intentional" => intentional_str, @@ -215,26 +216,26 @@ impl InternalEvent for BufferEventsDropped { .increment(self.byte_size); // DEPRECATED: buffer-bytes-events-metrics gauge!( - "buffer_events", + GaugeName::BufferEvents, "buffer_id" => self.buffer_id.clone(), "stage" => self.idx.to_string() ) .set(self.total_count as f64); gauge!( - "buffer_size_events", + GaugeName::BufferSizeEvents, "buffer_id" => self.buffer_id.clone(), "stage" => self.idx.to_string() ) .set(self.total_count as f64); gauge!( - "buffer_size_bytes", + GaugeName::BufferSizeBytes, "buffer_id" => self.buffer_id.clone(), "stage" => self.idx.to_string() ) .set(self.total_byte_size as f64); // DEPRECATED: buffer-bytes-events-metrics gauge!( - "buffer_byte_size", + GaugeName::BufferByteSize, "buffer_id" => self.buffer_id, "stage" => self.idx.to_string() ) @@ -258,7 +259,7 @@ impl InternalEvent for BufferReadError { stage = "processing", ); counter!( - "buffer_errors_total", "error_code" => self.error_code, + CounterName::BufferErrorsTotal, "error_code" => self.error_code, "error_type" => "reader_failed", "stage" => "processing", ) @@ -270,7 +271,7 @@ registered_event! { BufferSendDuration { stage: usize, } => { - send_duration: Histogram = histogram!("buffer_send_duration_seconds", "stage" => self.stage.to_string()), + send_duration: Histogram = histogram!(HistogramName::BufferSendDurationSeconds, "stage" => self.stage.to_string()), } fn emit(&self, duration: Duration) { diff --git a/lib/vector-buffers/src/topology/channel/limited_queue.rs b/lib/vector-buffers/src/topology/channel/limited_queue.rs index a246aa9659812..3e1f3f7c98c93 100644 --- a/lib/vector-buffers/src/topology/channel/limited_queue.rs +++ b/lib/vector-buffers/src/topology/channel/limited_queue.rs @@ -127,6 +127,7 @@ struct Metrics { impl Metrics { #[expect(clippy::cast_precision_loss)] // We have to convert buffer sizes for a gauge, it's okay to lose precision here. + #[allow(clippy::disallowed_macros)] // Metric names are constructed dynamically from runtime prefixes. fn new( limit: MemoryBufferSize, metadata: ChannelMetricMetadata, diff --git a/lib/vector-common/Cargo.toml b/lib/vector-common/Cargo.toml index 95098f19b1bd3..a583641c68cec 100644 --- a/lib/vector-common/Cargo.toml +++ b/lib/vector-common/Cargo.toml @@ -48,6 +48,7 @@ pin-project.workspace = true serde.workspace = true serde_json.workspace = true smallvec = { version = "1", default-features = false } +strum.workspace = true stream-cancel = { version = "0.8.2", default-features = false } tokio = { workspace = true, features = ["macros", "time"] } tracing.workspace = true diff --git a/lib/vector-common/src/internal_event/bytes_received.rs b/lib/vector-common/src/internal_event/bytes_received.rs index 98044ecb84570..4264c3cbc365a 100644 --- a/lib/vector-common/src/internal_event/bytes_received.rs +++ b/lib/vector-common/src/internal_event/bytes_received.rs @@ -1,12 +1,14 @@ -use metrics::{Counter, counter}; +use metrics::Counter; -use super::{ByteSize, Protocol, SharedString}; +use crate::counter; + +use super::{ByteSize, CounterName, Protocol, SharedString}; crate::registered_event!( BytesReceived { protocol: SharedString, } => { - received_bytes: Counter = counter!("component_received_bytes_total", "protocol" => self.protocol.clone()), + received_bytes: Counter = counter!(CounterName::ComponentReceivedBytesTotal, "protocol" => self.protocol.clone()), protocol: SharedString = self.protocol, } diff --git a/lib/vector-common/src/internal_event/bytes_sent.rs b/lib/vector-common/src/internal_event/bytes_sent.rs index 0b2a88247f96c..b363893783f43 100644 --- a/lib/vector-common/src/internal_event/bytes_sent.rs +++ b/lib/vector-common/src/internal_event/bytes_sent.rs @@ -1,13 +1,15 @@ -use metrics::{Counter, counter}; +use metrics::Counter; + +use crate::counter; use tracing::trace; -use super::{ByteSize, Protocol, SharedString}; +use super::{ByteSize, CounterName, Protocol, SharedString}; crate::registered_event!( BytesSent { protocol: SharedString, } => { - bytes_sent: Counter = counter!("component_sent_bytes_total", "protocol" => self.protocol.clone()), + bytes_sent: Counter = counter!(CounterName::ComponentSentBytesTotal, "protocol" => self.protocol.clone()), protocol: SharedString = self.protocol, } diff --git a/lib/vector-common/src/internal_event/cached_event.rs b/lib/vector-common/src/internal_event/cached_event.rs index 782faeb5a1d01..ec3d492998ab4 100644 --- a/lib/vector-common/src/internal_event/cached_event.rs +++ b/lib/vector-common/src/internal_event/cached_event.rs @@ -91,34 +91,36 @@ where #[cfg(test)] mod tests { #![allow(unreachable_pub)] - use metrics::{Counter, counter}; + use metrics::Counter; + use strum::IntoEnumIterator; use super::*; - - crate::registered_event!( - TestEvent { - fixed: String, - dynamic: String, - } => { - event: Counter = { - counter!("test_event_total", "fixed" => self.fixed, "dynamic" => self.dynamic) - }, - } - - fn emit(&self, count: u64) { - self.event.increment(count); - } - - fn register(fixed: String, dynamic: String) { - crate::internal_event::register(TestEvent { - fixed, - dynamic, - }) - } - ); + use crate::internal_event::CounterName; #[test] fn test_fixed_tag() { + crate::registered_event!( + TestEvent { + fixed: String, + dynamic: String, + } => { + event: Counter = { + crate::counter!(CounterName::iter().next().unwrap(), "fixed" => self.fixed, "dynamic" => self.dynamic) + }, + } + + fn emit(&self, count: u64) { + self.event.increment(count); + } + + fn register(fixed: String, dynamic: String) { + crate::internal_event::register(TestEvent { + fixed, + dynamic, + }) + } + ); + let event: RegisteredEventCache = RegisteredEventCache::new("fixed".to_string()); diff --git a/lib/vector-common/src/internal_event/component_events_dropped.rs b/lib/vector-common/src/internal_event/component_events_dropped.rs index bf45c054e45e1..8421d1d0f49ae 100644 --- a/lib/vector-common/src/internal_event/component_events_dropped.rs +++ b/lib/vector-common/src/internal_event/component_events_dropped.rs @@ -1,6 +1,8 @@ -use metrics::{Counter, counter}; +use metrics::Counter; -use super::{Count, InternalEvent, InternalEventHandle, RegisterInternalEvent}; +use crate::counter; + +use super::{Count, CounterName, InternalEvent, InternalEventHandle, RegisterInternalEvent}; use crate::NamedInternalEvent; pub const INTENTIONAL: bool = true; @@ -32,7 +34,7 @@ impl<'a, const INTENTIONAL: bool> RegisterInternalEvent fn register(self) -> Self::Handle { Self::Handle { discarded_events: counter!( - "component_discarded_events_total", + CounterName::ComponentDiscardedEventsTotal, "intentional" => if INTENTIONAL { "true" } else { "false" }, ), reason: self.reason, diff --git a/lib/vector-common/src/internal_event/component_events_timed_out.rs b/lib/vector-common/src/internal_event/component_events_timed_out.rs index bf138dd1481c7..4662322fe9776 100644 --- a/lib/vector-common/src/internal_event/component_events_timed_out.rs +++ b/lib/vector-common/src/internal_event/component_events_timed_out.rs @@ -1,13 +1,15 @@ -use metrics::{Counter, counter}; +use metrics::Counter; -use super::Count; +use crate::counter; + +use super::{Count, CounterName}; crate::registered_event! { ComponentEventsTimedOut { reason: &'static str, } => { - timed_out_events: Counter = counter!("component_timed_out_events_total"), - timed_out_requests: Counter = counter!("component_timed_out_requests_total"), + timed_out_events: Counter = counter!(CounterName::ComponentTimedOutEventsTotal), + timed_out_requests: Counter = counter!(CounterName::ComponentTimedOutRequestsTotal), reason: &'static str = self.reason, } diff --git a/lib/vector-common/src/internal_event/events_received.rs b/lib/vector-common/src/internal_event/events_received.rs index 40ba999a8cfca..a08ebc6e8438b 100644 --- a/lib/vector-common/src/internal_event/events_received.rs +++ b/lib/vector-common/src/internal_event/events_received.rs @@ -1,13 +1,15 @@ -use metrics::{Counter, Histogram, counter, histogram}; +use metrics::{Counter, Histogram}; + +use crate::{counter, histogram}; use tracing::trace; -use super::CountByteSize; +use super::{CountByteSize, CounterName, HistogramName}; crate::registered_event!( EventsReceived => { - events_count: Histogram = histogram!("component_received_events_count"), - events: Counter = counter!("component_received_events_total"), - event_bytes: Counter = counter!("component_received_event_bytes_total"), + events_count: Histogram = histogram!(HistogramName::ComponentReceivedEventsCount), + events: Counter = counter!(CounterName::ComponentReceivedEventsTotal), + event_bytes: Counter = counter!(CounterName::ComponentReceivedEventBytesTotal), } fn emit(&self, data: CountByteSize) { diff --git a/lib/vector-common/src/internal_event/events_sent.rs b/lib/vector-common/src/internal_event/events_sent.rs index 21e11260b2d38..3ae63db761381 100644 --- a/lib/vector-common/src/internal_event/events_sent.rs +++ b/lib/vector-common/src/internal_event/events_sent.rs @@ -1,9 +1,11 @@ use std::sync::Arc; -use metrics::{Counter, counter}; +use metrics::Counter; + +use crate::counter; use tracing::trace; -use super::{CountByteSize, OptionalTag, Output, SharedString}; +use super::{CountByteSize, CounterName, OptionalTag, Output, SharedString}; use crate::config::ComponentKey; pub const DEFAULT_OUTPUT: &str = "_default"; @@ -13,14 +15,14 @@ crate::registered_event!( output: Option, } => { events: Counter = if let Some(output) = &self.output { - counter!("component_sent_events_total", "output" => output.clone()) + counter!(CounterName::ComponentSentEventsTotal, "output" => output.clone()) } else { - counter!("component_sent_events_total") + counter!(CounterName::ComponentSentEventsTotal) }, event_bytes: Counter = if let Some(output) = &self.output { - counter!("component_sent_event_bytes_total", "output" => output.clone()) + counter!(CounterName::ComponentSentEventBytesTotal, "output" => output.clone()) } else { - counter!("component_sent_event_bytes_total") + counter!(CounterName::ComponentSentEventBytesTotal) }, output: Option = self.output, } @@ -75,10 +77,10 @@ crate::registered_event!( service: OptionalTag, } => { events: Counter = { - counter!("component_sent_events_total", &make_tags(&self.source, &self.service)) + counter!(CounterName::ComponentSentEventsTotal, &make_tags(&self.source, &self.service)) }, event_bytes: Counter = { - counter!("component_sent_event_bytes_total", &make_tags(&self.source, &self.service)) + counter!(CounterName::ComponentSentEventBytesTotal, &make_tags(&self.source, &self.service)) }, } diff --git a/lib/vector-common/src/internal_event/metric_name.rs b/lib/vector-common/src/internal_event/metric_name.rs new file mode 100644 index 0000000000000..21314695a13bf --- /dev/null +++ b/lib/vector-common/src/internal_event/metric_name.rs @@ -0,0 +1,337 @@ +use strum::{AsRefStr, Display, EnumIter}; + +/// Canonical list of all per-component internal metric names emitted by Vector. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Display, AsRefStr, EnumIter)] +#[strum(serialize_all = "snake_case")] +pub enum CounterName { + ComponentReceivedEventsTotal, + ComponentReceivedEventBytesTotal, + ComponentReceivedBytesTotal, + ComponentSentEventsTotal, + ComponentSentEventBytesTotal, + ComponentSentBytesTotal, + ComponentDiscardedEventsTotal, + ComponentErrorsTotal, + ComponentTimedOutEventsTotal, + ComponentTimedOutRequestsTotal, + BufferReceivedEventsTotal, + BufferReceivedBytesTotal, + BufferSentEventsTotal, + BufferSentBytesTotal, + BufferDiscardedEventsTotal, + BufferDiscardedBytesTotal, + BufferErrorsTotal, + // Internal events from src/internal_events/ + AggregateEventsRecordedTotal, + AggregateFailedUpdates, + AggregateFlushesTotal, + ApiStartedTotal, + CheckpointsTotal, + ChecksumErrorsTotal, + CollectCompletedTotal, + CommandExecutedTotal, + ConnectionEstablishedTotal, + ConnectionSendErrorsTotal, + ConnectionShutdownTotal, + ContainerProcessedEventsTotal, + ContainersUnwatchedTotal, + ContainersWatchedTotal, + DecoderBomRemovalsTotal, + DecoderMalformedReplacementWarningsTotal, + DorisBytesLoadedTotal, + DorisRowsFilteredTotal, + DorisRowsLoadedTotal, + EncoderUnmappableReplacementWarningsTotal, + EventsDiscardedTotal, + FilesAddedTotal, + FilesDeletedTotal, + FilesResumedTotal, + FilesUnwatchedTotal, + GrpcServerMessagesReceivedTotal, + GrpcServerMessagesSentTotal, + HttpClientErrorsTotal, + HttpClientRequestsSentTotal, + HttpClientResponsesTotal, + HttpServerRequestsReceivedTotal, + HttpServerResponsesSentTotal, + KafkaConsumedMessagesBytesTotal, + KafkaConsumedMessagesTotal, + KafkaProducedMessagesBytesTotal, + KafkaProducedMessagesTotal, + KafkaRequestsBytesTotal, + KafkaRequestsTotal, + KafkaResponsesBytesTotal, + KafkaResponsesTotal, + MetadataRefreshFailedTotal, + MetadataRefreshSuccessfulTotal, + ParseErrorsTotal, + QuitTotal, + ReloadedTotal, + RewrittenTimestampEventsTotal, + SqsMessageDeferSucceededTotal, + SqsMessageDeleteSucceededTotal, + SqsMessageProcessingSucceededTotal, + SqsMessageReceiveSucceededTotal, + SqsMessageReceivedMessagesTotal, + StaleEventsFlushedTotal, + StartedTotal, + StoppedTotal, + TagValueLimitExceededTotal, + ValueLimitReachedTotal, + WebsocketBytesSentTotal, + WebsocketMessagesSentTotal, + WindowsServiceInstallTotal, + WindowsServiceRestartTotal, + WindowsServiceStartTotal, + WindowsServiceStopTotal, + WindowsServiceUninstallTotal, + K8sEventNamespaceAnnotationFailuresTotal, + K8sEventNodeAnnotationFailuresTotal, + K8sFormatPickerEdgeCasesTotal, + K8sDockerFormatParseFailuresTotal, + SqsS3EventRecordIgnoredTotal, + ComponentAllocatedBytesTotal, + ComponentDeallocatedBytesTotal, + MemoryEnrichmentTableFailedInsertions, + MemoryEnrichmentTableFailedReads, + MemoryEnrichmentTableFlushesTotal, + MemoryEnrichmentTableInsertionsTotal, + MemoryEnrichmentTableReadsTotal, + MemoryEnrichmentTableTtlExpirations, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Display, AsRefStr, EnumIter)] +#[strum(serialize_all = "snake_case")] +pub enum HistogramName { + ComponentReceivedEventsCount, + ComponentReceivedBytes, + BufferSendDurationSeconds, + ComponentLatencySeconds, + SourceLagTimeSeconds, + SourceSendLatencySeconds, + SourceSendBatchLatencySeconds, + AdaptiveConcurrencyAveragedRtt, + AdaptiveConcurrencyBackPressure, + AdaptiveConcurrencyInFlight, + AdaptiveConcurrencyLimit, + AdaptiveConcurrencyObservedRtt, + AdaptiveConcurrencyPastRttMean, + AdaptiveConcurrencyReachedLimit, + S3ObjectProcessingSucceededDurationSeconds, + S3ObjectProcessingFailedDurationSeconds, + CollectDurationSeconds, + CommandExecutionDurationSeconds, + GrpcServerHandlerDurationSeconds, + HttpServerHandlerDurationSeconds, + HttpClientRttSeconds, + HttpClientResponseRttSeconds, + HttpClientErrorRttSeconds, +} + +impl HistogramName { + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::ComponentReceivedEventsCount => "component_received_events_count", + Self::ComponentReceivedBytes => "component_received_bytes", + Self::BufferSendDurationSeconds => "buffer_send_duration_seconds", + Self::ComponentLatencySeconds => "component_latency_seconds", + Self::SourceLagTimeSeconds => "source_lag_time_seconds", + Self::SourceSendLatencySeconds => "source_send_latency_seconds", + Self::SourceSendBatchLatencySeconds => "source_send_batch_latency_seconds", + Self::AdaptiveConcurrencyAveragedRtt => "adaptive_concurrency_averaged_rtt", + Self::AdaptiveConcurrencyBackPressure => "adaptive_concurrency_back_pressure", + Self::AdaptiveConcurrencyInFlight => "adaptive_concurrency_in_flight", + Self::AdaptiveConcurrencyLimit => "adaptive_concurrency_limit", + Self::AdaptiveConcurrencyObservedRtt => "adaptive_concurrency_observed_rtt", + Self::AdaptiveConcurrencyPastRttMean => "adaptive_concurrency_past_rtt_mean", + Self::AdaptiveConcurrencyReachedLimit => "adaptive_concurrency_reached_limit", + Self::S3ObjectProcessingSucceededDurationSeconds => { + "s3_object_processing_succeeded_duration_seconds" + } + Self::S3ObjectProcessingFailedDurationSeconds => { + "s3_object_processing_failed_duration_seconds" + } + Self::CollectDurationSeconds => "collect_duration_seconds", + Self::CommandExecutionDurationSeconds => "command_execution_duration_seconds", + Self::GrpcServerHandlerDurationSeconds => "grpc_server_handler_duration_seconds", + Self::HttpServerHandlerDurationSeconds => "http_server_handler_duration_seconds", + Self::HttpClientRttSeconds => "http_client_rtt_seconds", + Self::HttpClientResponseRttSeconds => "http_client_response_rtt_seconds", + Self::HttpClientErrorRttSeconds => "http_client_error_rtt_seconds", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Display, AsRefStr, EnumIter)] +#[strum(serialize_all = "snake_case")] +pub enum GaugeName { + ComponentLatencyMeanSeconds, + BufferMaxSizeEvents, + BufferMaxEventSize, + BufferMaxSizeBytes, + BufferMaxByteSize, + BufferEvents, + BufferSizeEvents, + BufferSizeBytes, + BufferByteSize, + Utilization, + ComponentAllocatedBytes, + OpenFiles, + UptimeSeconds, + BuildInfo, + KafkaQueueMessages, + KafkaQueueMessagesBytes, + KafkaConsumerLag, + LuaMemoryUsedBytes, + OpenConnections, + ActiveEndpoints, + SplunkPendingAcks, + ActiveClients, + MemoryEnrichmentTableObjectsCount, + MemoryEnrichmentTableByteSize, +} + +impl GaugeName { + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::ComponentLatencyMeanSeconds => "component_latency_mean_seconds", + Self::BufferMaxSizeEvents => "buffer_max_size_events", + Self::BufferMaxEventSize => "buffer_max_event_size", + Self::BufferMaxSizeBytes => "buffer_max_size_bytes", + Self::BufferMaxByteSize => "buffer_max_byte_size", + Self::BufferEvents => "buffer_events", + Self::BufferSizeEvents => "buffer_size_events", + Self::BufferSizeBytes => "buffer_size_bytes", + Self::BufferByteSize => "buffer_byte_size", + Self::Utilization => "utilization", + Self::ComponentAllocatedBytes => "component_allocated_bytes", + Self::OpenFiles => "open_files", + Self::UptimeSeconds => "uptime_seconds", + Self::BuildInfo => "build_info", + Self::KafkaQueueMessages => "kafka_queue_messages", + Self::KafkaQueueMessagesBytes => "kafka_queue_messages_bytes", + Self::KafkaConsumerLag => "kafka_consumer_lag", + Self::LuaMemoryUsedBytes => "lua_memory_used_bytes", + Self::OpenConnections => "open_connections", + Self::ActiveEndpoints => "active_endpoints", + Self::SplunkPendingAcks => "splunk_pending_acks", + Self::ActiveClients => "active_clients", + Self::MemoryEnrichmentTableObjectsCount => "memory_enrichment_table_objects_count", + Self::MemoryEnrichmentTableByteSize => "memory_enrichment_table_byte_size", + } + } +} + +impl CounterName { + #[allow(clippy::too_many_lines)] + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::ComponentReceivedEventsTotal => "component_received_events_total", + Self::ComponentReceivedEventBytesTotal => "component_received_event_bytes_total", + Self::ComponentReceivedBytesTotal => "component_received_bytes_total", + Self::ComponentSentEventsTotal => "component_sent_events_total", + Self::ComponentSentEventBytesTotal => "component_sent_event_bytes_total", + Self::ComponentSentBytesTotal => "component_sent_bytes_total", + Self::ComponentDiscardedEventsTotal => "component_discarded_events_total", + Self::ComponentErrorsTotal => "component_errors_total", + Self::ComponentTimedOutEventsTotal => "component_timed_out_events_total", + Self::ComponentTimedOutRequestsTotal => "component_timed_out_requests_total", + Self::BufferReceivedEventsTotal => "buffer_received_events_total", + Self::BufferReceivedBytesTotal => "buffer_received_bytes_total", + Self::BufferSentEventsTotal => "buffer_sent_events_total", + Self::BufferSentBytesTotal => "buffer_sent_bytes_total", + Self::BufferDiscardedEventsTotal => "buffer_discarded_events_total", + Self::BufferDiscardedBytesTotal => "buffer_discarded_bytes_total", + Self::BufferErrorsTotal => "buffer_errors_total", + Self::AggregateEventsRecordedTotal => "aggregate_events_recorded_total", + Self::AggregateFailedUpdates => "aggregate_failed_updates", + Self::AggregateFlushesTotal => "aggregate_flushes_total", + Self::ApiStartedTotal => "api_started_total", + Self::CheckpointsTotal => "checkpoints_total", + Self::ChecksumErrorsTotal => "checksum_errors_total", + Self::CollectCompletedTotal => "collect_completed_total", + Self::CommandExecutedTotal => "command_executed_total", + Self::ConnectionEstablishedTotal => "connection_established_total", + Self::ConnectionSendErrorsTotal => "connection_send_errors_total", + Self::ConnectionShutdownTotal => "connection_shutdown_total", + Self::ContainerProcessedEventsTotal => "container_processed_events_total", + Self::ContainersUnwatchedTotal => "containers_unwatched_total", + Self::ContainersWatchedTotal => "containers_watched_total", + Self::DecoderBomRemovalsTotal => "decoder_bom_removals_total", + Self::DecoderMalformedReplacementWarningsTotal => { + "decoder_malformed_replacement_warnings_total" + } + Self::DorisBytesLoadedTotal => "doris_bytes_loaded_total", + Self::DorisRowsFilteredTotal => "doris_rows_filtered_total", + Self::DorisRowsLoadedTotal => "doris_rows_loaded_total", + Self::EncoderUnmappableReplacementWarningsTotal => { + "encoder_unmappable_replacement_warnings_total" + } + Self::EventsDiscardedTotal => "events_discarded_total", + Self::FilesAddedTotal => "files_added_total", + Self::FilesDeletedTotal => "files_deleted_total", + Self::FilesResumedTotal => "files_resumed_total", + Self::FilesUnwatchedTotal => "files_unwatched_total", + Self::GrpcServerMessagesReceivedTotal => "grpc_server_messages_received_total", + Self::GrpcServerMessagesSentTotal => "grpc_server_messages_sent_total", + Self::HttpClientErrorsTotal => "http_client_errors_total", + Self::HttpClientRequestsSentTotal => "http_client_requests_sent_total", + Self::HttpClientResponsesTotal => "http_client_responses_total", + Self::HttpServerRequestsReceivedTotal => "http_server_requests_received_total", + Self::HttpServerResponsesSentTotal => "http_server_responses_sent_total", + Self::KafkaConsumedMessagesBytesTotal => "kafka_consumed_messages_bytes_total", + Self::KafkaConsumedMessagesTotal => "kafka_consumed_messages_total", + Self::KafkaProducedMessagesBytesTotal => "kafka_produced_messages_bytes_total", + Self::KafkaProducedMessagesTotal => "kafka_produced_messages_total", + Self::KafkaRequestsBytesTotal => "kafka_requests_bytes_total", + Self::KafkaRequestsTotal => "kafka_requests_total", + Self::KafkaResponsesBytesTotal => "kafka_responses_bytes_total", + Self::KafkaResponsesTotal => "kafka_responses_total", + Self::MetadataRefreshFailedTotal => "metadata_refresh_failed_total", + Self::MetadataRefreshSuccessfulTotal => "metadata_refresh_successful_total", + Self::ParseErrorsTotal => "parse_errors_total", + Self::QuitTotal => "quit_total", + Self::ReloadedTotal => "reloaded_total", + Self::RewrittenTimestampEventsTotal => "rewritten_timestamp_events_total", + Self::SqsMessageDeferSucceededTotal => "sqs_message_defer_succeeded_total", + Self::SqsMessageDeleteSucceededTotal => "sqs_message_delete_succeeded_total", + Self::SqsMessageProcessingSucceededTotal => "sqs_message_processing_succeeded_total", + Self::SqsMessageReceiveSucceededTotal => "sqs_message_receive_succeeded_total", + Self::SqsMessageReceivedMessagesTotal => "sqs_message_received_messages_total", + Self::StaleEventsFlushedTotal => "stale_events_flushed_total", + Self::StartedTotal => "started_total", + Self::StoppedTotal => "stopped_total", + Self::TagValueLimitExceededTotal => "tag_value_limit_exceeded_total", + Self::ValueLimitReachedTotal => "value_limit_reached_total", + Self::WebsocketBytesSentTotal => "websocket_bytes_sent_total", + Self::WebsocketMessagesSentTotal => "websocket_messages_sent_total", + Self::WindowsServiceInstallTotal => "windows_service_install_total", + Self::WindowsServiceRestartTotal => "windows_service_restart_total", + Self::WindowsServiceStartTotal => "windows_service_start_total", + Self::WindowsServiceStopTotal => "windows_service_stop_total", + Self::WindowsServiceUninstallTotal => "windows_service_uninstall_total", + Self::K8sEventNamespaceAnnotationFailuresTotal => { + "k8s_event_namespace_annotation_failures_total" + } + Self::K8sEventNodeAnnotationFailuresTotal => "k8s_event_node_annotation_failures_total", + Self::K8sFormatPickerEdgeCasesTotal => "k8s_format_picker_edge_cases_total", + Self::K8sDockerFormatParseFailuresTotal => "k8s_docker_format_parse_failures_total", + Self::SqsS3EventRecordIgnoredTotal => "sqs_s3_event_record_ignored_total", + Self::ComponentAllocatedBytesTotal => "component_allocated_bytes_total", + Self::ComponentDeallocatedBytesTotal => "component_deallocated_bytes_total", + Self::MemoryEnrichmentTableFailedInsertions => { + "memory_enrichment_table_failed_insertions" + } + Self::MemoryEnrichmentTableFailedReads => "memory_enrichment_table_failed_reads", + Self::MemoryEnrichmentTableFlushesTotal => "memory_enrichment_table_flushes_total", + Self::MemoryEnrichmentTableInsertionsTotal => { + "memory_enrichment_table_insertions_total" + } + Self::MemoryEnrichmentTableReadsTotal => "memory_enrichment_table_reads_total", + Self::MemoryEnrichmentTableTtlExpirations => "memory_enrichment_table_ttl_expirations", + } + } +} diff --git a/lib/vector-common/src/internal_event/mod.rs b/lib/vector-common/src/internal_event/mod.rs index 272e2344900e3..e776e89553172 100644 --- a/lib/vector-common/src/internal_event/mod.rs +++ b/lib/vector-common/src/internal_event/mod.rs @@ -5,6 +5,7 @@ pub mod component_events_dropped; pub mod component_events_timed_out; mod events_received; mod events_sent; +pub mod metric_name; mod optional_tag; mod prelude; pub mod service; @@ -19,6 +20,7 @@ pub use component_events_dropped::{ComponentEventsDropped, INTENTIONAL, UNINTENT pub use component_events_timed_out::ComponentEventsTimedOut; pub use events_received::{EventsReceived, EventsReceivedHandle}; pub use events_sent::{DEFAULT_OUTPUT, EventsSent, TaggedEventsSent}; +pub use metric_name::{CounterName, GaugeName, HistogramName}; pub use metrics::SharedString; pub use optional_tag::OptionalTag; pub use prelude::{error_stage, error_type}; diff --git a/lib/vector-common/src/internal_event/service.rs b/lib/vector-common/src/internal_event/service.rs index 93a3c9b745722..60d615b3d61a0 100644 --- a/lib/vector-common/src/internal_event/service.rs +++ b/lib/vector-common/src/internal_event/service.rs @@ -1,4 +1,6 @@ -use metrics::counter; +use super::CounterName; + +use crate::counter; use super::{ComponentEventsDropped, InternalEvent, UNINTENTIONAL, emit, error_stage, error_type}; use crate::NamedInternalEvent; @@ -17,7 +19,7 @@ impl InternalEvent for PollReadyError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::REQUEST_FAILED, "stage" => error_stage::SENDING, ) @@ -43,7 +45,7 @@ impl InternalEvent for CallError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::REQUEST_FAILED, "stage" => error_stage::SENDING, ) diff --git a/lib/vector-common/src/lib.rs b/lib/vector-common/src/lib.rs index 099a068578090..75849aac75884 100644 --- a/lib/vector-common/src/lib.rs +++ b/lib/vector-common/src/lib.rs @@ -66,6 +66,63 @@ pub mod trigger; #[macro_use] extern crate tracing; +/// Typed wrapper around `metrics::counter!` that only accepts [`internal_event::CounterName`]. +#[macro_export] +macro_rules! counter { + ($name:expr) => {{ + let _name: $crate::internal_event::CounterName = $name; + #[allow(clippy::disallowed_macros)] + { + metrics::counter!(_name.as_str()) + } + }}; + ($name:expr, $($rest:tt)*) => {{ + let _name: $crate::internal_event::CounterName = $name; + #[allow(clippy::disallowed_macros)] + { + metrics::counter!(_name.as_str(), $($rest)*) + } + }}; +} + +/// Typed wrapper around `metrics::histogram!` that only accepts [`internal_event::HistogramName`]. +#[macro_export] +macro_rules! histogram { + ($name:expr) => {{ + let _name: $crate::internal_event::HistogramName = $name; + #[allow(clippy::disallowed_macros)] + { + metrics::histogram!(_name.as_str()) + } + }}; + ($name:expr, $($rest:tt)*) => {{ + let _name: $crate::internal_event::HistogramName = $name; + #[allow(clippy::disallowed_macros)] + { + metrics::histogram!(_name.as_str(), $($rest)*) + } + }}; +} + +/// Typed wrapper around `metrics::gauge!` that only accepts [`internal_event::GaugeName`]. +#[macro_export] +macro_rules! gauge { + ($name:expr) => {{ + let _name: $crate::internal_event::GaugeName = $name; + #[allow(clippy::disallowed_macros)] + { + metrics::gauge!(_name.as_str()) + } + }}; + ($name:expr, $($rest:tt)*) => {{ + let _name: $crate::internal_event::GaugeName = $name; + #[allow(clippy::disallowed_macros)] + { + metrics::gauge!(_name.as_str(), $($rest)*) + } + }}; +} + /// Vector's basic error type, dynamically dispatched and safe to send across /// threads. pub type Error = Box; diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml index 5c3370ac1a526..edf7b43a4a74f 100644 --- a/lib/vector-core/Cargo.toml +++ b/lib/vector-core/Cargo.toml @@ -78,6 +78,7 @@ prost-build.workspace = true [dev-dependencies] base64 = "0.22.1" +strum.workspace = true chrono-tz.workspace = true criterion = { workspace = true, features = ["html_reports"] } env-test-util = "1.0.1" diff --git a/lib/vector-core/src/latency.rs b/lib/vector-core/src/latency.rs index 536a8ef532d0d..b25ca73427749 100644 --- a/lib/vector-core/src/latency.rs +++ b/lib/vector-core/src/latency.rs @@ -1,12 +1,14 @@ use std::time::Instant; -use metrics::{Histogram, gauge, histogram}; +use metrics::Histogram; use vector_common::stats::EwmaGauge; +use vector_common::{ + gauge, histogram, + internal_event::{GaugeName, HistogramName}, +}; use crate::event::EventArray; -const COMPONENT_LATENCY: &str = "component_latency_seconds"; -const COMPONENT_LATENCY_MEAN: &str = "component_latency_mean_seconds"; const DEFAULT_LATENCY_EWMA_ALPHA: f64 = 0.9; #[derive(Debug)] @@ -18,9 +20,9 @@ pub struct LatencyRecorder { impl LatencyRecorder { pub fn new(ewma_alpha: Option) -> Self { Self { - histogram: histogram!(COMPONENT_LATENCY), + histogram: histogram!(HistogramName::ComponentLatencySeconds), gauge: EwmaGauge::new( - gauge!(COMPONENT_LATENCY_MEAN), + gauge!(GaugeName::ComponentLatencyMeanSeconds), ewma_alpha.or(Some(DEFAULT_LATENCY_EWMA_ALPHA)), ), } diff --git a/lib/vector-core/src/metrics/mod.rs b/lib/vector-core/src/metrics/mod.rs index cd900d392f004..ea4c040163c7f 100644 --- a/lib/vector-core/src/metrics/mod.rs +++ b/lib/vector-core/src/metrics/mod.rs @@ -202,6 +202,12 @@ impl Controller { #[cfg(test)] mod tests { + use strum::IntoEnumIterator; + use vector_common::{ + counter, gauge, + internal_event::{CounterName, GaugeName}, + }; + use super::*; use crate::{ config::metrics_expiration::{ @@ -224,8 +230,9 @@ mod tests { let controller = Controller::get().unwrap(); controller.reset(); + let name = CounterName::iter().next().unwrap(); for idx in 0..cardinality { - metrics::counter!("test", "idx" => idx.to_string()).increment(1); + counter!(name, "idx" => idx.to_string()).increment(1); } let metrics = controller.capture_metrics(); @@ -253,11 +260,11 @@ mod tests { fn handles_registered_metrics() { let controller = init_metrics(); - let counter = metrics::counter!("test7"); + let counter = counter!(CounterName::iter().next().unwrap()); assert_eq!(controller.capture_metrics().len(), 3); counter.increment(1); assert_eq!(controller.capture_metrics().len(), 3); - let gauge = metrics::gauge!("test8"); + let gauge = gauge!(GaugeName::iter().next().unwrap()); assert_eq!(controller.capture_metrics().len(), 4); gauge.set(1.0); assert_eq!(controller.capture_metrics().len(), 4); @@ -270,12 +277,16 @@ mod tests { .set_expiry(Some(IDLE_TIMEOUT), Vec::new()) .unwrap(); - metrics::counter!("test2").increment(1); - metrics::counter!("test3").increment(2); + let mut names = CounterName::iter(); + let name_a = names.next().unwrap(); + let name_b = names.next().unwrap(); + + counter!(name_a).increment(1); + counter!(name_b).increment(2); assert_eq!(controller.capture_metrics().len(), 4); std::thread::sleep(Duration::from_secs_f64(IDLE_TIMEOUT * 2.0)); - metrics::counter!("test2").increment(3); + counter!(name_a).increment(3); assert_eq!(controller.capture_metrics().len(), 3); } @@ -286,12 +297,13 @@ mod tests { .set_expiry(Some(IDLE_TIMEOUT), Vec::new()) .unwrap(); - metrics::counter!("test4", "tag" => "value1").increment(1); - metrics::counter!("test4", "tag" => "value2").increment(2); + let name = CounterName::iter().next().unwrap(); + counter!(name, "tag" => "value1").increment(1); + counter!(name, "tag" => "value2").increment(2); assert_eq!(controller.capture_metrics().len(), 4); std::thread::sleep(Duration::from_secs_f64(IDLE_TIMEOUT * 2.0)); - metrics::counter!("test4", "tag" => "value1").increment(3); + counter!(name, "tag" => "value1").increment(3); assert_eq!(controller.capture_metrics().len(), 3); } @@ -302,8 +314,12 @@ mod tests { .set_expiry(Some(IDLE_TIMEOUT), Vec::new()) .unwrap(); - let a = metrics::counter!("test5"); - metrics::counter!("test6").increment(5); + let mut names = CounterName::iter(); + let name_a = names.next().unwrap(); + let name_b = names.next().unwrap(); + + let a = counter!(name_a); + counter!(name_b).increment(5); assert_eq!(controller.capture_metrics().len(), 4); a.increment(1); assert_eq!(controller.capture_metrics().len(), 4); @@ -316,7 +332,7 @@ mod tests { assert_eq!(metrics.len(), 3); let metric = metrics .into_iter() - .find(|metric| metric.name() == "test5") + .find(|metric| metric.name() == name_a.as_str()) .expect("Test metric is not present"); match metric.value() { MetricValue::Counter { value } => assert_eq!(*value, 2.0), @@ -327,12 +343,17 @@ mod tests { #[test] fn expires_metrics_per_set() { let controller = init_metrics(); + + let mut names = CounterName::iter(); + let name_a = names.next().unwrap(); + let name_b = names.next().unwrap(); + controller .set_expiry( None, vec![PerMetricSetExpiration { name: Some(MetricNameMatcherConfig::Exact { - value: "test3".to_string(), + value: name_b.as_str().to_string(), }), labels: None, expire_secs: IDLE_TIMEOUT, @@ -340,25 +361,32 @@ mod tests { ) .unwrap(); - metrics::counter!("test2").increment(1); - metrics::counter!("test3").increment(2); + counter!(name_a).increment(1); + counter!(name_b).increment(2); assert_eq!(controller.capture_metrics().len(), 4); std::thread::sleep(Duration::from_secs_f64(IDLE_TIMEOUT * 2.0)); - metrics::counter!("test2").increment(3); + counter!(name_a).increment(3); assert_eq!(controller.capture_metrics().len(), 3); } #[test] fn expires_metrics_multiple_different_sets() { let controller = init_metrics(); + + let mut names = CounterName::iter(); + let name_a = names.next().unwrap(); + let name_b = names.next().unwrap(); + let name_c = names.next().unwrap(); + let name_d = names.next().unwrap(); + controller .set_expiry( Some(IDLE_TIMEOUT * 3.0), vec![ PerMetricSetExpiration { name: Some(MetricNameMatcherConfig::Exact { - value: "test3".to_string(), + value: name_c.as_str().to_string(), }), labels: None, expire_secs: IDLE_TIMEOUT, @@ -377,22 +405,22 @@ mod tests { ) .unwrap(); - metrics::counter!("test1").increment(1); - metrics::counter!("test2").increment(1); - metrics::counter!("test3").increment(2); - metrics::counter!("test4", "tag" => "value1").increment(3); + counter!(name_a).increment(1); + counter!(name_b).increment(1); + counter!(name_c).increment(2); + counter!(name_d, "tag" => "value1").increment(3); assert_eq!(controller.capture_metrics().len(), 6); std::thread::sleep(Duration::from_secs_f64(IDLE_TIMEOUT * 1.5)); - metrics::counter!("test2").increment(3); + counter!(name_b).increment(3); assert_eq!(controller.capture_metrics().len(), 5); std::thread::sleep(Duration::from_secs_f64(IDLE_TIMEOUT)); - metrics::counter!("test2").increment(3); + counter!(name_b).increment(3); assert_eq!(controller.capture_metrics().len(), 4); std::thread::sleep(Duration::from_secs_f64(IDLE_TIMEOUT)); - metrics::counter!("test2").increment(3); + counter!(name_b).increment(3); assert_eq!(controller.capture_metrics().len(), 3); } } diff --git a/lib/vector-core/src/source_sender/builder.rs b/lib/vector-core/src/source_sender/builder.rs index f3051726a7f56..7f9f5d522497a 100644 --- a/lib/vector-core/src/source_sender/builder.rs +++ b/lib/vector-core/src/source_sender/builder.rs @@ -1,7 +1,7 @@ use std::{collections::HashMap, time::Duration}; -use metrics::histogram; use vector_buffers::topology::channel::LimitedReceiver; +use vector_common::histogram; use vector_common::internal_event::DEFAULT_OUTPUT; use super::{ diff --git a/lib/vector-core/src/source_sender/mod.rs b/lib/vector-core/src/source_sender/mod.rs index fc0cb417890a0..24eb23688c0a1 100644 --- a/lib/vector-core/src/source_sender/mod.rs +++ b/lib/vector-core/src/source_sender/mod.rs @@ -22,6 +22,8 @@ pub const CHUNK_SIZE: usize = 1000; #[cfg(any(test, feature = "test"))] const TEST_BUFFER_SIZE: usize = 100; -const LAG_TIME_NAME: &str = "source_lag_time_seconds"; -const SEND_LATENCY_NAME: &str = "source_send_latency_seconds"; -const SEND_BATCH_LATENCY_NAME: &str = "source_send_batch_latency_seconds"; +use vector_common::internal_event::HistogramName; + +const LAG_TIME_NAME: HistogramName = HistogramName::SourceLagTimeSeconds; +const SEND_LATENCY_NAME: HistogramName = HistogramName::SourceSendLatencySeconds; +const SEND_BATCH_LATENCY_NAME: HistogramName = HistogramName::SourceSendBatchLatencySeconds; diff --git a/lib/vector-core/src/source_sender/sender.rs b/lib/vector-core/src/source_sender/sender.rs index 6041dad3dad9f..d5834ab0edff5 100644 --- a/lib/vector-core/src/source_sender/sender.rs +++ b/lib/vector-core/src/source_sender/sender.rs @@ -5,12 +5,12 @@ use std::{collections::HashMap, time::Instant}; use futures::Stream; #[cfg(any(test, feature = "test"))] use futures::StreamExt as _; -#[cfg(any(test, feature = "test"))] -use metrics::histogram; use vector_buffers::EventCount; #[cfg(any(test, feature = "test"))] use vector_buffers::topology::channel::LimitedReceiver; #[cfg(any(test, feature = "test"))] +use vector_common::histogram; +#[cfg(any(test, feature = "test"))] use vector_common::internal_event::DEFAULT_OUTPUT; #[cfg(doc)] use vector_common::internal_event::{ComponentEventsDropped, EventsSent}; diff --git a/lib/vector-lib/src/lib.rs b/lib/vector-lib/src/lib.rs index 82bb67cf9afce..277d28df18e92 100644 --- a/lib/vector-lib/src/lib.rs +++ b/lib/vector-lib/src/lib.rs @@ -11,9 +11,9 @@ pub use vector_buffers as buffers; pub use vector_common::event_test_util; pub use vector_common::{ Error, NamedInternalEvent, Result, TimeZone, assert_event_data_eq, atomic, btreemap, - byte_size_of, byte_size_of::ByteSizeOf, conversion, encode_logfmt, finalization, finalizer, id, - impl_event_data_eq, internal_event, json_size, registered_event, request_metadata, - sensitive_string, shutdown, stats, trigger, + byte_size_of, byte_size_of::ByteSizeOf, conversion, counter, encode_logfmt, finalization, + finalizer, gauge, histogram, id, impl_event_data_eq, internal_event, json_size, + registered_event, request_metadata, sensitive_string, shutdown, stats, trigger, }; pub use vector_config as configurable; pub use vector_config::impl_generate_config_from_default; diff --git a/scripts/check-events b/scripts/check-events index 5d4411b601802..18a3c61b85418 100755 --- a/scripts/check-events +++ b/scripts/check-events @@ -78,11 +78,22 @@ class Event # Scan for counter names and tags def scan_metrics(block) + # Match string literal names: counter!("metric_name", ...) block.scan(/ (counter|gauge|histogram)!\((?:\n\s+)?"([^"]+)",?(.+?)\)[;\n]/ms) \ do |type, name, tags| tags = Hash[tags.scan(/"([^"]+)" => (.+?)(?:,|$)/)] add_metric(type, name, tags) end + # Match typed enum names: counter!(CounterName::VariantName, ...) + block.scan(/ (counter|gauge|histogram)!\((?:\n\s+)?\w+Name::(\w+),?(.+?)\)[;\n]/ms) \ + do |type, variant, tags| + # Convert CamelCase variant name to snake_case metric name + name = variant.gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2') + .gsub(/([a-z\d])([A-Z])/, '\1_\2') + .downcase + tags = Hash[tags.scan(/"([^"]+)" => (.+?)(?:,|$)/)] + add_metric(type, name, tags) + end end # Scan the registered event macro block diff --git a/src/enrichment_tables/memory/internal_events.rs b/src/enrichment_tables/memory/internal_events.rs index 0799c6662a6af..dfd7b8d6f4c78 100644 --- a/src/enrichment_tables/memory/internal_events.rs +++ b/src/enrichment_tables/memory/internal_events.rs @@ -1,6 +1,8 @@ -use metrics::{counter, gauge}; use vector_lib::{ - NamedInternalEvent, configurable::configurable_component, internal_event::InternalEvent, + NamedInternalEvent, + configurable::configurable_component, + counter, gauge, + internal_event::{CounterName, GaugeName, InternalEvent}, }; /// Configuration of internal metrics for enrichment memory table. @@ -26,12 +28,12 @@ impl InternalEvent for MemoryEnrichmentTableRead<'_> { fn emit(self) { if self.include_key_metric_tag { counter!( - "memory_enrichment_table_reads_total", + CounterName::MemoryEnrichmentTableReadsTotal, "key" => self.key.to_owned() ) .increment(1); } else { - counter!("memory_enrichment_table_reads_total",).increment(1); + counter!(CounterName::MemoryEnrichmentTableReadsTotal,).increment(1); } } } @@ -46,12 +48,12 @@ impl InternalEvent for MemoryEnrichmentTableInserted<'_> { fn emit(self) { if self.include_key_metric_tag { counter!( - "memory_enrichment_table_insertions_total", + CounterName::MemoryEnrichmentTableInsertionsTotal, "key" => self.key.to_owned() ) .increment(1); } else { - counter!("memory_enrichment_table_insertions_total",).increment(1); + counter!(CounterName::MemoryEnrichmentTableInsertionsTotal,).increment(1); } } } @@ -64,9 +66,9 @@ pub(crate) struct MemoryEnrichmentTableFlushed { impl InternalEvent for MemoryEnrichmentTableFlushed { fn emit(self) { - counter!("memory_enrichment_table_flushes_total",).increment(1); - gauge!("memory_enrichment_table_objects_count",).set(self.new_objects_count as f64); - gauge!("memory_enrichment_table_byte_size",).set(self.new_byte_size as f64); + counter!(CounterName::MemoryEnrichmentTableFlushesTotal,).increment(1); + gauge!(GaugeName::MemoryEnrichmentTableObjectsCount,).set(self.new_objects_count as f64); + gauge!(GaugeName::MemoryEnrichmentTableByteSize,).set(self.new_byte_size as f64); } } @@ -80,12 +82,12 @@ impl InternalEvent for MemoryEnrichmentTableTtlExpired<'_> { fn emit(self) { if self.include_key_metric_tag { counter!( - "memory_enrichment_table_ttl_expirations", + CounterName::MemoryEnrichmentTableTtlExpirations, "key" => self.key.to_owned() ) .increment(1); } else { - counter!("memory_enrichment_table_ttl_expirations",).increment(1); + counter!(CounterName::MemoryEnrichmentTableTtlExpirations,).increment(1); } } } @@ -100,12 +102,12 @@ impl InternalEvent for MemoryEnrichmentTableReadFailed<'_> { fn emit(self) { if self.include_key_metric_tag { counter!( - "memory_enrichment_table_failed_reads", + CounterName::MemoryEnrichmentTableFailedReads, "key" => self.key.to_owned() ) .increment(1); } else { - counter!("memory_enrichment_table_failed_reads",).increment(1); + counter!(CounterName::MemoryEnrichmentTableFailedReads,).increment(1); } } } @@ -120,12 +122,12 @@ impl InternalEvent for MemoryEnrichmentTableInsertFailed<'_> { fn emit(self) { if self.include_key_metric_tag { counter!( - "memory_enrichment_table_failed_insertions", + CounterName::MemoryEnrichmentTableFailedInsertions, "key" => self.key.to_owned() ) .increment(1); } else { - counter!("memory_enrichment_table_failed_insertions",).increment(1); + counter!(CounterName::MemoryEnrichmentTableFailedInsertions,).increment(1); } } } diff --git a/src/internal_events/adaptive_concurrency.rs b/src/internal_events/adaptive_concurrency.rs index f815f7d4e2b4b..b43ec8452e20b 100644 --- a/src/internal_events/adaptive_concurrency.rs +++ b/src/internal_events/adaptive_concurrency.rs @@ -1,6 +1,7 @@ use std::time::Duration; -use metrics::{Histogram, histogram}; +use metrics::Histogram; +use vector_lib::{histogram, internal_event::HistogramName}; #[derive(Clone, Copy)] pub struct AdaptiveConcurrencyLimitData { @@ -17,10 +18,10 @@ registered_event! { // These are histograms, as they may have a number of different // values over each reporting interval, and each of those values // is valuable for diagnosis. - limit: Histogram = histogram!("adaptive_concurrency_limit"), - reached_limit: Histogram = histogram!("adaptive_concurrency_reached_limit"), - back_pressure: Histogram = histogram!("adaptive_concurrency_back_pressure"), - past_rtt_mean: Histogram = histogram!("adaptive_concurrency_past_rtt_mean"), + limit: Histogram = histogram!(HistogramName::AdaptiveConcurrencyLimit), + reached_limit: Histogram = histogram!(HistogramName::AdaptiveConcurrencyReachedLimit), + back_pressure: Histogram = histogram!(HistogramName::AdaptiveConcurrencyBackPressure), + past_rtt_mean: Histogram = histogram!(HistogramName::AdaptiveConcurrencyPastRttMean), } fn emit(&self, data: AdaptiveConcurrencyLimitData) { @@ -36,7 +37,7 @@ registered_event! { registered_event! { AdaptiveConcurrencyInFlight => { - in_flight: Histogram = histogram!("adaptive_concurrency_in_flight"), + in_flight: Histogram = histogram!(HistogramName::AdaptiveConcurrencyInFlight), } fn emit(&self, in_flight: u64) { @@ -46,7 +47,7 @@ registered_event! { registered_event! { AdaptiveConcurrencyObservedRtt => { - observed_rtt: Histogram = histogram!("adaptive_concurrency_observed_rtt"), + observed_rtt: Histogram = histogram!(HistogramName::AdaptiveConcurrencyObservedRtt), } fn emit(&self, rtt: Duration) { @@ -56,7 +57,7 @@ registered_event! { registered_event! { AdaptiveConcurrencyAveragedRtt => { - averaged_rtt: Histogram = histogram!("adaptive_concurrency_averaged_rtt"), + averaged_rtt: Histogram = histogram!(HistogramName::AdaptiveConcurrencyAveragedRtt), } fn emit(&self, rtt: Duration) { diff --git a/src/internal_events/aggregate.rs b/src/internal_events/aggregate.rs index 1b70c4350f614..96f48f5de013c 100644 --- a/src/internal_events/aggregate.rs +++ b/src/internal_events/aggregate.rs @@ -1,13 +1,14 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::InternalEvent; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent}, +}; #[derive(Debug, NamedInternalEvent)] pub struct AggregateEventRecorded; impl InternalEvent for AggregateEventRecorded { fn emit(self) { - counter!("aggregate_events_recorded_total").increment(1); + counter!(CounterName::AggregateEventsRecordedTotal).increment(1); } } @@ -16,7 +17,7 @@ pub struct AggregateFlushed; impl InternalEvent for AggregateFlushed { fn emit(self) { - counter!("aggregate_flushes_total").increment(1); + counter!(CounterName::AggregateFlushesTotal).increment(1); } } @@ -25,6 +26,6 @@ pub struct AggregateUpdateFailed; impl InternalEvent for AggregateUpdateFailed { fn emit(self) { - counter!("aggregate_failed_updates").increment(1); + counter!(CounterName::AggregateFailedUpdates).increment(1); } } diff --git a/src/internal_events/amqp.rs b/src/internal_events/amqp.rs index 45acdf201305a..e7864acc44e98 100644 --- a/src/internal_events/amqp.rs +++ b/src/internal_events/amqp.rs @@ -1,8 +1,9 @@ #[cfg(feature = "sources-amqp")] pub mod source { - use metrics::counter; - use vector_lib::NamedInternalEvent; - use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; + use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, + }; #[derive(Debug, NamedInternalEvent)] pub struct AmqpBytesReceived { @@ -18,7 +19,7 @@ pub mod source { protocol = %self.protocol, ); counter!( - "component_received_bytes_total", + CounterName::ComponentReceivedBytesTotal, "protocol" => self.protocol, ) .increment(self.byte_size as u64); @@ -38,7 +39,7 @@ pub mod source { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::REQUEST_FAILED, "stage" => error_stage::RECEIVING, ) @@ -59,7 +60,7 @@ pub mod source { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::ACKNOWLEDGMENT_FAILED, "stage" => error_stage::RECEIVING, ) @@ -80,7 +81,7 @@ pub mod source { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::COMMAND_FAILED, "stage" => error_stage::RECEIVING, ) diff --git a/src/internal_events/apache_metrics.rs b/src/internal_events/apache_metrics.rs index dd5e9c34e4cf5..544dda88ba5a6 100644 --- a/src/internal_events/apache_metrics.rs +++ b/src/internal_events/apache_metrics.rs @@ -1,7 +1,6 @@ -use metrics::counter; use vector_lib::{ - NamedInternalEvent, - internal_event::{InternalEvent, error_stage, error_type}, + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, json_size::JsonSize, }; @@ -19,12 +18,12 @@ impl InternalEvent for ApacheMetricsEventsReceived<'_> { fn emit(self) { trace!(message = "Events received.", count = %self.count, byte_size = %self.byte_size, endpoint = %self.endpoint); counter!( - "component_received_events_total", + CounterName::ComponentReceivedEventsTotal, "endpoint" => self.endpoint.to_owned(), ) .increment(self.count as u64); counter!( - "component_received_event_bytes_total", + CounterName::ComponentReceivedEventBytesTotal, "endpoint" => self.endpoint.to_owned(), ) .increment(self.byte_size.get() as u64); @@ -47,7 +46,7 @@ impl InternalEvent for ApacheMetricsParseError<'_> { endpoint = %self.endpoint, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "stage" => error_stage::PROCESSING, "error_type" => error_type::PARSER_FAILED, "endpoint" => self.endpoint.to_owned(), diff --git a/src/internal_events/api.rs b/src/internal_events/api.rs index de2cb9588537d..b9196989471c8 100644 --- a/src/internal_events/api.rs +++ b/src/internal_events/api.rs @@ -1,8 +1,9 @@ use std::net::SocketAddr; -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::InternalEvent; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent}, +}; #[derive(Debug, NamedInternalEvent)] pub struct ApiStarted { @@ -15,6 +16,6 @@ impl InternalEvent for ApiStarted { message = "API server running.", address = %self.addr, ); - counter!("api_started_total").increment(1); + counter!(CounterName::ApiStartedTotal).increment(1); } } diff --git a/src/internal_events/aws.rs b/src/internal_events/aws.rs index ff4da6c01e56b..d4c91776d7d55 100644 --- a/src/internal_events/aws.rs +++ b/src/internal_events/aws.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::InternalEvent; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent}, +}; #[derive(NamedInternalEvent)] pub struct AwsBytesSent { @@ -21,7 +22,7 @@ impl InternalEvent for AwsBytesSent { region = ?self.region, ); counter!( - "component_sent_bytes_total", + CounterName::ComponentSentBytesTotal, "protocol" => "https", "region" => region, ) diff --git a/src/internal_events/aws_cloudwatch_logs.rs b/src/internal_events/aws_cloudwatch_logs.rs index 6325d6c71b25d..b4964f25d84fe 100644 --- a/src/internal_events/aws_cloudwatch_logs.rs +++ b/src/internal_events/aws_cloudwatch_logs.rs @@ -1,8 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, error_stage, error_type, }; +use vector_lib::{NamedInternalEvent, counter}; #[derive(Debug, NamedInternalEvent)] pub struct AwsCloudwatchLogsMessageSizeError { @@ -22,7 +21,7 @@ impl InternalEvent for AwsCloudwatchLogsMessageSizeError { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "message_too_long", "error_type" => error_type::ENCODER_FAILED, "stage" => error_stage::PROCESSING, diff --git a/src/internal_events/aws_ec2_metadata.rs b/src/internal_events/aws_ec2_metadata.rs index 43c28aee61272..c59364c75348c 100644 --- a/src/internal_events/aws_ec2_metadata.rs +++ b/src/internal_events/aws_ec2_metadata.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; #[derive(Debug, NamedInternalEvent)] pub struct AwsEc2MetadataRefreshSuccessful; @@ -8,7 +9,7 @@ pub struct AwsEc2MetadataRefreshSuccessful; impl InternalEvent for AwsEc2MetadataRefreshSuccessful { fn emit(self) { debug!(message = "AWS EC2 metadata refreshed."); - counter!("metadata_refresh_successful_total").increment(1); + counter!(CounterName::MetadataRefreshSuccessfulTotal).increment(1); } } @@ -26,12 +27,12 @@ impl InternalEvent for AwsEc2MetadataRefreshError { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::REQUEST_FAILED, "stage" => error_stage::PROCESSING, ) .increment(1); // deprecated - counter!("metadata_refresh_failed_total").increment(1); + counter!(CounterName::MetadataRefreshFailedTotal).increment(1); } } diff --git a/src/internal_events/aws_ecs_metrics.rs b/src/internal_events/aws_ecs_metrics.rs index 86b0d32012713..59127134e2666 100644 --- a/src/internal_events/aws_ecs_metrics.rs +++ b/src/internal_events/aws_ecs_metrics.rs @@ -1,9 +1,8 @@ use std::borrow::Cow; -use metrics::counter; use vector_lib::{ - NamedInternalEvent, - internal_event::{InternalEvent, error_stage, error_type}, + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, json_size::JsonSize, }; @@ -24,12 +23,12 @@ impl InternalEvent for AwsEcsMetricsEventsReceived<'_> { endpoint = %self.endpoint, ); counter!( - "component_received_events_total", + CounterName::ComponentReceivedEventsTotal, "endpoint" => self.endpoint.to_string(), ) .increment(self.count as u64); counter!( - "component_received_event_bytes_total", + CounterName::ComponentReceivedEventBytesTotal, "endpoint" => self.endpoint.to_string(), ) .increment(self.byte_size.get() as u64); @@ -57,9 +56,9 @@ impl InternalEvent for AwsEcsMetricsParseError<'_> { endpoint = %self.endpoint, "Failed to parse response.", ); - counter!("parse_errors_total").increment(1); + counter!(CounterName::ParseErrorsTotal).increment(1); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "stage" => error_stage::PROCESSING, "error_type" => error_type::PARSER_FAILED, "endpoint" => self.endpoint.to_string(), diff --git a/src/internal_events/aws_kinesis.rs b/src/internal_events/aws_kinesis.rs index 9fa2fb4315118..f8a7dcff49de2 100644 --- a/src/internal_events/aws_kinesis.rs +++ b/src/internal_events/aws_kinesis.rs @@ -1,8 +1,8 @@ -/// Used in both `aws_kinesis_streams` and `aws_kinesis_firehose` sinks -use metrics::counter; use vector_lib::NamedInternalEvent; +/// Used in both `aws_kinesis_streams` and `aws_kinesis_firehose` sinks +use vector_lib::counter; use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, error_stage, error_type, }; #[derive(Debug, NamedInternalEvent)] @@ -22,7 +22,7 @@ impl InternalEvent for AwsKinesisStreamNoPartitionKeyError<'_> { ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, ) diff --git a/src/internal_events/aws_kinesis_firehose.rs b/src/internal_events/aws_kinesis_firehose.rs index 6cb4ae43ea51f..58f976839debd 100644 --- a/src/internal_events/aws_kinesis_firehose.rs +++ b/src/internal_events/aws_kinesis_firehose.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; use super::prelude::{http_error_code, io_error_code}; use crate::sources::aws_kinesis_firehose::Compression; @@ -49,7 +50,7 @@ impl InternalEvent for AwsKinesisFirehoseRequestError<'_> { request_id = %self.request_id.unwrap_or(""), ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "stage" => error_stage::RECEIVING, "error_type" => error_type::REQUEST_FAILED, "error_code" => self.error_code, @@ -75,7 +76,7 @@ impl InternalEvent for AwsKinesisFirehoseAutomaticRecordDecodeError { compression = %self.compression, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "stage" => error_stage::PROCESSING, "error_type" => error_type::PARSER_FAILED, "error_code" => io_error_code(&self.error), diff --git a/src/internal_events/aws_sqs.rs b/src/internal_events/aws_sqs.rs index 4bdcc8ef48e6c..51e8b78d23941 100644 --- a/src/internal_events/aws_sqs.rs +++ b/src/internal_events/aws_sqs.rs @@ -1,11 +1,13 @@ #![allow(dead_code)] // TODO requires optional feature compilation -use metrics::counter; #[cfg(feature = "sources-aws_s3")] pub use s3::*; +use vector_lib::counter; #[cfg(any(feature = "sources-aws_s3", feature = "sources-aws_sqs"))] -use vector_lib::internal_event::{error_stage, error_type}; -use vector_lib::{NamedInternalEvent, internal_event::InternalEvent}; +use vector_lib::{ + NamedInternalEvent, + internal_event::{CounterName, HistogramName, InternalEvent, error_stage, error_type}, +}; #[cfg(feature = "sources-aws_s3")] mod s3 { @@ -15,7 +17,7 @@ mod s3 { BatchResultErrorEntry, DeleteMessageBatchRequestEntry, DeleteMessageBatchResultEntry, SendMessageBatchRequestEntry, SendMessageBatchResultEntry, }; - use metrics::histogram; + use vector_lib::histogram; use super::*; use crate::sources::aws_s3::sqs::ProcessingError; @@ -34,7 +36,7 @@ mod s3 { duration_ms = %self.duration.as_millis(), ); histogram!( - "s3_object_processing_succeeded_duration_seconds", + HistogramName::S3ObjectProcessingSucceededDurationSeconds, "bucket" => self.bucket.to_owned(), ) .record(self.duration); @@ -55,7 +57,7 @@ mod s3 { duration_ms = %self.duration.as_millis(), ); histogram!( - "s3_object_processing_failed_duration_seconds", + HistogramName::S3ObjectProcessingFailedDurationSeconds, "bucket" => self.bucket.to_owned(), ) .record(self.duration); @@ -79,7 +81,7 @@ mod s3 { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "failed_processing_sqs_message", "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, @@ -100,7 +102,8 @@ mod s3 { .map(|x| x.id.as_str()) .collect::>() .join(", ")); - counter!("sqs_message_delete_succeeded_total").increment(self.message_ids.len() as u64); + counter!(CounterName::SqsMessageDeleteSucceededTotal) + .increment(self.message_ids.len() as u64); } } @@ -122,7 +125,7 @@ mod s3 { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "failed_deleting_some_sqs_messages", "error_type" => error_type::ACKNOWLEDGMENT_FAILED, "stage" => error_stage::PROCESSING, @@ -151,7 +154,7 @@ mod s3 { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "failed_deleting_all_sqs_messages", "error_type" => error_type::ACKNOWLEDGMENT_FAILED, "stage" => error_stage::PROCESSING, @@ -172,7 +175,8 @@ mod s3 { .map(|x| x.id.as_str()) .collect::>() .join(", ")); - counter!("sqs_message_defer_succeeded_total").increment(self.message_ids.len() as u64); + counter!(CounterName::SqsMessageDeferSucceededTotal) + .increment(self.message_ids.len() as u64); } } @@ -194,7 +198,7 @@ mod s3 { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "failed_deferring_some_sqs_messages", "error_type" => error_type::ACKNOWLEDGMENT_FAILED, "stage" => error_stage::PROCESSING, @@ -223,7 +227,7 @@ mod s3 { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "failed_deferring_all_sqs_messages", "error_type" => error_type::ACKNOWLEDGMENT_FAILED, "stage" => error_stage::PROCESSING, @@ -248,7 +252,7 @@ impl InternalEvent for SqsMessageReceiveError<'_, E> { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "failed_fetching_sqs_events", "error_type" => error_type::REQUEST_FAILED, "stage" => error_stage::RECEIVING, @@ -265,8 +269,8 @@ pub struct SqsMessageReceiveSucceeded { impl InternalEvent for SqsMessageReceiveSucceeded { fn emit(self) { trace!(message = "Received SQS messages.", count = %self.count); - counter!("sqs_message_receive_succeeded_total").increment(1); - counter!("sqs_message_received_messages_total").increment(self.count as u64); + counter!(CounterName::SqsMessageReceiveSucceededTotal).increment(1); + counter!(CounterName::SqsMessageReceivedMessagesTotal).increment(self.count as u64); } } @@ -278,7 +282,7 @@ pub struct SqsMessageProcessingSucceeded<'a> { impl InternalEvent for SqsMessageProcessingSucceeded<'_> { fn emit(self) { trace!(message = "Processed SQS message successfully.", message_id = %self.message_id); - counter!("sqs_message_processing_succeeded_total").increment(1); + counter!(CounterName::SqsMessageProcessingSucceededTotal).increment(1); } } @@ -300,7 +304,7 @@ impl InternalEvent for SqsMessageDeleteError<'_, E> { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::WRITER_FAILED, "stage" => error_stage::PROCESSING, ) @@ -322,7 +326,7 @@ impl InternalEvent for SqsS3EventRecordInvalidEventIgnored<'_> { fn emit(self) { warn!(message = "Ignored S3 record in SQS message for an event that was not ObjectCreated.", bucket = %self.bucket, key = %self.key, kind = %self.kind, name = %self.name); - counter!("sqs_s3_event_record_ignored_total", "ignore_type" => "invalid_event_kind") + counter!(CounterName::SqsS3EventRecordIgnoredTotal, "ignore_type" => "invalid_event_kind") .increment(1); } } diff --git a/src/internal_events/batch.rs b/src/internal_events/batch.rs index 8b7a4decfa770..bd553c1e9863d 100644 --- a/src/internal_events/batch.rs +++ b/src/internal_events/batch.rs @@ -1,8 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, error_stage, error_type, }; +use vector_lib::{NamedInternalEvent, counter}; #[derive(Debug, NamedInternalEvent)] pub struct LargeEventDroppedError { @@ -21,7 +20,7 @@ impl InternalEvent for LargeEventDroppedError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "oversized", "error_type" => error_type::CONDITION_FAILED, "stage" => error_stage::SENDING, diff --git a/src/internal_events/common.rs b/src/internal_events/common.rs index 69877e3db9ef5..5a6f9c50ede93 100644 --- a/src/internal_events/common.rs +++ b/src/internal_events/common.rs @@ -1,11 +1,12 @@ use std::time::Instant; -use metrics::{counter, histogram}; use vector_lib::NamedInternalEvent; pub use vector_lib::internal_event::EventsReceived; use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, HistogramName, InternalEvent, UNINTENTIONAL, error_stage, + error_type, }; +use vector_lib::{counter, histogram}; #[derive(Debug, NamedInternalEvent)] pub struct EndpointBytesReceived<'a> { @@ -23,7 +24,7 @@ impl InternalEvent for EndpointBytesReceived<'_> { endpoint = %self.endpoint, ); counter!( - "component_received_bytes_total", + CounterName::ComponentReceivedBytesTotal, "protocol" => self.protocol.to_owned(), "endpoint" => self.endpoint.to_owned(), ) @@ -47,7 +48,7 @@ impl InternalEvent for EndpointBytesSent<'_> { endpoint = %self.endpoint ); counter!( - "component_sent_bytes_total", + CounterName::ComponentSentBytesTotal, "protocol" => self.protocol.to_string(), "endpoint" => self.endpoint.to_string() ) @@ -70,7 +71,7 @@ impl InternalEvent for SocketOutgoingConnectionError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "failed_connecting", "error_type" => error_type::CONNECTION_FAILED, "stage" => error_stage::SENDING, @@ -95,7 +96,7 @@ impl InternalEvent for StreamClosedError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => STREAM_CLOSED, "error_type" => error_type::WRITER_FAILED, "stage" => error_stage::SENDING, @@ -117,8 +118,8 @@ pub struct CollectionCompleted { impl InternalEvent for CollectionCompleted { fn emit(self) { debug!(message = "Collection completed."); - counter!("collect_completed_total").increment(1); - histogram!("collect_duration_seconds").record(self.end - self.start); + counter!(CounterName::CollectCompletedTotal).increment(1); + histogram!(HistogramName::CollectDurationSeconds).record(self.end - self.start); } } @@ -139,7 +140,7 @@ impl InternalEvent for SinkRequestBuildError { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::ENCODER_FAILED, "stage" => error_stage::PROCESSING, ) diff --git a/src/internal_events/conditions.rs b/src/internal_events/conditions.rs index 96f7ae0ae0d50..e89f63b6a9c54 100644 --- a/src/internal_events/conditions.rs +++ b/src/internal_events/conditions.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; #[derive(Debug, Copy, Clone, NamedInternalEvent)] pub struct VrlConditionExecutionError<'a> { @@ -16,7 +17,7 @@ impl InternalEvent for VrlConditionExecutionError<'_> { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::SCRIPT_FAILED, "stage" => error_stage::PROCESSING, ) diff --git a/src/internal_events/datadog_agent.rs b/src/internal_events/datadog_agent.rs index 9ee5296407b24..84a4a8535c0c7 100644 --- a/src/internal_events/datadog_agent.rs +++ b/src/internal_events/datadog_agent.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; #[derive(Debug, NamedInternalEvent)] pub struct DatadogAgentJsonParseError<'a> { @@ -16,7 +17,7 @@ impl InternalEvent for DatadogAgentJsonParseError<'_> { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, ) diff --git a/src/internal_events/datadog_metrics.rs b/src/internal_events/datadog_metrics.rs index 5daa3ab87fa79..00e7d50a8a7d7 100644 --- a/src/internal_events/datadog_metrics.rs +++ b/src/internal_events/datadog_metrics.rs @@ -1,8 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, error_stage, error_type, }; +use vector_lib::{NamedInternalEvent, counter}; #[derive(Debug, NamedInternalEvent)] pub struct DatadogMetricsEncodingError<'a> { @@ -21,7 +20,7 @@ impl InternalEvent for DatadogMetricsEncodingError<'_> { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => self.error_code, "error_type" => error_type::ENCODER_FAILED, "stage" => error_stage::PROCESSING, diff --git a/src/internal_events/datadog_traces.rs b/src/internal_events/datadog_traces.rs index 06b48552fb840..016002681f81b 100644 --- a/src/internal_events/datadog_traces.rs +++ b/src/internal_events/datadog_traces.rs @@ -1,8 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, error_stage, error_type, }; +use vector_lib::{NamedInternalEvent, counter}; #[derive(Debug, NamedInternalEvent)] pub struct DatadogTracesEncodingError { @@ -22,7 +21,7 @@ impl InternalEvent for DatadogTracesEncodingError { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::ENCODER_FAILED, "stage" => error_stage::PROCESSING, ) @@ -51,7 +50,7 @@ impl InternalEvent for DatadogTracesAPMStatsError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::WRITER_FAILED, "stage" => error_stage::SENDING, ) diff --git a/src/internal_events/dedupe.rs b/src/internal_events/dedupe.rs index eb7511578cdd7..a1c36a3c4fa2b 100644 --- a/src/internal_events/dedupe.rs +++ b/src/internal_events/dedupe.rs @@ -1,5 +1,7 @@ -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{ComponentEventsDropped, INTENTIONAL, InternalEvent}; +use vector_lib::{ + NamedInternalEvent, + internal_event::{ComponentEventsDropped, INTENTIONAL, InternalEvent}, +}; #[derive(Debug, NamedInternalEvent)] pub struct DedupeEventsDropped { diff --git a/src/internal_events/demo_logs.rs b/src/internal_events/demo_logs.rs index 406a9856fe6e9..3958f41421ca7 100644 --- a/src/internal_events/demo_logs.rs +++ b/src/internal_events/demo_logs.rs @@ -1,5 +1,4 @@ -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::InternalEvent; +use vector_lib::{NamedInternalEvent, internal_event::InternalEvent}; #[derive(Debug, NamedInternalEvent)] pub struct DemoLogsEventProcessed; diff --git a/src/internal_events/dnstap.rs b/src/internal_events/dnstap.rs index 2fc20b95848d1..1ef67ee3001e0 100644 --- a/src/internal_events/dnstap.rs +++ b/src/internal_events/dnstap.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; #[derive(Debug, NamedInternalEvent)] pub(crate) struct DnstapParseError { @@ -16,7 +17,7 @@ impl InternalEvent for DnstapParseError { error_type = error_type::PARSER_FAILED, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "stage" => error_stage::PROCESSING, "error_type" => error_type::PARSER_FAILED, ) diff --git a/src/internal_events/docker_logs.rs b/src/internal_events/docker_logs.rs index bb44e112e05b8..fe24d41b6350b 100644 --- a/src/internal_events/docker_logs.rs +++ b/src/internal_events/docker_logs.rs @@ -1,9 +1,8 @@ use bollard::errors::Error; use chrono::ParseError; -use metrics::counter; use vector_lib::{ - NamedInternalEvent, - internal_event::{InternalEvent, error_stage, error_type}, + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, json_size::JsonSize, }; @@ -23,11 +22,11 @@ impl InternalEvent for DockerLogsEventsReceived<'_> { container_id = %self.container_id ); counter!( - "component_received_events_total", "container_name" => self.container_name.to_owned() + CounterName::ComponentReceivedEventsTotal, "container_name" => self.container_name.to_owned() ) .increment(1); counter!( - "component_received_event_bytes_total", "container_name" => self.container_name.to_owned() + CounterName::ComponentReceivedEventBytesTotal, "container_name" => self.container_name.to_owned() ).increment(self.byte_size.get() as u64); } } @@ -45,7 +44,7 @@ impl InternalEvent for DockerLogsContainerEventReceived<'_> { container_id = %self.container_id, action = %self.action, ); - counter!("container_processed_events_total").increment(1); + counter!(CounterName::ContainerProcessedEventsTotal).increment(1); } } @@ -60,7 +59,7 @@ impl InternalEvent for DockerLogsContainerWatch<'_> { message = "Started watching for container logs.", container_id = %self.container_id, ); - counter!("containers_watched_total").increment(1); + counter!(CounterName::ContainersWatchedTotal).increment(1); } } @@ -75,7 +74,7 @@ impl InternalEvent for DockerLogsContainerUnwatch<'_> { message = "Stopped watching for container logs.", container_id = %self.container_id, ); - counter!("containers_unwatched_total").increment(1); + counter!(CounterName::ContainersUnwatchedTotal).increment(1); } } @@ -95,7 +94,7 @@ impl InternalEvent for DockerLogsCommunicationError<'_> { container_id = ?self.container_id ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::CONNECTION_FAILED, "stage" => error_stage::RECEIVING, ) @@ -119,7 +118,7 @@ impl InternalEvent for DockerLogsContainerMetadataFetchError<'_> { container_id = ?self.container_id ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::REQUEST_FAILED, "stage" => error_stage::RECEIVING, "container_id" => self.container_id.to_owned(), @@ -144,7 +143,7 @@ impl InternalEvent for DockerLogsTimestampParseError<'_> { container_id = ?self.container_id ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, "container_id" => self.container_id.to_owned(), @@ -169,7 +168,7 @@ impl InternalEvent for DockerLogsLoggingDriverUnsupportedError<'_> { container_id = ?self.container_id, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::CONFIGURATION_FAILED, "stage" => error_stage::RECEIVING, "container_id" => self.container_id.to_owned(), diff --git a/src/internal_events/doris.rs b/src/internal_events/doris.rs index 0a2430869b7c5..15dffd26418b4 100644 --- a/src/internal_events/doris.rs +++ b/src/internal_events/doris.rs @@ -1,5 +1,7 @@ -use metrics::counter; -use vector_lib::{NamedInternalEvent, internal_event::InternalEvent}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent}, +}; /// Emitted when rows are successfully loaded into Doris. #[derive(Debug, NamedInternalEvent)] @@ -17,10 +19,10 @@ impl InternalEvent for DorisRowsLoaded { ); // Record the number of rows loaded - counter!("doris_rows_loaded_total").increment(self.loaded_rows as u64); + counter!(CounterName::DorisRowsLoadedTotal).increment(self.loaded_rows as u64); // Record the number of bytes loaded - counter!("doris_bytes_loaded_total").increment(self.load_bytes as u64); + counter!(CounterName::DorisBytesLoadedTotal).increment(self.load_bytes as u64); } } @@ -37,6 +39,6 @@ impl InternalEvent for DorisRowsFiltered { filtered_rows = %self.filtered_rows ); - counter!("doris_rows_filtered_total").increment(self.filtered_rows as u64); + counter!(CounterName::DorisRowsFilteredTotal).increment(self.filtered_rows as u64); } } diff --git a/src/internal_events/encoding_transcode.rs b/src/internal_events/encoding_transcode.rs index 76ad81792a63a..ec6280f1c703d 100644 --- a/src/internal_events/encoding_transcode.rs +++ b/src/internal_events/encoding_transcode.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::InternalEvent; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent}, +}; #[derive(Debug, NamedInternalEvent)] pub struct DecoderBomRemoval { @@ -13,7 +14,7 @@ impl InternalEvent for DecoderBomRemoval { message = "Removing initial BOM bytes from the final output while decoding to utf8.", from_encoding = %self.from_encoding ); - counter!("decoder_bom_removals_total").increment(1); + counter!(CounterName::DecoderBomRemovalsTotal).increment(1); } } @@ -30,7 +31,7 @@ impl InternalEvent for DecoderMalformedReplacement { ); // NOT the actual number of replacements in the output: there's no easy // way to get that from the lib we use here (encoding_rs) - counter!("decoder_malformed_replacement_warnings_total").increment(1); + counter!(CounterName::DecoderMalformedReplacementWarningsTotal).increment(1); } } @@ -47,6 +48,6 @@ impl InternalEvent for EncoderUnmappableReplacement { ); // NOT the actual number of replacements in the output: there's no easy // way to get that from the lib we use here (encoding_rs) - counter!("encoder_unmappable_replacement_warnings_total").increment(1); + counter!(CounterName::EncoderUnmappableReplacementWarningsTotal).increment(1); } } diff --git a/src/internal_events/eventstoredb_metrics.rs b/src/internal_events/eventstoredb_metrics.rs index c72b9c7dbeb7c..c7db7fad354a5 100644 --- a/src/internal_events/eventstoredb_metrics.rs +++ b/src/internal_events/eventstoredb_metrics.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; #[derive(Debug, NamedInternalEvent)] pub struct EventStoreDbMetricsHttpError { @@ -16,7 +17,7 @@ impl InternalEvent for EventStoreDbMetricsHttpError { error_type = error_type::REQUEST_FAILED, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "stage" => error_stage::RECEIVING, "error_type" => error_type::REQUEST_FAILED, ) @@ -38,7 +39,7 @@ impl InternalEvent for EventStoreDbStatsParsingError { error_type = error_type::PARSER_FAILED, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "stage" => error_stage::PROCESSING, "error_type" => error_type::PARSER_FAILED, ) diff --git a/src/internal_events/exec.rs b/src/internal_events/exec.rs index abe226897d5d3..21edc68f9df2a 100644 --- a/src/internal_events/exec.rs +++ b/src/internal_events/exec.rs @@ -1,11 +1,11 @@ use std::time::Duration; -use metrics::{counter, histogram}; use tokio::time::error::Elapsed; use vector_lib::{ - NamedInternalEvent, + NamedInternalEvent, counter, histogram, internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, HistogramName, InternalEvent, UNINTENTIONAL, + error_stage, error_type, }, json_size::JsonSize, }; @@ -28,12 +28,12 @@ impl InternalEvent for ExecEventsReceived<'_> { command = %self.command, ); counter!( - "component_received_events_total", + CounterName::ComponentReceivedEventsTotal, "command" => self.command.to_owned(), ) .increment(self.count as u64); counter!( - "component_received_event_bytes_total", + CounterName::ComponentReceivedEventBytesTotal, "command" => self.command.to_owned(), ) .increment(self.byte_size.get() as u64); @@ -57,7 +57,7 @@ impl InternalEvent for ExecFailedError<'_> { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "command" => self.command.to_owned(), "error_type" => error_type::COMMAND_FAILED, "error_code" => io_error_code(&self.error), @@ -85,7 +85,7 @@ impl InternalEvent for ExecTimeoutError<'_> { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "command" => self.command.to_owned(), "error_type" => error_type::TIMED_OUT, "stage" => error_stage::RECEIVING, @@ -120,14 +120,14 @@ impl InternalEvent for ExecCommandExecuted<'_> { elapsed_millis = %self.exec_duration.as_millis(), ); counter!( - "command_executed_total", + CounterName::CommandExecutedTotal, "command" => self.command.to_owned(), "exit_status" => exit_status.clone(), ) .increment(1); histogram!( - "command_execution_duration_seconds", + HistogramName::CommandExecutionDurationSeconds, "exit_status" => exit_status, "command" => self.command.to_owned(), ) @@ -196,7 +196,7 @@ impl InternalEvent for ExecFailedToSignalChildError<'_> { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "command" => format!("{:?}", self.command.as_std()), "error_code" => self.error.to_error_code(), "error_type" => error_type::COMMAND_FAILED, @@ -218,7 +218,7 @@ impl InternalEvent for ExecChannelClosedError { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::COMMAND_FAILED, "stage" => error_stage::RECEIVING, ) diff --git a/src/internal_events/expansion.rs b/src/internal_events/expansion.rs index 7b2633dd64f97..7f5b48914c69c 100644 --- a/src/internal_events/expansion.rs +++ b/src/internal_events/expansion.rs @@ -1,8 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, error_stage, error_type, }; +use vector_lib::{NamedInternalEvent, counter}; #[derive(NamedInternalEvent)] pub struct PairExpansionError<'a> { @@ -25,7 +24,7 @@ impl InternalEvent for PairExpansionError<'_> { ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, ) diff --git a/src/internal_events/file.rs b/src/internal_events/file.rs index 0e85c67150621..f1299ae751694 100644 --- a/src/internal_events/file.rs +++ b/src/internal_events/file.rs @@ -2,12 +2,13 @@ use std::borrow::Cow; -use metrics::{counter, gauge}; use vector_lib::{ NamedInternalEvent, configurable::configurable_component, + counter, gauge, internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, GaugeName, InternalEvent, UNINTENTIONAL, error_stage, + error_type, }, }; @@ -34,7 +35,7 @@ pub struct FileOpen { impl InternalEvent for FileOpen { fn emit(self) { - gauge!("open_files").set(self.count as f64); + gauge!(GaugeName::OpenFiles).set(self.count as f64); } } @@ -55,13 +56,13 @@ impl InternalEvent for FileBytesSent<'_> { ); if self.include_file_metric_tag { counter!( - "component_sent_bytes_total", + CounterName::ComponentSentBytesTotal, "protocol" => "file", "file" => self.file.clone().into_owned(), ) } else { counter!( - "component_sent_bytes_total", + CounterName::ComponentSentBytesTotal, "protocol" => "file", ) } @@ -89,7 +90,7 @@ impl InternalEvent for FileIoError<'_, P> { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => self.code, "error_type" => error_type::IO_FAILED, "stage" => error_stage::SENDING, @@ -110,11 +111,12 @@ mod source { use std::{io::Error, path::Path, time::Duration}; use bytes::BytesMut; - use metrics::counter; use vector_lib::{ - NamedInternalEvent, emit, + NamedInternalEvent, counter, emit, file_source_common::internal_events::FileSourceInternalEvents, - internal_event::{ComponentEventsDropped, INTENTIONAL, error_stage, error_type}, + internal_event::{ + ComponentEventsDropped, CounterName, INTENTIONAL, error_stage, error_type, + }, json_size::JsonSize, }; @@ -137,13 +139,13 @@ mod source { ); if self.include_file_metric_tag { counter!( - "component_received_bytes_total", + CounterName::ComponentReceivedBytesTotal, "protocol" => "file", "file" => self.file.to_owned() ) } else { counter!( - "component_received_bytes_total", + CounterName::ComponentReceivedBytesTotal, "protocol" => "file", ) } @@ -169,18 +171,18 @@ mod source { ); if self.include_file_metric_tag { counter!( - "component_received_events_total", + CounterName::ComponentReceivedEventsTotal, "file" => self.file.to_owned(), ) .increment(self.count as u64); counter!( - "component_received_event_bytes_total", + CounterName::ComponentReceivedEventBytesTotal, "file" => self.file.to_owned(), ) .increment(self.byte_size.get() as u64); } else { - counter!("component_received_events_total").increment(self.count as u64); - counter!("component_received_event_bytes_total") + counter!(CounterName::ComponentReceivedEventsTotal).increment(self.count as u64); + counter!(CounterName::ComponentReceivedEventBytesTotal) .increment(self.byte_size.get() as u64); } } @@ -200,11 +202,11 @@ mod source { ); if self.include_file_metric_tag { counter!( - "checksum_errors_total", + CounterName::ChecksumErrorsTotal, "file" => self.file.to_string_lossy().into_owned(), ) } else { - counter!("checksum_errors_total") + counter!(CounterName::ChecksumErrorsTotal) } .increment(1); } @@ -229,7 +231,7 @@ mod source { ); if self.include_file_metric_tag { counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "reading_fingerprint", "error_type" => error_type::READER_FAILED, "stage" => error_stage::RECEIVING, @@ -237,7 +239,7 @@ mod source { ) } else { counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "reading_fingerprint", "error_type" => error_type::READER_FAILED, "stage" => error_stage::RECEIVING, @@ -268,7 +270,7 @@ mod source { ); if self.include_file_metric_tag { counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "file" => self.file.to_string_lossy().into_owned(), "error_code" => DELETION_FAILED, "error_type" => error_type::COMMAND_FAILED, @@ -276,7 +278,7 @@ mod source { ) } else { counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => DELETION_FAILED, "error_type" => error_type::COMMAND_FAILED, "stage" => error_stage::RECEIVING, @@ -300,11 +302,11 @@ mod source { ); if self.include_file_metric_tag { counter!( - "files_deleted_total", + CounterName::FilesDeletedTotal, "file" => self.file.to_string_lossy().into_owned(), ) } else { - counter!("files_deleted_total") + counter!(CounterName::FilesDeletedTotal) } .increment(1); } @@ -327,13 +329,13 @@ mod source { ); if self.include_file_metric_tag { counter!( - "files_unwatched_total", + CounterName::FilesUnwatchedTotal, "file" => self.file.to_string_lossy().into_owned(), "reached_eof" => reached_eof, ) } else { counter!( - "files_unwatched_total", + CounterName::FilesUnwatchedTotal, "reached_eof" => reached_eof, ) } @@ -360,7 +362,7 @@ mod source { ); if self.include_file_metric_tag { counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "watching", "error_type" => error_type::COMMAND_FAILED, "stage" => error_stage::RECEIVING, @@ -368,7 +370,7 @@ mod source { ) } else { counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "watching", "error_type" => error_type::COMMAND_FAILED, "stage" => error_stage::RECEIVING, @@ -394,11 +396,11 @@ mod source { ); if self.include_file_metric_tag { counter!( - "files_resumed_total", + CounterName::FilesResumedTotal, "file" => self.file.to_string_lossy().into_owned(), ) } else { - counter!("files_resumed_total") + counter!(CounterName::FilesResumedTotal) } .increment(1); } @@ -418,11 +420,11 @@ mod source { ); if self.include_file_metric_tag { counter!( - "files_added_total", + CounterName::FilesAddedTotal, "file" => self.file.to_string_lossy().into_owned(), ) } else { - counter!("files_added_total") + counter!(CounterName::FilesAddedTotal) } .increment(1); } @@ -441,7 +443,7 @@ mod source { count = %self.count, duration_ms = self.duration.as_millis() as u64, ); - counter!("checkpoints_total").increment(self.count as u64); + counter!(CounterName::CheckpointsTotal).increment(self.count as u64); } } @@ -460,7 +462,7 @@ mod source { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "writing_checkpoints", "error_type" => error_type::WRITER_FAILED, "stage" => error_stage::RECEIVING, @@ -486,7 +488,7 @@ mod source { path = %self.path.display(), ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "globbing", "error_type" => error_type::READER_FAILED, "stage" => error_stage::RECEIVING, @@ -513,7 +515,7 @@ mod source { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "reading_line_from_file", "error_type" => error_type::CONDITION_FAILED, "stage" => error_stage::RECEIVING, diff --git a/src/internal_events/file_descriptor.rs b/src/internal_events/file_descriptor.rs index 7882ed4d95b12..8db4e7895729e 100644 --- a/src/internal_events/file_descriptor.rs +++ b/src/internal_events/file_descriptor.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; #[derive(Debug, NamedInternalEvent)] pub struct FileDescriptorReadError { @@ -19,7 +20,7 @@ where stage = error_stage::RECEIVING ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::CONNECTION_FAILED, "stage" => error_stage::RECEIVING, ) diff --git a/src/internal_events/fluent.rs b/src/internal_events/fluent.rs index 6d6bb408edc5f..f47b927d5cbe2 100644 --- a/src/internal_events/fluent.rs +++ b/src/internal_events/fluent.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; use crate::sources::fluent::DecodeError; @@ -31,7 +32,7 @@ impl InternalEvent for FluentMessageDecodeError<'_> { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, ) diff --git a/src/internal_events/gcp_pubsub.rs b/src/internal_events/gcp_pubsub.rs index 4066d71936529..752546cfa7d09 100644 --- a/src/internal_events/gcp_pubsub.rs +++ b/src/internal_events/gcp_pubsub.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; #[derive(NamedInternalEvent)] pub struct GcpPubsubConnectError { @@ -18,7 +19,7 @@ impl InternalEvent for GcpPubsubConnectError { ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "failed_connecting", "error_type" => error_type::CONNECTION_FAILED, "stage" => error_stage::RECEIVING, @@ -43,7 +44,7 @@ impl InternalEvent for GcpPubsubStreamingPullError { ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "failed_streaming_pull", "error_type" => error_type::REQUEST_FAILED, "stage" => error_stage::RECEIVING, @@ -68,7 +69,7 @@ impl InternalEvent for GcpPubsubReceiveError { ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "failed_fetching_events", "error_type" => error_type::REQUEST_FAILED, "stage" => error_stage::RECEIVING, diff --git a/src/internal_events/grpc.rs b/src/internal_events/grpc.rs index 882b1dbfaf33b..66da41e10b797 100644 --- a/src/internal_events/grpc.rs +++ b/src/internal_events/grpc.rs @@ -1,10 +1,11 @@ use std::time::Duration; use http::response::Response; -use metrics::{counter, histogram}; use tonic::Code; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, histogram, + internal_event::{CounterName, HistogramName, InternalEvent, error_stage, error_type}, +}; const GRPC_STATUS_LABEL: &str = "grpc_status"; @@ -13,7 +14,7 @@ pub struct GrpcServerRequestReceived; impl InternalEvent for GrpcServerRequestReceived { fn emit(self) { - counter!("grpc_server_messages_received_total").increment(1); + counter!(CounterName::GrpcServerMessagesReceivedTotal).increment(1); } } @@ -34,8 +35,8 @@ impl InternalEvent for GrpcServerResponseSent<'_, B> { let grpc_code = grpc_code_to_name(grpc_code); let labels = &[(GRPC_STATUS_LABEL, grpc_code)]; - counter!("grpc_server_messages_sent_total", labels).increment(1); - histogram!("grpc_server_handler_duration_seconds", labels).record(self.latency); + counter!(CounterName::GrpcServerMessagesSentTotal, labels).increment(1); + histogram!(HistogramName::GrpcServerHandlerDurationSeconds, labels).record(self.latency); } } @@ -53,7 +54,7 @@ impl InternalEvent for GrpcInvalidCompressionSchemeError<'_> { stage = error_stage::RECEIVING ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::REQUEST_FAILED, "stage" => error_stage::RECEIVING, ) @@ -78,7 +79,7 @@ where stage = error_stage::RECEIVING ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::REQUEST_FAILED, "stage" => error_stage::RECEIVING, ) diff --git a/src/internal_events/heartbeat.rs b/src/internal_events/heartbeat.rs index 658a24fcb8762..16191669e7317 100644 --- a/src/internal_events/heartbeat.rs +++ b/src/internal_events/heartbeat.rs @@ -1,8 +1,9 @@ use std::time::Instant; -use metrics::gauge; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::InternalEvent; +use vector_lib::{ + NamedInternalEvent, gauge, + internal_event::{GaugeName, InternalEvent}, +}; use crate::built_info; @@ -14,9 +15,9 @@ pub struct Heartbeat { impl InternalEvent for Heartbeat { fn emit(self) { trace!(target: "vector", message = "Beep."); - gauge!("uptime_seconds").set(self.since.elapsed().as_secs() as f64); + gauge!(GaugeName::UptimeSeconds).set(self.since.elapsed().as_secs() as f64); gauge!( - "build_info", + GaugeName::BuildInfo, "debug" => built_info::DEBUG, "version" => built_info::PKG_VERSION, "rust_version" => built_info::RUST_VERSION, diff --git a/src/internal_events/host_metrics.rs b/src/internal_events/host_metrics.rs index d60771644571e..7662ca5e808a4 100644 --- a/src/internal_events/host_metrics.rs +++ b/src/internal_events/host_metrics.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; #[derive(Debug, NamedInternalEvent)] pub struct HostMetricsScrapeError { @@ -16,7 +17,7 @@ impl InternalEvent for HostMetricsScrapeError { ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::READER_FAILED, "stage" => error_stage::RECEIVING, ) @@ -40,7 +41,7 @@ impl InternalEvent for HostMetricsScrapeDetailError { ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::READER_FAILED, "stage" => error_stage::RECEIVING, ) @@ -66,7 +67,7 @@ impl InternalEvent for HostMetricsScrapeFilesystemError { ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::READER_FAILED, "stage" => error_stage::RECEIVING, ) diff --git a/src/internal_events/http.rs b/src/internal_events/http.rs index a0d8f5d2a92d3..4aad267d7683b 100644 --- a/src/internal_events/http.rs +++ b/src/internal_events/http.rs @@ -1,10 +1,9 @@ use std::{error::Error, time::Duration}; use http::Response; -use metrics::{counter, histogram}; use vector_lib::{ - NamedInternalEvent, - internal_event::{InternalEvent, error_stage, error_type}, + NamedInternalEvent, counter, histogram, + internal_event::{CounterName, HistogramName, InternalEvent, error_stage, error_type}, json_size::JsonSize, }; @@ -16,7 +15,7 @@ pub struct HttpServerRequestReceived; impl InternalEvent for HttpServerRequestReceived { fn emit(self) { debug!(message = "Received HTTP request."); - counter!("http_server_requests_received_total").increment(1); + counter!(CounterName::HttpServerRequestsReceivedTotal).increment(1); } } @@ -32,8 +31,8 @@ impl InternalEvent for HttpServerResponseSent<'_, B> { HTTP_STATUS_LABEL, self.response.status().as_u16().to_string(), )]; - counter!("http_server_responses_sent_total", labels).increment(1); - histogram!("http_server_handler_duration_seconds", labels).record(self.latency); + counter!(CounterName::HttpServerResponsesSentTotal, labels).increment(1); + histogram!(HistogramName::HttpServerHandlerDurationSeconds, labels).record(self.latency); } } @@ -53,7 +52,7 @@ impl InternalEvent for HttpBytesReceived<'_> { protocol = %self.protocol ); counter!( - "component_received_bytes_total", + CounterName::ComponentReceivedBytesTotal, "http_path" => self.http_path.to_string(), "protocol" => self.protocol, ) @@ -79,15 +78,15 @@ impl InternalEvent for HttpEventsReceived<'_> { protocol = %self.protocol, ); - histogram!("component_received_events_count").record(self.count as f64); + histogram!(HistogramName::ComponentReceivedEventsCount).record(self.count as f64); counter!( - "component_received_events_total", + CounterName::ComponentReceivedEventsTotal, "http_path" => self.http_path.to_string(), "protocol" => self.protocol, ) .increment(self.count as u64); counter!( - "component_received_event_bytes_total", + CounterName::ComponentReceivedEventBytesTotal, "http_path" => self.http_path.to_string(), "protocol" => self.protocol, ) @@ -126,7 +125,7 @@ impl InternalEvent for HttpBadRequest<'_> { http_code = %self.code, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => self.error_code, "error_type" => error_type::REQUEST_FAILED, "error_stage" => error_stage::RECEIVING, @@ -152,7 +151,7 @@ impl InternalEvent for HttpDecompressError<'_> { encoding = %self.encoding ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "failed_decompressing_payload", "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::RECEIVING, @@ -174,7 +173,7 @@ impl InternalEvent for HttpInternalError<'_> { stage = error_stage::RECEIVING ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::CONNECTION_FAILED, "stage" => error_stage::RECEIVING, ) diff --git a/src/internal_events/http_client.rs b/src/internal_events/http_client.rs index d1e4c7549d15a..346192dfca141 100644 --- a/src/internal_events/http_client.rs +++ b/src/internal_events/http_client.rs @@ -5,9 +5,10 @@ use http::{ header::{self, HeaderMap, HeaderValue}, }; use hyper::{Error, body::HttpBody}; -use metrics::{counter, histogram}; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, histogram, + internal_event::{CounterName, HistogramName, InternalEvent, error_stage, error_type}, +}; #[derive(Debug, NamedInternalEvent)] pub struct AboutToSendHttpRequest<'a, T> { @@ -39,7 +40,7 @@ impl InternalEvent for AboutToSendHttpRequest<'_, T> { headers = ?remove_sensitive(self.request.headers()), body = %FormatBody(self.request.body()), ); - counter!("http_client_requests_sent_total", "method" => self.request.method().to_string()) + counter!(CounterName::HttpClientRequestsSentTotal, "method" => self.request.method().to_string()) .increment(1); } } @@ -60,13 +61,13 @@ impl InternalEvent for GotHttpResponse<'_, T> { body = %FormatBody(self.response.body()), ); counter!( - "http_client_responses_total", + CounterName::HttpClientResponsesTotal, "status" => self.response.status().as_u16().to_string(), ) .increment(1); - histogram!("http_client_rtt_seconds").record(self.roundtrip); + histogram!(HistogramName::HttpClientRttSeconds).record(self.roundtrip); histogram!( - "http_client_response_rtt_seconds", + HistogramName::HttpClientResponseRttSeconds, "status" => self.response.status().as_u16().to_string(), ) .record(self.roundtrip); @@ -87,9 +88,10 @@ impl InternalEvent for GotHttpWarning<'_> { error_type = error_type::REQUEST_FAILED, stage = error_stage::PROCESSING, ); - counter!("http_client_errors_total", "error_kind" => self.error.to_string()).increment(1); - histogram!("http_client_rtt_seconds").record(self.roundtrip); - histogram!("http_client_error_rtt_seconds", "error_kind" => self.error.to_string()) + counter!(CounterName::HttpClientErrorsTotal, "error_kind" => self.error.to_string()) + .increment(1); + histogram!(HistogramName::HttpClientRttSeconds).record(self.roundtrip); + histogram!(HistogramName::HttpClientErrorRttSeconds, "error_kind" => self.error.to_string()) .record(self.roundtrip); } } diff --git a/src/internal_events/http_client_source.rs b/src/internal_events/http_client_source.rs index ff67535c0c14b..0ba52d722dfee 100644 --- a/src/internal_events/http_client_source.rs +++ b/src/internal_events/http_client_source.rs @@ -1,9 +1,8 @@ #![allow(dead_code)] // TODO requires optional feature compilation -use metrics::counter; use vector_lib::{ - NamedInternalEvent, - internal_event::{InternalEvent, error_stage, error_type}, + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, json_size::JsonSize, }; @@ -25,12 +24,12 @@ impl InternalEvent for HttpClientEventsReceived { url = %self.url, ); counter!( - "component_received_events_total", + CounterName::ComponentReceivedEventsTotal, "uri" => self.url.clone(), ) .increment(self.count as u64); counter!( - "component_received_event_bytes_total", + CounterName::ComponentReceivedEventBytesTotal, "uri" => self.url.clone(), ) .increment(self.byte_size.get() as u64); @@ -53,7 +52,7 @@ impl InternalEvent for HttpClientHttpResponseError { error_code = %http_error_code(self.code.as_u16()), ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "url" => self.url, "stage" => error_stage::RECEIVING, "error_type" => error_type::REQUEST_FAILED, @@ -79,7 +78,7 @@ impl InternalEvent for HttpClientHttpError { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "url" => self.url, "error_type" => error_type::REQUEST_FAILED, "stage" => error_stage::RECEIVING, diff --git a/src/internal_events/influxdb.rs b/src/internal_events/influxdb.rs index 22b34c3dc532c..a142642140f6f 100644 --- a/src/internal_events/influxdb.rs +++ b/src/internal_events/influxdb.rs @@ -1,8 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, error_stage, error_type, }; +use vector_lib::{NamedInternalEvent, counter}; #[derive(Debug, NamedInternalEvent)] pub struct InfluxdbEncodingError { @@ -20,7 +19,7 @@ impl InternalEvent for InfluxdbEncodingError { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::ENCODER_FAILED, "stage" => error_stage::PROCESSING, ) diff --git a/src/internal_events/internal_logs.rs b/src/internal_events/internal_logs.rs index 63c520723c9c7..2f0674d380edd 100644 --- a/src/internal_events/internal_logs.rs +++ b/src/internal_events/internal_logs.rs @@ -1,5 +1,8 @@ -use metrics::counter; -use vector_lib::{NamedInternalEvent, internal_event::InternalEvent, json_size::JsonSize}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent}, + json_size::JsonSize, +}; #[derive(Debug, NamedInternalEvent)] pub struct InternalLogsBytesReceived { @@ -10,7 +13,7 @@ impl InternalEvent for InternalLogsBytesReceived { fn emit(self) { // MUST NOT emit logs here to avoid an infinite log loop counter!( - "component_received_bytes_total", + CounterName::ComponentReceivedBytesTotal, "protocol" => "internal", ) .increment(self.byte_size as u64); @@ -26,7 +29,8 @@ pub struct InternalLogsEventsReceived { impl InternalEvent for InternalLogsEventsReceived { fn emit(self) { // MUST NOT emit logs here to avoid an infinite log loop - counter!("component_received_events_total").increment(self.count as u64); - counter!("component_received_event_bytes_total").increment(self.byte_size.get() as u64); + counter!(CounterName::ComponentReceivedEventsTotal).increment(self.count as u64); + counter!(CounterName::ComponentReceivedEventBytesTotal) + .increment(self.byte_size.get() as u64); } } diff --git a/src/internal_events/journald.rs b/src/internal_events/journald.rs index 0debba0de5d29..02f6300cb6905 100644 --- a/src/internal_events/journald.rs +++ b/src/internal_events/journald.rs @@ -1,8 +1,8 @@ -use metrics::counter; use vector_lib::{ NamedInternalEvent, codecs::decoding::BoxedFramingError, - internal_event::{InternalEvent, error_stage, error_type}, + counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, }; #[derive(Debug, NamedInternalEvent)] @@ -21,7 +21,7 @@ impl InternalEvent for JournaldInvalidRecordError { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "stage" => error_stage::PROCESSING, "error_type" => error_type::PARSER_FAILED, ) @@ -43,7 +43,7 @@ impl InternalEvent for JournaldStartJournalctlError { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "stage" => error_stage::RECEIVING, "error_type" => error_type::COMMAND_FAILED, ) @@ -65,7 +65,7 @@ impl InternalEvent for JournaldReadError { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "stage" => error_stage::PROCESSING, "error_type" => error_type::READER_FAILED, ) @@ -89,7 +89,7 @@ impl InternalEvent for JournaldCheckpointSetError { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "stage" => error_stage::PROCESSING, "error_type" => error_type::IO_FAILED, ) @@ -113,7 +113,7 @@ impl InternalEvent for JournaldCheckpointFileOpenError { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "stage" => error_stage::RECEIVING, "error_type" => error_type::IO_FAILED, ) diff --git a/src/internal_events/kafka.rs b/src/internal_events/kafka.rs index 87ad581e1c453..1cbd2ba3ce8cb 100644 --- a/src/internal_events/kafka.rs +++ b/src/internal_events/kafka.rs @@ -1,9 +1,8 @@ #![allow(dead_code)] // TODO requires optional feature compilation -use metrics::{counter, gauge}; use vector_lib::{ - NamedInternalEvent, - internal_event::{InternalEvent, error_stage, error_type}, + NamedInternalEvent, counter, gauge, + internal_event::{CounterName, GaugeName, InternalEvent, error_stage, error_type}, json_size::JsonSize, }; use vrl::path::OwnedTargetPath; @@ -26,7 +25,7 @@ impl InternalEvent for KafkaBytesReceived<'_> { partition = %self.partition, ); counter!( - "component_received_bytes_total", + CounterName::ComponentReceivedBytesTotal, "protocol" => self.protocol, "topic" => self.topic.to_string(), "partition" => self.partition.to_string(), @@ -53,13 +52,13 @@ impl InternalEvent for KafkaEventsReceived<'_> { partition = %self.partition, ); counter!( - "component_received_events_total", + CounterName::ComponentReceivedEventsTotal, "topic" => self.topic.to_string(), "partition" => self.partition.to_string(), ) .increment(self.count as u64); counter!( - "component_received_event_bytes_total", + CounterName::ComponentReceivedEventBytesTotal, "topic" => self.topic.to_string(), "partition" => self.partition.to_string(), ) @@ -82,7 +81,7 @@ impl InternalEvent for KafkaOffsetUpdateError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "kafka_offset_update", "error_type" => error_type::READER_FAILED, "stage" => error_stage::SENDING, @@ -106,7 +105,7 @@ impl InternalEvent for KafkaReadError { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "reading_message", "error_type" => error_type::READER_FAILED, "stage" => error_stage::RECEIVING, @@ -123,24 +122,24 @@ pub struct KafkaStatisticsReceived<'a> { impl InternalEvent for KafkaStatisticsReceived<'_> { fn emit(self) { - gauge!("kafka_queue_messages").set(self.statistics.msg_cnt as f64); - gauge!("kafka_queue_messages_bytes").set(self.statistics.msg_size as f64); - counter!("kafka_requests_total").absolute(self.statistics.tx as u64); - counter!("kafka_requests_bytes_total").absolute(self.statistics.tx_bytes as u64); - counter!("kafka_responses_total").absolute(self.statistics.rx as u64); - counter!("kafka_responses_bytes_total").absolute(self.statistics.rx_bytes as u64); - counter!("kafka_produced_messages_total").absolute(self.statistics.txmsgs as u64); - counter!("kafka_produced_messages_bytes_total") + gauge!(GaugeName::KafkaQueueMessages).set(self.statistics.msg_cnt as f64); + gauge!(GaugeName::KafkaQueueMessagesBytes).set(self.statistics.msg_size as f64); + counter!(CounterName::KafkaRequestsTotal).absolute(self.statistics.tx as u64); + counter!(CounterName::KafkaRequestsBytesTotal).absolute(self.statistics.tx_bytes as u64); + counter!(CounterName::KafkaResponsesTotal).absolute(self.statistics.rx as u64); + counter!(CounterName::KafkaResponsesBytesTotal).absolute(self.statistics.rx_bytes as u64); + counter!(CounterName::KafkaProducedMessagesTotal).absolute(self.statistics.txmsgs as u64); + counter!(CounterName::KafkaProducedMessagesBytesTotal) .absolute(self.statistics.txmsg_bytes as u64); - counter!("kafka_consumed_messages_total").absolute(self.statistics.rxmsgs as u64); - counter!("kafka_consumed_messages_bytes_total") + counter!(CounterName::KafkaConsumedMessagesTotal).absolute(self.statistics.rxmsgs as u64); + counter!(CounterName::KafkaConsumedMessagesBytesTotal) .absolute(self.statistics.rxmsg_bytes as u64); if self.expose_lag_metrics { for (topic_id, topic) in &self.statistics.topics { for (partition_id, partition) in &topic.partitions { gauge!( - "kafka_consumer_lag", + GaugeName::KafkaConsumerLag, "topic_id" => topic_id.clone(), "partition_id" => partition_id.to_string(), ) @@ -166,7 +165,7 @@ impl InternalEvent for KafkaHeaderExtractionError<'_> { header_field = self.header_field.to_string(), ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "extracting_header", "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::RECEIVING, diff --git a/src/internal_events/kubernetes_logs.rs b/src/internal_events/kubernetes_logs.rs index c6edb03724169..15b4ca4e61ee0 100644 --- a/src/internal_events/kubernetes_logs.rs +++ b/src/internal_events/kubernetes_logs.rs @@ -1,8 +1,8 @@ -use metrics::counter; use vector_lib::{ - NamedInternalEvent, + NamedInternalEvent, counter, internal_event::{ - ComponentEventsDropped, INTENTIONAL, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, INTENTIONAL, InternalEvent, UNINTENTIONAL, + error_stage, error_type, }, json_size::JsonSize, }; @@ -37,21 +37,21 @@ impl InternalEvent for KubernetesLogsEventsReceived<'_> { let pod_namespace = pod_info.namespace; counter!( - "component_received_events_total", + CounterName::ComponentReceivedEventsTotal, "pod_name" => pod_name.clone(), "pod_namespace" => pod_namespace.clone(), ) .increment(1); counter!( - "component_received_event_bytes_total", + CounterName::ComponentReceivedEventBytesTotal, "pod_name" => pod_name, "pod_namespace" => pod_namespace, ) .increment(self.byte_size.get() as u64); } None => { - counter!("component_received_events_total").increment(1); - counter!("component_received_event_bytes_total") + counter!(CounterName::ComponentReceivedEventsTotal).increment(1); + counter!(CounterName::ComponentReceivedEventBytesTotal) .increment(self.byte_size.get() as u64); } } @@ -75,7 +75,7 @@ impl InternalEvent for KubernetesLogsEventAnnotationError<'_> { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => ANNOTATION_FAILED, "error_type" => error_type::READER_FAILED, "stage" => error_stage::PROCESSING, @@ -99,13 +99,13 @@ impl InternalEvent for KubernetesLogsEventNamespaceAnnotationError<'_> { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => ANNOTATION_FAILED, "error_type" => error_type::READER_FAILED, "stage" => error_stage::PROCESSING, ) .increment(1); - counter!("k8s_event_namespace_annotation_failures_total").increment(1); + counter!(CounterName::K8sEventNamespaceAnnotationFailuresTotal).increment(1); } } @@ -124,13 +124,13 @@ impl InternalEvent for KubernetesLogsEventNodeAnnotationError<'_> { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => ANNOTATION_FAILED, "error_type" => error_type::READER_FAILED, "stage" => error_stage::PROCESSING, ) .increment(1); - counter!("k8s_event_node_annotation_failures_total").increment(1); + counter!(CounterName::K8sEventNodeAnnotationFailuresTotal).increment(1); } } @@ -145,7 +145,7 @@ impl InternalEvent for KubernetesLogsFormatPickerEdgeCase { message = "Encountered format picker edge case.", what = %self.what, ); - counter!("k8s_format_picker_edge_cases_total").increment(1); + counter!(CounterName::K8sFormatPickerEdgeCasesTotal).increment(1); } } @@ -163,12 +163,12 @@ impl InternalEvent for KubernetesLogsDockerFormatParseError<'_> { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, ) .increment(1); - counter!("k8s_docker_format_parse_failures_total").increment(1); + counter!(CounterName::K8sDockerFormatParseFailuresTotal).increment(1); } } @@ -191,7 +191,7 @@ impl InternalEvent for KubernetesLifecycleError { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => KUBERNETES_LIFECYCLE, "error_type" => error_type::READER_FAILED, "stage" => error_stage::PROCESSING, @@ -222,7 +222,7 @@ impl InternalEvent for KubernetesMergedLineTooBigError<'_> { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "reading_line_from_kubernetes_log", "error_type" => error_type::CONDITION_FAILED, "stage" => error_stage::RECEIVING, diff --git a/src/internal_events/log_to_metric.rs b/src/internal_events/log_to_metric.rs index 9fbb64dba0622..e8e3da5b898b7 100644 --- a/src/internal_events/log_to_metric.rs +++ b/src/internal_events/log_to_metric.rs @@ -1,9 +1,10 @@ use std::num::ParseFloatError; -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{ + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, error_stage, error_type, + }, }; #[derive(NamedInternalEvent)] @@ -22,7 +23,7 @@ impl InternalEvent for LogToMetricFieldNullError<'_> { null_field = %self.field ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "field_null", "error_type" => error_type::CONDITION_FAILED, "stage" => error_stage::PROCESSING, @@ -52,7 +53,7 @@ impl InternalEvent for LogToMetricParseFloatError<'_> { stage = error_stage::PROCESSING ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "failed_parsing_float", "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, @@ -83,7 +84,7 @@ impl InternalEvent for MetricMetadataInvalidFieldValueError<'_> { stage = error_stage::PROCESSING ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "invalid_field_value", "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, @@ -112,7 +113,7 @@ impl InternalEvent for MetricMetadataParseError<'_> { stage = error_stage::PROCESSING ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => format!("failed_parsing_{}", self.kind), "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, @@ -137,7 +138,7 @@ impl InternalEvent for MetricMetadataMetricDetailsNotFoundError { stage = error_stage::PROCESSING ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "missing_metric_details", "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, diff --git a/src/internal_events/logplex.rs b/src/internal_events/logplex.rs index b28588c5c2d1e..4a45df6349b65 100644 --- a/src/internal_events/logplex.rs +++ b/src/internal_events/logplex.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; use super::prelude::io_error_code; @@ -37,7 +38,7 @@ impl InternalEvent for HerokuLogplexRequestReadError { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::READER_FAILED, "error_code" => io_error_code(&self.error), "stage" => error_stage::PROCESSING, diff --git a/src/internal_events/loki.rs b/src/internal_events/loki.rs index 523efd7e4f59b..3a5935e1a372a 100644 --- a/src/internal_events/loki.rs +++ b/src/internal_events/loki.rs @@ -1,8 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; use vector_lib::internal_event::{ - ComponentEventsDropped, INTENTIONAL, InternalEvent, error_stage, error_type, + ComponentEventsDropped, CounterName, INTENTIONAL, InternalEvent, error_stage, error_type, }; +use vector_lib::{NamedInternalEvent, counter}; #[derive(Debug, NamedInternalEvent)] pub struct LokiEventUnlabeledError; @@ -17,7 +16,7 @@ impl InternalEvent for LokiEventUnlabeledError { ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "unlabeled_event", "error_type" => error_type::CONDITION_FAILED, "stage" => error_stage::PROCESSING, @@ -48,7 +47,7 @@ impl InternalEvent for LokiOutOfOrderEventDroppedError { }); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "out_of_order", "error_type" => error_type::CONDITION_FAILED, "stage" => error_stage::PROCESSING, @@ -69,7 +68,7 @@ impl InternalEvent for LokiOutOfOrderEventRewritten { count = self.count, reason = "out_of_order", ); - counter!("rewritten_timestamp_events_total").increment(self.count as u64); + counter!(CounterName::RewrittenTimestampEventsTotal).increment(self.count as u64); } } @@ -90,7 +89,7 @@ impl InternalEvent for LokiTimestampNonParsableEventsDropped { emit!(ComponentEventsDropped:: { count: 1, reason }); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "non-parsable_timestamp", "error_type" => error_type::CONDITION_FAILED, "stage" => error_stage::PROCESSING, diff --git a/src/internal_events/lua.rs b/src/internal_events/lua.rs index feb9d04212b88..3a8779d9cd4b1 100644 --- a/src/internal_events/lua.rs +++ b/src/internal_events/lua.rs @@ -1,8 +1,9 @@ -use metrics::{counter, gauge}; use vector_lib::NamedInternalEvent; use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, GaugeName, InternalEvent, UNINTENTIONAL, error_stage, + error_type, }; +use vector_lib::{counter, gauge}; use crate::transforms::lua::v2::BuildError; @@ -13,7 +14,7 @@ pub struct LuaGcTriggered { impl InternalEvent for LuaGcTriggered { fn emit(self) { - gauge!("lua_memory_used_bytes").set(self.used_memory as f64); + gauge!(GaugeName::LuaMemoryUsedBytes).set(self.used_memory as f64); } } @@ -32,7 +33,7 @@ impl InternalEvent for LuaScriptError { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => mlua_error_code(&self.error), "error_type" => error_type::SCRIPT_FAILED, "stage" => error_stage::PROCESSING, @@ -62,7 +63,7 @@ impl InternalEvent for LuaBuildError { internal_log_rate_limit = false, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => lua_build_error_code(&self.error), "error_type" => error_type::SCRIPT_FAILED, "stage" => error_stage:: PROCESSING, diff --git a/src/internal_events/metric_to_log.rs b/src/internal_events/metric_to_log.rs index a536a1b24df24..a11ed286be17a 100644 --- a/src/internal_events/metric_to_log.rs +++ b/src/internal_events/metric_to_log.rs @@ -1,9 +1,8 @@ -use metrics::counter; use serde_json::Error; -use vector_lib::NamedInternalEvent; use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, error_stage, error_type, }; +use vector_lib::{NamedInternalEvent, counter}; #[derive(Debug, NamedInternalEvent)] pub struct MetricToLogSerializeError { @@ -20,7 +19,7 @@ impl InternalEvent for MetricToLogSerializeError { stage = error_stage::PROCESSING ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::ENCODER_FAILED, "stage" => error_stage::PROCESSING, ) diff --git a/src/internal_events/mongodb_metrics.rs b/src/internal_events/mongodb_metrics.rs index 3cad1e4ef97f9..b0023085f025f 100644 --- a/src/internal_events/mongodb_metrics.rs +++ b/src/internal_events/mongodb_metrics.rs @@ -1,8 +1,7 @@ -use metrics::counter; use mongodb::{bson, error::Error as MongoError}; use vector_lib::{ - NamedInternalEvent, - internal_event::{InternalEvent, error_stage, error_type}, + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, json_size::JsonSize, }; @@ -23,12 +22,12 @@ impl InternalEvent for MongoDbMetricsEventsReceived<'_> { endpoint = self.endpoint, ); counter!( - "component_received_events_total", + CounterName::ComponentReceivedEventsTotal, "endpoint" => self.endpoint.to_owned(), ) .increment(self.count as u64); counter!( - "component_received_event_bytes_total", + CounterName::ComponentReceivedEventBytesTotal, "endpoint" => self.endpoint.to_owned(), ) .increment(self.byte_size.get() as u64); @@ -51,7 +50,7 @@ impl InternalEvent for MongoDbMetricsRequestError<'_> { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::REQUEST_FAILED, "stage" => error_stage::RECEIVING, ) @@ -75,7 +74,7 @@ impl InternalEvent for MongoDbMetricsBsonParseError<'_> { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::RECEIVING, "endpoint" => self.endpoint.to_owned(), diff --git a/src/internal_events/mqtt.rs b/src/internal_events/mqtt.rs index 1e406a9494f1f..8d416c4c748ee 100644 --- a/src/internal_events/mqtt.rs +++ b/src/internal_events/mqtt.rs @@ -1,9 +1,10 @@ use std::fmt::Debug; -use metrics::counter; use rumqttc::ConnectionError; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; #[derive(Debug, NamedInternalEvent)] pub struct MqttConnectionError { @@ -20,7 +21,7 @@ impl InternalEvent for MqttConnectionError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "mqtt_connection_error", "error_type" => error_type::WRITER_FAILED, "stage" => error_stage::SENDING, diff --git a/src/internal_events/nginx_metrics.rs b/src/internal_events/nginx_metrics.rs index a3384d4ef0faf..82cd7ea3878b7 100644 --- a/src/internal_events/nginx_metrics.rs +++ b/src/internal_events/nginx_metrics.rs @@ -1,7 +1,6 @@ -use metrics::counter; use vector_lib::{ - NamedInternalEvent, - internal_event::{InternalEvent, error_stage, error_type}, + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, json_size::JsonSize, }; @@ -23,12 +22,12 @@ impl InternalEvent for NginxMetricsEventsReceived<'_> { endpoint = self.endpoint, ); counter!( - "component_received_events_total", + CounterName::ComponentReceivedEventsTotal, "endpoint" => self.endpoint.to_owned(), ) .increment(self.count as u64); counter!( - "component_received_event_bytes_total", + CounterName::ComponentReceivedEventBytesTotal, "endpoint" => self.endpoint.to_owned(), ) .increment(self.byte_size.get() as u64); @@ -51,7 +50,7 @@ impl InternalEvent for NginxMetricsRequestError<'_> { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "endpoint" => self.endpoint.to_owned(), "error_type" => error_type::REQUEST_FAILED, "stage" => error_stage::RECEIVING, @@ -76,7 +75,7 @@ impl InternalEvent for NginxMetricsStubStatusParseError<'_> { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "endpoint" => self.endpoint.to_owned(), "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, diff --git a/src/internal_events/open.rs b/src/internal_events/open.rs index e48e0ac337820..58e9c9d40b3f8 100644 --- a/src/internal_events/open.rs +++ b/src/internal_events/open.rs @@ -6,9 +6,10 @@ use std::{ }, }; -use metrics::gauge; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::InternalEvent; +use vector_lib::{ + NamedInternalEvent, gauge, + internal_event::{GaugeName, InternalEvent}, +}; #[derive(Debug, NamedInternalEvent)] pub struct ConnectionOpen { @@ -17,7 +18,7 @@ pub struct ConnectionOpen { impl InternalEvent for ConnectionOpen { fn emit(self) { - gauge!("open_connections").set(self.count as f64); + gauge!(GaugeName::OpenConnections).set(self.count as f64); } } @@ -28,7 +29,7 @@ pub struct EndpointsActive { impl InternalEvent for EndpointsActive { fn emit(self) { - gauge!("active_endpoints").set(self.count as f64); + gauge!(GaugeName::ActiveEndpoints).set(self.count as f64); } } diff --git a/src/internal_events/parser.rs b/src/internal_events/parser.rs index 85a3e1383afa3..2610ae5458958 100644 --- a/src/internal_events/parser.rs +++ b/src/internal_events/parser.rs @@ -2,10 +2,11 @@ use std::borrow::Cow; -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{ + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, error_stage, error_type, + }, }; fn truncate_string_at(s: &str, maxlen: usize) -> Cow<'_, str> { @@ -36,7 +37,7 @@ impl InternalEvent for ParserMatchError<'_> { field = &truncate_string_at(&String::from_utf8_lossy(self.value), 60)[..] ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "no_match_found", "error_type" => error_type::CONDITION_FAILED, "stage" => error_stage::PROCESSING, @@ -66,7 +67,7 @@ impl InternalEvent for ParserMissingFieldError<'_, DROP_ stage = error_stage::PROCESSING ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "field_not_found", "error_type" => error_type::CONDITION_FAILED, "stage" => error_stage::PROCESSING, @@ -97,7 +98,7 @@ impl InternalEvent for ParserConversionError<'_> { stage = error_stage::PROCESSING ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "type_conversion", "error_type" => error_type::CONVERSION_FAILED, "stage" => error_stage::PROCESSING, diff --git a/src/internal_events/postgresql_metrics.rs b/src/internal_events/postgresql_metrics.rs index d6a58f968ab7f..62fa4d18a722d 100644 --- a/src/internal_events/postgresql_metrics.rs +++ b/src/internal_events/postgresql_metrics.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; #[derive(Debug, NamedInternalEvent)] pub struct PostgresqlMetricsCollectError<'a> { @@ -18,7 +19,7 @@ impl InternalEvent for PostgresqlMetricsCollectError<'_> { endpoint = %self.endpoint, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::REQUEST_FAILED, "stage" => error_stage::RECEIVING, ) diff --git a/src/internal_events/process.rs b/src/internal_events/process.rs index 987724c2da18d..19701c6feaca8 100644 --- a/src/internal_events/process.rs +++ b/src/internal_events/process.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; use crate::{built_info, config}; @@ -17,7 +18,7 @@ impl InternalEvent for VectorStarted { arch = built_info::TARGET_ARCH, revision = built_info::VECTOR_BUILD_DESC.unwrap_or(""), ); - counter!("started_total").increment(1); + counter!(CounterName::StartedTotal).increment(1); } } @@ -34,7 +35,7 @@ impl InternalEvent for VectorReloaded<'_> { path = ?self.config_paths, internal_log_rate_limit = false, ); - counter!("reloaded_total").increment(1); + counter!(CounterName::ReloadedTotal).increment(1); } } @@ -59,7 +60,7 @@ impl InternalEvent for VectorStopped { target: "vector", message = "Vector has stopped.", ); - counter!("stopped_total").increment(1); + counter!(CounterName::StoppedTotal).increment(1); } } @@ -72,7 +73,7 @@ impl InternalEvent for VectorQuit { target: "vector", message = "Vector has quit.", ); - counter!("quit_total").increment(1); + counter!(CounterName::QuitTotal).increment(1); } } @@ -92,7 +93,7 @@ impl InternalEvent for VectorReloadError { internal_log_rate_limit = false, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "reload", "error_type" => error_type::CONFIGURATION_FAILED, "stage" => error_stage::PROCESSING, @@ -115,7 +116,7 @@ impl InternalEvent for VectorConfigLoadError { internal_log_rate_limit = false, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "config_load", "error_type" => error_type::CONFIGURATION_FAILED, "stage" => error_stage::PROCESSING, @@ -137,7 +138,7 @@ impl InternalEvent for VectorRecoveryError { internal_log_rate_limit = false, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "recovery", "error_type" => error_type::CONFIGURATION_FAILED, "stage" => error_stage::PROCESSING, diff --git a/src/internal_events/prometheus.rs b/src/internal_events/prometheus.rs index 27b28fb5ffafa..3ddd3c555f728 100644 --- a/src/internal_events/prometheus.rs +++ b/src/internal_events/prometheus.rs @@ -3,13 +3,14 @@ #[cfg(feature = "sources-prometheus-scrape")] use std::borrow::Cow; -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, -}; #[cfg(feature = "sources-prometheus-scrape")] use vector_lib::prometheus::parser::ParserError; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{ + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, error_stage, error_type, + }, +}; #[cfg(feature = "sources-prometheus-scrape")] #[derive(Debug, NamedInternalEvent)] @@ -34,7 +35,7 @@ impl InternalEvent for PrometheusParseError<'_> { url = %self.url ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, "url" => self.url.to_string(), @@ -57,7 +58,7 @@ impl InternalEvent for PrometheusRemoteWriteParseError { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, ) @@ -77,7 +78,7 @@ impl InternalEvent for PrometheusNormalizationError { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::CONVERSION_FAILED, "stage" => error_stage::PROCESSING, ) diff --git a/src/internal_events/pulsar.rs b/src/internal_events/pulsar.rs index a2212ef0d5971..0e2e11b061f3e 100644 --- a/src/internal_events/pulsar.rs +++ b/src/internal_events/pulsar.rs @@ -2,11 +2,10 @@ #[cfg(feature = "sources-pulsar")] use metrics::Counter; -use metrics::counter; -use vector_lib::NamedInternalEvent; use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, error_stage, error_type, }; +use vector_lib::{NamedInternalEvent, counter}; #[derive(Debug, NamedInternalEvent)] pub struct PulsarSendingError { @@ -24,7 +23,7 @@ impl InternalEvent for PulsarSendingError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::REQUEST_FAILED, "stage" => error_stage::SENDING, ) @@ -51,7 +50,7 @@ impl InternalEvent for PulsarPropertyExtractionError { property_field = %self.property_field, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "extracting_property", "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, @@ -77,21 +76,21 @@ pub struct PulsarErrorEventData { registered_event!( PulsarErrorEvent => { ack_errors: Counter = counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "acknowledge_message", "error_type" => error_type::ACKNOWLEDGMENT_FAILED, "stage" => error_stage::RECEIVING, ), nack_errors: Counter = counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "negative_acknowledge_message", "error_type" => error_type::ACKNOWLEDGMENT_FAILED, "stage" => error_stage::RECEIVING, ), read_errors: Counter = counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "reading_message", "error_type" => error_type::READER_FAILED, "stage" => error_stage::RECEIVING, diff --git a/src/internal_events/redis.rs b/src/internal_events/redis.rs index 45d76f3670dcd..87b9bc6d7b2f8 100644 --- a/src/internal_events/redis.rs +++ b/src/internal_events/redis.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; #[derive(Debug, NamedInternalEvent)] pub struct RedisReceiveEventError { @@ -25,7 +26,7 @@ impl InternalEvent for RedisReceiveEventError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => self.error_code, "error_type" => error_type::READER_FAILED, "stage" => error_stage::RECEIVING, diff --git a/src/internal_events/reduce.rs b/src/internal_events/reduce.rs index 8917c5aad0a5a..0962c6332a439 100644 --- a/src/internal_events/reduce.rs +++ b/src/internal_events/reduce.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; use vrl::{path::PathParseError, value::KeyString}; #[derive(Debug, NamedInternalEvent)] @@ -8,7 +9,7 @@ pub struct ReduceStaleEventFlushed; impl InternalEvent for ReduceStaleEventFlushed { fn emit(self) { - counter!("stale_events_flushed_total").increment(1); + counter!(CounterName::StaleEventsFlushedTotal).increment(1); } } @@ -28,7 +29,7 @@ impl InternalEvent for ReduceAddEventError { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, ) diff --git a/src/internal_events/remap.rs b/src/internal_events/remap.rs index 989a6aa064065..d0e6500d69341 100644 --- a/src/internal_events/remap.rs +++ b/src/internal_events/remap.rs @@ -1,8 +1,8 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; use vector_lib::internal_event::{ - ComponentEventsDropped, INTENTIONAL, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, INTENTIONAL, InternalEvent, UNINTENTIONAL, error_stage, + error_type, }; +use vector_lib::{NamedInternalEvent, counter}; #[derive(Debug, NamedInternalEvent)] pub struct RemapMappingError { @@ -21,7 +21,7 @@ impl InternalEvent for RemapMappingError { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::CONVERSION_FAILED, "stage" => error_stage::PROCESSING, ) diff --git a/src/internal_events/sample.rs b/src/internal_events/sample.rs index 3722f9530b668..825175dc42555 100644 --- a/src/internal_events/sample.rs +++ b/src/internal_events/sample.rs @@ -1,5 +1,7 @@ -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{ComponentEventsDropped, INTENTIONAL, InternalEvent}; +use vector_lib::{ + NamedInternalEvent, + internal_event::{ComponentEventsDropped, INTENTIONAL, InternalEvent}, +}; #[derive(Debug, NamedInternalEvent)] pub struct SampleEventDiscarded; diff --git a/src/internal_events/sematext_metrics.rs b/src/internal_events/sematext_metrics.rs index 65d23c53e00ef..cb03742a5d3d1 100644 --- a/src/internal_events/sematext_metrics.rs +++ b/src/internal_events/sematext_metrics.rs @@ -1,7 +1,8 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{ + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, error_stage, error_type, + }, }; use crate::event::metric::Metric; @@ -23,7 +24,7 @@ impl InternalEvent for SematextMetricsInvalidMetricError<'_> { kind = ?self.metric.kind(), ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "invalid_metric", "error_type" => error_type::ENCODER_FAILED, "stage" => error_stage::PROCESSING, @@ -49,7 +50,7 @@ impl InternalEvent for SematextMetricsEncodeEventError stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::ENCODER_FAILED, "stage" => error_stage::PROCESSING, ) diff --git a/src/internal_events/socket.rs b/src/internal_events/socket.rs index f0b58ce33bbd7..d38788f46767c 100644 --- a/src/internal_events/socket.rs +++ b/src/internal_events/socket.rs @@ -1,10 +1,10 @@ use std::net::Ipv4Addr; -use metrics::{counter, histogram}; use vector_lib::{ - NamedInternalEvent, + NamedInternalEvent, counter, histogram, internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, HistogramName, InternalEvent, UNINTENTIONAL, + error_stage, error_type, }, json_size::JsonSize, }; @@ -42,11 +42,11 @@ impl InternalEvent for SocketBytesReceived { %protocol, ); counter!( - "component_received_bytes_total", + CounterName::ComponentReceivedBytesTotal, "protocol" => protocol, ) .increment(self.byte_size as u64); - histogram!("component_received_bytes").record(self.byte_size as f64); + histogram!(HistogramName::ComponentReceivedBytes).record(self.byte_size as f64); } } @@ -66,10 +66,12 @@ impl InternalEvent for SocketEventsReceived { byte_size = self.byte_size.get(), %mode, ); - counter!("component_received_events_total", "mode" => mode).increment(self.count as u64); - counter!("component_received_event_bytes_total", "mode" => mode) + counter!(CounterName::ComponentReceivedEventsTotal, "mode" => mode) + .increment(self.count as u64); + counter!(CounterName::ComponentReceivedEventBytesTotal, "mode" => mode) .increment(self.byte_size.get() as u64); - histogram!("component_received_bytes", "mode" => mode).record(self.byte_size.get() as f64); + histogram!(HistogramName::ComponentReceivedBytes, "mode" => mode) + .record(self.byte_size.get() as f64); } } @@ -88,7 +90,7 @@ impl InternalEvent for SocketBytesSent { %protocol, ); counter!( - "component_sent_bytes_total", + CounterName::ComponentSentBytesTotal, "protocol" => protocol, ) .increment(self.byte_size as u64); @@ -105,8 +107,9 @@ pub struct SocketEventsSent { impl InternalEvent for SocketEventsSent { fn emit(self) { trace!(message = "Events sent.", count = %self.count, byte_size = %self.byte_size.get()); - counter!("component_sent_events_total", "mode" => self.mode.as_str()).increment(self.count); - counter!("component_sent_event_bytes_total", "mode" => self.mode.as_str()) + counter!(CounterName::ComponentSentEventsTotal, "mode" => self.mode.as_str()) + .increment(self.count); + counter!(CounterName::ComponentSentEventBytesTotal, "mode" => self.mode.as_str()) .increment(self.byte_size.get() as u64); } } @@ -129,7 +132,7 @@ impl InternalEvent for SocketBindError { %mode, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "socket_bind", "error_type" => error_type::IO_FAILED, "stage" => error_stage::INITIALIZING, @@ -164,7 +167,7 @@ impl InternalEvent for SocketMulticastGroupJoinError { %interface, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "socket_multicast_group_join", "error_type" => error_type::IO_FAILED, "stage" => error_stage::INITIALIZING, @@ -194,7 +197,7 @@ impl InternalEvent for SocketReceiveError { %mode, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "socket_receive", "error_type" => error_type::READER_FAILED, "stage" => error_stage::RECEIVING, @@ -223,7 +226,7 @@ impl InternalEvent for SocketSendError { %mode, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "socket_send", "error_type" => error_type::WRITER_FAILED, "stage" => error_stage::SENDING, diff --git a/src/internal_events/splunk_hec.rs b/src/internal_events/splunk_hec.rs index 6dcd63084d8e6..bd3fde83c2318 100644 --- a/src/internal_events/splunk_hec.rs +++ b/src/internal_events/splunk_hec.rs @@ -7,11 +7,13 @@ pub use self::source::*; #[cfg(feature = "sinks-splunk_hec")] mod sink { - use metrics::{counter, gauge}; use serde_json::Error; - use vector_lib::NamedInternalEvent; - use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, + use vector_lib::{ + NamedInternalEvent, counter, gauge, + internal_event::{ + ComponentEventsDropped, CounterName, GaugeName, InternalEvent, UNINTENTIONAL, + error_stage, error_type, + }, }; use crate::{ @@ -35,7 +37,7 @@ mod sink { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "serializing_json", "error_type" => error_type::ENCODER_FAILED, "stage" => error_stage::PROCESSING, @@ -63,13 +65,13 @@ mod sink { kind = ?self.kind, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::INVALID_METRIC, "stage" => error_stage::PROCESSING, ) .increment(1); counter!( - "component_discarded_events_total", + CounterName::ComponentDiscardedEventsTotal, "error_type" => error_type::INVALID_METRIC, "stage" => error_stage::PROCESSING, ) @@ -92,7 +94,7 @@ mod sink { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "invalid_response", "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::SENDING, @@ -117,7 +119,7 @@ mod sink { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "indexer_ack_failed", "error_type" => error_type::ACKNOWLEDGMENT_FAILED, "stage" => error_stage::SENDING, @@ -141,7 +143,7 @@ mod sink { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "indexer_ack_unavailable", "error_type" => error_type::ACKNOWLEDGMENT_FAILED, "stage" => error_stage::SENDING, @@ -155,7 +157,7 @@ mod sink { impl InternalEvent for SplunkIndexerAcknowledgementAckAdded { fn emit(self) { - gauge!("splunk_pending_acks").increment(1.0); + gauge!(GaugeName::SplunkPendingAcks).increment(1.0); } } @@ -166,7 +168,7 @@ mod sink { impl InternalEvent for SplunkIndexerAcknowledgementAcksRemoved { fn emit(self) { - gauge!("splunk_pending_acks").decrement(self.count); + gauge!(GaugeName::SplunkPendingAcks).decrement(self.count); } } @@ -197,9 +199,10 @@ mod sink { #[cfg(feature = "sources-splunk_hec")] mod source { - use metrics::counter; - use vector_lib::NamedInternalEvent; - use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; + use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, + }; use crate::sources::splunk_hec::ApiError; @@ -218,7 +221,7 @@ mod source { stage = error_stage::PROCESSING ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "invalid_request_body", "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, @@ -243,7 +246,7 @@ mod source { stage = error_stage::RECEIVING ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::AUTHENTICATION_FAILED, "stage" => error_stage::RECEIVING, ) @@ -257,7 +260,7 @@ mod source { stage = error_stage::RECEIVING ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::REQUEST_FAILED, "stage" => error_stage::RECEIVING, ) diff --git a/src/internal_events/statsd_sink.rs b/src/internal_events/statsd_sink.rs index 7f0ee81f48ddc..858084124b562 100644 --- a/src/internal_events/statsd_sink.rs +++ b/src/internal_events/statsd_sink.rs @@ -1,8 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, error_stage, error_type, }; +use vector_lib::{NamedInternalEvent, counter}; use crate::event::metric::{MetricKind, MetricValue}; @@ -24,7 +23,7 @@ impl InternalEvent for StatsdInvalidMetricError<'_> { kind = ?self.kind, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "invalid_metric", "error_type" => error_type::ENCODER_FAILED, "stage" => error_stage::PROCESSING, diff --git a/src/internal_events/tag_cardinality_limit.rs b/src/internal_events/tag_cardinality_limit.rs index 83a82acd342ae..9906f5c320f6b 100644 --- a/src/internal_events/tag_cardinality_limit.rs +++ b/src/internal_events/tag_cardinality_limit.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{ComponentEventsDropped, INTENTIONAL, InternalEvent}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{ComponentEventsDropped, CounterName, INTENTIONAL, InternalEvent}, +}; #[derive(NamedInternalEvent)] pub struct TagCardinalityLimitRejectingEvent<'a> { @@ -20,13 +21,13 @@ impl InternalEvent for TagCardinalityLimitRejectingEvent<'_> { ); if self.include_extended_tags { counter!( - "tag_value_limit_exceeded_total", + CounterName::TagValueLimitExceededTotal, "metric_name" => self.metric_name.to_string(), "tag_key" => self.tag_key.to_string(), ) .increment(1); } else { - counter!("tag_value_limit_exceeded_total").increment(1); + counter!(CounterName::TagValueLimitExceededTotal).increment(1); } emit!(ComponentEventsDropped:: { @@ -54,13 +55,13 @@ impl InternalEvent for TagCardinalityLimitRejectingTag<'_> { ); if self.include_extended_tags { counter!( - "tag_value_limit_exceeded_total", + CounterName::TagValueLimitExceededTotal, "metric_name" => self.metric_name.to_string(), "tag_key" => self.tag_key.to_string(), ) .increment(1); } else { - counter!("tag_value_limit_exceeded_total").increment(1); + counter!(CounterName::TagValueLimitExceededTotal).increment(1); } } } @@ -76,6 +77,6 @@ impl InternalEvent for TagCardinalityValueLimitReached<'_> { message = "Value_limit reached for key. New values for this key will be rejected.", key = %self.key, ); - counter!("value_limit_reached_total").increment(1); + counter!(CounterName::ValueLimitReachedTotal).increment(1); } } diff --git a/src/internal_events/tcp.rs b/src/internal_events/tcp.rs index a121eef1e0327..1ec0cfb6322e5 100644 --- a/src/internal_events/tcp.rs +++ b/src/internal_events/tcp.rs @@ -1,8 +1,9 @@ use std::net::SocketAddr; -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; use crate::{internal_events::SocketOutgoingConnectionError, tls::TlsError}; @@ -18,7 +19,7 @@ impl InternalEvent for TcpSocketConnectionEstablished { } else { debug!(message = "Connected.", peer_addr = "unknown"); } - counter!("connection_established_total", "mode" => "tcp").increment(1); + counter!(CounterName::ConnectionEstablishedTotal, "mode" => "tcp").increment(1); } } @@ -41,7 +42,7 @@ pub struct TcpSocketConnectionShutdown; impl InternalEvent for TcpSocketConnectionShutdown { fn emit(self) { warn!(message = "Received EOF from the server, shutdown."); - counter!("connection_shutdown_total", "mode" => "tcp").increment(1); + counter!(CounterName::ConnectionShutdownTotal, "mode" => "tcp").increment(1); } } @@ -63,7 +64,7 @@ impl InternalEvent for TcpSocketError<'_, E> { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::CONNECTION_FAILED, "stage" => error_stage::PROCESSING, ) @@ -100,7 +101,7 @@ impl InternalEvent for TcpSocketTlsConnectionError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "connection_failed", "error_type" => error_type::WRITER_FAILED, "stage" => error_stage::SENDING, @@ -127,7 +128,7 @@ impl InternalEvent for TcpSendAckError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "ack_failed", "error_type" => error_type::WRITER_FAILED, "stage" => error_stage::SENDING, @@ -152,7 +153,7 @@ impl InternalEvent for TcpBytesReceived { peer_addr = %self.peer_addr, ); counter!( - "component_received_bytes_total", "protocol" => "tcp" + CounterName::ComponentReceivedBytesTotal, "protocol" => "tcp" ) .increment(self.byte_size as u64); } diff --git a/src/internal_events/template.rs b/src/internal_events/template.rs index 147418042a0c6..db43081e2a22a 100644 --- a/src/internal_events/template.rs +++ b/src/internal_events/template.rs @@ -1,7 +1,8 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{ + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, error_stage, error_type, + }, }; #[derive(NamedInternalEvent)] @@ -29,7 +30,7 @@ impl InternalEvent for TemplateRenderingError<'_> { ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::TEMPLATE_FAILED, "stage" => error_stage::PROCESSING, ) diff --git a/src/internal_events/throttle.rs b/src/internal_events/throttle.rs index 274fedd7741b9..3757bbae40535 100644 --- a/src/internal_events/throttle.rs +++ b/src/internal_events/throttle.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{ComponentEventsDropped, INTENTIONAL, InternalEvent}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{ComponentEventsDropped, CounterName, INTENTIONAL, InternalEvent}, +}; #[derive(Debug, NamedInternalEvent)] pub(crate) struct ThrottleEventDiscarded { @@ -23,7 +24,7 @@ impl InternalEvent for ThrottleEventDiscarded { // if we should change the specification wording? Sort of a similar situation to the // `error_code` tag for the component errors metric, where it's meant to be optional and // only specified when relevant. - counter!("events_discarded_total", "key" => self.key).increment(1); // Deprecated. + counter!(CounterName::EventsDiscardedTotal, "key" => self.key).increment(1); // Deprecated. } emit!(ComponentEventsDropped:: { diff --git a/src/internal_events/udp.rs b/src/internal_events/udp.rs index 0602d4fdb3329..31e5f5c6533a4 100644 --- a/src/internal_events/udp.rs +++ b/src/internal_events/udp.rs @@ -1,7 +1,8 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{ + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, error_stage, error_type, + }, }; use crate::internal_events::SocketOutgoingConnectionError; @@ -14,7 +15,7 @@ pub struct UdpSocketConnectionEstablished; impl InternalEvent for UdpSocketConnectionEstablished { fn emit(self) { debug!(message = "Connected."); - counter!("connection_established_total", "mode" => "udp").increment(1); + counter!(CounterName::ConnectionEstablishedTotal, "mode" => "udp").increment(1); } } @@ -51,13 +52,13 @@ impl InternalEvent for UdpSendIncompleteError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::WRITER_FAILED, "stage" => error_stage::SENDING, ) .increment(1); // deprecated - counter!("connection_send_errors_total", "mode" => "udp").increment(1); + counter!(CounterName::ConnectionSendErrorsTotal, "mode" => "udp").increment(1); emit!(ComponentEventsDropped:: { count: 1, reason }); } @@ -80,7 +81,7 @@ impl InternalEvent for UdpChunkingError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::WRITER_FAILED, "stage" => error_stage::SENDING, ) diff --git a/src/internal_events/unix.rs b/src/internal_events/unix.rs index 6fab3348dc4d7..fc98ff7feacbd 100644 --- a/src/internal_events/unix.rs +++ b/src/internal_events/unix.rs @@ -2,11 +2,10 @@ use std::{io::Error, path::Path}; -use metrics::counter; -use vector_lib::NamedInternalEvent; use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, error_stage, error_type, }; +use vector_lib::{NamedInternalEvent, counter}; use crate::internal_events::SocketOutgoingConnectionError; @@ -18,7 +17,7 @@ pub struct UnixSocketConnectionEstablished<'a> { impl InternalEvent for UnixSocketConnectionEstablished<'_> { fn emit(self) { debug!(message = "Connected.", path = ?self.path); - counter!("connection_established_total", "mode" => "unix").increment(1); + counter!(CounterName::ConnectionEstablishedTotal, "mode" => "unix").increment(1); } } @@ -59,7 +58,7 @@ impl InternalEvent for UnixSocketError<'_, E> { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::CONNECTION_FAILED, "stage" => error_stage::PROCESSING, ) @@ -84,7 +83,7 @@ impl InternalEvent for UnixSocketSendError<'_, E> { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::WRITER_FAILED, "stage" => error_stage::SENDING, ) @@ -112,7 +111,7 @@ impl InternalEvent for UnixSendIncompleteError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::WRITER_FAILED, "stage" => error_stage::SENDING, ) @@ -139,7 +138,7 @@ impl InternalEvent for UnixSocketFileDeleteError<'_> { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "delete_socket_file", "error_type" => error_type::WRITER_FAILED, "stage" => error_stage::PROCESSING, diff --git a/src/internal_events/websocket.rs b/src/internal_events/websocket.rs index 960aec1916133..b842407ee0313 100644 --- a/src/internal_events/websocket.rs +++ b/src/internal_events/websocket.rs @@ -5,14 +5,15 @@ use std::{ fmt::{Debug, Display, Formatter, Result}, }; -use metrics::{counter, histogram}; use tokio_tungstenite::tungstenite::error::Error as TungsteniteError; use vector_common::{ internal_event::{error_stage, error_type}, json_size::JsonSize, }; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::InternalEvent; +use vector_lib::{ + NamedInternalEvent, counter, histogram, + internal_event::{CounterName, HistogramName, InternalEvent}, +}; pub const PROTOCOL: &str = "websocket"; @@ -22,7 +23,7 @@ pub struct WebSocketConnectionEstablished; impl InternalEvent for WebSocketConnectionEstablished { fn emit(self) { debug!(message = "Connected."); - counter!("connection_established_total").increment(1); + counter!(CounterName::ConnectionEstablishedTotal).increment(1); } } @@ -41,7 +42,7 @@ impl InternalEvent for WebSocketConnectionFailedError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "protocol" => PROTOCOL, "error_code" => "websocket_connection_failed", "error_type" => error_type::CONNECTION_FAILED, @@ -57,7 +58,7 @@ pub struct WebSocketConnectionShutdown; impl InternalEvent for WebSocketConnectionShutdown { fn emit(self) { warn!(message = "Closed by the server."); - counter!("connection_shutdown_total").increment(1); + counter!(CounterName::ConnectionShutdownTotal).increment(1); } } @@ -76,7 +77,7 @@ impl InternalEvent for WebSocketConnectionError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "protocol" => PROTOCOL, "error_code" => "websocket_connection_error", "error_type" => error_type::WRITER_FAILED, @@ -121,7 +122,7 @@ impl InternalEvent for WebSocketBytesReceived<'_> { kind = %self.kind ); let counter = counter!( - "component_received_bytes_total", + CounterName::ComponentReceivedBytesTotal, "url" => self.url.to_string(), "protocol" => self.protocol, "kind" => self.kind.to_string() @@ -150,17 +151,17 @@ impl InternalEvent for WebSocketMessageReceived<'_> { kind = %self.kind ); - let histogram = histogram!("component_received_events_count"); + let histogram = histogram!(HistogramName::ComponentReceivedEventsCount); histogram.record(self.count as f64); let counter = counter!( - "component_received_events_total", + CounterName::ComponentReceivedEventsTotal, "uri" => self.url.to_string(), "protocol" => PROTOCOL, "kind" => self.kind.to_string() ); counter.increment(self.count as u64); let counter = counter!( - "component_received_event_bytes_total", + CounterName::ComponentReceivedEventBytesTotal, "url" => self.url.to_string(), "protocol" => PROTOCOL, "kind" => self.kind.to_string() @@ -184,7 +185,7 @@ impl InternalEvent for WebSocketReceiveError<'_> { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "protocol" => PROTOCOL, "error_code" => "websocket_receive_error", "error_type" => error_type::CONNECTION_FAILED, @@ -209,7 +210,7 @@ impl InternalEvent for WebSocketSendError<'_> { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "protocol" => PROTOCOL, "error_code" => "websocket_send_error", "error_type" => error_type::CONNECTION_FAILED, diff --git a/src/internal_events/websocket_server.rs b/src/internal_events/websocket_server.rs index fd064c41f2f8a..bdace6b079da5 100644 --- a/src/internal_events/websocket_server.rs +++ b/src/internal_events/websocket_server.rs @@ -1,8 +1,9 @@ use std::{error::Error, fmt::Debug}; -use metrics::{counter, gauge}; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, gauge, + internal_event::{CounterName, GaugeName, InternalEvent, error_stage, error_type}, +}; #[derive(Debug, NamedInternalEvent)] pub struct WebSocketListenerConnectionEstablished { @@ -18,8 +19,8 @@ impl InternalEvent for WebSocketListenerConnectionEstablished { self.client_count ) ); - counter!("connection_established_total", &self.extra_tags).increment(1); - gauge!("active_clients", &self.extra_tags).set(self.client_count as f64); + counter!(CounterName::ConnectionEstablishedTotal, &self.extra_tags).increment(1); + gauge!(GaugeName::ActiveClients, &self.extra_tags).set(self.client_count as f64); } } @@ -49,7 +50,7 @@ impl InternalEvent for WebSocketListenerConnectionFailedError { ]); // Tags required by `component_errors_total` are dynamically added above. // ## skip check-validity-events ## - counter!("component_errors_total", &all_tags).increment(1); + counter!(CounterName::ComponentErrorsTotal, &all_tags).increment(1); } } @@ -67,8 +68,8 @@ impl InternalEvent for WebSocketListenerConnectionShutdown { self.client_count ) ); - counter!("connection_shutdown_total", &self.extra_tags).increment(1); - gauge!("active_clients", &self.extra_tags).set(self.client_count as f64); + counter!(CounterName::ConnectionShutdownTotal, &self.extra_tags).increment(1); + gauge!(GaugeName::ActiveClients, &self.extra_tags).set(self.client_count as f64); } } @@ -87,7 +88,7 @@ impl InternalEvent for WebSocketListenerSendError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "ws_server_connection_error", "error_type" => error_type::WRITER_FAILED, "stage" => error_stage::SENDING, @@ -104,8 +105,8 @@ pub struct WebSocketListenerMessageSent { impl InternalEvent for WebSocketListenerMessageSent { fn emit(self) { - counter!("websocket_messages_sent_total", &self.extra_tags).increment(1); - counter!("websocket_bytes_sent_total", &self.extra_tags) + counter!(CounterName::WebsocketMessagesSentTotal, &self.extra_tags).increment(1); + counter!(CounterName::WebsocketBytesSentTotal, &self.extra_tags) .increment(self.message_size as u64); } } diff --git a/src/internal_events/windows.rs b/src/internal_events/windows.rs index 7b36e2fa62466..5ea6bae62aa25 100644 --- a/src/internal_events/windows.rs +++ b/src/internal_events/windows.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; #[derive(Debug, NamedInternalEvent)] pub struct WindowsServiceStart<'a> { @@ -16,7 +17,7 @@ impl InternalEvent for WindowsServiceStart<'_> { "Started Windows Service.", ); counter!( - "windows_service_start_total", + CounterName::WindowsServiceStartTotal, "already_started" => self.already_started.to_string(), ) .increment(1); @@ -37,7 +38,7 @@ impl InternalEvent for WindowsServiceStop<'_> { "Stopped Windows Service.", ); counter!( - "windows_service_stop_total", + CounterName::WindowsServiceStopTotal, "already_stopped" => self.already_stopped.to_string(), ) .increment(1); @@ -55,7 +56,7 @@ impl InternalEvent for WindowsServiceRestart<'_> { name = ?self.name, "Restarted Windows Service." ); - counter!("windows_service_restart_total").increment(1) + counter!(CounterName::WindowsServiceRestartTotal).increment(1) } } @@ -70,7 +71,7 @@ impl InternalEvent for WindowsServiceInstall<'_> { name = ?self.name, "Installed Windows Service.", ); - counter!("windows_service_install_total").increment(1); + counter!(CounterName::WindowsServiceInstallTotal).increment(1); } } @@ -85,7 +86,7 @@ impl InternalEvent for WindowsServiceUninstall<'_> { name = ?self.name, "Uninstalled Windows Service.", ); - counter!("windows_service_uninstall_total").increment(1); + counter!(CounterName::WindowsServiceUninstallTotal).increment(1); } } @@ -104,7 +105,7 @@ impl InternalEvent for WindowsServiceDoesNotExistError<'_> { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "service_missing", "error_type" => error_type::CONDITION_FAILED, "stage" => error_stage::PROCESSING, diff --git a/src/internal_events/windows_event_log.rs b/src/internal_events/windows_event_log.rs index ae5363e2c52be..7ec356ba253ff 100644 --- a/src/internal_events/windows_event_log.rs +++ b/src/internal_events/windows_event_log.rs @@ -1,8 +1,7 @@ -use metrics::counter; use tracing::error; use vector_lib::{ - NamedInternalEvent, - internal_event::{InternalEvent, error_stage, error_type}, + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, }; #[derive(Debug, NamedInternalEvent)] @@ -25,7 +24,7 @@ impl InternalEvent for WindowsEventLogParseError { internal_log_rate_limit = true, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "parse_failed", "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, @@ -54,7 +53,7 @@ impl InternalEvent for WindowsEventLogQueryError { internal_log_rate_limit = true, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "query_failed", "error_type" => error_type::REQUEST_FAILED, "stage" => error_stage::RECEIVING, @@ -81,7 +80,7 @@ impl InternalEvent for WindowsEventLogBookmarkError { internal_log_rate_limit = true, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "bookmark_failed", "error_type" => error_type::REQUEST_FAILED, "stage" => error_stage::PROCESSING, diff --git a/src/internal_telemetry/allocations/mod.rs b/src/internal_telemetry/allocations/mod.rs index a2680256ec167..15ad82b3665fa 100644 --- a/src/internal_telemetry/allocations/mod.rs +++ b/src/internal_telemetry/allocations/mod.rs @@ -11,8 +11,11 @@ use std::{ }; use arr_macro::arr; -use metrics::{counter, gauge}; use rand_distr::num_traits::ToPrimitive; +use vector_common::{ + counter, gauge, + internal_event::{CounterName, GaugeName}, +}; use self::allocator::Tracer; pub(crate) use self::allocator::{ @@ -144,26 +147,26 @@ pub fn init_allocation_tracing() { let group_info = group.lock().unwrap(); if allocations_diff > 0 { counter!( - "component_allocated_bytes_total", "component_kind" => group_info.component_kind.clone(), + CounterName::ComponentAllocatedBytesTotal, "component_kind" => group_info.component_kind.clone(), "component_type" => group_info.component_type.clone(), "component_id" => group_info.component_id.clone()).increment(allocations_diff); } if deallocations_diff > 0 { counter!( - "component_deallocated_bytes_total", "component_kind" => group_info.component_kind.clone(), + CounterName::ComponentDeallocatedBytesTotal, "component_kind" => group_info.component_kind.clone(), "component_type" => group_info.component_type.clone(), "component_id" => group_info.component_id.clone()).increment(deallocations_diff); } if mem_used_diff > 0 { gauge!( - "component_allocated_bytes", "component_type" => group_info.component_type.clone(), + GaugeName::ComponentAllocatedBytes, "component_type" => group_info.component_type.clone(), "component_id" => group_info.component_id.clone(), "component_kind" => group_info.component_kind.clone()) .increment(mem_used_diff.to_f64().expect("failed to convert mem_used from int to float")); } if mem_used_diff < 0 { gauge!( - "component_allocated_bytes", "component_type" => group_info.component_type.clone(), + GaugeName::ComponentAllocatedBytes, "component_type" => group_info.component_type.clone(), "component_id" => group_info.component_id.clone(), "component_kind" => group_info.component_kind.clone()) .decrement(-mem_used_diff.to_f64().expect("failed to convert mem_used from int to float")); diff --git a/src/sources/internal_metrics.rs b/src/sources/internal_metrics.rs index f576efa2a71a1..76dfab8849997 100644 --- a/src/sources/internal_metrics.rs +++ b/src/sources/internal_metrics.rs @@ -196,8 +196,13 @@ impl InternalMetrics<'_> { mod tests { use std::collections::BTreeMap; - use metrics::{counter, gauge, histogram}; - use vector_lib::{metric_tags, metrics::Controller}; + use strum::IntoEnumIterator; + use vector_lib::{ + counter, gauge, histogram, + internal_event::{CounterName, GaugeName, HistogramName}, + metric_tags, + metrics::Controller, + }; use super::*; use crate::{ @@ -223,14 +228,20 @@ mod tests { // There *seems* to be a race condition here (CI was flaky), so add a slight delay. std::thread::sleep(std::time::Duration::from_millis(300)); - gauge!("foo").set(1.0); - gauge!("foo").set(2.0); - counter!("bar").increment(3); - counter!("bar").increment(4); - histogram!("baz").record(5.0); - histogram!("baz").record(6.0); - histogram!("quux", "host" => "foo").record(8.0); - histogram!("quux", "host" => "foo").record(8.1); + let gauge_name = GaugeName::iter().next().unwrap(); + let counter_name = CounterName::iter().next().unwrap(); + let mut histogram_iter = HistogramName::iter(); + let histogram_name = histogram_iter.next().unwrap(); + let histogram_tagged_name = histogram_iter.next().unwrap(); + + gauge!(gauge_name).set(1.0); + gauge!(gauge_name).set(2.0); + counter!(counter_name).increment(3); + counter!(counter_name).increment(4); + histogram!(histogram_name).record(5.0); + histogram!(histogram_name).record(6.0); + histogram!(histogram_tagged_name, "host" => "foo").record(8.0); + histogram!(histogram_tagged_name, "host" => "foo").record(8.1); let controller = Controller::get().expect("no controller"); @@ -243,10 +254,16 @@ mod tests { .map(|metric| (metric.name().to_string(), metric)) .collect::>(); - assert_eq!(&MetricValue::Gauge { value: 2.0 }, output["foo"].value()); - assert_eq!(&MetricValue::Counter { value: 7.0 }, output["bar"].value()); + assert_eq!( + &MetricValue::Gauge { value: 2.0 }, + output[gauge_name.as_str()].value() + ); + assert_eq!( + &MetricValue::Counter { value: 7.0 }, + output[counter_name.as_str()].value() + ); - match &output["baz"].value() { + match &output[histogram_name.as_str()].value() { MetricValue::AggregatedHistogram { buckets, count, @@ -263,7 +280,7 @@ mod tests { _ => panic!("wrong type"), } - match &output["quux"].value() { + match &output[histogram_tagged_name.as_str()].value() { MetricValue::AggregatedHistogram { buckets, count, @@ -282,7 +299,7 @@ mod tests { } let labels = metric_tags!("host" => "foo"); - assert_eq!(Some(&labels), output["quux"].tags()); + assert_eq!(Some(&labels), output[histogram_tagged_name.as_str()].tags()); } async fn event_from_config(config: InternalMetricsConfig) -> Event { diff --git a/src/topology/builder.rs b/src/topology/builder.rs index 8948109fa7fb7..a25a99d337a26 100644 --- a/src/topology/builder.rs +++ b/src/topology/builder.rs @@ -8,7 +8,6 @@ use std::{ use futures::{FutureExt, StreamExt, TryStreamExt, stream::FuturesOrdered}; use futures_util::stream::FuturesUnordered; -use metrics::gauge; use stream_cancel::{StreamExt as StreamCancelExt, Trigger, Tripwire}; use tokio::{ select, @@ -31,6 +30,7 @@ use vector_lib::{ source_sender::{CHUNK_SIZE, SourceSenderItem}, transform::update_runtime_schema_definition, }; +use vector_lib::{gauge, internal_event::GaugeName}; use vector_vrl_metrics::MetricsStorage; use super::{ @@ -627,7 +627,7 @@ impl<'a> Builder<'a> { let utilization_sender = self .utilization_registry - .add_component(key.clone(), gauge!("utilization")); + .add_component(key.clone(), gauge!(GaugeName::Utilization)); let component_key = key.clone(); let sink = async move { debug!("Sink starting."); @@ -747,7 +747,7 @@ impl<'a> Builder<'a> { let sender = self .utilization_registry - .add_component(node.key.clone(), gauge!("utilization")); + .add_component(node.key.clone(), gauge!(GaugeName::Utilization)); let runner = Runner::new( t, input_rx, @@ -803,7 +803,7 @@ impl<'a> Builder<'a> { let sender = self .utilization_registry - .add_component(key.clone(), gauge!("utilization")); + .add_component(key.clone(), gauge!(GaugeName::Utilization)); let output_sender = sender.clone(); let input_rx = Utilization::new(sender, key.clone(), input_rx.into_stream()); diff --git a/src/utilization.rs b/src/utilization.rs index b3496b998f1c5..21d5f670bf1d9 100644 --- a/src/utilization.rs +++ b/src/utilization.rs @@ -344,6 +344,9 @@ impl OutputUtilization { mod tests { use mock_instant::global::MockClock; use serial_test::serial; + use strum::IntoEnumIterator; + use vector_lib::gauge; + use vector_lib::internal_event::GaugeName; use super::*; @@ -353,7 +356,7 @@ mod tests { MockClock::set_time(Duration::from_secs(100)); Timer::new( - metrics::gauge!("test_utilization"), + gauge!(GaugeName::iter().next().unwrap()), #[cfg(debug_assertions)] "test_component".into(), ) @@ -522,7 +525,7 @@ mod tests { let (mut emitter, registry) = UtilizationEmitter::new(); let key = ComponentKey::from("test_transform"); - let sender = registry.add_component(key.clone(), metrics::gauge!("utilization")); + let sender = registry.add_component(key.clone(), gauge!(GaugeName::Utilization)); let output_sender = sender.clone(); // Upstream channel carrying EventArrays. From 48be543ff3b84dfbd56a49cc6a0a0aac450bbceb Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Mon, 4 May 2026 10:01:46 -0400 Subject: [PATCH 190/326] fix(aws_sqs source): scope HistogramName import to s3 module (#25353) `HistogramName` was only used inside the `mod s3` block (gated on `sources-aws_s3`) but was imported at the parent module level under a broader cfg, breaking `cargo check --features sources-aws_sqs` with `#[deny(warnings)]`. Co-authored-by: Claude Opus 4.7 (1M context) --- src/internal_events/aws_sqs.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/internal_events/aws_sqs.rs b/src/internal_events/aws_sqs.rs index 51e8b78d23941..c04cb21b117fb 100644 --- a/src/internal_events/aws_sqs.rs +++ b/src/internal_events/aws_sqs.rs @@ -6,7 +6,7 @@ use vector_lib::counter; #[cfg(any(feature = "sources-aws_s3", feature = "sources-aws_sqs"))] use vector_lib::{ NamedInternalEvent, - internal_event::{CounterName, HistogramName, InternalEvent, error_stage, error_type}, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, }; #[cfg(feature = "sources-aws_s3")] @@ -18,6 +18,7 @@ mod s3 { SendMessageBatchRequestEntry, SendMessageBatchResultEntry, }; use vector_lib::histogram; + use vector_lib::internal_event::HistogramName; use super::*; use crate::sources::aws_s3::sqs::ProcessingError; From 96db40ef65b0d1246f06581987e5b6428468edf0 Mon Sep 17 00:00:00 2001 From: ArunPiduguDD Date: Mon, 4 May 2026 13:47:01 -0400 Subject: [PATCH 191/326] enhancement(metrics): Make transform-related functions in aggregate & tag cardinality transforms public (#25358) Make transform-related functions in aggregate & tag cardinality processors public --- src/transforms/aggregate.rs | 4 ++-- src/transforms/tag_cardinality_limit/mod.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/transforms/aggregate.rs b/src/transforms/aggregate.rs index c00315ad4871a..67de0815c6d29 100644 --- a/src/transforms/aggregate.rs +++ b/src/transforms/aggregate.rs @@ -181,7 +181,7 @@ impl Aggregate { }) } - fn record(&mut self, event: Event) { + pub fn record(&mut self, event: Event) { let (series, data, metadata) = event.into_metric().into_parts(); match &mut self.mode { @@ -300,7 +300,7 @@ impl Aggregate { } } - fn flush_into(&mut self, output: &mut Vec) { + pub fn flush_into(&mut self, output: &mut Vec) { let map = std::mem::take(&mut self.map); for (series, entry) in map.clone().into_iter() { let mut metric = Metric::from_parts(series, entry.0, entry.1); diff --git a/src/transforms/tag_cardinality_limit/mod.rs b/src/transforms/tag_cardinality_limit/mod.rs index 2f26e214a5b61..6b936c59ce02c 100644 --- a/src/transforms/tag_cardinality_limit/mod.rs +++ b/src/transforms/tag_cardinality_limit/mod.rs @@ -121,7 +121,7 @@ impl TagCardinalityLimit { .insert(value.clone()); } - fn transform_one(&mut self, mut event: Event) -> Option { + pub fn transform_one(&mut self, mut event: Event) -> Option { let metric = event.as_mut_metric(); let metric_name = metric.name().to_string(); let metric_namespace = metric.namespace().map(|n| n.to_string()); From 8073e93b48352f781aace209f816ac55280a8935 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Mon, 4 May 2026 15:13:10 -0400 Subject: [PATCH 192/326] chore(ci): rename WIP label workflows to docs review label (#25355) Rename `add_wip_label.yml` and `remove_wip_label.yml` to `add_docs_review_label.yml` and `remove_docs_review_label.yml`, and switch the managed label from "work in progress" to "docs review on hold". The label string is now hoisted to a workflow-level `LABEL_NAME` env var so future renames are a one-line edit. Header comments note we can promote these to a reusable `workflow_call` workflow if a second use case appears. Co-authored-by: Claude Opus 4.7 (1M context) --- .github/CODEOWNERS | 2 +- ...ip_label.yml => add_docs_review_label.yml} | 12 +++++++++--- ...label.yml => remove_docs_review_label.yml} | 19 +++++++++++++------ 3 files changed, 23 insertions(+), 10 deletions(-) rename .github/workflows/{add_wip_label.yml => add_docs_review_label.yml} (81%) rename .github/workflows/{remove_wip_label.yml => remove_docs_review_label.yml} (68%) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ecd6fb374af9c..7534be2fb8652 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -3,7 +3,7 @@ .github/workflows/regression.yml @vectordotdev/vector @vectordotdev/single-machine-performance regression/config.yaml @vectordotdev/vector @vectordotdev/single-machine-performance -# Keep documentation team paths in sync with .github/workflows/add_wip_label.yml +# Keep documentation team paths in sync with .github/workflows/add_docs_review_label.yml docs/ @vectordotdev/vector @vectordotdev/documentation website/ @vectordotdev/vector website/content @vectordotdev/vector @vectordotdev/documentation diff --git a/.github/workflows/add_wip_label.yml b/.github/workflows/add_docs_review_label.yml similarity index 81% rename from .github/workflows/add_wip_label.yml rename to .github/workflows/add_docs_review_label.yml index 7924d4fade926..23f9a697397af 100644 --- a/.github/workflows/add_wip_label.yml +++ b/.github/workflows/add_docs_review_label.yml @@ -1,4 +1,7 @@ -name: Add Work In Progress Label +# Adds a label to PRs that touch documentation paths until a maintainer approves. +# Currently dedicated to the docs review flow. If a second use case appears, promote +# this into a reusable workflow (`workflow_call`) with the label as an input. +name: Add Docs Review Label permissions: pull-requests: write @@ -12,8 +15,11 @@ on: - "website/content/**" - "website/cue/reference/**" +env: + LABEL_NAME: "docs review on hold" + jobs: - add_wip_label: + add_label: runs-on: ubuntu-24.04 timeout-minutes: 5 steps: @@ -47,4 +53,4 @@ jobs: - if: steps.check.outputs.skip != 'true' uses: actions-ecosystem/action-add-labels@18f1af5e3544586314bbe15c0273249c770b2daf # v1.1.3 with: - labels: "work in progress" + labels: ${{ env.LABEL_NAME }} diff --git a/.github/workflows/remove_wip_label.yml b/.github/workflows/remove_docs_review_label.yml similarity index 68% rename from .github/workflows/remove_wip_label.yml rename to .github/workflows/remove_docs_review_label.yml index b4750b0f2b0d6..093c5a4bd5e90 100644 --- a/.github/workflows/remove_wip_label.yml +++ b/.github/workflows/remove_docs_review_label.yml @@ -1,4 +1,7 @@ -name: Remove Work In Progress Label +# Removes the docs review label once a maintainer approves the PR. +# Currently dedicated to the docs review flow. If a second use case appears, promote +# this into a reusable workflow (`workflow_call`) with the label as an input. +name: Remove Docs Review Label permissions: issues: write @@ -8,8 +11,11 @@ on: pull_request_review: types: [submitted] +env: + LABEL_NAME: "docs review on hold" + jobs: - remove_wip_label: + remove_label: if: github.event.review.state == 'approved' runs-on: ubuntu-24.04 timeout-minutes: 5 @@ -27,20 +33,21 @@ jobs: return; } + const labelName = process.env.LABEL_NAME; const { data: labels } = await github.rest.issues.listLabelsOnIssue({ owner: context.repo.owner, repo: context.repo.repo, issue_number: context.payload.pull_request.number, }); - const hasLabel = labels.some(l => l.name === 'work in progress'); + const hasLabel = labels.some(l => l.name === labelName); if (!hasLabel) { - core.info('Label "work in progress" not found, skipping'); + core.info(`Label "${labelName}" not found, skipping`); return; } await github.rest.issues.removeLabel({ owner: context.repo.owner, repo: context.repo.repo, issue_number: context.payload.pull_request.number, - name: 'work in progress', + name: labelName, }); - core.info('Removed "work in progress" label'); + core.info(`Removed "${labelName}" label`); From 4524b52c921447dfca228338add190f0363b582e Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 4 May 2026 19:08:38 -0400 Subject: [PATCH 193/326] chore(deps): upgrade hickory-proto to 0.26.1, ignore RUSTSEC-2026-0119 (#25354) * chore(deps): upgrade hickory-proto to 0.26.1, ignore RUSTSEC-2026-0119 * Format * fix(dnstap source): include EDNS OPT record in additional_count --- Cargo.lock | 42 +- Cargo.toml | 2 +- deny.toml | 1 + lib/dnsmsg-parser/src/dns_message_parser.rs | 410 ++++++++++---------- lib/dnsmsg-parser/src/ede.rs | 11 +- 5 files changed, 226 insertions(+), 240 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2e217d2d24c4f..cb014b2cd372a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3782,7 +3782,7 @@ version = "0.1.0" dependencies = [ "criterion", "data-encoding", - "hickory-proto 0.25.2", + "hickory-proto 0.26.1", "snafu 0.9.0", ] @@ -3796,7 +3796,7 @@ dependencies = [ "chrono", "chrono-tz", "dnsmsg-parser", - "hickory-proto 0.25.2", + "hickory-proto 0.26.1", "pastey", "prost 0.12.6", "prost-build 0.12.6", @@ -5134,7 +5134,7 @@ dependencies = [ "futures-channel", "futures-io", "futures-util", - "hickory-proto 0.26.0", + "hickory-proto 0.26.1", "idna", "ipnet", "jni 0.22.4", @@ -5172,37 +5172,11 @@ dependencies = [ [[package]] name = "hickory-proto" -version = "0.25.2" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8a6fe56c0038198998a6f217ca4e7ef3a5e51f46163bd6dd60b5c71ca6c6502" +checksum = "0bab31817bfb44672a252e97fe81cd0c18d1b2cf892108922f6818820df8c643" dependencies = [ - "async-trait", "bitflags 2.10.0", - "cfg-if", - "data-encoding", - "enum-as-inner", - "futures-channel", - "futures-io", - "futures-util", - "idna", - "ipnet", - "once_cell", - "rand 0.9.4", - "ring", - "rustls-pki-types", - "thiserror 2.0.17", - "time", - "tinyvec", - "tracing 0.1.44", - "url", -] - -[[package]] -name = "hickory-proto" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a916d0494600d99ecb15aadfab677ad97c4de559e8f1af0c129353a733ac1fcc" -dependencies = [ "data-encoding", "idna", "ipnet", @@ -5211,7 +5185,9 @@ dependencies = [ "prefix-trie", "rand 0.10.1", "ring", + "rustls-pki-types", "thiserror 2.0.17", + "time", "tinyvec", "tracing 0.1.44", "url", @@ -5247,7 +5223,7 @@ dependencies = [ "cfg-if", "futures-util", "hickory-net", - "hickory-proto 0.26.0", + "hickory-proto 0.26.1", "ipconfig", "ipnet", "jni 0.22.4", @@ -12854,7 +12830,7 @@ dependencies = [ "headers", "heim", "hex", - "hickory-proto 0.25.2", + "hickory-proto 0.26.1", "hostname 0.4.2", "http 0.2.12", "http 1.3.1", diff --git a/Cargo.toml b/Cargo.toml index e9cf7da0f400b..f28e8feabbfd5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -160,7 +160,7 @@ flate2 = { version = "1.1.2", default-features = false, features = ["zlib-rs"] } futures = { version = "0.3.31", default-features = false, features = ["std"] } futures-util = { version = "0.3.29", default-features = false } glob = { version = "0.3.3", default-features = false } -hickory-proto = { version = "0.25.2", default-features = false, features = ["dnssec-ring"] } +hickory-proto = { version = "0.26.1", default-features = false, features = ["dnssec-ring"] } humantime = { version = "2.3.0", default-features = false } indexmap = { version = "2.11.0", default-features = false, features = ["serde", "std"] } indoc = { version = "2.0.7" } diff --git a/deny.toml b/deny.toml index 0d563622a509e..37db54adf7956 100644 --- a/deny.toml +++ b/deny.toml @@ -50,4 +50,5 @@ ignore = [ { id = "RUSTSEC-2026-0097", reason = "rand 0.8.5 unsound with custom logger - transitive dependency, upstream crates have not updated to rand 0.9+" }, { id = "RUSTSEC-2026-0104", reason = "rustls-webpki 0.102/0.101 CRL parsing panic - tonic upgrade (https://github.com/vectordotdev/vector/issues/19179); 0.103 already patched to 0.103.13" }, { id = "RUSTSEC-2026-0105", reason = "core2 is unmaintained and all versions yanked - transitive dependency via libflate -> apache-avro, no safe upgrade available" }, + { id = "RUSTSEC-2026-0119", reason = "hickory-proto 0.24 - unpatched crate (https://github.com/mongodb/mongo-rust-driver/pull/1682)" }, ] diff --git a/lib/dnsmsg-parser/src/dns_message_parser.rs b/lib/dnsmsg-parser/src/dns_message_parser.rs index d0b271b2e7930..a22b630ffaca4 100644 --- a/lib/dnsmsg-parser/src/dns_message_parser.rs +++ b/lib/dnsmsg-parser/src/dns_message_parser.rs @@ -7,16 +7,13 @@ use hickory_proto::{ PublicKey, SupportedAlgorithms, Verifier, rdata::{CDNSKEY, CDS, DNSKEY, DNSSECRData, DS}, }, - op::{Query, message::Message as TrustDnsMessage}, + op::{Message as TrustDnsMessage, Query}, rr::{ - Name, RecordType, + Name, RData, Record, RecordType, rdata::{ A, AAAA, NULL, OPT, SVCB, - caa::Property, opt::{EdnsCode, EdnsOption}, }, - record_data::RData, - resource::Record, }, serialize::binary::{BinDecodable, BinDecoder}, }; @@ -101,8 +98,11 @@ impl DnsMessageParser { } pub fn parse_as_query_message(&mut self) -> DnsParserResult { - let msg = TrustDnsMessage::from_vec(&self.raw_message) - .map_err(|source| DnsMessageParserError::TrustDnsError { source })?; + let msg = TrustDnsMessage::from_vec(&self.raw_message).map_err(|source| { + DnsMessageParserError::TrustDnsError { + source: ProtoError::from(source), + } + })?; let header = parse_dns_query_message_header(&msg); let edns_section = parse_edns(&msg).transpose()?; let rcode_high = edns_section.as_ref().map_or(0, |edns| edns.extended_rcode); @@ -113,16 +113,19 @@ impl DnsMessageParser { response: parse_response_code(response_code), header, question_section: self.parse_dns_query_message_question_section(&msg), - answer_section: self.parse_dns_message_section(msg.answers())?, - authority_section: self.parse_dns_message_section(msg.name_servers())?, - additional_section: self.parse_dns_message_section(msg.additionals())?, + answer_section: self.parse_dns_message_section(&msg.answers)?, + authority_section: self.parse_dns_message_section(&msg.authorities)?, + additional_section: self.parse_dns_message_section(&msg.additionals)?, opt_pseudo_section: edns_section, }) } pub fn parse_as_update_message(&mut self) -> DnsParserResult { - let msg = TrustDnsMessage::from_vec(&self.raw_message) - .map_err(|source| DnsMessageParserError::TrustDnsError { source })?; + let msg = TrustDnsMessage::from_vec(&self.raw_message).map_err(|source| { + DnsMessageParserError::TrustDnsError { + source: ProtoError::from(source), + } + })?; let header = parse_dns_update_message_header(&msg); let response_code = (u16::from(header.rcode)) & 0x000F; Ok(DnsUpdateMessage { @@ -130,9 +133,9 @@ impl DnsMessageParser { response: parse_response_code(response_code), header, zone_to_update: self.parse_dns_update_message_zone_section(&msg)?, - prerequisite_section: self.parse_dns_message_section(msg.answers())?, - update_section: self.parse_dns_message_section(msg.name_servers())?, - additional_section: self.parse_dns_message_section(msg.additionals())?, + prerequisite_section: self.parse_dns_message_section(&msg.answers)?, + update_section: self.parse_dns_message_section(&msg.authorities)?, + additional_section: self.parse_dns_message_section(&msg.additionals)?, }) } @@ -141,7 +144,7 @@ impl DnsMessageParser { dns_message: &TrustDnsMessage, ) -> Vec { dns_message - .queries() + .queries .iter() .map(|query| self.parse_dns_query_question(query)) .collect() @@ -161,7 +164,7 @@ impl DnsMessageParser { dns_message: &TrustDnsMessage, ) -> DnsParserResult { let zones = dns_message - .queries() + .queries .iter() .map(|query| self.parse_dns_query_question(query).into()) .collect::>(); @@ -185,18 +188,18 @@ impl DnsMessageParser { } pub(crate) fn parse_dns_record(&mut self, record: &Record) -> DnsParserResult { - let record_data = match record.data() { - RData::Unknown { code, rdata } => self.format_unknown_rdata((*code).into(), rdata), + let record_data = match &record.data { + RData::Unknown { code, rdata } => self.format_unknown_rdata(u16::from(*code), rdata), RData::Update0(_) => Ok((Some(String::from("")), None)), // Previously none value rdata => self.format_rdata(rdata), }?; Ok(DnsRecord { - name: record.name().to_string_with_options(&self.options), - class: record.dns_class().to_string(), + name: record.name.to_string_with_options(&self.options), + class: record.dns_class.to_string(), record_type: format_record_type(record.record_type()), record_type_id: u16::from(record.record_type()), - ttl: record.ttl(), + ttl: record.ttl, rdata: record_data.0, rdata_bytes: record_data.1, }) @@ -381,30 +384,30 @@ impl DnsMessageParser { match code { dns_message::RTYPE_MB => { let options = self.options.clone(); - let mut decoder = self.get_rdata_decoder_with_raw_message(rdata.anything()); + let mut decoder = self.get_rdata_decoder_with_raw_message(&rdata.anything); let madname = Self::parse_domain_name(&mut decoder, &options)?; Ok((Some(madname), None)) } dns_message::RTYPE_MG => { let options = self.options.clone(); - let mut decoder = self.get_rdata_decoder_with_raw_message(rdata.anything()); + let mut decoder = self.get_rdata_decoder_with_raw_message(&rdata.anything); let mgname = Self::parse_domain_name(&mut decoder, &options)?; Ok((Some(mgname), None)) } dns_message::RTYPE_MR => { let options = self.options.clone(); - let mut decoder = self.get_rdata_decoder_with_raw_message(rdata.anything()); + let mut decoder = self.get_rdata_decoder_with_raw_message(&rdata.anything); let newname = Self::parse_domain_name(&mut decoder, &options)?; Ok((Some(newname), None)) } - dns_message::RTYPE_WKS => self.parse_wks_rdata(rdata.anything()), + dns_message::RTYPE_WKS => self.parse_wks_rdata(&rdata.anything), dns_message::RTYPE_MINFO => { let options = self.options.clone(); - let mut decoder = self.get_rdata_decoder_with_raw_message(rdata.anything()); + let mut decoder = self.get_rdata_decoder_with_raw_message(&rdata.anything); let rmailbx = Self::parse_domain_name(&mut decoder, &options)?; let emailbx = Self::parse_domain_name(&mut decoder, &options)?; Ok((Some(format!("{rmailbx} {emailbx}")), None)) @@ -412,7 +415,7 @@ impl DnsMessageParser { dns_message::RTYPE_RP => { let options = self.options.clone(); - let mut decoder = self.get_rdata_decoder_with_raw_message(rdata.anything()); + let mut decoder = self.get_rdata_decoder_with_raw_message(&rdata.anything); let mbox = Self::parse_domain_name(&mut decoder, &options)?; let txt = Self::parse_domain_name(&mut decoder, &options)?; Ok((Some(format!("{mbox} {txt}")), None)) @@ -420,14 +423,14 @@ impl DnsMessageParser { dns_message::RTYPE_AFSDB => { let options = self.options.clone(); - let mut decoder = self.get_rdata_decoder_with_raw_message(rdata.anything()); + let mut decoder = self.get_rdata_decoder_with_raw_message(&rdata.anything); let subtype = parse_u16(&mut decoder)?; let hostname = Self::parse_domain_name(&mut decoder, &options)?; Ok((Some(format!("{subtype} {hostname}")), None)) } dns_message::RTYPE_X25 => { - let mut decoder = BinDecoder::new(rdata.anything()); + let mut decoder = BinDecoder::new(&rdata.anything); let psdn_address = parse_character_string(&mut decoder)?; Ok(( Some(format!( @@ -439,7 +442,7 @@ impl DnsMessageParser { } dns_message::RTYPE_ISDN => { - let mut decoder = BinDecoder::new(rdata.anything()); + let mut decoder = BinDecoder::new(&rdata.anything); let address = parse_character_string(&mut decoder)?; if decoder.is_empty() { Ok(( @@ -464,14 +467,14 @@ impl DnsMessageParser { dns_message::RTYPE_RT => { let options = self.options.clone(); - let mut decoder = self.get_rdata_decoder_with_raw_message(rdata.anything()); + let mut decoder = self.get_rdata_decoder_with_raw_message(&rdata.anything); let preference = parse_u16(&mut decoder)?; let intermediate_host = Self::parse_domain_name(&mut decoder, &options)?; Ok((Some(format!("{preference} {intermediate_host}")), None)) } dns_message::RTYPE_NSAP => { - let raw_rdata = rdata.anything(); + let raw_rdata = &rdata.anything; let mut decoder = BinDecoder::new(raw_rdata); let rdata_len = raw_rdata.len() as u16; let nsap_rdata = HEXUPPER.encode(&parse_vec_with_u16_len(&mut decoder, rdata_len)?); @@ -480,27 +483,27 @@ impl DnsMessageParser { dns_message::RTYPE_PX => { let options = self.options.clone(); - let mut decoder = self.get_rdata_decoder_with_raw_message(rdata.anything()); + let mut decoder = self.get_rdata_decoder_with_raw_message(&rdata.anything); let preference = parse_u16(&mut decoder)?; let map822 = Self::parse_domain_name(&mut decoder, &options)?; let mapx400 = Self::parse_domain_name(&mut decoder, &options)?; Ok((Some(format!("{preference} {map822} {mapx400}")), None)) } - dns_message::RTYPE_LOC => self.parse_loc_rdata(rdata.anything()), + dns_message::RTYPE_LOC => self.parse_loc_rdata(&rdata.anything), dns_message::RTYPE_KX => { let options = self.options.clone(); - let mut decoder = self.get_rdata_decoder_with_raw_message(rdata.anything()); + let mut decoder = self.get_rdata_decoder_with_raw_message(&rdata.anything); let preference = parse_u16(&mut decoder)?; let exchanger = Self::parse_domain_name(&mut decoder, &options)?; Ok((Some(format!("{preference} {exchanger}")), None)) } - dns_message::RTYPE_A6 => self.parse_a6_rdata(rdata.anything()), + dns_message::RTYPE_A6 => self.parse_a6_rdata(&rdata.anything), dns_message::RTYPE_SINK => { - let raw_rdata = rdata.anything(); + let raw_rdata = &rdata.anything; let mut decoder = BinDecoder::new(raw_rdata); let meaning = parse_u8(&mut decoder)?; let coding = parse_u8(&mut decoder)?; @@ -511,10 +514,10 @@ impl DnsMessageParser { Ok((Some(format!("{meaning} {coding} {subcoding} {data}")), None)) } - dns_message::RTYPE_APL => self.parse_apl_rdata(rdata.anything()), + dns_message::RTYPE_APL => self.parse_apl_rdata(&rdata.anything), dns_message::RTYPE_DHCID => { - let raw_rdata = rdata.anything(); + let raw_rdata = &rdata.anything; let mut decoder = BinDecoder::new(raw_rdata); let raw_data_len = raw_rdata.len() as u16; let digest = BASE64.encode(&parse_vec_with_u16_len(&mut decoder, raw_data_len)?); @@ -522,7 +525,7 @@ impl DnsMessageParser { } dns_message::RTYPE_SPF => { - let mut decoder = BinDecoder::new(rdata.anything()); + let mut decoder = BinDecoder::new(&rdata.anything); let mut text = String::new(); while !decoder.is_empty() { text.push('\"'); @@ -532,7 +535,7 @@ impl DnsMessageParser { Ok((Some(text.trim_end().to_string()), None)) } - _ => Ok((None, Some(rdata.anything().to_vec()))), + _ => Ok((None, Some(rdata.anything.clone()))), } } @@ -543,13 +546,13 @@ impl DnsMessageParser { RData::ANAME(name) => Ok((Some(name.to_string_with_options(&self.options)), None)), RData::CNAME(name) => Ok((Some(name.to_string_with_options(&self.options)), None)), RData::CERT(cert) => { - let crl = BASE64.encode(&cert.cert_data()); + let crl = BASE64.encode(&cert.cert_data); Ok(( Some(format!( "{} {} {} {}", - u16::from(cert.cert_type()), - cert.key_tag(), - cert.algorithm(), + u16::from(cert.cert_type), + cert.key_tag, + cert.algorithm, crl )), None, @@ -564,15 +567,15 @@ impl DnsMessageParser { RData::MX(mx) => { let srv_rdata = format!( "{} {}", - mx.preference(), - mx.exchange().to_string_with_options(&self.options), + mx.preference, + mx.exchange.to_string_with_options(&self.options), ); Ok((Some(srv_rdata), None)) } - RData::NULL(null) => Ok((Some(BASE64.encode(null.anything())), None)), + RData::NULL(null) => Ok((Some(BASE64.encode(&null.anything)), None)), RData::NS(ns) => Ok((Some(ns.to_string_with_options(&self.options)), None)), RData::OPENPGPKEY(key) => { - if let Ok(key_string) = String::from_utf8(Vec::from(key.public_key())) { + if let Ok(key_string) = String::from_utf8(key.public_key.clone()) { Ok((Some(format!("({})", &key_string)), None)) } else { Err(DnsMessageParserError::SimpleError { @@ -584,29 +587,29 @@ impl DnsMessageParser { RData::SOA(soa) => Ok(( Some(format!( "{} {} {} {} {} {} {}", - soa.mname().to_string_with_options(&self.options), - soa.rname().to_string_with_options(&self.options), - soa.serial(), - soa.refresh(), - soa.retry(), - soa.expire(), - soa.minimum() + soa.mname.to_string_with_options(&self.options), + soa.rname.to_string_with_options(&self.options), + soa.serial, + soa.refresh, + soa.retry, + soa.expire, + soa.minimum )), None, )), RData::SRV(srv) => { let srv_rdata = format!( "{} {} {} {}", - srv.priority(), - srv.weight(), - srv.port(), - srv.target().to_string_with_options(&self.options) + srv.priority, + srv.weight, + srv.port, + srv.target.to_string_with_options(&self.options) ); Ok((Some(srv_rdata), None)) } RData::TXT(txt) => { let txt_rdata = txt - .txt_data() + .txt_data .iter() .map(|value| { format!( @@ -623,41 +626,35 @@ impl DnsMessageParser { RData::CAA(caa) => { let caa_rdata = format!( "{} {} \"{}\"", - caa.issuer_critical() as u8, - caa.tag().as_str(), - match caa.tag() { - Property::Iodef => { - let url = caa.value_as_iodef().map_err(|source| { - DnsMessageParserError::TrustDnsError { source } - })?; - url.as_str().to_string() - } - Property::Issue | Property::IssueWild => { - let (option_name, vec_keyvalue) = - caa.value_as_issue().map_err(|source| { - DnsMessageParserError::TrustDnsError { source } - })?; - - let mut final_issuer = String::new(); - if let Some(name) = option_name { - final_issuer.push_str(&name.to_string_with_options(&self.options)); - for keyvalue in vec_keyvalue.iter() { - final_issuer.push_str("; "); - final_issuer.push_str(keyvalue.key()); - final_issuer.push('='); - final_issuer.push_str(keyvalue.value()); - } + caa.issuer_critical as u8, + &caa.tag, + if caa.tag.eq_ignore_ascii_case("iodef") { + let url = caa + .value_as_iodef() + .map_err(|source| DnsMessageParserError::TrustDnsError { source })?; + url.as_str().to_string() + } else if caa.tag.eq_ignore_ascii_case("issue") + || caa.tag.eq_ignore_ascii_case("issuewild") + { + let (option_name, vec_keyvalue) = caa + .value_as_issue() + .map_err(|source| DnsMessageParserError::TrustDnsError { source })?; + + let mut final_issuer = String::new(); + if let Some(name) = option_name { + final_issuer.push_str(&name.to_string_with_options(&self.options)); + for keyvalue in vec_keyvalue.iter() { + final_issuer.push_str("; "); + final_issuer.push_str(keyvalue.key()); + final_issuer.push('='); + final_issuer.push_str(keyvalue.value()); } - final_issuer.trim_end().to_string() - } - Property::Unknown(_) => { - let unknown = caa.raw_value(); - std::str::from_utf8(unknown) - .map_err(|source| DnsMessageParserError::Utf8ParsingError { - source, - })? - .to_string() } + final_issuer.trim_end().to_string() + } else { + std::str::from_utf8(&caa.value) + .map_err(|source| DnsMessageParserError::Utf8ParsingError { source })? + .to_string() } ); Ok((Some(caa_rdata), None)) @@ -666,52 +663,52 @@ impl DnsMessageParser { RData::TLSA(tlsa) => { let tlsa_rdata = format!( "{} {} {} {}", - u8::from(tlsa.cert_usage()), - u8::from(tlsa.selector()), - u8::from(tlsa.matching()), - HEXUPPER.encode(tlsa.cert_data()) + u8::from(tlsa.cert_usage), + u8::from(tlsa.selector), + u8::from(tlsa.matching), + HEXUPPER.encode(&tlsa.cert_data) ); Ok((Some(tlsa_rdata), None)) } RData::SSHFP(sshfp) => { let sshfp_rdata = format!( "{} {} {}", - Into::::into(sshfp.algorithm()), - Into::::into(sshfp.fingerprint_type()), - HEXUPPER.encode(sshfp.fingerprint()) + Into::::into(sshfp.algorithm), + Into::::into(sshfp.fingerprint_type), + HEXUPPER.encode(&sshfp.fingerprint) ); Ok((Some(sshfp_rdata), None)) } RData::NAPTR(naptr) => { let naptr_rdata = format!( r#"{} {} "{}" "{}" "{}" {}"#, - naptr.order(), - naptr.preference(), + naptr.order, + naptr.preference, escape_string_for_text_representation( - std::str::from_utf8(naptr.flags()) + std::str::from_utf8(&naptr.flags) .map_err(|source| DnsMessageParserError::Utf8ParsingError { source })? .to_string() ), escape_string_for_text_representation( - std::str::from_utf8(naptr.services()) + std::str::from_utf8(&naptr.services) .map_err(|source| DnsMessageParserError::Utf8ParsingError { source })? .to_string() ), escape_string_for_text_representation( - std::str::from_utf8(naptr.regexp()) + std::str::from_utf8(&naptr.regexp) .map_err(|source| DnsMessageParserError::Utf8ParsingError { source })? .to_string() ), - naptr.replacement().to_string_with_options(&self.options) + naptr.replacement.to_string_with_options(&self.options) ); Ok((Some(naptr_rdata), None)) } RData::HINFO(hinfo) => { let hinfo_data = format!( r#""{}" "{}""#, - std::str::from_utf8(hinfo.cpu()) + std::str::from_utf8(&hinfo.cpu) .map_err(|source| DnsMessageParserError::Utf8ParsingError { source })?, - std::str::from_utf8(hinfo.os()) + std::str::from_utf8(&hinfo.os) .map_err(|source| DnsMessageParserError::Utf8ParsingError { source })?, ); Ok((Some(hinfo_data), None)) @@ -794,37 +791,41 @@ impl DnsMessageParser { DNSSECRData::SIG(sig) => { let sig_rdata = format!( "{} {} {} {} {} {} {} {} {}", - match format_record_type(sig.type_covered()) { + match format_record_type(sig.input().type_covered) { Some(record_type) => record_type, None => String::from("Unknown record type"), }, - u8::from(sig.algorithm()), - sig.num_labels(), - sig.original_ttl(), - sig.sig_expiration().get(), // currently in epoch convert to human readable ? - sig.sig_inception().get(), // currently in epoch convert to human readable ? - sig.key_tag(), - sig.signer_name().to_string_with_options(&self.options), + u8::from(sig.input().algorithm), + sig.input().num_labels, + sig.input().original_ttl, + sig.input().sig_expiration.get(), // currently in epoch convert to human readable ? + sig.input().sig_inception.get(), // currently in epoch convert to human readable ? + sig.input().key_tag, + sig.input() + .signer_name + .to_string_with_options(&self.options), BASE64.encode(sig.sig()) ); Ok((Some(sig_rdata), None)) } - // RSIG is a derivation of SIG but choosing to keep this duplicate code in lieu of the alternative + // RRSIG is a derivation of SIG but choosing to keep this duplicate code in lieu of the alternative // which is to allocate to the heap with Box in order to deref. DNSSECRData::RRSIG(sig) => { let sig_rdata = format!( "{} {} {} {} {} {} {} {} {}", - match format_record_type(sig.type_covered()) { + match format_record_type(sig.input().type_covered) { Some(record_type) => record_type, None => String::from("Unknown record type"), }, - u8::from(sig.algorithm()), - sig.num_labels(), - sig.original_ttl(), - sig.sig_expiration().get(), // currently in epoch convert to human readable ? - sig.sig_inception().get(), // currently in epoch convert to human readable ? - sig.key_tag(), - sig.signer_name().to_string_with_options(&self.options), + u8::from(sig.input().algorithm), + sig.input().num_labels, + sig.input().original_ttl, + sig.input().sig_expiration.get(), // currently in epoch convert to human readable ? + sig.input().sig_inception.get(), // currently in epoch convert to human readable ? + sig.input().key_tag, + sig.input() + .signer_name + .to_string_with_options(&self.options), BASE64.encode(sig.sig()) ); Ok((Some(sig_rdata), None)) @@ -839,9 +840,7 @@ impl DnsMessageParser { ); Ok((Some(key_rdata), None)) } - DNSSECRData::Unknown { code: _, rdata } => { - Ok((None, Some(rdata.anything().to_vec()))) - } + DNSSECRData::Unknown { code: _, rdata } => Ok((None, Some(rdata.anything.clone()))), _ => Err(DnsMessageParserError::SimpleError { cause: format!("Unsupported rdata {rdata:?}"), }), @@ -870,9 +869,9 @@ fn format_record_type(record_type: RecordType) -> Option { fn format_svcb_record(svcb: &SVCB, options: &DnsParserOptions) -> String { format!( "{} {} {}", - svcb.svc_priority(), - svcb.target_name().to_string_with_options(options), - svcb.svc_params() + svcb.svc_priority, + svcb.target_name.to_string_with_options(options), + svcb.svc_params .iter() .map(|(key, value)| format!(r#"{}="{}""#, key, value.to_string().trim_end_matches(','))) .collect::>() @@ -965,38 +964,38 @@ fn parse_response_code(rcode: u16) -> Option<&'static str> { fn parse_dns_query_message_header(dns_message: &TrustDnsMessage) -> QueryHeader { QueryHeader { - id: dns_message.header().id(), - opcode: dns_message.header().op_code().into(), - rcode: dns_message.header().response_code(), - qr: dns_message.header().message_type() as u8, - aa: dns_message.header().authoritative(), - tc: dns_message.header().truncated(), - rd: dns_message.header().recursion_desired(), - ra: dns_message.header().recursion_available(), - ad: dns_message.header().authentic_data(), - cd: dns_message.header().checking_disabled(), - question_count: dns_message.header().query_count(), - answer_count: dns_message.header().answer_count(), - authority_count: dns_message.header().name_server_count(), - additional_count: dns_message.header().additional_count(), + id: dns_message.id, + opcode: dns_message.op_code.into(), + rcode: dns_message.response_code, + qr: dns_message.message_type as u8, + aa: dns_message.authoritative, + tc: dns_message.truncation, + rd: dns_message.recursion_desired, + ra: dns_message.recursion_available, + ad: dns_message.authentic_data, + cd: dns_message.checking_disabled, + question_count: dns_message.queries.len() as u16, + answer_count: dns_message.answers.len() as u16, + authority_count: dns_message.authorities.len() as u16, + additional_count: dns_message.additionals.len() as u16 + dns_message.edns.is_some() as u16, } } fn parse_dns_update_message_header(dns_message: &TrustDnsMessage) -> UpdateHeader { UpdateHeader { - id: dns_message.header().id(), - opcode: dns_message.header().op_code().into(), - rcode: dns_message.header().response_code(), - qr: dns_message.header().message_type() as u8, - zone_count: dns_message.header().query_count(), - prerequisite_count: dns_message.header().answer_count(), - update_count: dns_message.header().name_server_count(), - additional_count: dns_message.header().additional_count(), + id: dns_message.id, + opcode: dns_message.op_code.into(), + rcode: dns_message.response_code, + qr: dns_message.message_type as u8, + zone_count: dns_message.queries.len() as u16, + prerequisite_count: dns_message.answers.len() as u16, + update_count: dns_message.authorities.len() as u16, + additional_count: dns_message.additionals.len() as u16 + dns_message.edns.is_some() as u16, } } fn parse_edns(dns_message: &TrustDnsMessage) -> Option> { - dns_message.extensions().as_ref().map(|edns| { + dns_message.edns.as_ref().map(|edns| { parse_edns_options(edns.options()).map(|(ede, rest)| OptPseudoSection { extended_rcode: edns.rcode_high(), version: edns.version(), @@ -1014,10 +1013,11 @@ fn parse_edns_options(edns: &OPT) -> DnsParserResult<(Vec, Vec) -> DnsParserResult { Ok(::read(decoder) - .map_err(|source| DnsMessageParserError::TrustDnsError { source })? + .map_err(|source| DnsMessageParserError::TrustDnsError { + source: ProtoError::from(source), + })? .to_string()) } fn parse_ipv4_address(decoder: &mut BinDecoder<'_>) -> DnsParserResult { Ok(::read(decoder) - .map_err(|source| DnsMessageParserError::TrustDnsError { source })? + .map_err(|source| DnsMessageParserError::TrustDnsError { + source: ProtoError::from(source), + })? .to_string()) } fn parse_domain_name(decoder: &mut BinDecoder<'_>) -> DnsParserResult { - Name::read(decoder).map_err(|source| DnsMessageParserError::TrustDnsError { source }) + Name::read(decoder).map_err(|source| DnsMessageParserError::TrustDnsError { + source: ProtoError::from(source), + }) } fn escape_string_for_text_representation(original_string: String) -> String { @@ -1305,11 +1311,13 @@ mod tests { dnssec::{ Algorithm as DNSSEC_Algorithm, DigestType, Nsec3HashAlgorithm, PublicKeyBuf, rdata::{ - KEY, NSEC, NSEC3, NSEC3PARAM, RRSIG, SIG, + KEY, NSEC, NSEC3, NSEC3PARAM, RRSIG, key::{KeyTrust, KeyUsage, Protocol}, + sig::SigInput, }, }, rr::{ + SerialNumber, domain::Name, rdata::{ CAA, CERT, CSYNC, HINFO, HTTPS, NAPTR, OPT, SSHFP, TLSA, TXT, @@ -1435,7 +1443,10 @@ mod tests { .expect_err("Expected TrustDnsError."); match err { DnsMessageParserError::TrustDnsError { source: e } => { - assert_eq!(e.to_string(), "unexpected end of input reached") + assert_eq!( + e.to_string(), + "decoding error: unexpected end of input reached" + ) } DnsMessageParserError::SimpleError { cause: e } => { panic!("Expected TrustDnsError, got {}.", &e) @@ -1859,29 +1870,26 @@ mod tests { #[test] fn test_format_rdata_for_sig_type() { - let rdata = RData::DNSSEC(DNSSECRData::SIG(SIG::new( - RecordType::NULL, - DNSSEC_Algorithm::RSASHA256, + // SIG wire: type_covered=NULL(10), alg=8, labels=0, orig_ttl=0, expire=2, inception=1, + // keytag=5, signer=www.example.com, sig=[0..=31] + let mut wire: Vec = Vec::new(); + wire.extend_from_slice(&10u16.to_be_bytes()); + wire.push(8u8); + wire.push(0u8); + wire.extend_from_slice(&0u32.to_be_bytes()); + wire.extend_from_slice(&2u32.to_be_bytes()); + wire.extend_from_slice(&1u32.to_be_bytes()); + wire.extend_from_slice(&5u16.to_be_bytes()); + wire.extend_from_slice(&[ + 3, b'w', b'w', b'w', 7, b'e', b'x', b'a', b'm', b'p', b'l', b'e', 3, b'c', b'o', b'm', 0, - 0, - 2, - 1, - 5, - Name::from_str("www.example.com").unwrap(), - vec![ - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, - 23, 24, 25, 26, 27, 28, 29, 29, 31, - ], - ))); - let rdata_text = format_rdata(&rdata); - assert!(rdata_text.is_ok()); - if let Ok((parsed, raw_rdata)) = rdata_text { - assert!(raw_rdata.is_none()); - assert_eq!( - "NULL 8 0 0 2 1 5 www.example.com AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHR8=", - parsed.unwrap() - ); - } + ]); + wire.extend(0u8..=31); + test_format_rdata( + &BASE64.encode(&wire), + 24, + "NULL 8 0 0 2 1 5 www.example.com. AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=", + ); } #[test] @@ -1918,26 +1926,26 @@ mod tests { // so there isn't really a great way to reduce code duplication here. #[test] fn test_format_rdata_for_rsig_type() { - let rdata = RData::DNSSEC(DNSSECRData::RRSIG(RRSIG::new( - RecordType::NULL, - DNSSEC_Algorithm::RSASHA256, - 0, - 0, - 2, - 1, - 5, - Name::from_str("www.example.com").unwrap(), - vec![ - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, - 23, 24, 25, 26, 27, 28, 29, 29, 31, - ], + let input = SigInput { + type_covered: RecordType::NULL, + algorithm: DNSSEC_Algorithm::RSASHA256, + num_labels: 0, + original_ttl: 0, + sig_expiration: SerialNumber::new(2), + sig_inception: SerialNumber::new(1), + key_tag: 5, + signer_name: Name::from_str("www.example.com").unwrap(), + }; + let rdata = RData::DNSSEC(DNSSECRData::RRSIG(RRSIG::from_sig( + input, + (0u8..=31).collect(), ))); let rdata_text = format_rdata(&rdata); assert!(rdata_text.is_ok()); if let Ok((parsed, raw_rdata)) = rdata_text { assert!(raw_rdata.is_none()); assert_eq!( - "NULL 8 0 0 2 1 5 www.example.com AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHR8=", + "NULL 8 0 0 2 1 5 www.example.com AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=", parsed.unwrap() ); } diff --git a/lib/dnsmsg-parser/src/ede.rs b/lib/dnsmsg-parser/src/ede.rs index 0c3ee6c2f12e6..26bd22ed008ea 100644 --- a/lib/dnsmsg-parser/src/ede.rs +++ b/lib/dnsmsg-parser/src/ede.rs @@ -1,6 +1,6 @@ use hickory_proto::{ ProtoError, - serialize::binary::{BinDecodable, BinDecoder, BinEncodable, BinEncoder}, + serialize::binary::{BinDecodable, BinDecoder, BinEncodable, BinEncoder, DecodeError}, }; pub const EDE_OPTION_CODE: u16 = 15u16; @@ -77,14 +77,15 @@ impl BinEncodable for EDE { } impl<'a> BinDecodable<'a> for EDE { - fn read(decoder: &mut BinDecoder<'a>) -> Result { + fn read(decoder: &mut BinDecoder<'a>) -> Result { let info_code = decoder.read_u16()?.unverified(); let extra_text = if decoder.is_empty() { None } else { - Some(String::from_utf8( - decoder.read_vec(decoder.len())?.unverified(), - )?) + Some( + String::from_utf8(decoder.read_vec(decoder.len())?.unverified()) + .map_err(DecodeError::Utf8)?, + ) }; Ok(Self { info_code, From 6c3116a6e0a1d44113e5e2ce0b7e5aeeef3db785 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 5 May 2026 09:33:38 -0400 Subject: [PATCH 194/326] chore(ci): retry apt fetches in deb-verify to reduce flakes (#25367) Set Acquire::Retries=5 and Acquire::http::Timeout=30 on the apt-get calls in the deb-verify job so transient connection failures to the Ubuntu mirrors don't fail the whole job. Co-authored-by: Claude Opus 4.7 (1M context) --- .github/workflows/publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 4f82bd06e3894..7e8455b60de58 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -204,8 +204,8 @@ jobs: image: ${{ matrix.container }} steps: - run: | - apt-get update && \ - apt-get install -y \ + apt-get -o Acquire::Retries=5 -o Acquire::http::Timeout=30 update && \ + apt-get -o Acquire::Retries=5 -o Acquire::http::Timeout=30 install -y \ ca-certificates \ curl \ git \ From 95756d72356406d1a71625dbcc8f83e49f43947d Mon Sep 17 00:00:00 2001 From: jh7459-gh <90462481+jh7459-gh@users.noreply.github.com> Date: Tue, 5 May 2026 10:34:08 -0500 Subject: [PATCH 195/326] enhancement(transforms): dynamic rate for sample (#25035) feat(sample transform): add ratio_field and rate_field for dynamic sampling Add per-event dynamic sampling to the `sample` transform. `ratio_field` reads a numeric ratio in `(0, 1]` from the event; `rate_field` reads a positive integer rate (`1/N`). Either falls back to the static `rate`/`ratio` configuration when the field is missing or invalid. - `ratio_field` and `rate_field` are mutually exclusive, and neither can combine with `key_field` (key-based coherence conflicts with per-event values). - Sampling is event-granular and uses a hash of (group_by_key, counter) so mixed dynamic values within the same `group_by` group sample at their respective expected rates. - `group_by` is supported alongside dynamic fields for independent per-group sampling. - The dynamic-rate path is `NonZeroU64`-typed end-to-end so the `is_multiple_of(0)` panic is unreachable by construction. Co-authored-by: Pavlos Rontidis Co-authored-by: Claude Opus 4.7 (1M context) --- .../sample_dynamic_ratio_field.feature.md | 3 + src/transforms/sample/config.rs | 195 ++++++++++- src/transforms/sample/tests.rs | 329 +++++++++++++++++- src/transforms/sample/transform.rs | 254 +++++++++++--- .../transforms/generated/sample.cue | 29 ++ 5 files changed, 749 insertions(+), 61 deletions(-) create mode 100644 changelog.d/sample_dynamic_ratio_field.feature.md diff --git a/changelog.d/sample_dynamic_ratio_field.feature.md b/changelog.d/sample_dynamic_ratio_field.feature.md new file mode 100644 index 0000000000000..d2e3c04839c19 --- /dev/null +++ b/changelog.d/sample_dynamic_ratio_field.feature.md @@ -0,0 +1,3 @@ +Added `ratio_field` and `rate_field` options to the `sample` transform to support dynamic per-event sampling, while requiring static `rate` or `ratio` fallback configuration and disallowing `ratio_field` and `rate_field` together. + +authors: jhammer diff --git a/src/transforms/sample/config.rs b/src/transforms/sample/config.rs index 209d2d655e7c9..1b5f7426d86a3 100644 --- a/src/transforms/sample/config.rs +++ b/src/transforms/sample/config.rs @@ -6,7 +6,7 @@ use vector_lib::{ }; use vrl::value::Kind; -use super::transform::{Sample, SampleMode}; +use super::transform::{DynamicSampleFields, Sample, SampleMode}; use crate::{ conditions::AnyCondition, config::{ @@ -29,10 +29,23 @@ pub enum SampleError { #[snafu(display("Only non-zero numbers are allowed values for `rate`"))] InvalidRate, + #[snafu(display("Only one value can be provided for either 'rate' or 'ratio', but not both"))] + InvalidStaticConfiguration, + + #[snafu(display( + "Only one value can be provided for either 'ratio_field' or 'rate_field', but not both" + ))] + InvalidDynamicConfiguration, + + #[snafu(display( + "Exactly one value must be provided for either 'rate' or 'ratio' to configure static sampling" + ))] + MissingStaticConfiguration, + #[snafu(display( - "Exactly one value must be provided for either 'rate' or 'ratio', but not both" + "'key_field' cannot be combined with 'ratio_field' or 'rate_field' because dynamic values can vary per event and break key-based coherence" ))] - InvalidConfiguration, + InvalidKeyFieldDynamicCombination, } /// Configuration for the `sample` transform. @@ -61,6 +74,24 @@ pub struct SampleConfig { #[configurable(validation(range(min = 0.0, max = 1.0)))] pub ratio: Option, + /// The event field whose numeric value is used as the sampling ratio on a per-event basis. + /// + /// Accepts integer, floating point, or string values that parse as a number. The value must be + /// in `(0, 1]` to be considered valid (for example, `0.25` keeps 25%). If the field is missing + /// or invalid, static sampling settings (`rate` or `ratio`) are used as a fallback. + /// This option cannot be used together with `rate_field`. + #[configurable(metadata(docs::examples = "sample_rate"))] + pub ratio_field: Option, + + /// The event field whose integer value is used as the sampling rate on a per-event basis, expressed as `1/N`. + /// + /// Accepts an integer, or a string that parses as a positive integer; floating point values + /// are rejected. The value must be a positive integer to be considered valid. If the field is + /// missing or invalid, static sampling settings (`rate` or `ratio`) are used as a fallback. + /// This option cannot be used together with `ratio_field`. + #[configurable(metadata(docs::examples = "sample_rate_n"))] + pub rate_field: Option, + /// The name of the field whose value is hashed to determine if the event should be /// sampled. /// @@ -72,6 +103,8 @@ pub struct SampleConfig { /// /// This can be useful to, for example, ensure that all logs for a given transaction are /// sampled together, but that overall `1/N` transactions are sampled. + /// + /// This option cannot be combined with `ratio_field` or `rate_field`. #[configurable(metadata(docs::examples = "message"))] pub key_field: Option, @@ -84,6 +117,9 @@ pub struct SampleConfig { /// /// If left unspecified, or if the event doesn't have `group_by`, then the event is not /// sampled separately. + /// + /// This can also be used with `ratio_field` or `rate_field` to apply dynamic sampling + /// independently per rendered group value. #[configurable(metadata( docs::examples = "{{ service }}", docs::examples = "{{ hostname }}-{{ service }}" @@ -96,6 +132,18 @@ pub struct SampleConfig { impl SampleConfig { fn sample_rate(&self) -> Result { + if self.ratio_field.is_some() && self.rate_field.is_some() { + return Err(SampleError::InvalidDynamicConfiguration); + } + + if self.key_field.is_some() && (self.ratio_field.is_some() || self.rate_field.is_some()) { + return Err(SampleError::InvalidKeyFieldDynamicCombination); + } + + if self.rate.is_some() && self.ratio.is_some() { + return Err(SampleError::InvalidStaticConfiguration); + } + match (self.rate, self.ratio) { (None, Some(ratio)) => { if ratio <= 0.0 { @@ -111,7 +159,8 @@ impl SampleConfig { Ok(SampleMode::new_rate(rate)) } } - _ => Err(SampleError::InvalidConfiguration), + (None, None) => Err(SampleError::MissingStaticConfiguration), + _ => Err(SampleError::InvalidStaticConfiguration), } } } @@ -121,6 +170,8 @@ impl GenerateConfig for SampleConfig { toml::Value::try_from(Self { rate: None, ratio: Some(0.1), + ratio_field: None, + rate_field: None, key_field: None, group_by: None, exclude: None::, @@ -134,19 +185,37 @@ impl GenerateConfig for SampleConfig { #[typetag::serde(name = "sample")] impl TransformConfig for SampleConfig { async fn build(&self, context: &TransformContext) -> crate::Result { - Ok(Transform::function(Sample::new( - Self::NAME.to_string(), - self.sample_rate()?, - self.key_field.clone(), - self.group_by.clone(), - self.exclude - .as_ref() - .map(|condition| { - condition.build(&context.enrichment_tables, &context.metrics_storage) - }) - .transpose()?, - self.sample_rate_key.clone(), - ))) + let sample_mode = self.sample_rate()?; + let exclude = self + .exclude + .as_ref() + .map(|condition| condition.build(&context.enrichment_tables, &context.metrics_storage)) + .transpose()?; + + let sample = if self.ratio_field.is_some() || self.rate_field.is_some() { + Sample::new_with_dynamic( + Self::NAME.to_string(), + sample_mode, + DynamicSampleFields { + ratio_field: self.ratio_field.clone(), + rate_field: self.rate_field.clone(), + }, + self.group_by.clone(), + exclude, + self.sample_rate_key.clone(), + ) + } else { + Sample::new( + Self::NAME.to_string(), + sample_mode, + self.key_field.clone(), + self.group_by.clone(), + exclude, + self.sample_rate_key.clone(), + ) + }; + + Ok(Transform::function(sample)) } fn input(&self) -> Input { @@ -191,10 +260,100 @@ pub fn default_sample_rate_key() -> OptionalValuePath { #[cfg(test)] mod tests { - use crate::transforms::sample::config::SampleConfig; + use crate::{ + config::TransformConfig, + transforms::sample::config::{SampleConfig, SampleError}, + }; #[test] fn generate_config() { crate::test_util::test_generate_config::(); } + + #[test] + fn rejects_dynamic_ratio_only_configuration() { + let config = SampleConfig { + rate: None, + ratio: None, + ratio_field: Some("sample_rate".to_string()), + rate_field: None, + key_field: None, + sample_rate_key: super::default_sample_rate_key(), + group_by: None, + exclude: None, + }; + + let err = config.sample_rate().unwrap_err(); + assert!(matches!(err, SampleError::MissingStaticConfiguration)); + } + + #[test] + fn rejects_dynamic_rate_only_configuration() { + let config = SampleConfig { + rate: None, + ratio: None, + ratio_field: None, + rate_field: Some("sample_rate_n".to_string()), + key_field: None, + sample_rate_key: super::default_sample_rate_key(), + group_by: None, + exclude: None, + }; + + let err = config.sample_rate().unwrap_err(); + assert!(matches!(err, SampleError::MissingStaticConfiguration)); + } + + #[test] + fn validates_static_with_dynamic_configuration() { + let config = SampleConfig { + rate: Some(10), + ratio: None, + ratio_field: None, + rate_field: Some("sample_rate_n".to_string()), + key_field: None, + sample_rate_key: super::default_sample_rate_key(), + group_by: None, + exclude: None, + }; + + assert!(config.validate(&crate::schema::Definition::any()).is_ok()); + } + + #[test] + fn rejects_both_dynamic_fields_configuration() { + let config = SampleConfig { + rate: Some(10), + ratio: None, + ratio_field: Some("sample_rate".to_string()), + rate_field: Some("sample_rate_n".to_string()), + key_field: None, + sample_rate_key: super::default_sample_rate_key(), + group_by: None, + exclude: None, + }; + + let err = config.sample_rate().unwrap_err(); + assert!(matches!(err, SampleError::InvalidDynamicConfiguration)); + } + + #[test] + fn rejects_key_field_with_dynamic_configuration() { + let config = SampleConfig { + rate: Some(10), + ratio: None, + ratio_field: Some("sample_ratio".to_string()), + rate_field: None, + key_field: Some("trace_id".to_string()), + sample_rate_key: super::default_sample_rate_key(), + group_by: None, + exclude: None, + }; + + let err = config.sample_rate().unwrap_err(); + assert!(matches!( + err, + SampleError::InvalidKeyFieldDynamicCombination + )); + } } diff --git a/src/transforms/sample/tests.rs b/src/transforms/sample/tests.rs index c705ff2cc4f1f..e6fec5c12a35d 100644 --- a/src/transforms/sample/tests.rs +++ b/src/transforms/sample/tests.rs @@ -1,4 +1,5 @@ use approx::assert_relative_eq; +use indoc::indoc; use tokio::sync::mpsc; use tokio_stream::wrappers::ReceiverStream; use vector_lib::lookup::lookup_v2::OptionalValuePath; @@ -14,7 +15,7 @@ use crate::{ FunctionTransform, OutputBuffer, sample::{ config::{SampleConfig, default_sample_rate_key}, - transform::{Sample, SampleMode}, + transform::{DynamicSampleFields, Sample, SampleMode}, }, test::{create_topology, transform_one}, }, @@ -26,6 +27,8 @@ async fn emits_internal_events() { let config = SampleConfig { rate: None, ratio: Some(1.0), + ratio_field: None, + rate_field: None, key_field: None, group_by: None, exclude: None, @@ -324,6 +327,330 @@ fn sample_at_rates_higher_then_half() { } } +#[test] +fn dynamic_ratio_field_overrides_static_ratio() { + let mut sampler = Sample::new_with_dynamic( + "sample".to_string(), + SampleMode::new_ratio(0.1), + DynamicSampleFields { + ratio_field: Some("dynamic_ratio".to_string()), + rate_field: None, + }, + None, + None, + default_sample_rate_key(), + ); + + let mut event = Event::Log(LogEvent::from("hello")); + let log = event.as_mut_log(); + log.insert("dynamic_ratio", 1.0); + + let output = transform_one(&mut sampler, event).expect("event should be sampled"); + assert_eq!(output.as_log()["sample_rate"], "1".into()); +} + +#[test] +fn dynamic_ratio_field_falls_back_to_static_ratio_when_missing() { + let mut sampler = Sample::new_with_dynamic( + "sample".to_string(), + SampleMode::new_ratio(1.0), + DynamicSampleFields { + ratio_field: Some("dynamic_ratio".to_string()), + rate_field: None, + }, + None, + None, + default_sample_rate_key(), + ); + + let event = Event::Log(LogEvent::from("hello")); + let output = transform_one(&mut sampler, event).expect("event should be sampled"); + assert_eq!(output.as_log()["sample_rate"], "1".into()); +} + +#[test] +fn dynamic_rate_field_overrides_static_ratio() { + let mut sampler = Sample::new_with_dynamic( + "sample".to_string(), + SampleMode::new_ratio(0.0), + DynamicSampleFields { + ratio_field: None, + rate_field: Some("dynamic_rate".to_string()), + }, + None, + None, + default_sample_rate_key(), + ); + + let mut event = Event::Log(LogEvent::from("hello")); + let log = event.as_mut_log(); + log.insert("dynamic_rate", 1); + + let output = transform_one(&mut sampler, event).expect("event should be sampled"); + assert_eq!(output.as_log()["sample_rate"], "1".into()); +} + +#[test] +fn dynamic_rate_field_falls_back_to_static_ratio_when_missing() { + let mut sampler = Sample::new_with_dynamic( + "sample".to_string(), + SampleMode::new_ratio(1.0), + DynamicSampleFields { + ratio_field: None, + rate_field: Some("dynamic_rate".to_string()), + }, + None, + None, + default_sample_rate_key(), + ); + + let event = Event::Log(LogEvent::from("hello")); + let output = transform_one(&mut sampler, event).expect("event should be sampled"); + assert_eq!(output.as_log()["sample_rate"], "1".into()); +} + +#[test] +fn dynamic_rate_field_rejects_float_and_falls_back_to_static_ratio() { + let mut sampler = Sample::new_with_dynamic( + "sample".to_string(), + SampleMode::new_ratio(1.0), + DynamicSampleFields { + ratio_field: None, + rate_field: Some("dynamic_rate".to_string()), + }, + None, + None, + default_sample_rate_key(), + ); + + let mut event = Event::Log(LogEvent::from("hello")); + let log = event.as_mut_log(); + log.insert("dynamic_rate", 2.0); + + let output = transform_one(&mut sampler, event).expect("event should be sampled"); + assert_eq!(output.as_log()["sample_rate"], "1".into()); +} + +#[test] +fn dynamic_ratio_honors_group_by_key() { + let ratio = 0.5_f64; + let events_per_service = 200; + let mut sampler = Sample::new_with_dynamic( + "sample".to_string(), + SampleMode::new_ratio(0.0), + DynamicSampleFields { + ratio_field: Some("dynamic_ratio".to_string()), + rate_field: None, + }, + Some(Template::try_from("{{ service }}").unwrap()), + None, + default_sample_rate_key(), + ); + + let mut sampled_service_a = 0; + let mut sampled_service_b = 0; + for _ in 0..events_per_service { + for service in ["service-a", "service-b"] { + let mut event = Event::Log(LogEvent::from("hello")); + let log = event.as_mut_log(); + log.insert("service", service); + log.insert("dynamic_ratio", ratio); + if let Some(output) = transform_one(&mut sampler, event) { + assert_eq!(output.as_log()["sample_rate"], "0.5".into()); + if service == "service-a" { + sampled_service_a += 1; + } else { + sampled_service_b += 1; + } + } + } + } + + assert!( + (60..140).contains(&sampled_service_a), + "service-a sampled {} out of {events_per_service}", + sampled_service_a + ); + assert!( + (60..140).contains(&sampled_service_b), + "service-b sampled {} out of {events_per_service}", + sampled_service_b + ); +} + +#[test] +fn dynamic_rate_honors_group_by_key() { + let rate = 2_i64; + let events_per_service = 200; + let mut sampler = Sample::new_with_dynamic( + "sample".to_string(), + SampleMode::new_ratio(0.0), + DynamicSampleFields { + ratio_field: None, + rate_field: Some("dynamic_rate".to_string()), + }, + Some(Template::try_from("{{ service }}").unwrap()), + None, + default_sample_rate_key(), + ); + + let mut sampled_service_a = 0; + let mut sampled_service_b = 0; + for _ in 0..events_per_service { + for service in ["service-a", "service-b"] { + let mut event = Event::Log(LogEvent::from("hello")); + let log = event.as_mut_log(); + log.insert("service", service); + log.insert("dynamic_rate", rate); + if let Some(output) = transform_one(&mut sampler, event) { + assert_eq!(output.as_log()["sample_rate"], "2".into()); + if service == "service-a" { + sampled_service_a += 1; + } else { + sampled_service_b += 1; + } + } + } + } + + assert!( + (60..140).contains(&sampled_service_a), + "service-a sampled {} out of {events_per_service}", + sampled_service_a + ); + assert!( + (60..140).contains(&sampled_service_b), + "service-b sampled {} out of {events_per_service}", + sampled_service_b + ); +} + +#[test] +fn dynamic_ratio_group_by_samples_mixed_ratios_at_expected_rates() { + let events_per_ratio = 500; + let mut sampler = Sample::new_with_dynamic( + "sample".to_string(), + SampleMode::new_ratio(0.0), + DynamicSampleFields { + ratio_field: Some("dynamic_ratio".to_string()), + rate_field: None, + }, + Some(Template::try_from("{{ service }}").unwrap()), + None, + default_sample_rate_key(), + ); + + let mut sampled_low_ratio = 0; + let mut sampled_high_ratio = 0; + for _ in 0..events_per_ratio { + for (ratio, is_low_ratio) in [(0.25_f64, true), (0.75_f64, false)] { + let mut event = Event::Log(LogEvent::from("hello")); + let log = event.as_mut_log(); + log.insert("service", "service-a"); + log.insert("dynamic_ratio", ratio); + if let Some(output) = transform_one(&mut sampler, event) { + assert_eq!(output.as_log()["sample_rate"], ratio.to_string().into()); + if is_low_ratio { + sampled_low_ratio += 1; + } else { + sampled_high_ratio += 1; + } + } + } + } + + assert!( + (80..220).contains(&sampled_low_ratio), + "ratio=0.25 sampled {} out of {events_per_ratio}", + sampled_low_ratio + ); + assert!( + (300..450).contains(&sampled_high_ratio), + "ratio=0.75 sampled {} out of {events_per_ratio}", + sampled_high_ratio + ); + assert!( + sampled_high_ratio > sampled_low_ratio, + "ratio=0.75 sampled {sampled_high_ratio}, ratio=0.25 sampled {sampled_low_ratio}" + ); +} + +#[test] +fn dynamic_rate_group_by_samples_mixed_rates_at_expected_rates() { + let events_per_rate = 600; + let mut sampler = Sample::new_with_dynamic( + "sample".to_string(), + SampleMode::new_ratio(0.0), + DynamicSampleFields { + ratio_field: None, + rate_field: Some("dynamic_rate".to_string()), + }, + Some(Template::try_from("{{ service }}").unwrap()), + None, + default_sample_rate_key(), + ); + + let mut sampled_rate_2 = 0; + let mut sampled_rate_3 = 0; + for _ in 0..events_per_rate { + for rate in [2_i64, 3_i64] { + let mut event = Event::Log(LogEvent::from("hello")); + let log = event.as_mut_log(); + log.insert("service", "service-a"); + log.insert("dynamic_rate", rate); + if let Some(output) = transform_one(&mut sampler, event) { + assert_eq!(output.as_log()["sample_rate"], rate.to_string().into()); + if rate == 2 { + sampled_rate_2 += 1; + } else { + sampled_rate_3 += 1; + } + } + } + } + + assert!( + (220..380).contains(&sampled_rate_2), + "rate=2 sampled {} out of {events_per_rate}", + sampled_rate_2 + ); + assert!( + (120..280).contains(&sampled_rate_3), + "rate=3 sampled {} out of {events_per_rate}", + sampled_rate_3 + ); + assert!( + sampled_rate_2 > sampled_rate_3, + "rate=2 sampled {sampled_rate_2}, rate=3 sampled {sampled_rate_3}" + ); +} + +#[tokio::test] +async fn dynamic_field_config_drives_dynamic_sampling() { + assert_transform_compliance(async move { + let config: SampleConfig = serde_yaml::from_str(indoc! {r#" + ratio_field: dynamic_ratio + ratio: 1.0 + "#}) + .expect("config should deserialize"); + + let (tx, rx) = mpsc::channel(1); + let (topology, mut out) = create_topology(ReceiverStream::new(rx), config).await; + + let mut log = LogEvent::from("hello"); + log.insert("dynamic_ratio", 1.0); + tx.send(log.into()).await.unwrap(); + + let event = out.recv().await.expect("event should be sampled"); + assert_eq!(event.as_log()["sample_rate"], "1".into()); + + drop(tx); + topology.stop().await; + assert_eq!(out.recv().await, None); + }) + .await +} + fn condition_contains(key: &str, needle: &str) -> Condition { let vrl_config = VrlConfig { source: format!(r#"contains!(."{key}", "{needle}")"#), diff --git a/src/transforms/sample/transform.rs b/src/transforms/sample/transform.rs index 74786a43fdc51..00728b076e68c 100644 --- a/src/transforms/sample/transform.rs +++ b/src/transforms/sample/transform.rs @@ -1,4 +1,9 @@ -use std::{collections::HashMap, fmt}; +use std::{ + collections::HashMap, + fmt, + hash::{Hash, Hasher}, + num::NonZeroU64, +}; use vector_lib::{ config::LegacyKey, @@ -91,6 +96,26 @@ impl SampleMode { } } +enum EventSampleMode { + Ratio(f64), + Rate(NonZeroU64), +} + +impl EventSampleMode { + fn sample_rate_label(&self) -> String { + match self { + Self::Ratio(ratio) => ratio.to_string(), + Self::Rate(rate) => rate.to_string(), + } + } +} + +#[derive(Clone, Default)] +pub struct DynamicSampleFields { + pub ratio_field: Option, + pub rate_field: Option, +} + impl fmt::Display for SampleMode { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { // Avoids the print of an additional '.0' which was not performed in the previous @@ -102,12 +127,24 @@ impl fmt::Display for SampleMode { } } +#[derive(Clone)] +pub enum SampleKeySource { + Static { + key_field: Option, + group_by: Option
- + {countries.map((d, i) => ( ))} {markers.map((m, i) => { - const coordinates: [number, number] = [ - m.coordinates[0], - m.coordinates[1], - ]; + const coordinates: [number, number] = [m.coordinates[0], m.coordinates[1]]; const position = projection(coordinates); const x = position ? position[0] : undefined; const y = position ? position[1] : undefined; - const invertedCoordinates = - projection.invert && projection.invert(center); + const invertedCoordinates = projection.invert && projection.invert(center); // hide the marker if rotated out of view - const hideMarker = - geoDistance(coordinates, invertedCoordinates || [0, 0]) > 1.4; + const hideMarker = geoDistance(coordinates, invertedCoordinates || [0, 0]) > 1.4; return ( = animated( } ); -export const RotatingGlobe: React.FC<{ size: number; duration?: number }> = ({ - size = 500, - duration = 50000, -}) => { +export const RotatingGlobe: React.FC<{ size: number; duration?: number }> = ({ size = 500, duration = 50000 }) => { // globe positioning state const [orientation, setOrientation] = React.useState({ lat: 0, long: 0, - degrees: 0, + degrees: 0 }); // set up spring animation @@ -129,8 +113,8 @@ export const RotatingGlobe: React.FC<{ size: number; duration?: number }> = ({ long: orientation.long, rotation: orientation.degrees, config: { - duration, - }, + duration + } }); // globe click handler @@ -146,7 +130,7 @@ export const RotatingGlobe: React.FC<{ size: number; duration?: number }> = ({ setOrientation({ lat: Math.floor(Math.random() * size), long: Math.floor(Math.random() * size), - degrees: orientation.degrees + 360, + degrees: orientation.degrees + 360 }); }, duration); @@ -162,7 +146,7 @@ export const RotatingGlobe: React.FC<{ size: number; duration?: number }> = ({ }; // Main page Vector diagram -function Diagram({className, height, width}) { +function Diagram({ className, height, width }) { const [_, updateState] = useState(); const defaultXPosition = 7; const defaultTextLength = 60; @@ -183,7 +167,12 @@ function Diagram({className, height, width}) { } return ( - + Vector Diagram A lightweight and ultra-fast tool for building observability pipelines @@ -202,118 +191,323 @@ function Diagram({className, height, width}) { - + - - - - + + + + - - - - + + + + - + - + - + - + - + - + - + - + - - - - - - - + + + + + + + - - - - - - - + + + + + + + - - - - - - - + + + + + + + - - - - - + + + + + - + ); -}; +} // Place the components in the DOM const globeRoot = createRoot(document.getElementById("globe")); diff --git a/website/assets/js/search.tsx b/website/assets/js/search.tsx index f322f463ed774..9fb384498f10e 100644 --- a/website/assets/js/search.tsx +++ b/website/assets/js/search.tsx @@ -1,13 +1,13 @@ -import { autocomplete } from '@algolia/autocomplete-js' -import Typesense from 'typesense' -import React, { createElement, Fragment, useEffect, useRef } from 'react' -import { createRoot } from 'react-dom/client' +import { autocomplete } from "@algolia/autocomplete-js"; +import Typesense from "typesense"; +import React, { createElement, Fragment, useEffect, useRef } from "react"; +import { createRoot } from "react-dom/client"; // // Algolia search // const appId = process.env.ALGOLIA_APP_ID -const apiKey = process.env.TYPESENSE_PUBLIC_API_KEY -const indexName = process.env.TYPESENSE_INDEX -const host = process.env.TYPESENSE_HOST +const apiKey = process.env.TYPESENSE_PUBLIC_API_KEY; +const indexName = process.env.TYPESENSE_INDEX; +const host = process.env.TYPESENSE_HOST; // const searchClient = algoliasearch(appId, apiKey) let searchClient = new Typesense.Client({ @@ -15,73 +15,53 @@ let searchClient = new Typesense.Client({ nearestNode: { host: `${host}.a1.typesense.net`, port: 443, - protocol: 'https', + protocol: "https" }, nodes: [ { host: `${host}-1.a1.typesense.net`, port: 443, - protocol: 'https', + protocol: "https" }, { host: `${host}-2.a1.typesense.net`, port: 443, - protocol: 'https', + protocol: "https" }, { host: `${host}-3.a1.typesense.net`, port: 443, - protocol: 'https', - }, + protocol: "https" + } ], - connectionTimeoutSeconds: 2, -}) - + connectionTimeoutSeconds: 2 +}); const CommandIcon: React.FC = ({ children }) => { return ( - + {children} - ) -} + ); +}; const Chevron: React.FC = () => { return ( - - + + - ) -} - + ); +}; const Result = ({ hit, components, category }) => { - const hierarchy = hit.document.hierarchy.concat(hit.document.title) - const isRootPage = hierarchy.length < 1 + const hierarchy = hit.document.hierarchy.concat(hit.document.title); + const isRootPage = hierarchy.length < 1; return ( -
- {category} -
+
{category}
{!isRootPage && @@ -100,34 +80,30 @@ const Result = ({ hit, components, category }) => { {isRootPage && }

- {hit.content && ( - - )} - {!hit.content && ( - {hit.document.itemUrl} - )} + {hit.content && } + {!hit.content && {hit.document.itemUrl}}

- ) -} + ); +}; const Autocomplete = (props) => { - const containerRef = useRef(null) - const panelRootRef = useRef({}) + const containerRef = useRef(null); + const panelRootRef = useRef({}); useEffect(() => { if (!containerRef.current) { - return undefined + return undefined; } const search = autocomplete({ container: containerRef.current, renderer: { createElement, Fragment }, render({ children, state, components }, root) { - const { preview } = state.context as any + const { preview } = state.context as any; if (!panelRootRef.current[root]) { - panelRootRef.current[root] = createRoot(root) + panelRootRef.current[root] = createRoot(root); } panelRootRef.current[root].render( @@ -168,18 +144,18 @@ const Autocomplete = (props) => {