diff --git a/.cargo/config.toml b/.cargo/config.toml index 2d2f4cbf67086..ef954fd8fe73a 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -2,12 +2,15 @@ vdev = "run --quiet --package vdev --" [env] -# Build with large pages so that Vector runs on systems with 64k pages or less (e.g. 4k) to support -# CentOS 7, 8, and a few other Linux distributions. +# Build jemalloc assuming 64K pages so binaries also run on aarch64 hosts whose +# kernels ship with a 64K page size (notably EL8 aarch64). On 4K-page hosts this +# coarsens jemalloc's allocation granularity; no regression was measured in +# https://github.com/vectordotdev/vector/pull/18481. JEMALLOC_SYS_WITH_LG_PAGE = "16" [target.'cfg(all())'] rustflags = [ + "-D", "warnings", "-Dclippy::print_stdout", "-Dclippy::print_stderr", "-Dclippy::dbg_macro", @@ -35,4 +38,4 @@ rustflags = ["-C", "link-args=-rdynamic"] [target.x86_64-pc-windows-msvc] # https://github.com/dtolnay/inventory/issues/58 -rustflags = ["-C", "codegen-units=1"] +rustflags = ["-C", "codegen-units=1", "-Ctarget-feature=+crt-static"] diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ced3f0fb7bc07..45bfb2b14f923 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -3,13 +3,15 @@ .github/workflows/regression.yml @vectordotdev/vector @vectordotdev/single-machine-performance regression/config.yaml @vectordotdev/vector @vectordotdev/single-machine-performance -docs/ @vectordotdev/vector @vectordotdev/documentation website/ @vectordotdev/vector -website/content @vectordotdev/vector @vectordotdev/documentation -website/cue/reference @vectordotdev/vector @vectordotdev/documentation - website/js @vectordotdev/vector @vectordotdev/vector-website website/layouts @vectordotdev/vector @vectordotdev/vector-website website/scripts @vectordotdev/vector @vectordotdev/vector-website website/data @vectordotdev/vector @vectordotdev/vector-website website/* @vectordotdev/vector @vectordotdev/vector-website + +# Keep documentation team paths in sync with .github/workflows/add_docs_review_label.yml +/*.md @vectordotdev/vector @vectordotdev/documentation +docs/ @vectordotdev/vector @vectordotdev/documentation +website/content @vectordotdev/vector @vectordotdev/documentation +website/cue/reference @vectordotdev/vector @vectordotdev/documentation diff --git a/.github/DISCUSSION_TEMPLATE/q-a.yml b/.github/DISCUSSION_TEMPLATE/q-a.yml index 23cb1f8467745..cc4942b0976ce 100644 --- a/.github/DISCUSSION_TEMPLATE/q-a.yml +++ b/.github/DISCUSSION_TEMPLATE/q-a.yml @@ -1,5 +1,5 @@ title: "Q&A" -labels: [ q-a ] +labels: [q-a] body: - type: markdown attributes: diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml index 47354ec1ba30e..ff1c7df5e83ca 100644 --- a/.github/ISSUE_TEMPLATE/bug.yml +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -1,94 +1,94 @@ name: Bug description: πŸ› Let us know about an unexpected error, a crash, or an incorrect behavior. -type: 'Bug' +type: "Bug" body: -- type: markdown - attributes: - value: | - Thank you for opening πŸ› bug report! + - type: markdown + attributes: + value: | + Thank you for opening πŸ› bug report! -- type: textarea - attributes: - label: A note for the community - value: | - - * Please vote on this issue by adding a πŸ‘ [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to the original issue to help the community and maintainers prioritize this request - * If you are interested in working on this issue or have submitted a pull request, please leave a comment - + - type: textarea + attributes: + label: A note for the community + value: | + + * Please vote on this issue by adding a πŸ‘ [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to the original issue to help the community and maintainers prioritize this request + * If you are interested in working on this issue or have submitted a pull request, please leave a comment + -- type: textarea - id: problem - attributes: - label: Problem - description: > - Please provide a clear and concise description of what the bug is, - including what currently happens and what you expected to happen. - validations: - required: true + - type: textarea + id: problem + attributes: + label: Problem + description: > + Please provide a clear and concise description of what the bug is, + including what currently happens and what you expected to happen. + validations: + required: true -- type: textarea - id: config - attributes: - label: Configuration - description: | - Paste the relevant parts of your Vector configuration file. + - type: textarea + id: config + attributes: + label: Configuration + description: | + Paste the relevant parts of your Vector configuration file. - !! If your config files contain sensitive information please remove it !! - render: text + !! If your config files contain sensitive information please remove it !! + render: text -- type: input - id: version - attributes: - label: Version - description: | - Please paste the output of running `vector --version`. + - type: input + id: version + attributes: + label: Version + description: | + Please paste the output of running `vector --version`. - If you are not running the latest version of Vector, please try upgrading - because your issue may have already been fixed. - validations: - required: true + If you are not running the latest version of Vector, please try upgrading + because your issue may have already been fixed. + validations: + required: true -- type: textarea - id: debug - attributes: - label: Debug Output - description: | - Full debug output can be obtained by running Vector with the following: + - type: textarea + id: debug + attributes: + label: Debug Output + description: | + Full debug output can be obtained by running Vector with the following: - ``` - RUST_BACKTRACE=full vector -vvv - ``` + ``` + RUST_BACKTRACE=full vector -vvv + ``` - If the debug output is long, please create a GitHub Gist containing the debug output and paste the link here. + If the debug output is long, please create a GitHub Gist containing the debug output and paste the link here. - !! Debug output may contain sensitive information. Please review it before posting publicly. !! - render: text + !! Debug output may contain sensitive information. Please review it before posting publicly. !! + render: text -- type: textarea - id: data - attributes: - label: Example Data - description: | - Please provide any example data that will help debug the issue, for example: + - type: textarea + id: data + attributes: + label: Example Data + description: | + Please provide any example data that will help debug the issue, for example: - ``` - 201.69.207.46 - kemmer6752 [07/06/2019:14:53:55 -0400] "PATCH /innovative/interfaces" 301 669 - ``` + ``` + 201.69.207.46 - kemmer6752 [07/06/2019:14:53:55 -0400] "PATCH /innovative/interfaces" 301 669 + ``` -- type: textarea - id: context - attributes: - label: Additional Context - description: | - Is there anything atypical about your situation that we should know? For - example: is Vector running in Kubernetes? Are you passing any unusual command - line options or environment variables to opt-in to non-default behavior? + - type: textarea + id: context + attributes: + label: Additional Context + description: | + Is there anything atypical about your situation that we should know? For + example: is Vector running in Kubernetes? Are you passing any unusual command + line options or environment variables to opt-in to non-default behavior? -- type: textarea - id: references - attributes: - label: References - description: | - Are there any other GitHub issues (open or closed) or Pull Requests that should be linked here? For example: + - type: textarea + id: references + attributes: + label: References + description: | + Are there any other GitHub issues (open or closed) or Pull Requests that should be linked here? For example: - - #6017 + - #6017 diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index ea44ff1b06562..df12e12af4f76 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -12,4 +12,3 @@ contact_links: - name: Twitter url: https://twitter.com/vectordotdev about: Follow us and stay up to date with Vector. - diff --git a/.github/ISSUE_TEMPLATE/feature.yml b/.github/ISSUE_TEMPLATE/feature.yml index f2e066bb37303..8b7a6fc139185 100644 --- a/.github/ISSUE_TEMPLATE/feature.yml +++ b/.github/ISSUE_TEMPLATE/feature.yml @@ -1,87 +1,86 @@ name: Feature -type: 'Feature' +type: "Feature" description: πŸš€ Suggest a new feature. body: -- type: markdown - attributes: - value: | - Thank you for opening πŸš€ feature request! - - For general questions about Vector usage, please see - https://discussions.vector.dev. - -- type: textarea - attributes: - label: A note for the community - value: | - - * Please vote on this issue by adding a πŸ‘ [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to the original issue to help the community and maintainers prioritize this request - * If you are interested in working on this issue or have submitted a pull request, please leave a comment - - -- type: textarea - id: use-cases - attributes: - label: Use Cases - description: | - In order to properly evaluate a feature request, it is necessary to - understand the use-cases for it. - - Please describe below the _end goal_ you are trying to achieve that has - led you to request this feature. - - Please keep this section focused on the problem and not on the suggested - solution. We'll get to that in a moment, below! - - -- type: textarea - id: attempted-solutions - attributes: - label: Attempted Solutions - description: | - If you've already tried to solve the problem within Vector's existing - features and found a limitation that prevented you from succeeding, please - describe it below in as much detail as possible. - - Ideally, this would include real configuration snippets that you tried - and what results you got in each case. - - Please remove any sensitive information such as passwords before sharing - configuration snippets and command lines. - -- type: textarea - id: proposal - attributes: - label: Proposal - description: | - If you have an idea for a way to address the problem via a change to - Vector features, please describe it below. - - In this section, it's helpful to include specific examples of how what - you are suggesting might look in configuration files, or on the command line, - since that allows us to understand the full picture of what you are proposing. - - If you're not sure of some details, don't worry! When we evaluate the - feature request we may suggest modifications as necessary to work within the - design constraints of Vector. - -- type: textarea - id: references - attributes: - label: References - description: | - Are there any other GitHub issues, whether open or closed, that are - related to the problem you've described above or to the suggested solution? If - so, please create a list below that mentions each of them. For example: - - - #7023 - -- type: input - id: version - attributes: - label: Version - description: | - Please paste the output of running `vector --version`. - - This will record which version was current at the time of your feature request, - to help manage the request backlog. + - type: markdown + attributes: + value: | + Thank you for opening πŸš€ feature request! + + For general questions about Vector usage, please see + https://discussions.vector.dev. + + - type: textarea + attributes: + label: A note for the community + value: | + + * Please vote on this issue by adding a πŸ‘ [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to the original issue to help the community and maintainers prioritize this request + * If you are interested in working on this issue or have submitted a pull request, please leave a comment + + + - type: textarea + id: use-cases + attributes: + label: Use Cases + description: | + In order to properly evaluate a feature request, it is necessary to + understand the use-cases for it. + + Please describe below the _end goal_ you are trying to achieve that has + led you to request this feature. + + Please keep this section focused on the problem and not on the suggested + solution. We'll get to that in a moment, below! + + - type: textarea + id: attempted-solutions + attributes: + label: Attempted Solutions + description: | + If you've already tried to solve the problem within Vector's existing + features and found a limitation that prevented you from succeeding, please + describe it below in as much detail as possible. + + Ideally, this would include real configuration snippets that you tried + and what results you got in each case. + + Please remove any sensitive information such as passwords before sharing + configuration snippets and command lines. + + - type: textarea + id: proposal + attributes: + label: Proposal + description: | + If you have an idea for a way to address the problem via a change to + Vector features, please describe it below. + + In this section, it's helpful to include specific examples of how what + you are suggesting might look in configuration files, or on the command line, + since that allows us to understand the full picture of what you are proposing. + + If you're not sure of some details, don't worry! When we evaluate the + feature request we may suggest modifications as necessary to work within the + design constraints of Vector. + + - type: textarea + id: references + attributes: + label: References + description: | + Are there any other GitHub issues, whether open or closed, that are + related to the problem you've described above or to the suggested solution? If + so, please create a list below that mentions each of them. For example: + + - #7023 + + - type: input + id: version + attributes: + label: Version + description: | + Please paste the output of running `vector --version`. + + This will record which version was current at the time of your feature request, + to help manage the request backlog. diff --git a/.github/ISSUE_TEMPLATE/minor-release.md b/.github/ISSUE_TEMPLATE/minor-release.md index 105299559632b..1c1172b1552ba 100644 --- a/.github/ISSUE_TEMPLATE/minor-release.md +++ b/.github/ISSUE_TEMPLATE/minor-release.md @@ -37,6 +37,7 @@ cargo vdev release prepare --version "${NEW_VECTOR_VERSION}" --vrl-version "${NE ``` Automated steps include: + - Create a new release branch from master to freeze commits - `git fetch && git checkout origin/master && git checkout -b "${RELEASE_BRANCH}" && git push -u` - Create a new release preparation branch from `master` @@ -44,7 +45,7 @@ Automated steps include: - Pin VRL to latest released version rather than `main` - Check if there is a newer version of [Alpine](https://alpinelinux.org/releases/) or [Debian](https://www.debian.org/releases/) available to update the release images in `distribution/docker/`. Update if so. -- Run `cargo vdev build release-cue` to generate a new cue file for the release +- Generate a new cue file for the release in `website/cue/reference/releases/` - Copy VRL changelogs from the VRL version in the last Vector release as a new changelog entry ([example](https://github.com/vectordotdev/vector/blob/9c67bba358195f5018febca2f228dfcb2be794b5/website/cue/reference/releases/0.41.0.cue#L33-L64)) - Update version number in `website/cue/reference/administration/interfaces/kubectl.cue` @@ -70,7 +71,8 @@ Automated steps include: # On the day of release - [ ] Make sure the release branch is in sync with origin/master and has only one squashed commit with all commits from the prepare branch. If you made a PR from the prepare branch into the release branch this should already be the case. - - [ ] `git checkout "${RELEASE_BRANCH}"` + - [ ] `git fetch origin` + - [ ] `git checkout "${RELEASE_BRANCH}" && git pull --ff-only origin "${RELEASE_BRANCH}"` - [ ] `git show --stat HEAD` - This should show the squashed prepare commit. - [ ] Ensure release date in `website/cue/reference/releases/0.XX.Y.cue` matches current date. - If this needs to be updated commit and squash it in the release branch. @@ -79,7 +81,7 @@ Automated steps include: - [ ] Squash the release preparation commits (but not the cherry-picked commits!) to a single commit. This makes it easier to cherry-pick to master after the release. - [ ] Merge release preparation branch into the release branch. - - `git switch "${RELEASE_BRANCH}" && git merge --ff-only "${PREP_BRANCH}"` + - `git switch "${RELEASE_BRANCH}" && git merge --ff-only "${PREP_BRANCH}"` - [ ] Tag new release - [ ] `git tag v"${NEW_VECTOR_VERSION}" -a -m v"${NEW_VECTOR_VERSION}"` @@ -87,7 +89,7 @@ Automated steps include: - [ ] Wait for release workflow to complete. - Discoverable via [release.yml](https://github.com/vectordotdev/vector/actions/workflows/release.yml) - [ ] Reset the `website` branch to the `HEAD` of the release branch to update https://vector.dev - - [ ] `git switch website && git reset --hard origin/"${RELEASE_BRANCH}" && git push` + - [ ] `git fetch origin && git switch website && git reset --hard origin/"${RELEASE_BRANCH}" && git push --force-with-lease` - [ ] Confirm that the release changelog was published to https://vector.dev/releases/ - Refer to the internal releasing doc to monitor the deployment. - [ ] Release Linux packages. Refer to the internal releasing doc. diff --git a/.github/ISSUE_TEMPLATE/patch-release.md b/.github/ISSUE_TEMPLATE/patch-release.md index afc5d8d826423..743d928ba70e7 100644 --- a/.github/ISSUE_TEMPLATE/patch-release.md +++ b/.github/ISSUE_TEMPLATE/patch-release.md @@ -24,8 +24,10 @@ export PREP_BRANCH=prepare-v-0-"${CURRENT_MINOR_VERSION}"-"${NEW_PATCH_VERSION}" - [ ] Cherry-pick in all commits to be released from the associated release milestone - If any merge conflicts occur, attempt to solve them and if needed enlist the aid of those familiar with the conflicting commits. - [ ] Bump the release number in the `Cargo.toml` to the current version number -- [ ] Run `cargo vdev build release-cue` to generate a new cue file for the release - - [ ] Add description key to the generated cue file with a description of the release (see +- [ ] Add a new cue file for the release at `website/cue/reference/releases/${NEW_VERSION}.cue` + by copying the previous patch release file and editing the version, date, commits, and + changelog entries to match this release. + - [ ] Add a description key to the cue file with a description of the release (see previous releases for examples). - [ ] Update version number in `distribution/install.sh` - [ ] Add new version to `website/cue/reference/versions.cue` @@ -62,5 +64,5 @@ export PREP_BRANCH=prepare-v-0-"${CURRENT_MINOR_VERSION}"-"${NEW_PATCH_VERSION}" - Follow the [instructions at the top of the mirror.yaml file](https://github.com/DataDog/images/blob/fbf12868e90d52e513ebca0389610dea8a3c7e1a/mirror.yaml#L33-L49). - [ ] Cherry-pick any release commits from the release branch that are not on `master`, to `master` - [ ] Reset the `website` branch to the `HEAD` of the release branch to update https://vector.dev - - [ ] `git checkout website && git reset --hard origin/"${RELEASE_BRANCH}" && git push` + - [ ] `git fetch origin && git checkout website && git reset --hard origin/"${RELEASE_BRANCH}" && git push --force-with-lease` - [ ] Kick-off post-mortems for any regressions resolved by the release diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 2d239269beeda..0aa52163fafd3 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -9,6 +9,7 @@ This should help the reviewers give feedback faster and with higher quality. --> ## Change Type + - [ ] Bug fix - [ ] New feature - [ ] Dependencies @@ -16,6 +17,7 @@ This should help the reviewers give feedback faster and with higher quality. --> - [ ] Performance ## Is this a breaking change? + - [ ] Yes - [ ] No @@ -36,6 +38,7 @@ Changes to CI, website, playground and similar are generally not considered user --> ## Notes + - Please read our [Vector contributor resources](https://github.com/vectordotdev/vector/tree/master/docs#getting-started). - Do not hesitate to use `@vectordotdev/vector` to reach out to us regarding this PR. - Some CI checks run only after we manually approve them. @@ -48,7 +51,7 @@ Changes to CI, website, playground and similar are generally not considered user - Feel free to push as many commits as you want. They will be squashed into one before merging. - For example, you can run `git merge origin master` and `git push`. - If this PR introduces changes Vector dependencies (modifies `Cargo.lock`), please - run `make build-licenses` to regenerate the [license inventory](https://github.com/vectordotdev/vrl/blob/main/LICENSE-3rdparty.csv) and commit the changes (if any). More details [here](https://crates.io/crates/dd-rust-license-tool). + run `make build-licenses` to regenerate the [license inventory](https://github.com/vectordotdev/vrl/blob/main/LICENSE-3rdparty.csv) and commit the changes (if any). More details on the [dd-rust-license-tool](https://crates.io/crates/dd-rust-license-tool). diff --git a/SECURITY.md b/SECURITY.md index 667a3f7b19d8e..e56f752785d91 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -19,9 +19,11 @@ possible on our security efforts. - [Open Source](#open-source) - [Workflow](#workflow) - [Version Control](#version-control) - - [Git](#git) + - [Pull Requests](#pull-requests) + - [Reviews & Approvals](#reviews--approvals) - [Signed Commits](#signed-commits) - [Protected Branches](#protected-branches) + - [Merge Policies](#merge-policies) - [Personnel](#personnel) - [Education](#education) - [Policies](#policies) @@ -34,26 +36,22 @@ possible on our security efforts. - [Unsafe Code](#unsafe-code) - [User Privileges](#user-privileges) - [Dependencies](#dependencies) - - [Change Control](#change-control) - - [Pull Requests](#pull-requests) - - [Reviews & Approvals](#reviews--approvals) - - [Merge Policies](#merge-policies) - [Automated Checks](#automated-checks) - [Vulnerability Scans & Security Advisories](#vulnerability-scans--security-advisories) - [Vulnerability Remediation](#vulnerability-remediation) - - [Fuzz Testing](#fuzz-testing) - [Infrastructure](#infrastructure) - [CI/CD](#cicd) - - [Runtime Isolation](#runtime-isolation) - [Network Security](#network-security) - - [Penetration Testing](#penetration-testing) - [Protocols](#protocols) - [Release Artifacts & Channels](#release-artifacts--channels) - [Asset Audit Logging](#asset-audit-logging) - [Asset Signatures & Checksums](#asset-signatures--checksums) +- [Vulnerability Reporting](#vulnerability-reporting) - [Meta](#meta) - [Review Schedule](#review-schedule) - [Vulnerability Reporting](#vulnerability-reporting) + - [Vector CI](#vector-ci) + - [Other reports](#other-reports) ## Project Structure @@ -84,10 +82,19 @@ are all publicly available. Version control ensures that all code changes are audited and authentic. -#### Git +Vector uses [Git][urls.git] to ensure that changes are auditable and traceable. + +#### Pull Requests + +All changes to Vector must go through a pull request review process. + +#### Reviews & Approvals + +All pull requests must be reviewed by at least one Vector team member. The +review process takes into account many factors, all of which are detailed in +our [Reviewing guide](REVIEWING.md). In exceptional circumstances, this +approval can be retroactive. -Vector leverages the [Git][urls.git] version-control system. This ensures all -changes are audited and traceable. #### Signed Commits @@ -107,6 +114,12 @@ are [protected][urls.github_protected_branches]. The exact requirements are: - Signed commits are required. - Administrators are included in these checks. +#### Merge Policies + +Vector requires pull requests to pass all [automated checks](#automated-checks). +Once passed, the pull request must be squashed and merged. This creates a clean +linear history with a Vector team member's co-sign. + ## Personnel ### Education @@ -116,7 +129,7 @@ the [contributing](CONTRIBUTING.md) and [reviewing](REVIEWING.md) documents. ### Policies -Vector maintains this security policy. Changed are communicated to all Vector +Vector maintains this security policy. Changes are communicated to all Vector team members. ### Two-factor Authentication @@ -150,8 +163,9 @@ catch many common sources of vulnerabilities at compile time. #### Unsafe Code -Vector does not allow the use of unsafe code except in circumstances where it -is required, such as dealing with CFFI. +Vector uses unsafe code sparingly. Unsafe is sometimes required, such as dealing +with CFFI. We may occasionally also use unsafe code for performance reasons but +those changes are kept to a minimum. #### User Privileges @@ -164,27 +178,6 @@ Vector aims to reduce the number of dependencies it relies on. If a dependency is added it goes through a comprehensive review process that is detailed in the [Reviewing guide](REVIEWING.md#dependencies). -### Change Control - -As noted above Vector uses the Git version control system on GitHub. - -#### Pull Requests - -All changes to Vector must go through a pull request review process. - -#### Reviews & Approvals - -All pull requests must be reviewed by at least one Vector team member. The -review process takes into account many factors, all of which are detailed in -our [Reviewing guide](REVIEWING.md). In exceptional circumstances, this -approval can be retroactive. - -#### Merge Policies - -Vector requires pull requests to pass all [automated checks](#automated-checks). -Once passed, the pull request must be squashed and merged. This creates a clean -linear history with a Vector team member's co-sign. - ### Automated Checks When possible, we'll create automated checks to enforce security policies. @@ -195,25 +188,21 @@ When possible, we'll create automated checks to enforce security policies. is part of the [Rust Security advisory database][urls.rust_sec]. The configuration, and a list of currently accepted advisories, are maintained in the [Cargo Deny configuration][urls.cargo_deny_configuration]. The check is run - [on every incoming PR][urls.cargo_deny_schedule] to the Vector project. + on every PR to the Vector project. - Vector implements [Dependabot][urls.dependabot] which performs automated upgrades on dependencies and [alerts][urls.dependabot_alerts] about any dependency-related security vulnerabilities. #### Vulnerability Remediation -If the advisory check fails then the PR will not be merged. We review each advisory to -determine what action to take. If possible, we update the dependency to a version -where the vulnerability has been addressed. If this isn't possible we either record -the acceptance of the vulnerability or replace the dependency. If we accept the -vulnerability we open a ticket to track its remediation, generally awaiting a fix -upstream. If the risk is deemed unacceptable we revisit the code and dependency -to find a more secure alternative. - -#### Fuzz Testing - -Vector implements automated fuzz testing to probe our code for other sources -of potential vulnerabilities. +If the advisory check fails due to changes made in the PR, it will not be +merged. We review each advisory to determine what action to take. Whenever +possible, we update the dependency to a version where the vulnerability has been +addressed. If this isn't possible we either record the acceptance of the +vulnerability or replace the dependency. If we accept the vulnerability we open +a ticket to track its remediation, generally awaiting a fix upstream. If the +risk is deemed unacceptable we revisit the code and dependency to find a more +secure alternative. ## Infrastructure @@ -223,16 +212,12 @@ Vector's infrastructure and how we secure them. ### CI/CD -#### Runtime Isolation - -All builds run in an isolated sandbox that is destroyed after each use. +All builds run in GitHub Actions runners which are ephemeral and don't maintain +state after the job is completed. We ensure we are following [OpenSSF best +practices](https://bestpractices.dev/) to minimize CI risk and exposure. ### Network Security -#### Penetration Testing - -Vector performs quarterly pen tests on vector.dev. - #### Protocols All network traffic is secured via TLS and SSH. This includes checking out @@ -260,7 +245,7 @@ Vector reviews this policy and all user access levels on a quarterly basis. We deeply appreciate any effort to discover and disclose security vulnerabilities responsibly. -## Vector CI +#### Vector CI If you would like to report a Vector CI vulnerability or have any security concerns with other Datadog products, please e-mail security@datadoghq.com. @@ -270,9 +255,9 @@ and verify the vulnerability before taking the necessary steps to fix it. After our initial reply to your disclosure, which should be directly after receiving it, we will periodically update you with the status of the fix. -## Other reports +#### Other reports -Due to the nature of a open-source project, Vector deployments are fully managed by users. Thus vulnerabilities in Vector deployments could +Due to the nature of an open-source project, Vector deployments are fully managed by users. Thus vulnerabilities in Vector deployments could potentially be exploited by malicious actors who already have access to the user’s infrastructure. We encourage responsible disclosure via opening an [open an issue][urls.new_security_report] so that risks can be properly assessed and mitigated. @@ -286,7 +271,6 @@ following when reporting: [urls.cargo_deny]: https://github.com/EmbarkStudios/cargo-deny [urls.cargo_deny_configuration]: https://github.com/vectordotdev/vector/blob/master/deny.toml -[urls.cargo_deny_schedule]: https://github.com/vectordotdev/vector/blob/master/.github/workflows/test.yml#L267 [urls.dependabot]: https://github.com/marketplace/dependabot-preview [urls.dependabot_alerts]: https://github.com/vectordotdev/vector/network/alerts [urls.git]: https://git-scm.com/ diff --git a/VERSIONING.md b/VERSIONING.md index a84b608f6ad80..9dd49bd92b68e 100644 --- a/VERSIONING.md +++ b/VERSIONING.md @@ -9,16 +9,16 @@ Please see the [FAQ](#faq) section for more info.** 1. [Convention](#convention) -1. [Public API](#public-api) +2. [Public API](#public-api) 1. [Areas that *are* covered](#areas-that-are-covered) 1. [Intended for *public* consumption](#intended-for-public-consumption) - 1. [Intended for *private* consumption](#intended-for-private-consumption) - 1. [Areas that are *NOT* covered](#areas-that-are-not-covered) -1. [FAQ](#faq) + 2. [Intended for *private* consumption](#intended-for-private-consumption) + 2. [Areas that are *NOT* covered](#areas-that-are-not-covered) +3. [FAQ](#faq) 1. [How often is Vector released?](#how-often-is-vector-released) - 1. [How does Vector treat patch and minor versions?](#how-does-vector-treat-patch-and-minor-versions) - 1. [How does Vector treat major versions \(breaking changes\)?](#how-does-vector-treat-major-versions-breaking-changes) - 1. [How does Vector treat pre-1.0 versions?](#how-does-vector-treat-pre-10-versions) + 2. [How does Vector treat patch and minor versions?](#how-does-vector-treat-patch-and-minor-versions) + 3. [How does Vector treat major versions \(breaking changes\)?](#how-does-vector-treat-major-versions-breaking-changes) + 4. [How does Vector treat pre-1.0 versions?](#how-does-vector-treat-pre-10-versions) diff --git a/benches/files.rs b/benches/files.rs index 703c4a984c8cd..6c6c00f367a66 100644 --- a/benches/files.rs +++ b/benches/files.rs @@ -1,9 +1,9 @@ use std::{convert::TryInto, path::PathBuf, time::Duration}; use bytes::Bytes; -use criterion::{BatchSize, Criterion, SamplingMode, Throughput, criterion_group}; +use criterion::{BatchSize, BenchmarkId, Criterion, SamplingMode, Throughput, criterion_group}; use futures::{SinkExt, StreamExt, stream}; -use tempfile::tempdir; +use tempfile::{TempDir, tempdir}; use tokio::fs::OpenOptions; use tokio_util::codec::{BytesCodec, FramedWrite}; use vector::{ @@ -12,6 +12,70 @@ use vector::{ }; use vector_lib::codecs::{TextSerializerConfig, encoding::FramingConfig}; +fn build_file_benchmark_environment( + idle_files: usize, +) -> ( + tokio::runtime::Runtime, + vector::topology::RunningTopology, + tokio::fs::File, + TempDir, +) { + let temp = tempdir().unwrap(); + let directory = temp.path().to_path_buf(); + + let mut data_dir = directory.clone(); + data_dir.push("data"); + std::fs::create_dir(&data_dir).unwrap(); + + let active_file = directory.join("active.log"); + std::fs::write(&active_file, []).unwrap(); + + for index in 0..idle_files { + let idle_file = directory.join(format!("idle-{index}.log")); + std::fs::write(idle_file, []).unwrap(); + } + + let output = directory.join("output.txt"); + + let mut source = sources::file::FileConfig::default(); + source.include = vec![directory.join("*.log")]; + source.data_dir = Some(data_dir); + + let mut config = config::Config::builder(); + config.add_source("in", source); + config.add_sink( + "out", + &["in"], + sinks::file::FileSinkConfig { + path: output.try_into().unwrap(), + idle_timeout: Duration::from_secs(30), + encoding: (None::, TextSerializerConfig::default()).into(), + compression: sinks::file::Compression::None, + acknowledgements: Default::default(), + timezone: Default::default(), + internal_metrics: Default::default(), + truncate: Default::default(), + }, + ); + + let rt = runtime(); + let (topology, input) = rt.block_on(async move { + let (topology, _crash) = start_topology(config.build().unwrap(), false).await; + + let mut options = OpenOptions::new(); + options.create(true).append(true).write(true); + + let input = options.open(active_file).await.unwrap(); + + // Give the source enough time to discover files and drive the idle watchers to EOF. + tokio::time::sleep(Duration::from_millis(250)).await; + + (topology, input) + }); + + (rt, topology, input, temp) +} + fn benchmark_files_no_partitions(c: &mut Criterion) { let num_lines: usize = 10_000; let line_size: usize = 100; @@ -99,10 +163,53 @@ fn benchmark_files_no_partitions(c: &mut Criterion) { group.finish(); } +fn benchmark_files_with_idle_watchers(c: &mut Criterion) { + let num_lines: usize = 10_000; + let line_size: usize = 100; + + let mut group = c.benchmark_group("files/idle_watchers"); + group.throughput(Throughput::Bytes((num_lines * line_size) as u64)); + group.sampling_mode(SamplingMode::Flat); + group.sample_size(10); + + for idle_files in [0usize, 128, 512] { + group.bench_with_input( + BenchmarkId::from_parameter(idle_files), + &idle_files, + |b, idle_files| { + b.iter_batched( + || build_file_benchmark_environment(*idle_files), + |(rt, topology, input, _temp)| { + rt.block_on(async move { + let mut sink = FramedWrite::new(input, BytesCodec::new()); + let raw_lines = + random_lines(line_size).take(num_lines).map(|mut line| { + line.push('\n'); + Bytes::from(line) + }); + let mut lines = stream::iter(raw_lines); + while let Some(line) = lines.next().await { + sink.send(line).await.unwrap(); + } + + // Keep the topology alive briefly so the benchmark includes watcher polling work. + tokio::time::sleep(Duration::from_millis(250)).await; + topology.stop().await; + }); + }, + BatchSize::LargeInput, + ) + }, + ); + } + + group.finish(); +} + criterion_group!( name = benches; // encapsulates inherent CI noise we saw in // https://github.com/vectordotdev/vector/issues/5394 config = Criterion::default().noise_threshold(0.05); - targets = benchmark_files_no_partitions + targets = benchmark_files_no_partitions, benchmark_files_with_idle_watchers ); diff --git a/benches/http.rs b/benches/http.rs index 0bc99dec551cf..5c8b5eaac7efe 100644 --- a/benches/http.rs +++ b/benches/http.rs @@ -55,7 +55,6 @@ fn benchmark_http(c: &mut Criterion) { compression: *compression, method: Default::default(), auth: Default::default(), - headers: Default::default(), payload_prefix: Default::default(), payload_suffix: Default::default(), batch, @@ -64,6 +63,7 @@ fn benchmark_http(c: &mut Criterion) { request: Default::default(), tls: Default::default(), acknowledgements: Default::default(), + retry_strategy: Default::default(), }, ); diff --git a/benches/metrics_snapshot.rs b/benches/metrics_snapshot.rs index 279911e5e65a4..4df220256f7b9 100644 --- a/benches/metrics_snapshot.rs +++ b/benches/metrics_snapshot.rs @@ -1,4 +1,7 @@ use criterion::{BenchmarkId, Criterion, criterion_group}; +use strum::IntoEnumIterator; +use vector_lib::counter; +use vector_lib::internal_event::CounterName; fn benchmark(c: &mut Criterion) { let mut group = c.benchmark_group("metrics_snapshot"); @@ -22,8 +25,9 @@ fn prepare_metrics(cardinality: usize) -> &'static vector::metrics::Controller { let controller = vector::metrics::Controller::get().unwrap(); controller.reset(); + let name = CounterName::iter().next().unwrap(); for idx in 0..cardinality { - metrics::counter!("test", "idx" => idx.to_string()).increment(1); + counter!(name, "idx" => idx.to_string()).increment(1); } controller diff --git a/build.rs b/build.rs index cf35846cbe61c..01cd4bea5e8ef 100644 --- a/build.rs +++ b/build.rs @@ -104,13 +104,30 @@ fn git_short_hash() -> std::io::Result { }) } +#[cfg(not(feature = "nightly"))] +fn git_path(path: &str) -> std::io::Result { + let output_result = Command::new("git") + .args(["rev-parse", "--git-path", path]) + .output(); + + output_result.map(|output| { + String::from_utf8(output.stdout) + .expect("valid UTF-8") + .trim_end_matches(['\r', '\n']) + .to_owned() + }) +} + fn main() { // Always rerun if the build script itself changes. println!("cargo:rerun-if-changed=build.rs"); // re-run if the HEAD has changed. This is only necessary for non-release and nightly builds. #[cfg(not(feature = "nightly"))] - println!("cargo:rerun-if-changed=.git/HEAD"); + println!( + "cargo:rerun-if-changed={}", + git_path("HEAD").expect("git HEAD path detection failed") + ); #[cfg(feature = "protobuf-build")] { @@ -120,6 +137,7 @@ fn main() { println!("cargo:rerun-if-changed=proto/vector/dd_trace.proto"); println!("cargo:rerun-if-changed=proto/vector/ddsketch_full.proto"); println!("cargo:rerun-if-changed=proto/vector/vector.proto"); + println!("cargo:rerun-if-changed=proto/vector/observability.proto"); // Create and store the "file descriptor set" from the compiled Protocol Buffers packages. // @@ -133,7 +151,8 @@ fn main() { let mut prost_build = prost_build::Config::new(); prost_build .btree_map(["."]) - .file_descriptor_set_path(protobuf_fds_path); + .file_descriptor_set_path(protobuf_fds_path) + .extern_path(".event", "crate::event::proto"); // Use existing event types from vector-core tonic_build::configure() .protoc_arg("--experimental_allow_proto3_optional") @@ -147,6 +166,7 @@ fn main() { "proto/third-party/google/pubsub/v1/pubsub.proto", "proto/third-party/google/rpc/status.proto", "proto/vector/vector.proto", + "proto/vector/observability.proto", ], &[ "proto/third-party", diff --git a/changelog.d/13882_azure_blob_storage_source.feature.md b/changelog.d/13882_azure_blob_storage_source.feature.md new file mode 100644 index 0000000000000..101e3faa71aa8 --- /dev/null +++ b/changelog.d/13882_azure_blob_storage_source.feature.md @@ -0,0 +1,18 @@ +New `azure_blob` source for reading logs from Azure Blob Storage + +A new `azure_blob` source has been added that allows Vector to read logs from Azure Blob Storage containers. The source processes events from Azure Storage Queues to detect when new blobs are created or modified, then reads and decodes the blob contents. The design followed an intended feature parity with AWS S3 source. + +Key features: + +- Supports connection string authentication +- Automatically processes Azure Event Grid notifications via Storage Queue +- Configurable compression support (gzip, zstd) with auto-detection from file extension (Content-Encoding detection NOT supported) +- Configurable framing for splitting blob contents (newline-delimited, character-delimited, bytes, etc.) +- Multiline aggregation support for combining related log lines (stack traces, multi-line JSON, etc.) +- Configurable decoding for various log formats +- Automatic enrichment of events with container, blob, and timestamp metadata +- Support for acknowledgements + +This enables efficient log collection from Azure Blob Storage without polling, making it suitable for high-throughput logging scenarios. + +authors: ldrozdz93 kwapik aq1l diff --git a/changelog.d/23657-grpc-health-protocol.enhancement.md b/changelog.d/23657-grpc-health-protocol.enhancement.md deleted file mode 100644 index a26e9ebf42b2c..0000000000000 --- a/changelog.d/23657-grpc-health-protocol.enhancement.md +++ /dev/null @@ -1,7 +0,0 @@ -`vector` source: Implement standard gRPC health checking protocol (`grpc.health.v1.Health`) -alongside the existing custom health check endpoint. This enables compatibility with standard -tools like `grpc-health-probe` for Kubernetes and other orchestration systems. - -Issue: https://github.com/vectordotdev/vector/issues/23657 - -authors: jpds diff --git a/changelog.d/24455_otel_source_per_signal_decoding.enhancement.md b/changelog.d/24455_otel_source_per_signal_decoding.enhancement.md deleted file mode 100644 index 79db8934df463..0000000000000 --- a/changelog.d/24455_otel_source_per_signal_decoding.enhancement.md +++ /dev/null @@ -1,21 +0,0 @@ -The `opentelemetry` source now supports independent configuration of OTLP decoding for logs, metrics, and traces. This allows more granular -control over which signal types are decoded, while maintaining backward compatibility with the existing boolean configuration. - -## Simple boolean form (applies to all signals) - -```yaml -use_otlp_decoding: true # All signals preserve OTLP format -# or -use_otlp_decoding: false # All signals use Vector native format (default) -``` - -## Per-signal configuration - -```yaml -use_otlp_decoding: - logs: false # Convert to Vector native format - metrics: false # Convert to Vector native format - traces: true # Preserve OTLP format -``` - -authors: pront diff --git a/changelog.d/25531_logstash_ack_windows.fix.md b/changelog.d/25531_logstash_ack_windows.fix.md new file mode 100644 index 0000000000000..786406ab211a4 --- /dev/null +++ b/changelog.d/25531_logstash_ack_windows.fix.md @@ -0,0 +1,3 @@ +Fixed the `logstash` source to preserve writer window boundaries when generating ACKs. This prevents batched reads from producing ACK sequences that advance past the current window, which could lead to "invalid sequence number received" errors and duplicate retransmits under load. + +authors: bruceg diff --git a/changelog.d/bump_kube_k8s_openapi.enhancement.md b/changelog.d/bump_kube_k8s_openapi.enhancement.md deleted file mode 100644 index 92b8fd24de684..0000000000000 --- a/changelog.d/bump_kube_k8s_openapi.enhancement.md +++ /dev/null @@ -1,2 +0,0 @@ -Bumped `kube` dependency from 0.93.0 to 3.0.1 and `k8s-openapi` from 0.22.0 to 0.27.0, adding support for Kubernetes API versions up to v1.35. -authors: hligit diff --git a/changelog.d/spawned_task_component_tags.fix.md b/changelog.d/spawned_task_component_tags.fix.md new file mode 100644 index 0000000000000..d2f5c2bb068a1 --- /dev/null +++ b/changelog.d/spawned_task_component_tags.fix.md @@ -0,0 +1,3 @@ +Internal telemetry (metrics and logs) emitted from work that Vector runs on spawned `tokio` tasks now correctly inherits the owning component's tags (`component_id`, `component_kind`, `component_type`). Previously, several components spawned background tasks without propagating the tracing span, so some internal events emitted from those tasks were missing their component tags. Affected emissions include the `datadog_logs` sink's `component_discarded_events_total` (events too large to encode), the `gcp_pubsub` source's `component_errors_total`/`component_discarded_events_total` from its per-stream tasks, and the `splunk_hec` sinks' acknowledgement-handling `component_errors_total`. + +authors: gwenaskell diff --git a/changelog.d/windows_event_log_source.feature.md b/changelog.d/windows_event_log_source.feature.md deleted file mode 100644 index 7f64f115a3391..0000000000000 --- a/changelog.d/windows_event_log_source.feature.md +++ /dev/null @@ -1,3 +0,0 @@ -Added a new `windows_event_log` source that collects logs from Windows Event Log channels using the native Windows Event Log API with pull-mode subscriptions, bookmark-based checkpointing, and configurable field filtering. - -authors: tot19 diff --git a/clippy.toml b/clippy.toml index cbd8c640fdb71..7cb042eba5a2d 100644 --- a/clippy.toml +++ b/clippy.toml @@ -8,9 +8,20 @@ disallowed-methods = [ { path = "vrl::stdlib::all", reason = "Use `vector_vrl_functions::all()` instead for consistency across all Vector VRL functions." }, ] +disallowed-macros = [ + { path = "metrics::counter", reason = "Use the `counter!` macro from `vector_common` with a `CounterName` variant instead of a raw string." }, + { path = "metrics::histogram", reason = "Use the `histogram!` macro from `vector_common` with a `HistogramName` variant instead of a raw string." }, + { path = "metrics::gauge", reason = "Use the `gauge!` macro from `vector_common` with a `GaugeName` variant instead of a raw string." }, +] + disallowed-types = [ { path = "once_cell::sync::OnceCell", reason = "Use `std::sync::OnceLock` instead." }, { path = "once_cell::unsync::OnceCell", reason = "Use `std::cell::OnceCell` instead." }, { path = "once_cell::sync::Lazy", reason = "Use `std::sync::LazyLock` instead." }, { path = "once_cell::unsync::Lazy", reason = "Use `std::sync::LazyCell` instead." }, ] + + +await-holding-invalid-types = [ + { path = "tracing::span::Entered", reason = "Holding a tracing span guard across an `.await` is thread-local and silently breaks when the task resumes on a different thread. Use `#[tracing::instrument]`, `future.instrument(span)`, or `Span::in_scope(|| ...)` instead." }, +] diff --git a/config/examples/environment_variables.yaml b/config/examples/environment_variables.yaml index 2f73528397ad1..d8421c1cc8e01 100644 --- a/config/examples/environment_variables.yaml +++ b/config/examples/environment_variables.yaml @@ -13,7 +13,7 @@ data_dir: "/var/lib/vector" sources: apache_logs: type: "file" - include: [ "/var/log/apache2/*.log" ] + include: ["/var/log/apache2/*.log"] # ignore files older than 1 day ignore_older_secs: 86400 @@ -21,7 +21,7 @@ sources: # Docs: https://vector.dev/docs/reference/configuration/transforms/remap transforms: add_host: - inputs: [ "apache_logs" ] + inputs: ["apache_logs"] type: "remap" source: | ''' @@ -32,7 +32,7 @@ transforms: # Docs: https://vector.dev/docs/reference/configuration/sinks/console sinks: out: - inputs: [ "add_host" ] + inputs: ["add_host"] type: "console" encoding: codec: "json" diff --git a/config/examples/file_to_cloudwatch_metrics.yaml b/config/examples/file_to_cloudwatch_metrics.yaml index f322ab8f091b0..138f9d20ceac2 100644 --- a/config/examples/file_to_cloudwatch_metrics.yaml +++ b/config/examples/file_to_cloudwatch_metrics.yaml @@ -8,13 +8,13 @@ data_dir: "/var/lib/vector" sources: file: type: "file" - include: [ "sample.log" ] + include: ["sample.log"] start_at_beginning: true # Structure and parse the data transforms: remap: - inputs: [ "file" ] + inputs: ["file"] type: "remap" drop_on_error: false source: | @@ -22,7 +22,7 @@ transforms: # Transform into metrics log_to_metric: - inputs: [ "remap" ] + inputs: ["remap"] type: "log_to_metric" metrics: - type: "counter" @@ -35,19 +35,19 @@ transforms: # Output data sinks: console_metrics: - inputs: [ "log_to_metric" ] + inputs: ["log_to_metric"] type: "console" encoding: codec: "json" console_logs: - inputs: [ "remap" ] + inputs: ["remap"] type: "console" encoding: codec: "json" cloudwatch: - inputs: [ "log_to_metric" ] + inputs: ["log_to_metric"] type: "aws_cloudwatch_metrics" namespace: "vector" endpoint: "http://localhost:4566" diff --git a/config/examples/http_sink_custom_retry.yaml b/config/examples/http_sink_custom_retry.yaml new file mode 100644 index 0000000000000..fd973c890e991 --- /dev/null +++ b/config/examples/http_sink_custom_retry.yaml @@ -0,0 +1,42 @@ +# HTTP sink example with a custom retry strategy +# ---------------------------------------------------- +# Sends demo logs to an HTTP endpoint and only retries the response codes +# that the upstream API documents as transient. + +data_dir: "/var/lib/vector" + +sources: + demo_logs: + type: "demo_logs" + format: "json" + interval: 1 + +sinks: + http_out: + type: "http" + inputs: ["demo_logs"] + uri: "https://example.com/ingest" + method: "post" + + # Skip the startup probe so the example can be adapted locally. + healthcheck: + enabled: false + + # Send newline-delimited JSON in the request body. + framing: + method: "newline_delimited" + encoding: + codec: "json" + + # Control how many retries are made and how quickly backoff grows. + request: + timeout_secs: 60 + retry_attempts: 8 + retry_initial_backoff_secs: 2 + retry_max_duration_secs: 30 + + # Retry only on the exact HTTP status codes that this destination + # treats as temporary failures. + retry_strategy: + type: "custom" + status_codes: [408, 425, 429, 503] diff --git a/config/examples/namespacing/sinks/es_cluster.yaml b/config/examples/namespacing/sinks/es_cluster.yaml index 820a49080b474..f916dac1b7180 100644 --- a/config/examples/namespacing/sinks/es_cluster.yaml +++ b/config/examples/namespacing/sinks/es_cluster.yaml @@ -1,6 +1,6 @@ # Send structured data to a short-term storage -inputs: ["apache_sample"] # only take sampled data +inputs: ["apache_sample"] # only take sampled data type: "elasticsearch" -endpoint: "http://79.12.221.222:9200" # local or external host +endpoint: "http://79.12.221.222:9200" # local or external host bulk: - index: "vector-%Y-%m-%d" # daily indices + index: "vector-%Y-%m-%d" # daily indices diff --git a/config/examples/namespacing/sinks/s3_archives.yaml b/config/examples/namespacing/sinks/s3_archives.yaml index a72c27027e8a1..6de3cd4df8f56 100644 --- a/config/examples/namespacing/sinks/s3_archives.yaml +++ b/config/examples/namespacing/sinks/s3_archives.yaml @@ -1,13 +1,13 @@ # Send structured data to a cost-effective long-term storage -inputs: ["apache_parser"] # don't sample for S3 +inputs: ["apache_parser"] # don't sample for S3 type: "aws_s3" region: "us-east-1" bucket: "my-log-archives" -key_prefix: "date=%Y-%m-%d" # daily partitions, hive friendly format -compression: "gzip" # compress final objects +key_prefix: "date=%Y-%m-%d" # daily partitions, hive friendly format +compression: "gzip" # compress final objects framing: - method: "newline_delimited" # new line delimited... + method: "newline_delimited" # new line delimited... encoding: - codec: "json" # ...JSON + codec: "json" # ...JSON batch: - max_bytes: 10000000 # 10mb uncompressed + max_bytes: 10000000 # 10mb uncompressed diff --git a/config/examples/namespacing/transforms/apache_sample.yaml b/config/examples/namespacing/transforms/apache_sample.yaml index 91b789e1016f2..cca25cf0989af 100644 --- a/config/examples/namespacing/transforms/apache_sample.yaml +++ b/config/examples/namespacing/transforms/apache_sample.yaml @@ -1,4 +1,4 @@ # Sample the data to save on cost inputs: ["apache_parser"] type: "sample" -rate: 2 # only keep 50% (1/`rate`) +rate: 2 # only keep 50% (1/`rate`) diff --git a/config/examples/stdio.yaml b/config/examples/stdio.yaml index cbe6429d8afae..7ca39a81876da 100644 --- a/config/examples/stdio.yaml +++ b/config/examples/stdio.yaml @@ -11,7 +11,7 @@ sources: sinks: out: - inputs: [ "in" ] + inputs: ["in"] type: "console" encoding: codec: "text" diff --git a/config/examples/wrapped_json.yaml b/config/examples/wrapped_json.yaml index 90fed711f2a18..4bd6ab0a79f58 100644 --- a/config/examples/wrapped_json.yaml +++ b/config/examples/wrapped_json.yaml @@ -12,14 +12,14 @@ data_dir: "/var/lib/vector" sources: logs: type: "file" - include: [ "/var/log/*.log" ] + include: ["/var/log/*.log"] ignore_older_secs: 86400 # 1 day # Parse the data as JSON # Docs: https://vector.dev/docs/reference/configuration/transforms/remap transforms: parse_json: - inputs: [ "logs" ] + inputs: ["logs"] type: "remap" drop_on_error: false source: | @@ -36,7 +36,7 @@ transforms: # Docs: https://vector.dev/docs/reference/configuration/sinks/console sinks: out: - inputs: [ "parse_json" ] + inputs: ["parse_json"] type: "console" encoding: codec: "json" diff --git a/config/vector.yaml b/config/vector.yaml index 590c2aa9ba152..c48be7abd7a31 100644 --- a/config/vector.yaml +++ b/config/vector.yaml @@ -41,9 +41,8 @@ sinks: json: pretty: true -# Vector's GraphQL API (disabled by default) -# Uncomment to try it out with the `vector top` command or -# in your browser at http://localhost:8686 +# Vector's API (disabled by default) +# Uncomment to try it out with the `vector top` command # api: # enabled: true # address: "127.0.0.1:8686" diff --git a/deny.toml b/deny.toml index c9a83079d192f..e6c3a90bd6e9c 100644 --- a/deny.toml +++ b/deny.toml @@ -40,14 +40,12 @@ license-files = [ [advisories] ignore = [ - # Vulnerability in `rsa` crate: https://rustsec.org/advisories/RUSTSEC-2023-0071.html - # There is not fix available yet. - # https://github.com/vectordotdev/vector/issues/19262 - "RUSTSEC-2023-0071", - { id = "RUSTSEC-2024-0388", reason = "derivative is unmaintained" }, - { id = "RUSTSEC-2024-0384", reason = "instant is unmaintained" }, - { id = "RUSTSEC-2020-0168", reason = "mach is unmaintained" }, - { id = "RUSTSEC-2025-0012", reason = "backoff is unmaintained" }, - # rustls-pemfile is unmaintained. Blocked by both async-nats and http 1.0.0 upgrade. - { id = "RUSTSEC-2025-0134", reason = "rustls-pemfile is unmaintained" }, + { id = "RUSTSEC-2023-0071", reason = "rsa marvin attack - unpatched upstream (https://github.com/vectordotdev/vector/issues/19262)" }, + { id = "RUSTSEC-2024-0388", reason = "derivative is unmaintained (https://github.com/vectordotdev/vector/issues/24940)" }, + { id = "RUSTSEC-2025-0134", reason = "rustls-pemfile is unmaintained - unpatched crate (https://github.com/bytebeamio/rumqtt/issues/1010) & tonic/reqwest upgrade (https://github.com/vectordotdev/vector/issues/19179)" }, + { id = "RUSTSEC-2026-0049", reason = "rustls-webpki 0.102 is vulnerable - tonic upgrade (https://github.com/vectordotdev/vector/issues/19179)" }, + { id = "RUSTSEC-2026-0098", reason = "rustls-webpki 0.102/0.101 is vulnerable - tonic upgrade (https://github.com/vectordotdev/vector/issues/19179)" }, + { id = "RUSTSEC-2026-0099", reason = "rustls-webpki 0.102/0.101 is vulnerable - tonic upgrade (https://github.com/vectordotdev/vector/issues/19179)" }, + { id = "RUSTSEC-2024-0436", reason = "paste crate is unmaintained - transitive dependency via parquet v56.2.0, no safe upgrade available" }, + { id = "RUSTSEC-2026-0104", reason = "rustls-webpki 0.102/0.101 CRL parsing panic - tonic upgrade (https://github.com/vectordotdev/vector/issues/19179); 0.103 already patched to 0.103.13" }, ] diff --git a/distribution/docker/README.md b/distribution/docker/README.md index b6e6a66ea65e5..b504cee5a2162 100644 --- a/distribution/docker/README.md +++ b/distribution/docker/README.md @@ -33,11 +33,19 @@ observability data with Vector. ## Configuring -As shown above, you can pass a custom -[Vector configuration file][docs.setup.configuration] via the `-c` flag. You'll want -to do this since the -[default `/etc/vector/vector.yaml` configuration file][urls.default_configuration] -doesn't do anything. +Vector images do not ship a default configuration. You must provide one via the +`-c` flag, typically by mounting a [Vector configuration file][docs.setup.configuration] +from the host: + +```bash +docker run \ + -v "$PWD/vector.yaml:/etc/vector/vector.yaml:ro" \ + timberio/vector:latest-alpine \ + -c /etc/vector/vector.yaml +``` + +A reference [example configuration][urls.example_configuration] is bundled at +`/usr/share/vector/examples/vector.yaml` inside the image. ## Deploying @@ -105,12 +113,12 @@ Vector maintains special tags that are automatically updated whenever Vector is | Version | URL | |:-----------------|:---------------------------------------------------------| -| Latest major | `timberio/vector:latest-alpine` | -| Latest minor | `timberio/vector:.X-alpine` | -| Latest patch | `timberio/vector:.X-alpine` | -| Specific version | `timberio/vector:-alpine` | -| Latest nightly | `timberio/vector:nightly-alpine` | -| Specific nightly | `timberio/vector:nightly--alpine` | +| Latest major | `timberio/vector:latest-alpine` | +| Latest minor | `timberio/vector:.X-alpine` | +| Latest patch | `timberio/vector:.X-alpine` | +| Specific version | `timberio/vector:-alpine` | +| Latest nightly | `timberio/vector:nightly-alpine` | +| Specific nightly | `timberio/vector:nightly--alpine` | ### Source Files @@ -128,7 +136,7 @@ Vector's Docker source files are located [docs.transforms]: https://vector.dev/docs/reference/configurationtransforms/ [pages.index#correctness]: https://vector.dev/#correctness [pages.index#performance]: https://vector.dev/#performance -[urls.default_configuration]: https://github.com/vectordotdev/vector/blob/master/config/vector.yaml +[urls.example_configuration]: https://github.com/vectordotdev/vector/blob/master/config/vector.yaml [urls.docker_alpine]: https://hub.docker.com/_/alpine [urls.docker_debian]: https://hub.docker.com/_/debian [urls.rust]: https://www.rust-lang.org/ diff --git a/distribution/docker/alpine/Dockerfile b/distribution/docker/alpine/Dockerfile index 426254abffd39..7da74246e0826 100644 --- a/distribution/docker/alpine/Dockerfile +++ b/distribution/docker/alpine/Dockerfile @@ -1,4 +1,4 @@ -FROM docker.io/alpine:3.23@sha256:25109184c71bdad752c8312a8623239686a9a2071e8825f20acb8f2198c3f659 AS builder +FROM docker.io/alpine:3.23@sha256:5b10f432ef3da1b8d4c7eb6c487f2f5a8f096bc91145e68878dd4a5019afde11 AS builder WORKDIR /vector @@ -12,7 +12,7 @@ RUN ARCH=$(if [ "$TARGETPLATFORM" = "linux/arm/v6" ]; then echo "arm"; else cat RUN mkdir -p /var/lib/vector -FROM docker.io/alpine:3.23@sha256:25109184c71bdad752c8312a8623239686a9a2071e8825f20acb8f2198c3f659 +FROM docker.io/alpine:3.23@sha256:5b10f432ef3da1b8d4c7eb6c487f2f5a8f096bc91145e68878dd4a5019afde11 # https://github.com/opencontainers/image-spec/blob/main/annotations.md LABEL org.opencontainers.image.url="https://vector.dev" @@ -24,7 +24,7 @@ LABEL org.opencontainers.image.documentation="https://vector.dev/docs" RUN apk --no-cache add ca-certificates tzdata COPY --from=builder /vector/bin/* /usr/local/bin/ -COPY --from=builder /vector/config/vector.yaml /etc/vector/vector.yaml +COPY --from=builder /vector/config/vector.yaml /usr/share/vector/examples/vector.yaml COPY --from=builder /var/lib/vector /var/lib/vector # Smoke test diff --git a/distribution/docker/debian/Dockerfile b/distribution/docker/debian/Dockerfile index 095989978231a..6f1d232a0a163 100644 --- a/distribution/docker/debian/Dockerfile +++ b/distribution/docker/debian/Dockerfile @@ -1,4 +1,4 @@ -FROM docker.io/debian:trixie-slim@sha256:1d3c811171a08a5adaa4a163fbafd96b61b87aa871bbc7aa15431ac275d3d430 AS builder +FROM docker.io/debian:trixie-slim@sha256:b6e2a152f22a40ff69d92cb397223c906017e1391a73c952b588e51af8883bf8 AS builder WORKDIR /vector @@ -7,7 +7,7 @@ RUN dpkg -i vector_*_"$(dpkg --print-architecture)".deb RUN mkdir -p /var/lib/vector -FROM docker.io/debian:trixie-slim@sha256:1d3c811171a08a5adaa4a163fbafd96b61b87aa871bbc7aa15431ac275d3d430 +FROM docker.io/debian:trixie-slim@sha256:b6e2a152f22a40ff69d92cb397223c906017e1391a73c952b588e51af8883bf8 # https://github.com/opencontainers/image-spec/blob/main/annotations.md LABEL org.opencontainers.image.url="https://vector.dev" @@ -16,7 +16,7 @@ LABEL org.opencontainers.image.documentation="https://vector.dev/docs" # we want the latest versions of these # hadolint ignore=DL3008 -RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates tzdata systemd && rm -rf /var/lib/apt/lists/* +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates tzdata systemd libsasl2-2 && rm -rf /var/lib/apt/lists/* COPY --from=builder /usr/bin/vector /usr/bin/vector COPY --from=builder /usr/share/vector /usr/share/vector diff --git a/distribution/docker/distroless-libc/Dockerfile b/distribution/docker/distroless-libc/Dockerfile index 2e3988b7403fe..c78cda7e7d7ee 100644 --- a/distribution/docker/distroless-libc/Dockerfile +++ b/distribution/docker/distroless-libc/Dockerfile @@ -1,4 +1,4 @@ -FROM docker.io/debian:trixie-slim@sha256:1d3c811171a08a5adaa4a163fbafd96b61b87aa871bbc7aa15431ac275d3d430 AS builder +FROM docker.io/debian:trixie-slim@sha256:b6e2a152f22a40ff69d92cb397223c906017e1391a73c952b588e51af8883bf8 AS builder WORKDIR /vector @@ -7,9 +7,15 @@ RUN dpkg -i vector_*_"$(dpkg --print-architecture)".deb RUN mkdir -p /var/lib/vector +# Stage libz at its arch-specific path so it can be copied into the distroless +# runtime. distroless/cc does not include zlib; vector (via rdkafka) links +# against it dynamically and we deliberately keep it as a runtime dependency. +RUN src=$(dpkg -L zlib1g | grep -E '/libz\.so\.1$') && \ + install -D "$(realpath "$src")" "/staging${src}" + # distroless doesn't use static tags # hadolint ignore=DL3007 -FROM gcr.io/distroless/cc-debian12:latest@sha256:329e54034ce498f9c6b345044e8f530c6691f99e94a92446f68c0adf9baa8464 +FROM gcr.io/distroless/cc-debian12:latest@sha256:aa0b7af67fa8211751ea6e00baa8373ba56cc1417ffc986ec9619bd0e1556b56 # https://github.com/opencontainers/image-spec/blob/main/annotations.md LABEL org.opencontainers.image.url="https://vector.dev" @@ -21,6 +27,7 @@ COPY --from=builder /usr/share/doc/vector /usr/share/doc/vector COPY --from=builder /usr/share/vector /usr/share/vector COPY --from=builder /etc/vector /etc/vector COPY --from=builder /var/lib/vector /var/lib/vector +COPY --from=builder /staging/ / # Smoke test RUN ["vector", "--version"] diff --git a/distribution/docker/distroless-static/Dockerfile b/distribution/docker/distroless-static/Dockerfile index 9518be0c4492c..fd1abf4d74960 100644 --- a/distribution/docker/distroless-static/Dockerfile +++ b/distribution/docker/distroless-static/Dockerfile @@ -1,4 +1,4 @@ -FROM docker.io/alpine:3.23@sha256:25109184c71bdad752c8312a8623239686a9a2071e8825f20acb8f2198c3f659 AS builder +FROM docker.io/alpine:3.23@sha256:5b10f432ef3da1b8d4c7eb6c487f2f5a8f096bc91145e68878dd4a5019afde11 AS builder WORKDIR /vector @@ -9,7 +9,7 @@ RUN mkdir -p /var/lib/vector # distroless doesn't use static tags # hadolint ignore=DL3007 -FROM gcr.io/distroless/static:latest@sha256:28efbe90d0b2f2a3ee465cc5b44f3f2cf5533514cf4d51447a977a5dc8e526d0 +FROM gcr.io/distroless/static:latest@sha256:3592aa8171c77482f62bbc4164e6a2d141c6122554ace66e5cc910cadb961ff0 # https://github.com/opencontainers/image-spec/blob/main/annotations.md LABEL org.opencontainers.image.url="https://vector.dev" @@ -17,7 +17,7 @@ LABEL org.opencontainers.image.source="https://github.com/vectordotdev/vector" LABEL org.opencontainers.image.documentation="https://vector.dev/docs" COPY --from=builder /vector/bin/* /usr/local/bin/ -COPY --from=builder /vector/config/vector.yaml /etc/vector/vector.yaml +COPY --from=builder /vector/config/vector.yaml /usr/share/vector/examples/vector.yaml COPY --from=builder /var/lib/vector /var/lib/vector # Smoke test diff --git a/distribution/install.sh b/distribution/install.sh index ceeb50c253862..6e4654b813b4f 100755 --- a/distribution/install.sh +++ b/distribution/install.sh @@ -13,7 +13,7 @@ set -u # If PACKAGE_ROOT is unset or empty, default it. PACKAGE_ROOT="${PACKAGE_ROOT:-"https://packages.timber.io/vector"}" # If VECTOR_VERSION is unset or empty, default it. -VECTOR_VERSION="${VECTOR_VERSION:-"0.54.0"}" +VECTOR_VERSION="${VECTOR_VERSION:-"0.56.0"}" _divider="--------------------------------------------------------------------------------" _prompt=">>>" _indent=" " diff --git a/distribution/kubernetes/vector-agent/README.md b/distribution/kubernetes/vector-agent/README.md index dfe0f05deb137..ed2114453dc2e 100644 --- a/distribution/kubernetes/vector-agent/README.md +++ b/distribution/kubernetes/vector-agent/README.md @@ -1,6 +1,6 @@ The kubernetes manifests found in this directory have been automatically generated from the [helm chart `vector/vector`](https://github.com/vectordotdev/helm-charts/tree/master/charts/vector) -version 0.51.0 with the following `values.yaml`: +version 0.56.0 with the following `values.yaml`: ```yaml role: Agent diff --git a/distribution/kubernetes/vector-agent/configmap.yaml b/distribution/kubernetes/vector-agent/configmap.yaml index e3d7499257236..45a8cb2b5d42b 100644 --- a/distribution/kubernetes/vector-agent/configmap.yaml +++ b/distribution/kubernetes/vector-agent/configmap.yaml @@ -9,14 +9,13 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Agent - app.kubernetes.io/version: "0.54.0-distroless-libc" + app.kubernetes.io/version: "0.56.0-distroless-libc" data: agent.yaml: | data_dir: /vector-data-dir api: enabled: true - address: 127.0.0.1:8686 - playground: false + address: 0.0.0.0:8686 sources: kubernetes_logs: type: kubernetes_logs diff --git a/distribution/kubernetes/vector-agent/daemonset.yaml b/distribution/kubernetes/vector-agent/daemonset.yaml index 4c468b88989a7..c71bdd78bce4f 100644 --- a/distribution/kubernetes/vector-agent/daemonset.yaml +++ b/distribution/kubernetes/vector-agent/daemonset.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Agent - app.kubernetes.io/version: "0.54.0-distroless-libc" + app.kubernetes.io/version: "0.56.0-distroless-libc" spec: selector: matchLabels: @@ -30,7 +30,7 @@ spec: dnsPolicy: ClusterFirst containers: - name: vector - image: "docker.io/timberio/vector:0.54.0-distroless-libc" + image: "docker.io/timberio/vector:0.56.0-distroless-libc" imagePullPolicy: IfNotPresent args: - --config-dir @@ -58,6 +58,10 @@ spec: - name: prom-exporter containerPort: 9090 protocol: TCP + readinessProbe: + httpGet: + path: /health + port: 8686 volumeMounts: - name: data mountPath: "/vector-data-dir" diff --git a/distribution/kubernetes/vector-agent/rbac.yaml b/distribution/kubernetes/vector-agent/rbac.yaml index 93b151826e238..735b6e9888bf0 100644 --- a/distribution/kubernetes/vector-agent/rbac.yaml +++ b/distribution/kubernetes/vector-agent/rbac.yaml @@ -10,7 +10,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Agent - app.kubernetes.io/version: "0.54.0-distroless-libc" + app.kubernetes.io/version: "0.56.0-distroless-libc" rules: - apiGroups: - "" @@ -31,7 +31,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Agent - app.kubernetes.io/version: "0.54.0-distroless-libc" + app.kubernetes.io/version: "0.56.0-distroless-libc" roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole diff --git a/distribution/kubernetes/vector-agent/service-headless.yaml b/distribution/kubernetes/vector-agent/service-headless.yaml index 2f3a72b2619fd..387a0f22c6c95 100644 --- a/distribution/kubernetes/vector-agent/service-headless.yaml +++ b/distribution/kubernetes/vector-agent/service-headless.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Agent - app.kubernetes.io/version: "0.54.0-distroless-libc" + app.kubernetes.io/version: "0.56.0-distroless-libc" annotations: spec: clusterIP: None diff --git a/distribution/kubernetes/vector-agent/serviceaccount.yaml b/distribution/kubernetes/vector-agent/serviceaccount.yaml index 9db1dca796f94..5c8e4754b56c2 100644 --- a/distribution/kubernetes/vector-agent/serviceaccount.yaml +++ b/distribution/kubernetes/vector-agent/serviceaccount.yaml @@ -9,5 +9,5 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Agent - app.kubernetes.io/version: "0.54.0-distroless-libc" + app.kubernetes.io/version: "0.56.0-distroless-libc" automountServiceAccountToken: true diff --git a/distribution/kubernetes/vector-aggregator/README.md b/distribution/kubernetes/vector-aggregator/README.md index 213cd06790f2e..8fba80518580e 100644 --- a/distribution/kubernetes/vector-aggregator/README.md +++ b/distribution/kubernetes/vector-aggregator/README.md @@ -1,6 +1,6 @@ The kubernetes manifests found in this directory have been automatically generated from the [helm chart `vector/vector`](https://github.com/vectordotdev/helm-charts/tree/master/charts/vector) -version 0.51.0 with the following `values.yaml`: +version 0.56.0 with the following `values.yaml`: ```yaml diff --git a/distribution/kubernetes/vector-aggregator/configmap.yaml b/distribution/kubernetes/vector-aggregator/configmap.yaml index 39fa1093bffa6..23fa0da68e6f0 100644 --- a/distribution/kubernetes/vector-aggregator/configmap.yaml +++ b/distribution/kubernetes/vector-aggregator/configmap.yaml @@ -9,14 +9,13 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Aggregator - app.kubernetes.io/version: "0.54.0-distroless-libc" + app.kubernetes.io/version: "0.56.0-distroless-libc" data: aggregator.yaml: | data_dir: /vector-data-dir api: enabled: true - address: 127.0.0.1:8686 - playground: false + address: 0.0.0.0:8686 sources: datadog_agent: address: 0.0.0.0:8282 diff --git a/distribution/kubernetes/vector-aggregator/service-headless.yaml b/distribution/kubernetes/vector-aggregator/service-headless.yaml index 3170f1c3751cd..5e5dafebd2908 100644 --- a/distribution/kubernetes/vector-aggregator/service-headless.yaml +++ b/distribution/kubernetes/vector-aggregator/service-headless.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Aggregator - app.kubernetes.io/version: "0.54.0-distroless-libc" + app.kubernetes.io/version: "0.56.0-distroless-libc" annotations: spec: clusterIP: None diff --git a/distribution/kubernetes/vector-aggregator/service.yaml b/distribution/kubernetes/vector-aggregator/service.yaml index 3c423d4590371..0c9d253d84991 100644 --- a/distribution/kubernetes/vector-aggregator/service.yaml +++ b/distribution/kubernetes/vector-aggregator/service.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Aggregator - app.kubernetes.io/version: "0.54.0-distroless-libc" + app.kubernetes.io/version: "0.56.0-distroless-libc" annotations: spec: ports: diff --git a/distribution/kubernetes/vector-aggregator/serviceaccount.yaml b/distribution/kubernetes/vector-aggregator/serviceaccount.yaml index 4f97844c7fee1..8c61ef1a3dff8 100644 --- a/distribution/kubernetes/vector-aggregator/serviceaccount.yaml +++ b/distribution/kubernetes/vector-aggregator/serviceaccount.yaml @@ -9,5 +9,5 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Aggregator - app.kubernetes.io/version: "0.54.0-distroless-libc" + app.kubernetes.io/version: "0.56.0-distroless-libc" automountServiceAccountToken: true diff --git a/distribution/kubernetes/vector-aggregator/statefulset.yaml b/distribution/kubernetes/vector-aggregator/statefulset.yaml index d107f7d55cd91..450015edae7c2 100644 --- a/distribution/kubernetes/vector-aggregator/statefulset.yaml +++ b/distribution/kubernetes/vector-aggregator/statefulset.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Aggregator - app.kubernetes.io/version: "0.54.0-distroless-libc" + app.kubernetes.io/version: "0.56.0-distroless-libc" annotations: {} spec: replicas: 1 @@ -34,7 +34,7 @@ spec: dnsPolicy: ClusterFirst containers: - name: vector - image: "docker.io/timberio/vector:0.54.0-distroless-libc" + image: "docker.io/timberio/vector:0.56.0-distroless-libc" imagePullPolicy: IfNotPresent args: - --config-dir @@ -67,6 +67,10 @@ spec: - name: prom-exporter containerPort: 9090 protocol: TCP + readinessProbe: + httpGet: + path: /health + port: 8686 volumeMounts: - name: data mountPath: "/vector-data-dir" diff --git a/distribution/kubernetes/vector-stateless-aggregator/README.md b/distribution/kubernetes/vector-stateless-aggregator/README.md index d03fb7eb5250b..614f6426226e4 100644 --- a/distribution/kubernetes/vector-stateless-aggregator/README.md +++ b/distribution/kubernetes/vector-stateless-aggregator/README.md @@ -1,6 +1,6 @@ The kubernetes manifests found in this directory have been automatically generated from the [helm chart `vector/vector`](https://github.com/vectordotdev/helm-charts/tree/master/charts/vector) -version 0.51.0 with the following `values.yaml`: +version 0.56.0 with the following `values.yaml`: ```yaml role: Stateless-Aggregator diff --git a/distribution/kubernetes/vector-stateless-aggregator/configmap.yaml b/distribution/kubernetes/vector-stateless-aggregator/configmap.yaml index e13b42240886c..11e9ecc0d0b20 100644 --- a/distribution/kubernetes/vector-stateless-aggregator/configmap.yaml +++ b/distribution/kubernetes/vector-stateless-aggregator/configmap.yaml @@ -9,14 +9,13 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Stateless-Aggregator - app.kubernetes.io/version: "0.54.0-distroless-libc" + app.kubernetes.io/version: "0.56.0-distroless-libc" data: aggregator.yaml: | data_dir: /vector-data-dir api: enabled: true - address: 127.0.0.1:8686 - playground: false + address: 0.0.0.0:8686 sources: datadog_agent: address: 0.0.0.0:8282 diff --git a/distribution/kubernetes/vector-stateless-aggregator/deployment.yaml b/distribution/kubernetes/vector-stateless-aggregator/deployment.yaml index 60e585952669b..345a6458281ab 100644 --- a/distribution/kubernetes/vector-stateless-aggregator/deployment.yaml +++ b/distribution/kubernetes/vector-stateless-aggregator/deployment.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Stateless-Aggregator - app.kubernetes.io/version: "0.54.0-distroless-libc" + app.kubernetes.io/version: "0.56.0-distroless-libc" annotations: {} spec: replicas: 1 @@ -32,7 +32,7 @@ spec: dnsPolicy: ClusterFirst containers: - name: vector - image: "docker.io/timberio/vector:0.54.0-distroless-libc" + image: "docker.io/timberio/vector:0.56.0-distroless-libc" imagePullPolicy: IfNotPresent args: - --config-dir @@ -65,6 +65,10 @@ spec: - name: prom-exporter containerPort: 9090 protocol: TCP + readinessProbe: + httpGet: + path: /health + port: 8686 volumeMounts: - name: data mountPath: "/vector-data-dir" diff --git a/distribution/kubernetes/vector-stateless-aggregator/service-headless.yaml b/distribution/kubernetes/vector-stateless-aggregator/service-headless.yaml index bb2c35189c395..bf51f36a508d7 100644 --- a/distribution/kubernetes/vector-stateless-aggregator/service-headless.yaml +++ b/distribution/kubernetes/vector-stateless-aggregator/service-headless.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Stateless-Aggregator - app.kubernetes.io/version: "0.54.0-distroless-libc" + app.kubernetes.io/version: "0.56.0-distroless-libc" annotations: spec: clusterIP: None diff --git a/distribution/kubernetes/vector-stateless-aggregator/service.yaml b/distribution/kubernetes/vector-stateless-aggregator/service.yaml index b96e58ef1ea31..f19a6123ab9b8 100644 --- a/distribution/kubernetes/vector-stateless-aggregator/service.yaml +++ b/distribution/kubernetes/vector-stateless-aggregator/service.yaml @@ -9,7 +9,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Stateless-Aggregator - app.kubernetes.io/version: "0.54.0-distroless-libc" + app.kubernetes.io/version: "0.56.0-distroless-libc" annotations: spec: ports: diff --git a/distribution/kubernetes/vector-stateless-aggregator/serviceaccount.yaml b/distribution/kubernetes/vector-stateless-aggregator/serviceaccount.yaml index 2636f02bed059..8c5dc213f27f0 100644 --- a/distribution/kubernetes/vector-stateless-aggregator/serviceaccount.yaml +++ b/distribution/kubernetes/vector-stateless-aggregator/serviceaccount.yaml @@ -9,5 +9,5 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Stateless-Aggregator - app.kubernetes.io/version: "0.54.0-distroless-libc" + app.kubernetes.io/version: "0.56.0-distroless-libc" automountServiceAccountToken: true diff --git a/distribution/rpm/vector.spec b/distribution/rpm/vector.spec index 0aaee7e7edbe8..a09809e221493 100644 --- a/distribution/rpm/vector.spec +++ b/distribution/rpm/vector.spec @@ -56,7 +56,8 @@ mkdir -p %{buildroot}%{_datadir}/%{_name} mkdir -p %{buildroot}%{_unitdir} cp -a %{_builddir}/bin/vector %{buildroot}%{_bindir} -cp -a %{_builddir}/config/vector.yaml %{buildroot}%{_sysconfdir}/%{_name}/vector.yaml +mkdir -p %{buildroot}%{_datadir}/%{_name}/examples +cp -a %{_builddir}/config/vector.yaml %{buildroot}%{_datadir}/%{_name}/examples/vector.yaml cp -a %{_builddir}/config/examples/. %{buildroot}%{_sysconfdir}/%{_name}/examples cp -a %{_builddir}/systemd/vector.service %{buildroot}%{_unitdir}/vector.service cp -a %{_builddir}/systemd/vector.default %{buildroot}%{_sysconfdir}/default/vector @@ -79,12 +80,16 @@ rm -rf %{buildroot} %defattr(-,root,root,-) %{_bindir}/* %{_unitdir}/vector.service -%config(noreplace) %{_sysconfdir}/%{_name}/vector.yaml %config(noreplace) %{_sysconfdir}/default/vector +# Older versions installed a demo config at this path; mark it as %ghost so +# rpm preserves any existing on-disk file during upgrade instead of removing +# it as orphaned. +%ghost %config(noreplace) %{_sysconfdir}/%{_name}/vector.yaml %config %{_sysconfdir}/%{_name}/examples/* %dir %{_sharedstatedir}/%{_name} %doc README.md %doc %{_datadir}/%{_name}/NOTICE +%doc %{_datadir}/%{_name}/examples/vector.yaml %doc %{_datadir}/%{_name}/licenses/* %doc %{_datadir}/%{_name}/LICENSE-3rdparty.csv %license LICENSE diff --git a/docs/DEPRECATIONS.md b/docs/DEPRECATIONS.md index cf3a9377bfe24..5d0e823165b1b 100644 --- a/docs/DEPRECATIONS.md +++ b/docs/DEPRECATIONS.md @@ -17,7 +17,11 @@ For example: - `v0.50.0` | `http-server-encoding` | The `encoding` field will be removed. Use `decoding` and `framing` instead. - `v0.53.0` | `buffer-bytes-events-metrics` | The `buffer_byte_size` and `buffer_events` gauges are deprecated in favor of the `buffer_size_bytes`/`buffer_size_events` metrics described in `docs/specs/buffer.md`. - `v0.58.0` | `azure-monitor-logs-sink` | The `azure_monitor_logs` sink is deprecated in favor of the new `azure_logs_ingestion` sink, which uses the Azure Monitor Logs Ingestion API. Users should migrate before Microsoft ends support for the old Data Collector API (scheduled for September 2026). +- `v0.57.0` | `bool-or-vector-compression` | The boolean syntax for the `compression` field in the `vector` sink (`compression: true`/`compression: false`) is deprecated. Use the string syntax instead (`compression: "gzip"`, `compression: "zstd"`, or `compression: "none"`). The `bool_or_vector_compression` deserializer should be removed once the boolean syntax is no longer supported. +- `v0.58.0` | `datadog-metrics-series-v1` | The `series_api_version: v1` option on the `datadog_metrics` sink is deprecated in favor of `v2` (the default). The v1 series endpoint (`/api/v1/series`) is a legacy endpoint; users should remove `series_api_version: v1` from their configuration or set it to `v2`. ## To be migrated ## To be removed + +- `v0.56.0` | `greptimedb-v0-support` | The `greptimedb_metrics` and `greptimedb_logs` sinks drop support for GreptimeDB v0.x. Users must upgrade their GreptimeDB instance to v1.x before upgrading Vector. diff --git a/docs/DEVELOPING.md b/docs/DEVELOPING.md index d286c9feca343..70aca379ec774 100644 --- a/docs/DEVELOPING.md +++ b/docs/DEVELOPING.md @@ -1,8 +1,6 @@ # Developing - [Setup](#setup) - - [Using a Docker or Podman environment](#using-a-docker-or-podman-environment) - - [Bring your own toolbox](#bring-your-own-toolbox) - [The basics](#the-basics) - [Directory structure](#directory-structure) - [Makefile](#makefile) @@ -41,95 +39,31 @@ ## Setup -We're super excited to have you interested in working on Vector! Before you start you should pick how you want to develop. - -For small or first-time contributions, we recommend the Docker method. Prefer to do it yourself? That's fine too! - -### Using a Docker or Podman environment - -> **Targets:** You can use this method to produce AARCH64, Arm6/7, as well as x86/64 Linux builds. - -Since not everyone has a full working native environment, we took our environment and stuffed it into a Docker (or Podman) container! - -This is ideal for users who want it to "Just work" and just want to start contributing. It's also what we use for our CI, so you know if it breaks we can't do anything else until we fix it. πŸ˜‰ - -**Before you go further, install Docker or Podman through your official package manager, or from the [Docker](https://docs.docker.com/get-docker/) or [Podman](https://podman.io/) sites.** - -```bash -# Optional: Only if you use `podman` -export CONTAINER_TOOL="podman" -``` - -If your Linux environment runs SELinux in Enforcing mode, you will need to relabel the vector source code checkout with `container_home_t` context. Otherwise, the container environment cannot read/write the code: - -```bash -cd your/checkout/of/vector/ -sudo semanage fcontext -a "${PWD}(/.*)?" -t container_file_t -sudo restorecon . -R -``` - -By default, `make environment` style tasks will do a `docker pull` from GitHub's container repository, you can **optionally** build your own environment while you make your morning coffee β˜•: - -```bash -# Optional: Only if you want to go make a coffee -make environment-prepare -``` - -Now that you have your coffee, you can enter the shell! - -```bash -# Enter a shell with optimized mounts for interactive processes. -# Inside here, you can use Vector like you have full toolchain (See below!) -make environment -# Try out a specific container tool. (Docker/Podman) -make environment CONTAINER_TOOL="podman" -# Add extra cli opts -make environment CLI_OPTS="--publish 3000:2000" -``` - -Now you can use the jobs detailed in **"Bring your own toolbox"** below. - -Want to run from outside of the environment? _Clever. Good thinking._ You can run any of the following: - -```bash -# Validate your code can compile -make check ENVIRONMENT=true -# Validate your code actually does compile (in dev mode) -make build-dev ENVIRONMENT=true -# Validate your test pass -make test SCOPE="sources::example" ENVIRONMENT=true -# Validate tests (that do not require other services) pass -make test ENVIRONMENT=true -# Validate your tests pass (starting required services in Docker) -make test-integration SCOPE="sources::example" ENVIRONMENT=true -# Validate your tests pass against a live service. -make test-integration SCOPE="sources::example" AUTOSPAWN=false ENVIRONMENT=true -# Validate all tests pass (starting required services in Docker) -make test-integration ENVIRONMENT=true -# Run your benchmarks -make bench SCOPE="transforms::example" ENVIRONMENT=true -# Format your code before pushing! -make fmt ENVIRONMENT=true -``` - -We use explicit environment opt-in as many contributors choose to keep their Rust toolchain local. - -### Bring your own toolbox - -> **Targets:** This option is required for MSVC/Mac/FreeBSD toolchains. It can be used to build for any environment or OS. +We're super excited to have you interested in working on Vector! To build Vector on your own host will require a fairly complete development environment! Loosely, you'll need the following: - **To build Vector:** Have working Rustup, Protobuf tools, C++/C build tools (LLVM, GCC, or MSVC), Python, and Perl, `make` (the GNU one preferably), `bash`, `cmake`, `GNU coreutils`, and `autotools`. + - The `default` feature does not enable Kerberos/GSSAPI SASL for kafka, so local dev builds (`cargo build`, `make build`) have no extra system prerequisites beyond the C build tools above. + - To enable GSSAPI for kafka, choose one of: + - `gssapi`: dynamically links against system `libsasl2`. Requires `libsasl2-dev` (Ubuntu: `sudo apt-get install -y libsasl2-dev`) or `cyrus-sasl` (macOS: `brew install cyrus-sasl`) installed. + - `gssapi-vendored` (or `vendored`, which includes it): builds `libsasl2` and `krb5` from bundled source. No system install needed. Linux only; the bundled path does not currently link on macOS arm64. + - **To run `make test`:** Install [`cargo-nextest`](https://nexte.st/) - **To run integration tests:** Have `docker` available, or a real live version of that service. (Use `AUTOSPAWN=false`) - **To run `make check-component-features`:** Have `remarshal` installed. - **To run `make check-licenses` or `make build-licenses`:** Have `dd-rust-license-tool` [installed](https://github.com/DataDog/rust-license-tool). - **To run `make generate-docs`:** Have `cue` [installed](https://cuelang.org/docs/install/). -If you find yourself needing to run something inside the Docker environment described above, that's totally fine, they won't collide or hurt each other. In this case, you'd just run `make environment-generate`. +**Tooling shortcut:** Once the system-level dependencies above are in place, you can install the Rust and npm tooling (`cargo-nextest`, `cargo-deny`, `dd-rust-license-tool`, `vdev`, `markdownlint-cli2`, `prettier`, and others) in one shot: + +```bash +bash scripts/environment/prepare.sh +``` + +This is the same script CI uses to provision its toolchain, so what you get locally matches what CI installs. We're interested in reducing our dependencies if simple options exist. Got an idea? Try it out, we'd love to hear of your successes and failures! @@ -141,7 +75,6 @@ cargo check make check # Validate your code actually does compile (in dev mode) cargo build -make build-dev # Validate your test pass cargo test sources::example make test SCOPE="sources::example" @@ -454,8 +387,8 @@ times: We use `flog` to build a sample set of log files to test sending logs from a file. This can be done with the following commands on Mac with `homebrew`. -Installation instruction for flog can be found -[here](https://github.com/mingrammer/flog#installation). +Installation instruction for flog can be found in the +[flog README](https://github.com/mingrammer/flog#installation). ```bash flog --bytes $((100 * 1024 * 1024)) > sample.log diff --git a/docs/DOCUMENTING.md b/docs/DOCUMENTING.md index e6a032fee8dcf..3faa1f0011ba9 100644 --- a/docs/DOCUMENTING.md +++ b/docs/DOCUMENTING.md @@ -147,9 +147,12 @@ watchexec "make check-docs" ### Changelog -Contributors do not need to maintain a changelog. This is automatically generated -via the `make release` command, made possible by the use of -[conventional commit](https://www.conventionalcommits.org/en/v1.0.0/) titles. +Contributors document user-facing changes by adding a fragment under +[`changelog.d/`](../changelog.d/README.md); the release CUE file's user-facing +changelog section is assembled from those fragments by `cargo vdev release +prepare` during release prep. The CUE file's `commits:` array is populated +from the git log via [conventional commit](https://www.conventionalcommits.org/en/v1.0.0/) +titles, so PR titles must follow that spec. ### Release highlights diff --git a/docs/README.md b/docs/README.md index cb3ed925a1158..443c706e38faa 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,8 +1,8 @@ # Vector Internal Documentation -**_This folder contains internal documentation for Vector contributors. If +_**This folder contains internal documentation for Vector contributors. If you're a Vector user, please visit , which is powered by -the [website](../website) directory._** +the [website](../website) directory.**_ ## Getting started diff --git a/docs/RUST_STYLE.md b/docs/RUST_STYLE.md index 5ee34496af7b8..5d51af6855704 100644 --- a/docs/RUST_STYLE.md +++ b/docs/RUST_STYLE.md @@ -90,6 +90,8 @@ Code in Vector should **NOT** panic under normal circumstances. ## Feature Flags +### Component Feature Flags + New components (sources, sinks, transforms) must be behind feature flags: ```bash @@ -98,3 +100,26 @@ cargo test --lib --no-default-features --features sinks-console sinks::console ``` See `features` section in `Cargo.toml` for examples. + +### Cargo Dependency Feature Placement + +[Cargo features are additive](https://doc.rust-lang.org/cargo/reference/features.html#feature-unification): once any crate in the dependency graph enables a feature, it is enabled for the entire build. This can be surprising β€” enabling a feature in one crate silently turns it on everywhere. + +Always set `default-features = false` in `[workspace.dependencies]`. For feature declarations, the rule is simple: + +- If only one crate needs a feature, declare it in that crate's own `Cargo.toml`. +- If multiple crates need it, declare it in `[workspace.dependencies]`. + +Add a short comment near the dependency when the feature setup is non-obvious. + +**Auditing:** when unsure whether a feature is truly needed or only transitively enabled, verify with: + +```bash +# Show the full feature tree for the workspace +cargo tree -e features + +# Narrow down to a specific dependency +cargo tree -e features -i +``` + +The `-i` flag shows which crates depend on `` and which features they activate, useful for tracing where a feature is coming from. If a feature only appears because another crate enables it, and your crate relies on that feature being present, declare it explicitly β€” otherwise your crate silently breaks if the other crate ever stops enabling it. diff --git a/docs/generated/find_enrichment_table_records.json b/docs/generated/find_enrichment_table_records.json index 105c16d7ca3bd..52840357ad746 100644 --- a/docs/generated/find_enrichment_table_records.json +++ b/docs/generated/find_enrichment_table_records.json @@ -2,7 +2,7 @@ "anchor": "find_enrichment_table_records", "name": "find_enrichment_table_records", "category": "Enrichment", - "description": "Searches an [enrichment table](/docs/reference/glossary/#enrichment-tables) for rows that match the provided condition.\n\nFor `file` enrichment tables, this condition needs to be a VRL object in which\nthe key-value pairs indicate a field to search mapped to a value to search in that field.\nThis function returns the rows that match the provided condition(s). _All_ fields need to\nmatch for rows to be returned; if any fields do not match, then no rows are returned.\n\nThere are three forms of search criteria:\n\n1. **Exact match search**. The given field must match the value exactly. Case sensitivity\n can be specified using the `case_sensitive` argument. An exact match search can use an\n index directly into the dataset, which should make this search fairly \"cheap\" from a\n performance perspective.\n\n2. **Wildcard match search**. The given fields specified by the exact match search may also\n be matched exactly to the value provided to the `wildcard` parameter.\n A wildcard match search can also use an index directly into the dataset.\n\n3. **Date range search**. The given field must be greater than or equal to the `from` date\n and/or less than or equal to the `to` date. A date range search involves\n sequentially scanning through the rows that have been located using any exact match\n criteria. This can be an expensive operation if there are many rows returned by any exact\n match criteria. Therefore, use date ranges as the _only_ criteria when the enrichment\n data set is very small.\n\nFor `geoip` and `mmdb` enrichment tables, this condition needs to be a VRL object with a single key-value pair\nwhose value needs to be a valid IP address. Example: `{\"ip\": .ip }`. If a return field is expected\nand without a value, `null` is used. This table can return the following fields:\n\n* ISP databases:\n * `autonomous_system_number`\n * `autonomous_system_organization`\n * `isp`\n * `organization`\n\n* City databases:\n * `city_name`\n * `continent_code`\n * `country_code`\n * `country_name`\n * `region_code`\n * `region_name`\n * `metro_code`\n * `latitude`\n * `longitude`\n * `postal_code`\n * `timezone`\n\n* Connection-Type databases:\n * `connection_type`\n\nTo use this function, you need to update your configuration to\ninclude an\n[`enrichment_tables`](/docs/reference/configuration/global-options/#enrichment_tables)\nparameter.", + "description": "Searches an [enrichment table](/docs/reference/glossary/#enrichment-tables) for rows that match the provided condition.\n\nFor `file` enrichment tables, this condition needs to be a VRL object in which\nthe key-value pairs indicate a field to search mapped to a value to search in that field.\nThis function returns the rows that match the provided condition(s). _All_ fields need to\nmatch for rows to be returned; if any fields do not match, then no rows are returned.\n\nThere are three forms of search criteria:\n\n1. **Exact match search**. The given field must match the value exactly. Case sensitivity\n can be specified using the `case_sensitive` argument. An exact match search can use an\n index directly into the dataset, which should make this search fairly \"cheap\" from a\n performance perspective.\n\n2. **Wildcard match search**. The given fields specified by the exact match search may also\n be matched exactly to the value provided to the `wildcard` parameter.\n A wildcard match search can also use an index directly into the dataset.\n\n3. **Date range search**. The given field must be greater than or equal to the `from` date\n and/or less than or equal to the `to` date. A date range search involves\n sequentially scanning through the rows that have been located using any exact match\n criteria. This can be an expensive operation if there are many rows returned by any exact\n match criteria. Therefore, use date ranges as the _only_ criteria when the enrichment\n data set is very small.\n\nFor `geoip` and `mmdb` enrichment tables, this condition needs to be a VRL object with a single key-value pair\nwhose value needs to be a valid IP address. Example: `{\"ip\": .ip }`. If a return field is expected\nand without a value, `null` is used. This table can return the following fields:\n\n* ISP databases:\n * `autonomous_system_number`\n * `autonomous_system_organization`\n * `isp`\n * `organization`\n * `network`\n\n* City databases:\n * `city_name`\n * `continent_code`\n * `country_code`\n * `country_name`\n * `region_code`\n * `region_name`\n * `metro_code`\n * `latitude`\n * `longitude`\n * `postal_code`\n * `timezone`\n * `network`\n\n* Connection-Type databases:\n * `connection_type`\n * `network`\n\n* Anonymous-IP databases:\n * `is_anonymous`\n * `is_anonymous_vpn`\n * `is_hosting_provider`\n * `is_public_proxy`\n * `is_residential_proxy`\n * `is_tor_exit_node`\n * `network`\n\nTo use this function, you need to update your configuration to\ninclude an\n[`enrichment_tables`](/docs/reference/configuration/global-options/#enrichment_tables)\nparameter.", "arguments": [ { "name": "table", diff --git a/docs/generated/get_enrichment_table_record.json b/docs/generated/get_enrichment_table_record.json index 947e8b3e65374..45a49c7606110 100644 --- a/docs/generated/get_enrichment_table_record.json +++ b/docs/generated/get_enrichment_table_record.json @@ -2,7 +2,7 @@ "anchor": "get_enrichment_table_record", "name": "get_enrichment_table_record", "category": "Enrichment", - "description": "Searches an [enrichment table](/docs/reference/glossary/#enrichment-tables) for a row that matches the provided condition. A single row must be matched. If no rows are found or more than one row is found, an error is returned.\n\nFor `file` enrichment tables, this condition needs to be a VRL object in which\nthe key-value pairs indicate a field to search mapped to a value to search in that field.\nThis function returns the rows that match the provided condition(s). _All_ fields need to\nmatch for rows to be returned; if any fields do not match, then no rows are returned.\n\nThere are three forms of search criteria:\n\n1. **Exact match search**. The given field must match the value exactly. Case sensitivity\n can be specified using the `case_sensitive` argument. An exact match search can use an\n index directly into the dataset, which should make this search fairly \"cheap\" from a\n performance perspective.\n\n2. **Wildcard match search**. The given fields specified by the exact match search may also\n be matched exactly to the value provided to the `wildcard` parameter.\n A wildcard match search can also use an index directly into the dataset.\n\n3. **Date range search**. The given field must be greater than or equal to the `from` date\n and/or less than or equal to the `to` date. A date range search involves\n sequentially scanning through the rows that have been located using any exact match\n criteria. This can be an expensive operation if there are many rows returned by any exact\n match criteria. Therefore, use date ranges as the _only_ criteria when the enrichment\n data set is very small.\n\nFor `geoip` and `mmdb` enrichment tables, this condition needs to be a VRL object with a single key-value pair\nwhose value needs to be a valid IP address. Example: `{\"ip\": .ip }`. If a return field is expected\nand without a value, `null` is used. This table can return the following fields:\n\n* ISP databases:\n * `autonomous_system_number`\n * `autonomous_system_organization`\n * `isp`\n * `organization`\n\n* City databases:\n * `city_name`\n * `continent_code`\n * `country_code`\n * `country_name`\n * `region_code`\n * `region_name`\n * `metro_code`\n * `latitude`\n * `longitude`\n * `postal_code`\n * `timezone`\n\n* Connection-Type databases:\n * `connection_type`\n\nTo use this function, you need to update your configuration to\ninclude an\n[`enrichment_tables`](/docs/reference/configuration/global-options/#enrichment_tables)\nparameter.", + "description": "Searches an [enrichment table](/docs/reference/glossary/#enrichment-tables) for a row that matches the provided condition. A single row must be matched. If no rows are found or more than one row is found, an error is returned.\n\nFor `file` enrichment tables, this condition needs to be a VRL object in which\nthe key-value pairs indicate a field to search mapped to a value to search in that field.\nThis function returns the rows that match the provided condition(s). _All_ fields need to\nmatch for rows to be returned; if any fields do not match, then no rows are returned.\n\nThere are three forms of search criteria:\n\n1. **Exact match search**. The given field must match the value exactly. Case sensitivity\n can be specified using the `case_sensitive` argument. An exact match search can use an\n index directly into the dataset, which should make this search fairly \"cheap\" from a\n performance perspective.\n\n2. **Wildcard match search**. The given fields specified by the exact match search may also\n be matched exactly to the value provided to the `wildcard` parameter.\n A wildcard match search can also use an index directly into the dataset.\n\n3. **Date range search**. The given field must be greater than or equal to the `from` date\n and/or less than or equal to the `to` date. A date range search involves\n sequentially scanning through the rows that have been located using any exact match\n criteria. This can be an expensive operation if there are many rows returned by any exact\n match criteria. Therefore, use date ranges as the _only_ criteria when the enrichment\n data set is very small.\n\nFor `geoip` and `mmdb` enrichment tables, this condition needs to be a VRL object with a single key-value pair\nwhose value needs to be a valid IP address. Example: `{\"ip\": .ip }`. If a return field is expected\nand without a value, `null` is used. This table can return the following fields:\n\n* ISP databases:\n * `autonomous_system_number`\n * `autonomous_system_organization`\n * `isp`\n * `organization`\n * `network`\n\n* City databases:\n * `city_name`\n * `continent_code`\n * `country_code`\n * `country_name`\n * `region_code`\n * `region_name`\n * `metro_code`\n * `latitude`\n * `longitude`\n * `postal_code`\n * `timezone`\n * `network`\n\n* Connection-Type databases:\n * `connection_type`\n * `network`\n\n* Anonymous-IP databases:\n * `is_anonymous`\n * `is_anonymous_vpn`\n * `is_hosting_provider`\n * `is_public_proxy`\n * `is_residential_proxy`\n * `is_tor_exit_node`\n * `network`\n\nTo use this function, you need to update your configuration to\ninclude an\n[`enrichment_tables`](/docs/reference/configuration/global-options/#enrichment_tables)\nparameter.", "arguments": [ { "name": "table", diff --git a/docs/generated/parse_dnstap.json b/docs/generated/parse_dnstap.json index 6bc7356fe5201..c30eb26893e02 100644 --- a/docs/generated/parse_dnstap.json +++ b/docs/generated/parse_dnstap.json @@ -41,6 +41,8 @@ "extraInfo": "", "messageType": "ResolverQuery", "messageTypeId": 3, + "requestMessageSize": 54, + "responseMessageSize": 100, "queryZone": "com.", "requestData": { "fullRcode": 0, diff --git a/docs/tutorials/sinks/1_basic_sink.md b/docs/tutorials/sinks/1_basic_sink.md index a099a5530602d..c823c6a26530f 100644 --- a/docs/tutorials/sinks/1_basic_sink.md +++ b/docs/tutorials/sinks/1_basic_sink.md @@ -2,7 +2,7 @@ Let's write a basic sink for Vector. Currently, there are two styles of sink in Vector - 'event' and 'event streams'. The 'event' style sinks are deprecated, but currently a significant portion of Vector's sinks are still developed in this style. A tracking issue that covers which sinks have been converted to -'event streams' can be found [here][event_streams_tracking]. +'event streams' can be found in the [event streams tracking issue][event_streams_tracking]. This tutorial covers writing an 'event stream' Sink. @@ -300,8 +300,8 @@ Change the body of `run_inner` to look like the following: } ``` -More details about instrumenting Vector can be found -[here](https://github.com/vectordotdev/vector/blob/master/docs/specs/instrumentation.md). +More details about instrumenting Vector can be found in the +[instrumentation specification](https://github.com/vectordotdev/vector/blob/master/docs/specs/instrumentation.md). # Running our sink @@ -324,7 +324,7 @@ This simply connects a `stdin` source to our `basic` sink. ## vdev Vector provides a build tool `vdev` that simplifies the task of building Vector. Install -`vdev` using the instructions [here][vdev_install]. +`vdev` using the [installation instructions][vdev_install]. With `vdev` installed we can run Vector using: diff --git a/lib/codecs/Cargo.toml b/lib/codecs/Cargo.toml index 0ed26d0b16a41..6a5f8424a61fd 100644 --- a/lib/codecs/Cargo.toml +++ b/lib/codecs/Cargo.toml @@ -15,9 +15,17 @@ path = "tests/bin/generate-avro-fixtures.rs" [dependencies] apache-avro = { version = "0.20.0", default-features = false } arrow = { version = "56.2.0", default-features = false, features = ["ipc", "json"], optional = true } +parquet = { version = "56.2.0", default-features = false, features = [ + "arrow", + "snap", + "zstd", + "lz4", + "flate2-rust_backened", +], optional = true } async-trait.workspace = true bytes.workspace = true chrono.workspace = true +chrono-tz.workspace = true rust_decimal.workspace = true csv-core = { version = "0.1.13", default-features = false } derivative.workspace = true @@ -37,12 +45,12 @@ prost-reflect.workspace = true rand.workspace = true regex.workspace = true serde.workspace = true -serde_with = { version = "3.14.0", default-features = false, features = ["std", "macros", "chrono_0_4"] } +serde_with.workspace = true serde_json.workspace = true serde-aux = { version = "4.5", optional = true } smallvec = { version = "1", default-features = false, features = ["union"] } snafu.workspace = true -strum = { version = "0.26.3", features = ["derive"], optional = true } +strum = { workspace = true, optional = true } syslog_loose = { version = "0.23", default-features = false, optional = true } tokio-util = { version = "0.7", default-features = false, features = ["codec"] } tokio = { workspace = true, features = ["full"] } @@ -69,7 +77,8 @@ uuid.workspace = true vrl.workspace = true [features] -arrow = ["dep:arrow"] +arrow = ["dep:arrow", "arrow/chrono-tz"] +parquet = ["dep:parquet", "arrow"] opentelemetry = ["dep:opentelemetry-proto"] syslog = ["dep:syslog_loose", "dep:strum", "dep:derive_more", "dep:serde-aux", "dep:toml"] test = [] diff --git a/lib/codecs/src/decoding/decoder.rs b/lib/codecs/src/decoding/decoder.rs index 85a0fce42148e..0ba05c32283e3 100644 --- a/lib/codecs/src/decoding/decoder.rs +++ b/lib/codecs/src/decoding/decoder.rs @@ -1,7 +1,10 @@ use bytes::{Bytes, BytesMut}; use smallvec::SmallVec; use vector_common::internal_event::emit; -use vector_core::{config::LogNamespace, event::Event}; +use vector_core::{ + config::LogNamespace, + event::{Event, EventMetadata}, +}; use crate::{ decoding::format::Deserializer as _, @@ -53,6 +56,13 @@ impl Decoder { self } + /// Attaches a per-decode-call metadata template to the inner deserializer, + /// allowing deserializers to read from and write to event metadata. + pub fn with_metadata_template(mut self, metadata: EventMetadata) -> Self { + self.deserializer = self.deserializer.with_metadata_template(metadata); + self + } + /// Handles the framing result and parses it into a structured event, if /// possible. /// diff --git a/lib/codecs/src/decoding/format/avro.rs b/lib/codecs/src/decoding/format/avro.rs index c21c09de21cfb..327c5d17acd78 100644 --- a/lib/codecs/src/decoding/format/avro.rs +++ b/lib/codecs/src/decoding/format/avro.rs @@ -103,8 +103,7 @@ pub struct AvroDeserializerOptions { ))] pub schema: String, - /// For Avro datum encoded in Kafka messages, the bytes are prefixed with the schema ID. Set this to `true` to strip the schema ID prefix. - /// According to [Confluent Kafka's document](https://docs.confluent.io/platform/current/schema-registry/fundamentals/serdes-develop/index.html#wire-format). + /// For Avro datum encoded in Kafka messages, the bytes are prefixed with the schema ID. Set this to `true` to strip the schema ID prefix, as described in [Confluent Kafka's documentation](https://docs.confluent.io/platform/current/schema-registry/fundamentals/serdes-develop/index.html#wire-format). pub strip_schema_id_prefix: bool, } diff --git a/lib/codecs/src/decoding/format/gelf.rs b/lib/codecs/src/decoding/format/gelf.rs index 377d437104cc4..021c6ef4df5ce 100644 --- a/lib/codecs/src/decoding/format/gelf.rs +++ b/lib/codecs/src/decoding/format/gelf.rs @@ -47,7 +47,7 @@ pub enum ValidationMode { /// Uses more relaxed validation that skips strict GELF specification checks. /// - /// This mode will not treat specification violations as errors, allowing the decoder + /// This mode does not treat specification violations as errors, allowing the decoder /// to accept messages from sources that don't strictly follow the GELF spec. Relaxed, } diff --git a/lib/codecs/src/decoding/format/mod.rs b/lib/codecs/src/decoding/format/mod.rs index 701d11c8403d5..cbb2172c3392d 100644 --- a/lib/codecs/src/decoding/format/mod.rs +++ b/lib/codecs/src/decoding/format/mod.rs @@ -50,6 +50,8 @@ pub trait Deserializer: DynClone + Send + Sync { /// by not requiring heap allocations for it. /// /// **Note**: The type of the produced events depends on the implementation. + /// + /// TODO: fn parse( &self, bytes: Bytes, diff --git a/lib/codecs/src/decoding/format/otlp.rs b/lib/codecs/src/decoding/format/otlp.rs index 2004df92269ca..3e81dec089dce 100644 --- a/lib/codecs/src/decoding/format/otlp.rs +++ b/lib/codecs/src/decoding/format/otlp.rs @@ -34,11 +34,11 @@ pub enum OtlpSignalType { pub struct OtlpDeserializerConfig { /// Signal types to attempt parsing, in priority order. /// - /// The deserializer will try parsing in the order specified. This allows you to optimize + /// The deserializer tries to parse signals in the specified order. This allows you to optimize /// performance when you know the expected signal types. For example, if you only receive /// traces, set this to `["traces"]` to avoid attempting to parse as logs or metrics first. /// - /// If not specified, defaults to trying all types in order: logs, metrics, traces. + /// If not specified, defaults to trying all types in this order: logs, metrics, traces. /// Duplicate signal types are automatically removed while preserving order. #[serde(default = "default_signal_types")] pub signal_types: IndexSet, @@ -181,6 +181,7 @@ impl Deserializer for OtlpDeserializer { } } OtlpSignalType::Traces => { + // TODO: if let Ok(mut events) = self.traces_deserializer.parse(bytes.clone(), log_namespace) && let Some(Event::Log(log)) = events.first() diff --git a/lib/codecs/src/decoding/format/protobuf.rs b/lib/codecs/src/decoding/format/protobuf.rs index f5b923e4d1480..7ccc1acef66d7 100644 --- a/lib/codecs/src/decoding/format/protobuf.rs +++ b/lib/codecs/src/decoding/format/protobuf.rs @@ -75,7 +75,7 @@ pub struct ProtobufDeserializerOptions { /// /// This file is the output of `protoc -I -o `. /// - /// You can read more [here](https://buf.build/docs/reference/images/#how-buf-images-work). + /// For more information, see [How Buf images work](https://buf.build/docs/reference/images/#how-buf-images-work). pub desc_file: PathBuf, /// The name of the message type to use for serializing. @@ -85,7 +85,7 @@ pub struct ProtobufDeserializerOptions { /// Use JSON field names (camelCase) instead of protobuf field names (snake_case). /// /// When enabled, the deserializer will output fields using their JSON names as defined - /// in the `.proto` file (e.g., `jobDescription` instead of `job_description`). + /// in the `.proto` file (for example, `jobDescription` instead of `job_description`). /// /// This is useful when working with data that needs to be converted to JSON or /// when interfacing with systems that use JSON naming conventions. diff --git a/lib/codecs/src/decoding/format/vrl.rs b/lib/codecs/src/decoding/format/vrl.rs index c67f60fdd8952..df990a767c7e8 100644 --- a/lib/codecs/src/decoding/format/vrl.rs +++ b/lib/codecs/src/decoding/format/vrl.rs @@ -1,5 +1,4 @@ use bytes::Bytes; -use derivative::Derivative; use smallvec::{SmallVec, smallvec}; use vector_config_macros::configurable_component; use vector_core::{ @@ -14,7 +13,9 @@ use vrl::{ value::Kind, }; -use crate::{BytesDeserializerConfig, decoding::format::Deserializer}; +use vector_core::event::EventMetadata; + +use crate::decoding::format::Deserializer; /// Config used to build a `VrlDeserializer`. #[configurable_component] @@ -26,12 +27,11 @@ pub struct VrlDeserializerConfig { /// VRL-specific decoding options. #[configurable_component] -#[derive(Debug, Clone, PartialEq, Eq, Derivative)] -#[derivative(Default)] +#[derive(Debug, Clone, PartialEq, Eq, Default)] pub struct VrlDeserializerOptions { /// The [Vector Remap Language][vrl] (VRL) program to execute for each event. - /// Note that the final contents of the `.` target will be used as the decoding result. - /// Compilation error or use of 'abort' in a program will result in a decoding error. + /// The final contents of the `.` target are used as the decoding result. + /// Compilation errors or use of `abort` in the program result in a decoding error. /// /// /// [vrl]: https://vector.dev/docs/reference/vrl @@ -66,6 +66,7 @@ impl VrlDeserializerConfig { Ok(result) => Ok(VrlDeserializer { program: result.program, timezone: self.vrl.timezone.unwrap_or(TimeZone::Local), + metadata_template: None, }), Err(diagnostics) => Err(Formatter::new(&self.vrl.source, diagnostics) .to_string() @@ -96,9 +97,28 @@ impl VrlDeserializerConfig { pub struct VrlDeserializer { program: Program, timezone: TimeZone, + /// Per-call metadata template set by the source before decoding. When + /// present, every `%`-prefixed path in the template is accessible from + /// within the VRL program (e.g. `%splunk_hec.host`, `%vector.secrets.*`). + metadata_template: Option, +} + +impl VrlDeserializer { + /// Attach a metadata template that will be pre-populated on each synthetic + /// event before the VRL program runs. + /// + /// Sources call this once per decode call with the per-request context they + /// have assembled (envelope fields, auth tokens, etc.). VRL can then read + /// those values via `%`-prefixed paths. + #[must_use] + pub fn with_metadata_template(mut self, metadata: EventMetadata) -> Self { + self.metadata_template = Some(metadata); + self + } } fn parse_bytes(bytes: Bytes, log_namespace: LogNamespace) -> Event { + use crate::BytesDeserializerConfig; let bytes_deserializer = BytesDeserializerConfig::new().build(); let log_event = bytes_deserializer.parse_single(bytes, log_namespace); Event::from(log_event) @@ -110,11 +130,14 @@ impl Deserializer for VrlDeserializer { bytes: Bytes, log_namespace: LogNamespace, ) -> vector_common::Result> { - let event = parse_bytes(bytes, log_namespace); - match self.run_vrl(event, log_namespace) { - Ok(events) => Ok(events), - Err(e) => Err(e), + let mut event = parse_bytes(bytes, log_namespace); + if let Some(template) = &self.metadata_template { + // Pre-populate the synthetic event with the source-assembled metadata so + // every `%`-prefixed path is in scope when VRL executes. This lets + // user programs read `%splunk_hec.host`, `%vector.secrets.*`, etc. + *event.metadata_mut() = template.clone(); } + self.run_vrl(event, log_namespace) } } @@ -322,4 +345,55 @@ mod tests { .to_string(); assert!(error.contains("aborted")); } + + fn metadata_with_secret(key: &str, value: &str) -> EventMetadata { + let mut metadata = EventMetadata::default(); + metadata.secrets_mut().insert(key, value); + metadata + } + + /// A VRL program that uses `get_secret!()` can read a secret injected via + /// `with_metadata_template`. + #[test] + fn test_with_metadata_template_vrl_can_read_secret() { + // VRL program copies the injected secret into an event field so we can + // assert on its value. The input bytes become `.message` (Legacy namespace) + // and we add `.secret_value` alongside it. + let decoder = make_decoder(r#".secret_value = get_secret!("my_token")"#) + .with_metadata_template(metadata_with_secret("my_token", "super-secret")); + + let bytes = Bytes::from(r#"hello"#); + let events = decoder + .parse(bytes, LogNamespace::Legacy) + .expect("parse should succeed"); + + assert_eq!(events.len(), 1); + assert_eq!( + *events[0].as_log().get("secret_value").unwrap(), + Value::from("super-secret") + ); + } + + /// Secrets explicitly set by the VRL program win over the template because + /// `set_secret!` runs after the template is pre-populated. + #[test] + fn test_with_metadata_template_codec_wins_on_collision() { + let decoder = make_decoder(r#"set_secret!("my_token", "codec-wins")"#) + .with_metadata_template(metadata_with_secret("my_token", "template-loses")); + + let bytes = Bytes::from(r#"hello"#); + let events = decoder + .parse(bytes, LogNamespace::Legacy) + .expect("parse should succeed"); + + assert_eq!( + events[0] + .metadata() + .secrets() + .get("my_token") + .unwrap() + .as_ref(), + "codec-wins" + ); + } } diff --git a/lib/codecs/src/decoding/framing/character_delimited.rs b/lib/codecs/src/decoding/framing/character_delimited.rs index 09587893be205..819be2c77b655 100644 --- a/lib/codecs/src/decoding/framing/character_delimited.rs +++ b/lib/codecs/src/decoding/framing/character_delimited.rs @@ -47,13 +47,13 @@ pub struct CharacterDelimitedDecoderOptions { /// /// This length does *not* include the trailing delimiter. /// - /// By default, there is no maximum length enforced. If events are malformed, this can lead to + /// By default, no maximum length is enforced. If events are malformed, this can lead to /// additional resource usage as events continue to be buffered in memory, and can potentially /// lead to memory exhaustion in extreme cases. /// /// If there is a risk of processing malformed data, such as logs with user-controlled input, /// consider setting the maximum length to a reasonably large value as a safety net. This - /// ensures that processing is not actually unbounded. + /// prevents processing from being unbounded. #[serde(skip_serializing_if = "vector_core::serde::is_default")] pub max_length: Option, } diff --git a/lib/codecs/src/decoding/framing/chunked_gelf.rs b/lib/codecs/src/decoding/framing/chunked_gelf.rs index 9a82333c87960..aaf457b6c5b49 100644 --- a/lib/codecs/src/decoding/framing/chunked_gelf.rs +++ b/lib/codecs/src/decoding/framing/chunked_gelf.rs @@ -54,7 +54,7 @@ impl ChunkedGelfDecoderConfig { #[derivative(Default)] pub struct ChunkedGelfDecoderOptions { /// The timeout, in seconds, for a message to be fully received. If the timeout is reached, the - /// decoder drops all the received chunks of the timed out message. + /// decoder drops all received chunks for the timed-out message. #[serde(default = "default_timeout_secs")] #[derivative(Default(value = "default_timeout_secs()"))] pub timeout_secs: f64, @@ -66,15 +66,15 @@ pub struct ChunkedGelfDecoderOptions { #[serde(default, skip_serializing_if = "vector_core::serde::is_default")] pub pending_messages_limit: Option, - /// The maximum length of a single GELF message, in bytes. Messages longer than this length will - /// be dropped. If this option is not set, the decoder does not limit the length of messages and + /// The maximum length of a single GELF message, in bytes. Messages longer than this length are + /// dropped. If this option is not set, the decoder does not limit the length of messages and /// the per-message memory is unbounded. /// - /// **Note**: A message can be composed of multiple chunks and this limit is applied to the whole + /// **Note**: A message can be composed of multiple chunks, and this limit applies to the whole /// message, not to individual chunks. /// - /// This limit takes only into account the message's payload and the GELF header bytes are excluded from the calculation. - /// The message's payload is the concatenation of all the chunks' payloads. + /// This limit takes into account only the message payload. GELF header bytes are excluded from the calculation. + /// The message payload is the concatenation of all chunk payloads. #[serde(default, skip_serializing_if = "vector_core::serde::is_default")] pub max_length: Option, @@ -85,11 +85,10 @@ pub struct ChunkedGelfDecoderOptions { /// Decompression options for ChunkedGelfDecoder. #[configurable_component] -#[derive(Clone, Copy, Debug, PartialEq, Eq, Derivative)] -#[derivative(Default)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Default)] pub enum ChunkedGelfDecompressionConfig { /// Automatically detect the decompression method based on the magic bytes of the message. - #[derivative(Default)] + #[default] Auto, /// Use Gzip decompression. Gzip, diff --git a/lib/codecs/src/decoding/framing/length_delimited.rs b/lib/codecs/src/decoding/framing/length_delimited.rs index 07a958dce1c52..16c562f653f2d 100644 --- a/lib/codecs/src/decoding/framing/length_delimited.rs +++ b/lib/codecs/src/decoding/framing/length_delimited.rs @@ -1,5 +1,4 @@ use bytes::{Bytes, BytesMut}; -use derivative::Derivative; use tokio_util::codec::Decoder; use vector_config::configurable_component; @@ -8,8 +7,7 @@ use crate::common::length_delimited::LengthDelimitedCoderOptions; /// Config used to build a `LengthDelimitedDecoder`. #[configurable_component] -#[derive(Debug, Clone, Derivative)] -#[derivative(Default)] +#[derive(Debug, Clone, Default)] pub struct LengthDelimitedDecoderConfig { /// Options for the length delimited decoder. #[serde(skip_serializing_if = "vector_core::serde::is_default")] diff --git a/lib/codecs/src/decoding/framing/newline_delimited.rs b/lib/codecs/src/decoding/framing/newline_delimited.rs index 7bdc3a60881e5..454aa08376c01 100644 --- a/lib/codecs/src/decoding/framing/newline_delimited.rs +++ b/lib/codecs/src/decoding/framing/newline_delimited.rs @@ -1,5 +1,4 @@ use bytes::{Bytes, BytesMut}; -use derivative::Derivative; use tokio_util::codec::Decoder; use vector_config::configurable_component; @@ -16,20 +15,19 @@ pub struct NewlineDelimitedDecoderConfig { /// Options for building a `NewlineDelimitedDecoder`. #[configurable_component] -#[derive(Clone, Debug, Derivative, PartialEq, Eq)] -#[derivative(Default)] +#[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct NewlineDelimitedDecoderOptions { /// The maximum length of the byte buffer. /// /// This length does *not* include the trailing delimiter. /// - /// By default, there is no maximum length enforced. If events are malformed, this can lead to + /// By default, no maximum length is enforced. If events are malformed, this can lead to /// additional resource usage as events continue to be buffered in memory, and can potentially /// lead to memory exhaustion in extreme cases. /// /// If there is a risk of processing malformed data, such as logs with user-controlled input, /// consider setting the maximum length to a reasonably large value as a safety net. This - /// ensures that processing is not actually unbounded. + /// prevents processing from being unbounded. #[serde(skip_serializing_if = "vector_core::serde::is_default")] pub max_length: Option, } diff --git a/lib/codecs/src/decoding/framing/octet_counting.rs b/lib/codecs/src/decoding/framing/octet_counting.rs index 749d66b8fffc9..80f5a28ee4aa4 100644 --- a/lib/codecs/src/decoding/framing/octet_counting.rs +++ b/lib/codecs/src/decoding/framing/octet_counting.rs @@ -1,7 +1,6 @@ use std::io; use bytes::{Buf, Bytes, BytesMut}; -use derivative::Derivative; use tokio_util::codec::{LinesCodec, LinesCodecError}; use tracing::trace; use vector_config::configurable_component; @@ -30,8 +29,7 @@ impl OctetCountingDecoderConfig { /// Options for building a `OctetCountingDecoder`. #[configurable_component] -#[derive(Clone, Debug, Derivative, PartialEq, Eq)] -#[derivative(Default)] +#[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct OctetCountingDecoderOptions { /// The maximum length of the byte buffer. #[serde(skip_serializing_if = "vector_core::serde::is_default")] diff --git a/lib/codecs/src/decoding/framing/varint_length_delimited.rs b/lib/codecs/src/decoding/framing/varint_length_delimited.rs index 7b9d26f0a0596..a1b870fe1bb51 100644 --- a/lib/codecs/src/decoding/framing/varint_length_delimited.rs +++ b/lib/codecs/src/decoding/framing/varint_length_delimited.rs @@ -1,5 +1,4 @@ use bytes::{Buf, Bytes, BytesMut}; -use derivative::Derivative; use snafu::Snafu; use tokio_util::codec::Decoder; use vector_config::configurable_component; @@ -38,8 +37,7 @@ impl FramingError for VarintFramingError { /// Config used to build a `VarintLengthDelimitedDecoder`. #[configurable_component] -#[derive(Debug, Clone, Derivative)] -#[derivative(Default)] +#[derive(Debug, Clone, Default)] pub struct VarintLengthDelimitedDecoderConfig { /// Maximum frame length #[serde(default = "default_max_frame_length")] diff --git a/lib/codecs/src/decoding/mod.rs b/lib/codecs/src/decoding/mod.rs index c87337856454a..0ed298554fea4 100644 --- a/lib/codecs/src/decoding/mod.rs +++ b/lib/codecs/src/decoding/mod.rs @@ -37,7 +37,7 @@ use smallvec::SmallVec; use vector_config::configurable_component; use vector_core::{ config::{DataType, LogNamespace}, - event::Event, + event::{Event, EventMetadata}, schema, }; @@ -270,7 +270,7 @@ pub enum DeserializerConfig { /// Decodes the raw bytes as [native Protocol Buffers format][vector_native_protobuf]. /// - /// This decoder can output all types of events (logs, metrics, traces). + /// This decoder can output all types of events: logs, metrics, and traces. /// /// This codec is **[experimental][experimental]**. /// @@ -280,7 +280,7 @@ pub enum DeserializerConfig { /// Decodes the raw bytes as [native JSON format][vector_native_json]. /// - /// This decoder can output all types of events (logs, metrics, traces). + /// This decoder can output all types of events: logs, metrics, and traces. /// /// This codec is **[experimental][experimental]**. /// @@ -294,13 +294,13 @@ pub enum DeserializerConfig { /// /// The GELF specification is more strict than the actual Graylog receiver. /// Vector's decoder adheres more strictly to the GELF spec, with - /// the exception that some characters such as `@` are allowed in field names. + /// the exception that some characters such as `@` are allowed in field names. /// - /// Other GELF codecs such as Loki's, use a [Go SDK][implementation] that is maintained - /// by Graylog, and is much more relaxed than the GELF spec. + /// Other GELF codecs, such as Loki's, use a [Go SDK][implementation] that is maintained + /// by Graylog and is much more relaxed than the GELF spec. /// - /// Going forward, Vector will use that [Go SDK][implementation] as the reference implementation, which means - /// the codec may continue to relax the enforcement of specification. + /// Going forward, Vector will use the [Go SDK][implementation] as the reference implementation, which means + /// the codec may continue to relax the enforcement of the specification. /// /// [gelf]: https://docs.graylog.org/docs/gelf /// [implementation]: https://github.com/Graylog2/go-gelf/blob/v2/gelf/reader.go @@ -426,6 +426,12 @@ impl DeserializerConfig { } } + /// Returns `true` when this is a VRL deserializer. + /// Sources can use this to decide whether to call `Decoder::with_metadata_template`. + pub fn is_vrl(&self) -> bool { + matches!(self, DeserializerConfig::Vrl(_)) + } + /// Return the type of event build by this deserializer. pub fn output_type(&self) -> DataType { match self { @@ -542,6 +548,17 @@ pub enum Deserializer { Vrl(VrlDeserializer), } +impl Deserializer { + /// Attaches a metadata template to the inner deserializer, if it supports + /// one. + pub fn with_metadata_template(self, metadata: EventMetadata) -> Self { + match self { + Deserializer::Vrl(d) => Deserializer::Vrl(d.with_metadata_template(metadata)), + other => other, + } + } +} + impl format::Deserializer for Deserializer { fn parse( &self, diff --git a/lib/codecs/src/encoding/encoder.rs b/lib/codecs/src/encoding/encoder.rs index 4924dd05447b1..7aac89ed1e100 100644 --- a/lib/codecs/src/encoding/encoder.rs +++ b/lib/codecs/src/encoding/encoder.rs @@ -5,17 +5,38 @@ use vector_core::event::Event; #[cfg(feature = "arrow")] use crate::encoding::ArrowStreamSerializer; +#[cfg(feature = "parquet")] +use crate::encoding::ParquetSerializer; use crate::{ - encoding::{Error, Framer, Serializer}, + encoding::{Error, Framer, ProtoBatchSerializer, Serializer}, internal_events::{EncoderFramingError, EncoderSerializeError}, }; +/// The output of a batch encoding operation. +/// +/// Different batch serializers produce different output types: +/// - Arrow serializer produces a `RecordBatch` +/// - Proto serializer produces individual byte buffers per event +#[derive(Debug)] +pub enum BatchOutput { + /// An Arrow RecordBatch containing all events encoded as columnar data. + #[cfg(feature = "arrow")] + Arrow(arrow::record_batch::RecordBatch), + /// A list of individually-serialized records (one per event). + Records(Vec>), +} + /// Serializers that support batch encoding (encoding all events at once). #[derive(Debug, Clone)] pub enum BatchSerializer { /// Arrow IPC stream format serializer. #[cfg(feature = "arrow")] Arrow(ArrowStreamSerializer), + /// Parquet format serializer. + #[cfg(feature = "parquet")] + Parquet(Box), + /// Protobuf batch serializer that encodes each event individually. + ProtoBatch(ProtoBatchSerializer), } /// An encoder that encodes batches of events. @@ -36,10 +57,47 @@ impl BatchEncoder { } /// Get the HTTP content type. - #[cfg(feature = "arrow")] - pub const fn content_type(&self) -> &'static str { + /// + /// Returns `None` for serializers that do not produce a single HTTP body + /// (e.g. `ProtoBatch`, which emits one record per event for an out-of-band + /// transport rather than an HTTP payload). + #[cfg(any(feature = "arrow", feature = "parquet"))] + pub const fn content_type(&self) -> Option<&'static str> { match &self.serializer { - BatchSerializer::Arrow(_) => "application/vnd.apache.arrow.stream", + #[cfg(feature = "arrow")] + BatchSerializer::Arrow(_) => Some("application/vnd.apache.arrow.stream"), + #[cfg(feature = "parquet")] + BatchSerializer::Parquet(_) => Some("application/vnd.apache.parquet"), + BatchSerializer::ProtoBatch(_) => None, + } + } + + /// Encode a batch of events into a `BatchOutput`. + pub fn encode_batch(&self, events: &[Event]) -> Result { + match &self.serializer { + #[cfg(feature = "arrow")] + BatchSerializer::Arrow(serializer) => { + let record_batch = serializer.encode_to_record_batch(events).map_err(|err| { + use crate::encoding::ArrowEncodingError; + match err { + ArrowEncodingError::NullConstraint { .. } => { + Error::SchemaConstraintViolation(Box::new(err)) + } + _ => Error::SerializingError(Box::new(err)), + } + })?; + Ok(BatchOutput::Arrow(record_batch)) + } + BatchSerializer::ProtoBatch(serializer) => { + let records = serializer + .encode_batch(events) + .map_err(|err| Error::SerializingError(Box::new(err)))?; + Ok(BatchOutput::Records(records)) + } + #[cfg(feature = "parquet")] + BatchSerializer::Parquet(_) => Err(Error::SerializingError(Box::from( + "Parquet serializer does not support encode_batch; use the tokio Encoder interface instead", + ))), } } } @@ -49,7 +107,6 @@ impl tokio_util::codec::Encoder> for BatchEncoder { #[allow(unused_variables)] fn encode(&mut self, events: Vec, buffer: &mut BytesMut) -> Result<(), Self::Error> { - #[allow(unreachable_patterns)] match &mut self.serializer { #[cfg(feature = "arrow")] BatchSerializer::Arrow(serializer) => { @@ -63,18 +120,23 @@ impl tokio_util::codec::Encoder> for BatchEncoder { } }) } - _ => unreachable!("BatchSerializer cannot be constructed without encode()"), + #[cfg(feature = "parquet")] + BatchSerializer::Parquet(serializer) => serializer + .encode(events, buffer) + .map_err(Error::SerializingError), + BatchSerializer::ProtoBatch(_) => Err(Error::SerializingError(Box::from( + "ProtoBatch serializer does not support the tokio Encoder interface; use BatchEncoder::encode_batch() instead", + ))), } } } -/// An wrapper that supports both framed and batch encoding modes. +/// A wrapper that supports both framed and batch encoding modes. #[derive(Debug, Clone)] pub enum EncoderKind { /// Uses framing to encode individual events Framed(Box>), /// Encodes events in batches without framing - #[cfg(feature = "arrow")] Batch(BatchEncoder), } diff --git a/lib/codecs/src/encoding/format/arrow.rs b/lib/codecs/src/encoding/format/arrow.rs index 236d6292d375e..0c67317b7f926 100644 --- a/lib/codecs/src/encoding/format/arrow.rs +++ b/lib/codecs/src/encoding/format/arrow.rs @@ -6,6 +6,7 @@ use arrow::{ datatypes::{DataType, Field, Fields, Schema, SchemaRef}, + error::ArrowError, ipc::writer::StreamWriter, json::reader::ReaderBuilder, record_batch::RecordBatch, @@ -39,12 +40,12 @@ pub struct ArrowStreamSerializerConfig { /// Allow null values for non-nullable fields in the schema. /// - /// When enabled, missing or incompatible values will be encoded as null even for fields + /// When enabled, missing or incompatible values are encoded as null, even for fields /// marked as non-nullable in the Arrow schema. This is useful when working with downstream /// systems that can handle null values through defaults, computed columns, or other mechanisms. /// - /// When disabled (default), missing values for non-nullable fields will cause encoding errors, - /// ensuring all required data is present before sending to the sink. + /// When disabled (default), missing values for non-nullable fields results in encoding errors. This is to + /// help ensure all required data is present before sending it to the sink. #[serde(default)] #[configurable(derived)] pub allow_nullable_fields: bool, @@ -92,6 +93,19 @@ pub struct ArrowStreamSerializer { } impl ArrowStreamSerializer { + /// Encode events into a `RecordBatch` without writing to IPC stream format. + pub fn encode_to_record_batch( + &self, + events: &[Event], + ) -> Result { + let values = vector_log_events_to_json_values(events).map_err(|e| { + ArrowEncodingError::RecordBatchCreation { + source: arrow::error::ArrowError::JsonError(e.to_string()), + } + })?; + build_record_batch(self.schema.clone(), &values) + } + /// Create a new ArrowStreamSerializer with the given configuration pub fn new(config: ArrowStreamSerializerConfig) -> Result { let schema = config.schema.ok_or(ArrowEncodingError::MissingSchema)?; @@ -203,7 +217,13 @@ pub fn encode_events_to_arrow_ipc_stream( return Err(ArrowEncodingError::NoEvents); } - let record_batch = build_record_batch(schema, events)?; + let json_values = vector_log_events_to_json_values(events).map_err(|e| { + ArrowEncodingError::RecordBatchCreation { + source: ArrowError::JsonError(e.to_string()), + } + })?; + + let record_batch = build_record_batch(schema, &json_values)?; let mut buffer = BytesMut::new().writer(); let mut writer = @@ -271,7 +291,7 @@ fn make_field_nullable(field: &Field) -> Result { /// Find non-nullable schema fields that are missing or null in any of the given events. pub fn find_null_non_nullable_fields<'a>( schema: &'a Schema, - values: &[&vrl::value::Value], + values: &[serde_json::Value], ) -> Vec<&'a str> { schema .fields() @@ -282,38 +302,40 @@ pub fn find_null_non_nullable_fields<'a>( value .as_object() .and_then(|map| map.get(field.name().as_str())) - .is_none_or(vrl::value::Value::is_null) + .is_none_or(serde_json::Value::is_null) }) }) .map(|field| field.name().as_str()) .collect() } -/// Build an Arrow RecordBatch from a slice of events using the provided schema. -fn build_record_batch( - schema: SchemaRef, +pub(crate) fn vector_log_events_to_json_values( events: &[Event], -) -> Result { - let values: Vec<_> = events +) -> Result, serde_json::Error> { + events .iter() .filter_map(Event::maybe_as_log) - .map(|log| log.value()) - .collect(); + .map(serde_json::to_value) + .collect() +} +/// Build an Arrow RecordBatch from a slice of events using the provided schema. +pub(crate) fn build_record_batch( + schema: SchemaRef, + values: &[serde_json::Value], +) -> Result { if values.is_empty() { return Err(ArrowEncodingError::NoEvents); } - let missing = find_null_non_nullable_fields(&schema, &values); + let missing = find_null_non_nullable_fields(&schema, values); if !missing.is_empty() { - for field_name in &missing { - let error: vector_common::Error = Box::new(ArrowEncodingError::NullConstraint { - field_name: field_name.to_string(), - }); - vector_common::internal_event::emit( - crate::internal_events::EncoderNullConstraintError { error: &error }, - ); - } + let error: vector_common::Error = Box::new(ArrowEncodingError::NullConstraint { + field_name: missing.join(", "), + }); + vector_common::internal_event::emit(crate::internal_events::EncoderNullConstraintError { + error: &error, + }); return Err(ArrowEncodingError::NullConstraint { field_name: missing.join(", "), }); @@ -321,12 +343,32 @@ fn build_record_batch( let mut decoder = ReaderBuilder::new(schema) .build_decoder() + .inspect_err(|e| { + vector_common::internal_event::emit(crate::internal_events::EncoderRecordBatchError { + error: e, + error_code: "arrow_record_batch_creation", + }); + }) .context(RecordBatchCreationSnafu)?; - decoder.serialize(&values).context(ArrowJsonDecodeSnafu)?; + decoder + .serialize(values) + .inspect_err(|e| { + vector_common::internal_event::emit(crate::internal_events::EncoderRecordBatchError { + error: e, + error_code: "arrow_json_decode", + }); + }) + .context(ArrowJsonDecodeSnafu)?; decoder .flush() + .inspect_err(|e| { + vector_common::internal_event::emit(crate::internal_events::EncoderRecordBatchError { + error: e, + error_code: "arrow_json_decode", + }); + }) .context(ArrowJsonDecodeSnafu)? .ok_or(ArrowEncodingError::NoEvents) } @@ -885,8 +927,10 @@ mod tests { ("a", Value::Bytes("val".into())), ("b", Value::Integer(42)), ]); - let value = event.as_log().value(); - let missing = find_null_non_nullable_fields(&schema, &[value]); + let missing = find_null_non_nullable_fields( + &schema, + &vector_log_events_to_json_values(&[event]).unwrap(), + ); assert!( missing.is_empty(), "Expected no missing fields, got: {missing:?}" @@ -898,8 +942,10 @@ mod tests { let schema = Schema::new(vec![Field::new("a", DataType::Utf8, false)]); let event = create_event(vec![("a", Value::Null)]); - let value = event.as_log().value(); - let missing = find_null_non_nullable_fields(&schema, &[value]); + let missing = find_null_non_nullable_fields( + &schema, + &vector_log_events_to_json_values(&[event]).unwrap(), + ); assert_eq!(missing, vec!["a"]); } } diff --git a/lib/codecs/src/encoding/format/mod.rs b/lib/codecs/src/encoding/format/mod.rs index 85bc094b26947..33030fd41ff2e 100644 --- a/lib/codecs/src/encoding/format/mod.rs +++ b/lib/codecs/src/encoding/format/mod.rs @@ -16,6 +16,9 @@ mod native; mod native_json; #[cfg(feature = "opentelemetry")] mod otlp; +#[cfg(feature = "parquet")] +mod parquet; +mod proto_batch; mod protobuf; mod raw_message; #[cfg(feature = "syslog")] @@ -24,6 +27,10 @@ mod text; use std::fmt::Debug; +#[cfg(feature = "parquet")] +pub use self::parquet::{ + ParquetCompression, ParquetSchemaMode, ParquetSerializer, ParquetSerializerConfig, +}; #[cfg(feature = "arrow")] pub use arrow::{ ArrowEncodingError, ArrowStreamSerializer, ArrowStreamSerializerConfig, SchemaProvider, @@ -39,6 +46,7 @@ pub use native::{NativeSerializer, NativeSerializerConfig}; pub use native_json::{NativeJsonSerializer, NativeJsonSerializerConfig}; #[cfg(feature = "opentelemetry")] pub use otlp::{OtlpSerializer, OtlpSerializerConfig}; +pub use proto_batch::{ProtoBatchEncodingError, ProtoBatchSerializer, ProtoBatchSerializerConfig}; pub use protobuf::{ProtobufSerializer, ProtobufSerializerConfig, ProtobufSerializerOptions}; pub use raw_message::{RawMessageSerializer, RawMessageSerializerConfig}; #[cfg(feature = "syslog")] diff --git a/lib/codecs/src/encoding/format/otlp.rs b/lib/codecs/src/encoding/format/otlp.rs index fd0ffffe47b04..c3e8d325a8abc 100644 --- a/lib/codecs/src/encoding/format/otlp.rs +++ b/lib/codecs/src/encoding/format/otlp.rs @@ -64,6 +64,7 @@ impl OtlpSerializer { pub fn new() -> vector_common::Result { let options = Options { use_json_names: true, + allow_lossy_string_coercion: true, }; let logs_descriptor = ProtobufSerializer::new_from_bytes( diff --git a/lib/codecs/src/encoding/format/parquet.rs b/lib/codecs/src/encoding/format/parquet.rs new file mode 100644 index 0000000000000..b8874d9b1de90 --- /dev/null +++ b/lib/codecs/src/encoding/format/parquet.rs @@ -0,0 +1,951 @@ +//! Parquet batch format codec for batched event encoding +//! +//! Provides Apache Parquet format encoding with schema file support and auto-inference. +//! Reuses the Arrow record batch building logic from the Arrow IPC codec, +//! then writes the batch as a complete Parquet file using `ArrowWriter`. + +use std::collections::HashSet; +use std::path::PathBuf; +use std::sync::Arc; + +use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; +use arrow::error::ArrowError; +use arrow::json::reader::infer_json_schema_from_iterator; +use arrow::record_batch::RecordBatch; +use bytes::{BufMut, BytesMut}; +use derivative::Derivative; +use parquet::arrow::ArrowWriter; +use parquet::basic::ZstdLevel; +use parquet::basic::{Compression as ParquetCodecCompression, GzipLevel}; +use parquet::file::properties::WriterProperties; +use std::io::{Error, ErrorKind}; +use tracing::warn; +use vector_common::internal_event::{ + ComponentEventsDropped, Count, InternalEventHandle, Registered, UNINTENTIONAL, emit, register, +}; +use vector_config::configurable_component; +use vector_core::event::Event; + +use super::arrow::{ArrowEncodingError, build_record_batch}; +use crate::encoding::format::arrow::vector_log_events_to_json_values; +use crate::internal_events::{ArrowWriterError, JsonSerializationError, SchemaGenerationError}; + +type EventsDroppedError = ComponentEventsDropped<'static, UNINTENTIONAL>; + +/// Compression algorithm and optional level for archive objects. +#[configurable_component] +#[derive(Default, Copy, Clone, Debug, PartialEq)] +#[configurable(metadata( + docs::enum_tag_description = "Compression codec applied per column page inside the Parquet file." +))] +#[serde(tag = "algorithm", rename_all = "snake_case")] +pub enum ParquetCompression { + /// Zstd compression. Level must be between 1 and 21. + Zstd { + /// Compression level (1–21). This is the range Vector supports; higher values compress more but are slower. + #[configurable(validation(range(min = 1, max = 21)))] + level: u8, + }, + /// Gzip compression. Level must be between 1 and 9. + Gzip { + /// Compression level (1–9). This is the range Vector supports; higher values compress more but are slower. + #[configurable(validation(range(min = 1, max = 9)))] + level: u8, + }, + + /// Snappy compression (no level). + #[default] + Snappy, + + /// LZ4 raw compression + Lz4, + + /// No compression + None, +} + +impl TryFrom for ParquetCodecCompression { + type Error = parquet::errors::ParquetError; + fn try_from( + value: ParquetCompression, + ) -> Result { + match value { + ParquetCompression::None => Ok(ParquetCodecCompression::UNCOMPRESSED), + ParquetCompression::Snappy => Ok(ParquetCodecCompression::SNAPPY), + ParquetCompression::Zstd { level } => Ok(ParquetCodecCompression::ZSTD( + ZstdLevel::try_new(level.into())?, + )), + ParquetCompression::Gzip { level } => Ok(ParquetCodecCompression::GZIP( + GzipLevel::try_new(level.into())?, + )), + ParquetCompression::Lz4 => Ok(ParquetCodecCompression::LZ4_RAW), + } + } +} + +/// Schema handling mode. +#[configurable_component] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ParquetSchemaMode { + /// Missing fields become null. Extra fields are silently dropped. + #[default] + Relaxed, + /// Missing fields become null. Extra fields cause an error. + Strict, + /// Auto infer schema based on the batch. No schema file needed. + AutoInfer, +} + +/// Configuration for the Parquet serializer. +/// +/// Encodes events as Apache Parquet columnar files, optimized for analytical queries +/// via Athena, Trino, Spark, and other columnar query engines. +/// +/// Either `schema_file` must be provided, or `schema_mode` must be set to `auto_infer`. +#[configurable_component] +#[derive(Clone, Debug, Default)] +pub struct ParquetSerializerConfig { + /// Path to a native Parquet schema file (`.schema`). + /// + /// Required unless `schema_mode` is `auto_infer`. The file must contain a valid + /// Parquet message type definition. + #[serde(default)] + pub schema_file: Option, + + /// Compression codec applied per column page inside the Parquet file. + #[serde(default)] + #[configurable(derived)] + pub compression: ParquetCompression, + + /// Controls how events with fields not present in the schema are handled. + #[serde(default)] + #[configurable(derived)] + pub schema_mode: ParquetSchemaMode, +} + +impl ParquetSerializerConfig { + /// Resolve the Arrow schema from the configured schema source. + fn resolve_schema(&self) -> Result> { + if self.schema_mode == ParquetSchemaMode::AutoInfer { + return Ok(Schema::empty()); + } + + let path = self + .schema_file + .as_ref() + .ok_or("schema_file is required unless schema_mode is auto_infer")?; + + let content = read_schema_file(path, "schema_file")?; + let parquet_type = parquet::schema::parser::parse_message_type(&content) + .map_err(|e| format!("Failed to parse Parquet schema: {e}"))?; + let schema_desc = parquet::schema::types::SchemaDescriptor::new(Arc::new(parquet_type)); + let arrow_schema = parquet::arrow::parquet_to_arrow_schema(&schema_desc, None) + .map_err(|e| format!("Failed to convert Parquet schema to Arrow: {e}"))?; + Ok(arrow_schema) + } + + /// The data type of events that are accepted by `ParquetSerializer`. + pub fn input_type(&self) -> vector_core::config::DataType { + vector_core::config::DataType::Log + } + + /// The schema required by the serializer. + pub fn schema_requirement(&self) -> vector_core::schema::Requirement { + vector_core::schema::Requirement::empty() + } +} + +fn read_schema_file( + path: &std::path::Path, + field_name: &str, +) -> Result> { + const MAX_SCHEMA_FILE_SIZE: u64 = 10 * 1024 * 1024; // 10 MB + let display = path.display(); + let metadata = std::fs::metadata(path) + .map_err(|e| format!("Failed to read {field_name} '{display}': {e}"))?; + if metadata.len() > MAX_SCHEMA_FILE_SIZE { + return Err(format!( + "{field_name} '{display}' is too large ({} bytes, max {MAX_SCHEMA_FILE_SIZE})", + metadata.len() + ) + .into()); + } + std::fs::read_to_string(path) + .map_err(|e| format!("Failed to read {field_name} '{display}': {e}").into()) +} + +/// Check the resolved Arrow schema for data types unsupported by the JSON-based +/// encode path (`arrow::json::reader::ReaderBuilder`). Binary variants are +/// accepted by Parquet/Arrow at the schema level but the JSON decoder rejects +/// them at runtime, so we fail fast here at config time. +fn reject_unsupported_arrow_types( + schema: &Schema, +) -> Result<(), Box> { + fn check_field(field: &Field, path: &str, bad: &mut Vec) { + let name = if path.is_empty() { + field.name().to_string() + } else { + format!("{path}.{}", field.name()) + }; + match field.data_type() { + DataType::Binary | DataType::LargeBinary | DataType::FixedSizeBinary(_) => { + bad.push(format!("'{name}' ({:?})", field.data_type())); + } + DataType::Struct(fields) => { + for f in fields { + check_field(f, &name, bad); + } + } + DataType::List(inner) | DataType::LargeList(inner) => { + check_field(inner, &name, bad); + } + DataType::Map(entries_field, _) => { + if let DataType::Struct(kv) = entries_field.data_type() { + for f in kv { + check_field(f, &name, bad); + } + } + } + _ => {} + } + } + + let mut bad = Vec::new(); + for field in schema.fields() { + check_field(field, "", &mut bad); + } + if !bad.is_empty() { + return Err(format!( + "Schema contains binary field(s) unsupported by the JSON-based Arrow encoder: {}. \ + Use Utf8 for base64/hex-encoded data instead.", + bad.join(", ") + ) + .into()); + } + Ok(()) +} + +/// Parquet batch serializer. +#[derive(Derivative)] +#[derivative(Debug, Clone)] +pub struct ParquetSerializer { + schema: SchemaRef, + writer_props: Arc, + schema_mode: ParquetSchemaMode, + /// Pre-built set of schema field names for O(1) strict-mode lookups. + schema_field_names: HashSet, + + #[derivative(Debug = "ignore")] + events_dropped_handle: Registered, +} + +impl ParquetSerializer { + /// Create a new `ParquetSerializer` from the given configuration. + pub fn new( + config: ParquetSerializerConfig, + ) -> Result> { + let schema = config.resolve_schema()?; + reject_unsupported_arrow_types(&schema)?; + let schema_ref = SchemaRef::new(schema); + + let schema_field_names = schema_ref + .fields() + .iter() + .map(|f| f.name().clone()) + .collect::>(); + + let writer_props = Arc::new( + WriterProperties::builder() + .set_compression(config.compression.try_into()?) + .build(), + ); + + Ok(Self { + schema: schema_ref, + writer_props, + schema_mode: config.schema_mode, + schema_field_names, + events_dropped_handle: register(EventsDroppedError::from( + "Events could not be serialized to parquet", + )), + }) + } + + /// Returns the MIME content type for Parquet data. + pub const fn content_type(&self) -> &'static str { + "application/vnd.apache.parquet" + } + + /// Writes `record_batch` into `buffer` as a complete Parquet file. + /// + /// On failure, emits an [`ArrowWriterError`] internal event (which + /// increments `component_errors_total`) before returning the error. + /// The caller is responsible for emitting `events_dropped`. + fn write_record_batch( + record_batch: &RecordBatch, + buffer: &mut BytesMut, + writer_props: &WriterProperties, + ) -> Result<(), parquet::errors::ParquetError> { + let mut writer = ArrowWriter::try_new( + buffer.writer(), + Arc::clone(record_batch.schema_ref()), + Some(writer_props.clone()), + ) + .inspect_err(|e| { + emit(ArrowWriterError { error: e }); + })?; + + writer.write(record_batch).inspect_err(|e| { + emit(ArrowWriterError { error: e }); + })?; + + writer.close().inspect_err(|e| { + emit(ArrowWriterError { error: e }); + })?; + + Ok(()) + } +} + +impl tokio_util::codec::Encoder> for ParquetSerializer { + type Error = vector_common::Error; + + fn encode(&mut self, events: Vec, buffer: &mut BytesMut) -> Result<(), Self::Error> { + if events.is_empty() { + return Ok(()); + } + + let json_values = match vector_log_events_to_json_values(&events) { + Ok(values) => values, + Err(e) => { + emit(JsonSerializationError { error: &e }); + return Err(Box::new(e)); + } + }; + + let non_log_count = events.len() - json_values.len(); + + if non_log_count > 0 { + warn!( + message = "Non-log events dropped by Parquet encoder ", + %non_log_count, + internal_log_rate_secs = 10, + ); + self.events_dropped_handle.emit(Count(non_log_count)) + } + + if json_values.is_empty() { + return Ok(()); + } + + match self.schema_mode { + // In strict mode, check for extra top-level fields not in the schema. + ParquetSchemaMode::Strict => { + for event in &events { + if let Some(log) = event.maybe_as_log() + && let Some(object_map) = log.as_map() + { + for top_level in object_map.keys() { + if !self.schema_field_names.contains(top_level.as_str()) { + return Err(Box::new(ArrowEncodingError::SchemaFetchError { + message: format!( + "Strict schema mode: event contains field '{top_level}' not in schema", + ), + })); + } + } + } + } + } + ParquetSchemaMode::AutoInfer => { + let schema = ParquetSchemaGenerator::infer_schema(&json_values)?; + self.schema = Arc::new(ParquetSchemaGenerator::try_normalize_schema( + &events, schema, + )); + } + ParquetSchemaMode::Relaxed => {} + } + + let record_batch = + build_record_batch(Arc::clone(&self.schema), &json_values).map_err(Box::new)?; + + Self::write_record_batch(&record_batch, buffer, &self.writer_props).map_err(Box::new)?; + + Ok(()) + } +} + +pub struct ParquetSchemaGenerator {} + +impl ParquetSchemaGenerator { + pub fn infer_schema(events: &[serde_json::Value]) -> Result { + let schema = infer_json_schema_from_iterator(events.iter().map(Ok::<_, ArrowError>)) + .map_err(|e| { + emit(SchemaGenerationError { error: &e }); + Error::new(ErrorKind::InvalidData, e.to_string()) + })?; + + Ok(schema) + } + + /// Attempt to modify schema to set timestamp fields as Timestamp instead of Utf8. + /// Only works for top-level fields. + fn try_normalize_schema(events: &[Event], schema: Schema) -> Schema { + let mut ts_seen: HashSet = HashSet::new(); + let mut non_ts_seen: HashSet = HashSet::new(); + + for event in events.iter().filter_map(Event::maybe_as_log) { + if let Some(object_map) = event.as_map() { + for (path, value) in object_map { + if value.is_timestamp() { + ts_seen.insert(path.to_string()); + } else if !value.is_null() { + non_ts_seen.insert(path.to_string()); + } + } + } + } + + let new_fields: Vec = schema + .fields() + .iter() + .map(|f| { + if ts_seen.contains(f.name()) && !non_ts_seen.contains(f.name()) { + Field::new( + f.name(), + DataType::Timestamp( + arrow::datatypes::TimeUnit::Microsecond, + Some("UTC".into()), + ), + f.is_nullable(), + ) + } else { + f.as_ref().clone() + } + }) + .collect(); + + Schema::new_with_metadata(new_fields, schema.metadata().clone()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use bytes::Bytes; + use parquet::file::reader::{FileReader, SerializedFileReader}; + use parquet::record::reader::RowIter; + use tokio_util::codec::Encoder; + use vector_core::event::LogEvent; + + fn create_event(fields: Vec<(&str, V)>) -> Event + where + V: Into, + { + let mut log = LogEvent::default(); + for (key, value) in fields { + log.insert(key, value.into()); + } + Event::Log(log) + } + + fn assert_parquet_magic(data: &[u8]) { + assert!(data.len() >= 4, "Output too short to be valid Parquet"); + assert_eq!(&data[..4], b"PAR1", "Missing Parquet magic bytes"); + } + + fn parquet_row_count(data: &[u8]) -> usize { + let reader = + SerializedFileReader::new(Bytes::copy_from_slice(data)).expect("Invalid Parquet file"); + let iter = RowIter::from_file_into(Box::new(reader)); + iter.count() + } + + fn parquet_column_names(data: &[u8]) -> Vec { + let reader = + SerializedFileReader::new(Bytes::copy_from_slice(data)).expect("Invalid Parquet file"); + let schema = reader.metadata().file_metadata().schema_descr(); + schema + .columns() + .iter() + .map(|c| c.name().to_string()) + .collect() + } + + fn parse_timestamp(s: &str) -> chrono::DateTime { + chrono::DateTime::parse_from_rfc3339(s) + .expect("invalid test timestamp") + .with_timezone(&chrono::Utc) + } + + fn demo_log_event( + message: &str, + timestamp: chrono::DateTime, + status_code: i64, + response_time_secs: f64, + ) -> Event { + use vector_core::event::Value; + let mut log = LogEvent::default(); + log.insert("host", "localhost"); + log.insert("message", message); + log.insert("service", "vector"); + log.insert("source_type", "demo_logs"); + log.insert("timestamp", Value::Timestamp(timestamp)); + log.insert("random_time", Value::Timestamp(timestamp)); + log.insert("status_code", Value::Integer(status_code)); + log.insert("response_time_secs", response_time_secs); + Event::Log(log) + } + + fn sample_events() -> Vec { + const EVENTS: [(&str, &str, i64, f64); 5] = [ + ( + "GET /api/v1/health HTTP/1.1", + "2026-03-05T20:49:08.037194Z", + 200, + 0.037, + ), + ( + "POST /api/v1/ingest HTTP/1.1", + "2026-03-05T20:49:09.038051Z", + 201, + 0.013, + ), + ( + "GET /metrics HTTP/1.1", + "2026-03-05T20:49:10.036612Z", + 200, + 0.022, + ), + ( + "DELETE /api/v1/resource HTTP/1.1", + "2026-03-05T20:49:11.537131Z", + 404, + 0.005, + ), + ( + "PATCH /api/v1/config HTTP/1.1", + "2026-03-05T20:49:12.037491Z", + 500, + 0.091, + ), + ]; + EVENTS + .iter() + .map(|(msg, ts, status, rt)| demo_log_event(msg, parse_timestamp(ts), *status, *rt)) + .collect() + } + + fn encode_autoinfer_and_read_schema( + events: Vec, + ) -> (arrow::datatypes::SchemaRef, usize) { + use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; + + let mut serializer = ParquetSerializer::new(ParquetSerializerConfig { + schema_mode: ParquetSchemaMode::AutoInfer, + ..Default::default() + }) + .expect("AutoInfer serializer should be created without a static schema"); + + let mut buffer = BytesMut::new(); + serializer + .encode(events, &mut buffer) + .expect("encoding should succeed"); + + let data = buffer.freeze(); + assert_parquet_magic(&data); + + let builder = ParquetRecordBatchReaderBuilder::try_new(data) + .expect("should build ParquetRecordBatchReaderBuilder"); + let schema = builder.schema().clone(); + let num_rows: usize = builder + .build() + .expect("should build reader") + .map(|b| b.expect("batch read error").num_rows()) + .sum(); + (schema, num_rows) + } + + /// Write a temporary Parquet schema file and return its path. + /// + /// `name` must be unique per test to avoid parallel-test races on the same file. + fn write_temp_schema(name: &str, content: &str) -> std::path::PathBuf { + use std::io::Write; + let path = std::env::temp_dir().join(format!( + "vector_parquet_test_{}_{}.schema", + std::process::id(), + name, + )); + let mut f = std::fs::File::create(&path).expect("Failed to create schema file"); + write!(f, "{content}").expect("Failed to write schema"); + path + } + + // ── AutoInfer mode ─────────────────────────────────────────────────────── + + #[test] + fn encode_input_produces_parquet_output() { + let events = sample_events(); + let n_events = events.len(); + let (schema, num_rows) = encode_autoinfer_and_read_schema(events); + + assert_eq!(num_rows, n_events, "row count should match event count"); + + for field_name in &["timestamp", "random_time"] { + let field = schema + .field_with_name(field_name) + .unwrap_or_else(|_| panic!("field '{field_name}' should exist in schema")); + assert!( + matches!( + field.data_type(), + DataType::Timestamp(arrow::datatypes::TimeUnit::Microsecond, Some(tz)) if tz.as_ref() == "UTC" + ), + "'{field_name}' should be Timestamp(Microsecond, UTC), got {:?}", + field.data_type() + ); + } + + let status_field = schema + .field_with_name("status_code") + .expect("status_code field should exist"); + assert_eq!(status_field.data_type(), &DataType::Int64); + + let rt_field = schema + .field_with_name("response_time_secs") + .expect("response_time_secs field should exist"); + assert_eq!(rt_field.data_type(), &DataType::Float64); + + for field_name in &["host", "message", "service", "source_type"] { + let field = schema + .field_with_name(field_name) + .unwrap_or_else(|_| panic!("field '{field_name}' should exist in schema")); + assert_eq!(field.data_type(), &DataType::Utf8); + } + } + + #[test] + fn test_parquet_empty_events() { + let mut serializer = ParquetSerializer::new(ParquetSerializerConfig { + schema_mode: ParquetSchemaMode::AutoInfer, + ..Default::default() + }) + .expect("AutoInfer serializer should succeed"); + + let events: Vec = vec![]; + let mut buffer = BytesMut::new(); + serializer + .encode(events, &mut buffer) + .expect("Empty events should succeed"); + + assert!(buffer.is_empty(), "Buffer should be empty for empty events"); + } + + #[test] + fn test_parquet_compression_variants() { + let events = vec![create_event(vec![("msg", "hello world")])]; + + let compressions = vec![ + ParquetCompression::None, + ParquetCompression::Snappy, + ParquetCompression::Zstd { level: 1 }, + ParquetCompression::Gzip { level: 1 }, + ParquetCompression::Lz4, + ]; + + for compression in compressions { + let mut serializer = ParquetSerializer::new(ParquetSerializerConfig { + schema_mode: ParquetSchemaMode::AutoInfer, + compression, + ..Default::default() + }) + .expect("Failed to create serializer"); + + let mut buffer = BytesMut::new(); + serializer + .encode(events.clone(), &mut buffer) + .unwrap_or_else(|e| panic!("Encoding with {:?} failed: {}", compression, e)); + + let data = buffer.freeze(); + assert_parquet_magic(&data); + assert_eq!( + parquet_row_count(&data), + 1, + "Wrong row count for {:?}", + compression + ); + } + } + + #[test] + fn test_parquet_output_has_footer() { + let mut serializer = ParquetSerializer::new(ParquetSerializerConfig { + schema_mode: ParquetSchemaMode::AutoInfer, + ..Default::default() + }) + .expect("AutoInfer serializer should succeed"); + + let events = vec![create_event(vec![("msg", "test")])]; + let mut buffer = BytesMut::new(); + serializer.encode(events, &mut buffer).unwrap(); + + let data = buffer.freeze(); + let len = data.len(); + assert!(len >= 8, "Parquet output too short"); + assert_eq!( + &data[len - 4..], + b"PAR1", + "Parquet footer magic bytes missing" + ); + } + + #[test] + fn test_writer_props_arc_shared() { + let serializer = ParquetSerializer::new(ParquetSerializerConfig { + schema_mode: ParquetSchemaMode::AutoInfer, + ..Default::default() + }) + .expect("AutoInfer serializer should succeed"); + let cloned = serializer.clone(); + + assert_eq!(Arc::strong_count(&serializer.writer_props), 2); + drop(cloned); + assert_eq!(Arc::strong_count(&serializer.writer_props), 1); + } + + #[test] + fn test_mixed_log_and_non_log_events() { + use vector_core::event::{Metric, MetricKind, MetricValue}; + + let mut serializer = ParquetSerializer::new(ParquetSerializerConfig { + schema_mode: ParquetSchemaMode::AutoInfer, + ..Default::default() + }) + .expect("AutoInfer serializer should succeed"); + + let metric = Metric::new( + "cpu.usage", + MetricKind::Absolute, + MetricValue::Gauge { value: 42.0 }, + ); + let events = vec![ + create_event(vec![("msg", "hello")]), + Event::Metric(metric), + create_event(vec![("msg", "world")]), + ]; + + let mut buffer = BytesMut::new(); + serializer + .encode(events, &mut buffer) + .expect("Mixed batch should succeed (non-log events dropped)"); + + assert_parquet_magic(&buffer); + assert_eq!(parquet_row_count(&buffer), 2); + } + + // ── Schema file mode ───────────────────────────────────────────────────── + + #[test] + fn test_parquet_schema_file() { + let schema_path = write_temp_schema( + "schema_file", + "message logs {\n required binary name (STRING);\n optional int64 age;\n}", + ); + + let config: ParquetSerializerConfig = serde_json::from_value(serde_json::json!({ + "schema_file": schema_path.to_str().unwrap() + })) + .expect("Config should deserialize"); + + let mut serializer = + ParquetSerializer::new(config).expect("Should create serializer from schema file"); + + let mut log = LogEvent::default(); + log.insert("name", "alice"); + + let mut buffer = BytesMut::new(); + serializer + .encode(vec![Event::Log(log)], &mut buffer) + .expect("Encoding with schema file should succeed"); + + let data = buffer.freeze(); + assert_parquet_magic(&data); + assert_eq!(parquet_row_count(&data), 1); + + let columns = parquet_column_names(&data); + assert_eq!(columns, vec!["name", "age"]); + } + + #[test] + fn test_parquet_schema_file_not_found_error() { + let config: ParquetSerializerConfig = serde_json::from_value(serde_json::json!({ + "schema_file": "/nonexistent/path/schema.parquet" + })) + .expect("Config should deserialize"); + + let result = ParquetSerializer::new(config); + assert!(result.is_err(), "Missing schema file should error"); + assert!( + result.unwrap_err().to_string().contains("Failed to read"), + "Error should mention file read failure" + ); + } + + #[test] + fn test_parquet_schema_file_invalid_syntax_error() { + let schema_path = write_temp_schema( + "invalid_syntax", + "this is not valid parquet schema syntax !!!", + ); + + let config: ParquetSerializerConfig = serde_json::from_value(serde_json::json!({ + "schema_file": schema_path.to_str().unwrap() + })) + .expect("Config should deserialize"); + + let result = ParquetSerializer::new(config); + assert!(result.is_err(), "Invalid Parquet schema should error"); + assert!( + result + .unwrap_err() + .to_string() + .contains("Failed to parse Parquet schema"), + "Error should mention parsing failure" + ); + } + + #[test] + fn test_parquet_no_schema_error() { + let config = ParquetSerializerConfig::default(); + let result = ParquetSerializer::new(config); + assert!( + result.is_err(), + "Should fail without schema_file or auto_infer" + ); + } + + // ── Schema mode: strict / relaxed ──────────────────────────────────────── + + #[test] + fn test_parquet_strict_mode_rejects_extra_fields() { + let schema_path = write_temp_schema( + "strict_rejects", + "message logs {\n required binary name (STRING);\n}", + ); + + let mut serializer = ParquetSerializer::new(ParquetSerializerConfig { + schema_file: Some(schema_path), + schema_mode: ParquetSchemaMode::Strict, + ..Default::default() + }) + .expect("Failed to create strict serializer"); + + let events = vec![create_event(vec![("name", "alice"), ("city", "paris")])]; + let mut buffer = BytesMut::new(); + let result = serializer.encode(events, &mut buffer); + assert!(result.is_err(), "Strict mode should reject extra fields"); + assert!(result.unwrap_err().to_string().contains("city")); + } + + #[test] + fn test_parquet_strict_mode_allows_schema_fields() { + let schema_path = write_temp_schema( + "strict_allows", + "message logs {\n required binary name (STRING);\n required binary level (STRING);\n}", + ); + + let mut serializer = ParquetSerializer::new(ParquetSerializerConfig { + schema_file: Some(schema_path), + schema_mode: ParquetSchemaMode::Strict, + ..Default::default() + }) + .expect("Failed to create strict serializer"); + + let mut log = LogEvent::default(); + log.insert("name", "test"); + log.insert("level", "info"); + + let mut buffer = BytesMut::new(); + assert!( + serializer + .encode(vec![Event::Log(log)], &mut buffer) + .is_ok(), + "Strict mode should pass when all fields match schema" + ); + } + + #[test] + fn test_parquet_relaxed_mode_drops_extra_fields() { + let schema_path = write_temp_schema( + "relaxed_drops", + "message logs {\n required binary name (STRING);\n}", + ); + + let mut serializer = ParquetSerializer::new(ParquetSerializerConfig { + schema_file: Some(schema_path), + schema_mode: ParquetSchemaMode::Relaxed, + ..Default::default() + }) + .expect("Failed to create relaxed serializer"); + + let events = vec![create_event(vec![("name", "alice"), ("city", "paris")])]; + let mut buffer = BytesMut::new(); + serializer + .encode(events, &mut buffer) + .expect("Relaxed mode should drop extra fields silently"); + + let data = buffer.freeze(); + assert_parquet_magic(&data); + assert_eq!(parquet_row_count(&data), 1); + let columns = parquet_column_names(&data); + assert_eq!(columns, vec!["name"]); + } + + #[test] + fn test_parquet_type_mismatch_returns_error() { + let schema_path = + write_temp_schema("type_mismatch", "message logs {\n required int64 name;\n}"); + + let mut serializer = ParquetSerializer::new(ParquetSerializerConfig { + schema_file: Some(schema_path), + schema_mode: ParquetSchemaMode::Relaxed, + ..Default::default() + }) + .expect("Failed to create serializer"); + + let events = vec![create_event(vec![("name", "not_an_integer")])]; + let mut buffer = BytesMut::new(); + let result = serializer.encode(events, &mut buffer); + assert!(result.is_err(), "Type mismatch should return an error"); + let err = result.unwrap_err().to_string(); + assert!( + err.contains("Int64"), + "Error should mention the expected type, got: {err}" + ); + } + + #[test] + fn test_parquet_schema_file_binary_without_string_annotation_rejected() { + // Native Parquet "binary" without (STRING) annotation resolves to Arrow Binary, + // which is rejected at config time. + let schema_path = write_temp_schema( + "binary_rejected", + "message logs {\n required binary name (STRING);\n optional binary raw_data;\n}", + ); + + let config: ParquetSerializerConfig = serde_json::from_value(serde_json::json!({ + "schema_file": schema_path.to_str().unwrap() + })) + .expect("Config should deserialize"); + + let result = ParquetSerializer::new(config); + assert!( + result.is_err(), + "Parquet binary without STRING annotation should be rejected" + ); + assert!( + result.unwrap_err().to_string().contains("raw_data"), + "Error should name the offending field" + ); + } +} diff --git a/lib/codecs/src/encoding/format/proto_batch.rs b/lib/codecs/src/encoding/format/proto_batch.rs new file mode 100644 index 0000000000000..3bdce405f6b70 --- /dev/null +++ b/lib/codecs/src/encoding/format/proto_batch.rs @@ -0,0 +1,285 @@ +//! Protobuf batch serializer for encoding events as individual protobuf records. +//! +//! Encodes each event in a batch independently into protobuf bytes, producing +//! a `Vec>` where each element is a single serialized protobuf message. + +use prost_reflect::{MessageDescriptor, prost::Message as _}; +use snafu::Snafu; +use std::sync::Arc; +use vector_config::configurable_component; +use vector_core::{config::DataType, event::Event, schema}; +use vrl::protobuf::encode::{Options, encode_message}; + +/// Errors that can occur during protobuf batch encoding +#[derive(Debug, Snafu)] +pub enum ProtoBatchEncodingError { + /// No events provided + #[snafu(display("Cannot encode an empty batch"))] + NoEvents, + + /// Unsupported event type + #[snafu(display("Unsupported event type: only Log events are supported"))] + UnsupportedEventType, + + /// Protobuf encoding failed + #[snafu(display("Protobuf encoding failed: {}", source))] + EncodingFailed { + /// The underlying encoding error + source: vector_common::Error, + }, + + /// Protobuf prost encoding failed + #[snafu(display("Protobuf prost encoding failed: {}", source))] + ProstEncodingFailed { + /// The underlying prost error + source: prost_reflect::prost::EncodeError, + }, +} + +/// Configuration for protobuf batch serialization +#[configurable_component] +#[derive(Clone, Default)] +pub struct ProtoBatchSerializerConfig { + /// The protobuf message descriptor to use for encoding. + #[serde(skip)] + #[configurable(derived)] + pub descriptor: Option, +} + +impl std::fmt::Debug for ProtoBatchSerializerConfig { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ProtoBatchSerializerConfig") + .field( + "descriptor", + &self.descriptor.as_ref().map(|d| d.full_name().to_string()), + ) + .finish() + } +} + +impl ProtoBatchSerializerConfig { + /// Create a new ProtoBatchSerializerConfig with a message descriptor + pub fn new(descriptor: MessageDescriptor) -> Self { + Self { + descriptor: Some(descriptor), + } + } + + /// The data type of events that are accepted by this serializer. + pub fn input_type(&self) -> DataType { + DataType::Log + } + + /// The schema required by the serializer. + pub fn schema_requirement(&self) -> schema::Requirement { + schema::Requirement::empty() + } +} + +/// Protobuf batch serializer that encodes each event into individual protobuf bytes. +#[derive(Clone, Debug)] +pub struct ProtoBatchSerializer { + descriptor: Arc, + options: Options, +} + +impl ProtoBatchSerializer { + /// Create a new ProtoBatchSerializer with the given configuration. + pub fn new(config: ProtoBatchSerializerConfig) -> Result { + let descriptor = config.descriptor.ok_or_else(|| { + vector_common::Error::from("Proto batch serializer requires a message descriptor.") + })?; + + Ok(Self { + descriptor: Arc::new(descriptor), + options: Options { + use_json_names: false, + allow_lossy_string_coercion: true, + }, + }) + } + + /// Encode a batch of events into individual protobuf byte buffers. + pub fn encode_batch(&self, events: &[Event]) -> Result>, ProtoBatchEncodingError> { + if events.is_empty() { + return Err(ProtoBatchEncodingError::NoEvents); + } + + let mut records = Vec::with_capacity(events.len()); + + for event in events { + let dynamic_message = match event { + Event::Log(log) => { + encode_message(&self.descriptor, log.value().clone(), &self.options) + } + Event::Trace(_) | Event::Metric(_) => { + return Err(ProtoBatchEncodingError::UnsupportedEventType); + } + } + .map_err(|source| ProtoBatchEncodingError::EncodingFailed { + source: source.into(), + })?; + + records.push(dynamic_message.encode_to_vec()); + } + + Ok(records) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use prost_reflect::{ + DescriptorPool, DynamicMessage, Value as ProstValue, + prost_types::{ + DescriptorProto, FieldDescriptorProto, FileDescriptorProto, FileDescriptorSet, + field_descriptor_proto::{Label, Type}, + }, + }; + use vector_core::event::{LogEvent, Metric, MetricKind, MetricValue, TraceEvent, Value}; + use vrl::btreemap; + + fn build_descriptor() -> MessageDescriptor { + // message Inner { string label = 1; } + let inner = DescriptorProto { + name: Some("Inner".to_string()), + field: vec![FieldDescriptorProto { + name: Some("label".to_string()), + number: Some(1), + label: Some(Label::Optional as i32), + r#type: Some(Type::String as i32), + ..Default::default() + }], + ..Default::default() + }; + + // message Outer { string name = 1; int64 count = 2; Inner inner = 3; } + let outer = DescriptorProto { + name: Some("Outer".to_string()), + field: vec![ + FieldDescriptorProto { + name: Some("name".to_string()), + number: Some(1), + label: Some(Label::Optional as i32), + r#type: Some(Type::String as i32), + ..Default::default() + }, + FieldDescriptorProto { + name: Some("count".to_string()), + number: Some(2), + label: Some(Label::Optional as i32), + r#type: Some(Type::Int64 as i32), + ..Default::default() + }, + FieldDescriptorProto { + name: Some("inner".to_string()), + number: Some(3), + label: Some(Label::Optional as i32), + r#type: Some(Type::Message as i32), + type_name: Some(".test.Inner".to_string()), + ..Default::default() + }, + ], + nested_type: vec![], + ..Default::default() + }; + + let file = FileDescriptorProto { + name: Some("test.proto".to_string()), + package: Some("test".to_string()), + message_type: vec![outer, inner], + syntax: Some("proto3".to_string()), + ..Default::default() + }; + + let pool = DescriptorPool::from_file_descriptor_set(FileDescriptorSet { file: vec![file] }) + .expect("descriptor pool builds"); + pool.get_message_by_name("test.Outer") + .expect("Outer message exists") + } + + fn make_serializer() -> ProtoBatchSerializer { + ProtoBatchSerializer::new(ProtoBatchSerializerConfig::new(build_descriptor())) + .expect("serializer builds") + } + + #[test] + fn empty_batch_returns_no_events_error() { + let serializer = make_serializer(); + let err = serializer + .encode_batch(&[]) + .expect_err("empty batch errors"); + assert!(matches!(err, ProtoBatchEncodingError::NoEvents)); + } + + #[test] + fn metric_event_is_rejected() { + let serializer = make_serializer(); + let metric = Event::Metric(Metric::new( + "test", + MetricKind::Absolute, + MetricValue::Counter { value: 1.0 }, + )); + let err = serializer + .encode_batch(&[metric]) + .expect_err("metric event errors"); + assert!(matches!(err, ProtoBatchEncodingError::UnsupportedEventType)); + } + + #[test] + fn trace_event_is_rejected() { + let serializer = make_serializer(); + let trace = Event::Trace(TraceEvent::default()); + let err = serializer + .encode_batch(&[trace]) + .expect_err("trace event errors"); + assert!(matches!(err, ProtoBatchEncodingError::UnsupportedEventType)); + } + + #[test] + fn round_trip_decode_preserves_field_mapping() { + let descriptor = build_descriptor(); + let serializer = + ProtoBatchSerializer::new(ProtoBatchSerializerConfig::new(descriptor.clone())) + .expect("serializer builds"); + + let event = Event::Log(LogEvent::from(btreemap! { + "name" => Value::from("hello"), + "count" => Value::from(42_i64), + "inner" => Value::from(btreemap! { + "label" => Value::from("nested"), + }), + })); + + let records = serializer + .encode_batch(&[event]) + .expect("encoding succeeds"); + assert_eq!(records.len(), 1); + + let decoded = + DynamicMessage::decode(descriptor, records[0].as_slice()).expect("decode succeeds"); + + let name_field = decoded + .get_field_by_name("name") + .expect("name field present"); + assert_eq!(name_field.as_str(), Some("hello")); + + let count_field = decoded + .get_field_by_name("count") + .expect("count field present"); + assert_eq!(count_field.as_i64(), Some(42)); + + let inner_field = decoded + .get_field_by_name("inner") + .expect("inner field present"); + let inner_msg = match &*inner_field { + ProstValue::Message(m) => m, + other => panic!("expected nested message, got {:?}", other), + }; + let label = inner_msg + .get_field_by_name("label") + .expect("label field present"); + assert_eq!(label.as_str(), Some("nested")); + } +} diff --git a/lib/codecs/src/encoding/format/protobuf.rs b/lib/codecs/src/encoding/format/protobuf.rs index f1c6730615707..b6ea35a5e848c 100644 --- a/lib/codecs/src/encoding/format/protobuf.rs +++ b/lib/codecs/src/encoding/format/protobuf.rs @@ -32,6 +32,7 @@ impl ProtobufSerializerConfig { message_descriptor, options: Options { use_json_names: self.protobuf.use_json_names, + allow_lossy_string_coercion: true, }, }) } diff --git a/lib/codecs/src/encoding/framing/length_delimited.rs b/lib/codecs/src/encoding/framing/length_delimited.rs index 660049ce5b305..152b3baf73673 100644 --- a/lib/codecs/src/encoding/framing/length_delimited.rs +++ b/lib/codecs/src/encoding/framing/length_delimited.rs @@ -1,5 +1,4 @@ use bytes::BytesMut; -use derivative::Derivative; use tokio_util::codec::{Encoder, LengthDelimitedCodec}; use vector_config::configurable_component; @@ -8,8 +7,7 @@ use crate::common::length_delimited::LengthDelimitedCoderOptions; /// Config used to build a `LengthDelimitedEncoder`. #[configurable_component] -#[derive(Debug, Clone, Derivative, Eq, PartialEq)] -#[derivative(Default)] +#[derive(Debug, Clone, Default, Eq, PartialEq)] pub struct LengthDelimitedEncoderConfig { /// Options for the length delimited decoder. #[serde(skip_serializing_if = "vector_core::serde::is_default")] diff --git a/lib/codecs/src/encoding/framing/varint_length_delimited.rs b/lib/codecs/src/encoding/framing/varint_length_delimited.rs index 87f3737fa3d86..f874fb3d04e48 100644 --- a/lib/codecs/src/encoding/framing/varint_length_delimited.rs +++ b/lib/codecs/src/encoding/framing/varint_length_delimited.rs @@ -1,5 +1,4 @@ use bytes::{BufMut, BytesMut}; -use derivative::Derivative; use snafu::Snafu; use tokio_util::codec::Encoder; use vector_config::configurable_component; @@ -17,8 +16,7 @@ impl FramingError for VarintFramingError {} /// Config used to build a `VarintLengthDelimitedEncoder`. #[configurable_component] -#[derive(Debug, Clone, PartialEq, Eq, Derivative)] -#[derivative(Default)] +#[derive(Debug, Clone, PartialEq, Eq, Default)] pub struct VarintLengthDelimitedEncoderConfig { /// Maximum frame length #[serde(default = "default_max_frame_length")] diff --git a/lib/codecs/src/encoding/mod.rs b/lib/codecs/src/encoding/mod.rs index 25f88406fbae9..2c8b485a49d8f 100644 --- a/lib/codecs/src/encoding/mod.rs +++ b/lib/codecs/src/encoding/mod.rs @@ -10,7 +10,7 @@ pub mod serializer; mod transformer; pub use chunking::{Chunker, Chunking, GelfChunker}; pub use config::{EncodingConfig, EncodingConfigWithFraming, SinkType}; -pub use encoder::{BatchEncoder, BatchSerializer, Encoder, EncoderKind}; +pub use encoder::{BatchEncoder, BatchOutput, BatchSerializer, Encoder, EncoderKind}; #[cfg(feature = "arrow")] pub use format::{ ArrowEncodingError, ArrowStreamSerializer, ArrowStreamSerializerConfig, SchemaProvider, @@ -21,12 +21,17 @@ pub use format::{ CefSerializerConfig, CsvSerializer, CsvSerializerConfig, GelfSerializer, GelfSerializerConfig, JsonSerializer, JsonSerializerConfig, JsonSerializerOptions, LogfmtSerializer, LogfmtSerializerConfig, NativeJsonSerializer, NativeJsonSerializerConfig, NativeSerializer, - NativeSerializerConfig, ProtobufSerializer, ProtobufSerializerConfig, + NativeSerializerConfig, ProtoBatchEncodingError, ProtoBatchSerializer, + ProtoBatchSerializerConfig, ProtobufSerializer, ProtobufSerializerConfig, ProtobufSerializerOptions, RawMessageSerializer, RawMessageSerializerConfig, TextSerializer, TextSerializerConfig, }; #[cfg(feature = "opentelemetry")] pub use format::{OtlpSerializer, OtlpSerializerConfig}; +#[cfg(feature = "parquet")] +pub use format::{ + ParquetCompression, ParquetSchemaMode, ParquetSerializer, ParquetSerializerConfig, +}; #[cfg(feature = "syslog")] pub use format::{SyslogSerializer, SyslogSerializerConfig}; pub use framing::{ @@ -36,9 +41,7 @@ pub use framing::{ NewlineDelimitedEncoderConfig, VarintLengthDelimitedEncoder, VarintLengthDelimitedEncoderConfig, }; -#[cfg(feature = "arrow")] -pub use serializer::BatchSerializerConfig; -pub use serializer::{Serializer, SerializerConfig}; +pub use serializer::{BatchSerializerConfig, Serializer, SerializerConfig}; pub use transformer::{TimestampFormat, Transformer}; /// An error that occurred while building an encoder. diff --git a/lib/codecs/src/encoding/serializer.rs b/lib/codecs/src/encoding/serializer.rs index 536f836ad8163..0a7b07a00fd04 100644 --- a/lib/codecs/src/encoding/serializer.rs +++ b/lib/codecs/src/encoding/serializer.rs @@ -8,6 +8,9 @@ use vector_core::{config::DataType, event::Event, schema}; use super::format::{ArrowStreamSerializer, ArrowStreamSerializerConfig}; #[cfg(feature = "opentelemetry")] use super::format::{OtlpSerializer, OtlpSerializerConfig}; +#[cfg(feature = "parquet")] +use super::format::{ParquetSerializer, ParquetSerializerConfig}; +use super::format::{ProtoBatchSerializer, ProtoBatchSerializerConfig}; #[cfg(feature = "syslog")] use super::format::{SyslogSerializer, SyslogSerializerConfig}; use super::{ @@ -160,17 +163,42 @@ pub enum BatchSerializerConfig { #[cfg(feature = "arrow")] #[serde(rename = "arrow_stream")] ArrowStream(ArrowStreamSerializerConfig), + /// Encodes events in [Apache Parquet][apache_parquet] columnar format. + /// + /// [apache_parquet]: https://parquet.apache.org/ + #[cfg(feature = "parquet")] + #[serde(rename = "parquet")] + Parquet(ParquetSerializerConfig), + + /// Encodes each event individually as a [Protocol Buffers][protobuf] message. + /// + /// Each event in the batch is serialized to protobuf bytes independently, + /// producing a list of byte buffers (one per event). + /// + /// [protobuf]: https://protobuf.dev/ + #[serde(rename = "proto_batch")] + ProtoBatch(ProtoBatchSerializerConfig), } -#[cfg(feature = "arrow")] impl BatchSerializerConfig { - /// Build the `ArrowStreamSerializer` from this configuration. - pub fn build( + /// Build the batch serializer from this configuration. + pub fn build_batch_serializer( &self, - ) -> Result> { + ) -> Result> { match self { + #[cfg(feature = "arrow")] BatchSerializerConfig::ArrowStream(arrow_config) => { - Ok(ArrowStreamSerializer::new(arrow_config.clone())?) + let serializer = ArrowStreamSerializer::new(arrow_config.clone())?; + Ok(super::BatchSerializer::Arrow(serializer)) + } + #[cfg(feature = "parquet")] + BatchSerializerConfig::Parquet(parquet_config) => { + let serializer = ParquetSerializer::new(parquet_config.clone())?; + Ok(super::BatchSerializer::Parquet(Box::new(serializer))) + } + BatchSerializerConfig::ProtoBatch(proto_config) => { + let serializer = ProtoBatchSerializer::new(proto_config.clone())?; + Ok(super::BatchSerializer::ProtoBatch(serializer)) } } } @@ -178,14 +206,22 @@ impl BatchSerializerConfig { /// The data type of events that are accepted by this batch serializer. pub fn input_type(&self) -> DataType { match self { + #[cfg(feature = "arrow")] BatchSerializerConfig::ArrowStream(arrow_config) => arrow_config.input_type(), + #[cfg(feature = "parquet")] + BatchSerializerConfig::Parquet(parquet_config) => parquet_config.input_type(), + BatchSerializerConfig::ProtoBatch(proto_config) => proto_config.input_type(), } } /// The schema required by the batch serializer. pub fn schema_requirement(&self) -> schema::Requirement { match self { + #[cfg(feature = "arrow")] BatchSerializerConfig::ArrowStream(arrow_config) => arrow_config.schema_requirement(), + #[cfg(feature = "parquet")] + BatchSerializerConfig::Parquet(parquet_config) => parquet_config.schema_requirement(), + BatchSerializerConfig::ProtoBatch(proto_config) => proto_config.schema_requirement(), } } } diff --git a/lib/codecs/src/internal_events.rs b/lib/codecs/src/internal_events.rs index 134fee16ecf8b..0518247e523a3 100644 --- a/lib/codecs/src/internal_events.rs +++ b/lib/codecs/src/internal_events.rs @@ -1,9 +1,12 @@ //! Internal events for codecs. -use metrics::counter; use tracing::error; -use vector_common::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, emit, error_stage, error_type, +use vector_common::{ + counter, + internal_event::{ + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, emit, error_stage, + error_type, + }, }; use vector_common_macros::NamedInternalEvent; @@ -24,7 +27,7 @@ impl InternalEvent for DecoderFramingError { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "decoder_frame", "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, @@ -50,7 +53,7 @@ impl InternalEvent for DecoderDeserializeError<'_> { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "decoder_deserialize", "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, @@ -77,7 +80,7 @@ impl InternalEvent for EncoderFramingError<'_> { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "encoder_frame", "error_type" => error_type::ENCODER_FAILED, "stage" => error_stage::SENDING, @@ -105,7 +108,7 @@ impl InternalEvent for EncoderSerializeError<'_> { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "encoder_serialize", "error_type" => error_type::ENCODER_FAILED, "stage" => error_stage::SENDING, @@ -137,7 +140,7 @@ impl InternalEvent for EncoderWriteError<'_, E> { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::ENCODER_FAILED, "stage" => error_stage::SENDING, ) @@ -162,24 +165,129 @@ pub struct EncoderNullConstraintError<'a> { #[cfg(feature = "arrow")] impl InternalEvent for EncoderNullConstraintError<'_> { fn emit(self) { - const CONSTRAINT_REASON: &str = "Schema constraint violation."; error!( - message = CONSTRAINT_REASON, + message = "Schema constraint violation.", error = %self.error, error_code = "encoding_null_constraint", error_type = error_type::ENCODER_FAILED, stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "encoding_null_constraint", "error_type" => error_type::ENCODER_FAILED, "stage" => error_stage::SENDING, ) .increment(1); - emit(ComponentEventsDropped:: { - count: 1, - reason: CONSTRAINT_REASON, - }); + } +} + +#[cfg(feature = "arrow")] +#[derive(Debug, NamedInternalEvent)] +/// Emitted when Arrow record batch construction fails (e.g. schema decoder build, +/// JSON-to-Arrow decoding such as type mismatches). +pub struct EncoderRecordBatchError<'a, E> { + /// The encoding error that occurred. + pub error: &'a E, + /// Stable error code identifying the failure mode. + pub error_code: &'static str, +} + +#[cfg(feature = "arrow")] +impl InternalEvent for EncoderRecordBatchError<'_, E> { + fn emit(self) { + error!( + message = "Failed to build Arrow record batch.", + error = %self.error, + error_code = self.error_code, + error_type = error_type::ENCODER_FAILED, + stage = error_stage::SENDING, + ); + counter!( + CounterName::ComponentErrorsTotal, + "error_code" => self.error_code, + "error_type" => error_type::ENCODER_FAILED, + "stage" => error_stage::SENDING, + ) + .increment(1); + } +} + +#[cfg(feature = "parquet")] +#[derive(NamedInternalEvent)] +pub(crate) struct SchemaGenerationError<'a> { + pub error: &'a arrow::error::ArrowError, +} + +#[cfg(feature = "parquet")] +impl InternalEvent for SchemaGenerationError<'_> { + fn emit(self) { + error!( + message = "Could not generate schema for batched events", + error = %self.error, + error_code = "parquet_schema_generation_failed", + error_type = error_type::ENCODER_FAILED, + stage = error_stage::SENDING, + internal_log_rate_limit = false, + ); + counter!( + CounterName::ComponentErrorsTotal, + "error_code" => "parquet_schema_generation_failed", + "error_type" => error_type::ENCODER_FAILED, + "stage" => error_stage::SENDING, + ) + .increment(1); + } +} + +#[cfg(feature = "parquet")] +#[derive(NamedInternalEvent)] +pub(crate) struct ArrowWriterError<'a> { + pub error: &'a parquet::errors::ParquetError, +} + +#[cfg(feature = "parquet")] +impl InternalEvent for ArrowWriterError<'_> { + fn emit(self) { + error!( + message = "Failed to write record batch with ArrowWriter.", + error = %self.error, + error_code = "parquet_arrow_writer_failed", + error_type = error_type::ENCODER_FAILED, + stage = error_stage::SENDING, + internal_log_rate_limit = false, + ); + counter!( + CounterName::ComponentErrorsTotal, + "error_code" => "parquet_arrow_writer_failed", + "error_type" => error_type::ENCODER_FAILED, + "stage" => error_stage::SENDING, + ) + .increment(1); + } +} + +#[cfg(feature = "parquet")] +#[derive(NamedInternalEvent)] +pub(crate) struct JsonSerializationError<'a> { + pub error: &'a serde_json::Error, +} + +#[cfg(feature = "parquet")] +impl InternalEvent for JsonSerializationError<'_> { + fn emit(self) { + error!( + message = "Could not serialize event to JSON.", + error = %self.error, + error_type = error_type::ENCODER_FAILED, + stage = error_stage::SENDING, + internal_log_rate_limit = true, + ); + counter!( + CounterName::ComponentErrorsTotal, + "error_type" => error_type::ENCODER_FAILED, + "stage" => error_stage::SENDING, + ) + .increment(1); } } diff --git a/lib/codecs/tests/data/native_encoding/README.md b/lib/codecs/tests/data/native_encoding/README.md index 0a84c98043c41..2cf2aaba974be 100644 --- a/lib/codecs/tests/data/native_encoding/README.md +++ b/lib/codecs/tests/data/native_encoding/README.md @@ -6,28 +6,23 @@ and we test that all the examples can be successfully parsed, parse the same across both formats, and match the current serialized format. In order to avoid small inherent serialization differences between JSON and -protobuf (e.g. float handling), some changes were made to the `Arbitrary` -implementation for `Event` to give simpler values. These are not changes we want -in most property testing scenarios, but they are appropriate in this case where -we only care about the overall structure of the events. +protobuf (e.g. float handling), the `generate-fixtures` feature flag in +`vector-core` activates a stricter `Arbitrary` implementation for `Event` that +produces simpler, round-trip-safe f64 values and non-empty field names. These +changes are intentionally scoped to fixture generation and not used in regular +property testing. -There is currently a multi-step procedure to re-generate the data files. -There are two diffs committed to this directory: - - `vector_generate_fixtures.patch` - - `vrl_generate_fixtures.patch` +## Re-generating fixtures -The `vrl_` one must be applied to the vectordotdev/vrl repo. -The `vector_` one must be applied to the vector repo (you are here). +Both this repo and the VRL repo have a `generate-fixtures` feature flag that +activates fixture-stable `Arbitrary` implementations. The vector-core +`generate-fixtures` feature automatically enables `vrl/generate-fixtures`. -Part of the vector patch file is a `roundtrip` unit test definition that needs -to be evoked from `lib/vector-core`. Before invoking it, the `_json` and `_proto` -directories need to be created. +### Run the generator ```bash - $ cd lib/vector-core - $ mkdir _json/ proto/ - $ cargo test event::test::serialization::roundtrip +cargo run -p vector-core --features generate-fixtures --bin generate-fixtures ``` -That test case writes out the appropriate files into the dirs, which then need to be -moved to their location here. +The binary writes files directly into this directory's `json/` and `proto/` +subdirectories, replacing the existing fixtures. diff --git a/lib/codecs/tests/data/native_encoding/json/0605.json b/lib/codecs/tests/data/native_encoding/json/0605.json index bcd39f82ae418..1c46a13cf2fb7 100644 --- a/lib/codecs/tests/data/native_encoding/json/0605.json +++ b/lib/codecs/tests/data/native_encoding/json/0605.json @@ -1 +1 @@ -{"metric":{"name":"l","tags":{"a":"l","e":"l","q":"n"},"timestamp":"1970-01-01T07:31:02.000011250Z","kind":"absolute","sketch":{"sketch":{"AgentDDSketch":{"bins":{"k":[],"n":[]},"count":0,"min":1.7976931348623157e308,"max":-1.7976931348623157e308,"sum":-209216.0,"avg":-113920.0}}}}} \ No newline at end of file +{"metric":{"name":"l","tags":{"a":"l","e":"l","q":"n"},"timestamp":"1970-01-01T07:31:02.000011250Z","kind":"absolute","sketch":{"sketch":{"AgentDDSketch":{"bins":{"k":[],"n":[]},"count":0,"min":1.7976931348623157e+308,"max":-1.7976931348623157e+308,"sum":-209216.0,"avg":-113920.0}}}}} \ No newline at end of file diff --git a/lib/codecs/tests/data/native_encoding/json/0638.json b/lib/codecs/tests/data/native_encoding/json/0638.json index 61f4ec83d0fb4..85fb25087875f 100644 --- a/lib/codecs/tests/data/native_encoding/json/0638.json +++ b/lib/codecs/tests/data/native_encoding/json/0638.json @@ -1 +1 @@ -{"metric":{"name":"t","namespace":"k","tags":{"s":"b","x":"y"},"timestamp":"1969-12-31T19:41:49.000002928Z","interval_ms":2984613283,"kind":"absolute","sketch":{"sketch":{"AgentDDSketch":{"bins":{"k":[],"n":[]},"count":0,"min":1.7976931348623157e308,"max":-1.7976931348623157e308,"sum":-667968.0,"avg":-404160.0}}}}} \ No newline at end of file +{"metric":{"name":"t","namespace":"k","tags":{"s":"b","x":"y"},"timestamp":"1969-12-31T19:41:49.000002928Z","interval_ms":2984613283,"kind":"absolute","sketch":{"sketch":{"AgentDDSketch":{"bins":{"k":[],"n":[]},"count":0,"min":1.7976931348623157e+308,"max":-1.7976931348623157e+308,"sum":-667968.0,"avg":-404160.0}}}}} \ No newline at end of file diff --git a/lib/codecs/tests/data/native_encoding/json/0639.json b/lib/codecs/tests/data/native_encoding/json/0639.json index 3f8fd591f1128..9bbe86b1aa695 100644 --- a/lib/codecs/tests/data/native_encoding/json/0639.json +++ b/lib/codecs/tests/data/native_encoding/json/0639.json @@ -1 +1 @@ -{"metric":{"name":"y","namespace":"d","kind":"incremental","sketch":{"sketch":{"AgentDDSketch":{"bins":{"k":[],"n":[]},"count":0,"min":1.7976931348623157e308,"max":-1.7976931348623157e308,"sum":-422464.0,"avg":906752.0}}}}} \ No newline at end of file +{"metric":{"name":"y","namespace":"d","kind":"incremental","sketch":{"sketch":{"AgentDDSketch":{"bins":{"k":[],"n":[]},"count":0,"min":1.7976931348623157e+308,"max":-1.7976931348623157e+308,"sum":-422464.0,"avg":906752.0}}}}} \ No newline at end of file diff --git a/lib/codecs/tests/data/native_encoding/json/0647.json b/lib/codecs/tests/data/native_encoding/json/0647.json index 8dbe86fb16fdd..c6e6cdb1dfbab 100644 --- a/lib/codecs/tests/data/native_encoding/json/0647.json +++ b/lib/codecs/tests/data/native_encoding/json/0647.json @@ -1 +1 @@ -{"metric":{"name":"b","interval_ms":586459638,"kind":"incremental","sketch":{"sketch":{"AgentDDSketch":{"bins":{"k":[],"n":[]},"count":0,"min":1.7976931348623157e308,"max":-1.7976931348623157e308,"sum":999808.0,"avg":-148288.0}}}}} \ No newline at end of file +{"metric":{"name":"b","interval_ms":586459638,"kind":"incremental","sketch":{"sketch":{"AgentDDSketch":{"bins":{"k":[],"n":[]},"count":0,"min":1.7976931348623157e+308,"max":-1.7976931348623157e+308,"sum":999808.0,"avg":-148288.0}}}}} \ No newline at end of file diff --git a/lib/codecs/tests/data/native_encoding/json/0700.json b/lib/codecs/tests/data/native_encoding/json/0700.json index 42fdfec5980f1..da5055a337409 100644 --- a/lib/codecs/tests/data/native_encoding/json/0700.json +++ b/lib/codecs/tests/data/native_encoding/json/0700.json @@ -1 +1 @@ -{"metric":{"name":"j","tags":{"f":"k"},"timestamp":"1969-12-31T18:05:17.000026769Z","kind":"absolute","sketch":{"sketch":{"AgentDDSketch":{"bins":{"k":[],"n":[]},"count":0,"min":1.7976931348623157e308,"max":-1.7976931348623157e308,"sum":-481728.0,"avg":590528.0}}}}} \ No newline at end of file +{"metric":{"name":"j","tags":{"f":"k"},"timestamp":"1969-12-31T18:05:17.000026769Z","kind":"absolute","sketch":{"sketch":{"AgentDDSketch":{"bins":{"k":[],"n":[]},"count":0,"min":1.7976931348623157e+308,"max":-1.7976931348623157e+308,"sum":-481728.0,"avg":590528.0}}}}} \ No newline at end of file diff --git a/lib/codecs/tests/data/native_encoding/json/0788.json b/lib/codecs/tests/data/native_encoding/json/0788.json index f458d97be6718..13f2d0913ec85 100644 --- a/lib/codecs/tests/data/native_encoding/json/0788.json +++ b/lib/codecs/tests/data/native_encoding/json/0788.json @@ -1 +1 @@ -{"metric":{"name":"w","interval_ms":870813348,"kind":"incremental","sketch":{"sketch":{"AgentDDSketch":{"bins":{"k":[],"n":[]},"count":0,"min":1.7976931348623157e308,"max":-1.7976931348623157e308,"sum":-356992.0,"avg":-572800.0}}}}} \ No newline at end of file +{"metric":{"name":"w","interval_ms":870813348,"kind":"incremental","sketch":{"sketch":{"AgentDDSketch":{"bins":{"k":[],"n":[]},"count":0,"min":1.7976931348623157e+308,"max":-1.7976931348623157e+308,"sum":-356992.0,"avg":-572800.0}}}}} \ No newline at end of file diff --git a/lib/codecs/tests/data/native_encoding/vector_generate_fixtures.patch b/lib/codecs/tests/data/native_encoding/vector_generate_fixtures.patch deleted file mode 100644 index a0262928b3b3b..0000000000000 --- a/lib/codecs/tests/data/native_encoding/vector_generate_fixtures.patch +++ /dev/null @@ -1,163 +0,0 @@ -diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml -index 0f51fc830..aadf99841 100644 ---- a/lib/vector-core/Cargo.toml -+++ b/lib/vector-core/Cargo.toml -@@ -95,7 +95,7 @@ rand = "0.8.5" - rand_distr = "0.4.3" - tracing-subscriber = { version = "0.3.17", default-features = false, features = ["env-filter", "fmt", "ansi", "registry"] } - vector-common = { path = "../vector-common", default-features = false, features = ["test"] } --vrl = { version = "0.9.0", features = ["cli", "test", "test_framework", "arbitrary"] } -+vrl = { path = "../../../vrl", features = ["cli", "test", "test_framework", "arbitrary"] } - - [features] - api = ["dep:async-graphql"] -diff --git a/lib/vector-core/src/event/test/common.rs b/lib/vector-core/src/event/test/common.rs -index c7ccca952..f851288ce 100644 ---- a/lib/vector-core/src/event/test/common.rs -+++ b/lib/vector-core/src/event/test/common.rs -@@ -26,6 +26,15 @@ const ALPHABET: [&str; 27] = [ - "t", "u", "v", "w", "x", "y", "z", "_", - ]; - -+fn make_simple_f64(g: &mut Gen) -> f64 { -+ let mut value = f64::arbitrary(g) % MAX_F64_SIZE; -+ while value.is_nan() || value == -0.0 { -+ value = f64::arbitrary(g) % MAX_F64_SIZE; -+ } -+ value = (value * 10_000.0).round() / 10_000.0; -+ value -+} -+ - #[derive(Debug, Clone)] - pub struct Name { - inner: String, -@@ -34,7 +43,7 @@ pub struct Name { - impl Arbitrary for Name { - fn arbitrary(g: &mut Gen) -> Self { - let mut name = String::with_capacity(MAX_STR_SIZE); -- for _ in 0..(g.size() % MAX_STR_SIZE) { -+ for _ in 0..(usize::max(1, g.size() % MAX_STR_SIZE)) { - let idx: usize = usize::arbitrary(g) % ALPHABET.len(); - name.push_str(ALPHABET[idx]); - } -@@ -182,10 +191,10 @@ impl Arbitrary for MetricValue { - // here toward `MetricValue::Counter` and `MetricValue::Gauge`. - match u8::arbitrary(g) % 7 { - 0 => MetricValue::Counter { -- value: f64::arbitrary(g) % MAX_F64_SIZE, -+ value: make_simple_f64(g), - }, - 1 => MetricValue::Gauge { -- value: f64::arbitrary(g) % MAX_F64_SIZE, -+ value: make_simple_f64(g), - }, - 2 => MetricValue::Set { - values: BTreeSet::arbitrary(g), -@@ -197,19 +206,20 @@ impl Arbitrary for MetricValue { - 4 => MetricValue::AggregatedHistogram { - buckets: Vec::arbitrary(g), - count: u64::arbitrary(g), -- sum: f64::arbitrary(g) % MAX_F64_SIZE, -+ sum: make_simple_f64(g), - }, - 5 => MetricValue::AggregatedSummary { - quantiles: Vec::arbitrary(g), - count: u64::arbitrary(g), -- sum: f64::arbitrary(g) % MAX_F64_SIZE, -+ sum: make_simple_f64(g), - }, - 6 => { - // We're working around quickcheck's limitations here, and - // should really migrate the tests in question to use proptest - let num_samples = u8::arbitrary(g); - let samples = std::iter::repeat_with(|| loop { -- let f = f64::arbitrary(g); -+ // let f = f64::arbitrary(g); -+ let f = make_simple_f64(g); - if f.is_normal() { - return f; - } -@@ -219,6 +229,8 @@ impl Arbitrary for MetricValue { - - let mut sketch = AgentDDSketch::with_agent_defaults(); - sketch.insert_many(&samples); -+ sketch.sum = make_simple_f64(g); -+ sketch.avg = make_simple_f64(g); - - MetricValue::Sketch { - sketch: MetricSketch::AgentDDSketch(sketch), -@@ -368,7 +380,7 @@ impl Arbitrary for MetricValue { - impl Arbitrary for Sample { - fn arbitrary(g: &mut Gen) -> Self { - Sample { -- value: f64::arbitrary(g) % MAX_F64_SIZE, -+ value: make_simple_f64(g), - rate: u32::arbitrary(g), - } - } -@@ -398,8 +410,8 @@ impl Arbitrary for Sample { - impl Arbitrary for Quantile { - fn arbitrary(g: &mut Gen) -> Self { - Quantile { -- quantile: f64::arbitrary(g) % MAX_F64_SIZE, -- value: f64::arbitrary(g) % MAX_F64_SIZE, -+ quantile: make_simple_f64(g), -+ value: make_simple_f64(g), - } - } - -@@ -428,7 +440,7 @@ impl Arbitrary for Quantile { - impl Arbitrary for Bucket { - fn arbitrary(g: &mut Gen) -> Self { - Bucket { -- upper_limit: f64::arbitrary(g) % MAX_F64_SIZE, -+ upper_limit: make_simple_f64(g), - count: u64::arbitrary(g), - } - } -diff --git a/lib/vector-core/src/event/test/serialization.rs b/lib/vector-core/src/event/test/serialization.rs -index aaab559da..5db6c0613 100644 ---- a/lib/vector-core/src/event/test/serialization.rs -+++ b/lib/vector-core/src/event/test/serialization.rs -@@ -96,3 +96,24 @@ fn type_serialization() { - assert_eq!(map["bool"], json!(true)); - assert_eq!(map["string"], json!("thisisastring")); - } -+ -+#[test] -+fn roundtrip() { -+ use prost::Message; -+ use quickcheck::{Arbitrary, Gen}; -+ use std::{fs::File, io::Write}; -+ -+ let mut gen = Gen::new(128); -+ for n in 0..1024 { -+ let mut json_out = File::create(format!("_json/{n:04}.json")).unwrap(); -+ let mut proto_out = File::create(format!("_proto/{n:04}.pb")).unwrap(); -+ let event = Event::arbitrary(&mut gen); -+ serde_json::to_writer(&mut json_out, &event).unwrap(); -+ -+ let array = EventArray::from(event); -+ let proto = proto::EventArray::from(array); -+ let mut buf = BytesMut::new(); -+ proto.encode(&mut buf).unwrap(); -+ proto_out.write_all(&buf).unwrap(); -+ } -+} -diff --git a/lib/vector-core/src/metrics/ddsketch.rs b/lib/vector-core/src/metrics/ddsketch.rs -index 3d0f80bb5..fcc6bcc97 100644 ---- a/lib/vector-core/src/metrics/ddsketch.rs -+++ b/lib/vector-core/src/metrics/ddsketch.rs -@@ -229,10 +229,10 @@ pub struct AgentDDSketch { - max: f64, - - /// The sum of all observations within the sketch. -- sum: f64, -+ pub sum: f64, - - /// The average value of all observations within the sketch. -- avg: f64, -+ pub avg: f64, - } - - impl AgentDDSketch { diff --git a/lib/codecs/tests/data/native_encoding/vrl_generate_fixtures.patch b/lib/codecs/tests/data/native_encoding/vrl_generate_fixtures.patch deleted file mode 100644 index 430b74abb8b66..0000000000000 --- a/lib/codecs/tests/data/native_encoding/vrl_generate_fixtures.patch +++ /dev/null @@ -1,69 +0,0 @@ -diff --git a/src/lib.rs b/src/lib.rs -index 29cad8dfc..0b166beb2 100644 ---- a/src/lib.rs -+++ b/src/lib.rs -@@ -1,4 +1,4 @@ --#![deny(warnings)] -+// #![deny(warnings)] - #![deny(clippy::all)] - #![deny(unused_allocation)] - #![deny(unused_extern_crates)] -diff --git a/src/value/value/arbitrary.rs b/src/value/value/arbitrary.rs -index dde213281..33b2f2518 100644 ---- a/src/value/value/arbitrary.rs -+++ b/src/value/value/arbitrary.rs -@@ -1,7 +1,6 @@ - use std::collections::BTreeMap; - - use bytes::Bytes; --use chrono::{DateTime, NaiveDateTime, Utc}; - use ordered_float::NotNan; - use quickcheck::{Arbitrary, Gen}; - -@@ -11,16 +10,13 @@ const MAX_ARRAY_SIZE: usize = 4; - const MAX_MAP_SIZE: usize = 4; - const MAX_F64_SIZE: f64 = 1_000_000.0; - --fn datetime(g: &mut Gen) -> DateTime { -- // `chrono` documents that there is an out-of-range for both second and -- // nanosecond values but doesn't actually document what the valid ranges -- // are. We just sort of arbitrarily restrict things. -- let secs = i64::arbitrary(g) % 32_000; -- let nanosecs = u32::arbitrary(g) % 32_000; -- DateTime::::from_utc( -- NaiveDateTime::from_timestamp_opt(secs, nanosecs).expect("invalid timestamp"), -- Utc, -- ) -+fn make_simple_f64(g: &mut Gen) -> f64 { -+ let mut value = f64::arbitrary(g) % MAX_F64_SIZE; -+ while value.is_nan() || value == -0.0 { -+ value = f64::arbitrary(g) % MAX_F64_SIZE; -+ } -+ value = (value * 10_000.0).round() / 10_000.0; -+ value - } - - impl Arbitrary for Value { -@@ -32,18 +28,18 @@ impl Arbitrary for Value { - // field picking. - match u8::arbitrary(g) % 8 { - 0 => { -- let bytes: Vec = Vec::arbitrary(g); -+ let bytes = String::arbitrary(g); - Self::Bytes(Bytes::from(bytes)) - } - 1 => Self::Integer(i64::arbitrary(g)), - 2 => { -- let f = f64::arbitrary(g) % MAX_F64_SIZE; -+ //let f = f64::arbitrary(g) % MAX_F64_SIZE; -+ let f = make_simple_f64(g); - let not_nan = NotNan::new(f).unwrap_or_else(|_| NotNan::new(0.0).unwrap()); - Self::from(not_nan) - } - 3 => Self::Boolean(bool::arbitrary(g)), -- 4 => Self::Timestamp(datetime(g)), -- 5 => { -+ 4 | 5 => { - let mut gen = Gen::new(MAX_MAP_SIZE); - Self::Object(BTreeMap::arbitrary(&mut gen)) - } diff --git a/lib/dnsmsg-parser/Cargo.toml b/lib/dnsmsg-parser/Cargo.toml index 9861bd93718a8..b392579014ef1 100644 --- a/lib/dnsmsg-parser/Cargo.toml +++ b/lib/dnsmsg-parser/Cargo.toml @@ -15,7 +15,7 @@ hickory-proto.workspace = true snafu.workspace = true [dev-dependencies] -criterion = "0.7" +criterion.workspace = true [lib] bench = false diff --git a/lib/dnsmsg-parser/src/dns_message_parser.rs b/lib/dnsmsg-parser/src/dns_message_parser.rs index d0b271b2e7930..a22b630ffaca4 100644 --- a/lib/dnsmsg-parser/src/dns_message_parser.rs +++ b/lib/dnsmsg-parser/src/dns_message_parser.rs @@ -7,16 +7,13 @@ use hickory_proto::{ PublicKey, SupportedAlgorithms, Verifier, rdata::{CDNSKEY, CDS, DNSKEY, DNSSECRData, DS}, }, - op::{Query, message::Message as TrustDnsMessage}, + op::{Message as TrustDnsMessage, Query}, rr::{ - Name, RecordType, + Name, RData, Record, RecordType, rdata::{ A, AAAA, NULL, OPT, SVCB, - caa::Property, opt::{EdnsCode, EdnsOption}, }, - record_data::RData, - resource::Record, }, serialize::binary::{BinDecodable, BinDecoder}, }; @@ -101,8 +98,11 @@ impl DnsMessageParser { } pub fn parse_as_query_message(&mut self) -> DnsParserResult { - let msg = TrustDnsMessage::from_vec(&self.raw_message) - .map_err(|source| DnsMessageParserError::TrustDnsError { source })?; + let msg = TrustDnsMessage::from_vec(&self.raw_message).map_err(|source| { + DnsMessageParserError::TrustDnsError { + source: ProtoError::from(source), + } + })?; let header = parse_dns_query_message_header(&msg); let edns_section = parse_edns(&msg).transpose()?; let rcode_high = edns_section.as_ref().map_or(0, |edns| edns.extended_rcode); @@ -113,16 +113,19 @@ impl DnsMessageParser { response: parse_response_code(response_code), header, question_section: self.parse_dns_query_message_question_section(&msg), - answer_section: self.parse_dns_message_section(msg.answers())?, - authority_section: self.parse_dns_message_section(msg.name_servers())?, - additional_section: self.parse_dns_message_section(msg.additionals())?, + answer_section: self.parse_dns_message_section(&msg.answers)?, + authority_section: self.parse_dns_message_section(&msg.authorities)?, + additional_section: self.parse_dns_message_section(&msg.additionals)?, opt_pseudo_section: edns_section, }) } pub fn parse_as_update_message(&mut self) -> DnsParserResult { - let msg = TrustDnsMessage::from_vec(&self.raw_message) - .map_err(|source| DnsMessageParserError::TrustDnsError { source })?; + let msg = TrustDnsMessage::from_vec(&self.raw_message).map_err(|source| { + DnsMessageParserError::TrustDnsError { + source: ProtoError::from(source), + } + })?; let header = parse_dns_update_message_header(&msg); let response_code = (u16::from(header.rcode)) & 0x000F; Ok(DnsUpdateMessage { @@ -130,9 +133,9 @@ impl DnsMessageParser { response: parse_response_code(response_code), header, zone_to_update: self.parse_dns_update_message_zone_section(&msg)?, - prerequisite_section: self.parse_dns_message_section(msg.answers())?, - update_section: self.parse_dns_message_section(msg.name_servers())?, - additional_section: self.parse_dns_message_section(msg.additionals())?, + prerequisite_section: self.parse_dns_message_section(&msg.answers)?, + update_section: self.parse_dns_message_section(&msg.authorities)?, + additional_section: self.parse_dns_message_section(&msg.additionals)?, }) } @@ -141,7 +144,7 @@ impl DnsMessageParser { dns_message: &TrustDnsMessage, ) -> Vec { dns_message - .queries() + .queries .iter() .map(|query| self.parse_dns_query_question(query)) .collect() @@ -161,7 +164,7 @@ impl DnsMessageParser { dns_message: &TrustDnsMessage, ) -> DnsParserResult { let zones = dns_message - .queries() + .queries .iter() .map(|query| self.parse_dns_query_question(query).into()) .collect::>(); @@ -185,18 +188,18 @@ impl DnsMessageParser { } pub(crate) fn parse_dns_record(&mut self, record: &Record) -> DnsParserResult { - let record_data = match record.data() { - RData::Unknown { code, rdata } => self.format_unknown_rdata((*code).into(), rdata), + let record_data = match &record.data { + RData::Unknown { code, rdata } => self.format_unknown_rdata(u16::from(*code), rdata), RData::Update0(_) => Ok((Some(String::from("")), None)), // Previously none value rdata => self.format_rdata(rdata), }?; Ok(DnsRecord { - name: record.name().to_string_with_options(&self.options), - class: record.dns_class().to_string(), + name: record.name.to_string_with_options(&self.options), + class: record.dns_class.to_string(), record_type: format_record_type(record.record_type()), record_type_id: u16::from(record.record_type()), - ttl: record.ttl(), + ttl: record.ttl, rdata: record_data.0, rdata_bytes: record_data.1, }) @@ -381,30 +384,30 @@ impl DnsMessageParser { match code { dns_message::RTYPE_MB => { let options = self.options.clone(); - let mut decoder = self.get_rdata_decoder_with_raw_message(rdata.anything()); + let mut decoder = self.get_rdata_decoder_with_raw_message(&rdata.anything); let madname = Self::parse_domain_name(&mut decoder, &options)?; Ok((Some(madname), None)) } dns_message::RTYPE_MG => { let options = self.options.clone(); - let mut decoder = self.get_rdata_decoder_with_raw_message(rdata.anything()); + let mut decoder = self.get_rdata_decoder_with_raw_message(&rdata.anything); let mgname = Self::parse_domain_name(&mut decoder, &options)?; Ok((Some(mgname), None)) } dns_message::RTYPE_MR => { let options = self.options.clone(); - let mut decoder = self.get_rdata_decoder_with_raw_message(rdata.anything()); + let mut decoder = self.get_rdata_decoder_with_raw_message(&rdata.anything); let newname = Self::parse_domain_name(&mut decoder, &options)?; Ok((Some(newname), None)) } - dns_message::RTYPE_WKS => self.parse_wks_rdata(rdata.anything()), + dns_message::RTYPE_WKS => self.parse_wks_rdata(&rdata.anything), dns_message::RTYPE_MINFO => { let options = self.options.clone(); - let mut decoder = self.get_rdata_decoder_with_raw_message(rdata.anything()); + let mut decoder = self.get_rdata_decoder_with_raw_message(&rdata.anything); let rmailbx = Self::parse_domain_name(&mut decoder, &options)?; let emailbx = Self::parse_domain_name(&mut decoder, &options)?; Ok((Some(format!("{rmailbx} {emailbx}")), None)) @@ -412,7 +415,7 @@ impl DnsMessageParser { dns_message::RTYPE_RP => { let options = self.options.clone(); - let mut decoder = self.get_rdata_decoder_with_raw_message(rdata.anything()); + let mut decoder = self.get_rdata_decoder_with_raw_message(&rdata.anything); let mbox = Self::parse_domain_name(&mut decoder, &options)?; let txt = Self::parse_domain_name(&mut decoder, &options)?; Ok((Some(format!("{mbox} {txt}")), None)) @@ -420,14 +423,14 @@ impl DnsMessageParser { dns_message::RTYPE_AFSDB => { let options = self.options.clone(); - let mut decoder = self.get_rdata_decoder_with_raw_message(rdata.anything()); + let mut decoder = self.get_rdata_decoder_with_raw_message(&rdata.anything); let subtype = parse_u16(&mut decoder)?; let hostname = Self::parse_domain_name(&mut decoder, &options)?; Ok((Some(format!("{subtype} {hostname}")), None)) } dns_message::RTYPE_X25 => { - let mut decoder = BinDecoder::new(rdata.anything()); + let mut decoder = BinDecoder::new(&rdata.anything); let psdn_address = parse_character_string(&mut decoder)?; Ok(( Some(format!( @@ -439,7 +442,7 @@ impl DnsMessageParser { } dns_message::RTYPE_ISDN => { - let mut decoder = BinDecoder::new(rdata.anything()); + let mut decoder = BinDecoder::new(&rdata.anything); let address = parse_character_string(&mut decoder)?; if decoder.is_empty() { Ok(( @@ -464,14 +467,14 @@ impl DnsMessageParser { dns_message::RTYPE_RT => { let options = self.options.clone(); - let mut decoder = self.get_rdata_decoder_with_raw_message(rdata.anything()); + let mut decoder = self.get_rdata_decoder_with_raw_message(&rdata.anything); let preference = parse_u16(&mut decoder)?; let intermediate_host = Self::parse_domain_name(&mut decoder, &options)?; Ok((Some(format!("{preference} {intermediate_host}")), None)) } dns_message::RTYPE_NSAP => { - let raw_rdata = rdata.anything(); + let raw_rdata = &rdata.anything; let mut decoder = BinDecoder::new(raw_rdata); let rdata_len = raw_rdata.len() as u16; let nsap_rdata = HEXUPPER.encode(&parse_vec_with_u16_len(&mut decoder, rdata_len)?); @@ -480,27 +483,27 @@ impl DnsMessageParser { dns_message::RTYPE_PX => { let options = self.options.clone(); - let mut decoder = self.get_rdata_decoder_with_raw_message(rdata.anything()); + let mut decoder = self.get_rdata_decoder_with_raw_message(&rdata.anything); let preference = parse_u16(&mut decoder)?; let map822 = Self::parse_domain_name(&mut decoder, &options)?; let mapx400 = Self::parse_domain_name(&mut decoder, &options)?; Ok((Some(format!("{preference} {map822} {mapx400}")), None)) } - dns_message::RTYPE_LOC => self.parse_loc_rdata(rdata.anything()), + dns_message::RTYPE_LOC => self.parse_loc_rdata(&rdata.anything), dns_message::RTYPE_KX => { let options = self.options.clone(); - let mut decoder = self.get_rdata_decoder_with_raw_message(rdata.anything()); + let mut decoder = self.get_rdata_decoder_with_raw_message(&rdata.anything); let preference = parse_u16(&mut decoder)?; let exchanger = Self::parse_domain_name(&mut decoder, &options)?; Ok((Some(format!("{preference} {exchanger}")), None)) } - dns_message::RTYPE_A6 => self.parse_a6_rdata(rdata.anything()), + dns_message::RTYPE_A6 => self.parse_a6_rdata(&rdata.anything), dns_message::RTYPE_SINK => { - let raw_rdata = rdata.anything(); + let raw_rdata = &rdata.anything; let mut decoder = BinDecoder::new(raw_rdata); let meaning = parse_u8(&mut decoder)?; let coding = parse_u8(&mut decoder)?; @@ -511,10 +514,10 @@ impl DnsMessageParser { Ok((Some(format!("{meaning} {coding} {subcoding} {data}")), None)) } - dns_message::RTYPE_APL => self.parse_apl_rdata(rdata.anything()), + dns_message::RTYPE_APL => self.parse_apl_rdata(&rdata.anything), dns_message::RTYPE_DHCID => { - let raw_rdata = rdata.anything(); + let raw_rdata = &rdata.anything; let mut decoder = BinDecoder::new(raw_rdata); let raw_data_len = raw_rdata.len() as u16; let digest = BASE64.encode(&parse_vec_with_u16_len(&mut decoder, raw_data_len)?); @@ -522,7 +525,7 @@ impl DnsMessageParser { } dns_message::RTYPE_SPF => { - let mut decoder = BinDecoder::new(rdata.anything()); + let mut decoder = BinDecoder::new(&rdata.anything); let mut text = String::new(); while !decoder.is_empty() { text.push('\"'); @@ -532,7 +535,7 @@ impl DnsMessageParser { Ok((Some(text.trim_end().to_string()), None)) } - _ => Ok((None, Some(rdata.anything().to_vec()))), + _ => Ok((None, Some(rdata.anything.clone()))), } } @@ -543,13 +546,13 @@ impl DnsMessageParser { RData::ANAME(name) => Ok((Some(name.to_string_with_options(&self.options)), None)), RData::CNAME(name) => Ok((Some(name.to_string_with_options(&self.options)), None)), RData::CERT(cert) => { - let crl = BASE64.encode(&cert.cert_data()); + let crl = BASE64.encode(&cert.cert_data); Ok(( Some(format!( "{} {} {} {}", - u16::from(cert.cert_type()), - cert.key_tag(), - cert.algorithm(), + u16::from(cert.cert_type), + cert.key_tag, + cert.algorithm, crl )), None, @@ -564,15 +567,15 @@ impl DnsMessageParser { RData::MX(mx) => { let srv_rdata = format!( "{} {}", - mx.preference(), - mx.exchange().to_string_with_options(&self.options), + mx.preference, + mx.exchange.to_string_with_options(&self.options), ); Ok((Some(srv_rdata), None)) } - RData::NULL(null) => Ok((Some(BASE64.encode(null.anything())), None)), + RData::NULL(null) => Ok((Some(BASE64.encode(&null.anything)), None)), RData::NS(ns) => Ok((Some(ns.to_string_with_options(&self.options)), None)), RData::OPENPGPKEY(key) => { - if let Ok(key_string) = String::from_utf8(Vec::from(key.public_key())) { + if let Ok(key_string) = String::from_utf8(key.public_key.clone()) { Ok((Some(format!("({})", &key_string)), None)) } else { Err(DnsMessageParserError::SimpleError { @@ -584,29 +587,29 @@ impl DnsMessageParser { RData::SOA(soa) => Ok(( Some(format!( "{} {} {} {} {} {} {}", - soa.mname().to_string_with_options(&self.options), - soa.rname().to_string_with_options(&self.options), - soa.serial(), - soa.refresh(), - soa.retry(), - soa.expire(), - soa.minimum() + soa.mname.to_string_with_options(&self.options), + soa.rname.to_string_with_options(&self.options), + soa.serial, + soa.refresh, + soa.retry, + soa.expire, + soa.minimum )), None, )), RData::SRV(srv) => { let srv_rdata = format!( "{} {} {} {}", - srv.priority(), - srv.weight(), - srv.port(), - srv.target().to_string_with_options(&self.options) + srv.priority, + srv.weight, + srv.port, + srv.target.to_string_with_options(&self.options) ); Ok((Some(srv_rdata), None)) } RData::TXT(txt) => { let txt_rdata = txt - .txt_data() + .txt_data .iter() .map(|value| { format!( @@ -623,41 +626,35 @@ impl DnsMessageParser { RData::CAA(caa) => { let caa_rdata = format!( "{} {} \"{}\"", - caa.issuer_critical() as u8, - caa.tag().as_str(), - match caa.tag() { - Property::Iodef => { - let url = caa.value_as_iodef().map_err(|source| { - DnsMessageParserError::TrustDnsError { source } - })?; - url.as_str().to_string() - } - Property::Issue | Property::IssueWild => { - let (option_name, vec_keyvalue) = - caa.value_as_issue().map_err(|source| { - DnsMessageParserError::TrustDnsError { source } - })?; - - let mut final_issuer = String::new(); - if let Some(name) = option_name { - final_issuer.push_str(&name.to_string_with_options(&self.options)); - for keyvalue in vec_keyvalue.iter() { - final_issuer.push_str("; "); - final_issuer.push_str(keyvalue.key()); - final_issuer.push('='); - final_issuer.push_str(keyvalue.value()); - } + caa.issuer_critical as u8, + &caa.tag, + if caa.tag.eq_ignore_ascii_case("iodef") { + let url = caa + .value_as_iodef() + .map_err(|source| DnsMessageParserError::TrustDnsError { source })?; + url.as_str().to_string() + } else if caa.tag.eq_ignore_ascii_case("issue") + || caa.tag.eq_ignore_ascii_case("issuewild") + { + let (option_name, vec_keyvalue) = caa + .value_as_issue() + .map_err(|source| DnsMessageParserError::TrustDnsError { source })?; + + let mut final_issuer = String::new(); + if let Some(name) = option_name { + final_issuer.push_str(&name.to_string_with_options(&self.options)); + for keyvalue in vec_keyvalue.iter() { + final_issuer.push_str("; "); + final_issuer.push_str(keyvalue.key()); + final_issuer.push('='); + final_issuer.push_str(keyvalue.value()); } - final_issuer.trim_end().to_string() - } - Property::Unknown(_) => { - let unknown = caa.raw_value(); - std::str::from_utf8(unknown) - .map_err(|source| DnsMessageParserError::Utf8ParsingError { - source, - })? - .to_string() } + final_issuer.trim_end().to_string() + } else { + std::str::from_utf8(&caa.value) + .map_err(|source| DnsMessageParserError::Utf8ParsingError { source })? + .to_string() } ); Ok((Some(caa_rdata), None)) @@ -666,52 +663,52 @@ impl DnsMessageParser { RData::TLSA(tlsa) => { let tlsa_rdata = format!( "{} {} {} {}", - u8::from(tlsa.cert_usage()), - u8::from(tlsa.selector()), - u8::from(tlsa.matching()), - HEXUPPER.encode(tlsa.cert_data()) + u8::from(tlsa.cert_usage), + u8::from(tlsa.selector), + u8::from(tlsa.matching), + HEXUPPER.encode(&tlsa.cert_data) ); Ok((Some(tlsa_rdata), None)) } RData::SSHFP(sshfp) => { let sshfp_rdata = format!( "{} {} {}", - Into::::into(sshfp.algorithm()), - Into::::into(sshfp.fingerprint_type()), - HEXUPPER.encode(sshfp.fingerprint()) + Into::::into(sshfp.algorithm), + Into::::into(sshfp.fingerprint_type), + HEXUPPER.encode(&sshfp.fingerprint) ); Ok((Some(sshfp_rdata), None)) } RData::NAPTR(naptr) => { let naptr_rdata = format!( r#"{} {} "{}" "{}" "{}" {}"#, - naptr.order(), - naptr.preference(), + naptr.order, + naptr.preference, escape_string_for_text_representation( - std::str::from_utf8(naptr.flags()) + std::str::from_utf8(&naptr.flags) .map_err(|source| DnsMessageParserError::Utf8ParsingError { source })? .to_string() ), escape_string_for_text_representation( - std::str::from_utf8(naptr.services()) + std::str::from_utf8(&naptr.services) .map_err(|source| DnsMessageParserError::Utf8ParsingError { source })? .to_string() ), escape_string_for_text_representation( - std::str::from_utf8(naptr.regexp()) + std::str::from_utf8(&naptr.regexp) .map_err(|source| DnsMessageParserError::Utf8ParsingError { source })? .to_string() ), - naptr.replacement().to_string_with_options(&self.options) + naptr.replacement.to_string_with_options(&self.options) ); Ok((Some(naptr_rdata), None)) } RData::HINFO(hinfo) => { let hinfo_data = format!( r#""{}" "{}""#, - std::str::from_utf8(hinfo.cpu()) + std::str::from_utf8(&hinfo.cpu) .map_err(|source| DnsMessageParserError::Utf8ParsingError { source })?, - std::str::from_utf8(hinfo.os()) + std::str::from_utf8(&hinfo.os) .map_err(|source| DnsMessageParserError::Utf8ParsingError { source })?, ); Ok((Some(hinfo_data), None)) @@ -794,37 +791,41 @@ impl DnsMessageParser { DNSSECRData::SIG(sig) => { let sig_rdata = format!( "{} {} {} {} {} {} {} {} {}", - match format_record_type(sig.type_covered()) { + match format_record_type(sig.input().type_covered) { Some(record_type) => record_type, None => String::from("Unknown record type"), }, - u8::from(sig.algorithm()), - sig.num_labels(), - sig.original_ttl(), - sig.sig_expiration().get(), // currently in epoch convert to human readable ? - sig.sig_inception().get(), // currently in epoch convert to human readable ? - sig.key_tag(), - sig.signer_name().to_string_with_options(&self.options), + u8::from(sig.input().algorithm), + sig.input().num_labels, + sig.input().original_ttl, + sig.input().sig_expiration.get(), // currently in epoch convert to human readable ? + sig.input().sig_inception.get(), // currently in epoch convert to human readable ? + sig.input().key_tag, + sig.input() + .signer_name + .to_string_with_options(&self.options), BASE64.encode(sig.sig()) ); Ok((Some(sig_rdata), None)) } - // RSIG is a derivation of SIG but choosing to keep this duplicate code in lieu of the alternative + // RRSIG is a derivation of SIG but choosing to keep this duplicate code in lieu of the alternative // which is to allocate to the heap with Box in order to deref. DNSSECRData::RRSIG(sig) => { let sig_rdata = format!( "{} {} {} {} {} {} {} {} {}", - match format_record_type(sig.type_covered()) { + match format_record_type(sig.input().type_covered) { Some(record_type) => record_type, None => String::from("Unknown record type"), }, - u8::from(sig.algorithm()), - sig.num_labels(), - sig.original_ttl(), - sig.sig_expiration().get(), // currently in epoch convert to human readable ? - sig.sig_inception().get(), // currently in epoch convert to human readable ? - sig.key_tag(), - sig.signer_name().to_string_with_options(&self.options), + u8::from(sig.input().algorithm), + sig.input().num_labels, + sig.input().original_ttl, + sig.input().sig_expiration.get(), // currently in epoch convert to human readable ? + sig.input().sig_inception.get(), // currently in epoch convert to human readable ? + sig.input().key_tag, + sig.input() + .signer_name + .to_string_with_options(&self.options), BASE64.encode(sig.sig()) ); Ok((Some(sig_rdata), None)) @@ -839,9 +840,7 @@ impl DnsMessageParser { ); Ok((Some(key_rdata), None)) } - DNSSECRData::Unknown { code: _, rdata } => { - Ok((None, Some(rdata.anything().to_vec()))) - } + DNSSECRData::Unknown { code: _, rdata } => Ok((None, Some(rdata.anything.clone()))), _ => Err(DnsMessageParserError::SimpleError { cause: format!("Unsupported rdata {rdata:?}"), }), @@ -870,9 +869,9 @@ fn format_record_type(record_type: RecordType) -> Option { fn format_svcb_record(svcb: &SVCB, options: &DnsParserOptions) -> String { format!( "{} {} {}", - svcb.svc_priority(), - svcb.target_name().to_string_with_options(options), - svcb.svc_params() + svcb.svc_priority, + svcb.target_name.to_string_with_options(options), + svcb.svc_params .iter() .map(|(key, value)| format!(r#"{}="{}""#, key, value.to_string().trim_end_matches(','))) .collect::>() @@ -965,38 +964,38 @@ fn parse_response_code(rcode: u16) -> Option<&'static str> { fn parse_dns_query_message_header(dns_message: &TrustDnsMessage) -> QueryHeader { QueryHeader { - id: dns_message.header().id(), - opcode: dns_message.header().op_code().into(), - rcode: dns_message.header().response_code(), - qr: dns_message.header().message_type() as u8, - aa: dns_message.header().authoritative(), - tc: dns_message.header().truncated(), - rd: dns_message.header().recursion_desired(), - ra: dns_message.header().recursion_available(), - ad: dns_message.header().authentic_data(), - cd: dns_message.header().checking_disabled(), - question_count: dns_message.header().query_count(), - answer_count: dns_message.header().answer_count(), - authority_count: dns_message.header().name_server_count(), - additional_count: dns_message.header().additional_count(), + id: dns_message.id, + opcode: dns_message.op_code.into(), + rcode: dns_message.response_code, + qr: dns_message.message_type as u8, + aa: dns_message.authoritative, + tc: dns_message.truncation, + rd: dns_message.recursion_desired, + ra: dns_message.recursion_available, + ad: dns_message.authentic_data, + cd: dns_message.checking_disabled, + question_count: dns_message.queries.len() as u16, + answer_count: dns_message.answers.len() as u16, + authority_count: dns_message.authorities.len() as u16, + additional_count: dns_message.additionals.len() as u16 + dns_message.edns.is_some() as u16, } } fn parse_dns_update_message_header(dns_message: &TrustDnsMessage) -> UpdateHeader { UpdateHeader { - id: dns_message.header().id(), - opcode: dns_message.header().op_code().into(), - rcode: dns_message.header().response_code(), - qr: dns_message.header().message_type() as u8, - zone_count: dns_message.header().query_count(), - prerequisite_count: dns_message.header().answer_count(), - update_count: dns_message.header().name_server_count(), - additional_count: dns_message.header().additional_count(), + id: dns_message.id, + opcode: dns_message.op_code.into(), + rcode: dns_message.response_code, + qr: dns_message.message_type as u8, + zone_count: dns_message.queries.len() as u16, + prerequisite_count: dns_message.answers.len() as u16, + update_count: dns_message.authorities.len() as u16, + additional_count: dns_message.additionals.len() as u16 + dns_message.edns.is_some() as u16, } } fn parse_edns(dns_message: &TrustDnsMessage) -> Option> { - dns_message.extensions().as_ref().map(|edns| { + dns_message.edns.as_ref().map(|edns| { parse_edns_options(edns.options()).map(|(ede, rest)| OptPseudoSection { extended_rcode: edns.rcode_high(), version: edns.version(), @@ -1014,10 +1013,11 @@ fn parse_edns_options(edns: &OPT) -> DnsParserResult<(Vec, Vec) -> DnsParserResult { Ok(::read(decoder) - .map_err(|source| DnsMessageParserError::TrustDnsError { source })? + .map_err(|source| DnsMessageParserError::TrustDnsError { + source: ProtoError::from(source), + })? .to_string()) } fn parse_ipv4_address(decoder: &mut BinDecoder<'_>) -> DnsParserResult { Ok(::read(decoder) - .map_err(|source| DnsMessageParserError::TrustDnsError { source })? + .map_err(|source| DnsMessageParserError::TrustDnsError { + source: ProtoError::from(source), + })? .to_string()) } fn parse_domain_name(decoder: &mut BinDecoder<'_>) -> DnsParserResult { - Name::read(decoder).map_err(|source| DnsMessageParserError::TrustDnsError { source }) + Name::read(decoder).map_err(|source| DnsMessageParserError::TrustDnsError { + source: ProtoError::from(source), + }) } fn escape_string_for_text_representation(original_string: String) -> String { @@ -1305,11 +1311,13 @@ mod tests { dnssec::{ Algorithm as DNSSEC_Algorithm, DigestType, Nsec3HashAlgorithm, PublicKeyBuf, rdata::{ - KEY, NSEC, NSEC3, NSEC3PARAM, RRSIG, SIG, + KEY, NSEC, NSEC3, NSEC3PARAM, RRSIG, key::{KeyTrust, KeyUsage, Protocol}, + sig::SigInput, }, }, rr::{ + SerialNumber, domain::Name, rdata::{ CAA, CERT, CSYNC, HINFO, HTTPS, NAPTR, OPT, SSHFP, TLSA, TXT, @@ -1435,7 +1443,10 @@ mod tests { .expect_err("Expected TrustDnsError."); match err { DnsMessageParserError::TrustDnsError { source: e } => { - assert_eq!(e.to_string(), "unexpected end of input reached") + assert_eq!( + e.to_string(), + "decoding error: unexpected end of input reached" + ) } DnsMessageParserError::SimpleError { cause: e } => { panic!("Expected TrustDnsError, got {}.", &e) @@ -1859,29 +1870,26 @@ mod tests { #[test] fn test_format_rdata_for_sig_type() { - let rdata = RData::DNSSEC(DNSSECRData::SIG(SIG::new( - RecordType::NULL, - DNSSEC_Algorithm::RSASHA256, + // SIG wire: type_covered=NULL(10), alg=8, labels=0, orig_ttl=0, expire=2, inception=1, + // keytag=5, signer=www.example.com, sig=[0..=31] + let mut wire: Vec = Vec::new(); + wire.extend_from_slice(&10u16.to_be_bytes()); + wire.push(8u8); + wire.push(0u8); + wire.extend_from_slice(&0u32.to_be_bytes()); + wire.extend_from_slice(&2u32.to_be_bytes()); + wire.extend_from_slice(&1u32.to_be_bytes()); + wire.extend_from_slice(&5u16.to_be_bytes()); + wire.extend_from_slice(&[ + 3, b'w', b'w', b'w', 7, b'e', b'x', b'a', b'm', b'p', b'l', b'e', 3, b'c', b'o', b'm', 0, - 0, - 2, - 1, - 5, - Name::from_str("www.example.com").unwrap(), - vec![ - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, - 23, 24, 25, 26, 27, 28, 29, 29, 31, - ], - ))); - let rdata_text = format_rdata(&rdata); - assert!(rdata_text.is_ok()); - if let Ok((parsed, raw_rdata)) = rdata_text { - assert!(raw_rdata.is_none()); - assert_eq!( - "NULL 8 0 0 2 1 5 www.example.com AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHR8=", - parsed.unwrap() - ); - } + ]); + wire.extend(0u8..=31); + test_format_rdata( + &BASE64.encode(&wire), + 24, + "NULL 8 0 0 2 1 5 www.example.com. AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=", + ); } #[test] @@ -1918,26 +1926,26 @@ mod tests { // so there isn't really a great way to reduce code duplication here. #[test] fn test_format_rdata_for_rsig_type() { - let rdata = RData::DNSSEC(DNSSECRData::RRSIG(RRSIG::new( - RecordType::NULL, - DNSSEC_Algorithm::RSASHA256, - 0, - 0, - 2, - 1, - 5, - Name::from_str("www.example.com").unwrap(), - vec![ - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, - 23, 24, 25, 26, 27, 28, 29, 29, 31, - ], + let input = SigInput { + type_covered: RecordType::NULL, + algorithm: DNSSEC_Algorithm::RSASHA256, + num_labels: 0, + original_ttl: 0, + sig_expiration: SerialNumber::new(2), + sig_inception: SerialNumber::new(1), + key_tag: 5, + signer_name: Name::from_str("www.example.com").unwrap(), + }; + let rdata = RData::DNSSEC(DNSSECRData::RRSIG(RRSIG::from_sig( + input, + (0u8..=31).collect(), ))); let rdata_text = format_rdata(&rdata); assert!(rdata_text.is_ok()); if let Ok((parsed, raw_rdata)) = rdata_text { assert!(raw_rdata.is_none()); assert_eq!( - "NULL 8 0 0 2 1 5 www.example.com AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHR8=", + "NULL 8 0 0 2 1 5 www.example.com AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=", parsed.unwrap() ); } diff --git a/lib/dnsmsg-parser/src/ede.rs b/lib/dnsmsg-parser/src/ede.rs index 0c3ee6c2f12e6..26bd22ed008ea 100644 --- a/lib/dnsmsg-parser/src/ede.rs +++ b/lib/dnsmsg-parser/src/ede.rs @@ -1,6 +1,6 @@ use hickory_proto::{ ProtoError, - serialize::binary::{BinDecodable, BinDecoder, BinEncodable, BinEncoder}, + serialize::binary::{BinDecodable, BinDecoder, BinEncodable, BinEncoder, DecodeError}, }; pub const EDE_OPTION_CODE: u16 = 15u16; @@ -77,14 +77,15 @@ impl BinEncodable for EDE { } impl<'a> BinDecodable<'a> for EDE { - fn read(decoder: &mut BinDecoder<'a>) -> Result { + fn read(decoder: &mut BinDecoder<'a>) -> Result { let info_code = decoder.read_u16()?.unverified(); let extra_text = if decoder.is_empty() { None } else { - Some(String::from_utf8( - decoder.read_vec(decoder.len())?.unverified(), - )?) + Some( + String::from_utf8(decoder.read_vec(decoder.len())?.unverified()) + .map_err(DecodeError::Utf8)?, + ) }; Ok(Self { info_code, diff --git a/lib/fakedata/Cargo.toml b/lib/fakedata/Cargo.toml index 708086e24002c..52ba22387ef89 100644 --- a/lib/fakedata/Cargo.toml +++ b/lib/fakedata/Cargo.toml @@ -8,5 +8,4 @@ license = "MPL-2.0" [dependencies] chrono.workspace = true -fakedata_generator = "0.5.0" rand.workspace = true diff --git a/lib/fakedata/src/logs.rs b/lib/fakedata/src/logs.rs index 1c3aa7683d2c4..768d67b06890d 100644 --- a/lib/fakedata/src/logs.rs +++ b/lib/fakedata/src/logs.rs @@ -3,9 +3,44 @@ use chrono::{ format::{DelayedFormat, StrftimeItems}, prelude::Local, }; -use fakedata_generator::{gen_domain, gen_ipv4, gen_username}; use rand::{Rng, rng}; +static FAKE_USERNAMES: [&str; 20] = [ + "log_whisperer", + "commit_conductor", + "cache_cowboy", + "compile_captain", + "latency_llama", + "yaml_yoda", + "regex_rider", + "semver_sage", + "kernel_keith", + "pixel_pilgrim", + "kubectl_kev", + "pipeline_pat", + "telemetry_tina", + "merge_maria", + "parser_pete", + "debug_duchess", + "nullable_nate", + "grep_greg", + "stderr_stan", + "segfault_sue", +]; + +static FAKE_DOMAIN_NAMES: [&str; 8] = [ + "acme", + "contoso", + "widgets", + "example", + "placeholder", + "sample", + "foobar", + "testbench", +]; + +static FAKE_DOMAIN_TLDS: [&str; 8] = ["com", "net", "org", "io", "dev", "co", "app", "biz"]; + static APPLICATION_NAMES: [&str; 10] = [ "auth", "data", "deploy", "etl", "scraper", "cron", "ingress", "egress", "alerter", "fwd", ]; @@ -156,7 +191,11 @@ fn application() -> &'static str { } fn domain() -> String { - gen_domain() + format!( + "{}.{}", + random_from_array(&FAKE_DOMAIN_NAMES), + random_from_array(&FAKE_DOMAIN_TLDS), + ) } fn error_level() -> &'static str { @@ -188,7 +227,14 @@ fn http_version() -> &'static str { } fn ipv4_address() -> String { - gen_ipv4() + let mut r = rng(); + format!( + "{}.{}.{}.{}", + r.random_range(1..255), + r.random_range(1..255), + r.random_range(1..255), + r.random_range(1..255), + ) } fn pid() -> usize { @@ -208,7 +254,7 @@ fn referer() -> String { } fn username() -> String { - gen_username() + (*random_from_array(&FAKE_USERNAMES)).to_string() } fn syslog_version() -> usize { diff --git a/lib/file-source-common/Cargo.toml b/lib/file-source-common/Cargo.toml index a53c52a27f6b8..f7a191d9cae7d 100644 --- a/lib/file-source-common/Cargo.toml +++ b/lib/file-source-common/Cargo.toml @@ -16,16 +16,16 @@ chrono.workspace = true tracing.workspace = true crc = "3.3.0" serde = { version = "1.0", default-features = false, features = ["derive"] } -serde_json = { version = "1.0.143", default-features = false } +serde_json.workspace = true bstr = { version = "1.12", default-features = false } bytes = { version = "1.11.1", default-features = false, features = ["serde"] } dashmap = { version = "6.1", default-features = false } -async-compression = { version = "0.4.41", features = ["tokio", "gzip"] } +async-compression = { version = "0.4.42", features = ["tokio", "gzip"] } vector-common = { path = "../vector-common", default-features = false } vector-config = { path = "../vector-config", default-features = false } tokio = { workspace = true, features = ["full"] } [dev-dependencies] -quickcheck = "1" +quickcheck.workspace = true tempfile.workspace = true similar-asserts = "1.7.0" diff --git a/lib/file-source-common/src/buffer.rs b/lib/file-source-common/src/buffer.rs index 097331fbf2be3..099d896f4e1e6 100644 --- a/lib/file-source-common/src/buffer.rs +++ b/lib/file-source-common/src/buffer.rs @@ -1,5 +1,5 @@ use crate::FilePosition; -use std::{cmp::min, io, pin::Pin}; +use std::{cmp::min, io}; use bstr::Finder; use bytes::BytesMut; @@ -34,7 +34,7 @@ pub struct ReadResult { /// GiB/s range for buffers of length 1KiB. For buffers any smaller than this /// the overhead of setup dominates our benchmarks. pub async fn read_until_with_max_size<'a, R: AsyncBufRead + ?Sized + Unpin>( - reader: Pin>, + reader: &'a mut R, position: &'a mut FilePosition, delim: &'a [u8], buf: &'a mut BytesMut, @@ -45,7 +45,6 @@ pub async fn read_until_with_max_size<'a, R: AsyncBufRead + ?Sized + Unpin>( let delim_finder = Finder::new(delim); let delim_len = delim.len(); let mut discarded_for_size_and_truncated = Vec::new(); - let mut reader = Box::new(reader); // Used to track partial delimiter matches across buffer boundaries. // Data is read in chunks from the reader (see `fill_buf` below). @@ -294,7 +293,7 @@ mod test { let mut reader = BufReader::new(Cursor::new(&chunk)); match read_until_with_max_size( - Box::pin(&mut reader), + &mut reader, &mut position, &delimiter, &mut buffer, @@ -425,7 +424,7 @@ mod test { for (i, expected_line) in expected_lines.iter().enumerate() { let mut buffer = BytesMut::new(); let result = read_until_with_max_size( - Box::pin(&mut reader), + &mut reader, &mut position, delimiter, &mut buffer, diff --git a/lib/file-source/Cargo.toml b/lib/file-source/Cargo.toml index 4b48b35b4f20e..a6b6e94c3b27f 100644 --- a/lib/file-source/Cargo.toml +++ b/lib/file-source/Cargo.toml @@ -21,9 +21,9 @@ futures = { version = "0.3.31", default-features = false, features = ["executor" futures-util.workspace = true vector-common = { path = "../vector-common", default-features = false } file-source-common = { path = "../file-source-common" } -async-compression = { version = "0.4.41", features = ["tokio", "gzip"] } +async-compression = { version = "0.4.42", features = ["tokio", "gzip"] } [dev-dependencies] tokio = { workspace = true, features = ["full"] } -quickcheck = "1" +quickcheck.workspace = true tempfile.workspace = true diff --git a/lib/file-source/src/file_server.rs b/lib/file-source/src/file_server.rs index 79f7273e3746c..17d80cf1586a4 100644 --- a/lib/file-source/src/file_server.rs +++ b/lib/file-source/src/file_server.rs @@ -148,7 +148,7 @@ where let mut stats = TimingStats::default(); // Spawn the checkpoint writer task - let checkpoint_task_handle = tokio::spawn(checkpoint_writer( + let checkpoint_task_handle = vector_common::spawn_in_current_span(checkpoint_writer( checkpointer, self.glob_minimum_cooldown, shutdown_checkpointer, @@ -411,7 +411,7 @@ where }; futures::pin_mut!(sleep); match select(shutdown_data, sleep).await { - Either::Left((_, _)) => { + Either::Left((_shutdown_token, _)) => { chans .close() .await @@ -423,6 +423,8 @@ where error!(?error, "Error writing checkpoints before shutdown"); } return Ok(Shutdown); + // _shutdown_token is dropped here, after checkpoints are written, + // which signals shutdown_done to the caller. } Either::Right((_, future)) => shutdown_data = future, } diff --git a/lib/file-source/src/file_watcher/mod.rs b/lib/file-source/src/file_watcher/mod.rs index b750a81433886..6e103cf13f39b 100644 --- a/lib/file-source/src/file_watcher/mod.rs +++ b/lib/file-source/src/file_watcher/mod.rs @@ -19,6 +19,9 @@ use file_source_common::{ buffer::{ReadResult, read_until_with_max_size}, }; +const EOF_READ_BACKOFF_MIN: Duration = Duration::from_millis(1); +const EOF_READ_BACKOFF_MAX: Duration = Duration::from_millis(250); + #[cfg(test)] mod tests; @@ -57,6 +60,7 @@ pub struct FileWatcher { reached_eof: bool, last_read_attempt: Instant, last_read_success: Instant, + read_retry_delay: Duration, last_seen: Instant, max_line_bytes: usize, line_delimiter: Bytes, @@ -166,6 +170,7 @@ impl FileWatcher { reached_eof: false, last_read_attempt: ts, last_read_success: ts, + read_retry_delay: EOF_READ_BACKOFF_MIN, last_seen: ts, max_line_bytes, line_delimiter, @@ -196,6 +201,8 @@ impl FileWatcher { self.devno = file_info.portable_dev(); self.inode = file_info.portable_ino(); } + self.reached_eof = false; + self.read_retry_delay = EOF_READ_BACKOFF_MIN; self.path = path; Ok(()) } @@ -235,7 +242,7 @@ impl FileWatcher { let file_position = &mut self.file_position; let initial_position = *file_position; match read_until_with_max_size( - Box::pin(reader), + reader.as_mut(), file_position, self.line_delimiter.as_ref(), &mut self.buf, @@ -283,7 +290,7 @@ impl FileWatcher { }) } } else { - self.reached_eof = true; + self.track_read_eof(); Ok(RawLineResult { raw_line: None, discarded_for_size_and_truncated, @@ -306,9 +313,24 @@ impl FileWatcher { #[inline] fn track_read_success(&mut self) { + self.reached_eof = false; + self.read_retry_delay = EOF_READ_BACKOFF_MIN; self.last_read_success = Instant::now(); } + #[inline] + fn track_read_eof(&mut self) { + self.read_retry_delay = if self.reached_eof { + std::cmp::min( + self.read_retry_delay.saturating_mul(2), + EOF_READ_BACKOFF_MAX, + ) + } else { + EOF_READ_BACKOFF_MIN + }; + self.reached_eof = true; + } + #[inline] pub fn last_read_success(&self) -> Instant { self.last_read_success @@ -316,6 +338,10 @@ impl FileWatcher { #[inline] pub fn should_read(&self) -> bool { + if self.reached_eof && self.last_read_attempt.elapsed() < self.read_retry_delay { + return false; + } + self.last_read_success.elapsed() < Duration::from_secs(10) || self.last_read_attempt.elapsed() > Duration::from_secs(10) } diff --git a/lib/file-source/src/file_watcher/tests/mod.rs b/lib/file-source/src/file_watcher/tests/mod.rs index 7a9cc92909db0..5384a43802d24 100644 --- a/lib/file-source/src/file_watcher/tests/mod.rs +++ b/lib/file-source/src/file_watcher/tests/mod.rs @@ -1,9 +1,13 @@ mod experiment; mod experiment_no_truncations; -use std::str; +use std::{path::PathBuf, str, thread}; +use bytes::{Bytes, BytesMut}; use quickcheck::{Arbitrary, Gen}; +use tokio::time::Instant; + +use super::{EOF_READ_BACKOFF_MAX, EOF_READ_BACKOFF_MIN, FileWatcher, null_reader}; // Welcome. // @@ -171,6 +175,68 @@ impl Arbitrary for FileWatcherAction { } } +fn watcher_for_timing() -> FileWatcher { + let now = Instant::now(); + + FileWatcher { + path: PathBuf::new(), + findable: true, + reader: Box::new(null_reader()), + file_position: 0, + devno: 0, + inode: 0, + is_dead: false, + reached_eof: false, + last_read_attempt: now, + last_read_success: now, + read_retry_delay: EOF_READ_BACKOFF_MIN, + last_seen: now, + max_line_bytes: 1024, + line_delimiter: Bytes::from_static(b"\n"), + buf: BytesMut::new(), + } +} + +#[test] +fn backs_off_after_eof() { + let mut watcher = watcher_for_timing(); + + watcher.track_read_attempt(); + watcher.track_read_eof(); + + assert_eq!(watcher.read_retry_delay, EOF_READ_BACKOFF_MIN); + assert!(!watcher.should_read()); + + thread::sleep(EOF_READ_BACKOFF_MIN); + + assert!(watcher.should_read()); + + watcher.track_read_attempt(); + watcher.track_read_eof(); + + assert_eq!( + watcher.read_retry_delay, + EOF_READ_BACKOFF_MIN.saturating_mul(2) + ); +} + +#[test] +fn caps_and_resets_eof_backoff() { + let mut watcher = watcher_for_timing(); + + for _ in 0..16 { + watcher.track_read_attempt(); + watcher.track_read_eof(); + } + + assert_eq!(watcher.read_retry_delay, EOF_READ_BACKOFF_MAX); + + watcher.track_read_success(); + + assert_eq!(watcher.read_retry_delay, EOF_READ_BACKOFF_MIN); + assert!(!watcher.reached_eof()); +} + #[inline] pub fn delay(attempts: u32) { let delay = match attempts { diff --git a/lib/k8s-e2e-tests/src/lib.rs b/lib/k8s-e2e-tests/src/lib.rs index 4d6d10f4a1875..8986b59ee03a1 100644 --- a/lib/k8s-e2e-tests/src/lib.rs +++ b/lib/k8s-e2e-tests/src/lib.rs @@ -20,6 +20,12 @@ pub mod metrics; pub const BUSYBOX_IMAGE: &str = "busybox:1.28"; +/// Returns the Helm chart repo to use for E2E tests. +/// Set `HELM_CHART_REPO` to override the default (e.g., a local chart path). +pub fn helm_chart_repo() -> String { + env::var("HELM_CHART_REPO").unwrap_or_else(|_| "https://helm.vector.dev".to_string()) +} + pub fn init() { _ = env_logger::builder().is_test(true).try_init(); } diff --git a/lib/k8s-e2e-tests/tests/vector-agent.rs b/lib/k8s-e2e-tests/tests/vector-agent.rs index 2bf56f32336c6..1a1176b8ce4a1 100644 --- a/lib/k8s-e2e-tests/tests/vector-agent.rs +++ b/lib/k8s-e2e-tests/tests/vector-agent.rs @@ -74,7 +74,7 @@ async fn default_agent() -> Result<(), Box> { &namespace, "vector", "vector", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![ &config_override_name(&override_name, true), @@ -199,7 +199,7 @@ async fn partial_merge() -> Result<(), Box> { &namespace, "vector", "vector", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![ &config_override_name(&override_name, true), @@ -351,7 +351,7 @@ async fn preexisting() -> Result<(), Box> { &namespace, "vector", "vector", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![ &config_override_name(&override_name, true), @@ -453,7 +453,7 @@ async fn multiple_lines() -> Result<(), Box> { &namespace, "vector", "vector", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![ &config_override_name(&override_name, true), @@ -580,7 +580,7 @@ async fn metadata_annotation() -> Result<(), Box> { &namespace, "vector", "vector", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![ &config_override_name(&override_name, true), @@ -769,7 +769,7 @@ async fn pod_filtering() -> Result<(), Box> { &namespace, "vector", "vector", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![ &config_override_name(&override_name, true), @@ -1012,7 +1012,7 @@ async fn custom_selectors() -> Result<(), Box> { &namespace, "vector", "vector", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![&config_override_name(&override_name, true), CONFIG], ..Default::default() @@ -1207,7 +1207,7 @@ async fn container_filtering() -> Result<(), Box> { &namespace, "vector", "vector", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![ &config_override_name(&override_name, true), @@ -1416,7 +1416,7 @@ async fn glob_pattern_filtering() -> Result<(), Box> { &namespace, "vector", "vector", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![&config_override_name(&override_name, true), CONFIG], ..Default::default() @@ -1576,7 +1576,7 @@ async fn multiple_ns() -> Result<(), Box> { &namespace, "vector", "vector", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![ &config_override_name(&override_name, true), @@ -1738,7 +1738,7 @@ async fn existing_config_file() -> Result<(), Box> { &namespace, "vector", "vector", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![ &config_override_name(&override_name, true), @@ -1839,7 +1839,7 @@ async fn metrics_pipeline() -> Result<(), Box> { &namespace, "vector", "vector", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![ &config_override_name(&override_name, true), @@ -1994,7 +1994,7 @@ async fn host_metrics() -> Result<(), Box> { &namespace, "vector", "vector", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![ &config_override_name(&override_name, true), @@ -2060,7 +2060,7 @@ async fn simple_checkpoint() -> Result<(), Box> { "test-vector", "vector", "vector", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![HELM_VALUES_AGENT], ..Default::default() diff --git a/lib/k8s-e2e-tests/tests/vector-aggregator.rs b/lib/k8s-e2e-tests/tests/vector-aggregator.rs index 07cb49ff4bf9d..d945fc5175051 100644 --- a/lib/k8s-e2e-tests/tests/vector-aggregator.rs +++ b/lib/k8s-e2e-tests/tests/vector-aggregator.rs @@ -19,7 +19,7 @@ async fn dummy_topology() -> Result<(), Box> { &namespace, "vector", "vector", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![&config_override_name(&override_name, false)], ..Default::default() @@ -54,7 +54,7 @@ async fn metrics_pipeline() -> Result<(), Box> { &namespace, "vector", "vector", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![&config_override_name(&override_name, false)], ..Default::default() diff --git a/lib/k8s-e2e-tests/tests/vector-dd-agent-aggregator.rs b/lib/k8s-e2e-tests/tests/vector-dd-agent-aggregator.rs index 25c7dc4ce6e0a..dde302827cd82 100644 --- a/lib/k8s-e2e-tests/tests/vector-dd-agent-aggregator.rs +++ b/lib/k8s-e2e-tests/tests/vector-dd-agent-aggregator.rs @@ -58,7 +58,7 @@ async fn datadog_to_vector() -> Result<(), Box> { &namespace, "vector", "vector", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![&config_override_name(&override_name, false)], ..Default::default() diff --git a/lib/k8s-e2e-tests/tests/vector.rs b/lib/k8s-e2e-tests/tests/vector.rs index 7d74442fb1146..e38f494236786 100644 --- a/lib/k8s-e2e-tests/tests/vector.rs +++ b/lib/k8s-e2e-tests/tests/vector.rs @@ -77,7 +77,7 @@ async fn logs() -> Result<(), Box> { &namespace, "vector", "aggregator", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { ..Default::default() }, @@ -97,7 +97,7 @@ async fn logs() -> Result<(), Box> { &namespace, "vector", "agent", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![&helm_values_stdout_sink(&agent_override_name)], ..Default::default() @@ -198,7 +198,7 @@ async fn haproxy() -> Result<(), Box> { &namespace, "vector", "aggregator", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![CONFIG], ..Default::default() @@ -227,7 +227,7 @@ async fn haproxy() -> Result<(), Box> { &namespace, "vector", "agent", - "https://helm.vector.dev", + &helm_chart_repo(), VectorConfig { custom_helm_values: vec![&helm_values_haproxy(&agent_override_name)], ..Default::default() diff --git a/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/logs/v1/logs_service_http.yaml b/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/logs/v1/logs_service_http.yaml index 4d913387252ef..4dd744f5d2c02 100644 --- a/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/logs/v1/logs_service_http.yaml +++ b/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/logs/v1/logs_service_http.yaml @@ -3,8 +3,7 @@ type: google.api.Service config_version: 3 http: - rules: - - selector: opentelemetry.proto.collector.logs.v1.LogsService.Export - post: /v1/logs - body: "*" - + rules: + - selector: opentelemetry.proto.collector.logs.v1.LogsService.Export + post: /v1/logs + body: "*" diff --git a/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/metrics/v1/metrics_service_http.yaml b/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/metrics/v1/metrics_service_http.yaml index 7ff39d4b406c6..7643be4125831 100644 --- a/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/metrics/v1/metrics_service_http.yaml +++ b/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/metrics/v1/metrics_service_http.yaml @@ -3,8 +3,7 @@ type: google.api.Service config_version: 3 http: - rules: - - selector: opentelemetry.proto.collector.metrics.v1.MetricsService.Export - post: /v1/metrics - body: "*" - + rules: + - selector: opentelemetry.proto.collector.metrics.v1.MetricsService.Export + post: /v1/metrics + body: "*" diff --git a/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/trace/v1/trace_service_http.yaml b/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/trace/v1/trace_service_http.yaml index d091b3a8d53d7..7eaa4e548c33a 100644 --- a/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/trace/v1/trace_service_http.yaml +++ b/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/collector/trace/v1/trace_service_http.yaml @@ -3,7 +3,7 @@ type: google.api.Service config_version: 3 http: - rules: - - selector: opentelemetry.proto.collector.trace.v1.TraceService.Export - post: /v1/traces - body: "*" + rules: + - selector: opentelemetry.proto.collector.trace.v1.TraceService.Export + post: /v1/traces + body: "*" diff --git a/lib/tracing-limit/Cargo.toml b/lib/tracing-limit/Cargo.toml index 2308c8ce81084..ba52c12fdea99 100644 --- a/lib/tracing-limit/Cargo.toml +++ b/lib/tracing-limit/Cargo.toml @@ -15,7 +15,7 @@ tracing-subscriber = { workspace = true, features = ["registry", "std"] } dashmap = { version = "6.1.0", default-features = false } [dev-dependencies] -criterion = "0.7" +criterion.workspace = true mock_instant.workspace = true serial_test.workspace = true tracing = "0.1.44" diff --git a/lib/vector-api-client/Cargo.toml b/lib/vector-api-client/Cargo.toml index 3fbc200af231b..260a84a927e76 100644 --- a/lib/vector-api-client/Cargo.toml +++ b/lib/vector-api-client/Cargo.toml @@ -1,34 +1,29 @@ [package] name = "vector-api-client" -version = "0.1.2" +version = "0.2.0" authors = ["Vector Contributors "] edition = "2024" publish = false license = "MPL-2.0" [dependencies] +# gRPC +tonic.workspace = true +tonic-health.workspace = true +prost.workspace = true +prost-types.workspace = true -# Serde -serde.workspace = true -serde_json.workspace = true - -# Error handling -anyhow.workspace = true - -# Tokio / Futures +# Async futures.workspace = true tokio = { workspace = true, features = ["macros", "rt", "sync"] } tokio-stream = { workspace = true, features = ["sync"] } -# GraphQL -graphql_client = { version = "0.14.0", default-features = false, features = ["graphql_query_derive"] } +# Error handling +snafu.workspace = true -# HTTP / WebSockets -reqwest = { version = "0.11.26", default-features = false, features = ["json"] } -tokio-tungstenite = { workspace = true, features = ["connect", "rustls"] } +# Utils +http = { version = "0.2", default-features = false } -# External libs -chrono.workspace = true -clap.workspace = true -url = { version = "2.5.4", default-features = false } -uuid.workspace = true +[build-dependencies] +tonic-build.workspace = true +prost-build.workspace = true diff --git a/lib/vector-api-client/build.rs b/lib/vector-api-client/build.rs new file mode 100644 index 0000000000000..a85641a4b2476 --- /dev/null +++ b/lib/vector-api-client/build.rs @@ -0,0 +1,34 @@ +fn main() -> Result<(), Box> { + println!("cargo:rerun-if-changed=../../proto/vector/observability.proto"); + println!("cargo:rerun-if-changed=../../proto"); + println!("cargo:rerun-if-changed=../../lib/vector-core/proto/event.proto"); + + // First, generate event.proto types + tonic_build::configure().build_server(false).compile( + &["../../lib/vector-core/proto/event.proto"], + &["../../lib/vector-core/proto", "../../proto/third-party"], + )?; + + // Then, generate observability.proto using extern_path to reference the event types + let mut prost_config = prost_build::Config::new(); + prost_config.extern_path(".event", "crate::proto::event"); + // Allow clippy warning for large enum variant in generated code + // TappedEvent contains a large EventWrapper while EventNotification is small + prost_config.type_attribute( + ".vector.observability.v1.StreamOutputEventsResponse", + "#[allow(clippy::large_enum_variant)]", + ); + + tonic_build::configure() + .build_server(false) + .compile_with_config( + prost_config, + &["../../proto/vector/observability.proto"], + &[ + "../../proto", + "../../proto/third-party", + "../../lib/vector-core/proto", + ], + )?; + Ok(()) +} diff --git a/lib/vector-api-client/graphql/queries/components.graphql b/lib/vector-api-client/graphql/queries/components.graphql deleted file mode 100644 index 3a6f78fc27989..0000000000000 --- a/lib/vector-api-client/graphql/queries/components.graphql +++ /dev/null @@ -1,62 +0,0 @@ -query ComponentsQuery($first: Int!) { - components(first: $first) { - edges { - node { - __typename - componentId - componentType - ... on Source { - outputs { - outputId - sentEventsTotal { - sentEventsTotal - } - } - metrics { - __typename - receivedBytesTotal { - receivedBytesTotal - } - receivedEventsTotal { - receivedEventsTotal - } - sentEventsTotal { - sentEventsTotal - } - } - } - ... on Transform { - outputs { - outputId - sentEventsTotal { - sentEventsTotal - } - } - metrics { - __typename - receivedEventsTotal { - receivedEventsTotal - } - sentEventsTotal { - sentEventsTotal - } - } - } - ... on Sink { - metrics { - __typename - receivedEventsTotal { - receivedEventsTotal - } - sentBytesTotal { - sentBytesTotal - } - sentEventsTotal { - sentEventsTotal - } - } - } - } - } - } -} diff --git a/lib/vector-api-client/graphql/queries/health.graphql b/lib/vector-api-client/graphql/queries/health.graphql deleted file mode 100644 index 6d4aa89849307..0000000000000 --- a/lib/vector-api-client/graphql/queries/health.graphql +++ /dev/null @@ -1,3 +0,0 @@ -query HealthQuery { - health -} diff --git a/lib/vector-api-client/graphql/queries/meta_version_string.graphql b/lib/vector-api-client/graphql/queries/meta_version_string.graphql deleted file mode 100644 index 82f41372efa07..0000000000000 --- a/lib/vector-api-client/graphql/queries/meta_version_string.graphql +++ /dev/null @@ -1,5 +0,0 @@ -query MetaVersionStringQuery { - meta { - versionString - } -} diff --git a/lib/vector-api-client/graphql/schema.json b/lib/vector-api-client/graphql/schema.json deleted file mode 100644 index 034780a80d718..0000000000000 --- a/lib/vector-api-client/graphql/schema.json +++ /dev/null @@ -1,7206 +0,0 @@ -{ - "data": { - "__schema": { - "queryType": { - "name": "Query" - }, - "mutationType": null, - "subscriptionType": { - "name": "Subscription" - }, - "types": [ - { - "kind": "OBJECT", - "name": "AllocatedBytes", - "description": null, - "fields": [ - { - "name": "timestamp", - "description": "Metric timestamp", - "args": [], - "type": { - "kind": "SCALAR", - "name": "DateTime", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "allocatedBytes", - "description": "Allocated bytes", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "SCALAR", - "name": "Boolean", - "description": "The `Boolean` scalar type represents `true` or `false`.", - "fields": null, - "inputFields": null, - "interfaces": null, - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "INTERFACE", - "name": "Component", - "description": null, - "fields": [ - { - "name": "componentId", - "description": null, - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "componentType", - "description": null, - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": null, - "enumValues": null, - "possibleTypes": [ - { - "kind": "OBJECT", - "name": "Source", - "ofType": null - }, - { - "kind": "OBJECT", - "name": "Transform", - "ofType": null - }, - { - "kind": "OBJECT", - "name": "Sink", - "ofType": null - } - ] - }, - { - "kind": "OBJECT", - "name": "ComponentAllocatedBytes", - "description": null, - "fields": [ - { - "name": "componentId", - "description": "Component id", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "metric", - "description": "Allocated bytes metric", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "AllocatedBytes", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "ComponentConnection", - "description": null, - "fields": [ - { - "name": "pageInfo", - "description": "Information to aid in pagination.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "PageInfo", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "edges", - "description": "A list of edges.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "ComponentEdge", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "nodes", - "description": "A list of nodes.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INTERFACE", - "name": "Component", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "totalCount", - "description": "Total result set count", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "ComponentEdge", - "description": "An edge in a connection.", - "fields": [ - { - "name": "cursor", - "description": "A cursor for use in pagination", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "node", - "description": "The item at the end of the edge", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INTERFACE", - "name": "Component", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "ComponentErrorsTotal", - "description": null, - "fields": [ - { - "name": "componentId", - "description": "Component id", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "metric", - "description": "Errors processed metric", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "ErrorsTotal", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "ENUM", - "name": "ComponentKind", - "description": null, - "fields": null, - "inputFields": null, - "interfaces": null, - "enumValues": [ - { - "name": "SOURCE", - "description": null, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "TRANSFORM", - "description": null, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "SINK", - "description": null, - "isDeprecated": false, - "deprecationReason": null - } - ], - "possibleTypes": null - }, - { - "kind": "INPUT_OBJECT", - "name": "ComponentKindFilter", - "description": null, - "fields": null, - "inputFields": [ - { - "name": "equals", - "description": null, - "type": { - "kind": "ENUM", - "name": "ComponentKind", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "notEquals", - "description": null, - "type": { - "kind": "ENUM", - "name": "ComponentKind", - "ofType": null - }, - "defaultValue": null - } - ], - "interfaces": null, - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "ComponentReceivedBytesThroughput", - "description": null, - "fields": [ - { - "name": "componentId", - "description": "Component ID.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "throughput", - "description": "Throughput of bytes sent.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "ComponentReceivedBytesTotal", - "description": null, - "fields": [ - { - "name": "componentId", - "description": "Component ID.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "metric", - "description": "Metric for total bytes received.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "ReceivedBytesTotal", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "ComponentReceivedEventsThroughput", - "description": null, - "fields": [ - { - "name": "componentId", - "description": "Component id", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "throughput", - "description": "Received events throughput", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "ComponentReceivedEventsTotal", - "description": null, - "fields": [ - { - "name": "componentId", - "description": "Component id", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "metric", - "description": "Total received events metric", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "ReceivedEventsTotal", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "ComponentSentBytesThroughput", - "description": null, - "fields": [ - { - "name": "componentId", - "description": "Component ID.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "throughput", - "description": "Throughput of bytes sent.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "ComponentSentBytesTotal", - "description": null, - "fields": [ - { - "name": "componentId", - "description": "Component ID.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "metric", - "description": "Metric for total bytes sent.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "SentBytesTotal", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "ComponentSentEventsThroughput", - "description": null, - "fields": [ - { - "name": "componentId", - "description": "Component id", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "throughput", - "description": "Total events processed throughput", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "outputs", - "description": "Output streams with throughputs", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "OutputThroughput", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "ComponentSentEventsTotal", - "description": null, - "fields": [ - { - "name": "componentId", - "description": "Component id", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "metric", - "description": "Total outgoing events metric", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "SentEventsTotal", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "outputs", - "description": "Output streams with outgoing events metrics", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "Output", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "INPUT_OBJECT", - "name": "ComponentsFilter", - "description": null, - "fields": null, - "inputFields": [ - { - "name": "componentId", - "description": null, - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INPUT_OBJECT", - "name": "StringFilter", - "ofType": null - } - } - }, - "defaultValue": null - }, - { - "name": "componentKind", - "description": null, - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INPUT_OBJECT", - "name": "ComponentKindFilter", - "ofType": null - } - } - }, - "defaultValue": null - }, - { - "name": "or", - "description": null, - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INPUT_OBJECT", - "name": "ComponentsFilter", - "ofType": null - } - } - }, - "defaultValue": null - } - ], - "interfaces": null, - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "INPUT_OBJECT", - "name": "ComponentsSortField", - "description": null, - "fields": null, - "inputFields": [ - { - "name": "field", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "ENUM", - "name": "ComponentsSortFieldName", - "ofType": null - } - }, - "defaultValue": null - }, - { - "name": "direction", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "ENUM", - "name": "Direction", - "ofType": null - } - }, - "defaultValue": "ASC" - } - ], - "interfaces": null, - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "ENUM", - "name": "ComponentsSortFieldName", - "description": null, - "fields": null, - "inputFields": null, - "interfaces": null, - "enumValues": [ - { - "name": "COMPONENT_KEY", - "description": null, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "COMPONENT_KIND", - "description": null, - "isDeprecated": false, - "deprecationReason": null - } - ], - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "CpuMetrics", - "description": null, - "fields": [ - { - "name": "cpuSecondsTotal", - "description": "CPU seconds total", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "SCALAR", - "name": "DateTime", - "description": "Implement the DateTime scalar\n\nThe input/output is a string in RFC3339 format.", - "fields": null, - "inputFields": null, - "interfaces": null, - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "ENUM", - "name": "Direction", - "description": null, - "fields": null, - "inputFields": null, - "interfaces": null, - "enumValues": [ - { - "name": "ASC", - "description": null, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "DESC", - "description": null, - "isDeprecated": false, - "deprecationReason": null - } - ], - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "DiskMetrics", - "description": null, - "fields": [ - { - "name": "readBytesTotal", - "description": "Total bytes read", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "readsCompletedTotal", - "description": "Total reads completed", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "writtenBytesTotal", - "description": "Total bytes written", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "writesCompletedTotal", - "description": "Total writes completed", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "ErrorsTotal", - "description": null, - "fields": [ - { - "name": "timestamp", - "description": "Metric timestamp", - "args": [], - "type": { - "kind": "SCALAR", - "name": "DateTime", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "errorsTotal", - "description": "Total error count", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "ENUM", - "name": "EventEncodingType", - "description": "Encoding format for the event", - "fields": null, - "inputFields": null, - "interfaces": null, - "enumValues": [ - { - "name": "JSON", - "description": null, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "YAML", - "description": null, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "LOGFMT", - "description": null, - "isDeprecated": false, - "deprecationReason": null - } - ], - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "EventNotification", - "description": "A notification regarding events observation", - "fields": [ - { - "name": "notification", - "description": "Notification details", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "UNION", - "name": "Notification", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "message", - "description": "The human-readable message associated with the notification", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "FileSourceMetricFile", - "description": null, - "fields": [ - { - "name": "name", - "description": "File name", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "receivedBytesTotal", - "description": "Metric indicating bytes received for the current file", - "args": [], - "type": { - "kind": "OBJECT", - "name": "ReceivedBytesTotal", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "receivedEventsTotal", - "description": "Metric indicating received events for the current file", - "args": [], - "type": { - "kind": "OBJECT", - "name": "ReceivedEventsTotal", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "sentEventsTotal", - "description": "Metric indicating outgoing events for the current file", - "args": [], - "type": { - "kind": "OBJECT", - "name": "SentEventsTotal", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "FileSourceMetricFileConnection", - "description": null, - "fields": [ - { - "name": "pageInfo", - "description": "Information to aid in pagination.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "PageInfo", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "edges", - "description": "A list of edges.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "FileSourceMetricFileEdge", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "nodes", - "description": "A list of nodes.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "FileSourceMetricFile", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "totalCount", - "description": "Total result set count", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "FileSourceMetricFileEdge", - "description": "An edge in a connection.", - "fields": [ - { - "name": "cursor", - "description": "A cursor for use in pagination", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "node", - "description": "The item at the end of the edge", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "FileSourceMetricFile", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "INPUT_OBJECT", - "name": "FileSourceMetricFilesSortField", - "description": null, - "fields": null, - "inputFields": [ - { - "name": "field", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "ENUM", - "name": "FileSourceMetricFilesSortFieldName", - "ofType": null - } - }, - "defaultValue": null - }, - { - "name": "direction", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "ENUM", - "name": "Direction", - "ofType": null - } - }, - "defaultValue": "ASC" - } - ], - "interfaces": null, - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "ENUM", - "name": "FileSourceMetricFilesSortFieldName", - "description": null, - "fields": null, - "inputFields": null, - "interfaces": null, - "enumValues": [ - { - "name": "NAME", - "description": null, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "RECEIVED_BYTES_TOTAL", - "description": null, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "RECEIVED_EVENTS_TOTAL", - "description": null, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "SENT_EVENTS_TOTAL", - "description": null, - "isDeprecated": false, - "deprecationReason": null - } - ], - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "FileSourceMetrics", - "description": null, - "fields": [ - { - "name": "files", - "description": "File metrics", - "args": [ - { - "name": "after", - "description": null, - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "before", - "description": null, - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "first", - "description": null, - "type": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "last", - "description": null, - "type": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "filter", - "description": null, - "type": { - "kind": "INPUT_OBJECT", - "name": "FileSourceMetricsFilesFilter", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "sort", - "description": null, - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INPUT_OBJECT", - "name": "FileSourceMetricFilesSortField", - "ofType": null - } - } - }, - "defaultValue": null - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "FileSourceMetricFileConnection", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "receivedBytesTotal", - "description": "Total received bytes for the current file source", - "args": [], - "type": { - "kind": "OBJECT", - "name": "ReceivedBytesTotal", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "receivedEventsTotal", - "description": "Total received events for the current file source", - "args": [], - "type": { - "kind": "OBJECT", - "name": "ReceivedEventsTotal", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "sentEventsTotal", - "description": "Total sent events for the current file source", - "args": [], - "type": { - "kind": "OBJECT", - "name": "SentEventsTotal", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [ - { - "kind": "INTERFACE", - "name": "SourceMetrics", - "ofType": null - } - ], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "INPUT_OBJECT", - "name": "FileSourceMetricsFilesFilter", - "description": null, - "fields": null, - "inputFields": [ - { - "name": "name", - "description": null, - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INPUT_OBJECT", - "name": "StringFilter", - "ofType": null - } - } - }, - "defaultValue": null - }, - { - "name": "or", - "description": null, - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INPUT_OBJECT", - "name": "FileSourceMetricsFilesFilter", - "ofType": null - } - } - }, - "defaultValue": null - } - ], - "interfaces": null, - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "FileSystemMetrics", - "description": null, - "fields": [ - { - "name": "freeBytes", - "description": "Free bytes", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "totalBytes", - "description": "Total bytes", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "usedBytes", - "description": "Used bytes", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "SCALAR", - "name": "Float", - "description": "The `Float` scalar type represents signed double-precision fractional values as specified by [IEEE 754](https://en.wikipedia.org/wiki/IEEE_floating_point).", - "fields": null, - "inputFields": null, - "interfaces": null, - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "GenericSinkMetrics", - "description": null, - "fields": [ - { - "name": "receivedEventsTotal", - "description": "Total received events for the current sink", - "args": [], - "type": { - "kind": "OBJECT", - "name": "ReceivedEventsTotal", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "sentBytesTotal", - "description": "Total sent bytes for the current sink", - "args": [], - "type": { - "kind": "OBJECT", - "name": "SentBytesTotal", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "sentEventsTotal", - "description": "Total sent events for the current sink", - "args": [], - "type": { - "kind": "OBJECT", - "name": "SentEventsTotal", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [ - { - "kind": "INTERFACE", - "name": "SinkMetrics", - "ofType": null - } - ], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "GenericSourceMetrics", - "description": null, - "fields": [ - { - "name": "receivedBytesTotal", - "description": "Total received bytes for the current source", - "args": [], - "type": { - "kind": "OBJECT", - "name": "ReceivedBytesTotal", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "receivedEventsTotal", - "description": "Total received events for the current source", - "args": [], - "type": { - "kind": "OBJECT", - "name": "ReceivedEventsTotal", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "sentEventsTotal", - "description": "Total sent events for the current source", - "args": [], - "type": { - "kind": "OBJECT", - "name": "SentEventsTotal", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [ - { - "kind": "INTERFACE", - "name": "SourceMetrics", - "ofType": null - } - ], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "GenericTransformMetrics", - "description": null, - "fields": [ - { - "name": "receivedEventsTotal", - "description": "Total received events for the current transform", - "args": [], - "type": { - "kind": "OBJECT", - "name": "ReceivedEventsTotal", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "sentEventsTotal", - "description": "Total sent events for the current transform", - "args": [], - "type": { - "kind": "OBJECT", - "name": "SentEventsTotal", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [ - { - "kind": "INTERFACE", - "name": "TransformMetrics", - "ofType": null - } - ], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "Heartbeat", - "description": null, - "fields": [ - { - "name": "utc", - "description": null, - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "DateTime", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "HostMetrics", - "description": "Vector host metrics", - "fields": [ - { - "name": "memory", - "description": "Memory metrics", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "MemoryMetrics", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "swap", - "description": "Swap metrics", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "SwapMetrics", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "cpu", - "description": "CPU metrics", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "CpuMetrics", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "loadAverage", - "description": "Load average metrics (*nix only)", - "args": [], - "type": { - "kind": "OBJECT", - "name": "LoadAverageMetrics", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "network", - "description": "Network metrics", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "NetworkMetrics", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "filesystem", - "description": "Filesystem metrics", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "FileSystemMetrics", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "disk", - "description": "Disk metrics", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "DiskMetrics", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "SCALAR", - "name": "ID", - "description": null, - "fields": null, - "inputFields": null, - "interfaces": null, - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "SCALAR", - "name": "Int", - "description": "The `Int` scalar type represents non-fractional whole numeric values.", - "fields": null, - "inputFields": null, - "interfaces": null, - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "InvalidMatch", - "description": "The pattern matched source(s) which cannot be tapped for inputs or sink(s)\nwhich cannot be tapped for outputs", - "fields": [ - { - "name": "pattern", - "description": "Pattern that raised the notification", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "invalidMatches", - "description": "Any invalid matches for the pattern", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "LoadAverageMetrics", - "description": null, - "fields": [ - { - "name": "load1", - "description": "Load 1 average", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "load5", - "description": "Load 5 average", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "load15", - "description": "Load 15 average", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "Log", - "description": "Log event with fields for querying log data", - "fields": [ - { - "name": "componentId", - "description": "Id of the component associated with the log event", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "componentType", - "description": "Type of component associated with the log event", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "componentKind", - "description": "Kind of component associated with the log event", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "message", - "description": "Log message", - "args": [], - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "timestamp", - "description": "Log timestamp", - "args": [], - "type": { - "kind": "SCALAR", - "name": "DateTime", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "string", - "description": "Log event as an encoded string format", - "args": [ - { - "name": "encoding", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "ENUM", - "name": "EventEncodingType", - "ofType": null - } - }, - "defaultValue": null - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "json", - "description": "Get JSON field data on the log event, by field name", - "args": [ - { - "name": "field", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "defaultValue": null - } - ], - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "Matched", - "description": "A component was found that matched the provided pattern", - "fields": [ - { - "name": "pattern", - "description": "Pattern that raised the notification", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "MemoryMetrics", - "description": "Host memory metrics", - "fields": [ - { - "name": "totalBytes", - "description": "Total bytes", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "freeBytes", - "description": "Free bytes", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "availableBytes", - "description": "Available bytes", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "activeBytes", - "description": "Active bytes (Linux/macOS only)", - "args": [], - "type": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "buffersBytes", - "description": "Buffers bytes (Linux only)", - "args": [], - "type": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "cachedBytes", - "description": "Cached bytes (Linux only)", - "args": [], - "type": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "sharedBytes", - "description": "Shared bytes (Linux only)", - "args": [], - "type": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "usedBytes", - "description": "Used bytes (Linux only)", - "args": [], - "type": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "inactiveBytes", - "description": "Inactive bytes (macOS only)", - "args": [], - "type": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "wiredBytes", - "description": "Wired bytes (macOS only)", - "args": [], - "type": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "Meta", - "description": null, - "fields": [ - { - "name": "versionString", - "description": "Vector version", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "hostname", - "description": "Hostname", - "args": [], - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "Metric", - "description": "Metric event with fields for querying metric data", - "fields": [ - { - "name": "componentId", - "description": "Id of the component associated with the metric event", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "componentType", - "description": "Type of component associated with the metric event", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "componentKind", - "description": "Kind of component associated with the metric event", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "timestamp", - "description": "Metric timestamp", - "args": [], - "type": { - "kind": "SCALAR", - "name": "DateTime", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "name", - "description": "Metric name", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "namespace", - "description": "Metric namespace", - "args": [], - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "kind", - "description": "Metric kind", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "ENUM", - "name": "MetricKind", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "valueType", - "description": "Metric type", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "value", - "description": "Metric value in human readable form", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "tags", - "description": "Metric tags", - "args": [], - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "MetricTag", - "ofType": null - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "string", - "description": "Metric event as an encoded string format", - "args": [ - { - "name": "encoding", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "ENUM", - "name": "EventEncodingType", - "ofType": null - } - }, - "defaultValue": null - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "ENUM", - "name": "MetricKind", - "description": null, - "fields": null, - "inputFields": null, - "interfaces": null, - "enumValues": [ - { - "name": "INCREMENTAL", - "description": "Incremental metrics update previous values", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "ABSOLUTE", - "description": "Absolute metrics set the reference value for future updates", - "isDeprecated": false, - "deprecationReason": null - } - ], - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "MetricTag", - "description": null, - "fields": [ - { - "name": "key", - "description": "Metric tag key", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "value", - "description": "Metric tag value", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "INTERFACE", - "name": "MetricType", - "description": null, - "fields": [ - { - "name": "timestamp", - "description": null, - "args": [], - "type": { - "kind": "SCALAR", - "name": "DateTime", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": null, - "enumValues": null, - "possibleTypes": [ - { - "kind": "OBJECT", - "name": "Uptime", - "ofType": null - } - ] - }, - { - "kind": "OBJECT", - "name": "NetworkMetrics", - "description": null, - "fields": [ - { - "name": "receiveBytesTotal", - "description": "Total bytes received", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "receiveErrsTotal", - "description": "Total errors received", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "receivePacketsTotal", - "description": "Total packets received", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "transmitBytesTotal", - "description": "Total bytes transmitted", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "transmitErrsTotal", - "description": "Total errors transmitted", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "transmitPacketsDropTotal", - "description": "Total transmission packets dropped (Linux/Windows only)", - "args": [], - "type": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "transmitPacketsTotal", - "description": "Total transmission packets (Linux/Windows only)", - "args": [], - "type": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "NotMatched", - "description": "There isn't currently a component that matches this pattern", - "fields": [ - { - "name": "pattern", - "description": "Pattern that raised the notification", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "UNION", - "name": "Notification", - "description": "A specific kind of notification with additional details", - "fields": null, - "inputFields": null, - "interfaces": null, - "enumValues": null, - "possibleTypes": [ - { - "kind": "OBJECT", - "name": "Matched", - "ofType": null - }, - { - "kind": "OBJECT", - "name": "NotMatched", - "ofType": null - }, - { - "kind": "OBJECT", - "name": "InvalidMatch", - "ofType": null - } - ] - }, - { - "kind": "OBJECT", - "name": "Output", - "description": null, - "fields": [ - { - "name": "outputId", - "description": "Id of the output stream", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "sentEventsTotal", - "description": "Total sent events for the current output stream", - "args": [], - "type": { - "kind": "OBJECT", - "name": "SentEventsTotal", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "UNION", - "name": "OutputEventsPayload", - "description": "An event or a notification", - "fields": null, - "inputFields": null, - "interfaces": null, - "enumValues": null, - "possibleTypes": [ - { - "kind": "OBJECT", - "name": "Log", - "ofType": null - }, - { - "kind": "OBJECT", - "name": "Metric", - "ofType": null - }, - { - "kind": "OBJECT", - "name": "EventNotification", - "ofType": null - }, - { - "kind": "OBJECT", - "name": "Trace", - "ofType": null - } - ] - }, - { - "kind": "OBJECT", - "name": "OutputThroughput", - "description": null, - "fields": [ - { - "name": "outputId", - "description": "Id of the output stream", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "throughput", - "description": "Throughput for the output stream", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "PageInfo", - "description": "Information about pagination in a connection", - "fields": [ - { - "name": "hasPreviousPage", - "description": "When paginating backwards, are there more items?", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Boolean", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "hasNextPage", - "description": "When paginating forwards, are there more items?", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Boolean", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "startCursor", - "description": "When paginating backwards, the cursor to continue.", - "args": [], - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "endCursor", - "description": "When paginating forwards, the cursor to continue.", - "args": [], - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "Query", - "description": null, - "fields": [ - { - "name": "health", - "description": "Returns `true` to denote the GraphQL server is reachable", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Boolean", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "components", - "description": "Configured components (sources/transforms/sinks)", - "args": [ - { - "name": "after", - "description": null, - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "before", - "description": null, - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "first", - "description": null, - "type": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "last", - "description": null, - "type": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "filter", - "description": null, - "type": { - "kind": "INPUT_OBJECT", - "name": "ComponentsFilter", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "sort", - "description": null, - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INPUT_OBJECT", - "name": "ComponentsSortField", - "ofType": null - } - } - }, - "defaultValue": null - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "ComponentConnection", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "sources", - "description": "Configured sources", - "args": [ - { - "name": "after", - "description": null, - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "before", - "description": null, - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "first", - "description": null, - "type": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "last", - "description": null, - "type": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "filter", - "description": null, - "type": { - "kind": "INPUT_OBJECT", - "name": "SourcesFilter", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "sort", - "description": null, - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INPUT_OBJECT", - "name": "SourcesSortField", - "ofType": null - } - } - }, - "defaultValue": null - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "SourceConnection", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "transforms", - "description": "Configured transforms", - "args": [ - { - "name": "after", - "description": null, - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "before", - "description": null, - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "first", - "description": null, - "type": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "last", - "description": null, - "type": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "filter", - "description": null, - "type": { - "kind": "INPUT_OBJECT", - "name": "TransformsFilter", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "sort", - "description": null, - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INPUT_OBJECT", - "name": "TransformsSortField", - "ofType": null - } - } - }, - "defaultValue": null - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "TransformConnection", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "sinks", - "description": "Configured sinks", - "args": [ - { - "name": "after", - "description": null, - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "before", - "description": null, - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "first", - "description": null, - "type": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "last", - "description": null, - "type": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "filter", - "description": null, - "type": { - "kind": "INPUT_OBJECT", - "name": "SinksFilter", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "sort", - "description": null, - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INPUT_OBJECT", - "name": "SinksSortField", - "ofType": null - } - } - }, - "defaultValue": null - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "SinkConnection", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "componentByComponentKey", - "description": "Gets a configured component by component_key", - "args": [ - { - "name": "componentId", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "defaultValue": null - } - ], - "type": { - "kind": "INTERFACE", - "name": "Component", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "hostMetrics", - "description": "Vector host metrics", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "HostMetrics", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "meta", - "description": null, - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "Meta", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "ReceivedBytesTotal", - "description": null, - "fields": [ - { - "name": "timestamp", - "description": "Metric timestamp.", - "args": [], - "type": { - "kind": "SCALAR", - "name": "DateTime", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "receivedBytesTotal", - "description": "Total number of bytes received.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "ReceivedEventsTotal", - "description": null, - "fields": [ - { - "name": "timestamp", - "description": "Metric timestamp", - "args": [], - "type": { - "kind": "SCALAR", - "name": "DateTime", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "receivedEventsTotal", - "description": "Total incoming events", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "SentBytesTotal", - "description": null, - "fields": [ - { - "name": "timestamp", - "description": "Metric timestamp.", - "args": [], - "type": { - "kind": "SCALAR", - "name": "DateTime", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "sentBytesTotal", - "description": "Total number of bytes sent.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "SentEventsTotal", - "description": null, - "fields": [ - { - "name": "timestamp", - "description": "Metric timestamp", - "args": [], - "type": { - "kind": "SCALAR", - "name": "DateTime", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "sentEventsTotal", - "description": "Total sent events", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "Sink", - "description": null, - "fields": [ - { - "name": "componentId", - "description": "Sink component_id", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "componentType", - "description": "Sink type", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "sources", - "description": "Source inputs", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "Source", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "transforms", - "description": "Transform inputs", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "Transform", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "metrics", - "description": "Sink metrics", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INTERFACE", - "name": "SinkMetrics", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [ - { - "kind": "INTERFACE", - "name": "Component", - "ofType": null - } - ], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "SinkConnection", - "description": null, - "fields": [ - { - "name": "pageInfo", - "description": "Information to aid in pagination.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "PageInfo", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "edges", - "description": "A list of edges.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "SinkEdge", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "nodes", - "description": "A list of nodes.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "Sink", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "totalCount", - "description": "Total result set count", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "SinkEdge", - "description": "An edge in a connection.", - "fields": [ - { - "name": "cursor", - "description": "A cursor for use in pagination", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "node", - "description": "The item at the end of the edge", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "Sink", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "INTERFACE", - "name": "SinkMetrics", - "description": null, - "fields": [ - { - "name": "receivedEventsTotal", - "description": null, - "args": [], - "type": { - "kind": "OBJECT", - "name": "ReceivedEventsTotal", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "sentBytesTotal", - "description": null, - "args": [], - "type": { - "kind": "OBJECT", - "name": "SentBytesTotal", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "sentEventsTotal", - "description": null, - "args": [], - "type": { - "kind": "OBJECT", - "name": "SentEventsTotal", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": null, - "enumValues": null, - "possibleTypes": [ - { - "kind": "OBJECT", - "name": "GenericSinkMetrics", - "ofType": null - } - ] - }, - { - "kind": "INPUT_OBJECT", - "name": "SinksFilter", - "description": null, - "fields": null, - "inputFields": [ - { - "name": "componentId", - "description": null, - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INPUT_OBJECT", - "name": "StringFilter", - "ofType": null - } - } - }, - "defaultValue": null - }, - { - "name": "componentType", - "description": null, - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INPUT_OBJECT", - "name": "StringFilter", - "ofType": null - } - } - }, - "defaultValue": null - }, - { - "name": "or", - "description": null, - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INPUT_OBJECT", - "name": "SinksFilter", - "ofType": null - } - } - }, - "defaultValue": null - } - ], - "interfaces": null, - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "INPUT_OBJECT", - "name": "SinksSortField", - "description": null, - "fields": null, - "inputFields": [ - { - "name": "field", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "ENUM", - "name": "SinksSortFieldName", - "ofType": null - } - }, - "defaultValue": null - }, - { - "name": "direction", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "ENUM", - "name": "Direction", - "ofType": null - } - }, - "defaultValue": "ASC" - } - ], - "interfaces": null, - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "ENUM", - "name": "SinksSortFieldName", - "description": null, - "fields": null, - "inputFields": null, - "interfaces": null, - "enumValues": [ - { - "name": "COMPONENT_KEY", - "description": null, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "COMPONENT_TYPE", - "description": null, - "isDeprecated": false, - "deprecationReason": null - } - ], - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "Source", - "description": null, - "fields": [ - { - "name": "componentId", - "description": "Source component_id", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "componentType", - "description": "Source type", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "outputTypes", - "description": "Source output type", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "ENUM", - "name": "SourceOutputType", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "outputs", - "description": "Source output streams", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "Output", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "transforms", - "description": "Transform outputs", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "Transform", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "sinks", - "description": "Sink outputs", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "Sink", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "metrics", - "description": "Source metrics", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INTERFACE", - "name": "SourceMetrics", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [ - { - "kind": "INTERFACE", - "name": "Component", - "ofType": null - } - ], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "SourceConnection", - "description": null, - "fields": [ - { - "name": "pageInfo", - "description": "Information to aid in pagination.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "PageInfo", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "edges", - "description": "A list of edges.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "SourceEdge", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "nodes", - "description": "A list of nodes.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "Source", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "totalCount", - "description": "Total result set count", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "SourceEdge", - "description": "An edge in a connection.", - "fields": [ - { - "name": "cursor", - "description": "A cursor for use in pagination", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "node", - "description": "The item at the end of the edge", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "Source", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "INTERFACE", - "name": "SourceMetrics", - "description": null, - "fields": [ - { - "name": "receivedBytesTotal", - "description": null, - "args": [], - "type": { - "kind": "OBJECT", - "name": "ReceivedBytesTotal", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "receivedEventsTotal", - "description": null, - "args": [], - "type": { - "kind": "OBJECT", - "name": "ReceivedEventsTotal", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "sentEventsTotal", - "description": null, - "args": [], - "type": { - "kind": "OBJECT", - "name": "SentEventsTotal", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": null, - "enumValues": null, - "possibleTypes": [ - { - "kind": "OBJECT", - "name": "GenericSourceMetrics", - "ofType": null - }, - { - "kind": "OBJECT", - "name": "FileSourceMetrics", - "ofType": null - } - ] - }, - { - "kind": "ENUM", - "name": "SourceOutputType", - "description": null, - "fields": null, - "inputFields": null, - "interfaces": null, - "enumValues": [ - { - "name": "LOG", - "description": null, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "METRIC", - "description": null, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "TRACE", - "description": null, - "isDeprecated": false, - "deprecationReason": null - } - ], - "possibleTypes": null - }, - { - "kind": "INPUT_OBJECT", - "name": "SourceOutputTypeFilter", - "description": null, - "fields": null, - "inputFields": [ - { - "name": "equals", - "description": null, - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "ENUM", - "name": "SourceOutputType", - "ofType": null - } - } - }, - "defaultValue": null - }, - { - "name": "notEquals", - "description": null, - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "ENUM", - "name": "SourceOutputType", - "ofType": null - } - } - }, - "defaultValue": null - }, - { - "name": "contains", - "description": null, - "type": { - "kind": "ENUM", - "name": "SourceOutputType", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "notContains", - "description": null, - "type": { - "kind": "ENUM", - "name": "SourceOutputType", - "ofType": null - }, - "defaultValue": null - } - ], - "interfaces": null, - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "INPUT_OBJECT", - "name": "SourcesFilter", - "description": null, - "fields": null, - "inputFields": [ - { - "name": "componentId", - "description": null, - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INPUT_OBJECT", - "name": "StringFilter", - "ofType": null - } - } - }, - "defaultValue": null - }, - { - "name": "componentType", - "description": null, - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INPUT_OBJECT", - "name": "StringFilter", - "ofType": null - } - } - }, - "defaultValue": null - }, - { - "name": "outputType", - "description": null, - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INPUT_OBJECT", - "name": "SourceOutputTypeFilter", - "ofType": null - } - } - }, - "defaultValue": null - }, - { - "name": "or", - "description": null, - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INPUT_OBJECT", - "name": "SourcesFilter", - "ofType": null - } - } - }, - "defaultValue": null - } - ], - "interfaces": null, - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "INPUT_OBJECT", - "name": "SourcesSortField", - "description": null, - "fields": null, - "inputFields": [ - { - "name": "field", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "ENUM", - "name": "SourcesSortFieldName", - "ofType": null - } - }, - "defaultValue": null - }, - { - "name": "direction", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "ENUM", - "name": "Direction", - "ofType": null - } - }, - "defaultValue": "ASC" - } - ], - "interfaces": null, - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "ENUM", - "name": "SourcesSortFieldName", - "description": null, - "fields": null, - "inputFields": null, - "interfaces": null, - "enumValues": [ - { - "name": "COMPONENT_KEY", - "description": null, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "COMPONENT_TYPE", - "description": null, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "OUTPUT_TYPE", - "description": null, - "isDeprecated": false, - "deprecationReason": null - } - ], - "possibleTypes": null - }, - { - "kind": "SCALAR", - "name": "String", - "description": "The `String` scalar type represents textual data, represented as UTF-8\ncharacter sequences. The String type is most often used by GraphQL to\nrepresent free-form human-readable text.", - "fields": null, - "inputFields": null, - "interfaces": null, - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "INPUT_OBJECT", - "name": "StringFilter", - "description": "Filter for String values", - "fields": null, - "inputFields": [ - { - "name": "equals", - "description": null, - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "notEquals", - "description": null, - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "contains", - "description": null, - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "notContains", - "description": null, - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "startsWith", - "description": null, - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "endsWith", - "description": null, - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "defaultValue": null - } - ], - "interfaces": null, - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "Subscription", - "description": null, - "fields": [ - { - "name": "heartbeat", - "description": "Heartbeat, containing the UTC timestamp of the last server-sent payload", - "args": [ - { - "name": "interval", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "defaultValue": "1000" - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "Heartbeat", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "uptime", - "description": "Metrics for how long the Vector instance has been running", - "args": [ - { - "name": "interval", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "defaultValue": "1000" - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "Uptime", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "receivedEventsTotal", - "description": "Total received events metrics", - "args": [ - { - "name": "interval", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "defaultValue": "1000" - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "ReceivedEventsTotal", - "ofType": null - } - }, - "isDeprecated": true, - "deprecationReason": "Use component_received_events_totals instead" - }, - { - "name": "receivedEventsThroughput", - "description": "Total received events throughput sampled over the provided millisecond `interval`", - "args": [ - { - "name": "interval", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "defaultValue": "1000" - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "isDeprecated": true, - "deprecationReason": "Use component_received_events_throughputs instead" - }, - { - "name": "componentReceivedEventsThroughputs", - "description": "Total incoming component events throughput metrics over `interval`", - "args": [ - { - "name": "interval", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "defaultValue": "1000" - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "ComponentReceivedEventsThroughput", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "componentReceivedEventsTotals", - "description": "Total received component event metrics over `interval`", - "args": [ - { - "name": "interval", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "defaultValue": "1000" - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "ComponentReceivedEventsTotal", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "sentEventsTotal", - "description": "Total sent events metrics", - "args": [ - { - "name": "interval", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "defaultValue": "1000" - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "SentEventsTotal", - "ofType": null - } - }, - "isDeprecated": true, - "deprecationReason": "Use component_sent_events_totals instead" - }, - { - "name": "sentEventsThroughput", - "description": "Total outgoing events throughput sampled over the provided millisecond `interval`", - "args": [ - { - "name": "interval", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "defaultValue": "1000" - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "isDeprecated": true, - "deprecationReason": "Use component_sent_events_throughputs instead" - }, - { - "name": "componentSentEventsThroughputs", - "description": "Total outgoing component event throughput metrics over `interval`", - "args": [ - { - "name": "interval", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "defaultValue": "1000" - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "ComponentSentEventsThroughput", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "componentSentEventsTotals", - "description": "Total outgoing component event metrics over `interval`", - "args": [ - { - "name": "interval", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "defaultValue": "1000" - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "ComponentSentEventsTotal", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "componentReceivedBytesTotals", - "description": "Component bytes received metrics over `interval`.", - "args": [ - { - "name": "interval", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "defaultValue": "1000" - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "ComponentReceivedBytesTotal", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "componentReceivedBytesThroughputs", - "description": "Component bytes received throughput over `interval`", - "args": [ - { - "name": "interval", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "defaultValue": "1000" - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "ComponentReceivedBytesThroughput", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "componentSentBytesTotals", - "description": "Component bytes sent metrics over `interval`.", - "args": [ - { - "name": "interval", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "defaultValue": "1000" - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "ComponentSentBytesTotal", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "componentSentBytesThroughputs", - "description": "Component bytes sent throughput over `interval`", - "args": [ - { - "name": "interval", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "defaultValue": "1000" - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "ComponentSentBytesThroughput", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "errorsTotal", - "description": "Total error metrics.", - "args": [ - { - "name": "interval", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "defaultValue": "1000" - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "ErrorsTotal", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "allocatedBytes", - "description": "Allocated bytes metrics.", - "args": [ - { - "name": "interval", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "defaultValue": "1000" - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "AllocatedBytes", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "componentAllocatedBytes", - "description": "Component allocation metrics", - "args": [ - { - "name": "interval", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "defaultValue": "1000" - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "ComponentAllocatedBytes", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "componentErrorsTotals", - "description": "Component error metrics over `interval`.", - "args": [ - { - "name": "interval", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "defaultValue": "1000" - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "ComponentErrorsTotal", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "metrics", - "description": "All metrics.", - "args": [ - { - "name": "interval", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "defaultValue": "1000" - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INTERFACE", - "name": "MetricType", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "componentAdded", - "description": "Subscribes to all newly added components", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INTERFACE", - "name": "Component", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "componentRemoved", - "description": "Subscribes to all removed components", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INTERFACE", - "name": "Component", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "outputEventsByComponentIdPatterns", - "description": "A stream of events emitted from matched component ID patterns", - "args": [ - { - "name": "outputsPatterns", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - } - } - }, - "defaultValue": null - }, - { - "name": "inputsPatterns", - "description": null, - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - } - }, - "defaultValue": null - }, - { - "name": "interval", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "defaultValue": "500" - }, - { - "name": "limit", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "defaultValue": "100" - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "UNION", - "name": "OutputEventsPayload", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "SwapMetrics", - "description": null, - "fields": [ - { - "name": "freeBytes", - "description": "Swap free bytes", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "totalBytes", - "description": "Swap total bytes", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "usedBytes", - "description": "Swap used bytes", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "swappedInBytesTotal", - "description": "Swapped in bytes total (not available on Windows)", - "args": [], - "type": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "swappedOutBytesTotal", - "description": "Swapped out bytes total (not available on Windows)", - "args": [], - "type": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "Trace", - "description": "Trace event with fields for querying trace data", - "fields": [ - { - "name": "componentId", - "description": "Id of the component associated with the trace event", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "componentType", - "description": "Type of component associated with the trace event", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "componentKind", - "description": "Kind of component associated with the trace event", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "string", - "description": "Trace event as an encoded string format", - "args": [ - { - "name": "encoding", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "ENUM", - "name": "EventEncodingType", - "ofType": null - } - }, - "defaultValue": null - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "json", - "description": "Get JSON field data on the trace event, by field name", - "args": [ - { - "name": "field", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "defaultValue": null - } - ], - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "Transform", - "description": null, - "fields": [ - { - "name": "componentId", - "description": "Transform component_id", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "componentType", - "description": "Transform type", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "outputs", - "description": "Transform output streams", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "Output", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "sources", - "description": "Source inputs", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "Source", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "transforms", - "description": "Transform outputs", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "Transform", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "sinks", - "description": "Sink outputs", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "Sink", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "metrics", - "description": "Transform metrics", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INTERFACE", - "name": "TransformMetrics", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [ - { - "kind": "INTERFACE", - "name": "Component", - "ofType": null - } - ], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "TransformConnection", - "description": null, - "fields": [ - { - "name": "pageInfo", - "description": "Information to aid in pagination.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "PageInfo", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "edges", - "description": "A list of edges.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "TransformEdge", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "nodes", - "description": "A list of nodes.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "Transform", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "totalCount", - "description": "Total result set count", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "TransformEdge", - "description": "An edge in a connection.", - "fields": [ - { - "name": "cursor", - "description": "A cursor for use in pagination", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "node", - "description": "The item at the end of the edge", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "Transform", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "INTERFACE", - "name": "TransformMetrics", - "description": null, - "fields": [ - { - "name": "receivedEventsTotal", - "description": null, - "args": [], - "type": { - "kind": "OBJECT", - "name": "ReceivedEventsTotal", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "sentEventsTotal", - "description": null, - "args": [], - "type": { - "kind": "OBJECT", - "name": "SentEventsTotal", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": null, - "enumValues": null, - "possibleTypes": [ - { - "kind": "OBJECT", - "name": "GenericTransformMetrics", - "ofType": null - } - ] - }, - { - "kind": "INPUT_OBJECT", - "name": "TransformsFilter", - "description": null, - "fields": null, - "inputFields": [ - { - "name": "componentId", - "description": null, - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INPUT_OBJECT", - "name": "StringFilter", - "ofType": null - } - } - }, - "defaultValue": null - }, - { - "name": "componentType", - "description": null, - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INPUT_OBJECT", - "name": "StringFilter", - "ofType": null - } - } - }, - "defaultValue": null - }, - { - "name": "or", - "description": null, - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INPUT_OBJECT", - "name": "TransformsFilter", - "ofType": null - } - } - }, - "defaultValue": null - } - ], - "interfaces": null, - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "INPUT_OBJECT", - "name": "TransformsSortField", - "description": null, - "fields": null, - "inputFields": [ - { - "name": "field", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "ENUM", - "name": "TransformsSortFieldName", - "ofType": null - } - }, - "defaultValue": null - }, - { - "name": "direction", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "ENUM", - "name": "Direction", - "ofType": null - } - }, - "defaultValue": "ASC" - } - ], - "interfaces": null, - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "ENUM", - "name": "TransformsSortFieldName", - "description": null, - "fields": null, - "inputFields": null, - "interfaces": null, - "enumValues": [ - { - "name": "COMPONENT_KEY", - "description": null, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "COMPONENT_TYPE", - "description": null, - "isDeprecated": false, - "deprecationReason": null - } - ], - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "Uptime", - "description": null, - "fields": [ - { - "name": "timestamp", - "description": "Metric timestamp", - "args": [], - "type": { - "kind": "SCALAR", - "name": "DateTime", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "seconds", - "description": "Number of seconds the Vector instance has been alive", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Float", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [ - { - "kind": "INTERFACE", - "name": "MetricType", - "ofType": null - } - ], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "__Directive", - "description": "A Directive provides a way to describe alternate runtime execution and type\nvalidation behavior in a GraphQL document.\n\nIn some cases, you need to provide options to alter GraphQL's execution\nbehavior in ways field arguments will not suffice, such as conditionally\nincluding or skipping a field. Directives provide this by describing\nadditional information to the executor.", - "fields": [ - { - "name": "name", - "description": null, - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "description", - "description": null, - "args": [], - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "locations", - "description": null, - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "ENUM", - "name": "__DirectiveLocation", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "args", - "description": null, - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "__InputValue", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "isRepeatable", - "description": null, - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Boolean", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "ENUM", - "name": "__DirectiveLocation", - "description": "A Directive can be adjacent to many parts of the GraphQL language, a\n__DirectiveLocation describes one such possible adjacencies.", - "fields": null, - "inputFields": null, - "interfaces": null, - "enumValues": [ - { - "name": "QUERY", - "description": "Location adjacent to a query operation.", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "MUTATION", - "description": "Location adjacent to a mutation operation.", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "SUBSCRIPTION", - "description": "Location adjacent to a subscription operation.", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "FIELD", - "description": "Location adjacent to a field.", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "FRAGMENT_DEFINITION", - "description": "Location adjacent to a fragment definition.", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "FRAGMENT_SPREAD", - "description": "Location adjacent to a fragment spread.", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "INLINE_FRAGMENT", - "description": "Location adjacent to an inline fragment.", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "VARIABLE_DEFINITION", - "description": "Location adjacent to a variable definition.", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "SCHEMA", - "description": "Location adjacent to a schema definition.", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "SCALAR", - "description": "Location adjacent to a scalar definition.", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "OBJECT", - "description": "Location adjacent to an object type definition.", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "FIELD_DEFINITION", - "description": "Location adjacent to a field definition.", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "ARGUMENT_DEFINITION", - "description": "Location adjacent to an argument definition.", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "INTERFACE", - "description": "Location adjacent to an interface definition.", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "UNION", - "description": "Location adjacent to a union definition.", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "ENUM", - "description": "Location adjacent to an enum definition.", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "ENUM_VALUE", - "description": "Location adjacent to an enum value definition.", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "INPUT_OBJECT", - "description": "Location adjacent to an input object type definition.", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "INPUT_FIELD_DEFINITION", - "description": "Location adjacent to an input object field definition.", - "isDeprecated": false, - "deprecationReason": null - } - ], - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "__EnumValue", - "description": "One possible value for a given Enum. Enum values are unique values, not a\nplaceholder for a string or numeric value. However an Enum value is returned\nin a JSON response as a string.", - "fields": [ - { - "name": "name", - "description": null, - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "description", - "description": null, - "args": [], - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "isDeprecated", - "description": null, - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Boolean", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "deprecationReason", - "description": null, - "args": [], - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "__Field", - "description": "Object and Interface types are described by a list of Fields, each of which\nhas a name, potentially a list of arguments, and a return type.", - "fields": [ - { - "name": "name", - "description": null, - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "description", - "description": null, - "args": [], - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "args", - "description": null, - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "__InputValue", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "type", - "description": null, - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "__Type", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "isDeprecated", - "description": null, - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Boolean", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "deprecationReason", - "description": null, - "args": [], - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "__InputValue", - "description": "Arguments provided to Fields or Directives and the input fields of an\nInputObject are represented as Input Values which describe their type and\noptionally a default value.", - "fields": [ - { - "name": "name", - "description": null, - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "String", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "description", - "description": null, - "args": [], - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "type", - "description": null, - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "__Type", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "defaultValue", - "description": null, - "args": [], - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "__Schema", - "description": "A GraphQL Schema defines the capabilities of a GraphQL server. It exposes\nall available types and directives on the server, as well as the entry\npoints for query, mutation, and subscription operations.", - "fields": [ - { - "name": "types", - "description": "A list of all types supported by this server.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "__Type", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "queryType", - "description": "The type that query operations will be rooted at.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "__Type", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "mutationType", - "description": "If this server supports mutation, the type that mutation operations will\nbe rooted at.", - "args": [], - "type": { - "kind": "OBJECT", - "name": "__Type", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "subscriptionType", - "description": "If this server support subscription, the type that subscription\noperations will be rooted at.", - "args": [], - "type": { - "kind": "OBJECT", - "name": "__Type", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "directives", - "description": "A list of all directives supported by this server.", - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "__Directive", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "OBJECT", - "name": "__Type", - "description": "The fundamental unit of any GraphQL Schema is the type. There are many kinds\nof types in GraphQL as represented by the `__TypeKind` enum.\n\nDepending on the kind of a type, certain fields describe information about\nthat type. Scalar types provide no information beyond a name and\ndescription, while Enum types provide their values. Object and Interface\ntypes provide the fields they describe. Abstract types, Union and Interface,\nprovide the Object types possible at runtime. List and NonNull types compose\nother types.", - "fields": [ - { - "name": "kind", - "description": null, - "args": [], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "ENUM", - "name": "__TypeKind", - "ofType": null - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "name", - "description": null, - "args": [], - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "description", - "description": null, - "args": [], - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "fields", - "description": null, - "args": [ - { - "name": "includeDeprecated", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Boolean", - "ofType": null - } - }, - "defaultValue": "false" - } - ], - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "__Field", - "ofType": null - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "interfaces", - "description": null, - "args": [], - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "__Type", - "ofType": null - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "possibleTypes", - "description": null, - "args": [], - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "__Type", - "ofType": null - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "enumValues", - "description": null, - "args": [ - { - "name": "includeDeprecated", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Boolean", - "ofType": null - } - }, - "defaultValue": "false" - } - ], - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "__EnumValue", - "ofType": null - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "inputFields", - "description": null, - "args": [], - "type": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "__InputValue", - "ofType": null - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "ofType", - "description": null, - "args": [], - "type": { - "kind": "OBJECT", - "name": "__Type", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "specifiedByURL", - "description": null, - "args": [], - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "isOneOf", - "description": null, - "args": [], - "type": { - "kind": "SCALAR", - "name": "Boolean", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, - { - "kind": "ENUM", - "name": "__TypeKind", - "description": "An enum describing what kind of type a given `__Type` is.", - "fields": null, - "inputFields": null, - "interfaces": null, - "enumValues": [ - { - "name": "SCALAR", - "description": "Indicates this type is a scalar.", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "OBJECT", - "description": "Indicates this type is an object. `fields` and `interfaces` are valid\nfields.", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "INTERFACE", - "description": "Indicates this type is an interface. `fields` and `possibleTypes` are\nvalid fields.", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "UNION", - "description": "Indicates this type is a union. `possibleTypes` is a valid field.", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "ENUM", - "description": "Indicates this type is an enum. `enumValues` is a valid field.", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "INPUT_OBJECT", - "description": "Indicates this type is an input object. `inputFields` is a valid field.", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "LIST", - "description": "Indicates this type is a list. `ofType` is a valid field.", - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "NON_NULL", - "description": "Indicates this type is a non-null. `ofType` is a valid field.", - "isDeprecated": false, - "deprecationReason": null - } - ], - "possibleTypes": null - } - ], - "directives": [ - { - "name": "include", - "description": "Directs the executor to include this field or fragment only when the `if` argument is true.", - "locations": [ - "FIELD", - "FRAGMENT_SPREAD", - "INLINE_FRAGMENT" - ], - "args": [ - { - "name": "if", - "description": "Included when true.", - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Boolean", - "ofType": null - } - }, - "defaultValue": null - } - ] - }, - { - "name": "skip", - "description": "Directs the executor to skip this field or fragment when the `if` argument is true.", - "locations": [ - "FIELD", - "FRAGMENT_SPREAD", - "INLINE_FRAGMENT" - ], - "args": [ - { - "name": "if", - "description": "Skipped when true.", - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "Boolean", - "ofType": null - } - }, - "defaultValue": null - } - ] - } - ] - } - } -} diff --git a/lib/vector-api-client/graphql/subscriptions/component_added.graphql b/lib/vector-api-client/graphql/subscriptions/component_added.graphql deleted file mode 100644 index 9197209bc053b..0000000000000 --- a/lib/vector-api-client/graphql/subscriptions/component_added.graphql +++ /dev/null @@ -1,7 +0,0 @@ -subscription ComponentAddedSubscription { - componentAdded { - __typename - componentId - componentType - } -} diff --git a/lib/vector-api-client/graphql/subscriptions/component_allocated_bytes.graphql b/lib/vector-api-client/graphql/subscriptions/component_allocated_bytes.graphql deleted file mode 100644 index 2624d3dc8dc75..0000000000000 --- a/lib/vector-api-client/graphql/subscriptions/component_allocated_bytes.graphql +++ /dev/null @@ -1,8 +0,0 @@ -subscription ComponentAllocatedBytesSubscription ($interval: Int!) { - componentAllocatedBytes(interval: $interval) { - componentId - metric { - allocatedBytes - } - } -} diff --git a/lib/vector-api-client/graphql/subscriptions/component_errors_totals.graphql b/lib/vector-api-client/graphql/subscriptions/component_errors_totals.graphql deleted file mode 100644 index d8e0b2554e74f..0000000000000 --- a/lib/vector-api-client/graphql/subscriptions/component_errors_totals.graphql +++ /dev/null @@ -1,8 +0,0 @@ -subscription ComponentErrorsTotalsSubscription ($interval: Int!) { - componentErrorsTotals(interval: $interval) { - componentId - metric { - errorsTotal - } - } -} diff --git a/lib/vector-api-client/graphql/subscriptions/component_received_bytes_throughputs.graphql b/lib/vector-api-client/graphql/subscriptions/component_received_bytes_throughputs.graphql deleted file mode 100644 index ec1b018d27107..0000000000000 --- a/lib/vector-api-client/graphql/subscriptions/component_received_bytes_throughputs.graphql +++ /dev/null @@ -1,6 +0,0 @@ -subscription ComponentReceivedBytesThroughputsSubscription($interval: Int!) { - componentReceivedBytesThroughputs(interval: $interval) { - componentId - throughput - } -} diff --git a/lib/vector-api-client/graphql/subscriptions/component_received_bytes_totals.graphql b/lib/vector-api-client/graphql/subscriptions/component_received_bytes_totals.graphql deleted file mode 100644 index 748b8930fb087..0000000000000 --- a/lib/vector-api-client/graphql/subscriptions/component_received_bytes_totals.graphql +++ /dev/null @@ -1,8 +0,0 @@ -subscription ComponentReceivedBytesTotalsSubscription($interval: Int!) { - componentReceivedBytesTotals(interval: $interval) { - componentId - metric { - receivedBytesTotal - } - } -} diff --git a/lib/vector-api-client/graphql/subscriptions/component_received_events_throughputs.graphql b/lib/vector-api-client/graphql/subscriptions/component_received_events_throughputs.graphql deleted file mode 100644 index 6ddea6400db92..0000000000000 --- a/lib/vector-api-client/graphql/subscriptions/component_received_events_throughputs.graphql +++ /dev/null @@ -1,6 +0,0 @@ -subscription ComponentReceivedEventsThroughputsSubscription($interval: Int!) { - componentReceivedEventsThroughputs(interval: $interval) { - componentId - throughput - } -} diff --git a/lib/vector-api-client/graphql/subscriptions/component_received_events_totals.graphql b/lib/vector-api-client/graphql/subscriptions/component_received_events_totals.graphql deleted file mode 100644 index 59bb58a614067..0000000000000 --- a/lib/vector-api-client/graphql/subscriptions/component_received_events_totals.graphql +++ /dev/null @@ -1,8 +0,0 @@ -subscription ComponentReceivedEventsTotalsSubscription($interval: Int!) { - componentReceivedEventsTotals(interval: $interval) { - componentId - metric { - receivedEventsTotal - } - } -} diff --git a/lib/vector-api-client/graphql/subscriptions/component_removed.graphql b/lib/vector-api-client/graphql/subscriptions/component_removed.graphql deleted file mode 100644 index 7232522a20498..0000000000000 --- a/lib/vector-api-client/graphql/subscriptions/component_removed.graphql +++ /dev/null @@ -1,6 +0,0 @@ -subscription ComponentRemovedSubscription { - componentRemoved { - __typename - componentId - } -} diff --git a/lib/vector-api-client/graphql/subscriptions/component_sent_bytes_throughputs.graphql b/lib/vector-api-client/graphql/subscriptions/component_sent_bytes_throughputs.graphql deleted file mode 100644 index a1b4f54c1a300..0000000000000 --- a/lib/vector-api-client/graphql/subscriptions/component_sent_bytes_throughputs.graphql +++ /dev/null @@ -1,6 +0,0 @@ -subscription ComponentSentBytesThroughputsSubscription($interval: Int!) { - componentSentBytesThroughputs(interval: $interval) { - componentId - throughput - } -} diff --git a/lib/vector-api-client/graphql/subscriptions/component_sent_bytes_totals.graphql b/lib/vector-api-client/graphql/subscriptions/component_sent_bytes_totals.graphql deleted file mode 100644 index aa2c2f6dd0832..0000000000000 --- a/lib/vector-api-client/graphql/subscriptions/component_sent_bytes_totals.graphql +++ /dev/null @@ -1,8 +0,0 @@ -subscription ComponentSentBytesTotalsSubscription($interval: Int!) { - componentSentBytesTotals(interval: $interval) { - componentId - metric { - sentBytesTotal - } - } -} diff --git a/lib/vector-api-client/graphql/subscriptions/component_sent_events_throughputs.graphql b/lib/vector-api-client/graphql/subscriptions/component_sent_events_throughputs.graphql deleted file mode 100644 index fe6c86298b0af..0000000000000 --- a/lib/vector-api-client/graphql/subscriptions/component_sent_events_throughputs.graphql +++ /dev/null @@ -1,10 +0,0 @@ -subscription ComponentSentEventsThroughputsSubscription($interval: Int!) { - componentSentEventsThroughputs(interval: $interval) { - componentId - outputs { - outputId - throughput - } - throughput - } -} diff --git a/lib/vector-api-client/graphql/subscriptions/component_sent_events_totals.graphql b/lib/vector-api-client/graphql/subscriptions/component_sent_events_totals.graphql deleted file mode 100644 index 9951c53e3ab16..0000000000000 --- a/lib/vector-api-client/graphql/subscriptions/component_sent_events_totals.graphql +++ /dev/null @@ -1,14 +0,0 @@ -subscription ComponentSentEventsTotalsSubscription($interval: Int!) { - componentSentEventsTotals(interval: $interval) { - componentId - outputs { - outputId - sentEventsTotal { - sentEventsTotal - } - } - metric { - sentEventsTotal - } - } -} diff --git a/lib/vector-api-client/graphql/subscriptions/heartbeat.graphql b/lib/vector-api-client/graphql/subscriptions/heartbeat.graphql deleted file mode 100644 index f5ae0b40f6d6b..0000000000000 --- a/lib/vector-api-client/graphql/subscriptions/heartbeat.graphql +++ /dev/null @@ -1,5 +0,0 @@ -subscription HeartbeatSubscription($interval: Int!) { - heartbeat(interval: $interval) { - utc - } -} diff --git a/lib/vector-api-client/graphql/subscriptions/output_events_by_component_id_patterns.graphql b/lib/vector-api-client/graphql/subscriptions/output_events_by_component_id_patterns.graphql deleted file mode 100644 index 6c17e197b8e7e..0000000000000 --- a/lib/vector-api-client/graphql/subscriptions/output_events_by_component_id_patterns.graphql +++ /dev/null @@ -1,30 +0,0 @@ -subscription OutputEventsByComponentIdPatternsSubscription( - $outputsPatterns: [String!]!, $inputsPatterns: [String!], $limit: Int!, $interval: Int!, $encoding: EventEncodingType!){ - outputEventsByComponentIdPatterns(outputsPatterns: $outputsPatterns, inputsPatterns: $inputsPatterns, limit: $limit, interval: $interval) { - __typename - ... on Log { - componentId - componentType - componentKind - message - timestamp - string(encoding: $encoding) - } - ... on Metric { - componentId - componentType - componentKind - timestamp - string(encoding: $encoding) - } - ... on Trace { - componentId - componentType - componentKind - string(encoding: $encoding) - } - ... on EventNotification { - message - } - } -} diff --git a/lib/vector-api-client/graphql/subscriptions/processed_bytes_throughput.graphql b/lib/vector-api-client/graphql/subscriptions/processed_bytes_throughput.graphql deleted file mode 100644 index d8dabdecc5ed4..0000000000000 --- a/lib/vector-api-client/graphql/subscriptions/processed_bytes_throughput.graphql +++ /dev/null @@ -1,3 +0,0 @@ -subscription ProcessedBytesThroughputSubscription($interval: Int!) { - processedBytesThroughput(interval: $interval) -} diff --git a/lib/vector-api-client/graphql/subscriptions/uptime.graphql b/lib/vector-api-client/graphql/subscriptions/uptime.graphql deleted file mode 100644 index 187d1e861877a..0000000000000 --- a/lib/vector-api-client/graphql/subscriptions/uptime.graphql +++ /dev/null @@ -1,5 +0,0 @@ -subscription UptimeSubscription { - uptime { - seconds - } -} diff --git a/lib/vector-api-client/src/client.rs b/lib/vector-api-client/src/client.rs index 941ffa7963f27..0d8fda1b15aaf 100644 --- a/lib/vector-api-client/src/client.rs +++ b/lib/vector-api-client/src/client.rs @@ -1,57 +1,221 @@ -use anyhow::Context; -use graphql_client::GraphQLQuery; -use url::Url; +use http::Uri; +use tokio_stream::{Stream, StreamExt}; +use tonic::transport::{Channel, Endpoint}; +use tonic_health::pb::{ + HealthCheckRequest, health_check_response::ServingStatus, health_client::HealthClient, +}; -use crate::gql::HealthQueryExt; +use crate::{ + error::{Error, Result}, + proto::{ + GetAllocationTracingStatusRequest, GetAllocationTracingStatusResponse, + GetComponentsRequest, GetComponentsResponse, GetMetaRequest, GetMetaResponse, MetricName, + StreamComponentAllocatedBytesRequest, StreamComponentAllocatedBytesResponse, + StreamComponentMetricsRequest, StreamComponentMetricsResponse, StreamHeartbeatRequest, + StreamHeartbeatResponse, StreamOutputEventsRequest, StreamOutputEventsResponse, + StreamUptimeRequest, StreamUptimeResponse, + observability_service_client::ObservabilityServiceClient, + }, +}; -/// Wrapped `Result` type, that returns deserialized GraphQL response data. -pub type QueryResult = - anyhow::Result::ResponseData>>; - -/// GraphQL query client over HTTP. -#[derive(Debug)] +/// gRPC client for the Vector observability API +#[derive(Debug, Clone)] pub struct Client { - url: Url, + endpoint: Endpoint, + channel: Option, + client: Option>, } impl Client { - /// Returns a new GraphQL query client, bound to the provided URL. - pub fn new(url: Url) -> Self { - Self { url } - } - - /// Send a health query - pub async fn healthcheck(&self) -> Result<(), ()> { - self.health_query().await.map(|_| ()).map_err(|_| ()) - } - - /// Issue a GraphQL query using Reqwest, serializing the response to the associated - /// GraphQL type for the given `request_body`. - pub async fn query( - &self, - request_body: &graphql_client::QueryBody, - ) -> QueryResult { - let client = reqwest::Client::new(); - - client - .post(self.url.clone()) - .json(request_body) - .send() - .await - .with_context(|| { - format!( - "Couldn't send '{}' query to {}", - request_body.operation_name, - &self.url.as_str() - ) - })? - .json() - .await - .with_context(|| { - format!( - "Couldn't serialize the response for '{}' query: {:?}", - request_body.operation_name, request_body.query - ) + /// Create a new gRPC client + /// + /// The client is not connected until `connect()` is called. + /// + /// # Arguments + /// + /// * `uri` - The gRPC server URI (e.g., `"http://localhost:9999".parse().unwrap()`) + pub fn new(uri: Uri) -> Self { + Self { + endpoint: Endpoint::from(uri), + channel: None, + client: None, + } + } + + /// Connect to the gRPC server + pub async fn connect(&mut self) -> Result<()> { + let channel = self.endpoint.connect().await?; + self.client = Some(ObservabilityServiceClient::new(channel.clone())); + self.channel = Some(channel); + Ok(()) + } + + /// Ensure the client is connected + fn ensure_connected(&mut self) -> Result<&mut ObservabilityServiceClient> { + self.client.as_mut().ok_or(Error::NotConnected) + } + + /// Get the underlying channel + fn channel(&self) -> Result<&Channel> { + self.channel.as_ref().ok_or(Error::NotConnected) + } + + // ========== Unary RPCs ========== + + /// Check if the API server is healthy using the standard gRPC health check + /// protocol (grpc.health.v1.Health/Check). + /// + /// Queries the empty service name (`""`), which represents whole-server + /// health. This is the default used by Kubernetes gRPC probes and + /// `grpc-health-probe`. + /// + /// Returns `Ok(())` if the server is `SERVING`, or an error otherwise. + pub async fn health(&mut self) -> Result<()> { + let channel = self.channel()?.clone(); + let mut health_client = HealthClient::new(channel); + let response = health_client + .check(HealthCheckRequest { + service: String::new(), + }) + .await?; + let status = response.into_inner().status; + if status != ServingStatus::Serving as i32 { + return Err(Error::NotServing { status }); + } + Ok(()) + } + + /// Get metadata about the Vector instance + pub async fn get_meta(&mut self) -> Result { + let client = self.ensure_connected()?; + let response = client.get_meta(GetMetaRequest {}).await?; + Ok(response.into_inner()) + } + + /// Get information about configured components + /// + /// # Arguments + /// + /// * `limit` - Maximum number of components to return (0 = no limit) + pub async fn get_components(&mut self, limit: i32) -> Result { + let client = self.ensure_connected()?; + let response = client + .get_components(GetComponentsRequest { limit }) + .await?; + Ok(response.into_inner()) + } + + /// Check whether allocation tracing is active on the connected Vector instance + pub async fn get_allocation_tracing_status( + &mut self, + ) -> Result { + let client = self.ensure_connected()?; + let response = client + .get_allocation_tracing_status(GetAllocationTracingStatusRequest {}) + .await?; + Ok(response.into_inner()) + } + + // ========== Streaming RPCs ========== + + /// Stream periodic heartbeat timestamps + /// + /// # Arguments + /// + /// * `interval_ms` - Update interval in milliseconds + pub async fn stream_heartbeat( + &mut self, + interval_ms: i32, + ) -> Result>> { + let client = self.ensure_connected()?; + let response = client + .stream_heartbeat(StreamHeartbeatRequest { interval_ms }) + .await?; + Ok(response.into_inner().map(|r| r.map_err(Error::from))) + } + + /// Stream uptime in seconds + /// + /// # Arguments + /// + /// * `interval_ms` - Update interval in milliseconds + pub async fn stream_uptime( + &mut self, + interval_ms: i32, + ) -> Result>> { + let client = self.ensure_connected()?; + let response = client + .stream_uptime(StreamUptimeRequest { interval_ms }) + .await?; + Ok(response.into_inner().map(|r| r.map_err(Error::from))) + } + + /// Stream memory allocated per component + /// + /// # Arguments + /// + /// * `interval_ms` - Update interval in milliseconds + pub async fn stream_component_allocated_bytes( + &mut self, + interval_ms: i32, + ) -> Result>> { + let client = self.ensure_connected()?; + let response = client + .stream_component_allocated_bytes(StreamComponentAllocatedBytesRequest { interval_ms }) + .await?; + Ok(response.into_inner().map(|r| r.map_err(Error::from))) + } + + /// Stream per-component metrics for a chosen metric name. + /// + /// # Arguments + /// + /// * `metric` - Which metric to stream + /// * `interval_ms` - Update interval in milliseconds + pub async fn stream_component_metrics( + &mut self, + metric: MetricName, + interval_ms: i32, + ) -> Result>> { + let client = self.ensure_connected()?; + let response = client + .stream_component_metrics(StreamComponentMetricsRequest { + interval_ms, + metric: metric as i32, }) + .await?; + Ok(response.into_inner().map(|r| r.map_err(Error::from))) + } + + /// Stream events from components matching patterns + /// + /// This is used by `vector tap` to capture events. + pub async fn stream_output_events( + &mut self, + request: StreamOutputEventsRequest, + ) -> Result> + use<>> { + let client = self.ensure_connected()?; + let response = client.stream_output_events(request).await?; + Ok(response.into_inner().map(|r| r.map_err(Error::from))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_not_connected_error() { + let mut client = Client::new("http://localhost:9999".parse().unwrap()); + let result = client.health().await; + assert!(matches!(result, Err(Error::NotConnected))); + } + + #[test] + fn test_ensure_connected() { + let mut client = Client::new("http://localhost:9999".parse().unwrap()); + let result = client.ensure_connected(); + assert!(result.is_err()); + assert!(matches!(result.unwrap_err(), Error::NotConnected)); } } diff --git a/lib/vector-api-client/src/error.rs b/lib/vector-api-client/src/error.rs new file mode 100644 index 0000000000000..34eb347af12d8 --- /dev/null +++ b/lib/vector-api-client/src/error.rs @@ -0,0 +1,80 @@ +use snafu::Snafu; + +/// Error types for the gRPC client +#[derive(Debug, Snafu)] +#[snafu(visibility(pub))] +pub enum Error { + #[snafu(display("Connection error: {}", source))] + Connection { source: tonic::transport::Error }, + + #[snafu(display("gRPC error: {}", source))] + Grpc { source: tonic::Status }, + + #[snafu(display("Invalid URL: {}", message))] + InvalidUrl { message: String }, + + #[snafu(display("Not connected to gRPC server"))] + NotConnected, + + #[snafu(display("Server is not serving (status: {})", status))] + NotServing { status: i32 }, + + #[snafu(display("Stream error: {}", message))] + Stream { message: String }, +} + +impl Error { + /// Returns true if this error is permanent and should not be retried. + pub fn is_fatal(&self) -> bool { + match self { + Error::InvalidUrl { .. } => true, + Error::Grpc { source } => matches!( + source.code(), + tonic::Code::InvalidArgument + | tonic::Code::OutOfRange + | tonic::Code::NotFound + | tonic::Code::PermissionDenied + | tonic::Code::Unauthenticated + | tonic::Code::Unimplemented + ), + _ => false, + } + } +} + +impl From for Error { + fn from(source: tonic::Status) -> Self { + Error::Grpc { source } + } +} + +impl From for Error { + fn from(source: tonic::transport::Error) -> Self { + Error::Connection { source } + } +} + +pub type Result = std::result::Result; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_error_display() { + let err = Error::NotConnected; + assert_eq!(err.to_string(), "Not connected to gRPC server"); + + let err = Error::InvalidUrl { + message: "invalid format".to_string(), + }; + assert_eq!(err.to_string(), "Invalid URL: invalid format"); + } + + #[test] + fn test_error_from_tonic_status() { + let status = tonic::Status::unavailable("service down"); + let err: Error = status.into(); + assert!(matches!(err, Error::Grpc { .. })); + } +} diff --git a/lib/vector-api-client/src/gql/components.rs b/lib/vector-api-client/src/gql/components.rs deleted file mode 100644 index d278b30c317c8..0000000000000 --- a/lib/vector-api-client/src/gql/components.rs +++ /dev/null @@ -1,224 +0,0 @@ -use std::fmt; - -use graphql_client::GraphQLQuery; - -use crate::{BoxedSubscription, QueryResult}; - -/// Components query for returning sources, transforms, and sinks -#[derive(GraphQLQuery, Debug, Copy, Clone)] -#[graphql( - schema_path = "graphql/schema.json", - query_path = "graphql/queries/components.graphql", - response_derives = "Debug" -)] -pub struct ComponentsQuery; - -/// Components subscription for notification when a component has been added -#[derive(GraphQLQuery, Debug, Copy, Clone)] -#[graphql( - schema_path = "graphql/schema.json", - query_path = "graphql/subscriptions/component_added.graphql", - response_derives = "Debug" -)] -pub struct ComponentAddedSubscription; - -/// Components subscription for notification when a component has been removed -#[derive(GraphQLQuery, Debug, Copy, Clone)] -#[graphql( - schema_path = "graphql/schema.json", - query_path = "graphql/subscriptions/component_removed.graphql", - response_derives = "Debug" -)] -pub struct ComponentRemovedSubscription; - -pub trait ComponentsQueryExt { - async fn components_query(&self, first: i64) -> crate::QueryResult; -} - -impl ComponentsQueryExt for crate::Client { - async fn components_query(&self, first: i64) -> QueryResult { - let request_body = ComponentsQuery::build_query(components_query::Variables { first }); - self.query::(&request_body).await - } -} - -pub trait ComponentsSubscriptionExt { - fn component_added(&self) -> crate::BoxedSubscription; - fn component_removed(&self) -> crate::BoxedSubscription; -} - -impl ComponentsSubscriptionExt for crate::SubscriptionClient { - /// Subscription for when a component has been added - fn component_added(&self) -> BoxedSubscription { - let request_body = - ComponentAddedSubscription::build_query(component_added_subscription::Variables); - - self.start::(&request_body) - } - - /// Subscription for when a component has been removed - fn component_removed(&self) -> BoxedSubscription { - let request_body = - ComponentRemovedSubscription::build_query(component_removed_subscription::Variables); - - self.start::(&request_body) - } -} - -impl components_query::ComponentsQueryComponentsEdgesNodeOn { - pub fn received_bytes_total(&self) -> i64 { - // This is network bytes received, and only sources can receive events. - match self { - components_query::ComponentsQueryComponentsEdgesNodeOn::Source(s) => s - .metrics - .received_bytes_total - .as_ref() - .map(|p| p.received_bytes_total as i64) - .unwrap_or(0), - components_query::ComponentsQueryComponentsEdgesNodeOn::Transform(_) => 0, - components_query::ComponentsQueryComponentsEdgesNodeOn::Sink(_) => 0, - } - } - - pub fn received_events_total(&self) -> i64 { - match self { - components_query::ComponentsQueryComponentsEdgesNodeOn::Source(s) => s - .metrics - .received_events_total - .as_ref() - .map(|p| p.received_events_total as i64) - .unwrap_or(0), - components_query::ComponentsQueryComponentsEdgesNodeOn::Transform(t) => t - .metrics - .received_events_total - .as_ref() - .map(|p| p.received_events_total as i64) - .unwrap_or(0), - components_query::ComponentsQueryComponentsEdgesNodeOn::Sink(s) => s - .metrics - .received_events_total - .as_ref() - .map(|p| p.received_events_total as i64) - .unwrap_or(0), - } - } - - pub fn sent_bytes_total(&self) -> i64 { - // This is network bytes sent, and only sinks can send out events. - match self { - components_query::ComponentsQueryComponentsEdgesNodeOn::Source(_) => 0, - components_query::ComponentsQueryComponentsEdgesNodeOn::Transform(_) => 0, - components_query::ComponentsQueryComponentsEdgesNodeOn::Sink(s) => s - .metrics - .sent_bytes_total - .as_ref() - .map(|p| p.sent_bytes_total as i64) - .unwrap_or(0), - } - } - - pub fn sent_events_total(&self) -> i64 { - match self { - components_query::ComponentsQueryComponentsEdgesNodeOn::Source(s) => s - .metrics - .sent_events_total - .as_ref() - .map(|p| p.sent_events_total as i64) - .unwrap_or(0), - components_query::ComponentsQueryComponentsEdgesNodeOn::Transform(t) => t - .metrics - .sent_events_total - .as_ref() - .map(|p| p.sent_events_total as i64) - .unwrap_or(0), - components_query::ComponentsQueryComponentsEdgesNodeOn::Sink(s) => s - .metrics - .sent_events_total - .as_ref() - .map(|p| p.sent_events_total as i64) - .unwrap_or(0), - } - } - - pub fn outputs(&self) -> Vec<(String, i64)> { - match self { - components_query::ComponentsQueryComponentsEdgesNodeOn::Source(s) => s - .outputs - .iter() - .map(|o| { - ( - o.output_id.clone(), - o.sent_events_total - .as_ref() - .map(|p| p.sent_events_total as i64) - .unwrap_or(0), - ) - }) - .collect(), - components_query::ComponentsQueryComponentsEdgesNodeOn::Transform(t) => t - .outputs - .iter() - .map(|o| { - ( - o.output_id.clone(), - o.sent_events_total - .as_ref() - .map(|p| p.sent_events_total as i64) - .unwrap_or(0), - ) - }) - .collect(), - components_query::ComponentsQueryComponentsEdgesNodeOn::Sink(_) => vec![], - } - } -} - -impl fmt::Display for components_query::ComponentsQueryComponentsEdgesNodeOn { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let res = match self { - components_query::ComponentsQueryComponentsEdgesNodeOn::Source(_) => "source", - components_query::ComponentsQueryComponentsEdgesNodeOn::Transform(_) => "transform", - components_query::ComponentsQueryComponentsEdgesNodeOn::Sink(_) => "sink", - }; - - write!(f, "{res}") - } -} - -impl fmt::Display for component_added_subscription::ComponentAddedSubscriptionComponentAddedOn { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let res = match self { - component_added_subscription::ComponentAddedSubscriptionComponentAddedOn::Source => { - "source" - } - component_added_subscription::ComponentAddedSubscriptionComponentAddedOn::Transform => { - "transform" - } - component_added_subscription::ComponentAddedSubscriptionComponentAddedOn::Sink => { - "sink" - } - }; - - write!(f, "{res}") - } -} - -impl fmt::Display - for component_removed_subscription::ComponentRemovedSubscriptionComponentRemovedOn -{ - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let res = match self { - component_removed_subscription::ComponentRemovedSubscriptionComponentRemovedOn::Source => { - "source" - } - component_removed_subscription::ComponentRemovedSubscriptionComponentRemovedOn::Transform => { - "transform" - } - component_removed_subscription::ComponentRemovedSubscriptionComponentRemovedOn::Sink => { - "sink" - } - }; - - write!(f, "{res}") - } -} diff --git a/lib/vector-api-client/src/gql/health.rs b/lib/vector-api-client/src/gql/health.rs deleted file mode 100644 index f3bd4968c8b30..0000000000000 --- a/lib/vector-api-client/src/gql/health.rs +++ /dev/null @@ -1,64 +0,0 @@ -//! Health queries/subscriptions, for asserting a GraphQL API server is alive. - -use graphql_client::GraphQLQuery; - -/// Shorthand for a Chrono datetime, set to UTC. -type DateTime = chrono::DateTime; - -/// HealthQuery is generally used to assert that the GraphQL API server is alive. -/// The `health` field returns true. -#[derive(GraphQLQuery, Debug, Copy, Clone)] -#[graphql( - schema_path = "graphql/schema.json", - query_path = "graphql/queries/health.graphql", - response_derives = "Debug" -)] -pub struct HealthQuery; - -/// HeartbeatSubscription is a subscription that returns a 'heartbeat' in the form -/// of a UTC timestamp. The use-case is allowing a client to assert that the server is -/// sending regular payloads, by using the timestamp to determine when the last healthcheck -/// was successful. -#[derive(GraphQLQuery, Debug, Copy, Clone)] -#[graphql( - schema_path = "graphql/schema.json", - query_path = "graphql/subscriptions/heartbeat.graphql", - response_derives = "Debug" -)] -pub struct HeartbeatSubscription; - -/// Extension methods for health queries. -pub trait HealthQueryExt { - /// Executes a health query. - async fn health_query(&self) -> crate::QueryResult; -} - -impl HealthQueryExt for crate::Client { - /// Executes a health query. - async fn health_query(&self) -> crate::QueryResult { - self.query::(&HealthQuery::build_query(health_query::Variables)) - .await - } -} - -/// Extension methods for health subscriptions -pub trait HealthSubscriptionExt { - /// Executes a heartbeat subscription, on a millisecond `interval`. - fn heartbeat_subscription( - &self, - interval: i64, - ) -> crate::BoxedSubscription; -} - -impl HealthSubscriptionExt for crate::SubscriptionClient { - /// Executes a heartbeat subscription, on a millisecond `interval`. - fn heartbeat_subscription( - &self, - interval: i64, - ) -> crate::BoxedSubscription { - let request_body = - HeartbeatSubscription::build_query(heartbeat_subscription::Variables { interval }); - - self.start::(&request_body) - } -} diff --git a/lib/vector-api-client/src/gql/meta.rs b/lib/vector-api-client/src/gql/meta.rs deleted file mode 100644 index d8d9a734342f3..0000000000000 --- a/lib/vector-api-client/src/gql/meta.rs +++ /dev/null @@ -1,26 +0,0 @@ -use graphql_client::GraphQLQuery; - -/// MetaVersionStringQuery returns the version string of the queried Vector instance. -#[derive(GraphQLQuery, Debug, Copy, Clone)] -#[graphql( - schema_path = "graphql/schema.json", - query_path = "graphql/queries/meta_version_string.graphql", - response_derives = "Debug" -)] -pub struct MetaVersionStringQuery; - -/// Extension methods for meta queries. -pub trait MetaQueryExt { - /// Executes a meta version string query. - async fn meta_version_string(&self) -> crate::QueryResult; -} - -impl MetaQueryExt for crate::Client { - /// Executes a meta version string query. - async fn meta_version_string(&self) -> crate::QueryResult { - self.query::(&MetaVersionStringQuery::build_query( - meta_version_string_query::Variables, - )) - .await - } -} diff --git a/lib/vector-api-client/src/gql/metrics.rs b/lib/vector-api-client/src/gql/metrics.rs deleted file mode 100644 index a9df90f7c0bd7..0000000000000 --- a/lib/vector-api-client/src/gql/metrics.rs +++ /dev/null @@ -1,341 +0,0 @@ -//! Metrics queries/subscriptions. - -use graphql_client::GraphQLQuery; - -use crate::BoxedSubscription; - -/// UptimeSubscription returns uptime metrics to determine how long the Vector -/// instance has been running. -#[derive(GraphQLQuery, Debug, Copy, Clone)] -#[graphql( - schema_path = "graphql/schema.json", - query_path = "graphql/subscriptions/uptime.graphql", - response_derives = "Debug" -)] -pub struct UptimeSubscription; - -/// ComponentAllocatedBytesSubscription contains metrics on the number of allocated bytes -/// that have been processed by a Vector instance, against specific components. -#[derive(GraphQLQuery, Debug, Copy, Clone)] -#[graphql( - schema_path = "graphql/schema.json", - query_path = "graphql/subscriptions/component_allocated_bytes.graphql", - response_derives = "Debug" -)] -pub struct ComponentAllocatedBytesSubscription; - -/// ComponentReceivedBytesThroughputsSubscription contains metrics on the number of bytes -/// that have been received between `interval` samples, against specific components. -#[derive(GraphQLQuery, Debug, Copy, Clone)] -#[graphql( - schema_path = "graphql/schema.json", - query_path = "graphql/subscriptions/component_received_bytes_throughputs.graphql", - response_derives = "Debug" -)] -pub struct ComponentReceivedBytesThroughputsSubscription; - -/// ComponentReceivedBytesTotalsSubscription contains metrics on the number of bytes -/// that have been received by a Vector instance, against a specific component. -#[derive(GraphQLQuery, Debug, Copy, Clone)] -#[graphql( - schema_path = "graphql/schema.json", - query_path = "graphql/subscriptions/component_received_bytes_totals.graphql", - response_derives = "Debug" -)] -pub struct ComponentReceivedBytesTotalsSubscription; - -/// ComponentReceivedEventsThroughputsSubscription contains metrics on the number of events -/// that have been accepted for processing between `interval` samples, against specific components. -#[derive(GraphQLQuery, Debug, Copy, Clone)] -#[graphql( - schema_path = "graphql/schema.json", - query_path = "graphql/subscriptions/component_received_events_throughputs.graphql", - response_derives = "Debug" -)] -pub struct ComponentReceivedEventsThroughputsSubscription; - -/// ComponentReceivedEventsTotalsSubscription contains metrics on the number of events -/// that have been accepted for processing by a Vector instance, against specific components. -#[derive(GraphQLQuery, Debug, Copy, Clone)] -#[graphql( - schema_path = "graphql/schema.json", - query_path = "graphql/subscriptions/component_received_events_totals.graphql", - response_derives = "Debug" -)] -pub struct ComponentReceivedEventsTotalsSubscription; - -/// ComponentSentBytesThroughputsSubscription contains metrics on the number of bytes -/// that have been received between `interval` samples, against specific components. -#[derive(GraphQLQuery, Debug, Copy, Clone)] -#[graphql( - schema_path = "graphql/schema.json", - query_path = "graphql/subscriptions/component_sent_bytes_throughputs.graphql", - response_derives = "Debug" -)] -pub struct ComponentSentBytesThroughputsSubscription; - -/// ComponentSentBytesTotalsSubscription contains metrics on the number of bytes -/// that have been received by a Vector instance, against a specific component. -#[derive(GraphQLQuery, Debug, Copy, Clone)] -#[graphql( - schema_path = "graphql/schema.json", - query_path = "graphql/subscriptions/component_sent_bytes_totals.graphql", - response_derives = "Debug" -)] -pub struct ComponentSentBytesTotalsSubscription; - -/// ComponentSentEventsThroughputsSubscription contains metrics on the number of events -/// that have been emitted between `interval` samples, against specific components. -#[derive(GraphQLQuery, Debug, Copy, Clone)] -#[graphql( - schema_path = "graphql/schema.json", - query_path = "graphql/subscriptions/component_sent_events_throughputs.graphql", - response_derives = "Debug" -)] -pub struct ComponentSentEventsThroughputsSubscription; - -/// ComponentSentEventsTotalsSubscription contains metrics on the number of events -/// that have been emitted by a Vector instance, against specific components. -#[derive(GraphQLQuery, Debug, Copy, Clone)] -#[graphql( - schema_path = "graphql/schema.json", - query_path = "graphql/subscriptions/component_sent_events_totals.graphql", - response_derives = "Debug" -)] -pub struct ComponentSentEventsTotalsSubscription; - -impl component_sent_events_totals_subscription::ComponentSentEventsTotalsSubscriptionComponentSentEventsTotals { - pub fn outputs(&self) -> Vec<(String, i64)> { - self.outputs - .iter() - .map(|output| { - ( - output.output_id.clone(), - output - .sent_events_total - .as_ref() - .map(|p| p.sent_events_total as i64) - .unwrap_or(0), - ) - }) - .collect() - } -} - -impl component_sent_events_throughputs_subscription::ComponentSentEventsThroughputsSubscriptionComponentSentEventsThroughputs { - pub fn outputs(&self) -> Vec<(String, i64)> { - self.outputs - .iter() - .map(|output| { - ( - output.output_id.clone(), - output.throughput, - ) - }) - .collect() - } - -} - -/// ComponentErrorsTotalsSubscription contains metrics on the number of errors -/// (metrics ending in `_errors_total`), against specific components. -#[derive(GraphQLQuery, Debug, Copy, Clone)] -#[graphql( - schema_path = "graphql/schema.json", - query_path = "graphql/subscriptions/component_errors_totals.graphql", - response_derives = "Debug" -)] -pub struct ComponentErrorsTotalsSubscription; - -/// Extension methods for metrics subscriptions -pub trait MetricsSubscriptionExt { - /// Executes an uptime metrics subscription. - fn uptime_subscription(&self) -> crate::BoxedSubscription; - - /// Executes an all component allocated bytes subscription. - fn component_allocated_bytes_subscription( - &self, - interval: i64, - ) -> BoxedSubscription; - - /// Executes a component bytes received totals subscription. - fn component_received_bytes_totals_subscription( - &self, - interval: i64, - ) -> crate::BoxedSubscription; - - /// Executes a component bytes received throughput subscription. - fn component_received_bytes_throughputs_subscription( - &self, - interval: i64, - ) -> crate::BoxedSubscription; - - /// Executes a component received events totals subscription. - fn component_received_events_totals_subscription( - &self, - interval: i64, - ) -> crate::BoxedSubscription; - - /// Executes an component events in throughputs subscription. - fn component_received_events_throughputs_subscription( - &self, - interval: i64, - ) -> crate::BoxedSubscription; - - /// Executes a component bytes sent totals subscription. - fn component_sent_bytes_totals_subscription( - &self, - interval: i64, - ) -> crate::BoxedSubscription; - - /// Executes a component bytes sent throughput subscription. - fn component_sent_bytes_throughputs_subscription( - &self, - interval: i64, - ) -> crate::BoxedSubscription; - - /// Executes a component events totals subscription. - fn component_sent_events_totals_subscription( - &self, - interval: i64, - ) -> crate::BoxedSubscription; - - /// Executes a component sent events throughputs subscription. - fn component_sent_events_throughputs_subscription( - &self, - interval: i64, - ) -> crate::BoxedSubscription; - - fn component_errors_totals_subscription( - &self, - interval: i64, - ) -> crate::BoxedSubscription; -} - -impl MetricsSubscriptionExt for crate::SubscriptionClient { - /// Executes an uptime metrics subscription. - fn uptime_subscription(&self) -> BoxedSubscription { - let request_body = UptimeSubscription::build_query(uptime_subscription::Variables); - - self.start::(&request_body) - } - - /// Executes an all component allocated bytes subscription. - fn component_allocated_bytes_subscription( - &self, - interval: i64, - ) -> BoxedSubscription { - let request_body = ComponentAllocatedBytesSubscription::build_query( - component_allocated_bytes_subscription::Variables { interval }, - ); - - self.start::(&request_body) - } - - /// Executes an all component bytes received totals subscription. - fn component_received_bytes_totals_subscription( - &self, - interval: i64, - ) -> BoxedSubscription { - let request_body = ComponentReceivedBytesTotalsSubscription::build_query( - component_received_bytes_totals_subscription::Variables { interval }, - ); - - self.start::(&request_body) - } - - /// Executes a component bytes received throughput subscription. - fn component_received_bytes_throughputs_subscription( - &self, - interval: i64, - ) -> BoxedSubscription { - let request_body = ComponentReceivedBytesThroughputsSubscription::build_query( - component_received_bytes_throughputs_subscription::Variables { interval }, - ); - - self.start::(&request_body) - } - - /// Executes an all component received events totals subscription. - fn component_received_events_totals_subscription( - &self, - interval: i64, - ) -> BoxedSubscription { - let request_body = ComponentReceivedEventsTotalsSubscription::build_query( - component_received_events_totals_subscription::Variables { interval }, - ); - - self.start::(&request_body) - } - - /// Executes an all component received events throughputs subscription. - fn component_received_events_throughputs_subscription( - &self, - interval: i64, - ) -> BoxedSubscription { - let request_body = ComponentReceivedEventsThroughputsSubscription::build_query( - component_received_events_throughputs_subscription::Variables { interval }, - ); - - self.start::(&request_body) - } - - /// Executes an all component bytes sent totals subscription. - fn component_sent_bytes_totals_subscription( - &self, - interval: i64, - ) -> BoxedSubscription { - let request_body = ComponentSentBytesTotalsSubscription::build_query( - component_sent_bytes_totals_subscription::Variables { interval }, - ); - - self.start::(&request_body) - } - - /// Executes a component bytes sent throughput subscription. - fn component_sent_bytes_throughputs_subscription( - &self, - interval: i64, - ) -> BoxedSubscription { - let request_body = ComponentSentBytesThroughputsSubscription::build_query( - component_sent_bytes_throughputs_subscription::Variables { interval }, - ); - - self.start::(&request_body) - } - - /// Executes a component sent events totals subscription. - fn component_sent_events_totals_subscription( - &self, - interval: i64, - ) -> crate::BoxedSubscription { - let request_body = ComponentSentEventsTotalsSubscription::build_query( - component_sent_events_totals_subscription::Variables { interval }, - ); - - self.start::(&request_body) - } - - /// Executes a component sent events throughputs subscription. - fn component_sent_events_throughputs_subscription( - &self, - interval: i64, - ) -> crate::BoxedSubscription { - let request_body = ComponentSentEventsThroughputsSubscription::build_query( - component_sent_events_throughputs_subscription::Variables { interval }, - ); - - self.start::(&request_body) - } - - fn component_errors_totals_subscription( - &self, - interval: i64, - ) -> BoxedSubscription { - let request_body = ComponentErrorsTotalsSubscription::build_query( - component_errors_totals_subscription::Variables { interval }, - ); - - self.start::(&request_body) - } -} diff --git a/lib/vector-api-client/src/gql/mod.rs b/lib/vector-api-client/src/gql/mod.rs deleted file mode 100644 index 6a690768f5ee9..0000000000000 --- a/lib/vector-api-client/src/gql/mod.rs +++ /dev/null @@ -1,14 +0,0 @@ -//! Queries, subscriptions, and extension methods for executing them - -mod components; -mod health; -mod meta; -mod metrics; -mod tap; - -pub use components::*; -pub use health::*; -pub use metrics::*; -pub use tap::*; - -pub use self::meta::*; diff --git a/lib/vector-api-client/src/gql/tap.rs b/lib/vector-api-client/src/gql/tap.rs deleted file mode 100644 index 1aef36838175c..0000000000000 --- a/lib/vector-api-client/src/gql/tap.rs +++ /dev/null @@ -1,90 +0,0 @@ -#![allow(clippy::upper_case_acronyms)] - -use graphql_client::GraphQLQuery; - -use crate::BoxedSubscription; - -/// Shorthand for a Chrono datetime, set to UTC. -type DateTime = chrono::DateTime; - -/// OutputEventsByComponentIdPatternsSubscription allows observability into the events that are -/// generated from component(s). -#[derive(GraphQLQuery, Debug, Copy, Clone)] -#[graphql( - schema_path = "graphql/schema.json", - query_path = "graphql/subscriptions/output_events_by_component_id_patterns.graphql", - response_derives = "Debug" -)] -pub struct OutputEventsByComponentIdPatternsSubscription; - -/// Tap encoding format type that is more convenient to use for public clients than the -/// generated `output_events_by_component_id_patterns_subscription::EventEncodingType`. -#[derive(clap::ValueEnum, Debug, Clone, Copy)] -pub enum TapEncodingFormat { - Json, - Yaml, - Logfmt, -} - -/// String -> TapEncodingFormat, typically for parsing user input. -impl std::str::FromStr for TapEncodingFormat { - type Err = String; - - fn from_str(s: &str) -> Result { - match s { - "json" => Ok(Self::Json), - "yaml" => Ok(Self::Yaml), - "logfmt" => Ok(Self::Logfmt), - _ => Err("Invalid encoding format".to_string()), - } - } -} - -/// Map the public-facing `TapEncodingFormat` to the internal `EventEncodingType`. -impl From - for output_events_by_component_id_patterns_subscription::EventEncodingType -{ - fn from(encoding: TapEncodingFormat) -> Self { - match encoding { - TapEncodingFormat::Json => Self::JSON, - TapEncodingFormat::Yaml => Self::YAML, - TapEncodingFormat::Logfmt => Self::LOGFMT, - } - } -} - -pub trait TapSubscriptionExt { - /// Executes an output events subscription. - fn output_events_by_component_id_patterns_subscription( - &self, - outputs_patterns: Vec, - inputs_patterns: Vec, - encoding: TapEncodingFormat, - limit: i64, - interval: i64, - ) -> crate::BoxedSubscription; -} - -impl TapSubscriptionExt for crate::SubscriptionClient { - /// Executes an output events subscription. - fn output_events_by_component_id_patterns_subscription( - &self, - outputs_patterns: Vec, - inputs_patterns: Vec, - encoding: TapEncodingFormat, - limit: i64, - interval: i64, - ) -> BoxedSubscription { - let request_body = OutputEventsByComponentIdPatternsSubscription::build_query( - output_events_by_component_id_patterns_subscription::Variables { - outputs_patterns, - inputs_patterns: Some(inputs_patterns), - limit, - interval, - encoding: encoding.into(), - }, - ); - - self.start::(&request_body) - } -} diff --git a/lib/vector-api-client/src/lib.rs b/lib/vector-api-client/src/lib.rs index e9850b80c235c..74b72f2f0ea5b 100644 --- a/lib/vector-api-client/src/lib.rs +++ b/lib/vector-api-client/src/lib.rs @@ -1,23 +1,43 @@ -//! Vector GraphQL client library, for the Vector GraphQL API server. +//! Vector gRPC API client library //! -//! Contains: +//! This library provides a Rust client for the Vector gRPC observability API. //! -//! 1. A GraphQL query client, for queries/mutations over HTTP(s) -//! 2. A GraphQL subscription client, for long-lived, multiplexed subscriptions over WebSockets -//! 3. GraphQL queries/mutations/subscriptions, defined in `graphql/**/*.graphql` files -//! 4. Extension methods for each client, for executing queries/subscriptions, and returning -//! deserialized JSON responses +//! # Example //! - -#![deny(warnings)] -#![deny(missing_debug_implementations, missing_copy_implementations)] -#![allow(async_fn_in_trait)] +//! ```no_run +//! use vector_api_client::Client; +//! +//! # async fn example() -> Result<(), Box> { +//! let mut client = Client::new("http://localhost:9999".parse().unwrap()); +//! client.connect().await?; +//! +//! // Check health (standard gRPC health check) +//! client.health().await?; +//! println!("Server is healthy"); +//! +//! // Get components +//! let components = client.get_components(0).await?; +//! for component in components.components { +//! println!("Component: {}", component.component_id); +//! } +//! # Ok(()) +//! # } +//! ``` mod client; -/// GraphQL queries -pub mod gql; -mod subscription; -pub mod test; +mod error; + +pub use client::Client; +pub use error::{Error, Result}; + +/// How long (ms) to wait before attempting to reconnect to the Vector API after a disconnect. +pub const RECONNECT_DELAY_MS: u64 = 5000; + +/// Re-export generated protobuf types +pub mod proto { + pub mod event { + tonic::include_proto!("event"); + } -pub use client::*; -pub use subscription::*; + tonic::include_proto!("vector.observability.v1"); +} diff --git a/lib/vector-api-client/src/subscription.rs b/lib/vector-api-client/src/subscription.rs deleted file mode 100644 index 0a4308c6ed7ae..0000000000000 --- a/lib/vector-api-client/src/subscription.rs +++ /dev/null @@ -1,199 +0,0 @@ -use std::{ - collections::HashMap, - pin::Pin, - sync::{Arc, Mutex}, -}; - -use futures::SinkExt; -use graphql_client::GraphQLQuery; -use serde::{Deserialize, Serialize}; -use serde_json::json; -use tokio::sync::{ - broadcast::{self, Sender}, - mpsc, oneshot, -}; -use tokio_stream::{Stream, StreamExt, wrappers::BroadcastStream}; -use tokio_tungstenite::{connect_async, tungstenite::Message}; -use url::Url; -use uuid::Uuid; - -/// Subscription GraphQL response, returned from an active stream. -pub type BoxedSubscription = Pin< - Box< - dyn Stream::ResponseData>>> - + Send - + Sync, - >, ->; - -/// Payload contains the raw data received back from a GraphQL subscription. At the point -/// of receiving data, the only known fields are { id, type }; what's contained inside the -/// `payload` field is unknown until we attempt to deserialize it against a generated -/// GraphQLQuery::ResponseData later. -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct Payload { - id: Uuid, - #[serde(rename = "type")] - payload_type: String, - payload: serde_json::Value, -} - -impl Payload { - /// Returns a "start" payload necessary for starting a new subscription. - pub fn start( - id: Uuid, - payload: &graphql_client::QueryBody, - ) -> Self { - Self { - id, - payload_type: "start".to_owned(), - payload: json!(payload), - } - } - - /// Returns a "stop" payload for terminating the subscription in the GraphQL server. - fn stop(id: Uuid) -> Self { - Self { - id, - payload_type: "stop".to_owned(), - payload: serde_json::Value::Null, - } - } - - /// Attempts to return a definitive ResponseData on the `payload` field, matched against - /// a generated `GraphQLQuery`. - fn response( - &self, - ) -> Option> { - serde_json::from_value::>(self.payload.clone()) - .ok() - } -} - -/// A single `SubscriptionClient` enables subscription multiplexing. -#[derive(Debug)] -pub struct SubscriptionClient { - tx: mpsc::UnboundedSender, - subscriptions: Arc>>>, - _shutdown_tx: oneshot::Sender<()>, -} - -impl SubscriptionClient { - /// Create a new subscription client. `tx` is a channel for sending `Payload`s to the - /// GraphQL server; `rx` is a channel for `Payload` back. - fn new(tx: mpsc::UnboundedSender, mut rx: mpsc::UnboundedReceiver) -> Self { - // Oneshot channel for cancelling the listener if SubscriptionClient is dropped - let (_shutdown_tx, mut shutdown_rx) = oneshot::channel::<()>(); - - let subscriptions = Arc::new(Mutex::new(HashMap::new())); - let subscriptions_clone = Arc::clone(&subscriptions); - - // Spawn a handler for shutdown, and relaying received `Payload`s back to the relevant - // subscription. - let tx_clone = tx.clone(); - tokio::spawn(async move { - loop { - tokio::select! { - // Break the loop if shutdown is triggered. This happens implicitly once - // the client goes out of scope - _ = &mut shutdown_rx => { - let subscriptions = subscriptions_clone.lock().unwrap(); - for id in subscriptions.keys() { - _ = tx_clone.send(Payload::stop(*id)); - } - break - }, - - // Handle receiving payloads back _from_ the server - message = rx.recv() => { - match message { - Some(p) => { - let subscriptions = subscriptions_clone.lock().unwrap(); - let s: Option<&Sender> = subscriptions.get::(&p.id); - if let Some(s) = s { - _ = s.send(p); - } - } - None => { - subscriptions_clone.lock().unwrap().clear(); - break; - }, - } - } - } - } - }); - - Self { - tx, - subscriptions, - _shutdown_tx, - } - } - - /// Start a new subscription request. - pub fn start( - &self, - request_body: &graphql_client::QueryBody, - ) -> BoxedSubscription - where - T: GraphQLQuery + Send + Sync, - ::ResponseData: Unpin + Send + Sync + 'static, - { - // Generate a unique ID for the subscription. Subscriptions can be multiplexed - // over a single connection, so we'll keep a copy of this against the client to - // handling routing responses back to the relevant subscriber. - let id = Uuid::new_v4(); - - let (tx, rx) = broadcast::channel::(100); - - self.subscriptions.lock().unwrap().insert(id, tx); - - // Send start subscription command with the relevant control messages. - _ = self.tx.send(Payload::start::(id, request_body)); - - Box::pin( - BroadcastStream::new(rx) - .filter(Result::is_ok) - .map(|p| p.unwrap().response::()), - ) - } -} - -/// Connect to a new WebSocket GraphQL server endpoint, and return a `SubscriptionClient`. -/// This method will a) connect to a ws(s):// endpoint, and perform the initial handshake, and b) -/// set up channel forwarding to expose just the returned `Payload`s to the client. -pub async fn connect_subscription_client( - url: Url, -) -> Result { - let (ws, _) = connect_async(url).await?; - let (mut ws_tx, mut ws_rx) = futures::StreamExt::split(ws); - - let (send_tx, mut send_rx) = mpsc::unbounded_channel::(); - let (recv_tx, recv_rx) = mpsc::unbounded_channel::(); - - // Initialize the connection - _ = ws_tx - .send(Message::Text(r#"{"type":"connection_init"}"#.to_string())) - .await; - - // Forwarded received messages back upstream to the GraphQL server - tokio::spawn(async move { - while let Some(p) = send_rx.recv().await { - _ = ws_tx - .send(Message::Text(serde_json::to_string(&p).unwrap())) - .await; - } - }); - - // Forward received messages to the receiver channel. - tokio::spawn(async move { - while let Some(Ok(Message::Text(m))) = ws_rx.next().await { - if let Ok(p) = serde_json::from_str::(&m) { - _ = recv_tx.send(p); - } - } - }); - - Ok(SubscriptionClient::new(send_tx, recv_rx)) -} diff --git a/lib/vector-api-client/src/test/mod.rs b/lib/vector-api-client/src/test/mod.rs deleted file mode 100644 index 1f7716d96214f..0000000000000 --- a/lib/vector-api-client/src/test/mod.rs +++ /dev/null @@ -1,160 +0,0 @@ -use graphql_client::GraphQLQuery; - -use crate::{BoxedSubscription, QueryResult}; - -/// Component links query for returning linked components for sources, transforms, and sinks -#[derive(GraphQLQuery, Debug, Copy, Clone)] -#[graphql( - schema_path = "graphql/schema.json", - query_path = "tests/queries/component_links.graphql", - response_derives = "Debug" -)] -pub struct ComponentLinksQuery; - -/// Errors total subscription -#[derive(GraphQLQuery, Debug, Copy, Clone)] -#[graphql( - schema_path = "graphql/schema.json", - query_path = "tests/subscriptions/errors_total.graphql", - response_derives = "Debug" -)] -pub struct ErrorsTotalSubscription; - -/// File source metrics query -#[derive(GraphQLQuery, Debug, Copy, Clone)] -#[graphql( - schema_path = "graphql/schema.json", - query_path = "tests/queries/file_source_metrics.graphql", - response_derives = "Debug" -)] -pub struct FileSourceMetricsQuery; - -/// Component by id query -#[derive(GraphQLQuery, Debug, Copy, Clone)] -#[graphql( - schema_path = "graphql/schema.json", - query_path = "tests/queries/component_by_component_key.graphql", - response_derives = "Debug" -)] -pub struct ComponentByComponentKeyQuery; - -/// Component by id query -#[derive(GraphQLQuery, Debug, Copy, Clone)] -#[graphql( - schema_path = "graphql/schema.json", - query_path = "tests/queries/components_connection.graphql", - response_derives = "Debug" -)] -pub struct ComponentsConnectionQuery; - -pub trait TestQueryExt { - async fn component_links_query( - &self, - after: Option, - before: Option, - first: Option, - last: Option, - ) -> crate::QueryResult; - async fn file_source_metrics_query( - &self, - after: Option, - before: Option, - first: Option, - last: Option, - ) -> crate::QueryResult; - async fn component_by_component_key_query( - &self, - component_id: &str, - ) -> crate::QueryResult; - async fn components_connection_query( - &self, - after: Option, - before: Option, - first: Option, - last: Option, - ) -> crate::QueryResult; -} - -impl TestQueryExt for crate::Client { - async fn component_links_query( - &self, - after: Option, - before: Option, - first: Option, - last: Option, - ) -> QueryResult { - let request_body = ComponentLinksQuery::build_query(component_links_query::Variables { - after, - before, - first, - last, - }); - self.query::(&request_body).await - } - - async fn file_source_metrics_query( - &self, - after: Option, - before: Option, - first: Option, - last: Option, - ) -> QueryResult { - let request_body = - FileSourceMetricsQuery::build_query(file_source_metrics_query::Variables { - after, - before, - first, - last, - }); - self.query::(&request_body).await - } - - async fn component_by_component_key_query( - &self, - component_id: &str, - ) -> QueryResult { - let request_body = ComponentByComponentKeyQuery::build_query( - component_by_component_key_query::Variables { - component_id: component_id.to_string(), - }, - ); - self.query::(&request_body) - .await - } - - async fn components_connection_query( - &self, - after: Option, - before: Option, - first: Option, - last: Option, - ) -> QueryResult { - let request_body = - ComponentsConnectionQuery::build_query(components_connection_query::Variables { - after, - before, - first, - last, - }); - self.query::(&request_body).await - } -} - -pub trait TestSubscriptionExt { - fn errors_total_subscription( - &self, - interval: i64, - ) -> crate::BoxedSubscription; -} - -impl TestSubscriptionExt for crate::SubscriptionClient { - fn errors_total_subscription( - &self, - interval: i64, - ) -> BoxedSubscription { - let request_body = - ErrorsTotalSubscription::build_query(errors_total_subscription::Variables { interval }); - - self.start::(&request_body) - } -} diff --git a/lib/vector-api-client/tests/queries/component_by_component_key.graphql b/lib/vector-api-client/tests/queries/component_by_component_key.graphql deleted file mode 100644 index e29946f9148c3..0000000000000 --- a/lib/vector-api-client/tests/queries/component_by_component_key.graphql +++ /dev/null @@ -1,6 +0,0 @@ -query ComponentByComponentKeyQuery($componentId: String!) { - componentByComponentKey(componentId: $componentId) { - __typename - componentId - } -} diff --git a/lib/vector-api-client/tests/queries/component_links.graphql b/lib/vector-api-client/tests/queries/component_links.graphql deleted file mode 100644 index 3259b6c9b6661..0000000000000 --- a/lib/vector-api-client/tests/queries/component_links.graphql +++ /dev/null @@ -1,45 +0,0 @@ -query ComponentLinksQuery($after: String, $before: String, $first: Int, $last: Int) { - sources(after: $after, before: $before, first: $first, last: $last) { - edges { - node { - componentId - transforms { - componentId - } - sinks { - componentId - } - } - } - } - transforms(after: $after, before: $before, first: $first, last: $last) { - edges { - node { - componentId - sources { - componentId - } - transforms { - componentId - } - sinks { - componentId - } - } - } - } - sinks(after: $after, before: $before, first: $first, last: $last) { - edges { - node { - componentId - sources { - componentId - } - transforms { - componentId - } - } - } - } -} - diff --git a/lib/vector-api-client/tests/queries/components_connection.graphql b/lib/vector-api-client/tests/queries/components_connection.graphql deleted file mode 100644 index 835908282634d..0000000000000 --- a/lib/vector-api-client/tests/queries/components_connection.graphql +++ /dev/null @@ -1,18 +0,0 @@ -query ComponentsConnectionQuery($after: String, $before: String, $first: Int, $last: Int) { - components(after: $after, before: $before, first: $first, last: $last) { - edges { - cursor - node { - __typename - componentId - } - } - pageInfo { - hasNextPage - hasPreviousPage - startCursor - endCursor - } - totalCount - } -} diff --git a/lib/vector-api-client/tests/queries/file_source_metrics.graphql b/lib/vector-api-client/tests/queries/file_source_metrics.graphql deleted file mode 100644 index 7b9d2afb0f10e..0000000000000 --- a/lib/vector-api-client/tests/queries/file_source_metrics.graphql +++ /dev/null @@ -1,25 +0,0 @@ -query FileSourceMetricsQuery($after: String, $before: String, $first: Int, $last: Int) { - sources { - __typename - edges { - node { - metrics { - __typename - ...on FileSourceMetrics { - __typename - files(after: $after, before: $before, first: $first, last: $last) { - edges { - node { - name - receivedEventsTotal { - receivedEventsTotal - } - } - } - } - } - } - } - } - } -} diff --git a/lib/vector-api-client/tests/subscriptions/errors_total.graphql b/lib/vector-api-client/tests/subscriptions/errors_total.graphql deleted file mode 100644 index b0e18d07f76b0..0000000000000 --- a/lib/vector-api-client/tests/subscriptions/errors_total.graphql +++ /dev/null @@ -1,5 +0,0 @@ -subscription ErrorsTotalSubscription($interval: Int!) { - errorsTotal(interval: $interval) { - errorsTotal - } -} diff --git a/lib/vector-buffers/Cargo.toml b/lib/vector-buffers/Cargo.toml index 02b43c29e382f..4f27f3ef369e1 100644 --- a/lib/vector-buffers/Cargo.toml +++ b/lib/vector-buffers/Cargo.toml @@ -37,13 +37,13 @@ ordered-float.workspace = true [dev-dependencies] clap.workspace = true -criterion = { version = "0.7", features = ["html_reports", "async_tokio"] } +criterion = { workspace = true, features = ["html_reports", "async_tokio"] } crossbeam-queue = "0.3.12" hdrhistogram = "7.5.4" metrics-tracing-context.workspace = true metrics-util = { workspace = true, features = ["debugging"] } -proptest = "1.10" -quickcheck = "1.0" +proptest.workspace = true +quickcheck.workspace = true rand.workspace = true serde_yaml.workspace = true temp-dir = "0.1.16" diff --git a/lib/vector-buffers/src/buffer_usage_data.rs b/lib/vector-buffers/src/buffer_usage_data.rs index 61bda12f91160..b418023657bb5 100644 --- a/lib/vector-buffers/src/buffer_usage_data.rs +++ b/lib/vector-buffers/src/buffer_usage_data.rs @@ -19,35 +19,8 @@ use crate::{ /// always used a "relaxed" ordering when updating them. const ORDERING: Ordering = Ordering::Relaxed; -fn increment_counter(counter: &AtomicU64, delta: u64) { - counter - .fetch_update(ORDERING, ORDERING, |current| { - Some(current.checked_add(delta).unwrap_or_else(|| { - warn!( - current, - delta, "Buffer counter overflowed. Clamping value to `u64::MAX`." - ); - u64::MAX - })) - }) - .ok(); -} - -fn decrement_counter(counter: &AtomicU64, delta: u64) { - counter - .fetch_update(ORDERING, ORDERING, |current| { - Some(current.checked_sub(delta).unwrap_or_else(|| { - warn!( - current, - delta, "Buffer counter underflowed. Clamping value to `0`." - ); - 0 - })) - }) - .ok(); -} - /// Snapshot of category metrics. +#[derive(Clone, Copy, Debug, Default)] struct CategorySnapshot { event_count: u64, event_byte_size: u64, @@ -62,10 +35,15 @@ impl CategorySnapshot { /// Per-category metrics. /// -/// This tracks the number of events, and their size in the buffer, that a given category has interacted with. A -/// category in this case could be something like the receive or send categories i.e. being written into the buffer, and -/// then read out of the buffer. Overall, it's a simple grouping mechanism because we often want to track the change in -/// both number of events, and their size as measured by the buffer. +/// This tracks the number of events, and their size in the buffer, that a given category has +/// interacted with. A category in this case could be something like the receive or send categories +/// i.e. being written into the buffer, and then read out of the buffer. Overall, it's a simple +/// grouping mechanism because we often want to track the change in both number of events, and their +/// size as measured by the buffer. +/// +/// At a sustained 1 GiB/sec, which is still faster than Vector can currently achieve, a `u64` byte +/// counter would take over 500 years to overflow. As such, we don't handle failures due to +/// overflow as it is effectively impossible. #[derive(Debug, Default)] struct CategoryMetrics { event_count: AtomicU64, @@ -75,14 +53,8 @@ struct CategoryMetrics { impl CategoryMetrics { /// Increments the event count and byte size by the given amounts. fn increment(&self, event_count: u64, event_byte_size: u64) { - increment_counter(&self.event_count, event_count); - increment_counter(&self.event_byte_size, event_byte_size); - } - - /// Decrements the event count and byte size by the given amounts. - fn decrement(&self, event_count: u64, event_byte_size: u64) { - decrement_counter(&self.event_count, event_count); - decrement_counter(&self.event_byte_size, event_byte_size); + self.event_count.fetch_add(event_count, ORDERING); + self.event_byte_size.fetch_add(event_byte_size, ORDERING); } /// Sets the event count and event byte size to the given amount. @@ -115,6 +87,56 @@ impl CategoryMetrics { } } +/// Running totals of events that have entered and left a buffer stage. +/// +/// Each reporting tick consumes the latest deltas from the atomic counters and +/// folds them into these cumulative totals. The difference +/// (`total_entered - total_left`) gives the approximate current buffer +/// occupancy without requiring a separate "current size" counter that would +/// itself be subject to cross-thread races. +#[derive(Clone, Copy, Debug, Default)] +struct ReporterCurrentMetrics { + total_entered: CategorySnapshot, + total_left: CategorySnapshot, +} + +impl ReporterCurrentMetrics { + fn add_received(&mut self, snapshot: CategorySnapshot) { + self.total_entered.event_count = self + .total_entered + .event_count + .saturating_add(snapshot.event_count); + self.total_entered.event_byte_size = self + .total_entered + .event_byte_size + .saturating_add(snapshot.event_byte_size); + } + + fn add_left(&mut self, snapshot: CategorySnapshot) { + self.total_left.event_count = self + .total_left + .event_count + .saturating_add(snapshot.event_count); + self.total_left.event_byte_size = self + .total_left + .event_byte_size + .saturating_add(snapshot.event_byte_size); + } + + fn current(&self) -> CategorySnapshot { + CategorySnapshot { + event_count: self + .total_entered + .event_count + .saturating_sub(self.total_left.event_count), + event_byte_size: self + .total_entered + .event_byte_size + .saturating_sub(self.total_left.event_byte_size), + } + } +} + /// Handle to buffer usage metrics for a specific buffer stage. #[derive(Clone, Debug)] pub struct BufferUsageHandle { @@ -155,7 +177,6 @@ impl BufferUsageHandle { pub fn increment_received_event_count_and_byte_size(&self, count: u64, byte_size: u64) { if count > 0 || byte_size > 0 { self.state.received.increment(count, byte_size); - self.state.current.increment(count, byte_size); } } @@ -165,7 +186,6 @@ impl BufferUsageHandle { pub fn increment_sent_event_count_and_byte_size(&self, count: u64, byte_size: u64) { if count > 0 || byte_size > 0 { self.state.sent.increment(count, byte_size); - self.state.current.decrement(count, byte_size); } } @@ -182,7 +202,6 @@ impl BufferUsageHandle { } else { self.state.dropped.increment(count, byte_size); } - self.state.current.decrement(count, byte_size); } } } @@ -195,7 +214,6 @@ struct BufferUsageData { dropped: CategoryMetrics, dropped_intentional: CategoryMetrics, max_size: CategoryMetrics, - current: CategoryMetrics, } impl BufferUsageData { @@ -229,6 +247,85 @@ impl BufferUsageData { .expect("should never be bigger than `usize`"), } } + + fn report(&self, current_metrics: &mut ReporterCurrentMetrics, buffer_id: &str) { + let max_size = self.max_size.get(); + emit(BufferCreated { + buffer_id: buffer_id.to_string(), + idx: self.idx, + max_size_bytes: max_size.event_byte_size, + max_size_events: max_size + .event_count + .try_into() + .expect("should never be bigger than `usize`"), + }); + + // Consume received before sent/dropped so that, in the presence of + // a race between producers and consumers, the computed current usage + // will err on the side of overcounting, which is more likely to be an + // accurate representation of the current usage than undercounting. + let received = self.received.consume(); + current_metrics.add_received(received); + + let sent = self.sent.consume(); + current_metrics.add_left(sent); + + let dropped = self.dropped.consume(); + current_metrics.add_left(dropped); + + let dropped_intentional = self.dropped_intentional.consume(); + current_metrics.add_left(dropped_intentional); + + let current = current_metrics.current(); + + if received.has_updates() { + emit(BufferEventsReceived { + buffer_id: buffer_id.to_string(), + idx: self.idx, + count: received.event_count, + byte_size: received.event_byte_size, + total_count: current.event_count, + total_byte_size: current.event_byte_size, + }); + } + + if sent.has_updates() { + emit(BufferEventsSent { + buffer_id: buffer_id.to_string(), + idx: self.idx, + count: sent.event_count, + byte_size: sent.event_byte_size, + total_count: current.event_count, + total_byte_size: current.event_byte_size, + }); + } + + if dropped.has_updates() { + emit(BufferEventsDropped { + buffer_id: buffer_id.to_string(), + idx: self.idx, + intentional: false, + reason: "corrupted_events", + count: dropped.event_count, + byte_size: dropped.event_byte_size, + total_count: current.event_count, + total_byte_size: current.event_byte_size, + }); + } + + if dropped_intentional.has_updates() { + emit(BufferEventsDropped { + buffer_id: buffer_id.to_string(), + idx: self.idx, + intentional: true, + reason: "drop_newest", + count: dropped_intentional.event_count, + byte_size: dropped_intentional.event_byte_size, + total_count: current.event_count, + total_byte_size: current.event_byte_size, + }); + } + } } /// Snapshot of buffer usage metrics. @@ -290,150 +387,122 @@ impl BufferUsage { pub fn install(self, buffer_id: &str) { let buffer_id = buffer_id.to_string(); let span = self.span; - let stages = self.stages; + let stages: Vec<_> = self + .stages + .into_iter() + .map(|stage| (stage, ReporterCurrentMetrics::default())) + .collect(); let task_name = format!("buffer usage reporter ({buffer_id})"); - let task = async move { - let mut interval = interval(Duration::from_secs(2)); - loop { - interval.tick().await; - - for stage in &stages { - let max_size = stage.max_size.get(); - emit(BufferCreated { - buffer_id: buffer_id.clone(), - idx: stage.idx, - max_size_bytes: max_size.event_byte_size, - max_size_events: max_size - .event_count - .try_into() - .expect("should never be bigger than `usize`"), - }); - - let current = stage.current.get(); - let received = stage.received.consume(); - if received.has_updates() { - emit(BufferEventsReceived { - buffer_id: buffer_id.clone(), - idx: stage.idx, - count: received.event_count, - byte_size: received.event_byte_size, - total_count: current.event_count, - total_byte_size: current.event_byte_size, - }); - } - - let sent = stage.sent.consume(); - if sent.has_updates() { - emit(BufferEventsSent { - buffer_id: buffer_id.clone(), - idx: stage.idx, - count: sent.event_count, - byte_size: sent.event_byte_size, - total_count: current.event_count, - total_byte_size: current.event_byte_size, - }); - } - - let dropped = stage.dropped.consume(); - if dropped.has_updates() { - emit(BufferEventsDropped { - buffer_id: buffer_id.clone(), - idx: stage.idx, - intentional: false, - reason: "corrupted_events", - count: dropped.event_count, - byte_size: dropped.event_byte_size, - total_count: current.event_count, - total_byte_size: current.event_byte_size, - }); - } - - let dropped_intentional = stage.dropped_intentional.consume(); - if dropped_intentional.has_updates() { - emit(BufferEventsDropped { - buffer_id: buffer_id.clone(), - idx: stage.idx, - intentional: true, - reason: "drop_newest", - count: dropped_intentional.event_count, - byte_size: dropped_intentional.event_byte_size, - total_count: current.event_count, - total_byte_size: current.event_byte_size, - }); - } - } - } - }; + let task = Self::report_buffer_usage(stages, buffer_id).instrument(span.or_current()); + spawn_named(task, task_name.as_str()); + } + + async fn report_buffer_usage( + mut stages: Vec<(Arc, ReporterCurrentMetrics)>, + buffer_id: String, + ) { + let mut interval = interval(Duration::from_secs(2)); + loop { + interval.tick().await; - spawn_named(task.instrument(span.or_current()), task_name.as_str()); + for (stage, current_metrics) in &mut stages { + stage.report(current_metrics, &buffer_id); + } + } } } #[cfg(test)] mod tests { - use std::thread; - use super::*; #[test] - fn test_multithreaded_updates_are_correct() { - const NUM_THREADS: u64 = 16; - const INCREMENTS_PER_THREAD: u64 = 10_000; - - let counter = Arc::new(AtomicU64::new(0)); - - let mut handles = vec![]; - - for _ in 0..NUM_THREADS { - let counter = Arc::clone(&counter); - let handle = thread::spawn(move || { - for _ in 0..INCREMENTS_PER_THREAD { - increment_counter(&counter, 1); - decrement_counter(&counter, 1); - } - }); - handles.push(handle); - } - - for handle in handles { - handle.join().unwrap(); - } - - assert_eq!(counter.load(ORDERING), 0); + fn reporter_current_usage_is_derived_from_entered_and_left_totals() { + let mut current = ReporterCurrentMetrics::default(); + current.add_received(CategorySnapshot { + event_count: 10, + event_byte_size: 1000, + }); + current.add_left(CategorySnapshot { + event_count: 3, + event_byte_size: 300, + }); + current.add_left(CategorySnapshot { + event_count: 2, + event_byte_size: 200, + }); + + let current = current.current(); + assert_eq!(current.event_count, 5); + assert_eq!(current.event_byte_size, 500); } #[test] - fn test_decrement_counter_prevents_negatives() { - let counter = AtomicU64::new(100); - - decrement_counter(&counter, 50); - assert_eq!(counter.load(ORDERING), 50); - - decrement_counter(&counter, 100); - assert_eq!(counter.load(ORDERING), 0); - - decrement_counter(&counter, 50); - assert_eq!(counter.load(ORDERING), 0); - - decrement_counter(&counter, u64::MAX); - assert_eq!(counter.load(ORDERING), 0); + fn reporter_current_usage_preserves_underflow_debt() { + let mut current = ReporterCurrentMetrics::default(); + current.add_left(CategorySnapshot { + event_count: 10, + event_byte_size: 1000, + }); + current.add_received(CategorySnapshot { + event_count: 15, + event_byte_size: 1500, + }); + + let current = current.current(); + assert_eq!(current.event_count, 5); + assert_eq!(current.event_byte_size, 500); } #[test] - fn test_increment_counter_prevents_overflow() { - let counter = AtomicU64::new(u64::MAX - 2); - - increment_counter(&counter, 1); - assert_eq!(counter.load(ORDERING), u64::MAX - 1); - - increment_counter(&counter, 1); - assert_eq!(counter.load(ORDERING), u64::MAX); + fn consume_resets_deltas_between_ticks() { + let data = BufferUsageData::new(0); + let mut metrics = ReporterCurrentMetrics::default(); + + data.received.increment(10, 1000); + data.sent.increment(3, 300); + data.report(&mut metrics, "test"); + let current = metrics.current(); + assert_eq!(current.event_count, 7); + assert_eq!(current.event_byte_size, 700); + + // Second tick with no new activity should report the same totals. + data.report(&mut metrics, "test"); + let current = metrics.current(); + assert_eq!(current.event_count, 7); + assert_eq!(current.event_byte_size, 700); + } - increment_counter(&counter, 1); - assert_eq!(counter.load(ORDERING), u64::MAX); + #[test] + fn accumulates_across_multiple_ticks() { + let data = BufferUsageData::new(0); + let mut metrics = ReporterCurrentMetrics::default(); + + data.received.increment(10, 1000); + data.report(&mut metrics, "test"); + + data.received.increment(5, 500); + data.sent.increment(8, 800); + data.report(&mut metrics, "test"); + let current = metrics.current(); + assert_eq!(current.event_count, 7); + assert_eq!(current.event_byte_size, 700); + } - increment_counter(&counter, u64::MAX); - assert_eq!(counter.load(ORDERING), u64::MAX); + #[test] + fn drops_count_as_leaving_the_buffer() { + let data = BufferUsageData::new(0); + let mut metrics = ReporterCurrentMetrics::default(); + + data.received.increment(20, 2000); + data.sent.increment(5, 500); + data.dropped.increment(3, 300); + data.dropped_intentional.increment(2, 200); + + data.report(&mut metrics, "test"); + let current = metrics.current(); + assert_eq!(current.event_count, 10); + assert_eq!(current.event_byte_size, 1000); } } diff --git a/lib/vector-buffers/src/config.rs b/lib/vector-buffers/src/config.rs index ea51bfce3cb57..0cc9d77cd29d1 100644 --- a/lib/vector-buffers/src/config.rs +++ b/lib/vector-buffers/src/config.rs @@ -371,6 +371,13 @@ impl Default for BufferConfig { } impl BufferConfig { + /// Returns true if any stage in this buffer configuration uses disk-based storage. + pub fn has_disk_stage(&self) -> bool { + self.stages() + .iter() + .any(|stage| matches!(stage, BufferType::DiskV2 { .. })) + } + /// Gets all of the configured stages for this buffer. pub fn stages(&self) -> &[BufferType] { match self { diff --git a/lib/vector-buffers/src/internal_events.rs b/lib/vector-buffers/src/internal_events.rs index 6d2735c7bb75f..31a6089322d0c 100644 --- a/lib/vector-buffers/src/internal_events.rs +++ b/lib/vector-buffers/src/internal_events.rs @@ -1,9 +1,10 @@ use std::time::Duration; -use metrics::{Histogram, counter, gauge, histogram}; +use metrics::Histogram; use vector_common::NamedInternalEvent; use vector_common::{ - internal_event::{InternalEvent, error_type}, + counter, gauge, histogram, + internal_event::{CounterName, GaugeName, HistogramName, InternalEvent, error_type}, registered_event, }; @@ -21,14 +22,14 @@ impl InternalEvent for BufferCreated { let stage = self.idx.to_string(); if self.max_size_events != 0 { gauge!( - "buffer_max_size_events", + GaugeName::BufferMaxSizeEvents, "buffer_id" => self.buffer_id.clone(), "stage" => stage.clone(), ) .set(self.max_size_events as f64); // DEPRECATED: buffer-bytes-events-metrics gauge!( - "buffer_max_event_size", + GaugeName::BufferMaxEventSize, "buffer_id" => self.buffer_id.clone(), "stage" => stage.clone(), ) @@ -36,14 +37,14 @@ impl InternalEvent for BufferCreated { } if self.max_size_bytes != 0 { gauge!( - "buffer_max_size_bytes", + GaugeName::BufferMaxSizeBytes, "buffer_id" => self.buffer_id.clone(), "stage" => stage.clone(), ) .set(self.max_size_bytes as f64); // DEPRECATED: buffer-bytes-events-metrics gauge!( - "buffer_max_byte_size", + GaugeName::BufferMaxByteSize, "buffer_id" => self.buffer_id, "stage" => stage, ) @@ -66,40 +67,40 @@ impl InternalEvent for BufferEventsReceived { #[expect(clippy::cast_precision_loss)] fn emit(self) { counter!( - "buffer_received_events_total", + CounterName::BufferReceivedEventsTotal, "buffer_id" => self.buffer_id.clone(), "stage" => self.idx.to_string() ) .increment(self.count); counter!( - "buffer_received_bytes_total", + CounterName::BufferReceivedBytesTotal, "buffer_id" => self.buffer_id.clone(), "stage" => self.idx.to_string() ) .increment(self.byte_size); // DEPRECATED: buffer-bytes-events-metrics gauge!( - "buffer_events", + GaugeName::BufferEvents, "buffer_id" => self.buffer_id.clone(), "stage" => self.idx.to_string() ) .set(self.total_count as f64); gauge!( - "buffer_size_events", + GaugeName::BufferSizeEvents, "buffer_id" => self.buffer_id.clone(), "stage" => self.idx.to_string() ) .set(self.total_count as f64); gauge!( - "buffer_size_bytes", + GaugeName::BufferSizeBytes, "buffer_id" => self.buffer_id.clone(), "stage" => self.idx.to_string() ) .set(self.total_byte_size as f64); // DEPRECATED: buffer-bytes-events-metrics gauge!( - "buffer_byte_size", + GaugeName::BufferByteSize, "buffer_id" => self.buffer_id, "stage" => self.idx.to_string() ) @@ -121,39 +122,39 @@ impl InternalEvent for BufferEventsSent { #[expect(clippy::cast_precision_loss)] fn emit(self) { counter!( - "buffer_sent_events_total", + CounterName::BufferSentEventsTotal, "buffer_id" => self.buffer_id.clone(), "stage" => self.idx.to_string() ) .increment(self.count); counter!( - "buffer_sent_bytes_total", + CounterName::BufferSentBytesTotal, "buffer_id" => self.buffer_id.clone(), "stage" => self.idx.to_string() ) .increment(self.byte_size); // DEPRECATED: buffer-bytes-events-metrics gauge!( - "buffer_events", + GaugeName::BufferEvents, "buffer_id" => self.buffer_id.clone(), "stage" => self.idx.to_string() ) .set(self.total_count as f64); gauge!( - "buffer_size_events", + GaugeName::BufferSizeEvents, "buffer_id" => self.buffer_id.clone(), "stage" => self.idx.to_string() ) .set(self.total_count as f64); gauge!( - "buffer_size_bytes", + GaugeName::BufferSizeBytes, "buffer_id" => self.buffer_id.clone(), "stage" => self.idx.to_string() ) .set(self.total_byte_size as f64); // DEPRECATED: buffer-bytes-events-metrics gauge!( - "buffer_byte_size", + GaugeName::BufferByteSize, "buffer_id" => self.buffer_id, "stage" => self.idx.to_string() ) @@ -200,14 +201,14 @@ impl InternalEvent for BufferEventsDropped { } counter!( - "buffer_discarded_events_total", + CounterName::BufferDiscardedEventsTotal, "buffer_id" => self.buffer_id.clone(), "stage" => self.idx.to_string(), "intentional" => intentional_str, ) .increment(self.count); counter!( - "buffer_discarded_bytes_total", + CounterName::BufferDiscardedBytesTotal, "buffer_id" => self.buffer_id.clone(), "stage" => self.idx.to_string(), "intentional" => intentional_str, @@ -215,26 +216,26 @@ impl InternalEvent for BufferEventsDropped { .increment(self.byte_size); // DEPRECATED: buffer-bytes-events-metrics gauge!( - "buffer_events", + GaugeName::BufferEvents, "buffer_id" => self.buffer_id.clone(), "stage" => self.idx.to_string() ) .set(self.total_count as f64); gauge!( - "buffer_size_events", + GaugeName::BufferSizeEvents, "buffer_id" => self.buffer_id.clone(), "stage" => self.idx.to_string() ) .set(self.total_count as f64); gauge!( - "buffer_size_bytes", + GaugeName::BufferSizeBytes, "buffer_id" => self.buffer_id.clone(), "stage" => self.idx.to_string() ) .set(self.total_byte_size as f64); // DEPRECATED: buffer-bytes-events-metrics gauge!( - "buffer_byte_size", + GaugeName::BufferByteSize, "buffer_id" => self.buffer_id, "stage" => self.idx.to_string() ) @@ -258,7 +259,7 @@ impl InternalEvent for BufferReadError { stage = "processing", ); counter!( - "buffer_errors_total", "error_code" => self.error_code, + CounterName::BufferErrorsTotal, "error_code" => self.error_code, "error_type" => "reader_failed", "stage" => "processing", ) @@ -270,7 +271,7 @@ registered_event! { BufferSendDuration { stage: usize, } => { - send_duration: Histogram = histogram!("buffer_send_duration_seconds", "stage" => self.stage.to_string()), + send_duration: Histogram = histogram!(HistogramName::BufferSendDurationSeconds, "stage" => self.stage.to_string()), } fn emit(&self, duration: Duration) { diff --git a/lib/vector-buffers/src/topology/channel/limited_queue.rs b/lib/vector-buffers/src/topology/channel/limited_queue.rs index a246aa9659812..b420396664c20 100644 --- a/lib/vector-buffers/src/topology/channel/limited_queue.rs +++ b/lib/vector-buffers/src/topology/channel/limited_queue.rs @@ -16,14 +16,23 @@ use std::sync::Mutex; use async_stream::stream; use crossbeam_queue::{ArrayQueue, SegQueue}; use futures::Stream; -use metrics::{Gauge, Histogram, gauge, histogram}; +use metrics::{Gauge, Histogram}; use tokio::sync::{Notify, OwnedSemaphorePermit, Semaphore, TryAcquireError}; +use vector_common::internal_event::{GaugeName, HistogramName}; use vector_common::stats::TimeEwmaGauge; +use vector_common::{gauge, histogram}; use crate::{InMemoryBufferable, config::MemoryBufferSize}; pub const DEFAULT_EWMA_HALF_LIFE_SECONDS: f64 = 5.0; +/// Identifies which buffer channel is being instrumented, determining the metric name prefix. +#[derive(Clone, Copy, Debug)] +pub enum BufferChannelKind { + Source, + Transform, +} + /// Error returned by `LimitedSender::send` when the receiver has disconnected. #[derive(Debug, PartialEq, Eq)] pub struct SendError(pub T); @@ -99,13 +108,13 @@ where #[derive(Clone, Debug)] pub struct ChannelMetricMetadata { - prefix: &'static str, + kind: BufferChannelKind, output: Option, } impl ChannelMetricMetadata { - pub fn new(prefix: &'static str, output: Option) -> Self { - Self { prefix, output } + pub fn new(kind: BufferChannelKind, output: Option) -> Self { + Self { kind, output } } } @@ -134,34 +143,55 @@ impl Metrics { ) -> Self { let ewma_half_life_seconds = ewma_half_life_seconds.unwrap_or(DEFAULT_EWMA_HALF_LIFE_SECONDS); - let ChannelMetricMetadata { prefix, output } = metadata; - let (legacy_suffix, gauge_suffix, max_value) = match limit { - MemoryBufferSize::MaxEvents(max_events) => ( - "_max_event_size", - "_max_size_events", - max_events.get() as f64, + let ChannelMetricMetadata { kind, output } = metadata; + + let (histogram_name, level_name, mean_name) = match kind { + BufferChannelKind::Source => ( + HistogramName::SourceBufferUtilization, + GaugeName::SourceBufferUtilizationLevel, + GaugeName::SourceBufferUtilizationMean, + ), + BufferChannelKind::Transform => ( + HistogramName::TransformBufferUtilization, + GaugeName::TransformBufferUtilizationLevel, + GaugeName::TransformBufferUtilizationMean, + ), + }; + + let (max_name, legacy_name, max_value) = match (kind, limit) { + (BufferChannelKind::Source, MemoryBufferSize::MaxEvents(n)) => ( + GaugeName::SourceBufferMaxSizeEvents, + GaugeName::SourceBufferMaxEventSize, + n.get() as f64, + ), + (BufferChannelKind::Source, MemoryBufferSize::MaxSize(n)) => ( + GaugeName::SourceBufferMaxSizeBytes, + GaugeName::SourceBufferMaxByteSize, + n.get() as f64, + ), + (BufferChannelKind::Transform, MemoryBufferSize::MaxEvents(n)) => ( + GaugeName::TransformBufferMaxSizeEvents, + GaugeName::TransformBufferMaxEventSize, + n.get() as f64, + ), + (BufferChannelKind::Transform, MemoryBufferSize::MaxSize(n)) => ( + GaugeName::TransformBufferMaxSizeBytes, + GaugeName::TransformBufferMaxByteSize, + n.get() as f64, ), - MemoryBufferSize::MaxSize(max_bytes) => { - ("_max_byte_size", "_max_size_bytes", max_bytes.get() as f64) - } }; - let max_gauge_name = format!("{prefix}{gauge_suffix}"); - let legacy_max_gauge_name = format!("{prefix}{legacy_suffix}"); - let histogram_name = format!("{prefix}_utilization"); - let gauge_name = format!("{prefix}_utilization_level"); - let mean_name = format!("{prefix}_utilization_mean"); #[cfg(test)] let recorded_values = Arc::new(Mutex::new(Vec::new())); if let Some(label_value) = output { - let max_gauge = gauge!(max_gauge_name, "output" => label_value.clone()); + let max_gauge = gauge!(max_name, "output" => label_value.clone()); max_gauge.set(max_value); let mean_gauge_handle = gauge!(mean_name, "output" => label_value.clone()); // DEPRECATED: buffer-bytes-events-metrics - let legacy_max_gauge = gauge!(legacy_max_gauge_name, "output" => label_value.clone()); + let legacy_max_gauge = gauge!(legacy_name, "output" => label_value.clone()); legacy_max_gauge.set(max_value); Self { histogram: histogram!(histogram_name, "output" => label_value.clone()), - gauge: gauge!(gauge_name, "output" => label_value.clone()), + gauge: gauge!(level_name, "output" => label_value.clone()), mean_gauge: TimeEwmaGauge::new(mean_gauge_handle, ewma_half_life_seconds), max_gauge, legacy_max_gauge, @@ -169,15 +199,15 @@ impl Metrics { recorded_values, } } else { - let max_gauge = gauge!(max_gauge_name); + let max_gauge = gauge!(max_name); max_gauge.set(max_value); let mean_gauge_handle = gauge!(mean_name); // DEPRECATED: buffer-bytes-events-metrics - let legacy_max_gauge = gauge!(legacy_max_gauge_name); + let legacy_max_gauge = gauge!(legacy_name); legacy_max_gauge.set(max_value); Self { histogram: histogram!(histogram_name), - gauge: gauge!(gauge_name), + gauge: gauge!(level_name), mean_gauge: TimeEwmaGauge::new(mean_gauge_handle, ewma_half_life_seconds), max_gauge, legacy_max_gauge, @@ -473,7 +503,9 @@ mod tests { use tokio_test::{assert_pending, assert_ready, task::spawn}; use vector_common::byte_size_of::ByteSizeOf; - use super::{ChannelMetricMetadata, LimitedReceiver, LimitedSender, limited}; + use super::{ + BufferChannelKind, ChannelMetricMetadata, LimitedReceiver, LimitedSender, limited, + }; use crate::{ MemoryBufferSize, test::MultiEventRecord, @@ -514,7 +546,32 @@ mod tests { let limit = MemoryBufferSize::MaxEvents(NonZeroUsize::new(2).unwrap()); let (mut tx, mut rx) = limited( limit, - Some(ChannelMetricMetadata::new("test_channel", None)), + Some(ChannelMetricMetadata::new(BufferChannelKind::Source, None)), + None, + ); + + let metrics = tx.inner.metrics.as_ref().unwrap().recorded_values.clone(); + + tx.send(Sample::new(1)).await.expect("send should succeed"); + let records = metrics.lock().unwrap().clone(); + assert_eq!(records.len(), 1); + assert_eq!(records.last().copied(), Some(1)); + + assert_eq!(Sample::new(1), rx.next().await.unwrap()); + let records = metrics.lock().unwrap(); + assert_eq!(records.len(), 2); + assert_eq!(records.last().copied(), Some(0)); + } + + #[tokio::test] + async fn records_utilization_transform_channel() { + let limit = MemoryBufferSize::MaxEvents(NonZeroUsize::new(2).unwrap()); + let (mut tx, mut rx) = limited( + limit, + Some(ChannelMetricMetadata::new( + BufferChannelKind::Transform, + None, + )), None, ); @@ -536,7 +593,7 @@ mod tests { let limit = MemoryBufferSize::MaxEvents(NonZeroUsize::new(2).unwrap()); let (mut tx, mut rx) = limited( limit, - Some(ChannelMetricMetadata::new("test_channel_oversized", None)), + Some(ChannelMetricMetadata::new(BufferChannelKind::Source, None)), None, ); let metrics = tx.inner.metrics.as_ref().unwrap().recorded_values.clone(); @@ -919,7 +976,7 @@ mod tests { let limit = NonZeroUsize::new(size * 10).unwrap(); let (tx, rx) = limited( MemoryBufferSize::MaxEvents(limit), - Some(ChannelMetricMetadata::new("test_channel_concurrent", None)), + Some(ChannelMetricMetadata::new(BufferChannelKind::Source, None)), None, ); let metrics = tx.inner.metrics.as_ref().unwrap().recorded_values.clone(); diff --git a/lib/vector-buffers/src/topology/channel/mod.rs b/lib/vector-buffers/src/topology/channel/mod.rs index e0180111819cb..ab8e0fa62e5ce 100644 --- a/lib/vector-buffers/src/topology/channel/mod.rs +++ b/lib/vector-buffers/src/topology/channel/mod.rs @@ -3,8 +3,8 @@ mod receiver; mod sender; pub use limited_queue::{ - ChannelMetricMetadata, DEFAULT_EWMA_HALF_LIFE_SECONDS, LimitedReceiver, LimitedSender, - SendError, limited, + BufferChannelKind, ChannelMetricMetadata, DEFAULT_EWMA_HALF_LIFE_SECONDS, LimitedReceiver, + LimitedSender, SendError, limited, }; pub use receiver::*; pub use sender::*; diff --git a/lib/vector-buffers/src/variants/disk_v2/ledger.rs b/lib/vector-buffers/src/variants/disk_v2/ledger.rs index e63fc5fb16973..a2d1c68f8d8e5 100644 --- a/lib/vector-buffers/src/variants/disk_v2/ledger.rs +++ b/lib/vector-buffers/src/variants/disk_v2/ledger.rs @@ -700,7 +700,7 @@ where #[must_use] pub(super) fn spawn_finalizer(self: Arc) -> OrderedFinalizer { let (finalizer, mut stream) = OrderedFinalizer::new(None); - tokio::spawn(async move { + vector_common::spawn_in_current_span(async move { while let Some((_status, amount)) = stream.next().await { self.increment_pending_acks(amount); self.notify_writer_waiters(); diff --git a/lib/vector-buffers/src/variants/disk_v2/tests/basic.rs b/lib/vector-buffers/src/variants/disk_v2/tests/basic.rs index 30cb47d3c265b..c523390b45796 100644 --- a/lib/vector-buffers/src/variants/disk_v2/tests/basic.rs +++ b/lib/vector-buffers/src/variants/disk_v2/tests/basic.rs @@ -1,4 +1,4 @@ -use std::{io::Cursor, time::Duration}; +use std::{io::Cursor, sync::Arc, time::Duration}; use futures::{StreamExt, stream}; use tokio::{select, time::sleep}; @@ -6,11 +6,14 @@ use tokio_test::{assert_pending, task::spawn}; use tracing::Instrument; use vector_common::finalization::Finalizable; -use super::{create_default_buffer_v2, read_next, read_next_some}; +use super::{ + create_default_buffer_v2, create_default_buffer_v2_with_usage, read_next, read_next_some, +}; use crate::{ EventCount, assert_buffer_is_empty, assert_buffer_records, + buffer_usage_data::BufferUsageHandle, test::{MultiEventRecord, SizedRecord, acknowledge, install_tracing_helpers, with_temp_dir}, - variants::disk_v2::{tests::create_default_buffer_v2_with_usage, writer::RecordWriter}, + variants::disk_v2::{BufferWriter, DiskBufferConfigBuilder, Ledger, writer::RecordWriter}, }; #[tokio::test] @@ -155,8 +158,26 @@ async fn initial_size_correct_with_multievents() { let data_dir = dir.to_path_buf(); async move { - // Create a regular buffer, no customizations required. - let (mut writer, _, _) = create_default_buffer_v2(data_dir.clone()).await; + // Build a write-only buffer without a finalizer task. Using + // `from_config_inner` would spawn a background finalizer that holds + // an Arc (and the lock file), causing a racy + // LedgerLockAlreadyHeld when we reopen the buffer below. + let config = DiskBufferConfigBuilder::from_path(&data_dir) + .build() + .expect("creating buffer config should not fail"); + let usage_handle = BufferUsageHandle::noop(); + let ledger = Ledger::load_or_create(config, usage_handle) + .await + .expect("ledger should not fail to load/create"); + let ledger = Arc::new(ledger); + + let mut writer = BufferWriter::new(Arc::clone(&ledger)); + writer + .validate_last_write() + .await + .expect("validate_last_write should not fail"); + + ledger.synchronize_buffer_usage(); let input_items = (512..768) .cycle() @@ -209,10 +230,14 @@ async fn initial_size_correct_with_multievents() { writer.flush().await.expect("writer flush should not fail"); writer.close(); - // Now drop our buffer and reopen it. + // Drop the first buffer. No background finalizer task exists, so the + // ledger lock is released immediately when the last Arc is dropped. drop(writer); + drop(ledger); + + // Reopen the buffer with a full reader to verify the persisted data. let (writer, mut reader, ledger, usage) = - create_default_buffer_v2_with_usage::<_, MultiEventRecord>(data_dir).await; + create_default_buffer_v2_with_usage::<_, MultiEventRecord>(&data_dir).await; drop(writer); // Make sure our usage data agrees with our expected event count and byte size: diff --git a/lib/vector-common/Cargo.toml b/lib/vector-common/Cargo.toml index 95098f19b1bd3..b329b981aa64c 100644 --- a/lib/vector-common/Cargo.toml +++ b/lib/vector-common/Cargo.toml @@ -48,8 +48,9 @@ pin-project.workspace = true serde.workspace = true serde_json.workspace = true smallvec = { version = "1", default-features = false } +strum.workspace = true stream-cancel = { version = "0.8.2", default-features = false } -tokio = { workspace = true, features = ["macros", "time"] } +tokio = { workspace = true, features = ["macros", "rt", "time"] } tracing.workspace = true vrl.workspace = true vector-config.workspace = true diff --git a/lib/vector-common/src/constants.rs b/lib/vector-common/src/constants.rs index 1eeadca45b5dc..8b336d7937344 100644 --- a/lib/vector-common/src/constants.rs +++ b/lib/vector-common/src/constants.rs @@ -1,3 +1,19 @@ pub const GZIP_MAGIC: &[u8] = &[0x1f, 0x8b]; pub const ZLIB_MAGIC: &[u8] = &[0x78]; pub const ZSTD_MAGIC: &[u8] = &[0x28, 0xB5, 0x2F, 0xFD]; + +/// Maximum size of a zlib stored (uncompressed) block in bytes. +/// See: +pub const ZLIB_STORED_BLOCK_SIZE: usize = 16384; + +/// Per-block overhead for zlib stored blocks: 1-byte header + 2-byte length + 2-byte ~length. +/// See: +pub const ZLIB_STORED_BLOCK_OVERHEAD: usize = 5; + +/// Zlib frame overhead: 2-byte header + 4-byte Adler-32 checksum trailer. +/// See: +pub const ZLIB_FRAME_OVERHEAD: usize = 6; + +/// Threshold below which zstd's `ZSTD_compressBound` adds extra margin (128 KiB). +/// See: (`ZSTD_compressBound`) +pub const ZSTD_SMALL_INPUT_THRESHOLD: usize = 128 << 10; diff --git a/lib/vector-common/src/internal_event/bytes_received.rs b/lib/vector-common/src/internal_event/bytes_received.rs index 98044ecb84570..4264c3cbc365a 100644 --- a/lib/vector-common/src/internal_event/bytes_received.rs +++ b/lib/vector-common/src/internal_event/bytes_received.rs @@ -1,12 +1,14 @@ -use metrics::{Counter, counter}; +use metrics::Counter; -use super::{ByteSize, Protocol, SharedString}; +use crate::counter; + +use super::{ByteSize, CounterName, Protocol, SharedString}; crate::registered_event!( BytesReceived { protocol: SharedString, } => { - received_bytes: Counter = counter!("component_received_bytes_total", "protocol" => self.protocol.clone()), + received_bytes: Counter = counter!(CounterName::ComponentReceivedBytesTotal, "protocol" => self.protocol.clone()), protocol: SharedString = self.protocol, } diff --git a/lib/vector-common/src/internal_event/bytes_sent.rs b/lib/vector-common/src/internal_event/bytes_sent.rs index 0b2a88247f96c..b363893783f43 100644 --- a/lib/vector-common/src/internal_event/bytes_sent.rs +++ b/lib/vector-common/src/internal_event/bytes_sent.rs @@ -1,13 +1,15 @@ -use metrics::{Counter, counter}; +use metrics::Counter; + +use crate::counter; use tracing::trace; -use super::{ByteSize, Protocol, SharedString}; +use super::{ByteSize, CounterName, Protocol, SharedString}; crate::registered_event!( BytesSent { protocol: SharedString, } => { - bytes_sent: Counter = counter!("component_sent_bytes_total", "protocol" => self.protocol.clone()), + bytes_sent: Counter = counter!(CounterName::ComponentSentBytesTotal, "protocol" => self.protocol.clone()), protocol: SharedString = self.protocol, } diff --git a/lib/vector-common/src/internal_event/cached_event.rs b/lib/vector-common/src/internal_event/cached_event.rs index 782faeb5a1d01..ec3d492998ab4 100644 --- a/lib/vector-common/src/internal_event/cached_event.rs +++ b/lib/vector-common/src/internal_event/cached_event.rs @@ -91,34 +91,36 @@ where #[cfg(test)] mod tests { #![allow(unreachable_pub)] - use metrics::{Counter, counter}; + use metrics::Counter; + use strum::IntoEnumIterator; use super::*; - - crate::registered_event!( - TestEvent { - fixed: String, - dynamic: String, - } => { - event: Counter = { - counter!("test_event_total", "fixed" => self.fixed, "dynamic" => self.dynamic) - }, - } - - fn emit(&self, count: u64) { - self.event.increment(count); - } - - fn register(fixed: String, dynamic: String) { - crate::internal_event::register(TestEvent { - fixed, - dynamic, - }) - } - ); + use crate::internal_event::CounterName; #[test] fn test_fixed_tag() { + crate::registered_event!( + TestEvent { + fixed: String, + dynamic: String, + } => { + event: Counter = { + crate::counter!(CounterName::iter().next().unwrap(), "fixed" => self.fixed, "dynamic" => self.dynamic) + }, + } + + fn emit(&self, count: u64) { + self.event.increment(count); + } + + fn register(fixed: String, dynamic: String) { + crate::internal_event::register(TestEvent { + fixed, + dynamic, + }) + } + ); + let event: RegisteredEventCache = RegisteredEventCache::new("fixed".to_string()); diff --git a/lib/vector-common/src/internal_event/component_events_dropped.rs b/lib/vector-common/src/internal_event/component_events_dropped.rs index bf45c054e45e1..d86891f4ba815 100644 --- a/lib/vector-common/src/internal_event/component_events_dropped.rs +++ b/lib/vector-common/src/internal_event/component_events_dropped.rs @@ -1,6 +1,8 @@ -use metrics::{Counter, counter}; +use metrics::Counter; -use super::{Count, InternalEvent, InternalEventHandle, RegisterInternalEvent}; +use crate::counter; + +use super::{Count, CounterName, InternalEvent, InternalEventHandle, RegisterInternalEvent}; use crate::NamedInternalEvent; pub const INTENTIONAL: bool = true; @@ -25,14 +27,17 @@ impl<'a, const INTENTIONAL: bool> From<&'a str> for ComponentEventsDropped<'a, I } } +// ComponentEventsDropped is the foundation type the `registered_event!` macro +// abstracts over, so we have to implement RegisterInternalEvent by hand here. impl<'a, const INTENTIONAL: bool> RegisterInternalEvent for ComponentEventsDropped<'a, INTENTIONAL> { + // ## skip check-validity-events ## type Handle = DroppedHandle<'a, INTENTIONAL>; fn register(self) -> Self::Handle { Self::Handle { discarded_events: counter!( - "component_discarded_events_total", + CounterName::ComponentDiscardedEventsTotal, "intentional" => if INTENTIONAL { "true" } else { "false" }, ), reason: self.reason, diff --git a/lib/vector-common/src/internal_event/component_events_timed_out.rs b/lib/vector-common/src/internal_event/component_events_timed_out.rs index bf138dd1481c7..4662322fe9776 100644 --- a/lib/vector-common/src/internal_event/component_events_timed_out.rs +++ b/lib/vector-common/src/internal_event/component_events_timed_out.rs @@ -1,13 +1,15 @@ -use metrics::{Counter, counter}; +use metrics::Counter; -use super::Count; +use crate::counter; + +use super::{Count, CounterName}; crate::registered_event! { ComponentEventsTimedOut { reason: &'static str, } => { - timed_out_events: Counter = counter!("component_timed_out_events_total"), - timed_out_requests: Counter = counter!("component_timed_out_requests_total"), + timed_out_events: Counter = counter!(CounterName::ComponentTimedOutEventsTotal), + timed_out_requests: Counter = counter!(CounterName::ComponentTimedOutRequestsTotal), reason: &'static str = self.reason, } diff --git a/lib/vector-common/src/internal_event/events_received.rs b/lib/vector-common/src/internal_event/events_received.rs index 40ba999a8cfca..a08ebc6e8438b 100644 --- a/lib/vector-common/src/internal_event/events_received.rs +++ b/lib/vector-common/src/internal_event/events_received.rs @@ -1,13 +1,15 @@ -use metrics::{Counter, Histogram, counter, histogram}; +use metrics::{Counter, Histogram}; + +use crate::{counter, histogram}; use tracing::trace; -use super::CountByteSize; +use super::{CountByteSize, CounterName, HistogramName}; crate::registered_event!( EventsReceived => { - events_count: Histogram = histogram!("component_received_events_count"), - events: Counter = counter!("component_received_events_total"), - event_bytes: Counter = counter!("component_received_event_bytes_total"), + events_count: Histogram = histogram!(HistogramName::ComponentReceivedEventsCount), + events: Counter = counter!(CounterName::ComponentReceivedEventsTotal), + event_bytes: Counter = counter!(CounterName::ComponentReceivedEventBytesTotal), } fn emit(&self, data: CountByteSize) { diff --git a/lib/vector-common/src/internal_event/events_sent.rs b/lib/vector-common/src/internal_event/events_sent.rs index 21e11260b2d38..3ae63db761381 100644 --- a/lib/vector-common/src/internal_event/events_sent.rs +++ b/lib/vector-common/src/internal_event/events_sent.rs @@ -1,9 +1,11 @@ use std::sync::Arc; -use metrics::{Counter, counter}; +use metrics::Counter; + +use crate::counter; use tracing::trace; -use super::{CountByteSize, OptionalTag, Output, SharedString}; +use super::{CountByteSize, CounterName, OptionalTag, Output, SharedString}; use crate::config::ComponentKey; pub const DEFAULT_OUTPUT: &str = "_default"; @@ -13,14 +15,14 @@ crate::registered_event!( output: Option, } => { events: Counter = if let Some(output) = &self.output { - counter!("component_sent_events_total", "output" => output.clone()) + counter!(CounterName::ComponentSentEventsTotal, "output" => output.clone()) } else { - counter!("component_sent_events_total") + counter!(CounterName::ComponentSentEventsTotal) }, event_bytes: Counter = if let Some(output) = &self.output { - counter!("component_sent_event_bytes_total", "output" => output.clone()) + counter!(CounterName::ComponentSentEventBytesTotal, "output" => output.clone()) } else { - counter!("component_sent_event_bytes_total") + counter!(CounterName::ComponentSentEventBytesTotal) }, output: Option = self.output, } @@ -75,10 +77,10 @@ crate::registered_event!( service: OptionalTag, } => { events: Counter = { - counter!("component_sent_events_total", &make_tags(&self.source, &self.service)) + counter!(CounterName::ComponentSentEventsTotal, &make_tags(&self.source, &self.service)) }, event_bytes: Counter = { - counter!("component_sent_event_bytes_total", &make_tags(&self.source, &self.service)) + counter!(CounterName::ComponentSentEventBytesTotal, &make_tags(&self.source, &self.service)) }, } diff --git a/lib/vector-common/src/internal_event/metric_name.rs b/lib/vector-common/src/internal_event/metric_name.rs new file mode 100644 index 0000000000000..8ca64f0d7ecf0 --- /dev/null +++ b/lib/vector-common/src/internal_event/metric_name.rs @@ -0,0 +1,383 @@ +use strum::{AsRefStr, Display, EnumIter}; + +/// Canonical list of all per-component internal metric names emitted by Vector. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Display, AsRefStr, EnumIter)] +#[strum(serialize_all = "snake_case")] +pub enum CounterName { + ComponentReceivedEventsTotal, + ComponentReceivedEventBytesTotal, + ComponentReceivedBytesTotal, + ComponentSentEventsTotal, + ComponentSentEventBytesTotal, + ComponentSentBytesTotal, + ComponentDiscardedEventsTotal, + ComponentErrorsTotal, + ComponentTimedOutEventsTotal, + ComponentTimedOutRequestsTotal, + BufferReceivedEventsTotal, + BufferReceivedBytesTotal, + BufferSentEventsTotal, + BufferSentBytesTotal, + BufferDiscardedEventsTotal, + BufferDiscardedBytesTotal, + BufferErrorsTotal, + // Internal events from src/internal_events/ + AggregateEventsRecordedTotal, + AggregateFailedUpdates, + AggregateFlushesTotal, + ApiStartedTotal, + AzureQueueEventIgnoredTotal, + AzureQueueMessageProcessingErroredTotal, + AzureQueueMessageProcessingRejectedTotal, + AzureQueueMessageProcessingSucceededTotal, + CheckpointsTotal, + ChecksumErrorsTotal, + CollectCompletedTotal, + CommandExecutedTotal, + ConnectionEstablishedTotal, + ConnectionSendErrorsTotal, + ConnectionShutdownTotal, + ContainerProcessedEventsTotal, + ContainersUnwatchedTotal, + ContainersWatchedTotal, + DecoderBomRemovalsTotal, + DecoderMalformedReplacementWarningsTotal, + DorisBytesLoadedTotal, + DorisRowsFilteredTotal, + DorisRowsLoadedTotal, + EncoderUnmappableReplacementWarningsTotal, + EventsDiscardedTotal, + FilesAddedTotal, + FilesDeletedTotal, + FilesResumedTotal, + FilesUnwatchedTotal, + GrpcServerMessagesReceivedTotal, + GrpcServerMessagesSentTotal, + HttpClientErrorsTotal, + HttpClientRequestsSentTotal, + HttpClientResponsesTotal, + HttpServerRequestsReceivedTotal, + HttpServerResponsesSentTotal, + KafkaConsumedMessagesBytesTotal, + KafkaConsumedMessagesTotal, + KafkaProducedMessagesBytesTotal, + KafkaProducedMessagesTotal, + KafkaRequestsBytesTotal, + KafkaRequestsTotal, + KafkaResponsesBytesTotal, + KafkaResponsesTotal, + MetadataRefreshFailedTotal, + MetadataRefreshSuccessfulTotal, + ParseErrorsTotal, + QuitTotal, + ReloadedTotal, + RewrittenTimestampEventsTotal, + SqsMessageDeferSucceededTotal, + SqsMessageDeleteSucceededTotal, + SqsMessageProcessingSucceededTotal, + SqsMessageReceiveSucceededTotal, + SqsMessageReceivedMessagesTotal, + StaleEventsFlushedTotal, + StartedTotal, + StoppedTotal, + TagCardinalityUntrackedEventsTotal, + TagValueLimitExceededTotal, + ValueLimitReachedTotal, + WebsocketBytesSentTotal, + WebsocketMessagesSentTotal, + WindowsServiceInstallTotal, + WindowsServiceRestartTotal, + WindowsServiceStartTotal, + WindowsServiceStopTotal, + WindowsServiceUninstallTotal, + K8sEventNamespaceAnnotationFailuresTotal, + K8sEventNodeAnnotationFailuresTotal, + K8sFormatPickerEdgeCasesTotal, + K8sDockerFormatParseFailuresTotal, + SqsS3EventRecordIgnoredTotal, + ComponentAllocatedBytesTotal, + ComponentDeallocatedBytesTotal, + MemoryEnrichmentTableFailedInsertions, + MemoryEnrichmentTableFailedReads, + MemoryEnrichmentTableFlushesTotal, + MemoryEnrichmentTableInsertionsTotal, + MemoryEnrichmentTableReadsTotal, + MemoryEnrichmentTableTtlExpirations, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Display, AsRefStr, EnumIter)] +#[strum(serialize_all = "snake_case")] +pub enum HistogramName { + ComponentReceivedEventsCount, + ComponentReceivedBytes, + BufferSendDurationSeconds, + ComponentLatencySeconds, + SourceLagTimeSeconds, + SourceSendLatencySeconds, + SourceSendBatchLatencySeconds, + AdaptiveConcurrencyAveragedRtt, + AdaptiveConcurrencyBackPressure, + AdaptiveConcurrencyInFlight, + AdaptiveConcurrencyLimit, + AdaptiveConcurrencyObservedRtt, + AdaptiveConcurrencyPastRttMean, + AdaptiveConcurrencyReachedLimit, + S3ObjectProcessingSucceededDurationSeconds, + S3ObjectProcessingFailedDurationSeconds, + CollectDurationSeconds, + CommandExecutionDurationSeconds, + GrpcServerHandlerDurationSeconds, + HttpServerHandlerDurationSeconds, + HttpClientRttSeconds, + HttpClientResponseRttSeconds, + HttpClientErrorRttSeconds, + SourceBufferUtilization, + TransformBufferUtilization, +} + +impl HistogramName { + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::ComponentReceivedEventsCount => "component_received_events_count", + Self::ComponentReceivedBytes => "component_received_bytes", + Self::BufferSendDurationSeconds => "buffer_send_duration_seconds", + Self::ComponentLatencySeconds => "component_latency_seconds", + Self::SourceLagTimeSeconds => "source_lag_time_seconds", + Self::SourceSendLatencySeconds => "source_send_latency_seconds", + Self::SourceSendBatchLatencySeconds => "source_send_batch_latency_seconds", + Self::AdaptiveConcurrencyAveragedRtt => "adaptive_concurrency_averaged_rtt", + Self::AdaptiveConcurrencyBackPressure => "adaptive_concurrency_back_pressure", + Self::AdaptiveConcurrencyInFlight => "adaptive_concurrency_in_flight", + Self::AdaptiveConcurrencyLimit => "adaptive_concurrency_limit", + Self::AdaptiveConcurrencyObservedRtt => "adaptive_concurrency_observed_rtt", + Self::AdaptiveConcurrencyPastRttMean => "adaptive_concurrency_past_rtt_mean", + Self::AdaptiveConcurrencyReachedLimit => "adaptive_concurrency_reached_limit", + Self::S3ObjectProcessingSucceededDurationSeconds => { + "s3_object_processing_succeeded_duration_seconds" + } + Self::S3ObjectProcessingFailedDurationSeconds => { + "s3_object_processing_failed_duration_seconds" + } + Self::CollectDurationSeconds => "collect_duration_seconds", + Self::CommandExecutionDurationSeconds => "command_execution_duration_seconds", + Self::GrpcServerHandlerDurationSeconds => "grpc_server_handler_duration_seconds", + Self::HttpServerHandlerDurationSeconds => "http_server_handler_duration_seconds", + Self::HttpClientRttSeconds => "http_client_rtt_seconds", + Self::HttpClientResponseRttSeconds => "http_client_response_rtt_seconds", + Self::HttpClientErrorRttSeconds => "http_client_error_rtt_seconds", + Self::SourceBufferUtilization => "source_buffer_utilization", + Self::TransformBufferUtilization => "transform_buffer_utilization", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Display, AsRefStr, EnumIter)] +#[strum(serialize_all = "snake_case")] +pub enum GaugeName { + ComponentLatencyMeanSeconds, + SourceBufferMaxSizeEvents, + SourceBufferMaxSizeBytes, + SourceBufferMaxEventSize, + SourceBufferMaxByteSize, + SourceBufferUtilizationLevel, + SourceBufferUtilizationMean, + TransformBufferMaxSizeEvents, + TransformBufferMaxSizeBytes, + TransformBufferMaxEventSize, + TransformBufferMaxByteSize, + TransformBufferUtilizationLevel, + TransformBufferUtilizationMean, + BufferMaxSizeEvents, + BufferMaxEventSize, + BufferMaxSizeBytes, + BufferMaxByteSize, + BufferEvents, + BufferSizeEvents, + BufferSizeBytes, + BufferByteSize, + Utilization, + ComponentAllocatedBytes, + OpenFiles, + UptimeSeconds, + BuildInfo, + KafkaQueueMessages, + KafkaQueueMessagesBytes, + KafkaConsumerLag, + LuaMemoryUsedBytes, + OpenConnections, + ActiveEndpoints, + SplunkPendingAcks, + ActiveClients, + MemoryEnrichmentTableObjectsCount, + MemoryEnrichmentTableByteSize, + TagCardinalityTrackedKeys, +} + +impl GaugeName { + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::ComponentLatencyMeanSeconds => "component_latency_mean_seconds", + Self::SourceBufferMaxSizeEvents => "source_buffer_max_size_events", + Self::SourceBufferMaxSizeBytes => "source_buffer_max_size_bytes", + Self::SourceBufferMaxEventSize => "source_buffer_max_event_size", + Self::SourceBufferMaxByteSize => "source_buffer_max_byte_size", + Self::SourceBufferUtilizationLevel => "source_buffer_utilization_level", + Self::SourceBufferUtilizationMean => "source_buffer_utilization_mean", + Self::TransformBufferMaxSizeEvents => "transform_buffer_max_size_events", + Self::TransformBufferMaxSizeBytes => "transform_buffer_max_size_bytes", + Self::TransformBufferMaxEventSize => "transform_buffer_max_event_size", + Self::TransformBufferMaxByteSize => "transform_buffer_max_byte_size", + Self::TransformBufferUtilizationLevel => "transform_buffer_utilization_level", + Self::TransformBufferUtilizationMean => "transform_buffer_utilization_mean", + Self::BufferMaxSizeEvents => "buffer_max_size_events", + Self::BufferMaxEventSize => "buffer_max_event_size", + Self::BufferMaxSizeBytes => "buffer_max_size_bytes", + Self::BufferMaxByteSize => "buffer_max_byte_size", + Self::BufferEvents => "buffer_events", + Self::BufferSizeEvents => "buffer_size_events", + Self::BufferSizeBytes => "buffer_size_bytes", + Self::BufferByteSize => "buffer_byte_size", + Self::Utilization => "utilization", + Self::ComponentAllocatedBytes => "component_allocated_bytes", + Self::OpenFiles => "open_files", + Self::UptimeSeconds => "uptime_seconds", + Self::BuildInfo => "build_info", + Self::KafkaQueueMessages => "kafka_queue_messages", + Self::KafkaQueueMessagesBytes => "kafka_queue_messages_bytes", + Self::KafkaConsumerLag => "kafka_consumer_lag", + Self::LuaMemoryUsedBytes => "lua_memory_used_bytes", + Self::OpenConnections => "open_connections", + Self::ActiveEndpoints => "active_endpoints", + Self::SplunkPendingAcks => "splunk_pending_acks", + Self::ActiveClients => "active_clients", + Self::MemoryEnrichmentTableObjectsCount => "memory_enrichment_table_objects_count", + Self::MemoryEnrichmentTableByteSize => "memory_enrichment_table_byte_size", + Self::TagCardinalityTrackedKeys => "tag_cardinality_tracked_keys", + } + } +} + +impl CounterName { + #[allow(clippy::too_many_lines)] + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::ComponentReceivedEventsTotal => "component_received_events_total", + Self::ComponentReceivedEventBytesTotal => "component_received_event_bytes_total", + Self::ComponentReceivedBytesTotal => "component_received_bytes_total", + Self::ComponentSentEventsTotal => "component_sent_events_total", + Self::ComponentSentEventBytesTotal => "component_sent_event_bytes_total", + Self::ComponentSentBytesTotal => "component_sent_bytes_total", + Self::ComponentDiscardedEventsTotal => "component_discarded_events_total", + Self::ComponentErrorsTotal => "component_errors_total", + Self::ComponentTimedOutEventsTotal => "component_timed_out_events_total", + Self::ComponentTimedOutRequestsTotal => "component_timed_out_requests_total", + Self::BufferReceivedEventsTotal => "buffer_received_events_total", + Self::BufferReceivedBytesTotal => "buffer_received_bytes_total", + Self::BufferSentEventsTotal => "buffer_sent_events_total", + Self::BufferSentBytesTotal => "buffer_sent_bytes_total", + Self::BufferDiscardedEventsTotal => "buffer_discarded_events_total", + Self::BufferDiscardedBytesTotal => "buffer_discarded_bytes_total", + Self::BufferErrorsTotal => "buffer_errors_total", + Self::AggregateEventsRecordedTotal => "aggregate_events_recorded_total", + Self::AggregateFailedUpdates => "aggregate_failed_updates", + Self::AggregateFlushesTotal => "aggregate_flushes_total", + Self::ApiStartedTotal => "api_started_total", + Self::AzureQueueEventIgnoredTotal => "azure_queue_event_ignored_total", + Self::AzureQueueMessageProcessingErroredTotal => { + "azure_queue_message_processing_errored_total" + } + Self::AzureQueueMessageProcessingRejectedTotal => { + "azure_queue_message_processing_rejected_total" + } + Self::AzureQueueMessageProcessingSucceededTotal => { + "azure_queue_message_processing_succeeded_total" + } + Self::CheckpointsTotal => "checkpoints_total", + Self::ChecksumErrorsTotal => "checksum_errors_total", + Self::CollectCompletedTotal => "collect_completed_total", + Self::CommandExecutedTotal => "command_executed_total", + Self::ConnectionEstablishedTotal => "connection_established_total", + Self::ConnectionSendErrorsTotal => "connection_send_errors_total", + Self::ConnectionShutdownTotal => "connection_shutdown_total", + Self::ContainerProcessedEventsTotal => "container_processed_events_total", + Self::ContainersUnwatchedTotal => "containers_unwatched_total", + Self::ContainersWatchedTotal => "containers_watched_total", + Self::DecoderBomRemovalsTotal => "decoder_bom_removals_total", + Self::DecoderMalformedReplacementWarningsTotal => { + "decoder_malformed_replacement_warnings_total" + } + Self::DorisBytesLoadedTotal => "doris_bytes_loaded_total", + Self::DorisRowsFilteredTotal => "doris_rows_filtered_total", + Self::DorisRowsLoadedTotal => "doris_rows_loaded_total", + Self::EncoderUnmappableReplacementWarningsTotal => { + "encoder_unmappable_replacement_warnings_total" + } + Self::EventsDiscardedTotal => "events_discarded_total", + Self::FilesAddedTotal => "files_added_total", + Self::FilesDeletedTotal => "files_deleted_total", + Self::FilesResumedTotal => "files_resumed_total", + Self::FilesUnwatchedTotal => "files_unwatched_total", + Self::GrpcServerMessagesReceivedTotal => "grpc_server_messages_received_total", + Self::GrpcServerMessagesSentTotal => "grpc_server_messages_sent_total", + Self::HttpClientErrorsTotal => "http_client_errors_total", + Self::HttpClientRequestsSentTotal => "http_client_requests_sent_total", + Self::HttpClientResponsesTotal => "http_client_responses_total", + Self::HttpServerRequestsReceivedTotal => "http_server_requests_received_total", + Self::HttpServerResponsesSentTotal => "http_server_responses_sent_total", + Self::KafkaConsumedMessagesBytesTotal => "kafka_consumed_messages_bytes_total", + Self::KafkaConsumedMessagesTotal => "kafka_consumed_messages_total", + Self::KafkaProducedMessagesBytesTotal => "kafka_produced_messages_bytes_total", + Self::KafkaProducedMessagesTotal => "kafka_produced_messages_total", + Self::KafkaRequestsBytesTotal => "kafka_requests_bytes_total", + Self::KafkaRequestsTotal => "kafka_requests_total", + Self::KafkaResponsesBytesTotal => "kafka_responses_bytes_total", + Self::KafkaResponsesTotal => "kafka_responses_total", + Self::MetadataRefreshFailedTotal => "metadata_refresh_failed_total", + Self::MetadataRefreshSuccessfulTotal => "metadata_refresh_successful_total", + Self::ParseErrorsTotal => "parse_errors_total", + Self::QuitTotal => "quit_total", + Self::ReloadedTotal => "reloaded_total", + Self::RewrittenTimestampEventsTotal => "rewritten_timestamp_events_total", + Self::SqsMessageDeferSucceededTotal => "sqs_message_defer_succeeded_total", + Self::SqsMessageDeleteSucceededTotal => "sqs_message_delete_succeeded_total", + Self::SqsMessageProcessingSucceededTotal => "sqs_message_processing_succeeded_total", + Self::SqsMessageReceiveSucceededTotal => "sqs_message_receive_succeeded_total", + Self::SqsMessageReceivedMessagesTotal => "sqs_message_received_messages_total", + Self::StaleEventsFlushedTotal => "stale_events_flushed_total", + Self::StartedTotal => "started_total", + Self::StoppedTotal => "stopped_total", + Self::TagCardinalityUntrackedEventsTotal => "tag_cardinality_untracked_events_total", + Self::TagValueLimitExceededTotal => "tag_value_limit_exceeded_total", + Self::ValueLimitReachedTotal => "value_limit_reached_total", + Self::WebsocketBytesSentTotal => "websocket_bytes_sent_total", + Self::WebsocketMessagesSentTotal => "websocket_messages_sent_total", + Self::WindowsServiceInstallTotal => "windows_service_install_total", + Self::WindowsServiceRestartTotal => "windows_service_restart_total", + Self::WindowsServiceStartTotal => "windows_service_start_total", + Self::WindowsServiceStopTotal => "windows_service_stop_total", + Self::WindowsServiceUninstallTotal => "windows_service_uninstall_total", + Self::K8sEventNamespaceAnnotationFailuresTotal => { + "k8s_event_namespace_annotation_failures_total" + } + Self::K8sEventNodeAnnotationFailuresTotal => "k8s_event_node_annotation_failures_total", + Self::K8sFormatPickerEdgeCasesTotal => "k8s_format_picker_edge_cases_total", + Self::K8sDockerFormatParseFailuresTotal => "k8s_docker_format_parse_failures_total", + Self::SqsS3EventRecordIgnoredTotal => "sqs_s3_event_record_ignored_total", + Self::ComponentAllocatedBytesTotal => "component_allocated_bytes_total", + Self::ComponentDeallocatedBytesTotal => "component_deallocated_bytes_total", + Self::MemoryEnrichmentTableFailedInsertions => { + "memory_enrichment_table_failed_insertions" + } + Self::MemoryEnrichmentTableFailedReads => "memory_enrichment_table_failed_reads", + Self::MemoryEnrichmentTableFlushesTotal => "memory_enrichment_table_flushes_total", + Self::MemoryEnrichmentTableInsertionsTotal => { + "memory_enrichment_table_insertions_total" + } + Self::MemoryEnrichmentTableReadsTotal => "memory_enrichment_table_reads_total", + Self::MemoryEnrichmentTableTtlExpirations => "memory_enrichment_table_ttl_expirations", + } + } +} diff --git a/lib/vector-common/src/internal_event/mod.rs b/lib/vector-common/src/internal_event/mod.rs index 272e2344900e3..e776e89553172 100644 --- a/lib/vector-common/src/internal_event/mod.rs +++ b/lib/vector-common/src/internal_event/mod.rs @@ -5,6 +5,7 @@ pub mod component_events_dropped; pub mod component_events_timed_out; mod events_received; mod events_sent; +pub mod metric_name; mod optional_tag; mod prelude; pub mod service; @@ -19,6 +20,7 @@ pub use component_events_dropped::{ComponentEventsDropped, INTENTIONAL, UNINTENT pub use component_events_timed_out::ComponentEventsTimedOut; pub use events_received::{EventsReceived, EventsReceivedHandle}; pub use events_sent::{DEFAULT_OUTPUT, EventsSent, TaggedEventsSent}; +pub use metric_name::{CounterName, GaugeName, HistogramName}; pub use metrics::SharedString; pub use optional_tag::OptionalTag; pub use prelude::{error_stage, error_type}; diff --git a/lib/vector-common/src/internal_event/prelude.rs b/lib/vector-common/src/internal_event/prelude.rs index 92aa160fe509e..bcc68aef26c6b 100644 --- a/lib/vector-common/src/internal_event/prelude.rs +++ b/lib/vector-common/src/internal_event/prelude.rs @@ -15,6 +15,8 @@ pub mod error_type { // When a condition for the event to be valid failed. // This is used for example when a field is missing or should be a string. pub const CONDITION_FAILED: &str = "condition_failed"; + // When the component received a request with missing or invalid authentication credentials. + pub const AUTHENTICATION_FAILED: &str = "authentication_failed"; // When the component or the service on which it depends is not configured properly. pub const CONFIGURATION_FAILED: &str = "configuration_failed"; // When the component failed to connect to an external service. diff --git a/lib/vector-common/src/internal_event/service.rs b/lib/vector-common/src/internal_event/service.rs index 93a3c9b745722..60d615b3d61a0 100644 --- a/lib/vector-common/src/internal_event/service.rs +++ b/lib/vector-common/src/internal_event/service.rs @@ -1,4 +1,6 @@ -use metrics::counter; +use super::CounterName; + +use crate::counter; use super::{ComponentEventsDropped, InternalEvent, UNINTENTIONAL, emit, error_stage, error_type}; use crate::NamedInternalEvent; @@ -17,7 +19,7 @@ impl InternalEvent for PollReadyError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::REQUEST_FAILED, "stage" => error_stage::SENDING, ) @@ -43,7 +45,7 @@ impl InternalEvent for CallError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::REQUEST_FAILED, "stage" => error_stage::SENDING, ) diff --git a/lib/vector-common/src/lib.rs b/lib/vector-common/src/lib.rs index 099a068578090..cda7e4b921242 100644 --- a/lib/vector-common/src/lib.rs +++ b/lib/vector-common/src/lib.rs @@ -66,6 +66,63 @@ pub mod trigger; #[macro_use] extern crate tracing; +/// Typed wrapper around `metrics::counter!` that only accepts [`internal_event::CounterName`]. +#[macro_export] +macro_rules! counter { + ($name:expr) => {{ + let _name: $crate::internal_event::CounterName = $name; + #[allow(clippy::disallowed_macros)] + { + metrics::counter!(_name.as_str()) + } + }}; + ($name:expr, $($rest:tt)*) => {{ + let _name: $crate::internal_event::CounterName = $name; + #[allow(clippy::disallowed_macros)] + { + metrics::counter!(_name.as_str(), $($rest)*) + } + }}; +} + +/// Typed wrapper around `metrics::histogram!` that only accepts [`internal_event::HistogramName`]. +#[macro_export] +macro_rules! histogram { + ($name:expr) => {{ + let _name: $crate::internal_event::HistogramName = $name; + #[allow(clippy::disallowed_macros)] + { + metrics::histogram!(_name.as_str()) + } + }}; + ($name:expr, $($rest:tt)*) => {{ + let _name: $crate::internal_event::HistogramName = $name; + #[allow(clippy::disallowed_macros)] + { + metrics::histogram!(_name.as_str(), $($rest)*) + } + }}; +} + +/// Typed wrapper around `metrics::gauge!` that only accepts [`internal_event::GaugeName`]. +#[macro_export] +macro_rules! gauge { + ($name:expr) => {{ + let _name: $crate::internal_event::GaugeName = $name; + #[allow(clippy::disallowed_macros)] + { + metrics::gauge!(_name.as_str()) + } + }}; + ($name:expr, $($rest:tt)*) => {{ + let _name: $crate::internal_event::GaugeName = $name; + #[allow(clippy::disallowed_macros)] + { + metrics::gauge!(_name.as_str(), $($rest)*) + } + }}; +} + /// Vector's basic error type, dynamically dispatched and safe to send across /// threads. pub type Error = Box; @@ -73,3 +130,19 @@ pub type Error = Box; /// Vector's basic result type, defined in terms of [`Error`] and generic over /// `T`. pub type Result = std::result::Result; + +/// Spawn a future on the current tokio runtime, propagating the current tracing span into the +/// spawned task. This ensures that any logs or internal metrics emitted by the task retain the +/// component tags (`component_id`, `component_kind`, `component_type`) of the caller. +/// +/// Prefer this over `tokio::spawn(future.in_current_span())` to keep call sites concise. +#[track_caller] +pub fn spawn_in_current_span( + task: impl std::future::Future + Send + 'static, +) -> tokio::task::JoinHandle +where + T: Send + 'static, +{ + use tracing::Instrument as _; + tokio::spawn(task.in_current_span()) +} diff --git a/lib/vector-config-common/Cargo.toml b/lib/vector-config-common/Cargo.toml index 350c1cb1678f4..946bc48e17e70 100644 --- a/lib/vector-config-common/Cargo.toml +++ b/lib/vector-config-common/Cargo.toml @@ -5,7 +5,7 @@ edition = "2024" license = "MPL-2.0" [dependencies] -convert_case = { version = "0.8", default-features = false } +convert_case.workspace = true darling.workspace = true proc-macro2 = { version = "1.0", default-features = false } serde.workspace = true diff --git a/lib/vector-config/Cargo.toml b/lib/vector-config/Cargo.toml index 2283df3d133e1..5f4cfdbdcf1b2 100644 --- a/lib/vector-config/Cargo.toml +++ b/lib/vector-config/Cargo.toml @@ -20,7 +20,7 @@ no-proxy = { version = "0.3.6", default-features = false, features = ["serialize num-traits = { version = "0.2.19", default-features = false } serde.workspace = true serde_json.workspace = true -serde_with = { version = "3.14.0", default-features = false, features = ["std"] } +serde_with.workspace = true snafu.workspace = true toml.workspace = true tracing.workspace = true @@ -32,4 +32,4 @@ vector-config-macros = { path = "../vector-config-macros" } [dev-dependencies] assert-json-diff = { version = "2", default-features = false } -serde_with = { version = "3.14.0", default-features = false, features = ["std", "macros"] } +serde_with.workspace = true diff --git a/lib/vector-config/src/http.rs b/lib/vector-config/src/http.rs index 94029610f0441..1f61630bf7cbd 100644 --- a/lib/vector-config/src/http.rs +++ b/lib/vector-config/src/http.rs @@ -2,9 +2,11 @@ use std::cell::RefCell; use http::StatusCode; use serde_json::Value; +use vector_config_common::{attributes::CustomAttribute, constants}; use crate::{ Configurable, GenerateError, Metadata, ToValue, + num::NumberClass, schema::{SchemaGenerator, SchemaObject, generate_number_schema}, }; @@ -26,7 +28,10 @@ impl Configurable for StatusCode { fn metadata() -> Metadata { let mut metadata = Metadata::default(); metadata.set_description("HTTP response status code"); - metadata.set_default_value(StatusCode::OK); + metadata.add_custom_attribute(CustomAttribute::kv( + constants::DOCS_META_NUMERIC_TYPE, + NumberClass::Unsigned, + )); metadata } diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml index d416d018d3fbb..1ec303bbb272b 100644 --- a/lib/vector-core/Cargo.toml +++ b/lib/vector-core/Cargo.toml @@ -34,18 +34,18 @@ metrics-util.workspace = true mlua = { version = "0.10.5", default-features = false, features = ["lua54", "send", "vendored"], optional = true } no-proxy = { version = "0.3.6", default-features = false, features = ["serialize"] } ordered-float.workspace = true -openssl = { version = "0.10.75", default-features = false, features = ["vendored"] } +openssl = { version = "0.10.80", default-features = false, features = ["vendored"] } parking_lot = { version = "0.12.5", default-features = false } pin-project.workspace = true -proptest = { version = "1.10", optional = true } +proptest = { workspace = true, optional = true } prost-types.workspace = true prost.workspace = true quanta = { version = "0.12.6", default-features = false } -regex = { version = "1.11.2", default-features = false, features = ["std", "perf"] } -ryu = { version = "1", default-features = false } +regex = { version = "1.12.3", default-features = false, features = ["std", "perf"] } +zmij = { version = "1", default-features = false } serde.workspace = true serde_json.workspace = true -serde_with = { version = "3.14.0", default-features = false, features = ["std", "macros"] } +serde_with.workspace = true smallvec = { version = "1", default-features = false, features = ["serde", "const_generics"] } snafu.workspace = true socket2.workspace = true @@ -65,24 +65,26 @@ vector-config = { path = "../vector-config" } vector-config-common = { path = "../vector-config-common" } vrl.workspace = true cfg-if.workspace = true +quickcheck = { workspace = true, optional = true } [target.'cfg(target_os = "macos")'.dependencies] -security-framework = "3.5.1" +security-framework = "3.6.0" [target.'cfg(windows)'.dependencies] -schannel = "0.1.28" +schannel = "0.1.29" [build-dependencies] prost-build.workspace = true [dev-dependencies] base64 = "0.22.1" +strum.workspace = true chrono-tz.workspace = true -criterion = { version = "0.7.0", features = ["html_reports"] } +criterion = { workspace = true, features = ["html_reports"] } env-test-util = "1.0.1" -quickcheck = "1" +quickcheck.workspace = true quickcheck_macros = "1" -proptest = "1.10" +proptest.workspace = true similar-asserts = "1.7.0" tokio-test.workspace = true toml.workspace = true @@ -100,6 +102,12 @@ default = [] lua = ["dep:mlua", "dep:tokio-stream", "vrl/lua"] vrl = [] test = ["vector-common/test", "proptest"] +generate-fixtures = ["vrl/generate-fixtures", "dep:quickcheck"] + +[[bin]] +name = "generate-fixtures" +path = "src/bin/generate_fixtures.rs" +required-features = ["generate-fixtures"] [[bench]] name = "event" diff --git a/lib/vector-core/src/bin/generate_fixtures.rs b/lib/vector-core/src/bin/generate_fixtures.rs new file mode 100644 index 0000000000000..9ab133757bbf8 --- /dev/null +++ b/lib/vector-core/src/bin/generate_fixtures.rs @@ -0,0 +1,38 @@ +use std::{fs::File, io::Write, path::PathBuf}; + +use bytes::BytesMut; +use prost::Message; +use quickcheck::{Arbitrary as _, Gen}; +use vector_core::event::{Event, EventArray, proto}; + +const SEED: u64 = 0; +const GEN_SIZE: usize = 128; + +fn main() { + let fixture_dir = + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../codecs/tests/data/native_encoding"); + let json_dir = fixture_dir.join("json"); + let proto_dir = fixture_dir.join("proto"); + std::fs::create_dir_all(&json_dir).unwrap(); + std::fs::create_dir_all(&proto_dir).unwrap(); + + let mut rng = Gen::from_size_and_seed(GEN_SIZE, SEED); + for n in 0..1024_usize { + let event = Event::arbitrary(&mut rng); + + let mut json_out = File::create(json_dir.join(format!("{n:04}.json"))).unwrap(); + serde_json::to_writer(&mut json_out, &event).unwrap(); + + let mut proto_out = File::create(proto_dir.join(format!("{n:04}.pb"))).unwrap(); + let mut buf = BytesMut::new(); + proto::EventArray::from(EventArray::from(event)) + .encode(&mut buf) + .unwrap(); + proto_out.write_all(&buf).unwrap(); + } + + #[allow(clippy::print_stdout)] + { + println!("Written 1024 fixtures to {}", fixture_dir.display()); + } +} diff --git a/lib/vector-core/src/config/global_options.rs b/lib/vector-core/src/config/global_options.rs index d86df49803552..2eb5fd3cdbda4 100644 --- a/lib/vector-core/src/config/global_options.rs +++ b/lib/vector-core/src/config/global_options.rs @@ -84,7 +84,11 @@ pub struct GlobalOptions { /// This is used if a component does not have its own specific log schema. All events use a log /// schema, whether or not the default is used, to assign event fields on incoming events. #[serde(default, skip_serializing_if = "crate::serde::is_default")] - #[configurable(metadata(docs::common = false, docs::required = false))] + #[configurable(metadata( + docs::common = false, + docs::required = false, + docs::warnings = "These settings are ignored when `schema.log_namespace` is set to `true`." + ))] pub log_schema: LogSchema, /// Telemetry options. diff --git a/lib/vector-core/src/config/mod.rs b/lib/vector-core/src/config/mod.rs index c86848d7b0be5..4e822bd84f59c 100644 --- a/lib/vector-core/src/config/mod.rs +++ b/lib/vector-core/src/config/mod.rs @@ -27,7 +27,7 @@ pub const MEMORY_BUFFER_DEFAULT_MAX_EVENTS: NonZeroUsize = vector_buffers::config::memory_buffer_default_max_events(); // This enum should be kept alphabetically sorted as the bitmask value is used when -// sorting sources by data type in the GraphQL API. +// sorting sources by data type in the API. #[bitmask(u8)] #[bitmask_config(flags_iter)] pub enum DataType { diff --git a/lib/vector-core/src/event/test/common.rs b/lib/vector-core/src/event/arbitrary_impl.rs similarity index 91% rename from lib/vector-core/src/event/test/common.rs rename to lib/vector-core/src/event/arbitrary_impl.rs index ab9460a7fa90f..fe02b846aa9b2 100644 --- a/lib/vector-core/src/event/test/common.rs +++ b/lib/vector-core/src/event/arbitrary_impl.rs @@ -4,7 +4,7 @@ use chrono::{DateTime, Utc}; use quickcheck::{Arbitrary, Gen, empty_shrinker}; use vrl::value::{ObjectMap, Value}; -use super::super::{ +use super::{ Event, EventMetadata, LogEvent, Metric, MetricKind, MetricValue, StatisticKind, TraceEvent, metric::{ Bucket, MetricData, MetricName, MetricSeries, MetricSketch, MetricTags, MetricTime, @@ -21,6 +21,27 @@ const ALPHABET: [&str; 27] = [ "t", "u", "v", "w", "x", "y", "z", "_", ]; +// When generating fixtures we need f64 values that survive a JSON round-trip +// without any loss of precision or serialization ambiguity (NaN, -0.0). +// Under the `generate-fixtures` feature the helper produces clean values; +// otherwise it falls back to the standard quickcheck approach. +fn f64_for_arbitrary(g: &mut Gen) -> f64 { + #[cfg(feature = "generate-fixtures")] + { + let mut value = f64::arbitrary(g) % MAX_F64_SIZE; + while value.is_nan() || value == -0.0 { + value = f64::arbitrary(g) % MAX_F64_SIZE; + } + let rounded = (value * 10_000.0).round() / 10_000.0; + // Rounding can produce -0.0 from small negatives; normalize to +0.0. + if rounded == -0.0_f64 { 0.0 } else { rounded } + } + #[cfg(not(feature = "generate-fixtures"))] + { + f64::arbitrary(g) % MAX_F64_SIZE + } +} + #[derive(Debug, Clone)] pub struct Name { inner: String, @@ -29,7 +50,11 @@ pub struct Name { impl Arbitrary for Name { fn arbitrary(g: &mut Gen) -> Self { let mut name = String::with_capacity(MAX_STR_SIZE); - for _ in 0..(g.size() % MAX_STR_SIZE) { + #[cfg(feature = "generate-fixtures")] + let len = usize::max(1, g.size() % MAX_STR_SIZE); + #[cfg(not(feature = "generate-fixtures"))] + let len = g.size() % MAX_STR_SIZE; + for _ in 0..len { let idx: usize = usize::arbitrary(g) % ALPHABET.len(); name.push_str(ALPHABET[idx]); } @@ -76,6 +101,9 @@ impl Arbitrary for Event { impl Arbitrary for LogEvent { fn arbitrary(g: &mut Gen) -> Self { + #[cfg(feature = "generate-fixtures")] + let mut generator = Gen::from_size_and_seed(MAX_MAP_SIZE, u64::arbitrary(g)); + #[cfg(not(feature = "generate-fixtures"))] let mut generator = Gen::new(MAX_MAP_SIZE); let map: ObjectMap = ObjectMap::arbitrary(&mut generator); let metadata: EventMetadata = EventMetadata::arbitrary(g); @@ -175,10 +203,10 @@ impl Arbitrary for MetricValue { // here toward `MetricValue::Counter` and `MetricValue::Gauge`. match u8::arbitrary(g) % 7 { 0 => MetricValue::Counter { - value: f64::arbitrary(g) % MAX_F64_SIZE, + value: f64_for_arbitrary(g), }, 1 => MetricValue::Gauge { - value: f64::arbitrary(g) % MAX_F64_SIZE, + value: f64_for_arbitrary(g), }, 2 => MetricValue::Set { values: BTreeSet::arbitrary(g), @@ -190,12 +218,12 @@ impl Arbitrary for MetricValue { 4 => MetricValue::AggregatedHistogram { buckets: Vec::arbitrary(g), count: u64::arbitrary(g), - sum: f64::arbitrary(g) % MAX_F64_SIZE, + sum: f64_for_arbitrary(g), }, 5 => MetricValue::AggregatedSummary { quantiles: Vec::arbitrary(g), count: u64::arbitrary(g), - sum: f64::arbitrary(g) % MAX_F64_SIZE, + sum: f64_for_arbitrary(g), }, 6 => { // We're working around quickcheck's limitations here, and @@ -203,7 +231,7 @@ impl Arbitrary for MetricValue { let num_samples = u8::arbitrary(g); let samples = std::iter::repeat_with(|| { loop { - let f = f64::arbitrary(g); + let f = f64_for_arbitrary(g); if f.is_normal() { return f; } @@ -214,6 +242,8 @@ impl Arbitrary for MetricValue { let mut sketch = AgentDDSketch::with_agent_defaults(); sketch.insert_many(&samples); + #[cfg(feature = "generate-fixtures")] + sketch.set_sum_avg(f64_for_arbitrary(g), f64_for_arbitrary(g)); MetricValue::Sketch { sketch: MetricSketch::AgentDDSketch(sketch), @@ -363,7 +393,7 @@ impl Arbitrary for MetricValue { impl Arbitrary for Sample { fn arbitrary(g: &mut Gen) -> Self { Sample { - value: f64::arbitrary(g) % MAX_F64_SIZE, + value: f64_for_arbitrary(g), rate: u32::arbitrary(g), } } @@ -393,8 +423,8 @@ impl Arbitrary for Sample { impl Arbitrary for Quantile { fn arbitrary(g: &mut Gen) -> Self { Quantile { - quantile: f64::arbitrary(g) % MAX_F64_SIZE, - value: f64::arbitrary(g) % MAX_F64_SIZE, + quantile: f64_for_arbitrary(g), + value: f64_for_arbitrary(g), } } @@ -423,7 +453,7 @@ impl Arbitrary for Quantile { impl Arbitrary for Bucket { fn arbitrary(g: &mut Gen) -> Self { Bucket { - upper_limit: f64::arbitrary(g) % MAX_F64_SIZE, + upper_limit: f64_for_arbitrary(g), count: u64::arbitrary(g), } } diff --git a/lib/vector-core/src/event/estimated_json_encoded_size_of.rs b/lib/vector-core/src/event/estimated_json_encoded_size_of.rs index 832dc70cfd343..c6e8c17bc95b2 100644 --- a/lib/vector-core/src/event/estimated_json_encoded_size_of.rs +++ b/lib/vector-core/src/event/estimated_json_encoded_size_of.rs @@ -120,13 +120,13 @@ impl EstimatedJsonEncodedSizeOf for bool { impl EstimatedJsonEncodedSizeOf for f64 { fn estimated_json_encoded_size_of(&self) -> JsonSize { - ryu::Buffer::new().format_finite(*self).len().into() + zmij::Buffer::new().format_finite(*self).len().into() } } impl EstimatedJsonEncodedSizeOf for f32 { fn estimated_json_encoded_size_of(&self) -> JsonSize { - ryu::Buffer::new().format_finite(*self).len().into() + zmij::Buffer::new().format_finite(*self).len().into() } } diff --git a/lib/vector-core/src/event/metric/tags.rs b/lib/vector-core/src/event/metric/tags.rs index 416f239af5ab5..d83813732be48 100644 --- a/lib/vector-core/src/event/metric/tags.rs +++ b/lib/vector-core/src/event/metric/tags.rs @@ -528,6 +528,12 @@ impl MetricTags { self.0.remove(name).and_then(TagValueSet::into_single) } + /// Remove a tag and return its full [`TagValueSet`], preserving all values rather than reducing + /// to a single string like [`Self::remove`]. + pub fn remove_set(&mut self, name: &str) -> Option { + self.0.remove(name) + } + pub fn keys(&self) -> impl Iterator { self.0.keys().map(String::as_str) } @@ -603,13 +609,13 @@ impl ByteSizeOf for MetricTags { } } -#[cfg(test)] +#[cfg(any(test, feature = "generate-fixtures"))] mod test_support { use std::collections::HashSet; use quickcheck::{Arbitrary, Gen}; - use super::*; + use super::{BTreeMap, MetricTags, TagValue, TagValueSet}; impl Arbitrary for TagValue { fn arbitrary(g: &mut Gen) -> Self { @@ -640,6 +646,33 @@ mod tests { use super::*; + fn make_tags(pairs: &[(&str, &str)]) -> MetricTags { + pairs + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect() + } + + #[test] + fn remove_set_returns_full_value_set() { + let mut tags = MetricTags::default(); + tags.insert("k".to_string(), TagValue::Value("a".to_string())); + tags.insert("k".to_string(), TagValue::Value("b".to_string())); + + let removed = tags.remove_set("k").expect("tag should exist"); + assert_eq!(removed.len(), 2); + assert!(removed.contains(&TagValue::Value("a".to_string()))); + assert!(removed.contains(&TagValue::Value("b".to_string()))); + assert!(!tags.contains_key("k")); + } + + #[test] + fn remove_set_missing_returns_none() { + let mut tags = make_tags(&[("a", "1")]); + assert!(tags.remove_set("missing").is_none()); + assert!(tags.contains_key("a")); + } + proptest! { #[test] fn reduces_set_to_simple(mut values: TagValueSet) { diff --git a/lib/vector-core/src/event/mod.rs b/lib/vector-core/src/event/mod.rs index 9d72301e85804..fd9d9c7c88bbd 100644 --- a/lib/vector-core/src/event/mod.rs +++ b/lib/vector-core/src/event/mod.rs @@ -7,7 +7,7 @@ pub use finalization::{ Finalizable, }; pub use log_event::LogEvent; -pub use metadata::{DatadogMetricOriginMetadata, EventMetadata, WithMetadata}; +pub use metadata::{DatadogMetricOriginMetadata, EventMetadata, Secrets, WithMetadata}; pub use metric::{Metric, MetricKind, MetricTags, MetricValue, StatisticKind}; pub use r#ref::{EventMutRef, EventRef}; use serde::{Deserialize, Serialize}; @@ -23,6 +23,8 @@ pub use vrl_target::{TargetEvents, VrlTarget}; use crate::config::{LogNamespace, OutputId}; +#[cfg(any(test, feature = "generate-fixtures"))] +pub(crate) mod arbitrary_impl; pub mod array; pub mod discriminant; mod estimated_json_encoded_size_of; @@ -35,6 +37,7 @@ pub mod metric; pub mod proto; mod r#ref; mod ser; + #[cfg(test)] mod test; mod trace; diff --git a/lib/vector-core/src/event/test/mod.rs b/lib/vector-core/src/event/test/mod.rs index 3762a53363602..8f9cbad4da2c4 100644 --- a/lib/vector-core/src/event/test/mod.rs +++ b/lib/vector-core/src/event/test/mod.rs @@ -1,4 +1,3 @@ -mod common; mod serialization; mod size_of; diff --git a/lib/vector-core/src/event/test/serialization.rs b/lib/vector-core/src/event/test/serialization.rs index 75304e3960d54..4bd7ac369e5c8 100644 --- a/lib/vector-core/src/event/test/serialization.rs +++ b/lib/vector-core/src/event/test/serialization.rs @@ -1,12 +1,11 @@ +use super::*; +use crate::config::log_schema; use bytes::{Buf, BufMut, BytesMut}; use quickcheck::{QuickCheck, TestResult}; use regex::Regex; use similar_asserts::assert_eq; use vector_buffers::encoding::Encodable; -use super::*; -use crate::config::log_schema; - fn encode_value(value: T, buffer: &mut B) { value.encode(buffer).expect("encoding should not fail"); } diff --git a/lib/vector-core/src/event/test/size_of.rs b/lib/vector-core/src/event/test/size_of.rs index 2428bc5ac6491..36bb1a0e23c4a 100644 --- a/lib/vector-core/src/event/test/size_of.rs +++ b/lib/vector-core/src/event/test/size_of.rs @@ -4,7 +4,8 @@ use lookup::{PathPrefix, path}; use quickcheck::{Arbitrary, Gen, QuickCheck, TestResult}; use vector_common::byte_size_of::ByteSizeOf; -use super::{common::Name, *}; +use super::*; +use crate::event::arbitrary_impl::Name; #[test] fn at_least_wrapper_size() { diff --git a/lib/vector-core/src/fanout.rs b/lib/vector-core/src/fanout.rs index 0fd5cb034b703..500b6b8583c51 100644 --- a/lib/vector-core/src/fanout.rs +++ b/lib/vector-core/src/fanout.rs @@ -7,7 +7,10 @@ use tokio::sync::mpsc; use tokio_util::sync::ReusableBoxFuture; use vector_buffers::topology::channel::BufferSender; -use crate::{config::ComponentKey, event::EventArray}; +use crate::{ + config::ComponentKey, + event::{EventArray, EventContainer}, +}; pub enum ControlMessage { /// Adds a new sink to the fanout. @@ -45,15 +48,17 @@ pub type ControlChannel = mpsc::UnboundedSender; pub struct Fanout { senders: IndexMap>, control_channel: mpsc::UnboundedReceiver, + upstream_component: ComponentKey, } impl Fanout { - pub fn new() -> (Self, ControlChannel) { + pub fn new(upstream_component: ComponentKey) -> (Self, ControlChannel) { let (control_tx, control_rx) = mpsc::unbounded_channel(); let fanout = Self { senders: Default::default(), control_channel: control_rx, + upstream_component, }; (fanout, control_tx) @@ -108,6 +113,20 @@ impl Fanout { } } + /// Waits for the next control message and applies it. + /// + /// Returns `true` if a message was processed, `false` if the control + /// channel was closed. + pub async fn recv_control_message(&mut self) -> bool { + match self.control_channel.recv().await { + Some(msg) => { + self.apply_control_message(msg); + true + } + None => false, + } + } + /// Apply a control message directly against this instance. /// /// This method should not be used if there is an active `SendGroup` being processed. @@ -206,6 +225,27 @@ impl Fanout { // Wait for any senders that are paused to be replaced first before continuing with the send. self.wait_for_replacements().await; + // Drop empty event batches before they reach any downstream buffer, this is technically + // programmer error. In debug/test builds the `debug_assert!` makes the underlying bug fail + // loudly. + debug_assert!( + !events.is_empty(), + "Fanout received empty event batch from upstream component '{}'", + self.upstream_component, + ); + // TODO: Wrap the conditional below with `std::hint::unlikely` once it stabilizes. This is an + // applicable situation to use it in since the following conditional should never evaluate to + // true. + #[cfg(not(debug_assertions))] + if events.is_empty() { + warn!( + message = "Dropping empty event batch emitted by upstream component. This is likely a bug in that component.", + component_id = %self.upstream_component, + downstream_count = self.senders.len(), + ); + return Ok(()); + } + // Nothing to send if we have no sender. if self.senders.is_empty() { trace!("No senders present."); @@ -318,6 +358,10 @@ impl<'a> SendGroup<'a> { fn try_detach_send(&mut self, id: &ComponentKey) -> bool { if let Some(send) = self.sends.remove(id) { + // Deliberately not instrumented with the current span: this drains a send to a sink + // that has just been detached from the topology, so it is unrelated to the upstream + // component that owns this fanout. Attaching the current span would mis-tag this + // task's logs with the upstream component's identity rather than the detached sink's. tokio::spawn(async move { if let Err(e) = send.await { warn!( @@ -511,7 +555,7 @@ mod tests { UnboundedSender, Vec>, ) { - let (mut fanout, control) = Fanout::new(); + let (mut fanout, control) = Fanout::new(ComponentKey::from("test_upstream")); let pairs = build_sender_pairs(capacities); let mut receivers = Vec::new(); @@ -813,7 +857,7 @@ mod tests { #[tokio::test] async fn fanout_no_sinks() { - let (mut fanout, _) = Fanout::new(); + let (mut fanout, _) = Fanout::new(ComponentKey::from("test_upstream")); let events = make_events(2); fanout @@ -826,6 +870,15 @@ mod tests { .expect("should not fail"); } + #[tokio::test] + #[should_panic(expected = "Fanout received empty event batch from upstream component")] + async fn fanout_panics_on_empty_event_array_in_debug_builds() { + let (mut fanout, _, _receivers) = fanout_from_senders(&[2, 2]); + let empty: EventArray = Vec::::new().into(); + + let _ = fanout.send(empty, None).await; + } + #[tokio::test] async fn fanout_replace() { let (mut fanout, control, mut receivers) = fanout_from_senders(&[4, 4, 4]); diff --git a/lib/vector-core/src/latency.rs b/lib/vector-core/src/latency.rs index 536a8ef532d0d..b25ca73427749 100644 --- a/lib/vector-core/src/latency.rs +++ b/lib/vector-core/src/latency.rs @@ -1,12 +1,14 @@ use std::time::Instant; -use metrics::{Histogram, gauge, histogram}; +use metrics::Histogram; use vector_common::stats::EwmaGauge; +use vector_common::{ + gauge, histogram, + internal_event::{GaugeName, HistogramName}, +}; use crate::event::EventArray; -const COMPONENT_LATENCY: &str = "component_latency_seconds"; -const COMPONENT_LATENCY_MEAN: &str = "component_latency_mean_seconds"; const DEFAULT_LATENCY_EWMA_ALPHA: f64 = 0.9; #[derive(Debug)] @@ -18,9 +20,9 @@ pub struct LatencyRecorder { impl LatencyRecorder { pub fn new(ewma_alpha: Option) -> Self { Self { - histogram: histogram!(COMPONENT_LATENCY), + histogram: histogram!(HistogramName::ComponentLatencySeconds), gauge: EwmaGauge::new( - gauge!(COMPONENT_LATENCY_MEAN), + gauge!(GaugeName::ComponentLatencyMeanSeconds), ewma_alpha.or(Some(DEFAULT_LATENCY_EWMA_ALPHA)), ), } diff --git a/lib/vector-core/src/lib.rs b/lib/vector-core/src/lib.rs index ad1b36d1db516..02d202c7780bc 100644 --- a/lib/vector-core/src/lib.rs +++ b/lib/vector-core/src/lib.rs @@ -39,6 +39,7 @@ pub mod serde; pub mod sink; pub mod source; pub mod source_sender; +pub mod span_fields; pub mod tcp; #[cfg(test)] mod test_util; @@ -83,3 +84,10 @@ macro_rules! register { vector_lib::internal_event::register($event) }; } + +pub use span_fields::SpanField; + +// Re-export `inventory` so `register_extra_span_field!` can resolve `submit!` through this +// crate without forcing downstream callers to declare `inventory` as a direct dependency. +#[doc(hidden)] +pub use inventory as __inventory; diff --git a/lib/vector-core/src/metrics/ddsketch.rs b/lib/vector-core/src/metrics/ddsketch.rs index b66f25ceb150b..886a5c190bf32 100644 --- a/lib/vector-core/src/metrics/ddsketch.rs +++ b/lib/vector-core/src/metrics/ddsketch.rs @@ -284,6 +284,17 @@ impl AgentDDSketch { }) } + /// Overrides `sum` and `avg` with arbitrary values. + /// + /// Only available under the `generate-fixtures` feature, where we need to + /// produce sketches with independently-randomized summary statistics to + /// exercise round-trip serialization of those fields. + #[cfg(feature = "generate-fixtures")] + pub fn set_sum_avg(&mut self, sum: f64, avg: f64) { + self.sum = sum; + self.avg = avg; + } + pub fn gamma(&self) -> f64 { self.config.gamma_v } diff --git a/lib/vector-core/src/metrics/label_filter.rs b/lib/vector-core/src/metrics/label_filter.rs index 8390620502a47..233f9e030e6ac 100644 --- a/lib/vector-core/src/metrics/label_filter.rs +++ b/lib/vector-core/src/metrics/label_filter.rs @@ -1,6 +1,35 @@ +use std::collections::HashSet; +use std::sync::LazyLock; + use metrics::{KeyName, Label}; use metrics_tracing_context::LabelFilter; +/// A label name that should be preserved on metric keys when present as a tracing-span field. +/// +/// Both Vector's own built-in global labels and downstream-registered labels go through this +/// type. Registrations are collected at link time via [`inventory`]. Downstream crates use the +/// [`register_extra_span_field!`](crate::register_extra_span_field) macro to add one. +#[derive(Debug)] +pub struct MetricLabel(pub &'static str); + +inventory::collect!(MetricLabel); + +// Vector's own global labels are registered through the same mechanism so the filter only has +// one allowlist to consult. +inventory::submit!(MetricLabel("component_id")); +inventory::submit!(MetricLabel("component_type")); +inventory::submit!(MetricLabel("component_kind")); +inventory::submit!(MetricLabel("buffer_type")); + +/// Snapshot of every registered [`MetricLabel`], built on first access. `inventory` populates +/// all submissions before `main`, so the snapshot is guaranteed to capture every entry β€” the +/// hot path is then a single set lookup against this static. +pub static LABELS: LazyLock> = LazyLock::new(|| { + inventory::iter::() + .map(|label| label.0) + .collect() +}); + #[derive(Debug, Clone)] pub(crate) struct VectorLabelFilter; @@ -19,10 +48,56 @@ impl LabelFilter for VectorLabelFilter { { return true; } - // Global labels - label_key == "component_id" - || label_key == "component_type" - || label_key == "component_kind" - || label_key == "buffer_type" + // Globally-registered labels: Vector's own built-ins plus any registered by downstream + // crates via `register_extra_span_field!`. + LABELS.contains(label_key) + } +} + +#[cfg(test)] +mod tests { + use metrics::{KeyName, Label}; + use metrics_tracing_context::LabelFilter; + + use super::VectorLabelFilter; + + // Use the macro so that removing either arm (MetricLabel or SpanField) from + // register_extra_span_field! would cause the tests below to fail, locking the + // dual-registration contract. + crate::register_extra_span_field!("test_extra_label"); + + fn key(name: &'static str) -> KeyName { + KeyName::from_const_str(name) + } + + #[test] + fn includes_globally_registered_label() { + let filter = VectorLabelFilter; + let label = Label::new("test_extra_label", "value"); + assert!(filter.should_include_label(&key("any_metric"), &label)); + } + + #[test] + fn excludes_unregistered_label() { + let filter = VectorLabelFilter; + let label = Label::new("not_registered", "value"); + assert!(!filter.should_include_label(&key("any_metric"), &label)); + } + + #[test] + fn includes_built_in_global_labels() { + let filter = VectorLabelFilter; + for builtin in [ + "component_id", + "component_type", + "component_kind", + "buffer_type", + ] { + let label = Label::new(builtin, "value"); + assert!( + filter.should_include_label(&key("any_metric"), &label), + "expected built-in label `{builtin}` to remain included", + ); + } } } diff --git a/lib/vector-core/src/metrics/mod.rs b/lib/vector-core/src/metrics/mod.rs index 660f29c000db3..0e56d28112159 100644 --- a/lib/vector-core/src/metrics/mod.rs +++ b/lib/vector-core/src/metrics/mod.rs @@ -15,6 +15,7 @@ use metrics_util::layers::Layer; use snafu::Snafu; pub use self::ddsketch::{AgentDDSketch, BinMap, Config}; +pub use self::label_filter::{LABELS, MetricLabel}; use self::{ label_filter::VectorLabelFilter, recorder::{Registry, VectorRecorder}, @@ -200,52 +201,14 @@ impl Controller { } } -#[macro_export] -/// This macro is used to emit metrics as a `counter` while simultaneously -/// converting from absolute values to incremental values. -/// -/// Values that do not arrive in strictly monotonically increasing order are -/// ignored and will not be emitted. -macro_rules! update_counter { - ($label:literal, $value:expr) => {{ - use ::std::sync::atomic::{AtomicU64, Ordering}; - - static PREVIOUS_VALUE: AtomicU64 = AtomicU64::new(0); - - let new_value = $value; - let mut previous_value = PREVIOUS_VALUE.load(Ordering::Relaxed); - - loop { - // Either a new greater value has been emitted before this thread updated the counter - // or values were provided that are not in strictly monotonically increasing order. - // Ignore. - if new_value <= previous_value { - break; - } - - match PREVIOUS_VALUE.compare_exchange_weak( - previous_value, - new_value, - Ordering::SeqCst, - Ordering::Relaxed, - ) { - // Another thread has written a new value before us. Re-enter loop. - Err(value) => previous_value = value, - // Calculate delta to last emitted value and emit it. - Ok(_) => { - let delta = new_value - previous_value; - // Albeit very unlikely, note that this sequence of deltas might be emitted in - // a different order than they were calculated. - counter!($label).increment(delta); - break; - } - } - } - }}; -} - #[cfg(test)] mod tests { + use strum::IntoEnumIterator; + use vector_common::{ + counter, gauge, + internal_event::{CounterName, GaugeName}, + }; + use super::*; use crate::{ config::metrics_expiration::{ @@ -268,8 +231,9 @@ mod tests { let controller = Controller::get().unwrap(); controller.reset(); + let name = CounterName::iter().next().unwrap(); for idx in 0..cardinality { - metrics::counter!("test", "idx" => idx.to_string()).increment(1); + counter!(name, "idx" => idx.to_string()).increment(1); } let metrics = controller.capture_metrics(); @@ -297,11 +261,11 @@ mod tests { fn handles_registered_metrics() { let controller = init_metrics(); - let counter = metrics::counter!("test7"); + let counter = counter!(CounterName::iter().next().unwrap()); assert_eq!(controller.capture_metrics().len(), 3); counter.increment(1); assert_eq!(controller.capture_metrics().len(), 3); - let gauge = metrics::gauge!("test8"); + let gauge = gauge!(GaugeName::iter().next().unwrap()); assert_eq!(controller.capture_metrics().len(), 4); gauge.set(1.0); assert_eq!(controller.capture_metrics().len(), 4); @@ -314,12 +278,16 @@ mod tests { .set_expiry(Some(IDLE_TIMEOUT), Vec::new()) .unwrap(); - metrics::counter!("test2").increment(1); - metrics::counter!("test3").increment(2); + let mut names = CounterName::iter(); + let name_a = names.next().unwrap(); + let name_b = names.next().unwrap(); + + counter!(name_a).increment(1); + counter!(name_b).increment(2); assert_eq!(controller.capture_metrics().len(), 4); std::thread::sleep(Duration::from_secs_f64(IDLE_TIMEOUT * 2.0)); - metrics::counter!("test2").increment(3); + counter!(name_a).increment(3); assert_eq!(controller.capture_metrics().len(), 3); } @@ -330,12 +298,13 @@ mod tests { .set_expiry(Some(IDLE_TIMEOUT), Vec::new()) .unwrap(); - metrics::counter!("test4", "tag" => "value1").increment(1); - metrics::counter!("test4", "tag" => "value2").increment(2); + let name = CounterName::iter().next().unwrap(); + counter!(name, "tag" => "value1").increment(1); + counter!(name, "tag" => "value2").increment(2); assert_eq!(controller.capture_metrics().len(), 4); std::thread::sleep(Duration::from_secs_f64(IDLE_TIMEOUT * 2.0)); - metrics::counter!("test4", "tag" => "value1").increment(3); + counter!(name, "tag" => "value1").increment(3); assert_eq!(controller.capture_metrics().len(), 3); } @@ -346,8 +315,12 @@ mod tests { .set_expiry(Some(IDLE_TIMEOUT), Vec::new()) .unwrap(); - let a = metrics::counter!("test5"); - metrics::counter!("test6").increment(5); + let mut names = CounterName::iter(); + let name_a = names.next().unwrap(); + let name_b = names.next().unwrap(); + + let a = counter!(name_a); + counter!(name_b).increment(5); assert_eq!(controller.capture_metrics().len(), 4); a.increment(1); assert_eq!(controller.capture_metrics().len(), 4); @@ -360,7 +333,7 @@ mod tests { assert_eq!(metrics.len(), 3); let metric = metrics .into_iter() - .find(|metric| metric.name() == "test5") + .find(|metric| metric.name() == name_a.as_str()) .expect("Test metric is not present"); match metric.value() { MetricValue::Counter { value } => assert_eq!(*value, 2.0), @@ -371,12 +344,17 @@ mod tests { #[test] fn expires_metrics_per_set() { let controller = init_metrics(); + + let mut names = CounterName::iter(); + let name_a = names.next().unwrap(); + let name_b = names.next().unwrap(); + controller .set_expiry( None, vec![PerMetricSetExpiration { name: Some(MetricNameMatcherConfig::Exact { - value: "test3".to_string(), + value: name_b.as_str().to_string(), }), labels: None, expire_secs: IDLE_TIMEOUT, @@ -384,25 +362,32 @@ mod tests { ) .unwrap(); - metrics::counter!("test2").increment(1); - metrics::counter!("test3").increment(2); + counter!(name_a).increment(1); + counter!(name_b).increment(2); assert_eq!(controller.capture_metrics().len(), 4); std::thread::sleep(Duration::from_secs_f64(IDLE_TIMEOUT * 2.0)); - metrics::counter!("test2").increment(3); + counter!(name_a).increment(3); assert_eq!(controller.capture_metrics().len(), 3); } #[test] fn expires_metrics_multiple_different_sets() { let controller = init_metrics(); + + let mut names = CounterName::iter(); + let name_a = names.next().unwrap(); + let name_b = names.next().unwrap(); + let name_c = names.next().unwrap(); + let name_d = names.next().unwrap(); + controller .set_expiry( Some(IDLE_TIMEOUT * 3.0), vec![ PerMetricSetExpiration { name: Some(MetricNameMatcherConfig::Exact { - value: "test3".to_string(), + value: name_c.as_str().to_string(), }), labels: None, expire_secs: IDLE_TIMEOUT, @@ -421,22 +406,22 @@ mod tests { ) .unwrap(); - metrics::counter!("test1").increment(1); - metrics::counter!("test2").increment(1); - metrics::counter!("test3").increment(2); - metrics::counter!("test4", "tag" => "value1").increment(3); + counter!(name_a).increment(1); + counter!(name_b).increment(1); + counter!(name_c).increment(2); + counter!(name_d, "tag" => "value1").increment(3); assert_eq!(controller.capture_metrics().len(), 6); std::thread::sleep(Duration::from_secs_f64(IDLE_TIMEOUT * 1.5)); - metrics::counter!("test2").increment(3); + counter!(name_b).increment(3); assert_eq!(controller.capture_metrics().len(), 5); std::thread::sleep(Duration::from_secs_f64(IDLE_TIMEOUT)); - metrics::counter!("test2").increment(3); + counter!(name_b).increment(3); assert_eq!(controller.capture_metrics().len(), 4); std::thread::sleep(Duration::from_secs_f64(IDLE_TIMEOUT)); - metrics::counter!("test2").increment(3); + counter!(name_b).increment(3); assert_eq!(controller.capture_metrics().len(), 3); } } diff --git a/lib/vector-core/src/source_sender/builder.rs b/lib/vector-core/src/source_sender/builder.rs index ec16d54d482a9..7f9f5d522497a 100644 --- a/lib/vector-core/src/source_sender/builder.rs +++ b/lib/vector-core/src/source_sender/builder.rs @@ -1,17 +1,20 @@ use std::{collections::HashMap, time::Duration}; -use metrics::{Histogram, histogram}; use vector_buffers::topology::channel::LimitedReceiver; +use vector_common::histogram; use vector_common::internal_event::DEFAULT_OUTPUT; -use super::{CHUNK_SIZE, LAG_TIME_NAME, Output, SourceSender, SourceSenderItem}; +use super::{ + CHUNK_SIZE, LAG_TIME_NAME, Output, OutputMetrics, SEND_BATCH_LATENCY_NAME, SEND_LATENCY_NAME, + SourceSender, SourceSenderItem, +}; use crate::config::{ComponentKey, OutputId, SourceOutput}; pub struct Builder { buf_size: usize, default_output: Option, named_outputs: HashMap, - lag_time: Option, + output_metrics: OutputMetrics, timeout: Option, ewma_half_life_seconds: Option, } @@ -22,7 +25,11 @@ impl Default for Builder { buf_size: CHUNK_SIZE, default_output: None, named_outputs: Default::default(), - lag_time: Some(histogram!(LAG_TIME_NAME)), + output_metrics: OutputMetrics::new( + Some(histogram!(LAG_TIME_NAME)), + Some(histogram!(SEND_LATENCY_NAME)), + Some(histogram!(SEND_BATCH_LATENCY_NAME)), + ), timeout: None, ewma_half_life_seconds: None, } @@ -53,7 +60,6 @@ impl Builder { output: SourceOutput, component_key: ComponentKey, ) -> LimitedReceiver { - let lag_time = self.lag_time.clone(); let log_definition = output.schema_definition.clone(); let output_id = OutputId { component: component_key, @@ -64,7 +70,7 @@ impl Builder { let (output, rx) = Output::new_with_buffer( self.buf_size, DEFAULT_OUTPUT.to_owned(), - lag_time, + self.output_metrics.clone(), log_definition, output_id, self.timeout, @@ -77,7 +83,7 @@ impl Builder { let (output, rx) = Output::new_with_buffer( self.buf_size, name.clone(), - lag_time, + self.output_metrics.clone(), log_definition, output_id, self.timeout, diff --git a/lib/vector-core/src/source_sender/mod.rs b/lib/vector-core/src/source_sender/mod.rs index c8af2db8bbf87..24eb23688c0a1 100644 --- a/lib/vector-core/src/source_sender/mod.rs +++ b/lib/vector-core/src/source_sender/mod.rs @@ -14,7 +14,7 @@ mod tests; pub use builder::Builder; pub use errors::SendError; -use output::Output; +use output::{Output, OutputMetrics}; pub use sender::{SourceSender, SourceSenderItem}; pub const CHUNK_SIZE: usize = 1000; @@ -22,4 +22,8 @@ pub const CHUNK_SIZE: usize = 1000; #[cfg(any(test, feature = "test"))] const TEST_BUFFER_SIZE: usize = 100; -const LAG_TIME_NAME: &str = "source_lag_time_seconds"; +use vector_common::internal_event::HistogramName; + +const LAG_TIME_NAME: HistogramName = HistogramName::SourceLagTimeSeconds; +const SEND_LATENCY_NAME: HistogramName = HistogramName::SourceSendLatencySeconds; +const SEND_BATCH_LATENCY_NAME: HistogramName = HistogramName::SourceSendBatchLatencySeconds; diff --git a/lib/vector-core/src/source_sender/output.rs b/lib/vector-core/src/source_sender/output.rs index df0bee2a43da0..fc596c6b9a50b 100644 --- a/lib/vector-core/src/source_sender/output.rs +++ b/lib/vector-core/src/source_sender/output.rs @@ -11,7 +11,9 @@ use metrics::Histogram; use tracing::Span; use vector_buffers::{ config::MemoryBufferSize, - topology::channel::{self, ChannelMetricMetadata, LimitedReceiver, LimitedSender}, + topology::channel::{ + self, BufferChannelKind, ChannelMetricMetadata, LimitedReceiver, LimitedSender, + }, }; use vector_common::{ byte_size_of::ByteSizeOf, @@ -30,8 +32,6 @@ use crate::{ schema::Definition, }; -const UTILIZATION_METRIC_PREFIX: &str = "source_buffer"; - /// UnsentEvents tracks the number of events yet to be sent in the buffer. This is used to /// increment the appropriate counters when a future is not polled to completion. Particularly, /// this is known to happen in a Warp server when a client sends a new HTTP request on a TCP @@ -85,7 +85,7 @@ impl Drop for UnsentEventCount { #[derive(Clone)] pub(super) struct Output { sender: LimitedSender, - lag_time: Option, + metrics: OutputMetrics, events_sent: Registered, /// The schema definition that will be attached to Log events sent through here log_definition: Option>, @@ -95,6 +95,27 @@ pub(super) struct Output { timeout: Option, } +#[derive(Clone, Default)] +pub(super) struct OutputMetrics { + lag_time: Option, + send_latency: Option, + send_batch_latency: Option, +} + +impl OutputMetrics { + pub(super) fn new( + lag_time: Option, + send_latency: Option, + send_batch_latency: Option, + ) -> Self { + Self { + lag_time, + send_latency, + send_batch_latency, + } + } +} + #[expect(clippy::missing_fields_in_debug)] impl fmt::Debug for Output { fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { @@ -111,19 +132,20 @@ impl Output { pub(super) fn new_with_buffer( n: usize, output: String, - lag_time: Option, + metrics: OutputMetrics, log_definition: Option>, output_id: OutputId, timeout: Option, ewma_half_life_seconds: Option, ) -> (Self, LimitedReceiver) { let limit = MemoryBufferSize::MaxEvents(NonZeroUsize::new(n).unwrap()); - let metrics = ChannelMetricMetadata::new(UTILIZATION_METRIC_PREFIX, Some(output.clone())); - let (tx, rx) = channel::limited(limit, Some(metrics), ewma_half_life_seconds); + let channel_metrics = + ChannelMetricMetadata::new(BufferChannelKind::Source, Some(output.clone())); + let (tx, rx) = channel::limited(limit, Some(channel_metrics), ewma_half_life_seconds); ( Self { sender: tx, - lag_time, + metrics, events_sent: internal_event::register(EventsSent::from(internal_event::Output( Some(output.into()), ))), @@ -136,12 +158,21 @@ impl Output { } pub(super) async fn send( + &mut self, + events: EventArray, + unsent_event_count: &mut UnsentEventCount, + ) -> Result<(), SendError> { + let reference = Utc::now().timestamp_millis(); + self.send_inner(events, unsent_event_count, reference).await + } + + async fn send_inner( &mut self, mut events: EventArray, unsent_event_count: &mut UnsentEventCount, + reference: i64, ) -> Result<(), SendError> { let send_reference = Instant::now(); - let reference = Utc::now().timestamp_millis(); events .iter_events() .for_each(|event| self.emit_lag_time(event, reference)); @@ -156,7 +187,17 @@ impl Output { let byte_size = events.estimated_json_encoded_size_of(); let count = events.len(); - self.send_with_timeout(events, send_reference).await?; + + let send_start = Instant::now(); + + let send_result = self.send_with_timeout(events, send_reference).await; + + if let Some(send_latency) = &self.metrics.send_latency { + send_latency.record(send_start.elapsed().as_secs_f64()); + } + + send_result?; + self.events_sent.emit(CountByteSize(count, byte_size)); unsent_event_count.decr(count); Ok(()) @@ -218,25 +259,39 @@ impl Output { I: IntoIterator, ::IntoIter: ExactSizeIterator, { + // Capture a single reference timestamp for the entire batch so that lag time + // measurements are not inflated by channel-send latency for later chunks. + let reference = Utc::now().timestamp_millis(); + // It's possible that the caller stops polling this future while it is blocked waiting // on `self.send()`. When that happens, we use `UnsentEventCount` to correctly emit // `ComponentEventsDropped` events. let events = events.into_iter().map(Into::into); let mut unsent_event_count = UnsentEventCount::new(events.len()); + let send_batch_start = Instant::now(); + for events in array::events_into_arrays(events, Some(CHUNK_SIZE)) { - self.send(events, &mut unsent_event_count) + self.send_inner(events, &mut unsent_event_count, reference) .await - .inspect_err(|error| match error { - SendError::Timeout => { - unsent_event_count.timed_out(); + .inspect_err(|error| { + match error { + SendError::Timeout => { + unsent_event_count.timed_out(); + } + SendError::Closed => { + // The unsent event count is discarded here because the callee emits the + // `StreamClosedError`. + unsent_event_count.discard(); + } } - SendError::Closed => { - // The unsent event count is discarded here because the callee emits the - // `StreamClosedError`. - unsent_event_count.discard(); + if let Some(send_batch_latency) = &self.metrics.send_batch_latency { + send_batch_latency.record(send_batch_start.elapsed().as_secs_f64()); } })?; } + if let Some(send_batch_latency) = &self.metrics.send_batch_latency { + send_batch_latency.record(send_batch_start.elapsed().as_secs_f64()); + } Ok(()) } @@ -244,7 +299,7 @@ impl Output { /// timestamp stored in the given event reference, and emit the /// different, as expressed in milliseconds, as a histogram. pub(super) fn emit_lag_time(&self, event: EventRef<'_>, reference: i64) { - if let Some(lag_time_metric) = &self.lag_time { + if let Some(lag_time_metric) = &self.metrics.lag_time { let timestamp = match event { EventRef::Log(log) => { log_schema() diff --git a/lib/vector-core/src/source_sender/sender.rs b/lib/vector-core/src/source_sender/sender.rs index 88cb50172c70a..d5834ab0edff5 100644 --- a/lib/vector-core/src/source_sender/sender.rs +++ b/lib/vector-core/src/source_sender/sender.rs @@ -5,12 +5,12 @@ use std::{collections::HashMap, time::Instant}; use futures::Stream; #[cfg(any(test, feature = "test"))] use futures::StreamExt as _; -#[cfg(any(test, feature = "test"))] -use metrics::histogram; use vector_buffers::EventCount; #[cfg(any(test, feature = "test"))] use vector_buffers::topology::channel::LimitedReceiver; #[cfg(any(test, feature = "test"))] +use vector_common::histogram; +#[cfg(any(test, feature = "test"))] use vector_common::internal_event::DEFAULT_OUTPUT; #[cfg(doc)] use vector_common::internal_event::{ComponentEventsDropped, EventsSent}; @@ -22,7 +22,9 @@ use vector_common::{ use super::{Builder, Output, SendError}; #[cfg(any(test, feature = "test"))] -use super::{LAG_TIME_NAME, TEST_BUFFER_SIZE}; +use super::{ + LAG_TIME_NAME, OutputMetrics, SEND_BATCH_LATENCY_NAME, SEND_LATENCY_NAME, TEST_BUFFER_SIZE, +}; use crate::{ EstimatedJsonEncodedSizeOf, event::{Event, EventArray, EventContainer, array::EventArrayIntoIter}, @@ -108,6 +110,8 @@ impl SourceSender { timeout: Option, ) -> (Self, LimitedReceiver) { let lag_time = Some(histogram!(LAG_TIME_NAME)); + let send_latency = Some(histogram!(SEND_LATENCY_NAME)); + let send_batch_latency = Some(histogram!(SEND_BATCH_LATENCY_NAME)); let output_id = OutputId { component: "test".to_string().into(), port: None, @@ -115,7 +119,7 @@ impl SourceSender { let (default_output, rx) = Output::new_with_buffer( n, DEFAULT_OUTPUT.to_owned(), - lag_time, + OutputMetrics::new(lag_time, send_latency, send_batch_latency), None, output_id, timeout, @@ -192,8 +196,15 @@ impl SourceSender { component: "test".to_string().into(), port: Some(name.clone()), }; - let (output, recv) = - Output::new_with_buffer(100, name.clone(), None, None, output_id, None, None); + let (output, recv) = Output::new_with_buffer( + 100, + name.clone(), + OutputMetrics::default(), + None, + output_id, + None, + None, + ); let recv = recv.into_stream().map(move |mut item| { item.events.iter_events_mut().for_each(|mut event| { let metadata = event.metadata_mut(); diff --git a/lib/vector-core/src/span_fields.rs b/lib/vector-core/src/span_fields.rs new file mode 100644 index 0000000000000..2193c1dc8fad2 --- /dev/null +++ b/lib/vector-core/src/span_fields.rs @@ -0,0 +1,42 @@ +/// Span field name that should be captured onto log events emitted by the `internal_logs` +/// source. Vector's `SpanFields` visitor only captures fields `component_*` by default; +/// downstream crates can extend that set through this type. +/// +/// Use [`register_extra_span_field!`](crate::register_extra_span_field) to register one. +#[derive(Debug)] +pub struct SpanField(pub &'static str); // name of the span field + +inventory::collect!(SpanField); // collect the span field names + +/// Register a tracing-span field name that downstream crates want preserved on Vector's +/// internal observability output. +/// +/// A single registration covers both output channels: +/// +/// * On metrics, the field is added to the allowlist consulted by +/// [`VectorLabelFilter`](crate::metrics) (alongside Vector's built-in `component_id`, +/// `component_type`, `component_kind`, `buffer_type`), so `metrics-tracing-context` no +/// longer drops it before the metrics registry sees it. +/// * On logs/traces emitted via `internal_logs`, the field is added to the allowlist +/// consulted by `SpanFields` (alongside the existing `component_*` prefix gate), so it is +/// captured onto the log event under `vector.`. +/// +/// Example: an embedder that owns a "deployment-version" concept of its own can write +/// `register_extra_span_field!("deployment_version");` once at module scope and any internal +/// metric or log emitted from inside a span carrying that field will inherit it. +/// +/// Registrations are collected at link time via the `inventory` crate, so both read paths +/// are lock-free. The expansion goes through this crate's re-exports of `inventory`, +/// [`MetricLabel`](crate::metrics::MetricLabel), and [`SpanField`], so callers do not need +/// a direct `inventory` dependency. +#[macro_export] +macro_rules! register_extra_span_field { + ($key:expr) => { + $crate::__inventory::submit! { + $crate::metrics::MetricLabel($key) + } + $crate::__inventory::submit! { + $crate::SpanField($key) + } + }; +} diff --git a/lib/vector-core/src/transform/outputs.rs b/lib/vector-core/src/transform/outputs.rs index 7918fb008f764..113f5f2b3418f 100644 --- a/lib/vector-core/src/transform/outputs.rs +++ b/lib/vector-core/src/transform/outputs.rs @@ -42,7 +42,7 @@ impl TransformOutputs { let mut controls = HashMap::new(); for output in outputs_in { - let (fanout, control) = Fanout::new(); + let (fanout, control) = Fanout::new(component_key.clone()); let log_schema_definitions = output .log_schema_definitions diff --git a/lib/vector-core/tests/data/fixtures/log_event/malformed.json b/lib/vector-core/tests/data/fixtures/log_event/malformed.json index 6f956957a7b81..32603adb1fb2f 100644 --- a/lib/vector-core/tests/data/fixtures/log_event/malformed.json +++ b/lib/vector-core/tests/data/fixtures/log_event/malformed.json @@ -1,4 +1,4 @@ { "foo": "bar", "foo": "You're not allowed duplicate keys..." -} \ No newline at end of file +} diff --git a/lib/vector-lib/Cargo.toml b/lib/vector-lib/Cargo.toml index 6539c82dfaa80..6f42af4dbb627 100644 --- a/lib/vector-lib/Cargo.toml +++ b/lib/vector-lib/Cargo.toml @@ -27,6 +27,7 @@ vrl = { workspace = true, optional = true } allocation-tracing = ["vector-top?/allocation-tracing"] api-client = ["dep:vector-api-client"] arrow = ["codecs/arrow"] +parquet = ["codecs/parquet"] api = ["vector-tap/api"] file-source = ["dep:file-source", "dep:file-source-common"] lua = ["vector-core/lua"] diff --git a/lib/vector-lib/src/lib.rs b/lib/vector-lib/src/lib.rs index 82bb67cf9afce..1247085a058a3 100644 --- a/lib/vector-lib/src/lib.rs +++ b/lib/vector-lib/src/lib.rs @@ -11,18 +11,20 @@ pub use vector_buffers as buffers; pub use vector_common::event_test_util; pub use vector_common::{ Error, NamedInternalEvent, Result, TimeZone, assert_event_data_eq, atomic, btreemap, - byte_size_of, byte_size_of::ByteSizeOf, conversion, encode_logfmt, finalization, finalizer, id, - impl_event_data_eq, internal_event, json_size, registered_event, request_metadata, - sensitive_string, shutdown, stats, trigger, + byte_size_of, byte_size_of::ByteSizeOf, conversion, counter, encode_logfmt, finalization, + finalizer, gauge, histogram, id, impl_event_data_eq, internal_event, json_size, + registered_event, request_metadata, sensitive_string, shutdown, spawn_in_current_span, stats, + trigger, }; pub use vector_config as configurable; pub use vector_config::impl_generate_config_from_default; #[cfg(feature = "vrl")] pub use vector_core::compile_vrl; pub use vector_core::{ - EstimatedJsonEncodedSizeOf, buckets, default_data_dir, emit, event, fanout, ipallowlist, - latency, metric_tags, metrics, partition, quantiles, register, samples, schema, serde, sink, - source, source_sender, tcp, tls, transform, + EstimatedJsonEncodedSizeOf, SpanField, buckets, default_data_dir, emit, event, fanout, + ipallowlist, latency, metric_tags, metrics, partition, quantiles, register, + register_extra_span_field, samples, schema, serde, sink, source, source_sender, tcp, tls, + transform, }; pub use vector_lookup as lookup; pub use vector_stream as stream; diff --git a/lib/vector-lib/tests/register_extra_span_field.rs b/lib/vector-lib/tests/register_extra_span_field.rs new file mode 100644 index 0000000000000..71a853de03fa7 --- /dev/null +++ b/lib/vector-lib/tests/register_extra_span_field.rs @@ -0,0 +1,17 @@ +// No `use inventory` here β€” the whole point is that callers of the macro must +// not need a direct inventory dependency. +vector_lib::register_extra_span_field!("lib_integration_label"); + +/// Asserts that `register_extra_span_field!` registers the name as a `MetricLabel` +/// when called from a crate that has no direct `inventory` dependency. +/// +/// If the `MetricLabel` arm were removed from the macro, this test would fail +/// because `lib_integration_label` would be absent from `LABELS` and +/// `VectorLabelFilter` would drop it before the metrics registry ever sees it. +#[test] +fn macro_registers_metric_label_without_caller_importing_inventory() { + assert!( + vector_lib::metrics::LABELS.contains("lib_integration_label"), + "expected `lib_integration_label` to be in the MetricLabel allowlist", + ); +} diff --git a/lib/vector-stream/Cargo.toml b/lib/vector-stream/Cargo.toml index b16138436eb82..21b9f1a9a2d7b 100644 --- a/lib/vector-stream/Cargo.toml +++ b/lib/vector-stream/Cargo.toml @@ -19,7 +19,7 @@ vector-common = { path = "../vector-common" } vector-core = { path = "../vector-core" } [dev-dependencies] -proptest = "1.10" +proptest.workspace = true rand.workspace = true rand_distr.workspace = true tokio = { workspace = true, features = ["test-util"] } diff --git a/lib/vector-stream/src/concurrent_map.rs b/lib/vector-stream/src/concurrent_map.rs index 6b2920db1cc68..6dde7a31f7bbe 100644 --- a/lib/vector-stream/src/concurrent_map.rs +++ b/lib/vector-stream/src/concurrent_map.rs @@ -72,6 +72,11 @@ where Poll::Pending | Poll::Ready(None) => break, Poll::Ready(Some(item)) => { let fut = (this.f)(item); + // `ConcurrentMap` does not instrument the spawned future itself: the + // mapping closure runs on a detached task, so the current span at poll + // time is not necessarily meaningful for the work being performed. It is + // the caller's responsibility to propagate any span (e.g. the owning + // component's span for internal metric/log tagging) into `fut`. let handle = tokio::spawn(fut); this.in_flight.push_back(handle); } diff --git a/lib/vector-stream/src/partitioned_batcher.rs b/lib/vector-stream/src/partitioned_batcher.rs index 30cf039ec3840..0cd0413548595 100644 --- a/lib/vector-stream/src/partitioned_batcher.rs +++ b/lib/vector-stream/src/partitioned_batcher.rs @@ -211,10 +211,9 @@ where Prt::Key: Eq + Hash + Clone, Prt::Item: ByteSizeOf, C: BatchConfig, - F: Fn() -> C + Send, + F: Fn(&Prt::Key) -> C + Send, { - pub fn new(stream: St, partitioner: Prt, settings: F) -> Self { - let timeout = settings().timeout(); + pub fn new(stream: St, partitioner: Prt, timeout: Duration, settings: F) -> Self { Self { state: settings, batches: HashMap::default(), @@ -233,8 +232,8 @@ where Prt: Partitioner + Unpin, Prt::Key: Eq + Hash + Clone, Prt::Item: ByteSizeOf, - C: BatchConfig, - F: Fn() -> C + Send, + C: BatchConfig, + F: Fn(&Prt::Key) -> C + Send, { pub fn with_timer(stream: St, partitioner: Prt, timer: KT, settings: F) -> Self { Self { @@ -256,7 +255,7 @@ where Prt::Item: ByteSizeOf, KT: KeyedTimer, C: BatchConfig, - F: Fn() -> C + Send, + F: Fn(&Prt::Key) -> C + Send, { type Item = (Prt::Key, B); @@ -307,7 +306,7 @@ where let batch = if let Some(batch) = this.batches.get_mut(&item_key) { batch } else { - let batch = (this.state)(); + let batch = (this.state)(&item_key); this.batches.insert(item_key.clone(), batch); this.timer.insert(item_key.clone()); this.batches @@ -479,7 +478,7 @@ mod test { let batch_settings = BatcherSettings::new(Duration::from_secs(1), allocation_limit, item_limit); let batcher = PartitionedBatcher::with_timer(&mut stream, partitioner, timer, - Box::new(move || batch_settings.as_byte_size_config())); + move |_: &u8| batch_settings.as_byte_size_config::()); let batcher_size_hint = batcher.size_hint(); assert_eq!(stream_size_hint, batcher_size_hint); @@ -503,7 +502,7 @@ mod test { let allocation_limit = NonZeroUsize::new(allocation_limit as usize).unwrap(); let batch_settings = BatcherSettings::new(Duration::from_secs(1), allocation_limit, item_limit); let mut batcher = PartitionedBatcher::with_timer(&mut stream, partitioner, - timer, Box::new(move || batch_settings.as_byte_size_config())); + timer, move |_: &u8| batch_settings.as_byte_size_config::()); let mut batcher = Pin::new(&mut batcher); loop { @@ -574,7 +573,7 @@ mod test { let allocation_limit = NonZeroUsize::new(allocation_limit as usize).unwrap(); let batch_settings = BatcherSettings::new(Duration::from_secs(1), allocation_limit, item_limit); let mut batcher = PartitionedBatcher::with_timer(&mut stream, partitioner, - timer, Box::new(move || batch_settings.as_byte_size_config())); + timer, move |_: &u8| batch_settings.as_byte_size_config::()); let mut batcher = Pin::new(&mut batcher); loop { @@ -618,7 +617,7 @@ mod test { let allocation_limit = NonZeroUsize::new(allocation_limit as usize).unwrap(); let batch_settings = BatcherSettings::new(Duration::from_secs(1), allocation_limit, item_limit); let mut batcher = PartitionedBatcher::with_timer(&mut stream, partitioner, - timer, Box::new(move || batch_settings.as_byte_size_config())); + timer, move |_: &u8| batch_settings.as_byte_size_config::()); let mut batcher = Pin::new(&mut batcher); let mut observed_items = 0; diff --git a/lib/vector-tap/Cargo.toml b/lib/vector-tap/Cargo.toml index 976233f446915..127c2e1ea32bd 100644 --- a/lib/vector-tap/Cargo.toml +++ b/lib/vector-tap/Cargo.toml @@ -7,24 +7,43 @@ publish = false license = "MPL-2.0" [features] -api = ["dep:async-graphql"] +api = [ + "dep:bytes", + "dep:clap", + "dep:codecs", + "dep:colored", + "dep:prost", + "dep:serde_json", + "dep:serde_yaml", + "dep:tokio-stream", + "dep:tokio-util", + "dep:url", + "dep:vector-api-client", +] [dependencies] -async-graphql = { version = "7.0.17", default-features = false, features = ["playground"], optional = true } -colored.workspace = true +# Server-side (always compiled) futures.workspace = true glob.workspace = true -serde_yaml.workspace = true tokio = { workspace = true, features = ["time"] } -tokio-stream = { workspace = true, features = ["sync"] } -tokio-tungstenite.workspace = true tracing.workspace = true -url = { version = "2.5.4", default-features = false } uuid.workspace = true -vector-api-client = { path = "../vector-api-client" } +vector-buffers = { path = "../vector-buffers" } vector-common = { path = "../vector-common" } vector-core = { path = "../vector-core" } -vector-buffers = { path = "../vector-buffers" } + +# Client-side (gated behind the `api` feature) +bytes = { workspace = true, optional = true } +clap = { workspace = true, optional = true } +codecs = { path = "../codecs", optional = true } +colored = { workspace = true, optional = true } +prost = { workspace = true, optional = true } +serde_json = { workspace = true, optional = true } +serde_yaml = { workspace = true, optional = true } +tokio-stream = { workspace = true, features = ["sync"], optional = true } +tokio-util = { version = "0.7", default-features = false, features = ["codec"], optional = true } +url = { version = "2.5.4", default-features = false, optional = true } +vector-api-client = { path = "../vector-api-client", optional = true } [dev-dependencies] tokio = { workspace = true, features = ["test-util"] } diff --git a/lib/vector-tap/src/controller.rs b/lib/vector-tap/src/controller.rs index dba9a7ec7b885..f0b19ecde5a72 100644 --- a/lib/vector-tap/src/controller.rs +++ b/lib/vector-tap/src/controller.rs @@ -214,7 +214,7 @@ impl TapController { fn shutdown_trigger(control_tx: fanout::ControlChannel, sink_id: ComponentKey) -> ShutdownTx { let (shutdown_tx, shutdown_rx) = oneshot::channel(); - tokio::spawn(async move { + vector_common::spawn_in_current_span(async move { _ = shutdown_rx.await; if control_tx .send(fanout::ControlMessage::Remove(sink_id.clone())) @@ -306,6 +306,7 @@ async fn tap_handler( source_keys, sink_keys, removals, + .. } = watch_rx.borrow().clone(); // Remove tap sinks from components that have gone away/can no longer match. @@ -365,7 +366,7 @@ async fn tap_handler( ); let mut tap_transformer = TapTransformer::new(tx.clone(), output.clone()); - tokio::spawn(async move { + vector_common::spawn_in_current_span(async move { while let Some(events) = tap_buffer_rx.next().await { tap_transformer.try_send(events); } diff --git a/lib/vector-tap/src/lib.rs b/lib/vector-tap/src/lib.rs index 30ec0eca2aa3a..46291301a079d 100644 --- a/lib/vector-tap/src/lib.rs +++ b/lib/vector-tap/src/lib.rs @@ -7,255 +7,7 @@ pub mod controller; pub mod notification; pub mod topology; -use std::{borrow::Cow, collections::BTreeMap}; - -use colored::{ColoredString, Colorize}; -use tokio::{ - sync::mpsc as tokio_mpsc, - time::{Duration, Instant, timeout}, -}; -use tokio_stream::StreamExt; -use url::Url; -use vector_api_client::{ - connect_subscription_client, - gql::{ - TapEncodingFormat, TapSubscriptionExt, - output_events_by_component_id_patterns_subscription::OutputEventsByComponentIdPatternsSubscriptionOutputEventsByComponentIdPatterns as GraphQLTapOutputEvent, - }, -}; - -#[derive(Clone, Debug)] -pub struct EventFormatter { - meta: bool, - format: TapEncodingFormat, - component_id_label: ColoredString, - component_kind_label: ColoredString, - component_type_label: ColoredString, -} - -impl EventFormatter { - pub fn new(meta: bool, format: TapEncodingFormat) -> Self { - Self { - meta, - format, - component_id_label: "component_id".green(), - component_kind_label: "component_kind".green(), - component_type_label: "component_type".green(), - } - } - - pub fn format<'a>( - &self, - component_id: &str, - component_kind: &str, - component_type: &str, - event: &'a str, - ) -> Cow<'a, str> { - if self.meta { - match self.format { - TapEncodingFormat::Json => format!( - r#"{{"{}":"{}","{}":"{}","{}":"{}","event":{}}}"#, - self.component_id_label, - component_id.green(), - self.component_kind_label, - component_kind.green(), - self.component_type_label, - component_type.green(), - event - ) - .into(), - TapEncodingFormat::Yaml => { - let mut value: BTreeMap = BTreeMap::new(); - value.insert("event".to_string(), serde_yaml::from_str(event).unwrap()); - // We interpolate to include component_id rather than - // include it in the map to correctly preserve color - // formatting - format!( - "{}{}: {}\n{}: {}\n{}: {}\n", - serde_yaml::to_string(&value).unwrap(), - self.component_id_label, - component_id.green(), - self.component_kind_label, - component_kind.green(), - self.component_type_label, - component_type.green() - ) - .into() - } - TapEncodingFormat::Logfmt => format!( - "{}={} {}={} {}={} {}", - self.component_id_label, - component_id.green(), - self.component_kind_label, - component_kind.green(), - self.component_type_label, - component_type.green(), - event - ) - .into(), - } - } else { - event.into() - } - } -} - -#[derive(Clone, Debug)] -pub enum OutputChannel { - Stdout(EventFormatter), - AsyncChannel(tokio_mpsc::Sender>), -} - -/// Error type for DNS message parsing -#[derive(Debug)] -pub enum TapExecutorError { - ConnectionFailure(tokio_tungstenite::tungstenite::Error), - GraphQLError, -} - -#[derive(Debug)] -pub struct TapRunner<'a> { - url: &'a Url, - input_patterns: Vec, - output_patterns: Vec, - output_channel: &'a OutputChannel, - format: TapEncodingFormat, -} - -impl<'a> TapRunner<'a> { - pub fn new( - url: &'a Url, - input_patterns: Vec, - output_patterns: Vec, - output_channel: &'a OutputChannel, - format: TapEncodingFormat, - ) -> Self { - TapRunner { - url, - input_patterns, - output_patterns, - output_channel, - format, - } - } - - pub async fn run_tap( - &self, - interval: i64, - limit: i64, - duration_ms: Option, - quiet: bool, - ) -> Result<(), TapExecutorError> { - let subscription_client = connect_subscription_client((*self.url).clone()) - .await - .map_err(TapExecutorError::ConnectionFailure)?; - - tokio::pin! { - let stream = subscription_client.output_events_by_component_id_patterns_subscription( - self.output_patterns.clone(), - self.input_patterns.clone(), - self.format, - limit, - interval, - ); - } - - let start_time = Instant::now(); - let stream_duration = duration_ms - .map(Duration::from_millis) - .unwrap_or(Duration::MAX); - - // Loop over the returned results, printing out tap events. - loop { - let time_elapsed = start_time.elapsed(); - if time_elapsed >= stream_duration { - return Ok(()); - } - - let message = timeout(stream_duration - time_elapsed, stream.next()).await; - match message { - Ok(Some(Some(res))) => { - if let Some(d) = res.data { - let output_events: Vec = d - .output_events_by_component_id_patterns - .into_iter() - .filter(|event| { - !matches!( - (quiet, event), - (true, GraphQLTapOutputEvent::EventNotification(_)) - ) - }) - .collect(); - - match &self.output_channel { - OutputChannel::Stdout(formatter) => { - self.output_event_stdout(&output_events, formatter); - } - OutputChannel::AsyncChannel(sender_tx) => { - if let Err(error) = sender_tx.send(output_events).await { - error!("Could not send tap events: {error}"); - } - } - } - } - } - Err(_) => - // If the stream times out, that indicates the duration specified by the user - // has elapsed. We should exit gracefully. - { - return Ok(()); - } - Ok(_) => return Err(TapExecutorError::GraphQLError), - } - } - } - - #[allow(clippy::print_stdout)] - fn output_event_stdout( - &self, - output_events: &[GraphQLTapOutputEvent], - formatter: &EventFormatter, - ) { - for tap_event in output_events.iter() { - match tap_event { - GraphQLTapOutputEvent::Log(ev) => { - println!( - "{}", - formatter.format( - ev.component_id.as_ref(), - ev.component_kind.as_ref(), - ev.component_type.as_ref(), - ev.string.as_ref() - ) - ); - } - GraphQLTapOutputEvent::Metric(ev) => { - println!( - "{}", - formatter.format( - ev.component_id.as_ref(), - ev.component_kind.as_ref(), - ev.component_type.as_ref(), - ev.string.as_ref() - ) - ); - } - GraphQLTapOutputEvent::Trace(ev) => { - println!( - "{}", - formatter.format( - ev.component_id.as_ref(), - ev.component_kind.as_ref(), - ev.component_type.as_ref(), - ev.string.as_ref() - ) - ); - } - #[allow(clippy::print_stderr)] - GraphQLTapOutputEvent::EventNotification(ev) => { - eprintln!("{}", ev.message); - } - } - } - } -} +#[cfg(feature = "api")] +mod runner; +#[cfg(feature = "api")] +pub use runner::{EventFormatter, OutputChannel, TapEncodingFormat, TapExecutorError, TapRunner}; diff --git a/lib/vector-tap/src/notification.rs b/lib/vector-tap/src/notification.rs index 04ba715718993..785e780147ebc 100644 --- a/lib/vector-tap/src/notification.rs +++ b/lib/vector-tap/src/notification.rs @@ -1,11 +1,6 @@ -#[cfg(feature = "api")] -use async_graphql::{SimpleObject, Union}; - #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(feature = "api", derive(SimpleObject))] /// A component was found that matched the provided pattern pub struct Matched { - #[cfg_attr(feature = "api", graphql(skip))] message: String, /// Pattern that raised the notification pub pattern: String, @@ -21,10 +16,8 @@ impl Matched { } #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(feature = "api", derive(SimpleObject))] /// There isn't currently a component that matches this pattern pub struct NotMatched { - #[cfg_attr(feature = "api", graphql(skip))] message: String, /// Pattern that raised the notification pub pattern: String, @@ -42,11 +35,9 @@ impl NotMatched { } #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(feature = "api", derive(SimpleObject))] /// The pattern matched source(s) which cannot be tapped for inputs or sink(s) /// which cannot be tapped for outputs pub struct InvalidMatch { - #[cfg_attr(feature = "api", graphql(skip))] message: String, /// Pattern that raised the notification pattern: String, @@ -65,7 +56,6 @@ impl InvalidMatch { } #[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(feature = "api", derive(Union))] /// A specific kind of notification with additional details pub enum Notification { Matched(Matched), diff --git a/lib/vector-tap/src/runner.rs b/lib/vector-tap/src/runner.rs new file mode 100644 index 0000000000000..8c6ab815a8bf8 --- /dev/null +++ b/lib/vector-tap/src/runner.rs @@ -0,0 +1,344 @@ +use std::{borrow::Cow, collections::BTreeMap}; + +use bytes::Bytes; +use colored::{ColoredString, Colorize}; +use prost::Message; +use tokio::{ + sync::mpsc as tokio_mpsc, + time::{Duration, Instant, timeout}, +}; +use tokio_stream::StreamExt; +use tokio_util::codec::Encoder; +use url::Url; +use vector_api_client::{ + Client, + proto::{StreamOutputEventsRequest, StreamOutputEventsResponse}, +}; +use vector_core::event::Event; + +#[derive(Clone, Copy, Debug, clap::ValueEnum)] +pub enum TapEncodingFormat { + Json, + Yaml, + Logfmt, +} + +// Note: TapEncodingFormat is kept for CLI compatibility but not used in the gRPC API +// The server now sends proto events directly, which clients serialize as needed + +#[derive(Clone, Debug)] +pub struct EventFormatter { + meta: bool, + format: TapEncodingFormat, + component_id_label: ColoredString, + component_kind_label: ColoredString, + component_type_label: ColoredString, +} + +impl EventFormatter { + pub fn new(meta: bool, format: TapEncodingFormat) -> Self { + Self { + meta, + format, + component_id_label: "component_id".green(), + component_kind_label: "component_kind".green(), + component_type_label: "component_type".green(), + } + } + + pub fn format<'a>( + &self, + component_id: &str, + component_kind: &str, + component_type: &str, + event: &'a str, + ) -> Cow<'a, str> { + if self.meta { + match self.format { + TapEncodingFormat::Json => format!( + r#"{{"{}":"{}","{}":"{}","{}":"{}","event":{}}}"#, + self.component_id_label, + component_id.green(), + self.component_kind_label, + component_kind.green(), + self.component_type_label, + component_type.green(), + event + ) + .into(), + TapEncodingFormat::Yaml => { + let mut value: BTreeMap = BTreeMap::new(); + value.insert("event".to_string(), serde_yaml::from_str(event).unwrap()); + // We interpolate to include component_id rather than + // include it in the map to correctly preserve color + // formatting + format!( + "{}{}: {}\n{}: {}\n{}: {}\n", + serde_yaml::to_string(&value).unwrap(), + self.component_id_label, + component_id.green(), + self.component_kind_label, + component_kind.green(), + self.component_type_label, + component_type.green() + ) + .into() + } + TapEncodingFormat::Logfmt => format!( + "{}={} {}={} {}={} {}", + self.component_id_label, + component_id.green(), + self.component_kind_label, + component_kind.green(), + self.component_type_label, + component_type.green(), + event + ) + .into(), + } + } else { + event.into() + } + } +} + +#[derive(Clone, Debug)] +pub enum OutputChannel { + Stdout(EventFormatter), + AsyncChannel(tokio_mpsc::Sender>), +} + +/// Error type for tap execution +#[derive(Debug)] +pub enum TapExecutorError { + ConnectionFailure(String), + GrpcError(String), + /// Permanent error that should not trigger a reconnect (e.g. invalid arguments). + Fatal(String), +} + +impl TapExecutorError { + pub fn is_fatal(&self) -> bool { + matches!(self, TapExecutorError::Fatal(_)) + } +} + +impl From for TapExecutorError { + fn from(err: vector_api_client::Error) -> Self { + if err.is_fatal() { + TapExecutorError::Fatal(format!("{}", err)) + } else { + TapExecutorError::GrpcError(format!("{}", err)) + } + } +} + +#[derive(Debug)] +pub struct TapRunner<'a> { + url: &'a Url, + input_patterns: Vec, + output_patterns: Vec, + output_channel: &'a OutputChannel, +} + +impl<'a> TapRunner<'a> { + pub fn new( + url: &'a Url, + input_patterns: Vec, + output_patterns: Vec, + output_channel: &'a OutputChannel, + ) -> Self { + TapRunner { + url, + input_patterns, + output_patterns, + output_channel, + } + } + + pub async fn run_tap( + &self, + interval: i64, + limit: i64, + duration_ms: Option, + quiet: bool, + ) -> Result<(), TapExecutorError> { + let uri = self + .url + .as_str() + .parse() + .map_err(|e| TapExecutorError::Fatal(format!("Invalid URL: {e}")))?; + let mut client = Client::new(uri); + client.connect().await?; + self.run_tap_with_client(client, interval, limit, duration_ms, quiet) + .await + } + + /// Run tap using a pre-connected client (avoids an extra connection round-trip when the + /// caller has already connected and health-checked the client). + pub async fn run_tap_with_client( + &self, + mut client: Client, + interval: i64, + limit: i64, + duration_ms: Option, + quiet: bool, + ) -> Result<(), TapExecutorError> { + let request = StreamOutputEventsRequest { + outputs_patterns: self.output_patterns.clone(), + inputs_patterns: self.input_patterns.clone(), + limit: limit as i32, + interval_ms: interval as i32, + }; + + let mut stream = client.stream_output_events(request).await?; + + let start_time = Instant::now(); + let stream_duration = duration_ms + .map(Duration::from_millis) + .unwrap_or(Duration::MAX); + + // Loop over the returned results, processing tap events + loop { + let time_elapsed = start_time.elapsed(); + if time_elapsed >= stream_duration { + return Ok(()); + } + + let message = timeout(stream_duration - time_elapsed, stream.next()).await; + match message { + Ok(Some(Ok(output_event))) => { + // Filter out notifications if quiet mode is enabled + if quiet + && matches!( + output_event.event, + Some( + vector_api_client::proto::stream_output_events_response::Event::Notification( + _ + ) + ) + ) + { + continue; + } + + match &self.output_channel { + OutputChannel::Stdout(formatter) => { + self.output_event_stdout(&output_event, formatter); + } + OutputChannel::AsyncChannel(sender_tx) => { + if let Err(error) = sender_tx.send(vec![output_event]).await { + error!("Could not send tap events: {error}"); + } + } + } + } + Err(_) => + // If the stream times out, that indicates the duration specified by the user + // has elapsed. We should exit gracefully. + { + return Ok(()); + } + Ok(None) => { + return Err(TapExecutorError::GrpcError( + "Stream ended unexpectedly".to_string(), + )); + } + Ok(Some(Err(err))) => return Err(TapExecutorError::from(err)), + } + } + } + + /// Convert and serialize a protobuf EventWrapper to the requested format + fn serialize_event( + event_wrapper: &vector_api_client::proto::event::EventWrapper, + format: TapEncodingFormat, + ) -> Result { + // INTENTIONAL round-trip through protobuf bytes: `vector_api_client` compiles + // `event.proto` independently to avoid taking a dependency on `vector_core` (a large + // crate with many features). Both types share the same proto schema, so encoding one + // and decoding the other is always safe. If the schemas ever diverge this will surface + // as a runtime decode error. + let bytes = event_wrapper.encode_to_vec(); + + let core_event_wrapper = + vector_core::event::proto::EventWrapper::decode(Bytes::from(bytes)) + .map_err(|e| format!("Failed to decode event: {}", e))?; + + // Convert to vector-core Event (which has Serialize) + let event: Event = core_event_wrapper.into(); + + // Serialize based on format + match format { + TapEncodingFormat::Json => serde_json::to_string(&event) + .map_err(|e| format!("JSON serialization failed: {}", e)), + TapEncodingFormat::Yaml => serde_yaml::to_string(&event) + .map_err(|e| format!("YAML serialization failed: {}", e)), + TapEncodingFormat::Logfmt => { + // For logfmt, we need to extract the log event and serialize it + match event { + Event::Log(log_event) => { + let mut serializer = + codecs::encoding::format::LogfmtSerializerConfig.build(); + let mut bytes = bytes::BytesMut::new(); + // Wrap the LogEvent back into Event for the serializer + serializer + .encode(Event::Log(log_event), &mut bytes) + .map_err(|e| format!("Logfmt serialization failed: {}", e))?; + String::from_utf8(bytes.to_vec()) + .map_err(|e| format!("UTF-8 conversion failed: {}", e)) + } + Event::Metric(_) => { + Err("logfmt format is only supported for log events".to_string()) + } + Event::Trace(_) => { + Err("logfmt format is only supported for log events".to_string()) + } + } + } + } + } + + #[allow(clippy::print_stdout)] + fn output_event_stdout( + &self, + output_event: &StreamOutputEventsResponse, + formatter: &EventFormatter, + ) { + use vector_api_client::proto::stream_output_events_response::Event as OutputEventType; + + match &output_event.event { + Some(OutputEventType::TappedEvent(ev)) => { + // Format the proto event for display + let encoded_string = if let Some(ref event_wrapper) = ev.event { + match Self::serialize_event(event_wrapper, formatter.format) { + Ok(s) => s, + Err(e) => { + error!(message = "Failed to serialize event.", error = %e); + format!("{:?}", event_wrapper) + } + } + } else { + "No event data".to_string() + }; + + println!( + "{}", + formatter.format( + &ev.component_id, + &ev.component_kind, + &ev.component_type, + &encoded_string + ) + ); + } + #[allow(clippy::print_stderr)] + Some(OutputEventType::Notification(ev)) => { + eprintln!("{}", ev.message); + } + None => { + error!("Received StreamOutputEventsResponse with no event data"); + } + } + } +} diff --git a/lib/vector-tap/src/topology.rs b/lib/vector-tap/src/topology.rs index 7b281f7c47151..eb13e93e2a9fe 100644 --- a/lib/vector-tap/src/topology.rs +++ b/lib/vector-tap/src/topology.rs @@ -26,6 +26,24 @@ pub struct TapResource { pub sink_keys: Vec, // Components removed on a reload (used to drop TapSinks) pub removals: HashSet, + // Plugin type name for every component (e.g. "demo_logs", "remap", "console") + pub type_names: HashMap, +} + +impl TapResource { + /// Returns each component's output port names, derived from the full `outputs` snapshot. + /// + /// Keys are all components that have at least one output (sources and transforms). + /// The port value is `None` for the default output and `Some(name)` for named ports. + pub fn output_ports_by_component(&self) -> HashMap<&ComponentKey, Vec>> { + let mut map: HashMap<&ComponentKey, Vec>> = HashMap::new(); + for tap_output in self.outputs.keys() { + map.entry(&tap_output.output_id.component) + .or_default() + .push(tap_output.output_id.port.as_deref()); + } + map + } } // Watcher types for topology changes. diff --git a/lib/vector-top/Cargo.toml b/lib/vector-top/Cargo.toml index c0da764b4de51..869fec71484f1 100644 --- a/lib/vector-top/Cargo.toml +++ b/lib/vector-top/Cargo.toml @@ -22,6 +22,7 @@ unit-prefix = { version = "0.5.2", default-features = false, features = ["std"] num-format = { version = "0.4.4", default-features = false, features = ["with-num-bigint"] } ratatui = { version = "0.30.0", default-features = false, features = ["crossterm", "layout-cache"] } regex.workspace = true +http = { version = "0.2", default-features = false } vector-common = { path = "../vector-common" } vector-api-client = { path = "../vector-api-client" } diff --git a/lib/vector-top/src/dashboard.rs b/lib/vector-top/src/dashboard.rs index c4078ebde6afb..9e6a1907dff36 100644 --- a/lib/vector-top/src/dashboard.rs +++ b/lib/vector-top/src/dashboard.rs @@ -170,9 +170,9 @@ static HEADER: [&str; NUM_COLUMNS] = [ columns::BYTES_IN, columns::EVENTS_OUT, columns::BYTES_OUT, - columns::ERRORS, #[cfg(feature = "allocation-tracing")] columns::MEMORY_USED, + columns::ERRORS, ]; struct Widgets<'a> { @@ -348,13 +348,17 @@ impl<'a> Widgets<'a> { r.sent_bytes_throughput_sec, self.human_metrics, ), + #[cfg(feature = "allocation-tracing")] + if state.allocation_tracing_active { + r.allocated_bytes.human_format_bytes() + } else { + "disabled".to_string() + }, if self.human_metrics { r.errors.human_format() } else { r.errors.thousands_format() }, - #[cfg(feature = "allocation-tracing")] - r.allocated_bytes.human_format_bytes(), ]; data.extend_from_slice(&formatted_metrics); @@ -373,7 +377,7 @@ impl<'a> Widgets<'a> { .map(Cell::from) .collect::>(); data[1] = Cell::from(id.as_str()); - data[5] = Cell::from(sent_events_metric); + data[6] = Cell::from(sent_events_metric); items.push(Row::new(data).style(Style::default())); } } @@ -383,26 +387,26 @@ impl<'a> Widgets<'a> { &[ Constraint::Percentage(13), // ID Constraint::Percentage(8), // Output - Constraint::Percentage(4), // Kind + Constraint::Percentage(5), // Kind Constraint::Percentage(9), // Type Constraint::Percentage(10), // Events In Constraint::Percentage(12), // Bytes In Constraint::Percentage(10), // Events Out Constraint::Percentage(12), // Bytes Out - Constraint::Percentage(8), // Errors - Constraint::Percentage(14), // Allocated Bytes + Constraint::Percentage(14), // Memory Used + Constraint::Percentage(7), // Errors ] } else { &[ Constraint::Percentage(13), // ID Constraint::Percentage(12), // Output - Constraint::Percentage(9), // Kind + Constraint::Percentage(10), // Kind Constraint::Percentage(6), // Type Constraint::Percentage(12), // Events In Constraint::Percentage(14), // Bytes In Constraint::Percentage(12), // Events Out Constraint::Percentage(14), // Bytes Out - Constraint::Percentage(8), // Errors + Constraint::Percentage(7), // Errors ] }; let w = Table::new(items, widths) diff --git a/lib/vector-top/src/metrics.rs b/lib/vector-top/src/metrics.rs index 89012f80030ff..75aae4456fce6 100644 --- a/lib/vector-top/src/metrics.rs +++ b/lib/vector-top/src/metrics.rs @@ -1,14 +1,16 @@ use std::{ - collections::{BTreeMap, HashMap}, + collections::{BTreeMap, HashSet}, sync::Arc, + time::Duration, }; use glob::Pattern; +use http::Uri; use tokio::task::JoinHandle; use tokio_stream::StreamExt; use vector_api_client::{ - Client, SubscriptionClient, - gql::{ComponentsQueryExt, ComponentsSubscriptionExt, MetricsSubscriptionExt}, + Client, + proto::{Component, ComponentType, MetricName, stream_component_metrics_response::Value}, }; use crate::state::{self, OutputMetrics, SentEventsMetric}; @@ -24,528 +26,565 @@ fn component_matches_patterns(component_id: &str, components_patterns: &[Pattern .any(|pattern| pattern.matches(component_id)) } -/// Components that have been added -async fn component_added( - client: Arc, +/// Component polling task +/// +/// Polls for component changes every interval and diffs to detect adds/removes. +async fn poll_components( + mut client: Client, tx: state::EventTx, + interval_ms: i64, components_patterns: Arc>, + initial_components: HashSet, ) { - tokio::pin! { - let stream = client.component_added(); - }; + let mut known_components = initial_components; + let poll_interval = Duration::from_millis(interval_ms as u64); + let mut consecutive_errors = 0; + const MAX_CONSECUTIVE_ERRORS: u32 = 3; + + loop { + tokio::time::sleep(poll_interval).await; + + // Fetch current components + let Ok(response) = client.get_components(0).await else { + consecutive_errors += 1; + if consecutive_errors >= MAX_CONSECUTIVE_ERRORS { + // Exit to trigger reconnection after sustained failures + return; + } + continue; + }; + consecutive_errors = 0; + + let current_components: std::collections::HashSet = response + .components + .iter() + .map(|c| c.component_id.clone()) + .collect(); + + // Detect added components + for component in &response.components { + let component_id = &component.component_id; + if !known_components.contains(component_id) + && component_matches_patterns(component_id, &components_patterns) + { + let row = component_to_row(component); + _ = tx.send(state::EventType::ComponentAdded(row)).await; + } + } - while let Some(Some(res)) = stream.next().await { - if let Some(d) = res.data { - let c = d.component_added; - let component_id = c.component_id; - if !component_matches_patterns(&component_id, &components_patterns) { - continue; + // Detect removed components + for old_id in &known_components { + if !current_components.contains(old_id) + && component_matches_patterns(old_id, &components_patterns) + { + let key = ComponentKey::from(old_id.as_str()); + _ = tx.send(state::EventType::ComponentRemoved(key)).await; } - let key = ComponentKey::from(component_id); - _ = tx - .send(state::EventType::ComponentAdded(state::ComponentRow { - key, - kind: c.on.to_string(), - component_type: c.component_type, - outputs: HashMap::new(), - received_bytes_total: 0, - received_bytes_throughput_sec: 0, - received_events_total: 0, - received_events_throughput_sec: 0, - sent_bytes_total: 0, - sent_bytes_throughput_sec: 0, - sent_events_total: 0, - sent_events_throughput_sec: 0, - #[cfg(feature = "allocation-tracing")] - allocated_bytes: 0, - errors: 0, - })) - .await; } + + known_components = current_components; + } +} + +fn component_to_row(component: &Component) -> state::ComponentRow { + let key = ComponentKey::from(component.component_id.as_str()); + let metrics = component.metrics.as_ref(); + + state::ComponentRow { + key: key.clone(), + kind: match component.component_type() { + ComponentType::Unspecified => "unknown", + ComponentType::Source => "source", + ComponentType::Transform => "transform", + ComponentType::Sink => "sink", + } + .to_string(), + component_type: component.on_type.clone(), // actual plugin type e.g. "demo_logs", "kafka" + outputs: component + .outputs + .iter() + .map(|o| { + ( + o.output_id.clone(), + OutputMetrics::from(o.sent_events_total), + ) + }) + .collect(), + received_bytes_total: metrics.and_then(|m| m.received_bytes_total).unwrap_or(0), + received_bytes_throughput_sec: 0, + received_events_total: metrics.and_then(|m| m.received_events_total).unwrap_or(0), + received_events_throughput_sec: 0, + sent_bytes_total: metrics.and_then(|m| m.sent_bytes_total).unwrap_or(0), + sent_bytes_throughput_sec: 0, + sent_events_total: metrics.and_then(|m| m.sent_events_total).unwrap_or(0), + sent_events_throughput_sec: 0, + #[cfg(feature = "allocation-tracing")] + allocated_bytes: 0, + errors: 0, } } /// Allocated bytes per component #[cfg(feature = "allocation-tracing")] async fn allocated_bytes( - client: Arc, + mut client: Client, tx: state::EventTx, interval: i64, components_patterns: Arc>, ) { - tokio::pin! { - let stream = client.component_allocated_bytes_subscription(interval); + let Ok(mut stream) = client + .stream_component_allocated_bytes(interval as i32) + .await + else { + // Failed to establish stream, will retry on reconnection + return; }; - while let Some(Some(res)) = stream.next().await { - if let Some(d) = res.data { - let c = d.component_allocated_bytes; - _ = tx - .send(state::EventType::AllocatedBytes( - c.into_iter() - .filter(|c| { - component_matches_patterns(&c.component_id, &components_patterns) - }) - .map(|c| { - ( - ComponentKey::from(c.component_id), - c.metric.allocated_bytes as i64, - ) - }) - .collect(), - )) - .await; + while let Some(Ok(response)) = stream.next().await { + let component_id = &response.component_id; + if !component_matches_patterns(component_id, &components_patterns) { + continue; } - } -} -/// Components that have been removed -async fn component_removed( - client: Arc, - tx: state::EventTx, - components_patterns: Arc>, -) { - tokio::pin! { - let stream = client.component_removed(); - }; - while let Some(Some(res)) = stream.next().await { - if let Some(d) = res.data { - let c = d.component_removed; - let component_id = c.component_id; - if !component_matches_patterns(&component_id, &components_patterns) { - continue; - } - let id = ComponentKey::from(component_id); - _ = tx.send(state::EventType::ComponentRemoved(id)).await; - } + _ = tx + .send(state::EventType::AllocatedBytes(vec![( + ComponentKey::from(component_id.as_str()), + response.allocated_bytes, + )])) + .await; } } async fn received_bytes_totals( - client: Arc, + mut client: Client, tx: state::EventTx, interval: i64, components_patterns: Arc>, ) { - tokio::pin! { - let stream = client.component_received_bytes_totals_subscription(interval); + let Ok(mut stream) = client + .stream_component_metrics(MetricName::ReceivedBytesTotal, interval as i32) + .await + else { + return; }; - while let Some(Some(res)) = stream.next().await { - if let Some(d) = res.data { - let c = d.component_received_bytes_totals; - + while let Some(Ok(response)) = stream.next().await { + let component_id = &response.component_id; + if !component_matches_patterns(component_id, &components_patterns) { + continue; + } + if let Some(Value::Total(total)) = response.value { _ = tx - .send(state::EventType::ReceivedBytesTotals( - c.into_iter() - .filter(|c| { - component_matches_patterns(&c.component_id, &components_patterns) - }) - .map(|c| { - ( - ComponentKey::from(c.component_id), - c.metric.received_bytes_total as i64, - ) - }) - .collect(), - )) + .send(state::EventType::ReceivedBytesTotals(vec![( + ComponentKey::from(component_id.as_str()), + total.value, + )])) .await; } } } async fn received_bytes_throughputs( - client: Arc, + mut client: Client, tx: state::EventTx, interval: i64, components_patterns: Arc>, ) { - tokio::pin! { - let stream = client.component_received_bytes_throughputs_subscription(interval); + let Ok(mut stream) = client + .stream_component_metrics(MetricName::ReceivedBytesThroughput, interval as i32) + .await + else { + return; }; - while let Some(Some(res)) = stream.next().await { - if let Some(d) = res.data { - let c = d.component_received_bytes_throughputs; + while let Some(Ok(response)) = stream.next().await { + let component_id = &response.component_id; + if !component_matches_patterns(component_id, &components_patterns) { + continue; + } + if let Some(Value::Throughput(tp)) = response.value { _ = tx - .send(state::EventType::ReceivedBytesThroughputs( - interval, - c.into_iter() - .filter(|c| { - component_matches_patterns(&c.component_id, &components_patterns) - }) - .map(|c| (ComponentKey::from(c.component_id), c.throughput)) - .collect(), - )) + .send(state::EventType::ReceivedBytesThroughputs(vec![( + ComponentKey::from(component_id.as_str()), + tp.value as i64, + )])) .await; } } } async fn received_events_totals( - client: Arc, + mut client: Client, tx: state::EventTx, interval: i64, components_patterns: Arc>, ) { - tokio::pin! { - let stream = client.component_received_events_totals_subscription(interval); + let Ok(mut stream) = client + .stream_component_metrics(MetricName::ReceivedEventsTotal, interval as i32) + .await + else { + return; }; - while let Some(Some(res)) = stream.next().await { - if let Some(d) = res.data { - let c = d.component_received_events_totals; + while let Some(Ok(response)) = stream.next().await { + let component_id = &response.component_id; + if !component_matches_patterns(component_id, &components_patterns) { + continue; + } + if let Some(Value::Total(total)) = response.value { _ = tx - .send(state::EventType::ReceivedEventsTotals( - c.into_iter() - .filter(|c| { - component_matches_patterns(&c.component_id, &components_patterns) - }) - .map(|c| { - ( - ComponentKey::from(c.component_id), - c.metric.received_events_total as i64, - ) - }) - .collect(), - )) + .send(state::EventType::ReceivedEventsTotals(vec![( + ComponentKey::from(component_id.as_str()), + total.value, + )])) .await; } } } async fn received_events_throughputs( - client: Arc, + mut client: Client, tx: state::EventTx, interval: i64, components_patterns: Arc>, ) { - tokio::pin! { - let stream = client.component_received_events_throughputs_subscription(interval); + let Ok(mut stream) = client + .stream_component_metrics(MetricName::ReceivedEventsThroughput, interval as i32) + .await + else { + return; }; - while let Some(Some(res)) = stream.next().await { - if let Some(d) = res.data { - let c = d.component_received_events_throughputs; + while let Some(Ok(response)) = stream.next().await { + let component_id = &response.component_id; + if !component_matches_patterns(component_id, &components_patterns) { + continue; + } + if let Some(Value::Throughput(tp)) = response.value { _ = tx - .send(state::EventType::ReceivedEventsThroughputs( - interval, - c.into_iter() - .filter(|c| { - component_matches_patterns(&c.component_id, &components_patterns) - }) - .map(|c| (ComponentKey::from(c.component_id), c.throughput)) - .collect(), - )) + .send(state::EventType::ReceivedEventsThroughputs(vec![( + ComponentKey::from(component_id.as_str()), + tp.value as i64, + )])) .await; } } } async fn sent_bytes_totals( - client: Arc, + mut client: Client, tx: state::EventTx, interval: i64, components_patterns: Arc>, ) { - tokio::pin! { - let stream = client.component_sent_bytes_totals_subscription(interval); + let Ok(mut stream) = client + .stream_component_metrics(MetricName::SentBytesTotal, interval as i32) + .await + else { + return; }; - while let Some(Some(res)) = stream.next().await { - if let Some(d) = res.data { - let c = d.component_sent_bytes_totals; + while let Some(Ok(response)) = stream.next().await { + let component_id = &response.component_id; + if !component_matches_patterns(component_id, &components_patterns) { + continue; + } + if let Some(Value::Total(total)) = response.value { _ = tx - .send(state::EventType::SentBytesTotals( - c.into_iter() - .filter(|c| { - component_matches_patterns(&c.component_id, &components_patterns) - }) - .map(|c| { - ( - ComponentKey::from(c.component_id), - c.metric.sent_bytes_total as i64, - ) - }) - .collect(), - )) + .send(state::EventType::SentBytesTotals(vec![( + ComponentKey::from(component_id.as_str()), + total.value, + )])) .await; } } } async fn sent_bytes_throughputs( - client: Arc, + mut client: Client, tx: state::EventTx, interval: i64, components_patterns: Arc>, ) { - tokio::pin! { - let stream = client.component_sent_bytes_throughputs_subscription(interval); + let Ok(mut stream) = client + .stream_component_metrics(MetricName::SentBytesThroughput, interval as i32) + .await + else { + return; }; - while let Some(Some(res)) = stream.next().await { - if let Some(d) = res.data { - let c = d.component_sent_bytes_throughputs; + while let Some(Ok(response)) = stream.next().await { + let component_id = &response.component_id; + if !component_matches_patterns(component_id, &components_patterns) { + continue; + } + if let Some(Value::Throughput(tp)) = response.value { _ = tx - .send(state::EventType::SentBytesThroughputs( - interval, - c.into_iter() - .filter(|c| { - component_matches_patterns(&c.component_id, &components_patterns) - }) - .map(|c| (ComponentKey::from(c.component_id), c.throughput)) - .collect(), - )) + .send(state::EventType::SentBytesThroughputs(vec![( + ComponentKey::from(component_id.as_str()), + tp.value as i64, + )])) .await; } } } async fn sent_events_totals( - client: Arc, + mut client: Client, tx: state::EventTx, interval: i64, components_patterns: Arc>, ) { - tokio::pin! { - let stream = client.component_sent_events_totals_subscription(interval); + let Ok(mut stream) = client + .stream_component_metrics(MetricName::SentEventsTotal, interval as i32) + .await + else { + return; }; - while let Some(Some(res)) = stream.next().await { - if let Some(d) = res.data { - let c = d.component_sent_events_totals; + while let Some(Ok(response)) = stream.next().await { + let component_id = &response.component_id; + if !component_matches_patterns(component_id, &components_patterns) { + continue; + } + if let Some(Value::Total(total)) = response.value { _ = tx - .send(state::EventType::SentEventsTotals( - c.into_iter() - .filter(|c| { - component_matches_patterns(&c.component_id, &components_patterns) - }) - .map(|c| SentEventsMetric { - key: ComponentKey::from(c.component_id.as_str()), - total: c.metric.sent_events_total as i64, - outputs: c.outputs().into_iter().collect(), - }) - .collect(), - )) + .send(state::EventType::SentEventsTotals(vec![SentEventsMetric { + key: ComponentKey::from(component_id.as_str()), + total: total.value, + outputs: total.output_totals.into_iter().collect(), + }])) .await; } } } async fn sent_events_throughputs( - client: Arc, + mut client: Client, tx: state::EventTx, interval: i64, components_patterns: Arc>, ) { - tokio::pin! { - let stream = client.component_sent_events_throughputs_subscription(interval); + let Ok(mut stream) = client + .stream_component_metrics(MetricName::SentEventsThroughput, interval as i32) + .await + else { + return; }; - while let Some(Some(res)) = stream.next().await { - if let Some(d) = res.data { - let c = d.component_sent_events_throughputs; + while let Some(Ok(response)) = stream.next().await { + let component_id = &response.component_id; + if !component_matches_patterns(component_id, &components_patterns) { + continue; + } + if let Some(Value::Throughput(tp)) = response.value { _ = tx - .send(state::EventType::SentEventsThroughputs( - interval, - c.into_iter() - .filter(|c| { - component_matches_patterns(&c.component_id, &components_patterns) - }) - .map(|c| SentEventsMetric { - key: ComponentKey::from(c.component_id.as_str()), - total: c.throughput, - outputs: c.outputs().into_iter().collect(), - }) - .collect(), - )) + .send(state::EventType::SentEventsThroughputs(vec![ + SentEventsMetric { + key: ComponentKey::from(component_id.as_str()), + total: tp.value as i64, + outputs: tp + .output_throughputs + .into_iter() + .map(|(k, v)| (k, v as i64)) + .collect(), + }, + ])) .await; } } } async fn errors_totals( - client: Arc, + mut client: Client, tx: state::EventTx, interval: i64, components_patterns: Arc>, ) { - tokio::pin! { - let stream = client.component_errors_totals_subscription(interval); + let Ok(mut stream) = client + .stream_component_metrics(MetricName::ErrorsTotal, interval as i32) + .await + else { + return; }; - while let Some(Some(res)) = stream.next().await { - if let Some(d) = res.data { - let c = d.component_errors_totals; + while let Some(Ok(response)) = stream.next().await { + let component_id = &response.component_id; + if !component_matches_patterns(component_id, &components_patterns) { + continue; + } + if let Some(Value::Total(total)) = response.value { _ = tx - .send(state::EventType::ErrorsTotals( - c.into_iter() - .filter(|c| { - component_matches_patterns(&c.component_id, &components_patterns) - }) - .map(|c| { - ( - ComponentKey::from(c.component_id.as_str()), - c.metric.errors_total as i64, - ) - }) - .collect(), - )) + .send(state::EventType::ErrorsTotals(vec![( + ComponentKey::from(component_id.as_str()), + total.value, + )])) .await; } } } -async fn uptime_changed(client: Arc, tx: state::EventTx) { - tokio::pin! { - let stream = client.uptime_subscription(); +async fn uptime_changed(mut client: Client, tx: state::EventTx, interval: i64) { + let Ok(mut stream) = client.stream_uptime(interval as i32).await else { + return; }; - while let Some(Some(res)) = stream.next().await { - if let Some(d) = res.data { - _ = tx - .send(state::EventType::UptimeChanged(d.uptime.seconds)) - .await; - } + while let Some(Ok(response)) = stream.next().await { + _ = tx + .send(state::EventType::UptimeChanged( + response.uptime_seconds as f64, + )) + .await; } } -/// Subscribe to each metrics channel through a separate client. This is a temporary workaround -/// until client multiplexing is fixed. In future, we should be able to use a single client -pub fn subscribe( - client: SubscriptionClient, +/// Handles returned by [`subscribe`], split by lifecycle. +pub struct SubscribeHandles { + /// Metric stream tasks β€” exit when a gRPC stream closes or errors. + /// `cmd.rs` joins these to detect connection loss and trigger reconnection. + pub metric_handles: Vec>, + /// Polls `get_components` for topology changes. Designed to run indefinitely + /// while the server is healthy, so it must be aborted separately rather than + /// joined alongside the metric handles. + pub poll_handle: JoinHandle<()>, +} + +/// Subscribe to each metrics stream, all sharing a single underlying gRPC connection. +/// HTTP/2 multiplexes the concurrent streams β€” cloning a connected `Client` is cheap +/// (the tonic `Channel` is Arc-backed) and avoids redundant TCP/HTTP2 handshakes. +pub async fn subscribe( + uri: Uri, tx: state::EventTx, interval: i64, components_patterns: Vec, -) -> Vec> { - let client = Arc::new(client); + initial_components: HashSet, +) -> Result { let components_patterns = Arc::new(components_patterns); - vec![ - tokio::spawn(component_added( - Arc::clone(&client), - tx.clone(), - Arc::clone(&components_patterns), - )), - tokio::spawn(component_removed( - Arc::clone(&client), - tx.clone(), - Arc::clone(&components_patterns), - )), + + let mut client = Client::new(uri); + client.connect().await?; + + let poll_handle = tokio::spawn(poll_components( + client.clone(), + tx.clone(), + interval, + Arc::clone(&components_patterns), + initial_components, + )); + + #[cfg_attr(not(feature = "allocation-tracing"), allow(unused_mut))] + let mut metric_handles = vec![ tokio::spawn(received_bytes_totals( - Arc::clone(&client), + client.clone(), tx.clone(), interval, Arc::clone(&components_patterns), )), tokio::spawn(received_bytes_throughputs( - Arc::clone(&client), + client.clone(), tx.clone(), interval, Arc::clone(&components_patterns), )), tokio::spawn(received_events_totals( - Arc::clone(&client), + client.clone(), tx.clone(), interval, Arc::clone(&components_patterns), )), tokio::spawn(received_events_throughputs( - Arc::clone(&client), + client.clone(), tx.clone(), interval, Arc::clone(&components_patterns), )), tokio::spawn(sent_bytes_totals( - Arc::clone(&client), + client.clone(), tx.clone(), interval, Arc::clone(&components_patterns), )), tokio::spawn(sent_bytes_throughputs( - Arc::clone(&client), + client.clone(), tx.clone(), interval, Arc::clone(&components_patterns), )), tokio::spawn(sent_events_totals( - Arc::clone(&client), + client.clone(), tx.clone(), interval, Arc::clone(&components_patterns), )), tokio::spawn(sent_events_throughputs( - Arc::clone(&client), - tx.clone(), - interval, - Arc::clone(&components_patterns), - )), - #[cfg(feature = "allocation-tracing")] - tokio::spawn(allocated_bytes( - Arc::clone(&client), + client.clone(), tx.clone(), interval, Arc::clone(&components_patterns), )), tokio::spawn(errors_totals( - Arc::clone(&client), + client.clone(), tx.clone(), interval, Arc::clone(&components_patterns), )), - tokio::spawn(uptime_changed(Arc::clone(&client), tx)), - ] + tokio::spawn(uptime_changed(client.clone(), tx.clone(), interval)), + ]; + + #[cfg(feature = "allocation-tracing")] + metric_handles.push(tokio::spawn(allocated_bytes( + client, + tx, + interval, + Arc::clone(&components_patterns), + ))); + + Ok(SubscribeHandles { + metric_handles, + poll_handle, + }) } /// Retrieve the initial components/metrics for first paint. Further updating the metrics /// will be handled by subscriptions. pub async fn init_components( - client: &Client, + uri: Uri, components_patterns: &[Pattern], -) -> Result { - // Execute a query to get the latest components, and aggregate metrics for each resource. - // Since we don't know currently have a mechanism for scrolling/paging through results, - // we're using an artificially high page size to capture all likely component configurations. - let rows = client - .components_query(i16::MAX as i64) - .await - .map_err(|_| ())? - .data - .ok_or(())? +) -> Result { + let mut client = Client::new(uri); + client.connect().await?; + + // Get all components + let response = client.get_components(0).await?; + + let rows = response .components - .edges .into_iter() - .filter(|edge| component_matches_patterns(&edge.node.component_id, components_patterns)) - .map(|edge| { - let d = edge.node; - let key = ComponentKey::from(d.component_id); - ( - key.clone(), - state::ComponentRow { - key, - kind: d.on.to_string(), - component_type: d.component_type, - outputs: d - .on - .outputs() - .into_iter() - .map(|(id, sent_events_total)| (id, OutputMetrics::from(sent_events_total))) - .collect(), - received_bytes_total: d.on.received_bytes_total(), - received_bytes_throughput_sec: 0, - received_events_total: d.on.received_events_total(), - received_events_throughput_sec: 0, - sent_bytes_total: d.on.sent_bytes_total(), - sent_bytes_throughput_sec: 0, - sent_events_total: d.on.sent_events_total(), - sent_events_throughput_sec: 0, - #[cfg(feature = "allocation-tracing")] - allocated_bytes: 0, - errors: 0, - }, - ) + .filter(|component| { + component_matches_patterns(&component.component_id, components_patterns) + }) + .map(|component| { + let row = component_to_row(&component); + (row.key.clone(), row) }) .collect::>(); + #[cfg(feature = "allocation-tracing")] + { + // Allocation tracing is a compile-time + startup-time setting on the + // server, so querying once per connection is sufficient. On error + // (e.g. older server without this RPC) we default to false, matching + // pre-existing behavior. This is re-evaluated on every reconnect via + // the retry loop in `subscription()`. + let mut state = state::State::new(rows); + state.allocation_tracing_active = client + .get_allocation_tracing_status() + .await + .map(|r| r.enabled) + .unwrap_or(false); + Ok(state) + } + + #[cfg(not(feature = "allocation-tracing"))] Ok(state::State::new(rows)) } diff --git a/lib/vector-top/src/state.rs b/lib/vector-top/src/state.rs index e4c535f7851f0..30fc05ef76c13 100644 --- a/lib/vector-top/src/state.rs +++ b/lib/vector-top/src/state.rs @@ -21,7 +21,7 @@ use crate::dashboard::columns; type IdentifiedMetric = (ComponentKey, i64); -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct SentEventsMetric { pub key: ComponentKey, pub total: i64, @@ -33,18 +33,18 @@ pub enum EventType { InitializeState(State), UptimeChanged(f64), ReceivedBytesTotals(Vec), - /// Interval + identified metric - ReceivedBytesThroughputs(i64, Vec), + /// Throughput values already normalized to per-second by the server + ReceivedBytesThroughputs(Vec), ReceivedEventsTotals(Vec), - /// Interval in ms + identified metric - ReceivedEventsThroughputs(i64, Vec), + /// Throughput values already normalized to per-second by the server + ReceivedEventsThroughputs(Vec), SentBytesTotals(Vec), - /// Interval + identified metric - SentBytesThroughputs(i64, Vec), + /// Throughput values already normalized to per-second by the server + SentBytesThroughputs(Vec), // Identified overall metric + output-specific metrics SentEventsTotals(Vec), - /// Interval in ms + identified overall metric + output-specific metrics - SentEventsThroughputs(i64, Vec), + /// Throughput values already normalized to per-second by the server + SentEventsThroughputs(Vec), ErrorsTotals(Vec), #[cfg(feature = "allocation-tracing")] AllocatedBytes(Vec), @@ -122,6 +122,10 @@ pub struct State { pub sort_state: SortState, pub filter_state: FilterState, pub ui: UiState, + /// Set to `true` once we receive the first `AllocatedBytes` event, + /// indicating the connected Vector instance has allocation tracing active. + #[cfg(feature = "allocation-tracing")] + pub allocation_tracing_active: bool, } #[derive(Debug, Clone, Copy)] @@ -343,6 +347,8 @@ impl State { ui: UiState::default(), sort_state: SortState::default(), filter_state: FilterState::default(), + #[cfg(feature = "allocation-tracing")] + allocation_tracing_active: false, } } @@ -437,11 +443,10 @@ pub async fn updater(mut event_rx: EventRx, mut state: State) -> StateRx { } } } - EventType::ReceivedBytesThroughputs(interval, rows) => { + EventType::ReceivedBytesThroughputs(rows) => { for (key, v) in rows { if let Some(r) = state.components.get_mut(&key) { - r.received_bytes_throughput_sec = - (v as f64 * (1000.0 / interval as f64)) as i64; + r.received_bytes_throughput_sec = v; } } } @@ -452,11 +457,10 @@ pub async fn updater(mut event_rx: EventRx, mut state: State) -> StateRx { } } } - EventType::ReceivedEventsThroughputs(interval, rows) => { + EventType::ReceivedEventsThroughputs(rows) => { for (key, v) in rows { if let Some(r) = state.components.get_mut(&key) { - r.received_events_throughput_sec = - (v as f64 * (1000.0 / interval as f64)) as i64; + r.received_events_throughput_sec = v; } } } @@ -467,11 +471,10 @@ pub async fn updater(mut event_rx: EventRx, mut state: State) -> StateRx { } } } - EventType::SentBytesThroughputs(interval, rows) => { + EventType::SentBytesThroughputs(rows) => { for (key, v) in rows { if let Some(r) = state.components.get_mut(&key) { - r.sent_bytes_throughput_sec = - (v as f64 * (1000.0 / interval as f64)) as i64; + r.sent_bytes_throughput_sec = v; } } } @@ -488,17 +491,15 @@ pub async fn updater(mut event_rx: EventRx, mut state: State) -> StateRx { } } } - EventType::SentEventsThroughputs(interval, rows) => { + EventType::SentEventsThroughputs(rows) => { for m in rows { if let Some(r) = state.components.get_mut(&m.key) { - r.sent_events_throughput_sec = - (m.total as f64 * (1000.0 / interval as f64)) as i64; + r.sent_events_throughput_sec = m.total; for (id, v) in m.outputs { - let throughput = (v as f64 * (1000.0 / interval as f64)) as i64; r.outputs .entry(id) .or_insert_with(OutputMetrics::default) - .sent_events_throughput_sec = throughput; + .sent_events_throughput_sec = v; } } } diff --git a/lib/vector-vrl/category/Cargo.toml b/lib/vector-vrl/category/Cargo.toml index dafb938de89fa..886b7dce5e72f 100644 --- a/lib/vector-vrl/category/Cargo.toml +++ b/lib/vector-vrl/category/Cargo.toml @@ -7,4 +7,4 @@ publish = false license = "MPL-2.0" [dependencies] -strum = { version = "0.27", features = ["derive"] } +strum.workspace = true diff --git a/lib/vector-vrl/cli/Cargo.toml b/lib/vector-vrl/cli/Cargo.toml index 32f0b8ccce166..8e3c78597de8b 100644 --- a/lib/vector-vrl/cli/Cargo.toml +++ b/lib/vector-vrl/cli/Cargo.toml @@ -6,6 +6,10 @@ edition = "2024" publish = false license = "MPL-2.0" +[features] +# Enable the full VRL stdlib, which includes all available VRL functions. +default = ["vrl/stdlib"] + [dependencies] clap.workspace = true vector-vrl-functions.workspace = true diff --git a/lib/vector-vrl/dnstap-parser/src/parser.rs b/lib/vector-vrl/dnstap-parser/src/parser.rs index 3876ff363bec6..5e14303c4a5a2 100644 --- a/lib/vector-vrl/dnstap-parser/src/parser.rs +++ b/lib/vector-vrl/dnstap-parser/src/parser.rs @@ -260,6 +260,13 @@ impl DnstapParser { match dnstap_message_type_id { 1..=12 => { if let Some(query_message) = dnstap_message.query_message { + DnstapParser::insert( + event, + prefix.clone(), + &DNSTAP_VALUE_PATHS.request_message_size, + query_message.len(), + ); + let mut query_message_parser = DnsMessageParser::with_options(query_message, parsing_options.clone()); if let Err(error) = DnstapParser::parse_dns_query_message( @@ -278,6 +285,12 @@ impl DnstapParser { } if let Some(response_message) = dnstap_message.response_message { + DnstapParser::insert( + event, + prefix.clone(), + &DNSTAP_VALUE_PATHS.response_message_size, + response_message.len(), + ); let mut response_message_parser = DnsMessageParser::with_options(response_message, parsing_options); if let Err(error) = DnstapParser::parse_dns_query_message( @@ -297,6 +310,12 @@ impl DnstapParser { } 13 | 14 => { if let Some(update_request_message) = dnstap_message.query_message { + DnstapParser::insert( + event, + prefix.clone(), + &DNSTAP_VALUE_PATHS.request_message_size, + update_request_message.len(), + ); let mut update_request_message_parser = DnsMessageParser::with_options( update_request_message, parsing_options.clone(), @@ -317,6 +336,12 @@ impl DnstapParser { } if let Some(update_response_message) = dnstap_message.response_message { + DnstapParser::insert( + event, + prefix.clone(), + &DNSTAP_VALUE_PATHS.response_message_size, + update_response_message.len(), + ); let mut update_response_message_parser = DnsMessageParser::with_options(update_response_message, parsing_options); if let Err(error) = DnstapParser::parse_dns_update_message( @@ -1060,6 +1085,7 @@ mod tests { ("dataTypeId", Value::Integer(1)), ("messageType", Value::Bytes(Bytes::from("ResolverQuery"))), ("messageTypeId", Value::Integer(3)), + ("requestMessageSize", Value::Integer(54)), ("queryZone", Value::Bytes(Bytes::from("com."))), ("requestData.fullRcode", Value::Integer(0)), ("requestData.header.aa", Value::Boolean(false)), @@ -1256,6 +1282,8 @@ mod tests { ("dataTypeId", Value::Integer(1)), ("messageType", Value::Bytes(Bytes::from("UpdateResponse"))), ("messageTypeId", Value::Integer(14)), + ("requestMessageSize", Value::Integer(29)), + ("responseMessageSize", Value::Integer(29)), ("requestData.fullRcode", Value::Integer(0)), ("requestData.header.adCount", Value::Integer(0)), ("requestData.header.id", Value::Integer(28811)), diff --git a/lib/vector-vrl/dnstap-parser/src/schema.rs b/lib/vector-vrl/dnstap-parser/src/schema.rs index 29a1b861f6e69..c2fea507d60e1 100644 --- a/lib/vector-vrl/dnstap-parser/src/schema.rs +++ b/lib/vector-vrl/dnstap-parser/src/schema.rs @@ -174,6 +174,16 @@ impl DnstapEventSchema { Kind::object(self.request_message_schema_definition()), None, ) + .optional_field( + &DNSTAP_VALUE_PATHS.request_message_size, + Kind::integer(), + None, + ) + .optional_field( + &DNSTAP_VALUE_PATHS.response_message_size, + Kind::integer(), + None, + ) } /// The schema definition for a dns tap message. @@ -211,6 +221,8 @@ pub struct DnstapPaths { pub message_type_id: OwnedValuePath, pub request_message: OwnedValuePath, pub response_message: OwnedValuePath, + pub request_message_size: OwnedValuePath, + pub response_message_size: OwnedValuePath, // DnsQueryMessageSchema pub response_code: OwnedValuePath, @@ -311,6 +323,8 @@ pub static DNSTAP_VALUE_PATHS: LazyLock = LazyLock::new(|| DnstapPa message_type_id: owned_value_path!("messageTypeId"), request_message: owned_value_path!("requestData"), response_message: owned_value_path!("responseData"), + request_message_size: owned_value_path!("requestMessageSize"), + response_message_size: owned_value_path!("responseMessageSize"), response_code: owned_value_path!("fullRcode"), response: owned_value_path!("rcodeName"), header: owned_value_path!("header"), diff --git a/lib/vector-vrl/dnstap-parser/src/vrl_functions/parse_dnstap.rs b/lib/vector-vrl/dnstap-parser/src/vrl_functions/parse_dnstap.rs index 8c0a5d07f35e7..b534226b68cb7 100644 --- a/lib/vector-vrl/dnstap-parser/src/vrl_functions/parse_dnstap.rs +++ b/lib/vector-vrl/dnstap-parser/src/vrl_functions/parse_dnstap.rs @@ -67,6 +67,8 @@ impl Function for ParseDnstap { "extraInfo": "", "messageType": "ResolverQuery", "messageTypeId": 3, + "requestMessageSize": 54, + "responseMessageSize": 100, "queryZone": "com.", "requestData": { "fullRcode": 0, @@ -244,6 +246,8 @@ mod tests { extraInfo: "", messageType: "ResolverQuery", messageTypeId: 3, + requestMessageSize: 54, + responseMessageSize: 100, queryZone: "com.", requestData: { fullRcode: 0, @@ -358,6 +362,8 @@ mod tests { dataTypeId: 1, messageType: "UpdateResponse", messageTypeId: 14, + requestMessageSize: 29, + responseMessageSize: 29, requestData: { fullRcode: 0, header: { diff --git a/lib/vector-vrl/doc-builder/Cargo.toml b/lib/vector-vrl/doc-builder/Cargo.toml new file mode 100644 index 0000000000000..75a1d262f4708 --- /dev/null +++ b/lib/vector-vrl/doc-builder/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "vector-vrl-doc-builder" +version = "0.1.0" +authors = ["Vector Contributors "] +edition = "2024" +publish = false +license = "MPL-2.0" + +[dependencies] +anyhow.workspace = true +clap.workspace = true +serde_json = { workspace = true, features = ["preserve_order"] } +vector-vrl-functions = { path = "../functions", features = ["dnstap", "vrl-metrics"] } +vrl = { workspace = true, features = ["docs"] } diff --git a/lib/vector-vrl/doc-builder/src/main.rs b/lib/vector-vrl/doc-builder/src/main.rs new file mode 100644 index 0000000000000..92c38dfea307d --- /dev/null +++ b/lib/vector-vrl/doc-builder/src/main.rs @@ -0,0 +1,49 @@ +use anyhow::Result; +use std::path::PathBuf; +use vrl::docs::{build_functions_doc, document_functions_to_dir}; + +/// Generate Vector-specific VRL function documentation as JSON files. +/// +/// Two modes of operation: +/// --output Write one JSON file per function into DIR (uses --extension). +/// (no --output) Print all functions to stdout as a JSON array (uses --minify). +#[derive(clap::Parser, Debug)] +#[command()] +struct Cli { + /// Output directory to create JSON files. When omitted, output is written to stdout as a JSON + /// array. + #[arg(short, long, conflicts_with = "minify")] + output: Option, + + /// Whether to minify the JSON output (stdout mode only) + #[arg(short, long, default_value_t = false, conflicts_with = "output")] + minify: bool, + + /// File extension for generated files (directory mode only) + #[arg(short, long, default_value = "json", requires = "output")] + extension: String, +} + +#[allow(clippy::print_stdout)] +fn main() -> Result<()> { + let cli = ::parse(); + let functions = vector_vrl_functions::all_without_vrl_stdlib(); + if let Some(output) = &cli.output { + document_functions_to_dir(&functions, output, &cli.extension)?; + } else { + let built = build_functions_doc(&functions); + if cli.minify { + println!( + "{}", + serde_json::to_string(&built).expect("FunctionDoc serialization should not fail") + ); + } else { + println!( + "{}", + serde_json::to_string_pretty(&built) + .expect("FunctionDoc serialization should not fail") + ); + } + } + Ok(()) +} diff --git a/lib/vector-vrl/enrichment/src/lib.rs b/lib/vector-vrl/enrichment/src/lib.rs index df4575e91f08e..293a5db343d61 100644 --- a/lib/vector-vrl/enrichment/src/lib.rs +++ b/lib/vector-vrl/enrichment/src/lib.rs @@ -185,6 +185,7 @@ pub(crate) const ENRICHMENT_TABLE_EXPLAINER: &str = indoc! {r#" * `autonomous_system_organization` * `isp` * `organization` + * `network` * City databases: * `city_name` @@ -198,9 +199,20 @@ pub(crate) const ENRICHMENT_TABLE_EXPLAINER: &str = indoc! {r#" * `longitude` * `postal_code` * `timezone` + * `network` * Connection-Type databases: * `connection_type` + * `network` + + * Anonymous-IP databases: + * `is_anonymous` + * `is_anonymous_vpn` + * `is_hosting_provider` + * `is_public_proxy` + * `is_residential_proxy` + * `is_tor_exit_node` + * `network` To use this function, you need to update your configuration to include an diff --git a/lib/vector-vrl/tests/Cargo.toml b/lib/vector-vrl/tests/Cargo.toml index ca477616e3b43..791c51f24c77e 100644 --- a/lib/vector-vrl/tests/Cargo.toml +++ b/lib/vector-vrl/tests/Cargo.toml @@ -20,7 +20,7 @@ serde_json.workspace = true tracing-subscriber = { workspace = true, features = ["fmt"] } [target.'cfg(not(target_env = "msvc"))'.dependencies] -tikv-jemallocator = { version = "0.6.1" } +tikv-jemallocator = { version = "0.7.0" } [features] default = [] diff --git a/lib/vector-vrl/tests/resources/json-schema_definition.json b/lib/vector-vrl/tests/resources/json-schema_definition.json index 9c4c8a00d0b28..a2e51e99c51d1 100644 --- a/lib/vector-vrl/tests/resources/json-schema_definition.json +++ b/lib/vector-vrl/tests/resources/json-schema_definition.json @@ -1 +1,10 @@ -{"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"https://example.com/product.schema.json","title":"Product","description":"A product from Acme's catalog","type":"object","properties":{"productUser":{"description":"The unique identifier for a product user","type":"string","format":"email"}}} +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.com/product.schema.json", + "title": "Product", + "description": "A product from Acme's catalog", + "type": "object", + "properties": { + "productUser": { "description": "The unique identifier for a product user", "type": "string", "format": "email" } + } +} diff --git a/lib/vector-vrl/web-playground/Cargo.toml b/lib/vector-vrl/web-playground/Cargo.toml index d84b2c8d69ad4..59db0427b3698 100644 --- a/lib/vector-vrl/web-playground/Cargo.toml +++ b/lib/vector-vrl/web-playground/Cargo.toml @@ -11,6 +11,10 @@ license = "MPL-2.0" [lib] crate-type = ["cdylib"] +[features] +# Enable the full VRL stdlib, which includes all available VRL functions. +default = ["vrl/stdlib"] + [dependencies] wasm-bindgen = "0.2" vrl.workspace = true @@ -21,6 +25,7 @@ vector-vrl-functions.workspace = true enrichment = { path = "../enrichment" } # Required per https://docs.rs/getrandom/latest/getrandom/#webassembly-support getrandom = { version = "0.2.15", features = ["js"] } +getrandom_v04 = { package = "getrandom", version = "0.4", features = ["wasm_js"] } [build-dependencies] -cargo-lock = "10.1.0" +cargo-lock = "11.0.1" diff --git a/lib/vector-vrl/web-playground/public/index.js b/lib/vector-vrl/web-playground/public/index.js index 4cdc2bf4e0dcc..9f810ec29e74a 100644 --- a/lib/vector-vrl/web-playground/public/index.js +++ b/lib/vector-vrl/web-playground/public/index.js @@ -1,5 +1,5 @@ -import init, {run_vrl, vector_link, vector_version, vrl_link, vrl_version} from "./pkg/vector_vrl_web_playground.js"; -import {vrlLanguageDefinition, vrlThemeDefinition} from "./vrl-highlighter.js"; +import init, { run_vrl, vector_link, vector_version, vrl_link, vrl_version } from "./pkg/vector_vrl_web_playground.js"; +import { vrlLanguageDefinition, vrlThemeDefinition } from "./vrl-highlighter.js"; const PROGRAM_EDITOR_DEFAULT_VALUE = `# Remove some fields del(.foo) @@ -42,309 +42,313 @@ You can try validating your JSON here: https://jsonlint.com/ `; function loadMonaco() { - return new Promise((resolve, reject) => { - // require is provided by loader.min.js. - require.config({ - paths: {vs: "https://cdnjs.cloudflare.com/ajax/libs/monaco-editor/0.26.1/min/vs"}, - }); - require(["vs/editor/editor.main"], () => resolve(window.monaco), reject); + return new Promise((resolve, reject) => { + // require is provided by loader.min.js. + require.config({ + paths: { vs: "https://cdnjs.cloudflare.com/ajax/libs/monaco-editor/0.26.1/min/vs" } }); + require(["vs/editor/editor.main"], () => resolve(window.monaco), reject); + }); } export class VrlWebPlayground { - static async create() { - const instance = new VrlWebPlayground(true); - await instance._initAsync(); - return instance; + static async create() { + const instance = new VrlWebPlayground(true); + await instance._initAsync(); + return instance; + } + + constructor(_internal = false) { + if (!_internal) { + // Prefer factory: VrlWebPlayground.create() + this._initAsync(); // fire-and-forget fallback } - - constructor(_internal = false) { - if (!_internal) { - // Prefer factory: VrlWebPlayground.create() - this._initAsync(); // fire-and-forget fallback - } + } + + async _initAsync() { + // Load wasm/runtime + await init(); + + // Bind native funcs/versions + this.run_vrl = run_vrl; + this.vector_version = vector_version(); + this.vector_link = vector_link(); + this.vrl_version = vrl_version(); + this.vrl_link = vrl_link(); + + // Load Monaco + this.monaco = await loadMonaco(); + + // VRL lang + theme + this.monaco.languages.register({ id: "vrl" }); + this.monaco.editor.defineTheme("vrl-theme", vrlThemeDefinition); + this.monaco.languages.setMonarchTokensProvider("vrl", vrlLanguageDefinition); + + // Editors + this.eventEditor = this.createDefaultEditor("container-event", EVENT_EDITOR_DEFAULT_VALUE, "json", "vs-light"); + this.outputEditor = this.createDefaultEditor("container-output", OUTPUT_EDITOR_DEFAULT_VALUE, "json", "vs-light"); + this.programEditor = this.createDefaultEditor( + "container-program", + PROGRAM_EDITOR_DEFAULT_VALUE, + "vrl", + "vrl-theme" + ); + + // Versions + this.addVersions(); + + // Handle shared state from URL (if present) + this._maybeLoadFromUrl(); + } + + _maybeLoadFromUrl() { + const qs = window.location.search; + if (!qs) return; + + const urlParams = new URLSearchParams(qs); + const stateParam = urlParams.get("state"); + if (!stateParam) return; + + try { + const decoded = atob(decodeURIComponent(stateParam)); + const urlState = JSON.parse(decoded); + + if (typeof urlState.program === "string") { + this.programEditor.setValue(urlState.program); + } + + if (urlState.is_jsonl === true && typeof urlState.event === "string") { + this.eventEditor.setValue(urlState.event); + } else if (urlState.event != null) { + this.eventEditor.setValue(JSON.stringify(urlState.event, null, "\t")); + } + + // Run immediately with the provided state + this.handleRunCode(urlState); + } catch (e) { + this.disableJsonLinting(); + this.outputEditor.setValue(`Error reading the shared URL\n${e}`); } + } - async _initAsync() { - // Load wasm/runtime - await init(); - - // Bind native funcs/versions - this.run_vrl = run_vrl; - this.vector_version = vector_version(); - this.vector_link = vector_link(); - this.vrl_version = vrl_version(); - this.vrl_link = vrl_link(); - - // Load Monaco - this.monaco = await loadMonaco(); - - // VRL lang + theme - this.monaco.languages.register({id: "vrl"}); - this.monaco.editor.defineTheme("vrl-theme", vrlThemeDefinition); - this.monaco.languages.setMonarchTokensProvider("vrl", vrlLanguageDefinition); - - // Editors - this.eventEditor = this.createDefaultEditor("container-event", EVENT_EDITOR_DEFAULT_VALUE, "json", "vs-light"); - this.outputEditor = this.createDefaultEditor("container-output", OUTPUT_EDITOR_DEFAULT_VALUE, "json", "vs-light"); - this.programEditor = this.createDefaultEditor("container-program", PROGRAM_EDITOR_DEFAULT_VALUE, "vrl", "vrl-theme"); - - // Versions - this.addVersions(); - - // Handle shared state from URL (if present) - this._maybeLoadFromUrl(); + addVersions() { + const vectorLinkElement = document.getElementById("vector-version-link"); + if (vectorLinkElement) { + vectorLinkElement.text = (this.vector_version || "").toString().substring(0, 8); + vectorLinkElement.href = this.vector_link || "#"; } - _maybeLoadFromUrl() { - const qs = window.location.search; - if (!qs) return; - - const urlParams = new URLSearchParams(qs); - const stateParam = urlParams.get("state"); - if (!stateParam) return; - - try { - const decoded = atob(decodeURIComponent(stateParam)); - const urlState = JSON.parse(decoded); - - if (typeof urlState.program === "string") { - this.programEditor.setValue(urlState.program); - } - - if (urlState.is_jsonl === true && typeof urlState.event === "string") { - this.eventEditor.setValue(urlState.event); - } else if (urlState.event != null) { - this.eventEditor.setValue(JSON.stringify(urlState.event, null, "\t")); - } - - // Run immediately with the provided state - this.handleRunCode(urlState); - } catch (e) { - this.disableJsonLinting(); - this.outputEditor.setValue(`Error reading the shared URL\n${e}`); - } + const vrlLinkElement = document.getElementById("vrl-version-link"); + if (vrlLinkElement) { + vrlLinkElement.text = (this.vrl_version || "").toString().substring(0, 8); + vrlLinkElement.href = this.vrl_link || "#"; } + } - addVersions() { - const vectorLinkElement = document.getElementById("vector-version-link"); - if (vectorLinkElement) { - vectorLinkElement.text = (this.vector_version || "").toString().substring(0, 8); - vectorLinkElement.href = this.vector_link || "#"; - } - - const vrlLinkElement = document.getElementById("vrl-version-link"); - if (vrlLinkElement) { - vrlLinkElement.text = (this.vrl_version || "").toString().substring(0, 8); - vrlLinkElement.href = this.vrl_link || "#"; - } + createDefaultEditor(elementId, value, language, theme) { + const el = document.getElementById(elementId); + if (!el) { + console.warn(`Editor container #${elementId} not found`); + return null; } + return this.monaco.editor.create(el, { + value, + language, + theme, + minimap: { enabled: false }, + automaticLayout: true, + wordWrap: "on" + }); + } - createDefaultEditor(elementId, value, language, theme) { - const el = document.getElementById(elementId); - if (!el) { - console.warn(`Editor container #${elementId} not found`); - return null; - } - return this.monaco.editor.create(el, { - value, - language, - theme, - minimap: {enabled: false}, - automaticLayout: true, - wordWrap: 'on', - }); + _clearOutput() { + if (this.outputEditor) { + // wipe the buffer so stale values never linger + this.outputEditor.setValue(""); } - - _clearOutput() { - if (this.outputEditor) { - // wipe the buffer so stale values never linger - this.outputEditor.setValue(""); - } - const elapsedEl = document.getElementById("elapsed-time"); - if (elapsedEl) { - elapsedEl.textContent = ""; - } + const elapsedEl = document.getElementById("elapsed-time"); + if (elapsedEl) { + elapsedEl.textContent = ""; } + } - _formatRunResult(runResult) { - if (runResult?.target_value != null) { - const isJson = typeof runResult.target_value === "object"; - const text = isJson - ? JSON.stringify(runResult.target_value, null, "\t") - : String(runResult.target_value); - return {text, isJson}; - } - if (runResult?.msg != null) { - return {text: String(runResult.msg), isJson: false}; - } - return {text: "Error - VRL did not return a result.", isJson: false}; + _formatRunResult(runResult) { + if (runResult?.target_value != null) { + const isJson = typeof runResult.target_value === "object"; + const text = isJson ? JSON.stringify(runResult.target_value, null, "\t") : String(runResult.target_value); + return { text, isJson }; } - - _setElapsed(elapsed_time) { - const elapsedEl = document.getElementById("elapsed-time"); - if (elapsedEl && elapsed_time != null) { - const ms = elapsed_time.toFixed(4) - elapsedEl.textContent = `Duration: ${ms} milliseconds`; - } + if (runResult?.msg != null) { + return { text: String(runResult.msg), isJson: false }; } - - _safeGet(editor, fallback = "") { - return editor?.getValue?.() ?? fallback; + return { text: "Error - VRL did not return a result.", isJson: false }; + } + + _setElapsed(elapsed_time) { + const elapsedEl = document.getElementById("elapsed-time"); + if (elapsedEl && elapsed_time != null) { + const ms = elapsed_time.toFixed(4); + elapsedEl.textContent = `Duration: ${ms} milliseconds`; } - - getState() { - if (this.eventEditorIsJsonl()) { - return { - program: this._safeGet(this.programEditor), - event: this.eventEditor.getModel().getLinesContent().join("\n"), - is_jsonl: true, - error: null, - }; - } - - const editorValue = this._safeGet(this.eventEditor); - try { - return { - program: this._safeGet(this.programEditor), - event: JSON.parse(editorValue.length === 0 ? "{}" : editorValue), - is_jsonl: false, - error: null, - }; - } catch (_err) { - return { - program: this._safeGet(this.programEditor), - event: null, - is_jsonl: false, - error: `Could not parse JSON event:\n${editorValue}`, - }; - } + } + + _safeGet(editor, fallback = "") { + return editor?.getValue?.() ?? fallback; + } + + getState() { + if (this.eventEditorIsJsonl()) { + return { + program: this._safeGet(this.programEditor), + event: this.eventEditor.getModel().getLinesContent().join("\n"), + is_jsonl: true, + error: null + }; } - disableJsonLinting() { - this.monaco.languages.json.jsonDefaults.setDiagnosticsOptions({validate: false}); + const editorValue = this._safeGet(this.eventEditor); + try { + return { + program: this._safeGet(this.programEditor), + event: JSON.parse(editorValue.length === 0 ? "{}" : editorValue), + is_jsonl: false, + error: null + }; + } catch (_err) { + return { + program: this._safeGet(this.programEditor), + event: null, + is_jsonl: false, + error: `Could not parse JSON event:\n${editorValue}` + }; } - - enableJsonLinting() { - this.monaco.languages.json.jsonDefaults.setDiagnosticsOptions({validate: true}); + } + + disableJsonLinting() { + this.monaco.languages.json.jsonDefaults.setDiagnosticsOptions({ validate: false }); + } + + enableJsonLinting() { + this.monaco.languages.json.jsonDefaults.setDiagnosticsOptions({ validate: true }); + } + + tryJsonParse(str) { + try { + return JSON.parse(str); + } catch (_e) { + this.disableJsonLinting(); + const err = ERROR_INVALID_JSONL_EVENT_MSG.toString().replace("{{str}}", str); + this.outputEditor.setValue(err); + throw new Error(err); } - - tryJsonParse(str) { - try { - return JSON.parse(str); - } catch (_e) { - this.disableJsonLinting(); - const err = ERROR_INVALID_JSONL_EVENT_MSG.toString().replace("{{str}}", str); - this.outputEditor.setValue(err); - throw new Error(err); - } + } + + /** + * Treat as JSONL if there are >1 non-empty lines and at least the second non-empty + * line *appears* to be a JSON object. Robust to whitespace. + */ + eventEditorIsJsonl() { + const model = this.eventEditor?.getModel?.(); + if (!model) return false; + + const rawLines = model.getLinesContent(); + const lines = rawLines.map((l) => l.trim()).filter((l) => l.length > 0); + if (lines.length <= 1) return false; + + const second = lines[1]; + return second.startsWith("{") && second.endsWith("}"); + } + + _getTimezoneOrDefault() { + const tzEl = document.getElementById("timezone-input"); + return tzEl?.value && tzEl.value.trim().length > 0 ? tzEl.value.trim() : "Default"; + } + + handleRunCode(input) { + this._clearOutput(); + + // JSONL path short-circuit + if (this.eventEditorIsJsonl()) { + return this.handleRunCodeJsonl(); } - /** - * Treat as JSONL if there are >1 non-empty lines and at least the second non-empty - * line *appears* to be a JSON object. Robust to whitespace. - */ - eventEditorIsJsonl() { - const model = this.eventEditor?.getModel?.(); - if (!model) return false; - - const rawLines = model.getLinesContent(); - const lines = rawLines.map((l) => l.trim()).filter((l) => l.length > 0); - if (lines.length <= 1) return false; - - const second = lines[1]; - return second.startsWith("{") && second.endsWith("}"); + if (input == null) { + input = this.getState(); } - _getTimezoneOrDefault() { - const tzEl = document.getElementById("timezone-input"); - return tzEl?.value && tzEl.value.trim().length > 0 ? tzEl.value.trim() : "Default"; + if (input.error) { + console.error(input.error); + this.disableJsonLinting(); + this.outputEditor.setValue(input.error); + return input; } - handleRunCode(input) { - this._clearOutput(); - - // JSONL path short-circuit - if (this.eventEditorIsJsonl()) { - return this.handleRunCodeJsonl(); - } - - if (input == null) { - input = this.getState(); - } - - if (input.error) { - console.error(input.error); - this.disableJsonLinting(); - this.outputEditor.setValue(input.error); - return input; - } - - const timezone = this._getTimezoneOrDefault(); - console.debug("Selected timezone: ", timezone); - const runResult = this.run_vrl(input, timezone); - console.log("Run result: ", runResult); - - const {text, isJson} = this._formatRunResult(runResult); - if (isJson) this.enableJsonLinting(); else this.disableJsonLinting(); - this.outputEditor.setValue(text); - - this._setElapsed(runResult?.elapsed_time); - return runResult; - } - - handleRunCodeJsonl() { - this._clearOutput(); - - const program = this._safeGet(this.programEditor); - const model = this.eventEditor?.getModel?.(); - const rawLines = model ? model.getLinesContent() : []; - const lines = rawLines.map(l => l.trim()).filter(l => l.length > 0); - - const timezone = this._getTimezoneOrDefault(); - - // Build inputs while validating JSON per line - const inputs = lines.map(line => ({ - program, - event: this.tryJsonParse(line), - is_jsonl: true, - })); - - // Run and collect results - const results = inputs.map(input => this.run_vrl(input, timezone)); - - const outputs = results.map(r => this._formatRunResult(r).text); - - // Output is not pure JSON (multiple objects / possible errors) - this.disableJsonLinting(); - this.outputEditor.setValue(outputs.join("\n")); - - // Aggregate elapsed time, rounded - const total = results.reduce((sum, r) => sum + (typeof r?.elapsed_time === "number" ? r.elapsed_time : 0), 0); - this._setElapsed(total); - - return results; - } - - handleShareCode() { - const state = this.getState(); - try { - const encoded = encodeURIComponent(btoa(JSON.stringify(state))); - window.history.pushState(state, "", `?state=${encoded}`); - return true; - } catch (e) { - this.disableJsonLinting(); - this.outputEditor.setValue(`Error encoding state for URL\n${e}`); - return false; - } + const timezone = this._getTimezoneOrDefault(); + console.debug("Selected timezone: ", timezone); + const runResult = this.run_vrl(input, timezone); + console.log("Run result: ", runResult); + + const { text, isJson } = this._formatRunResult(runResult); + if (isJson) this.enableJsonLinting(); + else this.disableJsonLinting(); + this.outputEditor.setValue(text); + + this._setElapsed(runResult?.elapsed_time); + return runResult; + } + + handleRunCodeJsonl() { + this._clearOutput(); + + const program = this._safeGet(this.programEditor); + const model = this.eventEditor?.getModel?.(); + const rawLines = model ? model.getLinesContent() : []; + const lines = rawLines.map((l) => l.trim()).filter((l) => l.length > 0); + + const timezone = this._getTimezoneOrDefault(); + + // Build inputs while validating JSON per line + const inputs = lines.map((line) => ({ + program, + event: this.tryJsonParse(line), + is_jsonl: true + })); + + // Run and collect results + const results = inputs.map((input) => this.run_vrl(input, timezone)); + + const outputs = results.map((r) => this._formatRunResult(r).text); + + // Output is not pure JSON (multiple objects / possible errors) + this.disableJsonLinting(); + this.outputEditor.setValue(outputs.join("\n")); + + // Aggregate elapsed time, rounded + const total = results.reduce((sum, r) => sum + (typeof r?.elapsed_time === "number" ? r.elapsed_time : 0), 0); + this._setElapsed(total); + + return results; + } + + handleShareCode() { + const state = this.getState(); + try { + const encoded = encodeURIComponent(btoa(JSON.stringify(state))); + window.history.pushState(state, "", `?state=${encoded}`); + return true; + } catch (e) { + this.disableJsonLinting(); + this.outputEditor.setValue(`Error encoding state for URL\n${e}`); + return false; } + } } // Prefer the async factory to ensure everything is loaded before use: VrlWebPlayground.create() - .then((instance) => { - window.vrlPlayground = instance; - }) - .catch((err) => { - console.error("Failed to initialize VrlWebPlayground:", err); - }); + .then((instance) => { + window.vrlPlayground = instance; + }) + .catch((err) => { + console.error("Failed to initialize VrlWebPlayground:", err); + }); diff --git a/lib/vector-vrl/web-playground/public/vrl-highlighter.js b/lib/vector-vrl/web-playground/public/vrl-highlighter.js index 3efaed67a4a4b..527934106530f 100644 --- a/lib/vector-vrl/web-playground/public/vrl-highlighter.js +++ b/lib/vector-vrl/web-playground/public/vrl-highlighter.js @@ -1,636 +1,633 @@ // VRL Language Definition export let vrlLanguageDefinition = { - defaultToken: "invalid", - ignoreCase: true, - tokenPostfix: ".vrl", + defaultToken: "invalid", + ignoreCase: true, + tokenPostfix: ".vrl", - brackets: [ - { open: "{", close: "}", token: "delimiter.curly" }, - { open: "[", close: "]", token: "delimiter.square" }, - { open: "(", close: ")", token: "delimiter.parenthesis" }, - ], + brackets: [ + { open: "{", close: "}", token: "delimiter.curly" }, + { open: "[", close: "]", token: "delimiter.square" }, + { open: "(", close: ")", token: "delimiter.parenthesis" } + ], - regEx: /\/(?!\/\/)(?:[^\/\\]|\\.)*\/[igm]*/, + regEx: /\/(?!\/\/)(?:[^\/\\]|\\.)*\/[igm]*/, - keywords: [ - "abort", - "as", - "break", - "continue", - "else", - "false", - "for", - "if", - "impl", - "in", - "let", - "loop", - "null", - "return", - "self", - "std", - "then", - "this", - "true", - "type", - "until", - "use", - "while", - ], + keywords: [ + "abort", + "as", + "break", + "continue", + "else", + "false", + "for", + "if", + "impl", + "in", + "let", + "loop", + "null", + "return", + "self", + "std", + "then", + "this", + "true", + "type", + "until", + "use", + "while" + ], - // we include these common regular expressions - symbols: /[=> output_totals = 2; +} + +// Throughput value (per-second rate) for a component metric. +// output_throughputs is only populated for METRIC_NAME_SENT_EVENTS_THROUGHPUT. +message ThroughputMetric { + double value = 1; + map output_throughputs = 2; +} + +message StreamComponentMetricsResponse { + string component_id = 1; + oneof value { + TotalMetric total = 2; + ThroughputMetric throughput = 3; + } +} + +message StreamHeartbeatRequest { + // Update interval in milliseconds + int32 interval_ms = 1; +} + +message StreamHeartbeatResponse { + google.protobuf.Timestamp utc = 1; +} + +message StreamUptimeRequest { + // Update interval in milliseconds + int32 interval_ms = 1; +} + +message StreamUptimeResponse { + int64 uptime_seconds = 1; +} + +message StreamComponentAllocatedBytesResponse { + string component_id = 1; + int64 allocated_bytes = 2; +} + +// ========== Event Tapping Messages ========== + +message StreamOutputEventsRequest { + // Glob patterns for output component IDs to match + repeated string outputs_patterns = 1; + + // Glob patterns for input component IDs to match (optional) + repeated string inputs_patterns = 2; + + // Maximum number of events to sample per interval. Must be >= 1. + // Unlike GetComponents, 0 is not valid here: limit controls both the + // reservoir size and the internal channel capacity, which require a + // positive value. + int32 limit = 3; + + // Sampling interval in milliseconds + int32 interval_ms = 4; +} + +message StreamOutputEventsResponse { + oneof event { + TappedEvent tapped_event = 1; + EventNotification notification = 2; + } +} + +// Event captured from a component with metadata about where it came from +message TappedEvent { + string component_id = 1; + string component_type = 2; + string component_kind = 3; + event.EventWrapper event = 4; // The actual event data (uses canonical proto definition) +} + +message EventNotification { + string message = 1; +} diff --git a/regression/Dockerfile b/regression/Dockerfile index dc1a037e47636..01be4e593b546 100644 --- a/regression/Dockerfile +++ b/regression/Dockerfile @@ -1,20 +1,33 @@ # # VECTOR BUILDER # -FROM docker.io/timberio/vector-dev:latest AS builder +ARG RUST_VERSION=1 +FROM docker.io/rust:${RUST_VERSION}-slim-trixie AS builder + +# Copy the whole scripts/environment/ tree so prepare.sh can source its sibling +# scripts on older checkouts in regression baseline builds. +COPY scripts/environment/ /scripts/environment/ + +RUN bash /scripts/environment/install-debian-build-deps.sh + WORKDIR /vector +COPY rust-toolchain.toml . + +RUN bash /scripts/environment/prepare.sh --modules=rustup +RUN bash /scripts/environment/install-protoc.sh + COPY . . RUN --mount=type=cache,target=/usr/local/cargo/registry \ --mount=type=cache,target=/usr/local/cargo/git \ --mount=type=cache,target=/vector/target \ - cargo build --bin vector --release && \ + /usr/bin/mold -run cargo build --bin vector --release && \ cp target/release/vector . # # TARGET # FROM docker.io/debian:trixie-slim@sha256:c85a2732e97694ea77237c61304b3bb410e0e961dd6ee945997a06c788c545bb -RUN apt-get update && apt-get dist-upgrade -y && apt-get -y --no-install-recommends install zlib1g ca-certificates && rm -rf /var/lib/apt/lists/* +RUN apt-get update && apt-get dist-upgrade -y && apt-get -y --no-install-recommends install zlib1g ca-certificates libsasl2-2 && rm -rf /var/lib/apt/lists/* COPY --from=builder /vector/vector /usr/bin/vector RUN mkdir --parents --mode=0777 /var/lib/vector diff --git a/regression/cases/datadog_agent_remap_blackhole/lading/lading.yaml b/regression/cases/datadog_agent_remap_blackhole/lading/lading.yaml index b870f2e2675bb..672e98c515b7b 100644 --- a/regression/cases/datadog_agent_remap_blackhole/lading/lading.yaml +++ b/regression/cases/datadog_agent_remap_blackhole/lading/lading.yaml @@ -1,7 +1,40 @@ generator: - http: - seed: [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, - 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131] + seed: + [ + 2, + 3, + 5, + 7, + 11, + 13, + 17, + 19, + 23, + 29, + 31, + 37, + 41, + 43, + 47, + 53, + 59, + 61, + 67, + 71, + 73, + 79, + 83, + 89, + 97, + 101, + 103, + 107, + 109, + 113, + 127, + 131 + ] headers: dd-api-key: "DEADBEEF" target_uri: "http://localhost:8282/v1/input" diff --git a/regression/cases/datadog_agent_remap_blackhole/vector/vector.yaml b/regression/cases/datadog_agent_remap_blackhole/vector/vector.yaml index 36704ea5986e3..9ec1a7665ff7f 100644 --- a/regression/cases/datadog_agent_remap_blackhole/vector/vector.yaml +++ b/regression/cases/datadog_agent_remap_blackhole/vector/vector.yaml @@ -9,9 +9,9 @@ sources: type: "internal_metrics" datadog_agent: - type: "datadog_agent" + type: "datadog_agent" acknowledgements: false - address: "0.0.0.0:8282" + address: "0.0.0.0:8282" ## ## Transforms @@ -19,21 +19,21 @@ sources: transforms: remap: - type: "remap" - inputs: [ "datadog_agent" ] + type: "remap" + inputs: ["datadog_agent"] source: | - .hostname = "vector" + .hostname = "vector" - if .status == "warning" { - .thing = upcase(.hostname) - } else if .status == "notice" { - .thung = downcase(.hostname) - } else { - .nong = upcase(.hostname) - } + if .status == "warning" { + .thing = upcase(.hostname) + } else if .status == "notice" { + .thung = downcase(.hostname) + } else { + .nong = upcase(.hostname) + } - .matches = { "name": .message, "num": "2" } - .origin, .err = .hostname + "/" + .matches.name + "/" + .matches.num + .matches = { "name": .message, "num": "2" } + .origin, .err = .hostname + "/" + .matches.name + "/" + .matches.num ## ## Sinks @@ -41,11 +41,11 @@ transforms: sinks: prometheus: - type: "prometheus_exporter" - inputs: [ "internal_metrics" ] + type: "prometheus_exporter" + inputs: ["internal_metrics"] address: "0.0.0.0:9090" blackhole: - type: "blackhole" + type: "blackhole" print_interval_secs: 0 - inputs: [ "remap" ] + inputs: ["remap"] diff --git a/regression/cases/datadog_agent_remap_blackhole_acks/lading/lading.yaml b/regression/cases/datadog_agent_remap_blackhole_acks/lading/lading.yaml index b870f2e2675bb..672e98c515b7b 100644 --- a/regression/cases/datadog_agent_remap_blackhole_acks/lading/lading.yaml +++ b/regression/cases/datadog_agent_remap_blackhole_acks/lading/lading.yaml @@ -1,7 +1,40 @@ generator: - http: - seed: [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, - 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131] + seed: + [ + 2, + 3, + 5, + 7, + 11, + 13, + 17, + 19, + 23, + 29, + 31, + 37, + 41, + 43, + 47, + 53, + 59, + 61, + 67, + 71, + 73, + 79, + 83, + 89, + 97, + 101, + 103, + 107, + 109, + 113, + 127, + 131 + ] headers: dd-api-key: "DEADBEEF" target_uri: "http://localhost:8282/v1/input" diff --git a/regression/cases/datadog_agent_remap_blackhole_acks/vector/vector.yaml b/regression/cases/datadog_agent_remap_blackhole_acks/vector/vector.yaml index aaf73d1570cfd..1054c161e4586 100644 --- a/regression/cases/datadog_agent_remap_blackhole_acks/vector/vector.yaml +++ b/regression/cases/datadog_agent_remap_blackhole_acks/vector/vector.yaml @@ -9,9 +9,9 @@ sources: type: "internal_metrics" datadog_agent: - type: "datadog_agent" + type: "datadog_agent" acknowledgements: true - address: "0.0.0.0:8282" + address: "0.0.0.0:8282" ## ## Transforms @@ -19,21 +19,21 @@ sources: transforms: remap: - type: "remap" - inputs: [ "datadog_agent" ] + type: "remap" + inputs: ["datadog_agent"] source: | - .hostname = "vector" + .hostname = "vector" - if .status == "warning" { - .thing = upcase(.hostname) - } else if .status == "notice" { - .thung = downcase(.hostname) - } else { - .nong = upcase(.hostname) - } + if .status == "warning" { + .thing = upcase(.hostname) + } else if .status == "notice" { + .thung = downcase(.hostname) + } else { + .nong = upcase(.hostname) + } - .matches = { "name": .message, "num": "2" } - .origin, .err = .hostname + "/" + .matches.name + "/" + .matches.num + .matches = { "name": .message, "num": "2" } + .origin, .err = .hostname + "/" + .matches.name + "/" + .matches.num ## ## Sinks @@ -41,11 +41,11 @@ transforms: sinks: prometheus: - type: "prometheus_exporter" - inputs: [ "internal_metrics" ] + type: "prometheus_exporter" + inputs: ["internal_metrics"] address: "0.0.0.0:9090" blackhole: - type: "blackhole" + type: "blackhole" print_interval_secs: 0 - inputs: [ "remap" ] + inputs: ["remap"] diff --git a/regression/cases/datadog_agent_remap_datadog_logs/lading/lading.yaml b/regression/cases/datadog_agent_remap_datadog_logs/lading/lading.yaml index dedc4de0b9819..41810ffad1cf2 100644 --- a/regression/cases/datadog_agent_remap_datadog_logs/lading/lading.yaml +++ b/regression/cases/datadog_agent_remap_datadog_logs/lading/lading.yaml @@ -1,6 +1,40 @@ generator: - http: - seed: [2, 3, 5, 7, 11, 13, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131, 137] + seed: + [ + 2, + 3, + 5, + 7, + 11, + 13, + 19, + 23, + 29, + 31, + 37, + 41, + 43, + 47, + 53, + 59, + 61, + 67, + 71, + 73, + 79, + 83, + 89, + 97, + 101, + 103, + 107, + 109, + 113, + 127, + 131, + 137 + ] headers: dd-api-key: "DEADBEEF" target_uri: "http://localhost:8282/v1/input" diff --git a/regression/cases/datadog_agent_remap_datadog_logs/vector/vector.yaml b/regression/cases/datadog_agent_remap_datadog_logs/vector/vector.yaml index 4691ea851f14c..faf04c536a0b7 100644 --- a/regression/cases/datadog_agent_remap_datadog_logs/vector/vector.yaml +++ b/regression/cases/datadog_agent_remap_datadog_logs/vector/vector.yaml @@ -9,9 +9,9 @@ sources: type: "internal_metrics" datadog_agent: - type: "datadog_agent" + type: "datadog_agent" acknowledgements: false - address: "0.0.0.0:8282" + address: "0.0.0.0:8282" ## ## Transforms @@ -19,13 +19,13 @@ sources: transforms: parse_message: - type: "remap" - inputs: [ "datadog_agent" ] + type: "remap" + inputs: ["datadog_agent"] source: | - pyld, err = parse_json(.message) - if err == null { - .message = pyld.mineral - } + pyld, err = parse_json(.message) + if err == null { + .message = pyld.mineral + } ## ## Sinks @@ -33,17 +33,17 @@ transforms: sinks: prometheus: - type: "prometheus_exporter" - inputs: [ "internal_metrics" ] + type: "prometheus_exporter" + inputs: ["internal_metrics"] address: "0.0.0.0:9090" datadog_logs: - type: "datadog_logs" - inputs: [ "parse_message" ] - endpoint: "http://localhost:8080" + type: "datadog_logs" + inputs: ["parse_message"] + endpoint: "http://localhost:8080" default_api_key: "DEADBEEF" healthcheck: enabled: false buffer: - type: "memory" - max_events: 50000 # buffer 50 payloads at a time + type: "memory" + max_events: 50000 # buffer 50 payloads at a time diff --git a/regression/cases/datadog_agent_remap_datadog_logs_acks/lading/lading.yaml b/regression/cases/datadog_agent_remap_datadog_logs_acks/lading/lading.yaml index dedc4de0b9819..41810ffad1cf2 100644 --- a/regression/cases/datadog_agent_remap_datadog_logs_acks/lading/lading.yaml +++ b/regression/cases/datadog_agent_remap_datadog_logs_acks/lading/lading.yaml @@ -1,6 +1,40 @@ generator: - http: - seed: [2, 3, 5, 7, 11, 13, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131, 137] + seed: + [ + 2, + 3, + 5, + 7, + 11, + 13, + 19, + 23, + 29, + 31, + 37, + 41, + 43, + 47, + 53, + 59, + 61, + 67, + 71, + 73, + 79, + 83, + 89, + 97, + 101, + 103, + 107, + 109, + 113, + 127, + 131, + 137 + ] headers: dd-api-key: "DEADBEEF" target_uri: "http://localhost:8282/v1/input" diff --git a/regression/cases/datadog_agent_remap_datadog_logs_acks/vector/vector.yaml b/regression/cases/datadog_agent_remap_datadog_logs_acks/vector/vector.yaml index 2e9d42fd6db20..7586039ee7465 100644 --- a/regression/cases/datadog_agent_remap_datadog_logs_acks/vector/vector.yaml +++ b/regression/cases/datadog_agent_remap_datadog_logs_acks/vector/vector.yaml @@ -9,9 +9,9 @@ sources: type: "internal_metrics" datadog_agent: - type: "datadog_agent" + type: "datadog_agent" acknowledgements: true - address: "0.0.0.0:8282" + address: "0.0.0.0:8282" ## ## Transforms @@ -19,13 +19,13 @@ sources: transforms: parse_message: - type: "remap" - inputs: [ "datadog_agent" ] + type: "remap" + inputs: ["datadog_agent"] source: | - pyld, err = parse_json(.message) - if err == null { - .message = pyld.mineral - } + pyld, err = parse_json(.message) + if err == null { + .message = pyld.mineral + } ## ## Sinks @@ -33,17 +33,17 @@ transforms: sinks: prometheus: - type: "prometheus_exporter" - inputs: [ "internal_metrics" ] + type: "prometheus_exporter" + inputs: ["internal_metrics"] address: "0.0.0.0:9090" datadog_logs: - type: "datadog_logs" - inputs: [ "parse_message" ] - endpoint: "http://localhost:8080" + type: "datadog_logs" + inputs: ["parse_message"] + endpoint: "http://localhost:8080" default_api_key: "DEADBEEF" healthcheck: enabled: false buffer: - type: "memory" - max_events: 50000 # buffer 50 payloads at a time + type: "memory" + max_events: 50000 # buffer 50 payloads at a time diff --git a/regression/cases/file_100_to_blackhole/lading/lading.yaml b/regression/cases/file_100_to_blackhole/lading/lading.yaml index 98cfbb9c9ab20..b44ea4e1915d2 100644 --- a/regression/cases/file_100_to_blackhole/lading/lading.yaml +++ b/regression/cases/file_100_to_blackhole/lading/lading.yaml @@ -1,8 +1,41 @@ generator: - file_gen: logrotate_fs: - seed: [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, - 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131] + seed: + [ + 2, + 3, + 5, + 7, + 11, + 13, + 17, + 19, + 23, + 29, + 31, + 37, + 41, + 43, + 47, + 53, + 59, + 61, + 67, + 71, + 73, + 79, + 83, + 89, + 97, + 101, + 103, + 107, + 109, + 113, + 127, + 131 + ] load_profile: constant: 100MiB concurrent_logs: 100 diff --git a/regression/cases/file_100_to_blackhole/vector/vector.yaml b/regression/cases/file_100_to_blackhole/vector/vector.yaml index 377ae72154fa9..f06546523893d 100644 --- a/regression/cases/file_100_to_blackhole/vector/vector.yaml +++ b/regression/cases/file_100_to_blackhole/vector/vector.yaml @@ -19,14 +19,14 @@ sources: sinks: prometheus: - type: "prometheus_exporter" - inputs: [ "internal_metrics" ] + type: "prometheus_exporter" + inputs: ["internal_metrics"] address: "0.0.0.0:9090" socket_sink: - type: "socket" - inputs: [ "file" ] - mode: "tcp" + type: "socket" + inputs: ["file"] + mode: "tcp" address: "127.0.0.1:15400" encoding: codec: "json" diff --git a/regression/cases/file_to_blackhole/lading/lading.yaml b/regression/cases/file_to_blackhole/lading/lading.yaml index 5017287363f08..2e6a1615b14e1 100644 --- a/regression/cases/file_to_blackhole/lading/lading.yaml +++ b/regression/cases/file_to_blackhole/lading/lading.yaml @@ -1,8 +1,41 @@ generator: - file_gen: logrotate_fs: - seed: [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, - 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131] + seed: + [ + 2, + 3, + 5, + 7, + 11, + 13, + 17, + 19, + 23, + 29, + 31, + 37, + 41, + 43, + 47, + 53, + 59, + 61, + 67, + 71, + 73, + 79, + 83, + 89, + 97, + 101, + 103, + 107, + 109, + 113, + 127, + 131 + ] load_profile: constant: 100MiB concurrent_logs: 4 diff --git a/regression/cases/file_to_blackhole/vector/vector.yaml b/regression/cases/file_to_blackhole/vector/vector.yaml index 377ae72154fa9..f06546523893d 100644 --- a/regression/cases/file_to_blackhole/vector/vector.yaml +++ b/regression/cases/file_to_blackhole/vector/vector.yaml @@ -19,14 +19,14 @@ sources: sinks: prometheus: - type: "prometheus_exporter" - inputs: [ "internal_metrics" ] + type: "prometheus_exporter" + inputs: ["internal_metrics"] address: "0.0.0.0:9090" socket_sink: - type: "socket" - inputs: [ "file" ] - mode: "tcp" + type: "socket" + inputs: ["file"] + mode: "tcp" address: "127.0.0.1:15400" encoding: codec: "json" diff --git a/regression/cases/fluent_elasticsearch/lading/lading.yaml b/regression/cases/fluent_elasticsearch/lading/lading.yaml index 80829c35e3539..392c39482fbd0 100644 --- a/regression/cases/fluent_elasticsearch/lading/lading.yaml +++ b/regression/cases/fluent_elasticsearch/lading/lading.yaml @@ -1,7 +1,40 @@ generator: - tcp: - seed: [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, - 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131] + seed: + [ + 2, + 3, + 5, + 7, + 11, + 13, + 17, + 19, + 23, + 29, + 31, + 37, + 41, + 43, + 47, + 53, + 59, + 61, + 67, + 71, + 73, + 79, + 83, + 89, + 97, + 101, + 103, + 107, + 109, + 113, + 127, + 131 + ] addr: "0.0.0.0:8282" variant: "fluent" bytes_per_second: "256 Mb" diff --git a/regression/cases/fluent_elasticsearch/vector/vector.yaml b/regression/cases/fluent_elasticsearch/vector/vector.yaml index 217d8b9cf9eb6..a62f315f716a4 100644 --- a/regression/cases/fluent_elasticsearch/vector/vector.yaml +++ b/regression/cases/fluent_elasticsearch/vector/vector.yaml @@ -9,7 +9,7 @@ sources: type: "internal_metrics" fluent: - type: "fluent" + type: "fluent" address: "0.0.0.0:8282" ## @@ -24,16 +24,16 @@ sources: sinks: prometheus: - type: "prometheus_exporter" - inputs: [ "internal_metrics" ] + type: "prometheus_exporter" + inputs: ["internal_metrics"] address: "0.0.0.0:9090" elasticsearch: - type: "elasticsearch" - inputs: [ "fluent" ] - endpoint: "http://0.0.0.0:8080" - mode: "bulk" - pipeline: "pipeline-name" + type: "elasticsearch" + inputs: ["fluent"] + endpoint: "http://0.0.0.0:8080" + mode: "bulk" + pipeline: "pipeline-name" compression: "none" bulk: index: "vector-%F" diff --git a/regression/cases/http_elasticsearch/lading/lading.yaml b/regression/cases/http_elasticsearch/lading/lading.yaml index 69cc36c1de926..a36bc0c04473e 100644 --- a/regression/cases/http_elasticsearch/lading/lading.yaml +++ b/regression/cases/http_elasticsearch/lading/lading.yaml @@ -1,7 +1,40 @@ generator: - http: - seed: [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, - 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131] + seed: + [ + 2, + 3, + 5, + 7, + 11, + 13, + 17, + 19, + 23, + 29, + 31, + 37, + 41, + 43, + 47, + 53, + 59, + 61, + 67, + 71, + 73, + 79, + 83, + 89, + 97, + 101, + 103, + 107, + 109, + 113, + 127, + 131 + ] target_uri: "http://localhost:8282/" bytes_per_second: "500 Mb" parallel_connections: 10 diff --git a/regression/cases/http_elasticsearch/vector/vector.yaml b/regression/cases/http_elasticsearch/vector/vector.yaml index e9f136f45a5ad..f34ce75aab7f2 100644 --- a/regression/cases/http_elasticsearch/vector/vector.yaml +++ b/regression/cases/http_elasticsearch/vector/vector.yaml @@ -7,7 +7,7 @@ sources: type: "internal_metrics" logs: - type: "http" + type: "http" address: "0.0.0.0:8282" decoding: codec: "json" @@ -24,17 +24,17 @@ sources: sinks: prometheus: - type: "prometheus_exporter" - inputs: [ "internal_metrics" ] + type: "prometheus_exporter" + inputs: ["internal_metrics"] address: "0.0.0.0:9090" elasticsearch: - type: "elasticsearch" - inputs: [ "logs" ] - endpoints: [ "http://localhost:8080" ] + type: "elasticsearch" + inputs: ["logs"] + endpoints: ["http://localhost:8080"] api_version: "v8" - mode: "bulk" - pipeline: "pipeline-name" + mode: "bulk" + pipeline: "pipeline-name" compression: "none" bulk: index: "vector-%F" diff --git a/regression/cases/http_text_to_http_json/lading/lading.yaml b/regression/cases/http_text_to_http_json/lading/lading.yaml index 97b45f7087245..f5608a87d4b41 100644 --- a/regression/cases/http_text_to_http_json/lading/lading.yaml +++ b/regression/cases/http_text_to_http_json/lading/lading.yaml @@ -1,7 +1,40 @@ generator: - http: - seed: [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, - 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131] + seed: + [ + 2, + 3, + 5, + 7, + 11, + 13, + 17, + 19, + 23, + 29, + 31, + 37, + 41, + 43, + 47, + 53, + 59, + 61, + 67, + 71, + 73, + 79, + 83, + 89, + 97, + 101, + 103, + 107, + 109, + 113, + 127, + 131 + ] headers: {} target_uri: "http://localhost:8282/" bytes_per_second: "500 Mb" diff --git a/regression/cases/http_text_to_http_json/vector/vector.yaml b/regression/cases/http_text_to_http_json/vector/vector.yaml index 36c3aa08578dc..9cdbd18b689ea 100644 --- a/regression/cases/http_text_to_http_json/vector/vector.yaml +++ b/regression/cases/http_text_to_http_json/vector/vector.yaml @@ -9,7 +9,7 @@ sources: type: "internal_metrics" logs: - type: "http" + type: "http" address: "0.0.0.0:8282" decoding: codec: "bytes" @@ -20,13 +20,13 @@ sources: sinks: prometheus: - type: "prometheus_exporter" - inputs: [ "internal_metrics" ] + type: "prometheus_exporter" + inputs: ["internal_metrics"] address: "0.0.0.0:9090" http_sink: type: "http" - uri: "http://localhost:8080" - inputs: [ "logs" ] + uri: "http://localhost:8080" + inputs: ["logs"] encoding: codec: "json" diff --git a/regression/cases/http_to_http_acks/lading/lading.yaml b/regression/cases/http_to_http_acks/lading/lading.yaml index 7f0bbeef95068..17cd16656b713 100644 --- a/regression/cases/http_to_http_acks/lading/lading.yaml +++ b/regression/cases/http_to_http_acks/lading/lading.yaml @@ -1,7 +1,40 @@ generator: - http: - seed: [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, - 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131] + seed: + [ + 2, + 3, + 5, + 7, + 11, + 13, + 17, + 19, + 23, + 29, + 31, + 37, + 41, + 43, + 47, + 53, + 59, + 61, + 67, + 71, + 73, + 79, + 83, + 89, + 97, + 101, + 103, + 107, + 109, + 113, + 127, + 131 + ] target_uri: "http://localhost:8282/" bytes_per_second: "100 Mb" parallel_connections: 10 diff --git a/regression/cases/http_to_http_acks/vector/vector.yaml b/regression/cases/http_to_http_acks/vector/vector.yaml index 3636dfe569d73..ec042dcdf086b 100644 --- a/regression/cases/http_to_http_acks/vector/vector.yaml +++ b/regression/cases/http_to_http_acks/vector/vector.yaml @@ -9,9 +9,9 @@ sources: type: "internal_metrics" http_source: - type: "http" + type: "http" acknowledgements: true - address: "0.0.0.0:8282" + address: "0.0.0.0:8282" ## ## Sinks @@ -19,18 +19,18 @@ sources: sinks: prometheus: - type: "prometheus_exporter" - inputs: [ "internal_metrics" ] + type: "prometheus_exporter" + inputs: ["internal_metrics"] address: "0.0.0.0:9090" http_sink: type: "http" - inputs: [ "http_source" ] - uri: "http://localhost:8080" + inputs: ["http_source"] + uri: "http://localhost:8080" encoding: codec: "text" healthcheck: enabled: false buffer: - type: "memory" - max_events: 50000 # buffer 50 payloads at a time + type: "memory" + max_events: 50000 # buffer 50 payloads at a time diff --git a/regression/cases/http_to_http_disk_buffer/experiment.yaml b/regression/cases/http_to_http_disk_buffer/experiment.yaml index e7faba7f54b2a..3650e2c1157a2 100644 --- a/regression/cases/http_to_http_disk_buffer/experiment.yaml +++ b/regression/cases/http_to_http_disk_buffer/experiment.yaml @@ -8,4 +8,3 @@ target: environment: VECTOR_THREADS: 4 - diff --git a/regression/cases/http_to_http_disk_buffer/lading/lading.yaml b/regression/cases/http_to_http_disk_buffer/lading/lading.yaml index a5b1c01a5c222..17cd16656b713 100644 --- a/regression/cases/http_to_http_disk_buffer/lading/lading.yaml +++ b/regression/cases/http_to_http_disk_buffer/lading/lading.yaml @@ -1,7 +1,40 @@ generator: - http: - seed: [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, - 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131] + seed: + [ + 2, + 3, + 5, + 7, + 11, + 13, + 17, + 19, + 23, + 29, + 31, + 37, + 41, + 43, + 47, + 53, + 59, + 61, + 67, + 71, + 73, + 79, + 83, + 89, + 97, + 101, + 103, + 107, + 109, + 113, + 127, + 131 + ] target_uri: "http://localhost:8282/" bytes_per_second: "100 Mb" parallel_connections: 10 @@ -18,4 +51,3 @@ blackhole: target_metrics: - prometheus: # internal telemetry uri: "http://127.0.0.1:9090/metrics" - diff --git a/regression/cases/http_to_http_disk_buffer/vector/vector.yaml b/regression/cases/http_to_http_disk_buffer/vector/vector.yaml index cd675e05c5ce7..23435dea23d1b 100644 --- a/regression/cases/http_to_http_disk_buffer/vector/vector.yaml +++ b/regression/cases/http_to_http_disk_buffer/vector/vector.yaml @@ -9,7 +9,7 @@ sources: type: "internal_metrics" http_source: - type: "http" + type: "http" acknowledgements: true address: "0.0.0.0:8282" @@ -19,19 +19,19 @@ sources: sinks: prometheus: - type: "prometheus_exporter" - inputs: [ "internal_metrics" ] + type: "prometheus_exporter" + inputs: ["internal_metrics"] address: "0.0.0.0:9090" http_sink: type: "http" - inputs: [ "http_source" ] - uri: "http://localhost:8080" + inputs: ["http_source"] + uri: "http://localhost:8080" encoding: codec: "text" healthcheck: enabled: false buffer: - type: "disk" + type: "disk" max_size: 4294967296 # 4GB diff --git a/regression/cases/http_to_http_json/lading/lading.yaml b/regression/cases/http_to_http_json/lading/lading.yaml index 68b9b25a6b6ff..25260a6683e33 100644 --- a/regression/cases/http_to_http_json/lading/lading.yaml +++ b/regression/cases/http_to_http_json/lading/lading.yaml @@ -1,7 +1,40 @@ generator: - http: - seed: [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, - 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131] + seed: + [ + 2, + 3, + 5, + 7, + 11, + 13, + 17, + 19, + 23, + 29, + 31, + 37, + 41, + 43, + 47, + 53, + 59, + 61, + 67, + 71, + 73, + 79, + 83, + 89, + 97, + 101, + 103, + 107, + 109, + 113, + 127, + 131 + ] target_uri: "http://localhost:8282/" bytes_per_second: "100 Mb" parallel_connections: 10 diff --git a/regression/cases/http_to_http_json/vector/vector.yaml b/regression/cases/http_to_http_json/vector/vector.yaml index 3d571dd5b407d..c4f229bba5195 100644 --- a/regression/cases/http_to_http_json/vector/vector.yaml +++ b/regression/cases/http_to_http_json/vector/vector.yaml @@ -9,9 +9,9 @@ sources: type: "internal_metrics" http_source: - type: "http" + type: "http" acknowledgements: false - address: "0.0.0.0:8282" + address: "0.0.0.0:8282" ## ## Sinks @@ -19,18 +19,18 @@ sources: sinks: prometheus: - type: "prometheus_exporter" - inputs: [ "internal_metrics" ] + type: "prometheus_exporter" + inputs: ["internal_metrics"] address: "0.0.0.0:9090" http_sink: type: "http" - inputs: [ "http_source" ] - uri: "http://localhost:8080" + inputs: ["http_source"] + uri: "http://localhost:8080" encoding: codec: "text" healthcheck: enabled: false buffer: - type: "memory" - max_events: 50000 # buffer 50 payloads at a time + type: "memory" + max_events: 50000 # buffer 50 payloads at a time diff --git a/regression/cases/http_to_http_noack/lading/lading.yaml b/regression/cases/http_to_http_noack/lading/lading.yaml index 7f0bbeef95068..17cd16656b713 100644 --- a/regression/cases/http_to_http_noack/lading/lading.yaml +++ b/regression/cases/http_to_http_noack/lading/lading.yaml @@ -1,7 +1,40 @@ generator: - http: - seed: [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, - 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131] + seed: + [ + 2, + 3, + 5, + 7, + 11, + 13, + 17, + 19, + 23, + 29, + 31, + 37, + 41, + 43, + 47, + 53, + 59, + 61, + 67, + 71, + 73, + 79, + 83, + 89, + 97, + 101, + 103, + 107, + 109, + 113, + 127, + 131 + ] target_uri: "http://localhost:8282/" bytes_per_second: "100 Mb" parallel_connections: 10 diff --git a/regression/cases/http_to_http_noack/vector/vector.yaml b/regression/cases/http_to_http_noack/vector/vector.yaml index 3d571dd5b407d..c4f229bba5195 100644 --- a/regression/cases/http_to_http_noack/vector/vector.yaml +++ b/regression/cases/http_to_http_noack/vector/vector.yaml @@ -9,9 +9,9 @@ sources: type: "internal_metrics" http_source: - type: "http" + type: "http" acknowledgements: false - address: "0.0.0.0:8282" + address: "0.0.0.0:8282" ## ## Sinks @@ -19,18 +19,18 @@ sources: sinks: prometheus: - type: "prometheus_exporter" - inputs: [ "internal_metrics" ] + type: "prometheus_exporter" + inputs: ["internal_metrics"] address: "0.0.0.0:9090" http_sink: type: "http" - inputs: [ "http_source" ] - uri: "http://localhost:8080" + inputs: ["http_source"] + uri: "http://localhost:8080" encoding: codec: "text" healthcheck: enabled: false buffer: - type: "memory" - max_events: 50000 # buffer 50 payloads at a time + type: "memory" + max_events: 50000 # buffer 50 payloads at a time diff --git a/regression/cases/http_to_s3/lading/lading.yaml b/regression/cases/http_to_s3/lading/lading.yaml index 0ce93d58e2ab4..25260a6683e33 100644 --- a/regression/cases/http_to_s3/lading/lading.yaml +++ b/regression/cases/http_to_s3/lading/lading.yaml @@ -1,6 +1,40 @@ generator: - http: - seed: [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131] + seed: + [ + 2, + 3, + 5, + 7, + 11, + 13, + 17, + 19, + 23, + 29, + 31, + 37, + 41, + 43, + 47, + 53, + 59, + 61, + 67, + 71, + 73, + 79, + 83, + 89, + 97, + 101, + 103, + 107, + 109, + 113, + 127, + 131 + ] target_uri: "http://localhost:8282/" bytes_per_second: "100 Mb" parallel_connections: 10 diff --git a/regression/cases/http_to_s3/vector/vector.yaml b/regression/cases/http_to_s3/vector/vector.yaml index a41bc700ef805..bf6a915f7e298 100644 --- a/regression/cases/http_to_s3/vector/vector.yaml +++ b/regression/cases/http_to_s3/vector/vector.yaml @@ -9,9 +9,9 @@ sources: type: "internal_metrics" http_source: - type: "http" + type: "http" acknowledgements: false - address: "0.0.0.0:8282" + address: "0.0.0.0:8282" decoding: codec: "json" @@ -21,20 +21,20 @@ sources: sinks: prometheus: - type: "prometheus_exporter" - inputs: [ "internal_metrics" ] + type: "prometheus_exporter" + inputs: ["internal_metrics"] address: "0.0.0.0:9090" s3_service: - type: "aws_s3" - inputs: [ "http_source" ] - region: "us-west-2" + type: "aws_s3" + inputs: ["http_source"] + region: "us-west-2" endpoint: "http://localhost:8080" - bucket: "vector-soak-service" + bucket: "vector-soak-service" framing: method: "newline_delimited" encoding: codec: "json" auth: - access_key_id: "BADDCAFE" + access_key_id: "BADDCAFE" secret_access_key: "BADDCAFE" diff --git a/regression/cases/otlp_grpc_to_blackhole/lading/lading.yaml b/regression/cases/otlp_grpc_to_blackhole/lading/lading.yaml index 763b301d51bf9..e860fff329203 100644 --- a/regression/cases/otlp_grpc_to_blackhole/lading/lading.yaml +++ b/regression/cases/otlp_grpc_to_blackhole/lading/lading.yaml @@ -1,7 +1,40 @@ generator: - grpc: - seed: [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, - 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131] + seed: + [ + 2, + 3, + 5, + 7, + 11, + 13, + 17, + 19, + 23, + 29, + 31, + 37, + 41, + 43, + 47, + 53, + 59, + 61, + 67, + 71, + 73, + 79, + 83, + 89, + 97, + 101, + 103, + 107, + 109, + 113, + 127, + 131 + ] target_uri: "http://127.0.0.1:4317/opentelemetry.proto.collector.logs.v1.LogsService/Export" bytes_per_second: "100 Mb" parallel_connections: 5 diff --git a/regression/cases/otlp_grpc_to_blackhole/vector/vector.yaml b/regression/cases/otlp_grpc_to_blackhole/vector/vector.yaml index ad267d1b9ec8d..31bf236873ef2 100644 --- a/regression/cases/otlp_grpc_to_blackhole/vector/vector.yaml +++ b/regression/cases/otlp_grpc_to_blackhole/vector/vector.yaml @@ -21,14 +21,14 @@ sources: sinks: prometheus: - type: "prometheus_exporter" - inputs: [ "internal_metrics" ] + type: "prometheus_exporter" + inputs: ["internal_metrics"] address: "127.0.0.1:9090" socket_sink: - type: "socket" - inputs: [ "opentelemetry.logs" ] - mode: "tcp" + type: "socket" + inputs: ["opentelemetry.logs"] + mode: "tcp" address: "127.0.0.1:15400" encoding: codec: "json" diff --git a/regression/cases/otlp_http_to_blackhole/lading/lading.yaml b/regression/cases/otlp_http_to_blackhole/lading/lading.yaml index 7f71aa751b070..74ac89acd261c 100644 --- a/regression/cases/otlp_http_to_blackhole/lading/lading.yaml +++ b/regression/cases/otlp_http_to_blackhole/lading/lading.yaml @@ -1,7 +1,40 @@ generator: - http: - seed: [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, - 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131] + seed: + [ + 2, + 3, + 5, + 7, + 11, + 13, + 17, + 19, + 23, + 29, + 31, + 37, + 41, + 43, + 47, + 53, + 59, + 61, + 67, + 71, + 73, + 79, + 83, + 89, + 97, + 101, + 103, + 107, + 109, + 113, + 127, + 131 + ] headers: content-type: "application/x-protobuf" target_uri: "http://127.0.0.1:4318/v1/logs" diff --git a/regression/cases/otlp_http_to_blackhole/vector/vector.yaml b/regression/cases/otlp_http_to_blackhole/vector/vector.yaml index ad267d1b9ec8d..31bf236873ef2 100644 --- a/regression/cases/otlp_http_to_blackhole/vector/vector.yaml +++ b/regression/cases/otlp_http_to_blackhole/vector/vector.yaml @@ -21,14 +21,14 @@ sources: sinks: prometheus: - type: "prometheus_exporter" - inputs: [ "internal_metrics" ] + type: "prometheus_exporter" + inputs: ["internal_metrics"] address: "127.0.0.1:9090" socket_sink: - type: "socket" - inputs: [ "opentelemetry.logs" ] - mode: "tcp" + type: "socket" + inputs: ["opentelemetry.logs"] + mode: "tcp" address: "127.0.0.1:15400" encoding: codec: "json" diff --git a/regression/cases/socket_to_socket_blackhole/lading/lading.yaml b/regression/cases/socket_to_socket_blackhole/lading/lading.yaml index 6a6b12171d473..5f0b017588d9e 100644 --- a/regression/cases/socket_to_socket_blackhole/lading/lading.yaml +++ b/regression/cases/socket_to_socket_blackhole/lading/lading.yaml @@ -1,7 +1,40 @@ generator: - tcp: - seed: [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, - 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131] + seed: + [ + 2, + 3, + 5, + 7, + 11, + 13, + 17, + 19, + 23, + 29, + 31, + 37, + 41, + 43, + 47, + 53, + 59, + 61, + 67, + 71, + 73, + 79, + 83, + 89, + 97, + 101, + 103, + 107, + 109, + 113, + 127, + 131 + ] addr: "0.0.0.0:8282" variant: "syslog5424" bytes_per_second: "500 Mb" diff --git a/regression/cases/socket_to_socket_blackhole/vector/vector.yaml b/regression/cases/socket_to_socket_blackhole/vector/vector.yaml index 76ddd6acf6919..6a43b8037ab68 100644 --- a/regression/cases/socket_to_socket_blackhole/vector/vector.yaml +++ b/regression/cases/socket_to_socket_blackhole/vector/vector.yaml @@ -9,9 +9,9 @@ sources: type: "internal_metrics" socket_source: - type: "socket" + type: "socket" address: "0.0.0.0:8282" - mode: "tcp" + mode: "tcp" ## ## Sinks @@ -19,14 +19,14 @@ sources: sinks: prometheus: - type: "prometheus_exporter" - inputs: [ "internal_metrics" ] + type: "prometheus_exporter" + inputs: ["internal_metrics"] address: "0.0.0.0:9090" socket_sink: - type: "socket" - inputs: [ "socket_source" ] - mode: "tcp" + type: "socket" + inputs: ["socket_source"] + mode: "tcp" address: "localhost:8080" encoding: codec: "json" diff --git a/regression/cases/splunk_hec_indexer_ack_blackhole/lading/lading.yaml b/regression/cases/splunk_hec_indexer_ack_blackhole/lading/lading.yaml index 560a72de2d84c..87f0ee7b8c328 100644 --- a/regression/cases/splunk_hec_indexer_ack_blackhole/lading/lading.yaml +++ b/regression/cases/splunk_hec_indexer_ack_blackhole/lading/lading.yaml @@ -1,7 +1,40 @@ generator: - splunk_hec: - seed: [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, - 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131] + seed: + [ + 2, + 3, + 5, + 7, + 11, + 13, + 17, + 19, + 23, + 29, + 31, + 37, + 41, + 43, + 47, + 53, + 59, + 61, + 67, + 71, + 73, + 79, + 83, + 89, + 97, + 101, + 103, + 107, + 109, + 113, + 127, + 131 + ] target_uri: "http://localhost:8282/" format: "json" token: "abcd1234" diff --git a/regression/cases/splunk_hec_indexer_ack_blackhole/vector/vector.yaml b/regression/cases/splunk_hec_indexer_ack_blackhole/vector/vector.yaml index d9b277f336517..342fed6840d53 100644 --- a/regression/cases/splunk_hec_indexer_ack_blackhole/vector/vector.yaml +++ b/regression/cases/splunk_hec_indexer_ack_blackhole/vector/vector.yaml @@ -9,7 +9,7 @@ sources: type: "internal_metrics" splunk_hec: - type: "splunk_hec" + type: "splunk_hec" address: "0.0.0.0:8282" acknowledgements: enabled: true @@ -20,11 +20,11 @@ sources: sinks: prometheus: - type: "prometheus_exporter" - inputs: [ "internal_metrics" ] + type: "prometheus_exporter" + inputs: ["internal_metrics"] address: "0.0.0.0:9090" blackhole: - type: "blackhole" + type: "blackhole" print_interval_secs: 0 - inputs: [ "splunk_hec" ] + inputs: ["splunk_hec"] diff --git a/regression/cases/splunk_hec_route_s3/lading/lading.yaml b/regression/cases/splunk_hec_route_s3/lading/lading.yaml index 308e3c05ce98c..efbc2dc0a9799 100644 --- a/regression/cases/splunk_hec_route_s3/lading/lading.yaml +++ b/regression/cases/splunk_hec_route_s3/lading/lading.yaml @@ -1,6 +1,40 @@ generator: - splunk_hec: - seed: [2, 3, 5, 7, 11, 13, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131, 137] + seed: + [ + 2, + 3, + 5, + 7, + 11, + 13, + 19, + 23, + 29, + 31, + 37, + 41, + 43, + 47, + 53, + 59, + 61, + 67, + 71, + 73, + 79, + 83, + 89, + 97, + 101, + 103, + 107, + 109, + 113, + 127, + 131, + 137 + ] target_uri: "http://localhost:8282/" format: "json" token: "abcd1234" diff --git a/regression/cases/splunk_hec_route_s3/vector/vector.yaml b/regression/cases/splunk_hec_route_s3/vector/vector.yaml index c81d227c190d7..801adcd5e814a 100644 --- a/regression/cases/splunk_hec_route_s3/vector/vector.yaml +++ b/regression/cases/splunk_hec_route_s3/vector/vector.yaml @@ -10,7 +10,7 @@ sources: splunk: address: "0.0.0.0:8282" - type: "splunk_hec" + type: "splunk_hec" ## ## Transforms @@ -19,7 +19,7 @@ sources: transforms: container_type: type: "route" - inputs: [ "splunk" ] + inputs: ["splunk"] route: service: '.attrs.c2cContainerType == "service"' sidecar: '.attrs.c2cContainerType != "service"' @@ -30,38 +30,38 @@ transforms: sinks: prometheus: - type: "prometheus_exporter" - inputs: [ "internal_metrics" ] + type: "prometheus_exporter" + inputs: ["internal_metrics"] address: "0.0.0.0:9090" s3_sidecar: - type: "aws_s3" - inputs: [ "container_type.sidecar" ] - region: "us-west-2" - endpoint: "http://localhost:8080" - bucket: "vector-soak-sidecar" + type: "aws_s3" + inputs: ["container_type.sidecar"] + region: "us-west-2" + endpoint: "http://localhost:8080" + bucket: "vector-soak-sidecar" framing: method: "newline_delimited" encoding: codec: "json" - except_fields: [ "timestamp" ] + except_fields: ["timestamp"] key_prefix: "v1/source_type/sidecar/aws_account_id/{{attrs.aws_account}}/system_id/{{attrs.systemid}}/service/{{attrs.c2cService}}/partition/{{attrs.c2cPartition}}/stage/{{attrs.c2cStage}}/year/%Y/month/%m/day/%d/hour/%H" auth: - access_key_id: "BADDCAFE" + access_key_id: "BADDCAFE" secret_access_key: "BADDCAFE" s3_service: - type: "aws_s3" - inputs: [ "container_type.service" ] - region: "us-west-2" - endpoint: "http://localhost:8080" - bucket: "vector-soak-service" + type: "aws_s3" + inputs: ["container_type.service"] + region: "us-west-2" + endpoint: "http://localhost:8080" + bucket: "vector-soak-service" framing: method: "newline_delimited" encoding: codec: "json" - except_fields: [ "timestamp" ] + except_fields: ["timestamp"] key_prefix: "v1/source_type/app/system_id/{{attrs.systemid}}/service/{{attrs.c2cService}}/partition/{{attrs.c2cPartition}}/stage/{{attrs.c2cStage}}/year/%Y/month/%m/day/%d/hour/%H" auth: - access_key_id: "BADDCAFE" + access_key_id: "BADDCAFE" secret_access_key: "BADDCAFE" diff --git a/regression/cases/splunk_hec_to_splunk_hec_logs_acks/lading/lading.yaml b/regression/cases/splunk_hec_to_splunk_hec_logs_acks/lading/lading.yaml index a84c3fb60727b..2e28e6c294f85 100644 --- a/regression/cases/splunk_hec_to_splunk_hec_logs_acks/lading/lading.yaml +++ b/regression/cases/splunk_hec_to_splunk_hec_logs_acks/lading/lading.yaml @@ -1,6 +1,40 @@ generator: - splunk_hec: - seed: [2, 3, 5, 7, 11, 13, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131, 137] + seed: + [ + 2, + 3, + 5, + 7, + 11, + 13, + 19, + 23, + 29, + 31, + 37, + 41, + 43, + 47, + 53, + 59, + 61, + 67, + 71, + 73, + 79, + 83, + 89, + 97, + 101, + 103, + 107, + 109, + 113, + 127, + 131, + 137 + ] target_uri: "http://localhost:8282/" format: "json" token: "abcd1234" diff --git a/regression/cases/splunk_hec_to_splunk_hec_logs_acks/vector/vector.yaml b/regression/cases/splunk_hec_to_splunk_hec_logs_acks/vector/vector.yaml index 39cb209c758f5..4f1818291b97d 100644 --- a/regression/cases/splunk_hec_to_splunk_hec_logs_acks/vector/vector.yaml +++ b/regression/cases/splunk_hec_to_splunk_hec_logs_acks/vector/vector.yaml @@ -9,9 +9,9 @@ sources: type: "internal_metrics" splunk_hec: - type: "splunk_hec" + type: "splunk_hec" acknowledgements: true - address: "0.0.0.0:8282" + address: "0.0.0.0:8282" ## ## Sinks @@ -19,17 +19,17 @@ sources: sinks: prometheus: - type: "prometheus_exporter" - inputs: [ "internal_metrics" ] + type: "prometheus_exporter" + inputs: ["internal_metrics"] address: "0.0.0.0:9090" splunk_hec_logs: - type: "splunk_hec_logs" - inputs: [ "splunk_hec" ] + type: "splunk_hec_logs" + inputs: ["splunk_hec"] endpoint: "http://localhost:8080" encoding: codec: "json" - token: "abcd1234" + token: "abcd1234" healthcheck: enabled: false acknowledgements: diff --git a/regression/cases/splunk_hec_to_splunk_hec_logs_noack/lading/lading.yaml b/regression/cases/splunk_hec_to_splunk_hec_logs_noack/lading/lading.yaml index a487150adf69b..97264370ad03f 100644 --- a/regression/cases/splunk_hec_to_splunk_hec_logs_noack/lading/lading.yaml +++ b/regression/cases/splunk_hec_to_splunk_hec_logs_noack/lading/lading.yaml @@ -1,6 +1,40 @@ generator: - splunk_hec: - seed: [2, 3, 5, 7, 11, 13, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131, 137] + seed: + [ + 2, + 3, + 5, + 7, + 11, + 13, + 19, + 23, + 29, + 31, + 37, + 41, + 43, + 47, + 53, + 59, + 61, + 67, + 71, + 73, + 79, + 83, + 89, + 97, + 101, + 103, + 107, + 109, + 113, + 127, + 131, + 137 + ] target_uri: "http://localhost:8282/" format: "json" token: "abcd1234" diff --git a/regression/cases/splunk_hec_to_splunk_hec_logs_noack/vector/vector.yaml b/regression/cases/splunk_hec_to_splunk_hec_logs_noack/vector/vector.yaml index b19ec94b5dcd2..b2de7a3241798 100644 --- a/regression/cases/splunk_hec_to_splunk_hec_logs_noack/vector/vector.yaml +++ b/regression/cases/splunk_hec_to_splunk_hec_logs_noack/vector/vector.yaml @@ -9,9 +9,9 @@ sources: type: "internal_metrics" splunk_hec: - type: "splunk_hec" + type: "splunk_hec" acknowledgements: false - address: "0.0.0.0:8282" + address: "0.0.0.0:8282" ## ## Sinks @@ -19,17 +19,17 @@ sources: sinks: prometheus: - type: "prometheus_exporter" - inputs: [ "internal_metrics" ] + type: "prometheus_exporter" + inputs: ["internal_metrics"] address: "0.0.0.0:9090" splunk_hec_logs: - type: "splunk_hec_logs" - inputs: [ "splunk_hec" ] + type: "splunk_hec_logs" + inputs: ["splunk_hec"] endpoint: "http://localhost:8080" encoding: codec: "json" - token: "abcd1234" + token: "abcd1234" healthcheck: enabled: false acknowledgements: diff --git a/regression/cases/statsd_to_datadog_metrics/experiment.yaml b/regression/cases/statsd_to_datadog_metrics/experiment.yaml new file mode 100644 index 0000000000000..3650e2c1157a2 --- /dev/null +++ b/regression/cases/statsd_to_datadog_metrics/experiment.yaml @@ -0,0 +1,10 @@ +optimization_goal: ingress_throughput + +target: + name: vector + command: /usr/bin/vector + cpu_allotment: 6 + memory_allotment: 8GiB + + environment: + VECTOR_THREADS: 4 diff --git a/regression/cases/statsd_to_datadog_metrics/lading/lading.yaml b/regression/cases/statsd_to_datadog_metrics/lading/lading.yaml new file mode 100644 index 0000000000000..43eda824248c0 --- /dev/null +++ b/regression/cases/statsd_to_datadog_metrics/lading/lading.yaml @@ -0,0 +1,50 @@ +generator: + - tcp: + seed: + [ + 2, + 3, + 5, + 7, + 11, + 13, + 19, + 23, + 29, + 31, + 37, + 41, + 43, + 47, + 53, + 59, + 61, + 67, + 71, + 73, + 79, + 83, + 89, + 97, + 101, + 103, + 107, + 109, + 113, + 127, + 131, + 137 + ] + addr: "0.0.0.0:8125" + variant: + dogstatsd: {} + bytes_per_second: "500 Mb" + maximum_prebuild_cache_size_bytes: "256 Mb" + +blackhole: + - http: + binding_addr: "0.0.0.0:8080" + +target_metrics: + - prometheus: + uri: "http://127.0.0.1:9090/metrics" diff --git a/regression/cases/statsd_to_datadog_metrics/vector/vector.yaml b/regression/cases/statsd_to_datadog_metrics/vector/vector.yaml new file mode 100644 index 0000000000000..990d4d00c349f --- /dev/null +++ b/regression/cases/statsd_to_datadog_metrics/vector/vector.yaml @@ -0,0 +1,25 @@ +data_dir: "/var/lib/vector" + +sources: + internal_metrics: + type: "internal_metrics" + + statsd: + type: "statsd" + address: "0.0.0.0:8125" + mode: "tcp" + +sinks: + prometheus: + type: "prometheus_exporter" + inputs: ["internal_metrics"] + address: "0.0.0.0:9090" + + datadog_metrics: + type: "datadog_metrics" + inputs: ["statsd"] + endpoint: "http://0.0.0.0:8080" + default_api_key: "DEADBEEF" + default_namespace: "vector" + healthcheck: + enabled: false diff --git a/regression/cases/syslog_humio_logs/lading/lading.yaml b/regression/cases/syslog_humio_logs/lading/lading.yaml index 8c8c616ea4488..170cc63205295 100644 --- a/regression/cases/syslog_humio_logs/lading/lading.yaml +++ b/regression/cases/syslog_humio_logs/lading/lading.yaml @@ -1,6 +1,40 @@ generator: - tcp: - seed: [2, 3, 5, 7, 11, 13, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131, 137] + seed: + [ + 2, + 3, + 5, + 7, + 11, + 13, + 19, + 23, + 29, + 31, + 37, + 41, + 43, + 47, + 53, + 59, + 61, + 67, + 71, + 73, + 79, + 83, + 89, + 97, + 101, + 103, + 107, + 109, + 113, + 127, + 131, + 137 + ] addr: "0.0.0.0:8282" variant: "syslog5424" bytes_per_second: "500 Mb" diff --git a/regression/cases/syslog_humio_logs/vector/vector.yaml b/regression/cases/syslog_humio_logs/vector/vector.yaml index 485e2a8911d61..b464dac8c238d 100644 --- a/regression/cases/syslog_humio_logs/vector/vector.yaml +++ b/regression/cases/syslog_humio_logs/vector/vector.yaml @@ -9,10 +9,10 @@ sources: type: "internal_metrics" syslog: - type: "syslog" - address: "0.0.0.0:8282" + type: "syslog" + address: "0.0.0.0:8282" max_length: 1500000 - mode: "tcp" + mode: "tcp" ## ## Sinks @@ -20,16 +20,16 @@ sources: sinks: prometheus: - type: "prometheus_exporter" - inputs: [ "internal_metrics" ] + type: "prometheus_exporter" + inputs: ["internal_metrics"] address: "0.0.0.0:9090" humio_logs: - type: "humio_logs" - inputs: [ "syslog" ] + type: "humio_logs" + inputs: ["syslog"] endpoint: "http://0.0.0.0:8080" encoding: codec: "json" - token: "humio_token" + token: "humio_token" healthcheck: enabled: false diff --git a/regression/cases/syslog_log2metric_humio_metrics/lading/lading.yaml b/regression/cases/syslog_log2metric_humio_metrics/lading/lading.yaml index 8c8c616ea4488..170cc63205295 100644 --- a/regression/cases/syslog_log2metric_humio_metrics/lading/lading.yaml +++ b/regression/cases/syslog_log2metric_humio_metrics/lading/lading.yaml @@ -1,6 +1,40 @@ generator: - tcp: - seed: [2, 3, 5, 7, 11, 13, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131, 137] + seed: + [ + 2, + 3, + 5, + 7, + 11, + 13, + 19, + 23, + 29, + 31, + 37, + 41, + 43, + 47, + 53, + 59, + 61, + 67, + 71, + 73, + 79, + 83, + 89, + 97, + 101, + 103, + 107, + 109, + 113, + 127, + 131, + 137 + ] addr: "0.0.0.0:8282" variant: "syslog5424" bytes_per_second: "500 Mb" diff --git a/regression/cases/syslog_log2metric_humio_metrics/vector/vector.yaml b/regression/cases/syslog_log2metric_humio_metrics/vector/vector.yaml index 780fc384c4ea2..91a652737852e 100644 --- a/regression/cases/syslog_log2metric_humio_metrics/vector/vector.yaml +++ b/regression/cases/syslog_log2metric_humio_metrics/vector/vector.yaml @@ -7,10 +7,10 @@ sources: type: "internal_metrics" syslog: - type: "syslog" - address: "0.0.0.0:8282" + type: "syslog" + address: "0.0.0.0:8282" max_length: 1500000 - mode: "tcp" + mode: "tcp" ## ## Transforms @@ -19,9 +19,9 @@ sources: transforms: log2metric: type: "log_to_metric" - inputs: [ "syslog" ] + inputs: ["syslog"] metrics: - - type: "gauge" + - type: "gauge" field: "procid" tags: hostname: "{{ hostname }}" @@ -34,14 +34,14 @@ transforms: sinks: prometheus: - type: "prometheus_exporter" - inputs: [ "internal_metrics" ] + type: "prometheus_exporter" + inputs: ["internal_metrics"] address: "0.0.0.0:9090" humio_metrics: - type: "humio_metrics" - inputs: [ "log2metric" ] + type: "humio_metrics" + inputs: ["log2metric"] endpoint: "http://0.0.0.0:8080" - token: "humio_token" + token: "humio_token" healthcheck: enabled: false diff --git a/regression/cases/syslog_log2metric_splunk_hec_metrics/lading/lading.yaml b/regression/cases/syslog_log2metric_splunk_hec_metrics/lading/lading.yaml index 8c8c616ea4488..170cc63205295 100644 --- a/regression/cases/syslog_log2metric_splunk_hec_metrics/lading/lading.yaml +++ b/regression/cases/syslog_log2metric_splunk_hec_metrics/lading/lading.yaml @@ -1,6 +1,40 @@ generator: - tcp: - seed: [2, 3, 5, 7, 11, 13, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131, 137] + seed: + [ + 2, + 3, + 5, + 7, + 11, + 13, + 19, + 23, + 29, + 31, + 37, + 41, + 43, + 47, + 53, + 59, + 61, + 67, + 71, + 73, + 79, + 83, + 89, + 97, + 101, + 103, + 107, + 109, + 113, + 127, + 131, + 137 + ] addr: "0.0.0.0:8282" variant: "syslog5424" bytes_per_second: "500 Mb" diff --git a/regression/cases/syslog_log2metric_splunk_hec_metrics/vector/vector.yaml b/regression/cases/syslog_log2metric_splunk_hec_metrics/vector/vector.yaml index 86036373115d4..05a0144d2c1a8 100644 --- a/regression/cases/syslog_log2metric_splunk_hec_metrics/vector/vector.yaml +++ b/regression/cases/syslog_log2metric_splunk_hec_metrics/vector/vector.yaml @@ -9,10 +9,10 @@ sources: type: "internal_metrics" syslog: - type: "syslog" - address: "0.0.0.0:8282" + type: "syslog" + address: "0.0.0.0:8282" max_length: 1500000 - mode: "tcp" + mode: "tcp" ## ## Transforms @@ -21,9 +21,9 @@ sources: transforms: log2metric: type: "log_to_metric" - inputs: [ "syslog" ] + inputs: ["syslog"] metrics: - - type: "gauge" + - type: "gauge" field: "procid" tags: hostname: "{{ hostname }}" @@ -36,14 +36,14 @@ transforms: sinks: prometheus: - type: "prometheus_exporter" - inputs: [ "internal_metrics" ] + type: "prometheus_exporter" + inputs: ["internal_metrics"] address: "0.0.0.0:9090" splunk_hec_metrics: - type: "splunk_hec_metrics" - inputs: [ "log2metric" ] + type: "splunk_hec_metrics" + inputs: ["log2metric"] endpoint: "http://0.0.0.0:8080" - token: "splunk_token" + token: "splunk_token" healthcheck: enabled: false diff --git a/regression/cases/syslog_log2metric_tag_cardinality_limit_blackhole/lading/lading.yaml b/regression/cases/syslog_log2metric_tag_cardinality_limit_blackhole/lading/lading.yaml index e8d45f0936c2c..c054ff9d2fc7f 100644 --- a/regression/cases/syslog_log2metric_tag_cardinality_limit_blackhole/lading/lading.yaml +++ b/regression/cases/syslog_log2metric_tag_cardinality_limit_blackhole/lading/lading.yaml @@ -1,6 +1,40 @@ generator: - tcp: - seed: [2, 3, 5, 7, 11, 13, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131, 137] + seed: + [ + 2, + 3, + 5, + 7, + 11, + 13, + 19, + 23, + 29, + 31, + 37, + 41, + 43, + 47, + 53, + 59, + 61, + 67, + 71, + 73, + 79, + 83, + 89, + 97, + 101, + 103, + 107, + 109, + 113, + 127, + 131, + 137 + ] addr: "0.0.0.0:8282" variant: "syslog5424" bytes_per_second: "500 Mb" diff --git a/regression/cases/syslog_log2metric_tag_cardinality_limit_blackhole/vector/vector.yaml b/regression/cases/syslog_log2metric_tag_cardinality_limit_blackhole/vector/vector.yaml index e3ed776dac8bb..74fac88991b7d 100644 --- a/regression/cases/syslog_log2metric_tag_cardinality_limit_blackhole/vector/vector.yaml +++ b/regression/cases/syslog_log2metric_tag_cardinality_limit_blackhole/vector/vector.yaml @@ -9,10 +9,10 @@ sources: type: "internal_metrics" syslog: - type: "syslog" - address: "0.0.0.0:8282" + type: "syslog" + address: "0.0.0.0:8282" max_length: 1500000 - mode: "tcp" + mode: "tcp" ## ## Transforms @@ -20,16 +20,16 @@ sources: transforms: remap: - type: "remap" - inputs: [ "syslog" ] + type: "remap" + inputs: ["syslog"] source: |- .cardinality_1000 = random_int(0, 1000) log2metric: type: "log_to_metric" - inputs: [ "remap" ] + inputs: ["remap"] metrics: - - type: "counter" + - type: "counter" field: "procid" tags: hostname: "{{ host }}" @@ -39,7 +39,7 @@ transforms: cardinality_control: type: "tag_cardinality_limit" - inputs: [ "log2metric" ] + inputs: ["log2metric"] limit_exceeded_action: "drop_tag" mode: "probabilistic" value_limit: 100 @@ -50,11 +50,11 @@ transforms: sinks: prometheus: - type: "prometheus_exporter" - inputs: [ "internal_metrics" ] + type: "prometheus_exporter" + inputs: ["internal_metrics"] address: "0.0.0.0:9090" blackhole: type: "blackhole" print_interval_secs: 0 - inputs: [ "cardinality_control" ] + inputs: ["cardinality_control"] diff --git a/regression/cases/syslog_loki/lading/lading.yaml b/regression/cases/syslog_loki/lading/lading.yaml index adf4f48e5ef5d..c84225159e11e 100644 --- a/regression/cases/syslog_loki/lading/lading.yaml +++ b/regression/cases/syslog_loki/lading/lading.yaml @@ -1,7 +1,40 @@ generator: - tcp: - seed: [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, - 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131] + seed: + [ + 2, + 3, + 5, + 7, + 11, + 13, + 17, + 19, + 23, + 29, + 31, + 37, + 41, + 43, + 47, + 53, + 59, + 61, + 67, + 71, + 73, + 79, + 83, + 89, + 97, + 101, + 103, + 107, + 109, + 113, + 127, + 131 + ] addr: "0.0.0.0:8282" variant: "syslog5424" bytes_per_second: "500 Mb" diff --git a/regression/cases/syslog_loki/vector/vector.yaml b/regression/cases/syslog_loki/vector/vector.yaml index 778842f59b526..c91513aeec217 100644 --- a/regression/cases/syslog_loki/vector/vector.yaml +++ b/regression/cases/syslog_loki/vector/vector.yaml @@ -9,10 +9,10 @@ sources: type: "internal_metrics" syslog: - type: "syslog" - address: "0.0.0.0:8282" + type: "syslog" + address: "0.0.0.0:8282" max_length: 1500000 - mode: "tcp" + mode: "tcp" ## ## Sinks @@ -20,14 +20,14 @@ sources: sinks: prometheus: - type: "prometheus_exporter" - inputs: [ "internal_metrics" ] + type: "prometheus_exporter" + inputs: ["internal_metrics"] address: "0.0.0.0:9090" loki: - type: "loki" - inputs: [ "syslog" ] - endpoint: "http://0.0.0.0:8080" + type: "loki" + inputs: ["syslog"] + endpoint: "http://0.0.0.0:8080" encoding: codec: "json" out_of_order_action: "accept" diff --git a/regression/cases/syslog_regex_logs2metric_ddmetrics/lading/lading.yaml b/regression/cases/syslog_regex_logs2metric_ddmetrics/lading/lading.yaml index 8c8c616ea4488..170cc63205295 100644 --- a/regression/cases/syslog_regex_logs2metric_ddmetrics/lading/lading.yaml +++ b/regression/cases/syslog_regex_logs2metric_ddmetrics/lading/lading.yaml @@ -1,6 +1,40 @@ generator: - tcp: - seed: [2, 3, 5, 7, 11, 13, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131, 137] + seed: + [ + 2, + 3, + 5, + 7, + 11, + 13, + 19, + 23, + 29, + 31, + 37, + 41, + 43, + 47, + 53, + 59, + 61, + 67, + 71, + 73, + 79, + 83, + 89, + 97, + 101, + 103, + 107, + 109, + 113, + 127, + 131, + 137 + ] addr: "0.0.0.0:8282" variant: "syslog5424" bytes_per_second: "500 Mb" diff --git a/regression/cases/syslog_regex_logs2metric_ddmetrics/vector/vector.yaml b/regression/cases/syslog_regex_logs2metric_ddmetrics/vector/vector.yaml index 5d50c63e57e6e..7b195176258fc 100644 --- a/regression/cases/syslog_regex_logs2metric_ddmetrics/vector/vector.yaml +++ b/regression/cases/syslog_regex_logs2metric_ddmetrics/vector/vector.yaml @@ -9,10 +9,10 @@ sources: type: "internal_metrics" syslog: - type: "syslog" - address: "0.0.0.0:8282" + type: "syslog" + address: "0.0.0.0:8282" max_length: 1500000 - mode: "tcp" + mode: "tcp" ## ## Transforms @@ -20,15 +20,15 @@ sources: transforms: remap: - type: "remap" - inputs: [ "syslog" ] + type: "remap" + inputs: ["syslog"] source: ". |= parse_regex!(.host, r'^(?P[a-z]+)\\.(?P[a-z]+)')" log2metric: type: "log_to_metric" - inputs: [ "remap" ] + inputs: ["remap"] metrics: - - type: "counter" + - type: "counter" field: "procid" tags: hostname: "{{ hostname }}" @@ -41,13 +41,13 @@ transforms: sinks: prometheus: - type: "prometheus_exporter" - inputs: [ "internal_metrics" ] + type: "prometheus_exporter" + inputs: ["internal_metrics"] address: "0.0.0.0:9090" datadog_metrics: - type: "datadog_metrics" - inputs: [ "log2metric" ] - endpoint: "http://0.0.0.0:8080" - default_api_key: "DEADBEEF" + type: "datadog_metrics" + inputs: ["log2metric"] + endpoint: "http://0.0.0.0:8080" + default_api_key: "DEADBEEF" default_namespace: "vector" diff --git a/regression/cases/syslog_splunk_hec_logs/lading/lading.yaml b/regression/cases/syslog_splunk_hec_logs/lading/lading.yaml index 8c8c616ea4488..170cc63205295 100644 --- a/regression/cases/syslog_splunk_hec_logs/lading/lading.yaml +++ b/regression/cases/syslog_splunk_hec_logs/lading/lading.yaml @@ -1,6 +1,40 @@ generator: - tcp: - seed: [2, 3, 5, 7, 11, 13, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131, 137] + seed: + [ + 2, + 3, + 5, + 7, + 11, + 13, + 19, + 23, + 29, + 31, + 37, + 41, + 43, + 47, + 53, + 59, + 61, + 67, + 71, + 73, + 79, + 83, + 89, + 97, + 101, + 103, + 107, + 109, + 113, + 127, + 131, + 137 + ] addr: "0.0.0.0:8282" variant: "syslog5424" bytes_per_second: "500 Mb" diff --git a/regression/cases/syslog_splunk_hec_logs/vector/vector.yaml b/regression/cases/syslog_splunk_hec_logs/vector/vector.yaml index e4d63f4e23b41..8c5874e22b718 100644 --- a/regression/cases/syslog_splunk_hec_logs/vector/vector.yaml +++ b/regression/cases/syslog_splunk_hec_logs/vector/vector.yaml @@ -9,10 +9,10 @@ sources: type: "internal_metrics" syslog: - type: "syslog" - address: "0.0.0.0:8282" + type: "syslog" + address: "0.0.0.0:8282" max_length: 1500000 - mode: "tcp" + mode: "tcp" ## ## Sinks @@ -20,16 +20,16 @@ sources: sinks: prometheus: - type: "prometheus_exporter" - inputs: [ "internal_metrics" ] + type: "prometheus_exporter" + inputs: ["internal_metrics"] address: "0.0.0.0:9090" splunk_hec_logs: - type: "splunk_hec_logs" - inputs: [ "syslog" ] + type: "splunk_hec_logs" + inputs: ["syslog"] endpoint: "http://0.0.0.0:8080" encoding: codec: "json" - token: "abcd1234" + token: "abcd1234" healthcheck: enabled: false diff --git a/regression/config.yaml b/regression/config.yaml index e7a3f5b1e5dab..9b80bae63a965 100644 --- a/regression/config.yaml +++ b/regression/config.yaml @@ -3,7 +3,6 @@ lading: target: - # Link templates for reports. # # Values may be removed to disable corresponding links in reports. diff --git a/rfcs/2020-03-06-1999-api-extensions-for-lua-transform.md b/rfcs/2020-03-06-1999-api-extensions-for-lua-transform.md index 4b8b75c032d66..abec1d261a80b 100644 --- a/rfcs/2020-03-06-1999-api-extensions-for-lua-transform.md +++ b/rfcs/2020-03-06-1999-api-extensions-for-lua-transform.md @@ -28,7 +28,7 @@ This RFC proposes a new API for the `lua` transform. Currently, the [`lua` transform](https://vector.dev/docs/reference/transforms/lua/) has some limitations in its API. In particular, the following features are missing: -* **Nested Fields** +* **Nested Fields** Currently accessing nested fields is possible using the field path notation: @@ -44,7 +44,7 @@ Currently, the [`lua` transform](https://vector.dev/docs/reference/transforms/lu See [#706](https://github.com/vectordotdev/vector/issues/706) and [#1406](https://github.com/vectordotdev/vector/issues/1406). -* **Setup Code** +* **Setup Code** Some scripts require expensive setup steps, for example, loading of modules or invoking shell commands. These steps should not be part of the main transform code. @@ -79,7 +79,7 @@ Currently, the [`lua` transform](https://vector.dev/docs/reference/transforms/lu See [#1864](https://github.com/vectordotdev/vector/issues/1864). -* **Control Flow** +* **Control Flow** It should be possible to define channels for output events, similarly to how it is done in [`swimlanes`](https://vector.dev/docs/reference/transforms/swimlanes/) transform. @@ -136,7 +136,7 @@ This example is a log to metric transform which produces metric events from inco 1. There is an internal counter which is increased on each incoming log event. 2. The log events are discarded. -2. Each 10 seconds the transform produces a metric event with the count of received log events. +3. Each 10 seconds the transform produces a metric event with the count of received log events. 4. Edge cases are handled in the following way: 1. If there are no incoming invents, the metric event with the counter equal to 0 still has to be produced. 2. On Vector's shutdown the transform has to produce the final metric event with the count of received events since the last flush. @@ -645,10 +645,10 @@ The mapping between Vector data types and Lua data types is the following: | Vector Type | Lua Type | Comment | | :----------- | :-------- | :------- | -| [`String`](https://vector.dev/docs/architecture/data-model/log/#strings) | [`string`](https://www.lua.org/pil/2.4.html) || -| [`Integer`](https://vector.dev/docs/architecture/data-model/log/#ints) | [`integer`](https://docs.rs/mlua/0.6.0/mlua/type.Integer.html) || -| [`Float`](https://vector.dev/docs/architecture/data-model/log/#floats) | [`number`](https://docs.rs/mlua/0.6.0/mlua/type.Number.html) || -| [`Boolean`](https://vector.dev/docs/architecture/data-model/log/#booleans) | [`boolean`](https://www.lua.org/pil/2.2.html) || +| [`String`](https://vector.dev/docs/architecture/data-model/log/#strings) | [`string`](https://www.lua.org/pil/2.4.html) | | +| [`Integer`](https://vector.dev/docs/architecture/data-model/log/#ints) | [`integer`](https://docs.rs/mlua/0.6.0/mlua/type.Integer.html) | | +| [`Float`](https://vector.dev/docs/architecture/data-model/log/#floats) | [`number`](https://docs.rs/mlua/0.6.0/mlua/type.Number.html) | | +| [`Boolean`](https://vector.dev/docs/architecture/data-model/log/#booleans) | [`boolean`](https://www.lua.org/pil/2.2.html) | | | [`Timestamp`](https://vector.dev/docs/architecture/data-model/log/#timestamps) | [`userdata`](https://www.lua.org/pil/28.1.html) | There is no dedicated timestamp type in Lua. However, there is a standard library function [`os.date`](https://www.lua.org/manual/5.1/manual.html#pdf-os.date) which returns a table with fields `year`, `month`, `day`, `hour`, `min`, `sec`, and some others. Other standard library functions, such as [`os.time`](https://www.lua.org/manual/5.1/manual.html#pdf-os.time), support tables with these fields as arguments. Because of that, Vector timestamps passed to the transform are represented as `userdata` with the same set of accessible fields. In order to have one-to-one correspondence between Vector timestamps and Lua timestamps, `os.date` function from the standard library is patched to return not a table, but `userdata` with the same set of fields as it usually would return instead. This approach makes it possible to have both compatibility with the standard library functions and a dedicated data type for timestamps. | | [`Null`](https://vector.dev/docs/architecture/data-model/log/#null-values) | empty string | In Lua setting a table field to `nil` means deletion of this field. Furthermore, setting an array element to `nil` leads to deletion of this element. In order to avoid inconsistencies, already present `Null` values are visible represented as empty strings from Lua code, and it is impossible to create a new `Null` value in the user-defined code. | | [`Map`](https://vector.dev/docs/architecture/data-model/log/#maps) | [`userdata`](https://www.lua.org/pil/28.1.html) or [`table`](https://www.lua.org/pil/2.5.html) | Maps which are parts of events passed to the transform from Vector have `userdata` type. User-created maps have `table` type. Both types are converted to Vector's `Map` type when they are emitted from the transform. | diff --git a/rfcs/2020-04-04-2221-kubernetes-integration.md b/rfcs/2020-04-04-2221-kubernetes-integration.md index e95ee2b4a5dbc..b2a06a68be1ad 100644 --- a/rfcs/2020-04-04-2221-kubernetes-integration.md +++ b/rfcs/2020-04-04-2221-kubernetes-integration.md @@ -135,7 +135,7 @@ it has to be deployed on every [`Node`][k8s_docs_node] in your cluster. The following diagram demonstrates how this works: - +Kubernetes deployment topology ### What We'll Accomplish @@ -149,7 +149,7 @@ The following diagram demonstrates how this works: #### Deploy using `kubectl` -1. Configure Vector: +1. Configure Vector: Before we can deploy Vector we must configure. This is done by creating a Kubernetes `ConfigMap`: @@ -173,7 +173,7 @@ The following diagram demonstrates how this works: kubectl create secret generic vector-config --from-file=vector.toml=vector.toml ``` -2. Deploy Vector! +2. Deploy Vector! Now that you have your custom `ConfigMap` ready it's time to deploy Vector. Create a `Namespace` and apply your `ConfigMap` and our recommended @@ -190,20 +190,20 @@ The following diagram demonstrates how this works: #### Deploy using Helm -1. Install [`helm`][helm_install]. +1. Install [`helm`][helm_install]. -2. Add our Helm Chart repo. +2. Add our Helm Chart repo. ```shell helm repo add vector https://charts.vector.dev helm repo update ``` -3. Configure Vector. +3. Configure Vector. TODO: address this when we decide on the helm chart internals. -4. Deploy Vector! +4. Deploy Vector! ```shell kubectl create namespace vector @@ -227,9 +227,9 @@ The following diagram demonstrates how this works: #### Deploy using Kustomize -1. Install [`kustomize`][kustomize]. +1. Install [`kustomize`][kustomize]. -1. Prepare `kustomization.yaml`. +2. Prepare `kustomization.yaml`. Use the same config as in [`kubectl` guide][anchor_tutorial_kubectl]. @@ -243,7 +243,7 @@ The following diagram demonstrates how this works: - vector-configmap.yaml ``` -1. Deploy Vector! +3. Deploy Vector! ```shell kustomize build . | kubectl apply -f - @@ -327,9 +327,7 @@ logs such that they're accessible from the following locations: - `/var/log/containers` - legacy location, kept for backward compatibility with pre `1.14` clusters. -To make our lives easier, here's a [link][k8s_src_build_container_logs_directory] -to the part of the k8s source that's responsible for building the path to the -log file. If we encounter issues, this would be a good starting point to unwrap +To make our lives easier, here's a reference to the [k8s source code responsible for building the container log path][k8s_src_build_container_logs_directory]. If we encounter issues, this would be a good starting point to unwrap the k8s code. #### Log file format @@ -603,7 +601,7 @@ This worth a separate dedicated RFC though. #### Security considerations on deployment configuration Security considerations on deployment configuration are grouped together with -other security-related measures. See [here](#deployment-hardening). +other security-related measures. See the [deployment hardening](#deployment-hardening) section. #### Other notable [`PodSpec`][k8s_api_pod_spec] properties @@ -1059,7 +1057,7 @@ We have a matrix of concerns, we'd like to ensure Vectors works properly with. - OCI (via [CRI-O](https://cri-o.io/) or [containerd](https://containerd.io/)) - [runc](https://github.com/opencontainers/runc) - [runhcs](https://github.com/Microsoft/hcsshim/tree/master/cmd/runhcs) - - see more [here][windows_in_kubernetes] + see [Windows in Kubernetes][windows_in_kubernetes] - [Kata Containers](https://github.com/kata-containers/runtime) - [gVisor](https://github.com/google/gvisor) - [Firecracker](https://github.com/firecracker-microvm/firecracker-containerd) @@ -1508,16 +1506,16 @@ nightly!) the supported Vector versions. ## Prior Art 1. [Filebeat k8s integration] -1. [Fluentbit k8s integration] -1. [Fluentd k8s integration] -1. [LogDNA k8s integration] -1. [Honeycomb integration] -1. [Bonzai logging operator] - This is approach is likely outside of the scope +2. [Fluentbit k8s integration] +3. [Fluentd k8s integration] +4. [LogDNA k8s integration] +5. [Honeycomb integration] +6. [Bonzai logging operator] - This is approach is likely outside of the scope of Vector's initial Kubernetes integration because it focuses more on deployment strategies and topologies. There are likely some very useful and interesting tactics in their approach though. -1. [Influx Helm charts] -1. [Awesome Operators List] - an "awesome list" of operators. +7. [Influx Helm charts] +8. [Awesome Operators List] - an "awesome list" of operators. ## Sales Pitch @@ -1541,7 +1539,7 @@ See [motivation](#motivation). namespaces? We'd just need to configure Vector to exclude this namespace?~~ See the [Origin filtering][anchor_origin_filtering] section. -1. ~~From what I understand, Vector requires the Kubernetes `watch` verb in order +2. ~~From what I understand, Vector requires the Kubernetes `watch` verb in order to receive updates to k8s cluster changes. This is required for the `kubernetes_pod_metadata` transform. Yet, Fluentbit [requires the `get`, `list`, and `watch` verbs][fluentbit_role]. Why don't we require the same?~~ @@ -1550,7 +1548,7 @@ See [motivation](#motivation). complete the implementation. It's really trivial to determine from a set of API calls used. See the [Deployment Hardening](#deployment-hardening) section. -1. What are some of the details that set Vector's Kubernetes integration apart? +3. What are some of the details that set Vector's Kubernetes integration apart? This is for marketing purposes and also helps us "raise the bar". ### From Mike @@ -1559,12 +1557,12 @@ See [motivation](#motivation). we want to test against? Some clusters use `docker`, some use `CRI-O`, [etc][container_runtimes]. Some even use [gVisor] or [Firecracker]. There might be differences in how different container runtimes handle logs. -1. How do we want to approach Helm Chart Repository management. -1. How do we implement liveness, readiness and startup probes? +2. How do we want to approach Helm Chart Repository management. +3. How do we implement liveness, readiness and startup probes? Readiness probe is a tricky one. See [Container probes](#container-probes). -1. Can we populate file at `terminationMessagePath` with some meaningful +4. Can we populate file at `terminationMessagePath` with some meaningful information when we exit or crash? -1. Can we allow passing arbitrary fields from the `Pod` object to the event? +5. Can we allow passing arbitrary fields from the `Pod` object to the event? Currently we only to pass `pod_id`, pod `annotations` and pod `labels`. ## Plan Of Attack diff --git a/rfcs/2020-05-25-2685-dev-workflow-simplification.md b/rfcs/2020-05-25-2685-dev-workflow-simplification.md index b5dfb5f41eed6..ae50463376f76 100644 --- a/rfcs/2020-05-25-2685-dev-workflow-simplification.md +++ b/rfcs/2020-05-25-2685-dev-workflow-simplification.md @@ -1,5 +1,7 @@ # RFC 2685 - 2020-05-28 - Dev Workflow Simplification +> **Archived:** This RFC is kept for historical reference only. The Docker/Podman dev environment it proposed (`make environment`, `ENVIRONMENT=true`, the `timberio/vector-dev` image) is no longer maintained and the supporting Makefile targets, workflow, and image have been removed. For current contributor instructions, see [CONTRIBUTING.md](../CONTRIBUTING.md) and [docs/DEVELOPING.md](../docs/DEVELOPING.md). + Vector's `Makefile` serves a variety of purposes, and this RFC attempts to tame the complexity of common dev tasks, improving contributor and team member experience. It proposes a practical base `environment` container that merges the functionality of our non-integration test containers into one. It then suggests making common dev `make` tasks to rely on the caller environment having all dependencies, done at the same time it suggests adding `make` tasks to run common `make` tasks inside the environment. Finally, it suggests updating documentation to suggest users can use their native toolchains, `docker`. @@ -183,7 +185,7 @@ There are tools like `hab` (from the Habitat project) and `packer` that can be u ## Outstanding Questions - Windows/Mac/FreeBSD builds via `make build` et all will produce native binaries natively, we should be review those docs. -- This RFC does not scope in integration tests beyond letting the `environment` run them. We may find motivation to explore a more **_slick_** solution in the future. +- This RFC does not scope in integration tests beyond letting the `environment` run them. We may find motivation to explore a more _**slick**_ solution in the future. ## Rationale & Alternatives @@ -207,6 +209,6 @@ Alternatives: 2. Get preliminary consensus this is good path forward 3. Add `DOCKER_SOCKET` passing and support integration testing 4. Cross-OS testing -6. Explore handling `make environment-%` commands via wildcard -7. Acceptance testing (Test including new contributor test) -8. Merge preliminary support +5. Explore handling `make environment-%` commands via wildcard +6. Acceptance testing (Test including new contributor test) +7. Merge preliminary support diff --git a/rfcs/2020-05-25-2692-more-usable-logevents.md b/rfcs/2020-05-25-2692-more-usable-logevents.md index 09ab89b35d7e5..601ef610f626a 100644 --- a/rfcs/2020-05-25-2692-more-usable-logevents.md +++ b/rfcs/2020-05-25-2692-more-usable-logevents.md @@ -183,14 +183,14 @@ In the [WASM transform](https://github.com/vectordotdev/vector/pull/2006/files) This RFC ultimately proposes the following steps: 1. Add UX improvements on `LogEvent`, particularly turning JSON into or from `LogEvent`. -1. Refactor the `PathIter` to make `vector::event::Lookup` type. -1. Add UX improvements on `Lookup` , particularly an internal `String` ↔ `Lookup` with an `Into`/`From` that does not do path parsing, as well as a `::from_str(s: String)` that does. (This also enables `"foo.bar".parse::()?`) -1. Refactor all `LogEvent` to accept `Into` values. +2. Refactor the `PathIter` to make `vector::event::Lookup` type. +3. Add UX improvements on `Lookup` , particularly an internal `String` ↔ `Lookup` with an `Into`/`From` that does not do path parsing, as well as a `::from_str(s: String)` that does. (This also enables `"foo.bar".parse::()?`) +4. Refactor all `LogEvent` to accept `Into` values. 1. Remove obsolete functionality like `insert_path` since the new `Lookup` type covers this. 2. Refactor the `keys` function to return an `Iterator` -1. Add an `Entry` style API to `LogEvent`. +5. Add an `Entry` style API to `LogEvent`. 1. Remove functionality rendered obsolete by the Entry API like `try_insert`, moving them to use the new Entry API -1. Provide `iter` and `iter_mut` functions that yield `(Lookup, Value)`. +6. Provide `iter` and `iter_mut` functions that yield `(Lookup, Value)`. 1. Remove the `all_fields` function, moving them to the new iterator. We believe these steps will provide a more ergonomic and consistent API. diff --git a/rfcs/2020-07-28-3642-jmx_rfc.md b/rfcs/2020-07-28-3642-jmx_rfc.md index b872b276574f5..7f0181b65620c 100644 --- a/rfcs/2020-07-28-3642-jmx_rfc.md +++ b/rfcs/2020-07-28-3642-jmx_rfc.md @@ -161,7 +161,7 @@ The query will return these metrics by parsing the query results and converting The type of metric for any metric marked `untyped` is unclear but likely gauges. We'll need to do deeper research to determine: 1. Whether we want to keep them. -1. What type they are if we retain them. +2. What type they are if we retain them. Naming of metrics is determined via: @@ -215,7 +215,7 @@ Additionally, as part of Vector's vision to be the "one tool" for ingesting and 1. Having users run telegraf or Prom node exporter and using Vector's Prometheus source to scrape it. We could not add the source directly to Vector and instead instruct users to run Prometheus' `jmx_exporter` and point Vector at the resulting data. -1. Or we could use something like [jmxtrans](https://github.com/jmxtrans/jmxtrans) and add a Vector OutputWriter. +2. Or we could use something like [jmxtrans](https://github.com/jmxtrans/jmxtrans) and add a Vector OutputWriter. I decided against both of these this as they would be in conflict with one of the listed principles of Vector: diff --git a/rfcs/2020-10-15-3480-file-source-rework.md b/rfcs/2020-10-15-3480-file-source-rework.md index da199d16e5c56..e7becd118c5d1 100644 --- a/rfcs/2020-10-15-3480-file-source-rework.md +++ b/rfcs/2020-10-15-3480-file-source-rework.md @@ -194,10 +194,10 @@ In the pseudocode above, read scheduling is controlled by `sort`, `should_read`, would need to answer the following questions: 1. In what order should I read the available files? -1. Should I finish one file before moving on to the next? -1. Should I back off reads to this file? -1. Should I back off reads to all files (i.e. sleep)? -1. How long should I spend working on a single file? +2. Should I finish one file before moving on to the next? +3. Should I back off reads to this file? +4. Should I back off reads to all files (i.e. sleep)? +5. How long should I spend working on a single file? These questions would then be answered by a combination of configuration, file metadata, and gathered statistics. @@ -226,9 +226,9 @@ out a magic identifier, but also the logic to update our list of watched files based on those identifiers. It needs to answer the following: 1. Given a visible path, does it contain a file I've seen before? -1. If I have seen it before, has it been renamed? -1. If I have seen it before, am I now seeing it in multiple places? -1. If I'm seeing duplicates, how should I choose which to follow? +2. If I have seen it before, has it been renamed? +3. If I have seen it before, am I now seeing it in multiple places? +4. If I'm seeing duplicates, how should I choose which to follow? This logic is mostly in one place in the current implementation, so it should not be terribly difficult to extract it. The use of `Fingerprinter` should @@ -238,9 +238,9 @@ In addition to simply consolidating the logic, we can expand and make the use of `Fingerprinter` more intelligent. We currently have three ways it can work: 1. Checksum (usually reliable but frustrating for small files) -1. Device and inode (simple and works with small files, but doesn't handle edge +2. Device and inode (simple and works with small files, but doesn't handle edge cases well) -1. First line checksum (solid for intended use case but not yet general) +3. First line checksum (solid for intended use case but not yet general) I'd first propose that we drop device and inode fingerprinting and add path-based fingerprinting in its place. This gives users the option to do the @@ -255,8 +255,8 @@ in place, the algorithm can look something like the following: 1. Read up to `max_line_length` bytes from the file starting at `ignored_header_bytes` -1. Return no fingerprint if there is no newline in the returned bytes -1. Otherwise, return the checksum of the bytes up to the first newline +2. Return no fingerprint if there is no newline in the returned bytes +3. Otherwise, return the checksum of the bytes up to the first newline This should give a good balance between usability and flexibility for the default strategy. As we implement it, we should evolve the current @@ -278,11 +278,11 @@ about providing an understandable config UI than designing the right interface. This should be driven by real world use cases. For example: 1. Ignoring existing checkpoints -1. Start at the beginning or end of existing files, optionally taking into +2. Start at the beginning or end of existing files, optionally taking into account factors like mtime -1. Start at the beginning or end of files added while we're watching (this can +3. Start at the beginning or end of files added while we're watching (this can be tricky) -1. Ordering which of the above concerns take precedence +4. Ordering which of the above concerns take precedence I would suggest a config like the following: @@ -327,8 +327,8 @@ reads, but it will require a bit of experimentation before we're able to determine if it's worthwhile. There are a few possible approaches: 1. Dispatch reads to an explicit threadpool -1. Spawn a limited number of blocking tokio tasks -1. Implement something with `iouring` +2. Spawn a limited number of blocking tokio tasks +3. Implement something with `iouring` The first two both introduce the questions of sizing and the ability of the underlying file system to enable concurrent access in a way that actually adds diff --git a/rfcs/2021-01-08-5843-encoding-decoding-format-vector.md b/rfcs/2021-01-08-5843-encoding-decoding-format-vector.md index 9687c50e06992..59bfd92566555 100644 --- a/rfcs/2021-01-08-5843-encoding-decoding-format-vector.md +++ b/rfcs/2021-01-08-5843-encoding-decoding-format-vector.md @@ -248,7 +248,7 @@ Per the data format itself - I haven't found any single conglomerate conclusion ## Drawbacks -The only drawback to adopting protobuf as the encoding format is that Protobufs can be slightly slower than other schema'd data formats and we don't shed the protobuf tooling overhead. +The only drawback to adopting protobuf as the encoding format is that Protobufs can be slightly slower than other schema-based data formats and we don't shed the protobuf tooling overhead. Drawbacks to adopting HTTP/2 are that (like literally any other transport we could pick) it alters our kubernetes deployment handling and if we don't decide to continue to maintain other transports it means needing to appropriately handle the deprecation and removal of our TCP and HTTP/1.1 implementations fairly soon afterwards. diff --git a/rfcs/2021-02-23-6531-performance-testing.md b/rfcs/2021-02-23-6531-performance-testing.md index e60a21b02f577..fbb5205bcb3ed 100644 --- a/rfcs/2021-02-23-6531-performance-testing.md +++ b/rfcs/2021-02-23-6531-performance-testing.md @@ -47,10 +47,10 @@ processes in our work on Vector: 1. Performance is a first-class testing concern for Vector. We will drive our process to identify regressions or opportunities for optimization as close to introduction as possible. -1. Identifying _that_ a regression has happened is often easier than _why_. We +2. Identifying _that_ a regression has happened is often easier than _why_. We will continuously improve Vector’s diagnosis tooling to reduce the time to debug and repair detected issues. -1. Performance regressions will inevitably, unintentionally make their way +3. Performance regressions will inevitably, unintentionally make their way into a release. When this happens we will treat this just like we would a correctness regression, relying on our diagnostic tools and rolling the experiences of repair back into the tooling. diff --git a/rfcs/2021-03-26-6517-end-to-end-acknowledgement.md b/rfcs/2021-03-26-6517-end-to-end-acknowledgement.md index b669c9ebc7c68..a69bdbc719c2f 100644 --- a/rfcs/2021-03-26-6517-end-to-end-acknowledgement.md +++ b/rfcs/2021-03-26-6517-end-to-end-acknowledgement.md @@ -330,39 +330,39 @@ source handles acknowledgements. The above structure provides for several considerations: -1. This the minimum amount of data that can be added to the metadata to +1. This the minimum amount of data that can be added to the metadata to fully support this feature, amounting to a single shared reference as the `Option` is optimized into the `Arc`. -2. The use of `Arc` reference counting for finalization prevents events +2. The use of `Arc` reference counting for finalization prevents events from "escaping" without providing a status indication. -3. If a source does not need or is not configured to require +3. If a source does not need or is not configured to require finalization, it will not contribute to the list of sources and so has no additional event overhead, and no additional allocations when the event is created. -4. Sending the finalization status to the source does not require any +4. Sending the finalization status to the source does not require any lookups or topology traversal. -5. No additional work is required to handle dropped sources due to +5. No additional work is required to handle dropped sources due to topology reconfiguration, other than the expected checking for a closed channel when sending. ## Drawbacks -1. This adds a base size overhead to each event, even for +1. This adds a base size overhead to each event, even for configurations that do not support or require end-to-end acknowledgement. ## Alternatives -1. The set of sources could be stored in a more customary `Vec`. This +1. The set of sources could be stored in a more customary `Vec`. This provides for merging multiple sources with a minimum of code. However, merged events already have other overhead, and it increases the data required for this array by an additional word. -2. The source could be stored as simply the unique identifier +2. The source could be stored as simply the unique identifier string. This requires that all reporting of finalization status proceed through a dictionary lookup instead of simply sending it through a channel, increasing the run-time overhead. diff --git a/rfcs/2021-07-19-8216-multiple-pipelines.md b/rfcs/2021-07-19-8216-multiple-pipelines.md index a628fe1843e08..6176d99ec2272 100644 --- a/rfcs/2021-07-19-8216-multiple-pipelines.md +++ b/rfcs/2021-07-19-8216-multiple-pipelines.md @@ -36,8 +36,8 @@ Large Vector users often require complex Vector topologies to facilitate the col This change will introduce the concept of pipelines to users. A pipeline is defined as: 1. A collection of transforms defined together, outside of the top-level configuration file -1. Able to draw input from and send output to components defined in the top-level configuration, but are isolated from other pipelines -1. Having each contained component's internal metrics tagged with the `id` of the pipeline +2. Able to draw input from and send output to components defined in the top-level configuration, but are isolated from other pipelines +3. Having each contained component's internal metrics tagged with the `id` of the pipeline Pipelines will be loaded from a `pipelines` sub-directory relative to the Vector configuration directory (e.g., `/etc/vector/pipelines`). Therefore, if a user changes the location of the Vector configuration directory they will also change the `pipelines` directory path. They are coupled. diff --git a/rfcs/2021-08-13-8025-internal-tracing.md b/rfcs/2021-08-13-8025-internal-tracing.md index 37dc48afc418b..d86a9f8ce4035 100644 --- a/rfcs/2021-08-13-8025-internal-tracing.md +++ b/rfcs/2021-08-13-8025-internal-tracing.md @@ -81,10 +81,10 @@ The step after that is a bit fuzzy, however. You could jump straight to profiling, but it has a few weaknesses: 1. It's not something that can easily be run in customer environments -1. The focus on CPU time vs wall clock time means it can miss issues like bad +2. The focus on CPU time vs wall clock time means it can miss issues like bad rate limits, waiting on downstream components, slow IO operations or syscalls, etc -1. The output tends to require interpretation by an experienced engineer, and +3. The output tends to require interpretation by an experienced engineer, and doesn't always indicate clearly where time is being spent from a Vector perspective @@ -251,10 +251,10 @@ as follows: corresponding visualizations means it would take more interpretation and external knowledge to derive the same signal. -1. Their aggregated nature would put a limit on the level of detail (e.g. no +2. Their aggregated nature would put a limit on the level of detail (e.g. no file name field on a `read` span from the file source). -1. Collecting timings directly is likely to require more explicit +3. Collecting timings directly is likely to require more explicit instrumentation code than simply adding spans. ### Do nothing diff --git a/rfcs/2021-08-29-8381-vrl-iteration-support.md b/rfcs/2021-08-29-8381-vrl-iteration-support.md index 39ab28cc9f717..020e9be84c66b 100644 --- a/rfcs/2021-08-29-8381-vrl-iteration-support.md +++ b/rfcs/2021-08-29-8381-vrl-iteration-support.md @@ -338,14 +338,14 @@ individual use-cases, this list shows one available solution per use-case. . = map_keys(., recursive: true) -> |key| { trim_start(key, "_") } ``` -11. [add prefix to all keys](https://discord.com/channels/742820443487993987/764187584452493323/883274684576182302) +1. [add prefix to all keys](https://discord.com/channels/742820443487993987/764187584452493323/883274684576182302) ```coffee . = map_keys(., recursive: true) -> |key| { "my_" + key } ``` -12. [parse message using list of Grok patterns until one matches](https://discord.com/channels/742820443487993987/764187584452493323/870353108692271104) +2. [parse message using list of Grok patterns until one matches](https://discord.com/channels/742820443487993987/764187584452493323/870353108692271104) ```coffee patterns = [] @@ -359,7 +359,7 @@ individual use-cases, this list shows one available solution per use-case. } ``` -13. [find match against list of regular expressions](https://discord.com/channels/742820443487993987/764187584452493323/864496206947942400) +3. [find match against list of regular expressions](https://discord.com/channels/742820443487993987/764187584452493323/864496206947942400) ```coffee matched = false @@ -377,13 +377,13 @@ individual use-cases, this list shows one available solution per use-case. matched = any(patterns) -> |pattern| { match(.message, pattern) } ``` -14. [remove prefix from keys](https://discord.com/channels/742820443487993987/764187584452493323/864496206947942400) +4. [remove prefix from keys](https://discord.com/channels/742820443487993987/764187584452493323/864496206947942400) ```coffee . = map_keys(. ,recursive: true) -> |key| { replace(key, "my_prefix_", "") } ``` -15. [run `encode_json` on all top-level object fields](https://discord.com/channels/742820443487993987/746070591097798688/841787442271879209) +5. [run `encode_json` on all top-level object fields](https://discord.com/channels/742820443487993987/746070591097798688/841787442271879209) ```coffee . = map_values(.) -> |value| { @@ -395,7 +395,7 @@ individual use-cases, this list shows one available solution per use-case. } ``` -16. [map key/value pairs to object with ”key” and ”value” fields](https://discord.com/channels/742820443487993987/746070591097798688/832684085771370587) +6. [map key/value pairs to object with ”key” and ”value” fields](https://discord.com/channels/742820443487993987/746070591097798688/832684085771370587) ```coffee . = { "labels": { "key1": "value1", "key2": "value2" } } @@ -405,7 +405,8 @@ individual use-cases, this list shows one available solution per use-case. } .labels = new_labels - ``` + + ```text **NOTE** this is similar to [Jq’s `to_entries` function](https://stedolan.github.io/jq/manual/#to_entries,from_entries,with_entries), @@ -422,7 +423,7 @@ individual use-cases, this list shows one available solution per use-case. . = to_entries(.) ``` -17. [run `parse_json` on multiple strings in array, and emit as multiple +1. [run `parse_json` on multiple strings in array, and emit as multiple events](https://discord.com/channels/742820443487993987/746070591097798688/832257215506415657) ```coffee @@ -432,7 +433,7 @@ individual use-cases, this list shows one available solution per use-case. ``` -18. [convert object to specific string format](https://discord.com/channels/742820443487993987/764187584452493323/824574475495407639) +2. [convert object to specific string format](https://discord.com/channels/742820443487993987/764187584452493323/824574475495407639) ```coffee . = { "key1": "value1", "key2": "value2" } @@ -451,7 +452,7 @@ individual use-cases, this list shows one available solution per use-case. "{" + join(strings, ",") + "}" ``` -19. [re-introduce previous `only_fields` functionality using iteration](https://github.com/vectordotdev/vector/issues/7347) +3. [re-introduce previous `only_fields` functionality using iteration](https://github.com/vectordotdev/vector/issues/7347) ```coffee @@ -471,7 +472,7 @@ individual use-cases, this list shows one available solution per use-case. . = filter(.) -> |key, _| { includes(only_fields, key) } ``` -20. [map complex dynamic object based on conditionals](https://github.com/vectordotdev/vector/discussions/12387#discussioncomment-2639876) +4. [map complex dynamic object based on conditionals](https://github.com/vectordotdev/vector/discussions/12387#discussioncomment-2639876) ```coffee .input = map_values(.input) -> |input| { @@ -507,7 +508,7 @@ individual use-cases, this list shows one available solution per use-case. } ``` -21. merge array of objects into single object +5. merge array of objects into single object ```coffee result = {} diff --git a/rfcs/2021-09-01-8547-accept-metrics-in-datadog-agent-source.md b/rfcs/2021-09-01-8547-accept-metrics-in-datadog-agent-source.md index ac95ef626762e..c0471ed349985 100644 --- a/rfcs/2021-09-01-8547-accept-metrics-in-datadog-agent-source.md +++ b/rfcs/2021-09-01-8547-accept-metrics-in-datadog-agent-source.md @@ -78,7 +78,7 @@ https://vector.mycompany.tld` to forward metrics to a Vector deployment. The current `dd_url` endpoint configuration has a [conditional behavior](https://github.com/DataDog/datadog-agent/blob/main/pkg/config/config.go#L1199-L1201) (also -[here](https://github.com/DataDog/datadog-agent/blob/main/pkg/forwarder/forwarder_health.go#L131-L143)). I.e. if +[in the forwarder health check](https://github.com/DataDog/datadog-agent/blob/main/pkg/forwarder/forwarder_health.go#L131-L143)). I.e. if `dd_url` contains a known pattern (i.e. it has a suffix that matches a Datadog site) some extra hostname manipulation happens. But overall, the following paths are expected to be supported on the host behind `dd_url`: @@ -100,20 +100,20 @@ A few details about the Datadog Agents & [Datadog metrics](https://docs.datadogh [`MetricSample`](https://github.com/DataDog/datadog-agent/blob/main/pkg/metrics/metric_sample.go#L81-L94) and can be of [several types](https://github.com/DataDog/datadog-agent/blob/main/pkg/metrics/metric_sample.go#L20-L31) * Major Agent usecases: - * Metrics are send from corechecks (i.e. go code) - [here](https://github.com/DataDog/datadog-agent/blob/main/pkg/aggregator/sender.go#L227-L252) - * Dogstatsd metrics are converted to the `MetricSample` structure - [here](https://github.com/DataDog/datadog-agent/blob/main/pkg/dogstatsd/enrich.go#L87-L137) However Datadog Agents + * Metrics are send from corechecks (i.e. go code) via the + [aggregator sender](https://github.com/DataDog/datadog-agent/blob/main/pkg/aggregator/sender.go#L227-L252) + * Dogstatsd metrics are converted to the `MetricSample` structure in the + [dogstatsd enrich module](https://github.com/DataDog/datadog-agent/blob/main/pkg/dogstatsd/enrich.go#L87-L137). However Datadog Agents metrics are transformed before being sent, ultimately metrics accounts for two different kind of payload: * The count, gauge and rate series kind of payload, sent to `/api/v1/series` using the [JSON schema officially documented](https://docs.datadoghq.com/api/latest/metrics) with few undocumented [additional fields](https://github.com/DataDog/datadog-agent/blob/main/pkg/metrics/series.go#L45-L57), but this align very well with the existing `datadog_metrics` sinks. -* The sketches kind of payload, sent to `/api/beta/sketches` and serialized as protobuf as shown - [here](https://github.com/DataDog/datadog-agent/blob/main/pkg/serializer/serializer.go#L315-L338) (it ultimately lands - [here](https://github.com/DataDog/datadog-agent/blob/main/pkg/metrics/sketch_series.go#L103-L269)). Public `.proto` - definition can be found - [here](https://github.com/DataDog/agent-payload/blob/master/proto/metrics/agent_payload.proto#L47-L81). +* The sketches kind of payload, sent to `/api/beta/sketches` and serialized as protobuf as shown in the + [serializer](https://github.com/DataDog/datadog-agent/blob/main/pkg/serializer/serializer.go#L315-L338) (it ultimately lands in the + [sketch_series module](https://github.com/DataDog/datadog-agent/blob/main/pkg/metrics/sketch_series.go#L103-L269)). Public `.proto` + definition can be found in the + [agent-payload proto](https://github.com/DataDog/agent-payload/blob/master/proto/metrics/agent_payload.proto#L47-L81). Vector has a nice description of its [metrics data model](https://vector.dev/docs/architecture/data-model/metric/) and a [concise enum for diff --git a/rfcs/2021-10-29-8621-framing-and-codecs-sinks.md b/rfcs/2021-10-29-8621-framing-and-codecs-sinks.md index 4b2c7ace88349..87a28793d93ec 100644 --- a/rfcs/2021-10-29-8621-framing-and-codecs-sinks.md +++ b/rfcs/2021-10-29-8621-framing-and-codecs-sinks.md @@ -94,37 +94,37 @@ Incremental steps to execute this change. These will be converted to issues afte Overview for the current state of sinks regarding encoding: -|sink|encoding config|`.apply_rules`|notes| -|-|-|-|-| -|`aws_cloudwatch_logs`| `EncodingConfig` | βœ”οΈŽ | Enveloped in `rusoto_logs::InputLogEvent`. `Text` reads message_key() -|`aws_kinesis_firehose`| `EncodingConfig` | βœ”οΈŽ | Enveloped in `rusoto_firehose::Record` that serializes to base64. `Text` reads `message_key()` | - -|`aws_kinesis_streams`| `EncodingConfig` | βœ”οΈŽ | Enveloped in `rusoto_kinesis::PutRecordsRequestEntry`. `Text` reads `message_key()` -|`aws_s3`| `EncodingConfig` | βœ”οΈŽ | Uses `util::{RequestBuilder, Encoder, Compressor}`. `Text` reads `message_key()` -|`aws_sqs`| `EncodingConfig` | βœ”οΈŽ | Enveloped in `EncodedEvent`. `Text` reads `message_key()` -|`azure_blob`| `EncodingConfig` | βœ”οΈŽ | Enveloped in `EncodedEvent`. `Text` reads `message_key()` -|`azure_monitor_logs`| `EncodingConfigWithDefault` | βœ”οΈŽ | Serializes to JSON. Enveloped in HTTP request -|`blackhole`| - | - | - -|`clickhouse`| `EncodingConfigWithDefault` | βœ”οΈŽ | Serializes to JSON. Enveloped in HTTP request -|`console`| `EncodingConfig` | βœ”οΈŽ | `Text` reads `message_key()` -|`datadog_logs`| `EncodingConfigFixed` | βœ”οΈŽ | Doesn't provide options to encode the event payload separately from the protocol -|`datadog_events`| - | βœ— | - -|`datadog_archives`| - | βœ— | Uses custom `DatadogArchivesEncoding`, which has a field `inner: StandardEncodings` which is not user-configurable -|`elasticsearch`| `EncodingConfigFixed` | βœ”οΈŽ | Reshapes event internally and implements custom `ElasticsearchEncoder` to serialize for the Elasticsearch protocol -|`file`| `EncodingConfig` | βœ”οΈŽ | `Text` reads `message_key()` -|`gcp`| `EncodingConfig` | βœ”οΈŽ | Enveloped in HTTP request via `sinks::util::request_builder::RequestBuild`. Sets HTTP request header depending on encoding config -|`honeycomb`| - | βœ— | Embeds event as JSON under a `data` key. Enveloped in HTTP request -|`http`| `EncodingConfig` | βœ”οΈŽ | Enveloped in HTTP request. Request-level compression. Sets HTTP request header depending on encoding config -|`humio`| `EncodingConfig` | βœ”οΈŽ | Wrapper, see `splunk_hec` for more information -|`influxdb`| `EncodingConfigWithDefault` | βœ”οΈŽ | Encoding (for the protocol envelope) is done by routing event fields either into a "tags" or "fields" map that are passed into an internal function `influx_line_protocol` -|`kafka`| `EncodingConfig` | βœ”οΈŽ | Uses `Encoder` for `StandardEncodings` in `encode_input`, enveloped in `KafkaRequest` -|`logdna`| `EncodingConfigWithDefault` | βœ”οΈŽ | Builds a message by manually picking fields from the event. Enveloped in HTTP request -|`loki`| `EncodingConfig` | βœ”οΈŽ | Uses reshaping. `Text` reads `message_key()`, `Logfmt` build a key-value string. Sink has config to preprocess event by adding/removing label fields and timestamp. Enveloped in HTTP request -|`nats`| `EncodingConfig` | βœ”οΈŽ | `Text` reads `message_key()` -|`new_relic_logs`| `EncodingConfigWithDefault` | βœ”οΈŽ | Defers to HTTP sink, uses encoding config to reshape only and convert to JSON -|`papertrail`| `EncodingConfig` | βœ”οΈŽ | `Text` reads `message_key()`. Serializes event using syslog and sends buffer via TCP -|`pulsar`| `EncodingConfig` | βœ”οΈŽ | `Text` reads `message_key()`, `Avro` expects another dedicated key for the serialization schema. Serialized buffer is sent to Pulsar producer -|`redis`| `EncodingConfig` | βœ”οΈŽ | `Text` reads `message_key()`. Encoded message is serialized to buffer -|`sematext`| `EncodingConfigFixed` | βœ”οΈŽ | Wrapper, see `elasticsearch` for more information -|`socket`| `EncodingConfig` | βœ”οΈŽ | `Text` reads `message_key()` -|`splunk_hec`| `EncodingConfig` | βœ”οΈŽ | Encoding is used to create a message according to the Splunk HEC protocol. There is no separate control over encoding the payload itself -|`vector`| - | - | - +| sink | encoding config | `.apply_rules` | notes | +| - | - | - | - | +| `aws_cloudwatch_logs` | `EncodingConfig` | βœ”οΈŽ | Enveloped in `rusoto_logs::InputLogEvent`. `Text` reads message_key() | +| `aws_kinesis_firehose` | `EncodingConfig` | βœ”οΈŽ | Enveloped in `rusoto_firehose::Record` that serializes to base64. `Text` reads `message_key()` | +| `aws_kinesis_streams` | `EncodingConfig` | βœ”οΈŽ | Enveloped in `rusoto_kinesis::PutRecordsRequestEntry`. `Text` reads `message_key()` | +| `aws_s3` | `EncodingConfig` | βœ”οΈŽ | Uses `util::{RequestBuilder, Encoder, Compressor}`. `Text` reads `message_key()` | +| `aws_sqs` | `EncodingConfig` | βœ”οΈŽ | Enveloped in `EncodedEvent`. `Text` reads `message_key()` | +| `azure_blob` | `EncodingConfig` | βœ”οΈŽ | Enveloped in `EncodedEvent`. `Text` reads `message_key()` | +| `azure_monitor_logs` | `EncodingConfigWithDefault` | βœ”οΈŽ | Serializes to JSON. Enveloped in HTTP request | +| `blackhole` | - | - | - | +| `clickhouse` | `EncodingConfigWithDefault` | βœ”οΈŽ | Serializes to JSON. Enveloped in HTTP request | +| `console` | `EncodingConfig` | βœ”οΈŽ | `Text` reads `message_key()` | +| `datadog_logs` | `EncodingConfigFixed` | βœ”οΈŽ | Doesn't provide options to encode the event payload separately from the protocol | +| `datadog_events` | - | βœ— | - | +| `datadog_archives` | - | βœ— | Uses custom `DatadogArchivesEncoding`, which has a field `inner: StandardEncodings` which is not user-configurable | +| `elasticsearch` | `EncodingConfigFixed` | βœ”οΈŽ | Reshapes event internally and implements custom `ElasticsearchEncoder` to serialize for the Elasticsearch protocol | +| `file` | `EncodingConfig` | βœ”οΈŽ | `Text` reads `message_key()` | +| `gcp` | `EncodingConfig` | βœ”οΈŽ | Enveloped in HTTP request via `sinks::util::request_builder::RequestBuild`. Sets HTTP request header depending on encoding config | +| `honeycomb` | - | βœ— | Embeds event as JSON under a `data` key. Enveloped in HTTP request | +| `http` | `EncodingConfig` | βœ”οΈŽ | Enveloped in HTTP request. Request-level compression. Sets HTTP request header depending on encoding config | +| `humio` | `EncodingConfig` | βœ”οΈŽ | Wrapper, see `splunk_hec` for more information | +| `influxdb` | `EncodingConfigWithDefault` | βœ”οΈŽ | Encoding (for the protocol envelope) is done by routing event fields either into a "tags" or "fields" map that are passed into an internal function `influx_line_protocol` | +| `kafka` | `EncodingConfig` | βœ”οΈŽ | Uses `Encoder` for `StandardEncodings` in `encode_input`, enveloped in `KafkaRequest` | +| `logdna` | `EncodingConfigWithDefault` | βœ”οΈŽ | Builds a message by manually picking fields from the event. Enveloped in HTTP request | +| `loki` | `EncodingConfig` | βœ”οΈŽ | Uses reshaping. `Text` reads `message_key()`, `Logfmt` build a key-value string. Sink has config to preprocess event by adding/removing label fields and timestamp. Enveloped in HTTP request | +| `nats` | `EncodingConfig` | βœ”οΈŽ | `Text` reads `message_key()` | +| `new_relic_logs` | `EncodingConfigWithDefault` | βœ”οΈŽ | Defers to HTTP sink, uses encoding config to reshape only and convert to JSON | +| `papertrail` | `EncodingConfig` | βœ”οΈŽ | `Text` reads `message_key()`. Serializes event using syslog and sends buffer via TCP | +| `pulsar` | `EncodingConfig` | βœ”οΈŽ | `Text` reads `message_key()`, `Avro` expects another dedicated key for the serialization schema. Serialized buffer is sent to Pulsar producer | +| `redis` | `EncodingConfig` | βœ”οΈŽ | `Text` reads `message_key()`. Encoded message is serialized to buffer | +| `sematext` | `EncodingConfigFixed` | βœ”οΈŽ | Wrapper, see `elasticsearch` for more information | +| `socket` | `EncodingConfig` | βœ”οΈŽ | `Text` reads `message_key()` | +| `splunk_hec` | `EncodingConfig` | βœ”οΈŽ | Encoding is used to create a message according to the Splunk HEC protocol. There is no separate control over encoding the payload itself | +| `vector` | - | - | - | diff --git a/rfcs/2021-11-03-9862-ingest-apm-stats-along-traces-in-dd-agent-source.md b/rfcs/2021-11-03-9862-ingest-apm-stats-along-traces-in-dd-agent-source.md index 0dd290232319a..c34109a576289 100644 --- a/rfcs/2021-11-03-9862-ingest-apm-stats-along-traces-in-dd-agent-source.md +++ b/rfcs/2021-11-03-9862-ingest-apm-stats-along-traces-in-dd-agent-source.md @@ -100,7 +100,7 @@ found in the trace support [RFC][trace-support-pr] and may provide relevant cont ## Cross cutting concerns * Ongoing work on transforms to add `named_outputs` that is laying the ground for the same feature but on `sources`, - [one PR][named-outputs-pr] has already be merged while scheduled work is tracked [here][named-outputs-improvements]. + [one PR][named-outputs-pr] has already be merged while scheduled work is tracked in the [named outputs improvements issue][named-outputs-improvements]. * [Ongoing work on schemas][schema-rfc] will ultimately offer a programatic way of validating required fields and express constrains on incoming event for a given sink. Traces & APM stats are a good fit for that because they will be represented as standard Vector events, but sinks handling thos will expect some mandatory information. diff --git a/rfcs/2021-11-12-9811-vrl-vm.md b/rfcs/2021-11-12-9811-vrl-vm.md index 0cb103fa9cf6a..6f6450cf3903d 100644 --- a/rfcs/2021-11-12-9811-vrl-vm.md +++ b/rfcs/2021-11-12-9811-vrl-vm.md @@ -323,8 +323,8 @@ functionality. The implementation needs to occur in well defined stages to prevent dumping a massive PR that never gets over the line. -- [ ] Submit a PR with spike-level code _roughly_ demonstrating the change for - the VM. [here](https://github.com/vectordotdev/vector/pull/9829) +- [ ] Submit a [PR with spike-level code](https://github.com/vectordotdev/vector/pull/9829) _roughly_ demonstrating the change for + the VM. - [ ] Incorporate (unit and property) tests. - [ ] Document and comment the VM and the functions used to emit OpCodes. - [ ] Develop a safe API surrounding the VM (in particular function calls). diff --git a/rfcs/2021-12-06-10298-kubernetes_logs-rewrite.md b/rfcs/2021-12-06-10298-kubernetes_logs-rewrite.md index d2dccc735fd1d..f66ab9824613a 100644 --- a/rfcs/2021-12-06-10298-kubernetes_logs-rewrite.md +++ b/rfcs/2021-12-06-10298-kubernetes_logs-rewrite.md @@ -126,8 +126,8 @@ enough to not warrant a version split between the old and new code. - Replace contents of `src/kubernetes` with equivalents from `kube` 1. `src/kubernetes/client` replaced with `kube-client` - 1. `src/kubernetes/reflector` and dependencies replaced with `kube-runtime::reflector` - 1. `src/kubernetes/state` updates to minimize in-house code + 2. `src/kubernetes/reflector` and dependencies replaced with `kube-runtime::reflector` + 3. `src/kubernetes/state` updates to minimize in-house code - Ensure unit tests and integration tests show matching behavior before and after rewrite ## Future Improvements diff --git a/rfcs/2022-05-17-11532-chronicle-sink.md b/rfcs/2022-05-17-11532-chronicle-sink.md index 146008b8b9b3a..a65596fdfa3d7 100644 --- a/rfcs/2022-05-17-11532-chronicle-sink.md +++ b/rfcs/2022-05-17-11532-chronicle-sink.md @@ -128,16 +128,16 @@ The encoding of the message (text or json) can be set via an `encoding` field. ##### Schema A UDM message has the following sections. (To keep this RFC succinct, this is -not a comprehensive list, the full list can be found -[here](https://cloud.google.com/chronicle/docs/reference/udm-field-list#udm_event_data_model).) +not a comprehensive list, the full list can be found in the +[UDM event data model reference](https://cloud.google.com/chronicle/docs/reference/udm-field-list#udm_event_data_model).) - *Metadata* Contains metadata about the event. The fields available for metadata can be - found [here](https://cloud.google.com/chronicle/docs/reference/udm-field-list#metadata). + found in the [UDM metadata field reference](https://cloud.google.com/chronicle/docs/reference/udm-field-list#metadata). Mandatory fields are `event_type` and `event_timestamp`. The value of `event_type` has repercussions for which fields are mandatory - in the rest of the event. See [here](https://cloud.google.com/chronicle/docs/unified-data-model/udm-usage#required_and_optional_fields_based_on_event_type) + in the rest of the event. See the [required and optional fields by event type](https://cloud.google.com/chronicle/docs/unified-data-model/udm-usage#required_and_optional_fields_based_on_event_type) for the list. For example, if `event_type == "FILE_COPY"` then `src.file` becomes mandatory. **Currently it is not possible to validate a field based on the value of another field within Vector schema, so this will need to @@ -145,18 +145,18 @@ not a comprehensive list, the full list can be found - *Principal* The principal is the entity that originates the event. This field is mandatory - and can contain any of the fields defined - [here](https://cloud.google.com/chronicle/docs/reference/udm-field-list#noun). + and can contain any of the fields defined in the + [UDM noun field reference](https://cloud.google.com/chronicle/docs/reference/udm-field-list#noun). - *Src* The source entity being acted upon. Depending on the `event_type` this field - is optional. Can contain any of the fields defined - [here](https://cloud.google.com/chronicle/docs/reference/udm-field-list#noun). + is optional. Can contain any of the fields defined in the + [UDM noun field reference](https://cloud.google.com/chronicle/docs/reference/udm-field-list#noun). - *Target* The target entity being acted upon. Depending on the `event_type` this field - is optional. Can contain any of the fields defined - [here](https://cloud.google.com/chronicle/docs/reference/udm-field-list#noun). + is optional. Can contain any of the fields defined in the + [UDM noun field reference](https://cloud.google.com/chronicle/docs/reference/udm-field-list#noun). ### Implementation diff --git a/rfcs/2022-10-12-14742-flexible-metric-tags.md b/rfcs/2022-10-12-14742-flexible-metric-tags.md index ccb0a2340e167..283541491d255 100644 --- a/rfcs/2022-10-12-14742-flexible-metric-tags.md +++ b/rfcs/2022-10-12-14742-flexible-metric-tags.md @@ -127,9 +127,9 @@ supported in the presence of multi-valued tags, which will be selectable with co 1. The individual values are tracked as before. Events are dropped when any one tag's cardinality exceeds the limit, but only the tags that would exceed the limit are dropped. -1. The individual values are tracked as before. Events are dropped when any one tag's cardinality +2. The individual values are tracked as before. Events are dropped when any one tag's cardinality exceeds the limit, and all values of tags that would exceed the limit are dropped. -1. Values of multi-valued tags are combined before tracking. Events are dropped as before and all +3. Values of multi-valued tags are combined before tracking. Events are dropped as before and all values of tags that would exceed the limit are dropped. #### Sinks @@ -215,7 +215,7 @@ The use of an `IndexSet` for the tag value provides us with two useful invariant 1. Only unique values for each tag will be stored, which prevents repeated values from showing up in the output. -1. The values can be retrieved in the order they first appeared, which allows us to trivially +2. The values can be retrieved in the order they first appeared, which allows us to trivially retrieve either the first or last stored value. ## Drawbacks @@ -257,16 +257,16 @@ which will cause problems for users: 1. Unconditionally expose the tags as arrays of values using the existing naming, but still accept assignments using either single values or arrays of values. This will cause breakage to existing scripts that relies on the existing single value tag values. -1. For Lua or VRL scripts, conditionally expose the tags as single values or arrays, as described in +2. For Lua or VRL scripts, conditionally expose the tags as single values or arrays, as described in the proposal, but accept assignments following the native JSON codec scheme. In Lua, this could cause a breaking change where scripts that emit metrics that have the wrong tabs type to be accepted for transmission. In VRL, this would create headaches for type definitions, at best preventing proper validation of programs. -1. Expose the tags as single values using the existing naming, picking some arbitrary value when a +3. Expose the tags as single values using the existing naming, picking some arbitrary value when a tag has multiple values, and set up a secondary tags structure that exposes the arrays. This will lead to all kinds of confusion and conflicts when the same tag is assigned through different variables. -1. Add functions specifically for manipulating tag sets. This continues to make metrics management +4. Add functions specifically for manipulating tag sets. This continues to make metrics management look like a second-class afterthought, and doesn't ease any compatibility problems for existing scripts. @@ -284,8 +284,8 @@ that would support this feature but with different semantics: 1. `Vec` β€” Retains the ordering of tags as they appear, but allows for duplicate values and cannot support both bare tags and multiple values simultaneously. -1. `Vec>` β€” Same as above but supports bare tags and mutiple values simultaneously. -1. `BTreeSet>` β€” Duplicate values are merged but are sorted, likely putting the bare +2. `Vec>` β€” Same as above but supports bare tags and mutiple values simultaneously. +3. `BTreeSet>` β€” Duplicate values are merged but are sorted, likely putting the bare tag first for single-value uses. There are also at least two other container types that could possibly support this use case: diff --git a/rfcs/2022-10-31-15056-tooling-revamp.md b/rfcs/2022-10-31-15056-tooling-revamp.md index 70e83db6bb530..d0660f5361222 100644 --- a/rfcs/2022-10-31-15056-tooling-revamp.md +++ b/rfcs/2022-10-31-15056-tooling-revamp.md @@ -41,7 +41,7 @@ This RFC discusses improving Vector's developer tooling in order to ease mainten - Test failures are difficult to debug, esp. since [this change](https://github.com/vectordotdev/vector/pull/13128) - Windows is essentially unsupported since Make takes a great deal of effort to install and most [scripts](https://github.com/vectordotdev/vector/tree/v0.24.2/scripts) require Bash and utilities like `find` -- Adding tests for new integrations to CI ([here](https://github.com/vectordotdev/vector/blob/v0.24.2/Makefile#L333-L341) and [here](https://github.com/vectordotdev/vector/blob/v0.24.2/.github/workflows/integration-test.yml#L58-L91)) is a manual and occasionally forgotten step (see [outstanding questions](#outstanding-questions)) +- Adding tests for new integrations to CI ([Makefile](https://github.com/vectordotdev/vector/blob/v0.24.2/Makefile#L333-L341) and [integration-test workflow](https://github.com/vectordotdev/vector/blob/v0.24.2/.github/workflows/integration-test.yml#L58-L91)) is a manual and occasionally forgotten step (see [outstanding questions](#outstanding-questions)) - Makefiles and scripts can get messy fast and often are hard to scale well ## Proposal diff --git a/rfcs/2023-05-03-data-volume-metrics.md b/rfcs/2023-05-03-data-volume-metrics.md index 368fe874cd420..ba1b571653122 100644 --- a/rfcs/2023-05-03-data-volume-metrics.md +++ b/rfcs/2023-05-03-data-volume-metrics.md @@ -109,7 +109,7 @@ as the O(1) scan. #### `component_received_event_bytes_total` -This metric is emitted by the framework [here][source_sender], so it looks like +This metric is emitted by the [framework's source sender][source_sender], so it looks like the only change needed is to add the service tag. #### `component_sent_event_bytes_total` diff --git a/rfcs/README.md b/rfcs/README.md index 8c0f2a932f5f7..055819c005902 100644 --- a/rfcs/README.md +++ b/rfcs/README.md @@ -78,8 +78,8 @@ more info navigating your solution. 1. Search GitHub for [previous issues](https://github.com/vectordotdev/vector/issues) and [RFCs](https://github.com/vectordotdev/vector/tree/master/rfcs) on this topic. -1. If an RFC issue does not exist, [open one](https://github.com/vectordotdev/vector/issues/new/choose). -1. Use the issue to obtain consensus that an RFC is necessary. +2. If an RFC issue does not exist, [open one](https://github.com/vectordotdev/vector/issues/new/choose). +3. Use the issue to obtain consensus that an RFC is necessary. - The change might be quickly rejected. - The change might be on our long term roadmap and get deferred. - The change might be blocked by other work. @@ -87,22 +87,22 @@ more info navigating your solution. ### Creating an RFC 1. Create a new branch -1. Copy the [`rfcs/_YYYY-MM-DD-issue#-title.md`](rfcs/_YYYY-MM-DD-issue%23-title.md) template with the appropriate +2. Copy the [`rfcs/_YYYY-MM-DD-issue#-title.md`](rfcs/_YYYY-MM-DD-issue%23-title.md) template with the appropriate name. Be sure to use the issue number you created above. (e.g., `rfcs/2020-02-10-445-internal-observability.md`) -1. Fill in your RFC, pay attention the bullets and guidelines. Do not omit any sections. -1. Work with the Vector team to land on a confident solution. Allocate time for code-level spikes if necessary. -1. Submit your RFC as a pull request and tag reviewers for approval. +3. Fill in your RFC, pay attention the bullets and guidelines. Do not omit any sections. +4. Work with the Vector team to land on a confident solution. Allocate time for code-level spikes if necessary. +5. Submit your RFC as a pull request and tag reviewers for approval. ### Getting an RFC accepted 1. Schedule a "last call" meeting for your RFC. This should be 1 week after opening your pull request. The purpose is to efficiently obtain consensus. -1. At least 3 Vector team members must approve your RFC in the form of pull request approvals. -1. Once approved, self-merge your RFC, or ask a Vector team member to do it for you. +2. At least 3 Vector team members must approve your RFC in the form of pull request approvals. +3. Once approved, self-merge your RFC, or ask a Vector team member to do it for you. ### Implementing an RFC 1. Create issues from the "Plan Of Attack" section. Place them in an epic if necessary. -1. Coordinate with leadership to schedule your work. +2. Coordinate with leadership to schedule your work. ## FAQ diff --git a/scripts/Gemfile b/scripts/Gemfile deleted file mode 100644 index 91a2743ddd3fc..0000000000000 --- a/scripts/Gemfile +++ /dev/null @@ -1,10 +0,0 @@ -ruby '~> 3.1.0' - -# !!! -# Please try not to add more dependencies here. -# We are trying to remove Ruby entirely from this repo. -# !!! - -source 'https://rubygems.org' -gem 'paint', '~> 2.1' # for scripts/release-prepare.rb -gem 'logging', '~> 2.3.1' # for scripts/generate-component-docs.rb diff --git a/scripts/Gemfile.lock b/scripts/Gemfile.lock deleted file mode 100644 index 47f15be10da47..0000000000000 --- a/scripts/Gemfile.lock +++ /dev/null @@ -1,22 +0,0 @@ -GEM - remote: https://rubygems.org/ - specs: - little-plugger (1.1.4) - logging (2.3.1) - little-plugger (~> 1.1) - multi_json (~> 1.14) - multi_json (1.15.0) - paint (2.2.0) - -PLATFORMS - ruby - -DEPENDENCIES - logging (~> 2.3.1) - paint (~> 2.1) - -RUBY VERSION - ruby 3.1.4p223 - -BUNDLED WITH - 2.4.14 diff --git a/scripts/check-events b/scripts/check-events deleted file mode 100755 index 5d4411b601802..0000000000000 --- a/scripts/check-events +++ /dev/null @@ -1,504 +0,0 @@ -#!/usr/bin/env ruby -# coding: utf-8 - -require 'find' - -# These members/tags are common to multiple events -BYTE_SIZE_COUNT = ['byte_size', 'count'] - -# SUFFIX => [MESSAGE, COUNTERS, ADDITIONAL_TAGS] -EVENT_CLASSES = { - 'BytesReceived' => [ - 'Bytes received.', ['received_bytes'], ['byte_size', 'protocol'] - ], - 'EventsReceived' => [ - 'Events received.', ['received_events', 'received_event_bytes'], ['count', 'byte_size'] - ], - 'EventsSent' => [ - 'Events sent.', ['sent_events', 'sent_event_bytes'], ['count', 'byte_size'] - ], - 'BytesSent' => [ - 'Bytes sent.', ['sent_bytes'], ['byte_size', 'protocol'] - ], -} - -METRIC_NAME_EVENTS_DROPPED = 'component_discarded_events_total' -METRIC_NAME_ERROR = 'component_errors_total' - -def hash_array_add(hash, key, item) - arr = hash.fetch(key, Array::new) - arr.append(item) - hash[key] = arr -end - -def is_constant?(name) - name.start_with? '"' and name.end_with? '"' or name.match? /^(.+::)[A-Z0-9_]$/ -end - -def find_line_number(haystack, needle) - idx = haystack.index(needle) - if !idx.nil? - prior = haystack[0,idx] - prior.lines.count - else - nil - end -end - -# A class to hold error reports and common functionality -class Event - attr_accessor :path, :skip_dropped_events, :uses, :skip_duplicate_check, :skip_validity_check, :impl_internal_event, :impl_register_event, :impl_event_handle - attr_reader :name, :reports, :logs - attr_writer :members - - def initialize(name) - @path = nil - @skip_duplicate_check = false - @skip_validity_check = false - @skip_dropped_events = false - @emits_component_events_dropped = false - @name = name - @reports = [] - @members = {} - @counters = {} - @metrics = {} - @logs = [] - @uses = 0 - @impl_internal_event = false - @impl_register_event = false - @impl_event_handle = false - end - - def add_metric(type, name, tags) - @metrics["#{type}:#{name}"] = tags - if type == 'counter' - @counters[name] = tags - end - end - - # Scan for counter names and tags - def scan_metrics(block) - block.scan(/ (counter|gauge|histogram)!\((?:\n\s+)?"([^"]+)",?(.+?)\)[;\n]/ms) \ - do |type, name, tags| - tags = Hash[tags.scan(/"([^"]+)" => (.+?)(?:,|$)/)] - add_metric(type, name, tags) - end - end - - # Scan the registered event macro block - def scan_registered_event(event_fields, handle_fields, data_type, emit_block) - @members = event_fields.scan(/^ *([a-z0-9_]+): *(.+?),$/m) \ - .map { |member, type| [member, type] } - handle_fields.scan(/^ *([a-z0-9_]+): *(.+?) *= *(.+?),$/m) do |name, type, assignment| - self.scan_component_dropped_events(assignment) - # This is a _slightly_ different regex than the above, couldn't figure a way to unify them - assignment.match(/ (counter|gauge|histogram)!\((?:\n\s+)?"([^"]+)"(,.+)?\)/ms) \ - do - |type, name, tags| - tags = tags || '' - tags = Hash[tags.scan(/"([^"]+)" => (.+?)(?:,|$)/)] - add_metric(type, name, tags) - true - end - end - self.scan_logs(emit_block) - end - - def add_log(type, message, parameters) - @logs.append([type, message, parameters]) - end - - # Scan for log outputs and their parameters - def scan_logs(block) - block.scan(/ - (trace|debug|info|warn|error)! # The log type - \(\s*(?:message\s*=\s*)? # Skip any leading "message =" bit - (?:"([^({)][^("]+)"|([^,]+)) # The log message text - ([^;]*?) # Match the parameter list - \)(?:;|\n\s*}) # Normally would end with simply ");", but some are missing the semicolon - /mx) \ - do |type, raw_message, var_message, parameters| - parameters = parameters.scan(/([a-z0-9_]+) *= .|[?%]([a-z0-9_.]+)/) \ - .map { |assignment, simple| assignment or simple } - - message = raw_message.nil? ? var_message : raw_message - - add_log(type, message, parameters) - end - end - - # Scan for the emission of ComponentEventsDropped. - def scan_component_dropped_events(block) - if block.match?(/(emit|register)!\(\s*ComponentEventsDropped\b/) - @emits_component_events_dropped = true - end - end - - # The event signature is used to check for duplicates and is - # composed from the member names and their types, the metric types, - # names, and their tags, and the log messages and parameters. If no - # metrics and no logs are defined for the event, the signature is - # `nil` to skip duplicate checking. - def signature - if @metrics.length == 0 and @logs.length == 0 - nil - else - members = @members.map { |name, type| "#{name}:#{type}" }.sort.join(':') - metrics = @metrics.map do |name, value| - tags = value.keys.sort.join(',') - "#{name}(#{tags})" - end - metrics = metrics.sort.join(';') - logs = @logs.sort.join(';') - "#{members}[#{logs}][#{metrics}]" - end - end - - def valid? - valid_with_handle? self - end - - def valid_with_handle?(handle) - if @uses == 0 - append('Event has no uses.') - end - - EVENT_CLASSES.each do |suffix, (required_message, counters, additional_tags)| - if @name.end_with? suffix - handle.logs.each do |type, message, parameters| - if type != 'trace' - append('Log type MUST be \"trace!\".') - end - if message != required_message - append("Log message MUST be \"#{required_message}\" (is \"#{message}\").") - end - additional_tags.each do |tag_name| - unless parameters.include? tag_name - append("Log MUST contain tag \"#{tag_name}\"") - end - end - end - counters.each do |counter| - counter = "component_#{counter}_total" - counters_must_include_exclude_tags(counter, additional_tags - BYTE_SIZE_COUNT) - end - end - end - - has_error_logs = handle.logs.one? { |type, _, _| type == 'error' } - - is_events_dropped_event = (@name.end_with? 'EventsDropped' or @counters.include? METRIC_NAME_EVENTS_DROPPED) - - # Validate Error events - if (has_error_logs and !is_events_dropped_event) or @name.end_with? 'Error' - - # Name check - append('Error events MUST be named "___Error".') unless @name.end_with? 'Error' - # Outputs an error log - handle.log_level_exactly('error') - # Metric check - counters_must_include_exclude_tags(METRIC_NAME_ERROR, ['error_type', 'stage']) - - # Make sure Error events contain the required parameters - handle.logs.each do |type, message, parameters| - if type == 'error' - ['error_type', 'stage'].each do |parameter| - unless parameters.include? parameter - append("Error log for Error event MUST include parameter \"#{parameter}\".") - end - end - - ['error_code', 'error_type', 'stage'].each do |parameter| - if parameters.include? parameter and !@counters[METRIC_NAME_ERROR].include? parameter - append("Counter \"#{METRIC_NAME_ERROR}\" must include \"#{parameter}\" to match error log.") - end - end - end - end - end - - # TODO remove @skip_dropped_events check logic after DroppedEvents audit is complete - # (https://github.com/vectordotdev/vector/issues/13995) - - # Validate EventsDropped events - if is_events_dropped_event && !@skip_dropped_events - - # Don't run the checks on event structs which themselves emit ComponentEventsDropped, - # as the ComponentEventsDropped event is already checked. - # Instead, verify that component_discarded_events_total is not being over-incremented. - if @emits_component_events_dropped - if @counters.include? METRIC_NAME_EVENTS_DROPPED - append("Event emitting ComponentEventsDropped should not also increment counter `#{METRIC_NAME_EVENTS_DROPPED}`") - end - else - - # Name check - append('EventsDropped events MUST be named "___EventsDropped".') unless @name.end_with? 'EventsDropped' - - # Outputs an error log or debug log. Which level is dependent on the value of the param `intentional`, however - # because implementation can involve passing in the value of the `intentional` bool at compile time, we would need to - # scan all the source code for places that emit this event to determine that. - handle.log_level_one_of(['error', 'debug']) - - # Metric check - counters_must_include_exclude_tags(METRIC_NAME_EVENTS_DROPPED, ['intentional'], ['reason', 'count']) - - # Make sure EventsDropped events contain the required parameters - handle.logs.each do |type, message, parameters| - if type == 'error' - ['count', 'intentional', 'reason'].each do |parameter| - unless parameters.include? parameter - append("Error log for EventsDropped event MUST include parameter \"#{parameter}\".") - end - end - - ['intentional'].each do |parameter| - if parameters.include? parameter and !@counters[METRIC_NAME_EVENTS_DROPPED].include? parameter - append("Counter \"#{METRIC_NAME_EVENTS_DROPPED}\" must include \"#{parameter}\" to match error log.") - end - end - end - end - end - end - - @counters.each do |name, tags| - # Only component_errors_total and component_discarded_events_total metrics are considered - if ['component_errors_total', 'component_discarded_events_total'].include? name - # Make sure defined tags to counters are constants - tags.each do |tag, value| - if tag == 'stage' - if !value.start_with? 'error_stage::' - append("Counter \"#{name}\" tag \"#{tag}\" value must be an \"error_stage\" constant.") - end - elsif tag == 'error_type' - if !value.start_with? 'error_type::' - append("Counter \"#{name}\" tag \"#{tag}\" value must be an \"error_type\" constant.") - end - end - end - end - end - - @reports.empty? - end - - def log_level_one_of(levels) - if @logs.find_index { |type, message, parameters| levels.include? type }.nil? - append("This event MUST log with one of these levels: #{levels}.") - end - end - - def log_level_exactly(level) - log_level_one_of([level]) - end - - def append(report) - @reports.append(report) - end - - private - - def counters_must_include_exclude_tags(name, required_tags, exclude_tags = []) - unless @counters.include? name - append("This event MUST increment counter \"#{name}\".") - else - tags = @counters[name] - required_tags.each do |tag| - unless tags.include? tag - append("Counter \"#{name}\" MUST include tag \"#{tag}\".") - end - end - - exclude_tags.each do |tag| - if tags.include? tag - append("Counter \"#{name}\" MUST NOT include tag \"#{tag}\".") - end - end - end - end - -end - -$all_events = Hash::new { |hash, key| hash[key] = Event::new(key) } - -error_count = 0 - -# Scan sources and build internal structures -Find.find('./src', './lib') do |path| - if path.start_with? './' - path = path[2..] - end - - if path.end_with? '.rs' - text = File.read(path) - - text.scan(/\b(?:emit!?|register!?)\((?:[a-z][a-z0-9_:]+)?([A-Z][A-Za-z0-9]+)/) \ - do |event_name,| - $all_events[event_name].uses += 1 - end - - # Check log message texts for correct formatting. - if path.start_with? 'src/' - reports = [] - - # Try to find all general usage of the various `tracing` macros. - text.scan(/( - (trace|debug|info|warn|error)!\( # Log type. - ([^;]*?) # All parameters to the macro. - \)(?:;|\n\s*}) # Handles usages that lack a trailing semicolon. - )/mx) \ - do |full, type, params| - # Extract each parameter to the macros, which involves handling structured fields and - # string literals. We parse them further below so that we can iterate through them to try - # and determine what the actual log message is, depending on if it's set by using the - # `message` field, or implicitly with a string literal. - # - # We also have some special handling in there for `tracing`-specific "target" and "parent" - # settings which influence how the event is handled when being processed by a subscriber, - # which we don't care about _here_ but need to account for in our pattern to parse things. - params = params.scan(/("(?:[^"\\]++|\\.)*+"|(?:target|parent):\s*[^,]+|(\w+\s*=\s*(?:"(?:[^"\\]++|\\.)*+"|[%?]?[^,]+))|[%?][^,]+)/) \ - .map do |param| - if /^\".*\"$/.match?(param[0].strip) - { "type" => "litstr", "value" => param[0] } - elsif param[0].include? "=" - parts = param[0].split('=', 2).map { |part| part.strip } - { "type" => "named_field", "field" => parts[0], "value" => parts[1] } - else - { "type" => "field", "field" => param[0] } - end - end - - # See if we found a message field. - message_param = params.find { |param| - # Use the first string literal parameter. - param["type"] == "litstr" || - # Or the first named field called `message` that has a value that is a string literal. - (param["type"] == "named_field" && param["field"] == "message" && /^\".*\"$/.match?(param["value"])) - } - - # We further scrutinize the message field, if we believe we found one. This lets us avoid - # scenarios where variable interpolation is being used, since we can't reasonably detect if - # an interpolated variable at the beginning or end of the message is capitalized or has a - # trailing period, respectively. - has_message = !message_param.nil? - message = if has_message then message_param["value"].gsub(/^"|"$/, '') else nil end - is_capitalized = !has_message || (message[0] == "{" || !message.match?(/^[a-zA-Z]/) || message.match?(/^[[:upper:]]/)) - has_trailing_period = !has_message || (message[-1, 1] == "}" || message.match?(/\.$/)) - - match_reports = [] - match_reports.append('Message must start with a capital.') unless is_capitalized - match_reports.append('Message must end with a period.') unless has_trailing_period - unless match_reports.empty? - line_no = find_line_number(text, full) - match_reports.each { |report| reports.push(" #{report} (`#{type}` call on #{path}:#{line_no})") } - end - end - - unless reports.empty? - reports.each { |report| puts report } - error_count += reports.length - end - end - - # TODO remove @skip_dropped_events check logic after DroppedEvents audit is complete - # (https://github.com/vectordotdev/vector/issues/13995) - skip_dropped_events = text.match? /## skip check-dropped-events ##/i - - if (path.start_with? 'src/internal_events/' or path.start_with? 'lib/vector-common/src/internal_event/') - # Scan internal event structs for member names - text.scan(/[\n ]struct (\S+?)(?:<.+?>)?(?: {\n(.+?)\n\s*}|;)\n/m) do |struct_name, members| - event = $all_events[struct_name] - event.path = path - event.skip_dropped_events = skip_dropped_events - if members - members = members.scan(/ ([A-Za-z0-9_]+): +(.+?),/).map { |member, type| [member, type] } - event.members = members.to_h - end - end - - # Scan internal event implementation blocks for logs and metrics - text.scan(/^(\s*)impl(?:<.+?>)? (InternalEvent|RegisterInternalEvent|InternalEventHandle) for ([A-Za-z0-9_]+)(?:<.+?>)? {\n(.+?)\n\1}$/m) \ - do |_space, trait, event_name, block| - event = $all_events[event_name] - event.path = path - - event.skip_duplicate_check = block.match? /## skip check-duplicate-events ##/i - event.skip_validity_check = block.match? /## skip check-validity-events ##/i - - if trait == 'InternalEvent' - # Look-aside internal events that defer their implementation to a registered event. - if ! block.include? 'register(' - event.impl_internal_event = true - event.scan_metrics(block) - event.scan_logs(block) - event.scan_component_dropped_events(block) - end - elsif trait == 'RegisterInternalEvent' - # This is just a dummy name and will cause spurious errors, but it will at least surface - # the issue of using the macro. - event.impl_register_event = event_name - event.append("Do not implement RegisterInternalEvent manually. Use the registered_event! macro instead.") - elsif trait == 'InternalEventHandle' - event.impl_event_handle = true - event.scan_logs(block) - end - end - end - - # Scan for the `registered_event` macro - text.scan(/^(crate::|vector_common::|)registered_event! *[({]\n *([A-Za-z0-9_]+) *({(.*?)})? *=> *{(.+?)}$.*^ *fn emit\(\&self, [a-z0-9_]+: ([A-Za-z0-9_]+)\) {$(.+?)}\n(\);|\})$/m) \ - do |_, event_name, _, event_fields, handle_fields, data_type, emit_block, _| - event = $all_events[event_name] - event.path = path - event.scan_registered_event(event_fields || "", handle_fields, data_type, emit_block) - end - end -end - -$duplicates = Hash::new { |hash, key| hash[key] = [] } - -$all_events.each do |name, event| - # Check for duplicated signatures - if !event.skip_duplicate_check and (event.impl_internal_event or event.impl_event_handle) - signature = event.signature - if signature - $duplicates[event.signature].append(name) - end - end - - # Check events for validity - if !event.skip_validity_check - if event.impl_internal_event - unless event.valid? - puts "#{event.path}: Errors in event #{event.name}:" - event.reports.each { |report| puts " #{report}" } - error_count += 1 - end - elsif event.impl_register_event - handle = $all_events[event.impl_register_event] - if handle - unless event.valid_with_handle? handle - puts "#{event.path}: Errors in event #{event.name}:" - event.reports.each { |report| puts " #{report}" } - error_count += 1 - end - else - puts "Registered event #{event.name} references nonexistent handle #{event.impl_register_event}" - error_count += 1 - next - end - end - end -end - -$duplicates.each do |signature, dupes| - if dupes.length > 1 - dupes = dupes.join(', ') - puts "Duplicate events detected: #{dupes}" - error_count += 1 - end -end - -puts "#{error_count} error(s)" -exit 1 if error_count > 0 diff --git a/scripts/check_changelog_fragments.sh b/scripts/check_changelog_fragments.sh index 916ebe88dd53b..555f802866dbc 100755 --- a/scripts/check_changelog_fragments.sh +++ b/scripts/check_changelog_fragments.sh @@ -8,7 +8,7 @@ CHANGELOG_DIR="changelog.d" # NOTE: If these are altered, update both the 'changelog.d/README.md' and -# 'scripts/generate-release-cue.rb' accordingly. +# 'vdev/src/commands/release/generate_cue.rs' accordingly. FRAGMENT_TYPES="breaking|security|deprecation|feature|enhancement|fix" if [ ! -d "${CHANGELOG_DIR}" ]; then diff --git a/scripts/cross/Dockerfile b/scripts/cross/Dockerfile index 80a75caba683c..3abe8cd8b2d11 100644 --- a/scripts/cross/Dockerfile +++ b/scripts/cross/Dockerfile @@ -1,13 +1,23 @@ -ARG CROSS_VERSION=0.2.5 ARG TARGET=x86_64-unknown-linux-musl +ARG CROSS_DIGEST -FROM ghcr.io/cross-rs/${TARGET}:${CROSS_VERSION} +FROM ghcr.io/cross-rs/${TARGET}@${CROSS_DIGEST} # Common steps for all targets COPY scripts/cross/bootstrap-ubuntu.sh / COPY scripts/environment/install-protoc.sh / RUN /bootstrap-ubuntu.sh && bash /install-protoc.sh +# Allow cmake to find pre-built dependencies (e.g. curl from curl-sys) that +# land in CMAKE_PREFIX_PATH rather than the cross sysroot. The cross-rs +# toolchain.cmake sets CMAKE_FIND_ROOT_PATH_MODE_{LIBRARY,INCLUDE} to ONLY, +# which makes cmake ignore CMAKE_PREFIX_PATH entries outside the sysroot. +# Switching to BOTH lets FindCURL (and similar modules) locate libraries that +# Cargo crates like curl-sys build from source and expose via CMAKE_PREFIX_PATH. +RUN if [ -f /opt/toolchain.cmake ]; then \ + printf '\n# Allow finding pre-built cargo dependencies via CMAKE_PREFIX_PATH\nset(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY BOTH)\nset(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE BOTH)\n' >> /opt/toolchain.cmake; \ + fi + # Relocate libstdc++ for musl targets that need it (TODO: investigate if still required) RUN if [ "$TARGET" = "arm-unknown-linux-musleabi" ]; then \ LIBSTDC=/usr/local/arm-linux-musleabi/lib/libstdc++.a; \ diff --git a/scripts/cross/bootstrap-ubuntu.sh b/scripts/cross/bootstrap-ubuntu.sh index 88e208ae2ea56..d89ea85f37ca5 100755 --- a/scripts/cross/bootstrap-ubuntu.sh +++ b/scripts/cross/bootstrap-ubuntu.sh @@ -1,29 +1,26 @@ #!/bin/sh set -o errexit -echo 'Acquire::Retries "5";' > /etc/apt/apt.conf.d/80-retries +export DEBIAN_FRONTEND=noninteractive +export ACCEPT_EULA=Y -apt-get update -apt-get install -y \ - apt-transport-https \ - gnupg \ - wget - -# we need LLVM >= 3.9 for onig_sys/bindgen - -cat <<-EOF > /etc/apt/sources.list.d/llvm.list -deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-9 main -deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-9 main +# Configure apt for speed and efficiency +cat > /etc/apt/apt.conf.d/90-vector-optimizations </dev/null } diff --git a/scripts/environment/Dockerfile b/scripts/environment/Dockerfile deleted file mode 100644 index fa7d72075dae6..0000000000000 --- a/scripts/environment/Dockerfile +++ /dev/null @@ -1,37 +0,0 @@ -FROM docker.io/ubuntu:24.04@sha256:d1e2e92c075e5ca139d51a140fff46f84315c0fdce203eab2807c7e495eff4f9 -ENV DEBIAN_FRONTEND=noninteractive \ - TZ='America/New York' \ - PATH=/root/.cargo/bin:/root/.local/bin/:$PATH \ - LANG=en_US.UTF-8 \ - LANGUAGE=en_US.UTF-8 \ - LC_ALL=en_US.UTF-8 \ - CROSS_DOCKER_IN_DOCKER=true - -# Container junk -RUN echo $TZ > /etc/timezone - -RUN apt-get update && apt-get install -y git - -WORKDIR /git/vectordotdev/vector - -# Setup the env -COPY scripts/environment/*.sh scripts/environment/ -RUN ./scripts/environment/bootstrap-ubuntu-24.04.sh - -# Setup the toolchain -COPY scripts/Gemfile scripts/Gemfile.lock \ - /git/vectordotdev/vector/scripts/ -COPY rust-toolchain.toml \ - /git/vectordotdev/vector/ -RUN ./scripts/environment/prepare.sh && ./scripts/environment/setup-helm.sh - -# Declare volumes -VOLUME /vector -VOLUME /vector/target -VOLUME /root/.cargo -VOLUME /root/.rustup - -# Prepare for use -COPY ./scripts/environment/entrypoint.sh / -ENTRYPOINT [ "/entrypoint.sh" ] -CMD [ "bash" ] diff --git a/scripts/environment/bootstrap-macos.sh b/scripts/environment/bootstrap-macos.sh index 5cf014b31546c..8efff32ac98c5 100755 --- a/scripts/environment/bootstrap-macos.sh +++ b/scripts/environment/bootstrap-macos.sh @@ -2,12 +2,4 @@ set -e -o verbose brew update -brew install ruby@3 coreutils cue-lang/tap/cue protobuf - -gem install bundler - -echo "export PATH=\"/usr/local/opt/ruby/bin:\$PATH\"" >> "$HOME/.bash_profile" - -if [ -n "${CI-}" ] ; then - echo "/usr/local/opt/ruby/bin" >> "$GITHUB_PATH" -fi +brew install coreutils cue-lang/tap/cue protobuf diff --git a/scripts/environment/bootstrap-ubuntu-24.04.sh b/scripts/environment/bootstrap-ubuntu-24.04.sh deleted file mode 100755 index 5e59d383dbc1a..0000000000000 --- a/scripts/environment/bootstrap-ubuntu-24.04.sh +++ /dev/null @@ -1,179 +0,0 @@ -#!/usr/bin/env bash -# Refer to https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2404-Readme.md -# for all runner information such as OS version and installed software. - -set -e -o verbose - -if [ -n "$RUSTFLAGS" ] -then - # shellcheck disable=SC2016 - echo '$RUSTFLAGS MUST NOT be set in CI configs as it overrides settings in `.cargo/config.toml`.' - exit 1 -fi - -export DEBIAN_FRONTEND=noninteractive -export ACCEPT_EULA=Y - -# Configure apt for speed and efficiency -cat > /etc/apt/apt.conf.d/90-vector-optimizations <> "${GITHUB_PATH}" - # we often run into OOM issues in CI due to the low memory vs. CPU ratio on c5 instances - echo "CARGO_BUILD_JOBS=$(($(nproc) /2))" >> "${GITHUB_ENV}" -else - echo "export PATH=\"$HOME/.cargo/bin:\$PATH\"" >> "${HOME}/.bash_profile" -fi - -# Add repositories for Docker and Node.js first, then do a single update -NEED_UPDATE=0 - -if ! [ -x "$(command -v docker)" ]; then - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - add-apt-repository \ - "deb [arch=$(dpkg --print-architecture)] https://download.docker.com/linux/ubuntu \ - xenial \ - stable" - NEED_UPDATE=1 -fi - -if ! [ -x "$(command -v node)" ]; then - curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | apt-key add - - add-apt-repository \ - "deb [arch=$(dpkg --print-architecture)] https://deb.nodesource.com/node_18.x \ - nodistro \ - main" - NEED_UPDATE=1 -fi - -# Only update if we added new repositories -if [ $NEED_UPDATE -eq 1 ]; then - apt-get update --yes -fi - -# Install Docker if needed -if ! [ -x "$(command -v docker)" ]; then - apt-get install --yes docker-ce docker-ce-cli containerd.io - # ubuntu user doesn't exist in scripts/environment/Dockerfile which runs this - usermod --append --groups docker ubuntu || true -fi - -bash scripts/environment/install-protoc.sh - -# Install Node.js if needed -if ! [ -x "$(command -v node)" ]; then - apt-get install --yes nodejs - # enable corepack (enables the yarn and pnpm package managers) - # ref: https://nodejs.org/docs/latest-v18.x/api/corepack.html - corepack enable -fi - -# Apt cleanup -apt-get clean - -# Set up the default "deny all warnings" build flags -CARGO_OVERRIDE_DIR="${HOME}/.cargo" -CARGO_OVERRIDE_CONF="${CARGO_OVERRIDE_DIR}/config.toml" -cat <>"$CARGO_OVERRIDE_CONF" -[target.'cfg(linux)'] -rustflags = [ "-D", "warnings" ] -EOF - -# Install mold, because the system linker wastes a bunch of time. -# -# Notably, we don't install/configure it when we're going to do anything with `cross`, as `cross` takes the Cargo -# configuration from the host system and ships it over... which isn't good when we're overriding the `rustc-wrapper` -# and all of that. -if [ -z "${DISABLE_MOLD:-""}" ] ; then - # We explicitly put `mold-wrapper.so` right beside `mold` itself because it's hard-coded to look in the same directory - # first when trying to load the shared object, so we can dodge having to care about the "right" lib folder to put it in. - TEMP=$(mktemp -d) - MOLD_VERSION=1.2.1 - MOLD_TARGET=mold-${MOLD_VERSION}-$(uname -m)-linux - curl -fsSL "https://github.com/rui314/mold/releases/download/v${MOLD_VERSION}/${MOLD_TARGET}.tar.gz" \ - --output "$TEMP/${MOLD_TARGET}.tar.gz" - tar \ - -xvf "${TEMP}/${MOLD_TARGET}.tar.gz" \ - -C "${TEMP}" - cp "${TEMP}/${MOLD_TARGET}/bin/mold" /usr/bin/mold - cp "${TEMP}/${MOLD_TARGET}/lib/mold/mold-wrapper.so" /usr/bin/mold-wrapper.so - rm -rf "$TEMP" - - # Create our rustc wrapper script that we'll use to actually invoke `rustc` such that `mold` will wrap it and intercept - # anything linking calls to use `mold` instead of `ld`, etc. - CARGO_BIN_DIR="${CARGO_OVERRIDE_DIR}/bin" - mkdir -p "$CARGO_BIN_DIR" - - RUSTC_WRAPPER="${CARGO_BIN_DIR}/wrap-rustc" - cat <"$RUSTC_WRAPPER" -#!/bin/sh -exec mold -run "\$@" -EOF - chmod +x "$RUSTC_WRAPPER" - - # Now configure Cargo to use our rustc wrapper script. - { - echo "[build]" - echo "rustc-wrapper = \"${RUSTC_WRAPPER}\"" - } >> "$CARGO_OVERRIDE_CONF" -fi - -mkdir -p /var/lib/vector -chmod 777 /var/lib/vector diff --git a/scripts/environment/bootstrap-windows-2025.ps1 b/scripts/environment/bootstrap-windows-2025.ps1 index 862efc554a187..7d73eeccbc32d 100644 --- a/scripts/environment/bootstrap-windows-2025.ps1 +++ b/scripts/environment/bootstrap-windows-2025.ps1 @@ -2,29 +2,6 @@ $ErrorActionPreference = "Stop" Set-StrictMode -Version Latest -# Helper function to install choco packages with exponential backoff retry -function Install-ChocoPackage { - param( - [string]$Package, - [int]$MaxRetries = 5 - ) - - for ($attempt = 1; $attempt -le $MaxRetries; $attempt++) { - choco install $Package --execution-timeout=7200 -y - if ($LASTEXITCODE -eq 0) { - return - } - - if ($attempt -lt $MaxRetries) { - $delay = 5 * [math]::Pow(2, $attempt) # Exponential: 10, 20, 40, 80 seconds - Write-Host "choco install $Package failed (attempt $attempt of $MaxRetries). Retrying in $delay seconds..." - Start-Sleep -Seconds $delay - } else { - throw "choco install $Package failed after $MaxRetries attempts" - } - } -} - # Set up our Cargo path so we can do Rust-y things. echo "$HOME\.cargo\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append @@ -34,9 +11,19 @@ if ($env:RELEASE_BUILDER -ne "true") { bash scripts/environment/prepare.sh --modules=rustup } -# Install Chocolatey packages with exponential backoff retry -Install-ChocoPackage "make" -Install-ChocoPackage "protoc" +# Install protoc via the shared cross-platform script. It pins the same version +# used on Linux/macOS and downloads directly from the upstream GitHub release, +# so we avoid the recurring Chocolatey CDN failures. +$ProtocInstallDir = Join-Path $env:RUNNER_TEMP "protoc-bin" +bash scripts/environment/install-protoc.sh "$ProtocInstallDir" +if ($LASTEXITCODE -ne 0) { + throw "install-protoc.sh failed with exit code $LASTEXITCODE" +} +echo "$ProtocInstallDir" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + +# GNU make is already on PATH on the windows-2025 runner image via the +# pre-installed MinGW toolchain at C:\mingw64\bin, so no extra install is +# needed here. # Set a specific override path for libclang. echo "LIBCLANG_PATH=$( (gcm clang).source -replace "clang.exe" )" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append diff --git a/scripts/environment/entrypoint.sh b/scripts/environment/entrypoint.sh deleted file mode 100755 index b97484e991fc6..0000000000000 --- a/scripts/environment/entrypoint.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - -# set HOSTNAME to container id for `cross` -if [ -f /.docker-container-id ]; then - HOSTNAME="$(cat /.docker-container-id)" - export HOSTNAME -fi - -if [ -z "$HOSTNAME" ]; then - echo "Failed to properly set HOSTNAME, cross may not work" - # Fallback if everything else fails - HOSTNAME="vector-environment" - export HOSTNAME -fi - -exec "$@" diff --git a/scripts/environment/install-debian-build-deps.sh b/scripts/environment/install-debian-build-deps.sh new file mode 100755 index 0000000000000..4de7028a4fbb5 --- /dev/null +++ b/scripts/environment/install-debian-build-deps.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Canonical apt-get build dependencies for Vector's Debian-based builder images. + +apt-get update +apt-get install -y --no-install-recommends \ + build-essential \ + clang \ + cmake \ + curl \ + git \ + libclang-dev \ + libsasl2-dev \ + libssl-dev \ + libxxhash-dev \ + mold \ + perl \ + pkg-config \ + unzip \ + zlib1g-dev +rm -rf /var/lib/apt/lists/* diff --git a/scripts/environment/install-protoc.sh b/scripts/environment/install-protoc.sh index 4d1debe6c3251..1e75279a3aec2 100755 --- a/scripts/environment/install-protoc.sh +++ b/scripts/environment/install-protoc.sh @@ -23,13 +23,12 @@ trap 'rm -rf "${TMP_DIR?}"' EXIT get_platform() { local os os=$(uname) - if [[ "${os}" == "Darwin" ]]; then - echo "osx" - elif [[ "${os}" == "Linux" ]]; then - echo "linux" - else - >&2 echo "unsupported os: ${os}" && exit 1 - fi + case "${os}" in + Darwin) echo "osx" ;; + Linux) echo "linux" ;; + MINGW*|MSYS*|CYGWIN*) echo "win64" ;; + *) >&2 echo "unsupported os: ${os}" && exit 1 ;; + esac } get_arch() { @@ -47,20 +46,36 @@ get_arch() { fi } +get_bin_name() { + if [[ "$(get_platform)" == "win64" ]]; then + echo "protoc.exe" + else + echo "protoc" + fi +} + install_protoc() { local version=$1 local install_path=$2 local base_url="https://github.com/protocolbuffers/protobuf/releases/download" local url - url="${base_url}/v${version}/protoc-${version}-$(get_platform)-$(get_arch).zip" + if [[ "$(get_platform)" == "win64" ]]; then + # Windows release assets are named without an explicit arch suffix. + url="${base_url}/v${version}/protoc-${version}-win64.zip" + else + url="${base_url}/v${version}/protoc-${version}-$(get_platform)-$(get_arch).zip" + fi local download_path="${TMP_DIR}/protoc.zip" echo "Downloading ${url}" - curl -fsSL "${url}" -o "${download_path}" + # Stay compatible with the curl shipped by Ubuntu 20.04 focal (7.68.x) used + # in the ghcr.io/cross-rs/* base images: --retry-all-errors was only added + # in curl 7.71.0, so rely on the transport-level retry that --retry covers. + curl --retry 5 --retry-delay 10 -fsSL "${url}" -o "${download_path}" unzip -qq "${download_path}" -d "${TMP_DIR}" - mv -f -v "${TMP_DIR}/bin/protoc" "${install_path}" + mv -f -v "${TMP_DIR}/bin/$(get_bin_name)" "${install_path}" } -install_protoc "21.12" "${INSTALL_PATH}/protoc" +install_protoc "21.12" "${INSTALL_PATH}/$(get_bin_name)" diff --git a/scripts/environment/npm-tools/package-lock.json b/scripts/environment/npm-tools/package-lock.json new file mode 100644 index 0000000000000..6b596203f7eae --- /dev/null +++ b/scripts/environment/npm-tools/package-lock.json @@ -0,0 +1,1277 @@ +{ + "name": "npm-tools", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "dependencies": { + "@datadog/datadog-ci": "5.13.0", + "markdownlint-cli2": "0.22.1", + "prettier": "3.8.1" + } + }, + "node_modules/@datadog/datadog-ci": { + "version": "5.13.0", + "resolved": "https://registry.npmjs.org/@datadog/datadog-ci/-/datadog-ci-5.13.0.tgz", + "integrity": "sha512-QEooWT2OE6J//OV4gka3F8Ca8QkmVGEyKR8zpJVZ6EufJVsNSnBX2V2VE3tzkvndcKRe2/Q9ZCiL9GUiZ4wTLA==", + "license": "Apache-2.0", + "bin": { + "datadog-ci": "dist/bundle.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@nodelib/fs.scandir": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", + "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", + "license": "MIT", + "dependencies": { + "@nodelib/fs.stat": "2.0.5", + "run-parallel": "^1.1.9" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.stat": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", + "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.walk": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", + "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", + "license": "MIT", + "dependencies": { + "@nodelib/fs.scandir": "2.1.5", + "fastq": "^1.6.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@sindresorhus/merge-streams": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/merge-streams/-/merge-streams-4.0.0.tgz", + "integrity": "sha512-tlqY9xq5ukxTUZBmoOp+m61cqwQD5pHJtFY3Mn8CA8ps6yghLH/Hw8UPdqg4OLmFW3IFlcXnQNmo/dh8HzXYIQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@types/debug": { + "version": "4.1.13", + "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.13.tgz", + "integrity": "sha512-KSVgmQmzMwPlmtljOomayoR89W4FynCAi3E8PPs7vmDVPe84hT+vGPKkJfThkmXs0x0jAaa9U8uW8bbfyS2fWw==", + "license": "MIT", + "dependencies": { + "@types/ms": "*" + } + }, + "node_modules/@types/katex": { + "version": "0.16.8", + "resolved": "https://registry.npmjs.org/@types/katex/-/katex-0.16.8.tgz", + "integrity": "sha512-trgaNyfU+Xh2Tc+ABIb44a5AYUpicB3uwirOioeOkNPPbmgRNtcWyDeeFRzjPZENO9Vq8gvVqfhaaXWLlevVwg==", + "license": "MIT" + }, + "node_modules/@types/ms": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@types/ms/-/ms-2.1.0.tgz", + "integrity": "sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==", + "license": "MIT" + }, + "node_modules/@types/unist": { + "version": "2.0.11", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-2.0.11.tgz", + "integrity": "sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==", + "license": "MIT" + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "license": "Python-2.0" + }, + "node_modules/braces": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", + "license": "MIT", + "dependencies": { + "fill-range": "^7.1.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/character-entities": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-2.0.2.tgz", + "integrity": "sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-entities-legacy": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/character-entities-legacy/-/character-entities-legacy-3.0.0.tgz", + "integrity": "sha512-RpPp0asT/6ufRm//AJVwpViZbGM/MkjQFxJccQRHmISF/22NBtsHqAWmL+/pmkPWoIUJdWyeVleTl1wydHATVQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-reference-invalid": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/character-reference-invalid/-/character-reference-invalid-2.0.1.tgz", + "integrity": "sha512-iBZ4F4wRbyORVsu0jPV7gXkOsGYjGHPmAyv+HiHG8gi5PtC9KI2j1+v8/tlibRvjoWX027ypmG/n0HtO5t7unw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/commander": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-8.3.0.tgz", + "integrity": "sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/decode-named-character-reference": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/decode-named-character-reference/-/decode-named-character-reference-1.3.0.tgz", + "integrity": "sha512-GtpQYB283KrPp6nRw50q3U9/VfOutZOe103qlN7BPP6Ad27xYnOIWv4lPzo8HCAL+mMZofJ9KEy30fq6MfaK6Q==", + "license": "MIT", + "dependencies": { + "character-entities": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/devlop": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/devlop/-/devlop-1.1.0.tgz", + "integrity": "sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==", + "license": "MIT", + "dependencies": { + "dequal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/entities": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/fast-glob": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz", + "integrity": "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==", + "license": "MIT", + "dependencies": { + "@nodelib/fs.stat": "^2.0.2", + "@nodelib/fs.walk": "^1.2.3", + "glob-parent": "^5.1.2", + "merge2": "^1.3.0", + "micromatch": "^4.0.8" + }, + "engines": { + "node": ">=8.6.0" + } + }, + "node_modules/fastq": { + "version": "1.20.1", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.1.tgz", + "integrity": "sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw==", + "license": "ISC", + "dependencies": { + "reusify": "^1.0.4" + } + }, + "node_modules/fill-range": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", + "license": "MIT", + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/get-east-asian-width": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/get-east-asian-width/-/get-east-asian-width-1.5.0.tgz", + "integrity": "sha512-CQ+bEO+Tva/qlmw24dCejulK5pMzVnUOFOijVogd3KQs07HnRIgp8TGipvCCRT06xeYEbpbgwaCxglFyiuIcmA==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "license": "ISC", + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/globby": { + "version": "16.2.0", + "resolved": "https://registry.npmjs.org/globby/-/globby-16.2.0.tgz", + "integrity": "sha512-QrJia2qDf5BB/V6HYlDTs0I0lBahyjLzpGQg3KT7FnCdTonAyPy2RtY802m2k4ALx6Dp752f82WsOczEVr3l6Q==", + "license": "MIT", + "dependencies": { + "@sindresorhus/merge-streams": "^4.0.0", + "fast-glob": "^3.3.3", + "ignore": "^7.0.5", + "is-path-inside": "^4.0.0", + "slash": "^5.1.0", + "unicorn-magic": "^0.4.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ignore": { + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.5.tgz", + "integrity": "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/is-alphabetical": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-alphabetical/-/is-alphabetical-2.0.1.tgz", + "integrity": "sha512-FWyyY60MeTNyeSRpkM2Iry0G9hpr7/9kD40mD/cGQEuilcZYS4okz8SN2Q6rLCJ8gbCt6fN+rC+6tMGS99LaxQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/is-alphanumerical": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-alphanumerical/-/is-alphanumerical-2.0.1.tgz", + "integrity": "sha512-hmbYhX/9MUMF5uh7tOXyK/n0ZvWpad5caBA17GsC6vyuCqaWliRG5K1qS9inmUhEMaOBIW7/whAnSwveW/LtZw==", + "license": "MIT", + "dependencies": { + "is-alphabetical": "^2.0.0", + "is-decimal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/is-decimal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-decimal/-/is-decimal-2.0.1.tgz", + "integrity": "sha512-AAB9hiomQs5DXWcRB1rqsxGUstbRroFOPPVAomNk/3XHR5JyEZChOyTWe2oayKnsSsr/kcGqF+z6yuH6HHpN0A==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "license": "MIT", + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-hexadecimal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-hexadecimal/-/is-hexadecimal-2.0.1.tgz", + "integrity": "sha512-DgZQp241c8oO6cA1SbTEWiXeoxV42vlcJxgH+B3hi1AiqqKruZR3ZGF8In3fj4+/y/7rHvlOZLZtgJ/4ttYGZg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "license": "MIT", + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/is-path-inside": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-4.0.0.tgz", + "integrity": "sha512-lJJV/5dYS+RcL8uQdBDW9c9uWFLLBNRyFhnAKXw5tVqLlKZ4RMGZKv+YQ/IA3OhD+RpbJa1LLFM1FQPGyIXvOA==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/js-yaml": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", + "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/jsonc-parser": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-3.3.1.tgz", + "integrity": "sha512-HUgH65KyejrUFPvHFPbqOY0rsFip3Bo5wb4ngvdi1EpCYWUQDC5V+Y7mZws+DLkr4M//zQJoanu1SP+87Dv1oQ==", + "license": "MIT" + }, + "node_modules/jsonpointer": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/jsonpointer/-/jsonpointer-5.0.1.tgz", + "integrity": "sha512-p/nXbhSEcu3pZRdkW1OfJhpsVtW1gd4Wa1fnQc9YLiTfAjn0312eMKimbdIQzuZl9aa9xUGaRlP9T/CJE/ditQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/katex": { + "version": "0.16.44", + "resolved": "https://registry.npmjs.org/katex/-/katex-0.16.44.tgz", + "integrity": "sha512-EkxoDTk8ufHqHlf9QxGwcxeLkWRR3iOuYfRpfORgYfqc8s13bgb+YtRY59NK5ZpRaCwq1kqA6a5lpX8C/eLphQ==", + "funding": [ + "https://opencollective.com/katex", + "https://github.com/sponsors/katex" + ], + "license": "MIT", + "dependencies": { + "commander": "^8.3.0" + }, + "bin": { + "katex": "cli.js" + } + }, + "node_modules/linkify-it": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-5.0.0.tgz", + "integrity": "sha512-5aHCbzQRADcdP+ATqnDuhhJ/MRIqDkZX5pyjFHRRysS8vZ5AbqGEoFIb6pYHPZ+L/OC2Lc+xT8uHVVR5CAK/wQ==", + "license": "MIT", + "dependencies": { + "uc.micro": "^2.0.0" + } + }, + "node_modules/markdown-it": { + "version": "14.1.1", + "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-14.1.1.tgz", + "integrity": "sha512-BuU2qnTti9YKgK5N+IeMubp14ZUKUUw7yeJbkjtosvHiP0AZ5c8IAgEMk79D0eC8F23r4Ac/q8cAIFdm2FtyoA==", + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1", + "entities": "^4.4.0", + "linkify-it": "^5.0.0", + "mdurl": "^2.0.0", + "punycode.js": "^2.3.1", + "uc.micro": "^2.1.0" + }, + "bin": { + "markdown-it": "bin/markdown-it.mjs" + } + }, + "node_modules/markdownlint": { + "version": "0.40.0", + "resolved": "https://registry.npmjs.org/markdownlint/-/markdownlint-0.40.0.tgz", + "integrity": "sha512-UKybllYNheWac61Ia7T6fzuQNDZimFIpCg2w6hHjgV1Qu0w1TV0LlSgryUGzM0bkKQCBhy2FDhEELB73Kb0kAg==", + "license": "MIT", + "dependencies": { + "micromark": "4.0.2", + "micromark-core-commonmark": "2.0.3", + "micromark-extension-directive": "4.0.0", + "micromark-extension-gfm-autolink-literal": "2.1.0", + "micromark-extension-gfm-footnote": "2.1.0", + "micromark-extension-gfm-table": "2.1.1", + "micromark-extension-math": "3.1.0", + "micromark-util-types": "2.0.2", + "string-width": "8.1.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/DavidAnson" + } + }, + "node_modules/markdownlint-cli2": { + "version": "0.22.1", + "resolved": "https://registry.npmjs.org/markdownlint-cli2/-/markdownlint-cli2-0.22.1.tgz", + "integrity": "sha512-X14ZbytybDCXAViDmtN4DKLt9ZTrRn+oOrxTYlg3a65jS6QcYYbAkGPh/En2L/GDNbFYJ6lKaQSUNrrbN1bPrw==", + "license": "MIT", + "dependencies": { + "globby": "16.2.0", + "js-yaml": "4.1.1", + "jsonc-parser": "3.3.1", + "jsonpointer": "5.0.1", + "markdown-it": "14.1.1", + "markdownlint": "0.40.0", + "markdownlint-cli2-formatter-default": "0.0.6", + "micromatch": "4.0.8", + "smol-toml": "1.6.1" + }, + "bin": { + "markdownlint-cli2": "markdownlint-cli2-bin.mjs" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/DavidAnson" + } + }, + "node_modules/markdownlint-cli2-formatter-default": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/markdownlint-cli2-formatter-default/-/markdownlint-cli2-formatter-default-0.0.6.tgz", + "integrity": "sha512-VVDGKsq9sgzu378swJ0fcHfSicUnMxnL8gnLm/Q4J/xsNJ4e5bA6lvAz7PCzIl0/No0lHyaWdqVD2jotxOSFMQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/DavidAnson" + }, + "peerDependencies": { + "markdownlint-cli2": ">=0.0.4" + } + }, + "node_modules/markdownlint/node_modules/ansi-regex": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", + "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/markdownlint/node_modules/string-width": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-8.1.0.tgz", + "integrity": "sha512-Kxl3KJGb/gxkaUMOjRsQ8IrXiGW75O4E3RPjFIINOVH8AMl2SQ/yWdTzWwF3FevIX9LcMAjJW+GRwAlAbTSXdg==", + "license": "MIT", + "dependencies": { + "get-east-asian-width": "^1.3.0", + "strip-ansi": "^7.1.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/markdownlint/node_modules/strip-ansi": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz", + "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^6.2.2" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/mdurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-2.0.0.tgz", + "integrity": "sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w==", + "license": "MIT" + }, + "node_modules/merge2": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", + "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/micromark": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/micromark/-/micromark-4.0.2.tgz", + "integrity": "sha512-zpe98Q6kvavpCr1NPVSCMebCKfD7CA2NqZ+rykeNhONIJBpc1tFKt9hucLGwha3jNTNI8lHpctWJWoimVF4PfA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "@types/debug": "^4.0.0", + "debug": "^4.0.0", + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "micromark-core-commonmark": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-combine-extensions": "^2.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-encode": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "micromark-util-subtokenize": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-core-commonmark": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/micromark-core-commonmark/-/micromark-core-commonmark-2.0.3.tgz", + "integrity": "sha512-RDBrHEMSxVFLg6xvnXmb1Ayr2WzLAWjeSATAoxwKYJV94TeNavgoIdA0a9ytzDSVzBy2YKFK+emCPOEibLeCrg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "micromark-factory-destination": "^2.0.0", + "micromark-factory-label": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-factory-title": "^2.0.0", + "micromark-factory-whitespace": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-classify-character": "^2.0.0", + "micromark-util-html-tag-name": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-subtokenize": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-extension-directive": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/micromark-extension-directive/-/micromark-extension-directive-4.0.0.tgz", + "integrity": "sha512-/C2nqVmXXmiseSSuCdItCMho7ybwwop6RrrRPk0KbOHW21JKoCldC+8rFOaundDoRBUWBnJJcxeA/Kvi34WQXg==", + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-factory-whitespace": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0", + "parse-entities": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-autolink-literal": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-autolink-literal/-/micromark-extension-gfm-autolink-literal-2.1.0.tgz", + "integrity": "sha512-oOg7knzhicgQ3t4QCjCWgTmfNhvQbDDnJeVu9v81r7NltNCVmhPy1fJRX27pISafdjL+SVc4d3l48Gb6pbRypw==", + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-footnote": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-footnote/-/micromark-extension-gfm-footnote-2.1.0.tgz", + "integrity": "sha512-/yPhxI1ntnDNsiHtzLKYnE3vf9JZ6cAisqVDauhp4CEHxlb4uoOTxOCJ+9s51bIB8U1N1FJ1RXOKTIlD5B/gqw==", + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-core-commonmark": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-table": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-table/-/micromark-extension-gfm-table-2.1.1.tgz", + "integrity": "sha512-t2OU/dXXioARrC6yWfJ4hqB7rct14e8f7m0cbI5hUmDyyIlwv5vEtooptH8INkbLzOatzKuVbQmAYcbWoyz6Dg==", + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-math": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-math/-/micromark-extension-math-3.1.0.tgz", + "integrity": "sha512-lvEqd+fHjATVs+2v/8kg9i5Q0AP2k85H0WUOwpIVvUML8BapsMvh1XAogmQjOCsLpoKRCVQqEkQBB3NhVBcsOg==", + "license": "MIT", + "dependencies": { + "@types/katex": "^0.16.0", + "devlop": "^1.0.0", + "katex": "^0.16.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-factory-destination": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-destination/-/micromark-factory-destination-2.0.1.tgz", + "integrity": "sha512-Xe6rDdJlkmbFRExpTOmRj9N3MaWmbAgdpSrBQvCFqhezUn4AHqJHbaEnfbVYYiexVSs//tqOdY/DxhjdCiJnIA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-label": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-label/-/micromark-factory-label-2.0.1.tgz", + "integrity": "sha512-VFMekyQExqIW7xIChcXn4ok29YE3rnuyveW3wZQWWqF4Nv9Wk5rgJ99KzPvHjkmPXF93FXIbBp6YdW3t71/7Vg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-space": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-space/-/micromark-factory-space-2.0.1.tgz", + "integrity": "sha512-zRkxjtBxxLd2Sc0d+fbnEunsTj46SWXgXciZmHq0kDYGnck/ZSGj9/wULTV95uoeYiK5hRXP2mJ98Uo4cq/LQg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-title": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-title/-/micromark-factory-title-2.0.1.tgz", + "integrity": "sha512-5bZ+3CjhAd9eChYTHsjy6TGxpOFSKgKKJPJxr293jTbfry2KDoWkhBb6TcPVB4NmzaPhMs1Frm9AZH7OD4Cjzw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-whitespace": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-whitespace/-/micromark-factory-whitespace-2.0.1.tgz", + "integrity": "sha512-Ob0nuZ3PKt/n0hORHyvoD9uZhr+Za8sFoP+OnMcnWK5lngSzALgQYKMr9RJVOWLqQYuyn6ulqGWSXdwf6F80lQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-character": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/micromark-util-character/-/micromark-util-character-2.1.1.tgz", + "integrity": "sha512-wv8tdUTJ3thSFFFJKtpYKOYiGP2+v96Hvk4Tu8KpCAsTMs6yi+nVmGh1syvSCsaxz45J6Jbw+9DD6g97+NV67Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-chunked": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-chunked/-/micromark-util-chunked-2.0.1.tgz", + "integrity": "sha512-QUNFEOPELfmvv+4xiNg2sRYeS/P84pTW0TCgP5zc9FpXetHY0ab7SxKyAQCNCc1eK0459uoLI1y5oO5Vc1dbhA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-classify-character": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-classify-character/-/micromark-util-classify-character-2.0.1.tgz", + "integrity": "sha512-K0kHzM6afW/MbeWYWLjoHQv1sgg2Q9EccHEDzSkxiP/EaagNzCm7T/WMKZ3rjMbvIpvBiZgwR3dKMygtA4mG1Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-combine-extensions": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-combine-extensions/-/micromark-util-combine-extensions-2.0.1.tgz", + "integrity": "sha512-OnAnH8Ujmy59JcyZw8JSbK9cGpdVY44NKgSM7E9Eh7DiLS2E9RNQf0dONaGDzEG9yjEl5hcqeIsj4hfRkLH/Bg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-chunked": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-decode-numeric-character-reference": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/micromark-util-decode-numeric-character-reference/-/micromark-util-decode-numeric-character-reference-2.0.2.tgz", + "integrity": "sha512-ccUbYk6CwVdkmCQMyr64dXz42EfHGkPQlBj5p7YVGzq8I7CtjXZJrubAYezf7Rp+bjPseiROqe7G6foFd+lEuw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-encode": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-encode/-/micromark-util-encode-2.0.1.tgz", + "integrity": "sha512-c3cVx2y4KqUnwopcO9b/SCdo2O67LwJJ/UyqGfbigahfegL9myoEFoDYZgkT7f36T0bLrM9hZTAaAyH+PCAXjw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-html-tag-name": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-html-tag-name/-/micromark-util-html-tag-name-2.0.1.tgz", + "integrity": "sha512-2cNEiYDhCWKI+Gs9T0Tiysk136SnR13hhO8yW6BGNyhOC4qYFnwF1nKfD3HFAIXA5c45RrIG1ub11GiXeYd1xA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-normalize-identifier": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-normalize-identifier/-/micromark-util-normalize-identifier-2.0.1.tgz", + "integrity": "sha512-sxPqmo70LyARJs0w2UclACPUUEqltCkJ6PhKdMIDuJ3gSf/Q+/GIe3WKl0Ijb/GyH9lOpUkRAO2wp0GVkLvS9Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-resolve-all": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-resolve-all/-/micromark-util-resolve-all-2.0.1.tgz", + "integrity": "sha512-VdQyxFWFT2/FGJgwQnJYbe1jjQoNTS4RjglmSjTUlpUMa95Htx9NHeYW4rGDJzbjvCsl9eLjMQwGeElsqmzcHg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-sanitize-uri": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-sanitize-uri/-/micromark-util-sanitize-uri-2.0.1.tgz", + "integrity": "sha512-9N9IomZ/YuGGZZmQec1MbgxtlgougxTodVwDzzEouPKo3qFWvymFHWcnDi2vzV1ff6kas9ucW+o3yzJK9YB1AQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-encode": "^2.0.0", + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-subtokenize": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-util-subtokenize/-/micromark-util-subtokenize-2.1.0.tgz", + "integrity": "sha512-XQLu552iSctvnEcgXw6+Sx75GflAPNED1qx7eBJ+wydBb2KCbRZe+NwvIEEMM83uml1+2WSXpBAcp9IUCgCYWA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-symbol": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-2.0.1.tgz", + "integrity": "sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-types": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/micromark-util-types/-/micromark-util-types-2.0.2.tgz", + "integrity": "sha512-Yw0ECSpJoViF1qTU4DC6NwtC4aWGt1EkzaQB8KPPyCRR8z9TWeV0HbEFGTO+ZY1wB22zmxnJqhPyTpOVCpeHTA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromatch": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", + "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", + "license": "MIT", + "dependencies": { + "braces": "^3.0.3", + "picomatch": "^2.3.1" + }, + "engines": { + "node": ">=8.6" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/parse-entities": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/parse-entities/-/parse-entities-4.0.2.tgz", + "integrity": "sha512-GG2AQYWoLgL877gQIKeRPGO1xF9+eG1ujIb5soS5gPvLQ1y2o8FL90w2QWNdf9I361Mpp7726c+lj3U0qK1uGw==", + "license": "MIT", + "dependencies": { + "@types/unist": "^2.0.0", + "character-entities-legacy": "^3.0.0", + "character-reference-invalid": "^2.0.0", + "decode-named-character-reference": "^1.0.0", + "is-alphanumerical": "^2.0.0", + "is-decimal": "^2.0.0", + "is-hexadecimal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/picomatch": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", + "license": "MIT", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/prettier": { + "version": "3.8.1", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.8.1.tgz", + "integrity": "sha512-UOnG6LftzbdaHZcKoPFtOcCKztrQ57WkHDeRD9t/PTQtmT0NHSeWWepj6pS0z/N7+08BHFDQVUrfmfMRcZwbMg==", + "license": "MIT", + "bin": { + "prettier": "bin/prettier.cjs" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/prettier/prettier?sponsor=1" + } + }, + "node_modules/punycode.js": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode.js/-/punycode.js-2.3.1.tgz", + "integrity": "sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/queue-microtask": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", + "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/reusify": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz", + "integrity": "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==", + "license": "MIT", + "engines": { + "iojs": ">=1.0.0", + "node": ">=0.10.0" + } + }, + "node_modules/run-parallel": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", + "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "queue-microtask": "^1.2.2" + } + }, + "node_modules/slash": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-5.1.0.tgz", + "integrity": "sha512-ZA6oR3T/pEyuqwMgAKT0/hAv8oAXckzbkmR0UkUosQ+Mc4RxGoJkRmwHgHufaenlyAgE1Mxgpdcrf75y6XcnDg==", + "license": "MIT", + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/smol-toml": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.6.1.tgz", + "integrity": "sha512-dWUG8F5sIIARXih1DTaQAX4SsiTXhInKf1buxdY9DIg4ZYPZK5nGM1VRIYmEbDbsHt7USo99xSLFu5Q1IqTmsg==", + "license": "BSD-3-Clause", + "engines": { + "node": ">= 18" + }, + "funding": { + "url": "https://github.com/sponsors/cyyynthia" + } + }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "license": "MIT", + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/uc.micro": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-2.1.0.tgz", + "integrity": "sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==", + "license": "MIT" + }, + "node_modules/unicorn-magic": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/unicorn-magic/-/unicorn-magic-0.4.0.tgz", + "integrity": "sha512-wH590V9VNgYH9g3lH9wWjTrUoKsjLF6sGLjhR4sH1LWpLmCOH0Zf7PukhDA8BiS7KHe4oPNkcTHqYkj7SOGUOw==", + "license": "MIT", + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + } + } +} diff --git a/scripts/environment/npm-tools/package.json b/scripts/environment/npm-tools/package.json new file mode 100644 index 0000000000000..1f74c554bc665 --- /dev/null +++ b/scripts/environment/npm-tools/package.json @@ -0,0 +1,9 @@ +{ + "private": true, + "description": "Pinned npm tools for CI. Run 'npm ci' to install with exact versions from package-lock.json.", + "dependencies": { + "@datadog/datadog-ci": "5.13.0", + "markdownlint-cli2": "0.22.1", + "prettier": "3.8.1" + } +} diff --git a/scripts/environment/prepare.sh b/scripts/environment/prepare.sh index a06f50844c353..41c9d6de5fbb5 100755 --- a/scripts/environment/prepare.sh +++ b/scripts/environment/prepare.sh @@ -35,10 +35,11 @@ CARGO_DENY_VERSION="0.19.0" CARGO_MSRV_VERSION="0.18.4" CARGO_HACK_VERSION="0.6.43" DD_RUST_LICENSE_TOOL_VERSION="1.0.6" +CARGO_LLVM_COV_VERSION="0.8.4" WASM_PACK_VERSION="0.13.1" -MARKDOWNLINT_VERSION="0.45.0" -DATADOG_CI_VERSION="5.9.0" -VDEV_VERSION="0.3.0" +# npm tool versions are defined in scripts/environment/npm-tools/package.json +# and pinned (including transitive deps) in npm-tools/package-lock.json. +VDEV_VERSION="0.3.3" ALL_MODULES=( rustup @@ -48,11 +49,12 @@ ALL_MODULES=( cargo-deny cargo-msrv cargo-hack + cargo-llvm-cov dd-rust-license-tool wasm-pack - markdownlint + markdownlint-cli2 + prettier datadog-ci - release-flags # Not a tool - sources release-flags.sh to set CI env vars vdev ) @@ -87,9 +89,11 @@ Modules: cargo-deny cargo-msrv cargo-hack + cargo-llvm-cov dd-rust-license-tool wasm-pack - markdownlint + markdownlint-cli2 + prettier datadog-ci vdev @@ -135,34 +139,114 @@ maybe_install_cargo_tool() { version_cmd="cargo ${tool#cargo-}" fi + # vdev fails fast on missing prebuilts so a cache/asset miss can't + # reintroduce the source-compile path that previously stalled a release. + local installer=("${install[@]}") + if [[ "$tool" == "vdev" && "${installer[0]}" == "binstall" ]]; then + installer+=(--disable-strategies compile) + fi + if ! $version_cmd --version 2>/dev/null | grep -q "^${version_pattern}"; then - cargo "${install[@]}" "$tool" --version "$version" --force --locked + local should_install=true + # Outside CI, preserve a newer-than-pin version the user already has. + # `cargo install --force` would otherwise silently downgrade them. + if [[ -z "${CI:-}" ]]; then + local current + current=$($version_cmd --version 2>/dev/null | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1 || true) + if [[ -n "$current" ]] && [[ "$current" != "$version" ]]; then + local newest + newest=$(printf '%s\n%s\n' "$current" "$version" | sort -V | tail -1) + if [[ "$newest" == "$current" ]]; then + echo "Keeping ${tool} ${current} (newer than pin ${version}). Set CI=1 to force the pin." + should_install=false + fi + fi + fi + if [[ "$should_install" == "true" ]]; then + cargo "${installer[@]}" "$tool" --version "$version" --force --locked + fi + fi + + # cargo-llvm-cov requires the llvm-tools-preview rustup component + if [[ "$tool" == "cargo-llvm-cov" ]]; then + rustup component add llvm-tools-preview fi } -# Helper for NPM packages -# Usage: maybe_install_npm_package -maybe_install_npm_package() { - local tool="$1" - local package="$2" - local version="$3" - local version_pattern="${4:-$version}" - local version_cmd="${5:---version}" # Default to --version, can override with "version" etc. +# Install npm tools from the committed package-lock.json so that every +# transitive dependency version is pinned (no live registry resolution). +# Versions are defined in npm-tools/package.json; npm ci ensures exact lockfile match. +# Note: npm ci installs all packages in the lockfile even if only one tool +# is requested, since it does not support selective installation. +maybe_install_npm_tools() { + local npm_tools=(markdownlint-cli2 prettier datadog-ci) - if ! contains_module "$tool"; then + # Early return when no npm tool is requested, so hosts without npm + # (e.g. tests/e2e/Dockerfile calling prepare.sh --modules=cargo-nextest) + # are not broken by the npm commands below. + local any_requested=false + for tool in "${npm_tools[@]}"; do + if contains_module "$tool"; then + any_requested=true + break + fi + done + if [[ "$any_requested" == "false" ]]; then return 0 fi - if [[ "$("$tool" "$version_cmd" 2>/dev/null)" != "$version_pattern" ]]; then - sudo npm install -g "${package}@${version}" + local npm_tools_dir="${SCRIPT_DIR}/npm-tools" + local npm_bin_dir + npm_bin_dir="$(npm config get prefix -g)/bin" + local need_install=false + + for tool in "${npm_tools[@]}"; do + if contains_module "$tool"; then + local expected="${npm_tools_dir}/node_modules/.bin/${tool}" + if [[ "$(readlink "${npm_bin_dir}/${tool}" 2>/dev/null)" != "$expected" ]] || [[ ! -x "$expected" ]]; then + need_install=true + break + fi + fi + done + + if [[ "$need_install" == "false" ]]; then + return 0 fi + + npm ci --prefix "${npm_tools_dir}" + + # Outside CI, skip the global symlink to avoid a sudo write to /usr/local/bin + # (or equivalent). The Makefile prepends this directory to PATH, so `make` + # recipes find the tools automatically. + if [[ -z "${CI:-}" ]]; then + echo "npm tools installed under ${npm_tools_dir}/node_modules/.bin" + echo "Make recipes discover them automatically. To invoke directly from a" + echo "shell, add the directory to your PATH:" + echo " export PATH=\"${npm_tools_dir}/node_modules/.bin:\$PATH\"" + return 0 + fi + + # Use sudo only when the target directory is not writable (e.g. /usr/local/bin + # on Linux CI runners is root-owned, but Homebrew dirs on macOS are user-owned). + local ln_cmd=(ln -sf) + if [[ ! -w "${npm_bin_dir}" ]]; then + ln_cmd=(sudo ln -sf) + fi + for tool in "${npm_tools[@]}"; do + "${ln_cmd[@]}" "${npm_tools_dir}/node_modules/.bin/${tool}" "${npm_bin_dir}/${tool}" + done } -# Always ensure git safe.directory is set -git config --global --add safe.directory "$(pwd)" +# Set git safe.directory in CI where the repo may be checked out by a different +# uid than the user running git. Skipped on workstations: the contributor owns +# the checkout and a global config write is unnecessary. +if [[ -n "${CI:-}" ]]; then + git config --global --add safe.directory "$(pwd)" +fi -REQUIRES_RUSTUP=(dd-rust-license-tool cargo-deb cross cargo-nextest cargo-deny cargo-msrv cargo-hack wasm-pack vdev) -REQUIRES_BINSTALL=(cargo-deb cross cargo-nextest cargo-deny cargo-msrv cargo-hack wasm-pack vdev) +REQUIRES_RUSTUP=(dd-rust-license-tool cargo-deb cross cargo-nextest cargo-deny cargo-msrv cargo-hack cargo-llvm-cov wasm-pack vdev) +REQUIRES_BINSTALL=(cargo-deb cross cargo-nextest cargo-deny cargo-msrv cargo-hack cargo-llvm-cov wasm-pack vdev) require_binstall=false for tool in "${REQUIRES_BINSTALL[@]}"; do @@ -184,9 +268,6 @@ fi install=(install) if contains_module rustup; then - # shellcheck source=scripts/environment/release-flags.sh - . "${SCRIPT_DIR}"/release-flags.sh - ensure_active_toolchain_is_installed if [ "${require_binstall}" = "true" ]; then @@ -205,9 +286,9 @@ maybe_install_cargo_tool cargo-nextest "${CARGO_NEXTEST_VERSION}" maybe_install_cargo_tool cargo-deny "${CARGO_DENY_VERSION}" maybe_install_cargo_tool cargo-msrv "${CARGO_MSRV_VERSION}" maybe_install_cargo_tool cargo-hack "${CARGO_HACK_VERSION}" +maybe_install_cargo_tool cargo-llvm-cov "${CARGO_LLVM_COV_VERSION}" maybe_install_cargo_tool dd-rust-license-tool "${DD_RUST_LICENSE_TOOL_VERSION}" maybe_install_cargo_tool wasm-pack "${WASM_PACK_VERSION}" maybe_install_cargo_tool vdev "${VDEV_VERSION}" -maybe_install_npm_package markdownlint markdownlint-cli "${MARKDOWNLINT_VERSION}" -maybe_install_npm_package datadog-ci "@datadog/datadog-ci" "${DATADOG_CI_VERSION}" "v${DATADOG_CI_VERSION}" "version" +maybe_install_npm_tools diff --git a/scripts/environment/release-flags.sh b/scripts/environment/release-flags.sh deleted file mode 100755 index 4115517c437e1..0000000000000 --- a/scripts/environment/release-flags.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash -set -e -o verbose - -# We want to ensure we're building using "full" release capabilities when possible, which -# means full LTO and a single codegen unit. This maximizes performance of the resulting -# code, but increases compilation time. We only set this if we're in CI _and_ we haven't -# been instructed to use the debug profile (via PROFILE environment variable). -if [[ "${CI-}" == "true" && "${PROFILE-}" != "debug" ]]; then - { - echo "CARGO_PROFILE_RELEASE_LTO=fat"; - echo "CARGO_PROFILE_RELEASE_CODEGEN_UNITS=1"; - echo "CARGO_PROFILE_RELEASE_DEBUG=false"; - } >> "${GITHUB_ENV}" -fi diff --git a/scripts/environment/setup-helm.sh b/scripts/environment/setup-helm.sh deleted file mode 100755 index a16b731957b3c..0000000000000 --- a/scripts/environment/setup-helm.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -KUBERNETES_VERSION="v1.18.6" -HELM_VERSION="v3.2.4" - -curl -Lo kubectl \ - "https://storage.googleapis.com/kubernetes-release/release/${KUBERNETES_VERSION}/bin/linux/amd64/kubectl" -sudo install kubectl /usr/local/bin/ && rm kubectl - -curl -L "https://get.helm.sh/helm-${HELM_VERSION}-linux-amd64.tar.gz" \ - | tar -xzv --strip-components=1 --occurrence linux-amd64/helm -sudo install helm /usr/local/bin/ && rm helm - -curl -L "https://github.com/instrumenta/kubeval/releases/latest/download/kubeval-linux-amd64.tar.gz" \ - | tar -xzv -sudo install kubeval /usr/local/bin/ && rm kubeval && rm README.md && rm LICENSE diff --git a/scripts/generate-component-docs.rb b/scripts/generate-component-docs.rb deleted file mode 100755 index 9f5fa7f695434..0000000000000 --- a/scripts/generate-component-docs.rb +++ /dev/null @@ -1,1914 +0,0 @@ -#!/usr/bin/env ruby -# frozen_string_literal: true - -begin - require 'json' - require 'tempfile' -rescue LoadError => e - puts "Load error: #{e.message}" - exit 1 -end - -DEBUG_LEVEL = 1 -INFO_LEVEL = 2 -ERROR_LEVEL = 3 - -LEVEL_MAPPINGS = { - 'debug' => { 'numeric' => DEBUG_LEVEL, 'colored' => "\033[34mDEBUG\033[0m" }, - 'info' => { 'numeric' => INFO_LEVEL, 'colored' => "\033[32mINFO\033[0m" }, - 'error' => { 'numeric' => ERROR_LEVEL, 'colored' => "\033[31mERROR\033[0m" }, -} - -def numerical_level(level_str) - LEVEL_MAPPINGS.dig(level_str.downcase, 'numeric') if !level_str.nil? -end - -def colored_level(level_str) - LEVEL_MAPPINGS.dig(level_str.downcase, 'colored') if !level_str.nil? -end - -class Logger - def initialize - @level = numerical_level(ENV['LOG_LEVEL'] || '') || INFO_LEVEL - @is_tty = STDOUT.isatty - end - - def formatted_level(level) - if @is_tty - colored_level(level) - else - level.upcase - end - end - - def log(level, msg) - numeric_level = numerical_level(level) - if numeric_level >= @level - formatted_level = self.formatted_level(level) - dt = Time.now.strftime('%Y-%m-%dT%H:%M:%S') - puts "[#{dt}] #{formatted_level} #{msg}" - end - end - - def debug(msg) - self.log('debug', msg) - end - - def info(msg) - self.log('info', msg) - end - - def error(msg) - self.log('error', msg) - end -end - -@logger = Logger.new - -@integer_schema_types = %w[uint int] -@number_schema_types = %w[float] -@numeric_schema_types = @integer_schema_types + @number_schema_types - -# Cross-platform friendly method of finding if command exists on the current path. -# -# If the command is found, the full path to it is returned. Otherwise, `nil` is returned. -def find_command_on_path(command) - exts = ENV['PATHEXT'] ? ENV['PATHEXT'].split(';') : [''] - ENV['PATH'].split(File::PATH_SEPARATOR).each do |path| - exts.each do |ext| - maybe_command_path = File.join(path, "#{command}#{ext}") - return maybe_command_path if File.executable?(maybe_command_path) && !File.directory?(maybe_command_path) - end - end - nil -end - -@cue_binary_path = find_command_on_path('cue') - -# Helpers for caching resolved/expanded schemas and detecting schema resolution cycles. -@resolved_schema_cache = {} -@expanded_schema_cache = {} - -# Gets the schema of the given `name` from the resolved schema cache, if it exists. -def get_cached_resolved_schema(schema_name) - @resolved_schema_cache[schema_name] -end - -# Gets the schema of the given `name` from the expanded schema cache, if it exists. -def get_cached_expanded_schema(schema_name) - @expanded_schema_cache[schema_name] -end - -# Generic helpers for making working with Ruby a bit easier. -def to_pretty_json(value) - if value.is_a?(Hash) - JSON.pretty_generate(Hash[*value.sort.flatten]) - else - JSON.pretty_generate(value) - end -end - -def deep_copy(obj) - Marshal.load(Marshal.dump(obj)) -end - -def mergeable?(value) - value.is_a?(Hash) || value.is_a?(Array) -end - -def _nested_merge_impl(base, override, merger) - # Handle some basic cases first. - if base.nil? - return override - elsif override.nil? - return base - elsif !mergeable?(base) && !mergeable?(override) - return override - end - - deep_copy(base).merge(override.to_h, &merger) -end - -def nested_merge(base, override) - merger = proc { |_, v1, v2| - if v1.is_a?(Hash) && v2.is_a?(Hash) - v1.merge(v2, &merger) - elsif v1.is_a?(Array) && v2.is_a?(Array) - v1 | v2 - else - [:undefined, nil, :nil].include?(v2) ? v1 : v2 - end - } - _nested_merge_impl(base, override, merger) -end - -def schema_aware_nested_merge(base, override) - merger = proc { |key, v1, v2| - if v1.is_a?(Hash) && v2.is_a?(Hash) - # Special behavior for merging const schemas together so they can be properly enum-ified. - if key == 'const' && v1.has_key?('value') && v2.has_key?('value') - [v1].flatten | [v2].flatten - else - v1.merge(v2, &merger) - end - elsif v1.is_a?(Array) && v2.is_a?(Array) - v1 | v2 - else - [:undefined, nil, :nil].include?(v2) ? v1 : v2 - end - } - _nested_merge_impl(base, override, merger) -end - -def sort_hash_nested(input) - input.keys.sort.each_with_object({}) do |key, acc| - acc[key] = if input[key].is_a?(Hash) - sort_hash_nested(input[key]) - else - input[key] - end - end -end - -def write_to_temp_file(prefix, data) - file = Tempfile.new(prefix) - file.write(data) - file.close - - file.path -end - -# Gets the JSON Schema-compatible type name for the given Ruby Value. -def json_type_str(value) - if value.is_a?(String) - 'string' - elsif value.is_a?(Integer) - 'integer' - elsif value.is_a?(Float) - 'number' - elsif [true, false].include?(value) - 'boolean' - elsif value.is_a?(Array) - 'array' - elsif value.is_a?(Hash) - 'object' - else - 'null' - end -end - -# Gets the docs-compatible type name for the given Ruby value. -# -# This is slightly different from the JSON Schema types, and is mostly an artifact of the original -# documentation design, and not representative of anything fundamental. -def docs_type_str(value) - type_str = json_type_str(value) - type_str = 'bool' if type_str == 'boolean' - type_str -end - -# Gets the type of the resolved schema. -# -# If the resolved schema has more than one type defined, `nil` is returned. -def resolved_schema_type?(resolved_schema) - if resolved_schema['type'].length == 1 - resolved_schema['type'].keys.first - end -end - -# Gets the numeric type of the resolved schema. -# -# If the resolved schema has more than one type defined, or the type is not a numeric type, `nil` is -# returned. -def numeric_schema_type(resolved_schema) - schema_type = resolved_schema_type?(resolved_schema) - schema_type if @numeric_schema_types.include?(schema_type) -end - -# Gets the docs type for the given value's type. -# -# When dealing with a source schema, and trying to get the "docs"-compatible schema type, we need to -# cross-reference the original schema with the type of the value we have on the Ruby side. While -# some types like a string will always have a type of "string", numbers represent an area where we -# need to do some digging. -# -# For example, we encode the specific class of number on the source schema -- unsigned, signed, or -# floating-point -- and we can discern nearly as much from the Ruby value itself (integer vs float), -# but we need to be able to discern the precise type i.e. the unsigned vs signed vs floating-point -# bit. -# -# This function cross-references the Ruby value given with the source schema it is associated with -# and returns the appropriate "docs" schema type for that value. If the value is not recognized, or -# if the source schema does not match the given value, `nil` is returned. -# -# Otherwise, the most precise "docs" schema type for the given value is returned. -def get_docs_type_for_value(schema, value) - # If there's no schema to check against, or there is but it has no type field, that means we're - # dealing with something like a complex, overlapping `oneOf` subschema, where we couldn't - # declaratively figure out the right field to dig into if we were discerning an integer/number - # value, and so on. - # - # We just use the detected value type in that case. - schema_instance_type = get_json_schema_instance_type(schema) unless schema.nil? - if schema.nil? || schema_instance_type.nil? - return docs_type_str(value) - end - - # If the schema defines a type, see if it matches the value type. If it doesn't, that's a bad sign - # and we abort. Otherwise, we fallthrough below to make sure we're handling special cases i.e. - # numeric types. - value_type = json_type_str(value) - if value_type != schema_instance_type - @logger.error "Schema instance type and value type are a mismatch, which should not happen." - @logger.error "Schema instance type: #{schema_instance_type}" - @logger.error "Value: #{value} (type: #{value_type})" - exit 1 - end - - # For any numeric type, extract the value of `docs::numeric_type`, which must always be present in - # the schema for numeric fields. If the schema is `nil`, though, then it means we're dealing with - # a complex schema (like an overlapping `oneOf` subschema, etc) and we just fallback to the - # detected type. - if ['number', 'integer'].include?(value_type) - numeric_type = get_schema_metadata(schema, 'docs::numeric_type') - if numeric_type.nil? - @logger.error "All fields with numeric types should have 'docs::numeric_type' metadata included." + - "e.g. #[configurable(metadata(docs::numeric_type = \"bytes\"))]" - @logger.error "Value: #{value} (type: #{value_type})" - exit 1 - end - - return numeric_type - end - - # TODO: The documentation should really just use `boolean` to match JSON Schema, which would let - # us get rid of this weird `json_type_str`/`docs_type_str` dichotomy. - docs_type_str(value) -end - -# Gets the schema type field for the given value's type. -# -# Essentially, as we resolve a schema we end up with a hash that looks like this: -# -# { "type" => { "string" => { ... } } -# -# When we want to do something like specify a default value, or give example values, we need to set -# them on the hash that represents the actual property value type. If a schema resolves as a string -# schema, we can trivially take that default value, calculate its type, and know that we need to set -# further data under the `string` key in the above example. -# -# This gets trickier for numeric types, however, as we encode them more specifically -- unsigned -# integer, floating-point number, etc -- in the resolved schema... but can only determine (on the -# Ruby side) if a value is the `number` type. To handle this, for any value of the type `number`, we -# iteratively try and find a matching type definition in the resolved schema for any of the possible -# numeric types. -def get_json_schema_type_field_for_value(source_schema, resolved_schema, value) - value_type = get_docs_type_for_value(source_schema, value) - resolved_schema.dig('type', value_type) -end - -# Tries to find the schema for an object property nested in the given schema. -# -# This function will search through either the properties of the schema itself, if it is an object -# schema, or the properties of any object subschema that is present in `oneOf`/`allOf`. -# -# If no property is found, `nil` is returned. -def find_nested_object_property_schema(schema, property_name) - # See if we're checking an object schema directly. - if !schema['properties'].nil? - return schema['properties'][property_name] - end - - # The schema isn't an object schema, so check to see if it's a `oneOf`/`allOf`, and if so, - # recursively visit each of those subschemas, looking for object schemas along the way that we can - # check for the given property within. - matching_property_schemas = [] - unvisited_subschemas = schema['oneOf'].dup || schema['anyOf'].dup|| schema['allOf'].dup || [] - while !unvisited_subschemas.empty? do - unvisited_subschema = unvisited_subschemas.pop - - # If the subschema has object properties, it won't be `oneOf`/`allOf`, so just try and grab the - # property if it exists, and move on. - if !unvisited_subschema['properties'].nil? - subschema_property = unvisited_subschema.dig('properties', property_name) - matching_property_schemas.push(subschema_property) unless subschema_property.nil? - next - end - - # If the subschema had no object properties, see if it's an `oneOf`/`allOf` subschema, and if - # so, collect any of _those_ subschemas and add them to our list of subschemas to visit. - maybe_unvisited_subschemas = unvisited_subschema['oneOf'].dup || unvisited_subschema['anyOf'].dup || unvisited_subschema['allOf'].dup || [] - unvisited_subschemas.concat(maybe_unvisited_subschemas) unless maybe_unvisited_subschemas.nil? - end - - # Compare all matching property schemas to each other -- in their reduced form -- to see if they're - # identical. If they're not, or there were no matches, return `nil`. - # - # Otherwise, return the first matching property schema. - reduced_matching_property_schemas = matching_property_schemas.map { |schema| get_reduced_schema(schema) } - matching_property_schemas[0] unless reduced_matching_property_schemas.uniq.count != 1 -end - -def get_schema_metadata(schema, key) - schema.dig('_metadata', key) -end - -def get_schema_ref(schema) - schema['$ref'] -end - -# Gets the schema type for the given schema. -def get_json_schema_type(schema) - if schema.key?('allOf') - 'all-of' - elsif schema.key?('oneOf') - 'one-of' - elsif schema.key?('anyOf') - 'any-of' - elsif schema.key?('type') - get_json_schema_instance_type(schema) - elsif schema.key?('const') - 'const' - elsif schema.key?('enum') - 'enum' - end -end - -def get_json_schema_instance_type(schema) - maybe_type = schema['type'] - - # We don't deal with null instance types at all in the documentation generation phase. - if maybe_type == 'null' - return nil - end - - # If the schema specifies multiple instance types, see if `null` is one of them, and if so, - # remove it. After that, if only one value is left, return that value directly rather than - # wrapped in an array. - # - # Otherwise, return the original array. - if maybe_type.is_a?(Array) - filtered = maybe_type.reject { |instance_type| instance_type == "null" } - if filtered.length == 1 - return filtered[0] - end - end - - maybe_type -end - -# Fixes grouped enums by adjusting the schema type where necessary. -# -# For "grouped enums", these represent the sum of all possible enum values in an `enum` schema being -# grouped by their JSON type. For example, a set of enums such as `[0, 1, 2, true, false]` would be -# grouped as: -# -# { "bool": [true, false], "number": [0, 1, 2] } -# -# This is technically correct, but in the documentation output, that `number` key needs to be `uint` -# or `int` or what have you. Since `enum` schemas don't carry the "numeric type" information, we try -# and figure that out here. -# -# If we find a `number` group, we check all of its values to see if they fit neatly within the -# bounds of any of the possible numeric types `int`, `uint`, or `float`. We try and coalesce -# towards `uint` as it's by far the most common numeric type in Vector configuration, but after -# that, we aim for `int`, unless the values are too large, in which case we'll shift up to `float`. -def fix_grouped_enums_if_numeric!(grouped_enums) - ['integer', 'number'].each { |type_name| - number_group = grouped_enums.delete(type_name) - if !number_group.nil? - is_integer = number_group.all? { |n| n.is_a?(Integer) } - within_uint = number_group.all? { |n| n >= 0 && n <= 2 ** 64 } - within_int = number_group.all? { |n| n >= -(2 ** 63) && n <= (2 ** 63) - 1 } - - # If the values themselves are not all integers, or they are but not all of them can fit within - # a normal 64-bit signed/unsigned integer, then we use `float` as it's the only other type that - # could reasonably satisfy the constraints. - numeric_type = if !is_integer || (!within_int && !within_uint) - 'float' - else - if within_uint - 'uint' - elsif within_int - 'int' - else - # This should never actually happen, _but_, technically Ruby integers could be a "BigNum" - # aka arbitrary-precision integer, so this protects us if somehow we get a value that is an - # integer but doesn't actually fit neatly into 64 bits. - 'float' - end - end - - grouped_enums[numeric_type] = number_group - end - } -end - -# Gets a schema definition from the root schema, by name. -def get_schema_by_name(root_schema, schema_name) - schema_name = schema_name.gsub(%r{#/definitions/}, '') - schema_def = root_schema.dig('definitions', schema_name) - if schema_def.nil? - @logger.error "Could not find schema definition '#{schema_name}' in given schema." - exit 1 - end - - schema_def -end - -# Gets the dereferenced version of this schema. -# -# If the schema has no schema reference, `nil` is returned. -def dereferenced_schema(schema) - schema_name = get_schema_ref(schema) - if !schema_ref.nil? - get_schema_by_name(root_schema, schema_name) - end -end - -# Applies various fields to an object property. -# -# This includes items such as any default value that is present, or whether or not the property is -# required. -def apply_object_property_fields!(parent_schema, property_schema, property_name, property) - @logger.debug "Applying object property fields for '#{property_name}'..." - - required_properties = parent_schema['required'] || [] - has_self_default_value = !property_schema['default'].nil? - has_parent_default_value = !parent_schema.dig('default', property_name).nil? - has_default_value = has_self_default_value || has_parent_default_value - is_required = required_properties.include?(property_name) - - if has_self_default_value - @logger.debug "Property has self-defined default value: #{property_schema['default']}" - end - - if has_parent_default_value - @logger.debug "Property has parent-defined default value: #{parent_schema.dig('default', property_name)}" - end - - if is_required - @logger.debug "Property is marked as required." - end - - # Set whether or not this property is required. - property['required'] = required_properties.include?(property_name) && !has_default_value -end - -# Expands any schema references in the given schema. -# -# If the schema contains a top-level schema reference, or if any of the parts of its schema contain -# schema references (array items schema, any subschemas in `oneOf`/`allOf`, etc), then those -# references are expanded. Expansion happens recursively until all schema references -# -# For any overlapping fields in the given schema and the referenced schema, the fields from the -# given schema will win. -def expand_schema_references(root_schema, unexpanded_schema) - schema = deep_copy(unexpanded_schema) - - # Grab the existing title/description from our unexpanded schema, and reset them after - # merging. This avoids us adding a title where there was only a description, and so on, since - # we have special handling rules around titles vs descriptions. - # - # TODO: If we ever just get rid of the title/description dichotomy, we could clean up this - # logic. - original_title = unexpanded_schema['title'] - original_description = unexpanded_schema['description'] - - # If the schema has a top level reference, we expand it. - schema_ref = schema['$ref'] - if !schema_ref.nil? - expanded_schema_ref = get_cached_expanded_schema(schema_ref) - if expanded_schema_ref.nil? - @logger.debug "Expanding top-level schema ref of '#{schema_ref}'..." - - unexpanded_schema_ref = get_schema_by_name(root_schema, schema_ref) - expanded_schema_ref = expand_schema_references(root_schema, unexpanded_schema_ref) - - @expanded_schema_cache[schema_ref] = expanded_schema_ref - end - - schema.delete('$ref') - schema = nested_merge(expanded_schema_ref, schema) - end - - # If the schema is an array type and has a reference for its items, we expand that. - items_ref = schema.dig('items', '$ref') - if !items_ref.nil? - expanded_items_schema_ref = expand_schema_references(root_schema, schema['items']) - - schema['items'].delete('$ref') - schema['items'] = nested_merge(expanded_items_schema_ref, schema['items']) - end - - # If the schema has any object properties, we expand those. - if !schema['properties'].nil? - schema['properties'] = schema['properties'].transform_values { |property_schema| - new_property_schema = expand_schema_references(root_schema, property_schema) - new_property_schema - } - end - - # If the schema has any `allOf`/`oneOf` subschemas, we expand those, too. - if !schema['allOf'].nil? - schema['allOf'] = schema['allOf'].map { |subschema| - new_subschema = expand_schema_references(root_schema, subschema) - new_subschema - } - end - - if !schema['oneOf'].nil? - schema['oneOf'] = schema['oneOf'].map { |subschema| - new_subschema = expand_schema_references(root_schema, subschema) - new_subschema - } - end - - if !schema['anyOf'].nil? - schema['anyOf'] = schema['anyOf'].map { |subschema| - new_subschema = expand_schema_references(root_schema, subschema) - new_subschema - } - end - - # If the original schema had either a title or description, we forcefully reset both of them back - # to their original state, either in terms of their value or them not existing as fields. - # - # If neither were present, we allow the merged in title/description, if any, to persist, as this - # maintains the "#[configurable(derived)]" behavior of titles/descriptions for struct fields. - if !original_title.nil? || !original_description.nil? - if !original_title.nil? - schema['title'] = original_title - else - schema.delete('title') - end - - if - schema['description'] = original_description - else - schema.delete('description') - end - end - - schema -end - -# Gets a reduced version of a schema. -# -# The reduced version strips out extraneous fields from the given schema, such that a value should -# be returned that is suitable for comparison with other schemas, to determine if the schemas -- -# specifically the values that are allowed/valid -- are the same, while ignoring things like titles -# and descriptions. -def get_reduced_schema(schema) - schema = deep_copy(schema) - - allowed_properties = ['type', 'const', 'enum', 'allOf', 'oneOf', '$ref', 'items', 'properties'] - schema.delete_if { |key, _value| !allowed_properties.include?(key) } - - if schema.key?('items') - schema['items'] = get_reduced_schema(schema['items']) - end - - if schema.key?('properties') - schema['properties'] = schema['properties'].transform_values { |property_schema| get_reduced_schema(property_schema) } - end - - if schema.key?('allOf') - schema['allOf'] = schema['allOf'].map { |subschema| get_reduced_schema(subschema) } - end - - if schema.key?('oneOf') - schema['oneOf'] = schema['oneOf'].map { |subschema| get_reduced_schema(subschema) } - end - - schema -end - -# Gets a reduced version of a resolved schema. -# -# This is similar in purpose to `get_reduced_schema` but only cares about fields relevant to a -# resolved schema. -def get_reduced_resolved_schema(schema) - schema = deep_copy(schema) - - allowed_types = ['condition', 'object', 'array', 'enum', 'const', 'string', 'bool', 'float', 'int', 'uint'] - allowed_fields = [] - - # Clear out anything not related to the type definitions first. - schema.delete_if { |key, _value| key != 'type' } - type_defs = schema['type'] - if !type_defs.nil? - type_defs.delete_if { |key, _value| !allowed_types.include?(key) } - schema['type'].each { |type_name, type_def| - case type_name - when "object" - type_def.delete_if { |key, _value| key != 'options' } - type_def['options'].transform_values! { |property| - get_reduced_resolved_schema(property) - } - when "array" - type_def.delete_if { |key, _value| key != 'items' } - type_def['items'] = get_reduced_resolved_schema(type_def['items']) - else - type_def.delete_if { |key, _value| !allowed_types.include?(key) } - end - } - end - - schema -end - -# Fully resolves a schema definition, if it exists. -# -# This looks up a schema definition by the given `name` within `root_schema` and resolves it. -# If no schema definition exists for the given name, `nil` is returned. Otherwise, the schema -# definition is preprocessed (collapsing schema references, etc), and then resolved. Both the -# "source" schema (preprocessed value) and the resolved schema are returned. -# -# Resolved schemas are cached. -# -# See `resolve_schema` for more details. -def resolve_schema_by_name(root_schema, schema_name) - # If it's already cached, use that. - resolved = get_cached_resolved_schema(schema_name) - return deep_copy(resolved) unless resolved.nil? - - # It wasn't already cached, so we actually have to resolve it. - schema = get_schema_by_name(root_schema, schema_name) - resolved = resolve_schema(root_schema, schema) - @resolved_schema_cache[schema_name] = resolved - deep_copy(resolved) -end - -# Fully resolves the schema. -# -# This recursively resolves schema references, as well as flattening them into a single object, and -# transforming certain usages -- composite/enum (`allOf`, `oneOf`), etc -- into more human-readable -# forms. -def resolve_schema(root_schema, schema) - # If the schema we've been given if a schema reference, we expand that first. This happens - # recursively, such that the resulting expanded schema has no schema references left. We need this - # because in further steps, we need the access to the full input schema that was used to generate - # the resolved schema. - schema = expand_schema_references(root_schema, schema) - - # Skip any schema that is marked as hidden. - # - # While this is already sort of obvious, one non-obvious usage is for certain types that we - # manually merge after this script runs, such as the high-level "outer" (`SinkOuter`, etc) types. - # Those types include a field that uses the Big Enum -- an enum with all possible components of - # that type -- which, if we resolved it here, would spit out a ton of nonsense. - # - # We mark that field hidden so that it's excluded when we resolve the schema for `SinkOuter`, etc, - # and then we individually resolve the component schemas, and merge the two (outer + component - # itself) schemas back together. - if get_schema_metadata(schema, 'docs::hidden') - @logger.debug 'Instructed to skip resolution for the given schema.' - return - end - - # Handle schemas that have type overrides. - # - # In order to better represent specific field types in the documentation, we may opt to use a - # special type definition name, separate from the classic types like "bool" or "string" or - # "object", and so on, in order to let the documentation generation process inject more - # full-fledged output than we can curry from the Rust side, across the configuration schema. - # - # We intentially set no actual definition for these types, relying on the documentation generation - # process to provide the actual details. We only need to specify the custom type name. - # - # To handle u8 types as ascii characters and not there uint representation between 0 and 255 we - # added a special handling of these exact values. This means - # `#[configurable(metadata(docs::type_override = "ascii_char"))]` should only be used consciously - # for rust u8 type. See lib/codecs/src/encoding/format/csv.rs for an example and - # https://github.com/vectordotdev/vector/pull/20498 - type_override = get_schema_metadata(schema, 'docs::type_override') - if !type_override.nil? - if type_override == 'ascii_char' - if !schema['default'].nil? - resolved = { 'type' => { type_override.to_s => { 'default' => schema['default'].chr } } } - else - resolved = { 'type' => { type_override.to_s => { } } } - end - else - resolved = { 'type' => { type_override.to_s => {} } } - end - description = get_rendered_description_from_schema(schema) - resolved['description'] = description unless description.empty? - return resolved - end - - # Now that the schema is fully expanded and it didn't need special handling, we'll go ahead and - # fully resolve it. - resolved = resolve_bare_schema(root_schema, schema) - if resolved.nil? - return - end - - # If this is an array schema, remove the description from the schema used for the items, as we - # want the description for the overall property, using this array schema, to describe everything. - items_schema = resolved.dig('type', 'array', 'items') - if !items_schema.nil? - items_schema.delete('description') - end - - # Apply any necessary defaults, descriptions, etc, to the resolved schema. This must happen here - # because there could be callsite-specific overrides to defaults, descriptions, etc, for a given - # schema definition that have to be layered. - apply_schema_default_value!(schema, resolved) - apply_schema_metadata!(schema, resolved) - - description = get_rendered_description_from_schema(schema) - resolved['description'] = description unless description.empty? - - ## Resolve the deprecated flag. An optional deprecated message can also be set in the metadata. - if schema.fetch('deprecated', false) - resolved['deprecated'] = true - message = get_schema_metadata(schema, 'deprecated_message') - if message - resolved['deprecated_message'] = message - end - end - - # required for global option configuration - is_common_field = get_schema_metadata(schema, 'docs::common') - if !is_common_field.nil? - resolved['common'] = is_common_field - end - - is_required_field = get_schema_metadata(schema, 'docs::required') - if !is_required_field.nil? - resolved['required'] = is_required_field - end - - # Resolve any warnings attached to this option. - # - # Warnings can be specified in Rust via `#[configurable(metadata(docs::warnings = "..."))]`. - # Multiple warnings can be specified by repeating the attribute, and they will be emitted as an - # array in the CUE output. - warnings = get_schema_metadata(schema, 'docs::warnings') - if !warnings.nil? - resolved['warnings'] = warnings.is_a?(Array) ? warnings : [warnings] - end - - # Reconcile the resolve schema, which essentially gives us a chance to, once the schema is - # entirely resolved, check it for logical inconsistencies, fix up anything that we reasonably can, - # and so on. - reconcile_resolved_schema!(resolved) - - resolved -end - -# Fully resolves a bare schema. -# -# A bare schema is one that has no references to another schema, etc. -def resolve_bare_schema(root_schema, schema) - resolved = case get_json_schema_type(schema) - when 'all-of' - @logger.debug 'Resolving composite schema.' - - # Composite schemas are indeed the sum of all of their parts, so resolve each subschema and - # merge their resolved state together. - reduced = schema['allOf'].filter_map { |subschema| resolve_schema(root_schema, subschema) } - .reduce { |acc, item| nested_merge(acc, item) } - reduced['type'] - when 'one-of', 'any-of' - @logger.debug 'Resolving enum schema.' - - # We completely defer resolution of enum schemas to `resolve_enum_schema` because there's a - # lot of tricks and matching we need to do to suss out patterns that can be represented in more - # condensed resolved forms. - wrapped = resolve_enum_schema(root_schema, schema) - - # `resolve_enum_schema` always hands back the resolved schema under the key `_resolved`, so - # that we can add extra details about the resolved schema (anything that will help us render - # it better in the docs output) on the side. That's why we have to unwrap the resolved schema - # like this. - - # TODO: Do something with the extra detail (`annotations`). - - wrapped.dig('_resolved', 'type') - when 'array' - @logger.debug 'Resolving array schema.' - - { 'array' => { 'items' => resolve_schema(root_schema, schema['items']) } } - when 'object' - @logger.debug 'Resolving object schema.' - - # TODO: Not all objects have an actual set of properties, such as anything using - # `additionalProperties` to allow for arbitrary key/values to be set, which is why we're - # handling the case of nothing in `properties`... but we probably want to be able to better - # handle expressing this in the output.. or maybe it doesn't matter, dunno! - properties = schema['properties'] || {} - - options = properties.filter_map do |property_name, property_schema| - @logger.debug "Resolving object property '#{property_name}'..." - resolved_property = resolve_schema(root_schema, property_schema) - if !resolved_property.nil? - apply_object_property_fields!(schema, property_schema, property_name, resolved_property) - - @logger.debug "Resolved object property '#{property_name}'." - - [property_name, resolved_property] - else - @logger.debug "Resolution failed for '#{property_name}'." - - nil - end - end - - # If the object schema has `additionalProperties` set, we add an additional field - # (`*`) which uses the specified schema for that field. - additional_properties = schema['additionalProperties'] - if !additional_properties.nil? - @logger.debug "Handling additional properties." - - # Normally, we only get here if there's a hashmap field on a struct that can act as the - # catch-all for additional properties. That field, by definition, will be required to have a - # description, and maybe will have a title. - # - # That title/description makes sense for the field itself, but when we generate this new - # wildcard property, we generally want to have something short and simple, in the singular - # form. For example, if we have a field called "labels", the title/description might talk - # about what the labels are used for, any special requirements, and so on... and then for - # the wildcard property, we might want to have the description read as "A foobar label." - # just to make the UI look nice. - # - # Rather than try and derive this from the title/description on the field, we'll require - # such a description to be provided on the Rust side via the metadata attribute shown below. - singular_description = get_schema_metadata(schema, 'docs::additional_props_description') - if singular_description.nil? - @logger.error "Missing 'docs::additional_props_description' metadata for a wildcard field.\n\n" \ - "For map fields (`HashMap<...>`, etc), a description (in the singular form) must be provided by via `#[configurable(metadata(docs::additional_props_description = \"Description of the field.\"))]`.\n\n" \ - "The description on the field, derived from the code comments, is shown specifically for `field`, while the description provided via `docs::additional_props_description` is shown for the special `field.*` entry that denotes that the field is actually a map." - - @logger.error "Relevant schema: #{JSON.pretty_generate(schema)}" - exit 1 - end - - resolved_additional_properties = resolve_schema(root_schema, additional_properties) - resolved_additional_properties['required'] = true - resolved_additional_properties['description'] = singular_description - options.push(['*', resolved_additional_properties]) - end - - { 'object' => { 'options' => options.to_h } } - when 'string' - @logger.debug 'Resolving string schema.' - - string_def = {} - string_def['default'] = schema['default'] unless schema['default'].nil? - - { 'string' => string_def } - when 'number', 'integer' - @logger.debug 'Resolving number schema.' - - numeric_type = get_schema_metadata(schema, 'docs::numeric_type') || 'number' - number_def = {} - number_def['default'] = schema['default'] unless schema['default'].nil? - - @logger.debug 'Resolved number schema.' - - { numeric_type => number_def } - when 'boolean' - @logger.debug 'Resolving boolean schema.' - - bool_def = {} - bool_def['default'] = schema['default'] unless schema['default'].nil? - - { 'bool' => bool_def } - when 'const' - @logger.debug 'Resolving const schema.' - - # For `const` schemas, just figure out the type of the constant value so we can generate the - # resolved output. - const_type = get_docs_type_for_value(schema, schema['const']) - const_value = { 'value' => schema['const'] } - const_description = get_rendered_description_from_schema(schema) - const_value['description'] = const_description unless const_description.nil? - { const_type => { 'const' => const_value } } - when 'enum' - @logger.debug 'Resolving enum const schema.' - - # Similarly to `const` schemas, `enum` schemas are merely multiple possible constant values. Given - # that JSON Schema does allow for the constant values to differ in type, we group them all by - # type to get the resolved output. - enum_values = schema['enum'] - grouped = enum_values.group_by { |value| docs_type_str(value) } - fix_grouped_enums_if_numeric!(grouped) - grouped.transform_values! { |values| { 'enum' => values } } - grouped - when nil - # Unconstrained/empty schema (e.g., Value without constraints). - # Represent it as accepting any JSON type so downstream code can render it - # and attach defaults/examples based on actual values. - @logger.debug 'Resolving unconstrained schema (any type).' - - { '*' => {} } - else - @logger.error "Failed to resolve the schema. Schema: #{schema}" - exit 1 - end - - { 'type' => resolved } -end - -def resolve_enum_schema(root_schema, schema) - # Figure out if this is a one-of or any-of enum schema. Both at the same time is never correct. - subschemas = if schema.key?('oneOf') - schema['oneOf'] - elsif schema.key?('anyOf') - schema['anyOf'] - else - @logger.error "Enum schema had both `oneOf` and `anyOf` specified. Schema: #{schema}" - exit 1 - end - - # Filter out all subschemas which are purely null schemas used for indicating optionality, as well - # as any subschemas that are marked as being hidden. - is_optional = get_schema_metadata(schema, 'docs::optional') - subschemas = subschemas - .reject { |subschema| subschema['type'] == 'null' } - .reject { |subschema| get_schema_metadata(subschema, 'docs::hidden') } - subschema_count = subschemas.count - - # If we only have one subschema after filtering, check to see if it's an `allOf` or `oneOf` schema - # and `is_optional` is true. - # - # If it's an `allOf` subschema, then that means we originally had an `allOf` schema that we had to - # make optional, thus converting it to a `oneOf` with subschemas in the shape of `[null, allOf]`. - # In this case, we'll just remove the `oneOf` and move the `allOf` subschema up, as if it this - # schema was a `allOf` one all along. - # - # If so, we unwrap it such that we end up with a copy of `schema` that looks like it was an - # `allOf` schema all along. We do this to properly resolve `allOf` schemas that were wrapped as - # `oneOf` w/ a null schema in order to establish optionality. - if is_optional && subschema_count == 1 - if get_json_schema_type(subschemas[0]) == 'all-of' - @logger.debug "Detected optional all-of schema, unwrapping all-of schema to resolve..." - - # Copy the current schema and drop `oneOf` and set `allOf` with the subschema, which will get us the correct - # unwrapped structure. - unwrapped_schema = deep_copy(schema) - unwrapped_schema.delete('oneOf') - unwrapped_schema['allOf'] = deep_copy(subschemas[0]['allOf']) - - return { '_resolved' => resolve_schema(root_schema, unwrapped_schema) } - else - # For all other subschema types, we copy the current schema, drop the `oneOf`, and merge the - # subschema into it. This essentially unnests the schema. - unwrapped_schema = deep_copy(schema) - unwrapped_schema.delete('oneOf') - unwrapped_schema = schema_aware_nested_merge(unwrapped_schema, subschemas[0]) - - return { '_resolved' => resolve_schema(root_schema, unwrapped_schema) } - end - end - - # Collect all of the tagging mode information upfront. - enum_tagging = get_schema_metadata(schema, 'docs::enum_tagging') - if enum_tagging.nil? - @logger.error 'Enum schemas should never be missing the metadata for the enum tagging mode.' - @logger.error "Schema: #{JSON.pretty_generate(schema)}" - @logger.error "Filter subschemas: #{JSON.pretty_generate(subschemas)}" - exit 1 - end - - enum_tag_field = get_schema_metadata(schema, 'docs::enum_tag_field') - - # Schema pattern: X or array of X. - # - # We employ this pattern on the Vector side to allow specifying a single instance of X -- object, - # string, whatever -- or as an array of X. We just need to inspect both schemas to make sure one - # is an array of X and the other is the same as X, or vice versa. - if subschema_count == 2 - array_idx = subschemas.index { |subschema| subschema['type'] == 'array' } - unless array_idx.nil? - @logger.debug "Detected likely 'X or array of X' enum schema, applying further validation..." - - single_idx = array_idx.zero? ? 1 : 0 - - # We 'reduce' the subschemas, which strips out all things that aren't fundamental, which boils - # it down to only the shape of what the schema accepts, so no title or description or default - # values, and so on. - single_reduced_subschema = get_reduced_schema(subschemas[single_idx]) - array_reduced_subschema = get_reduced_schema(subschemas[array_idx]) - - if single_reduced_subschema == array_reduced_subschema['items'] - @logger.debug 'Reduced schemas match, fully resolving schema for X...' - - # The single subschema and the subschema for the array items are a match! We'll resolve this - # as the schema of the "single" option, but with an annotation that it can be specified - # multiple times. - - # Copy the subschema, and if the parent schema we're resolving has a default value with a - # type that matches the type of the "single" subschema, add it to the copy of that schema. - # - # It's hard to propagate the default from the configuration schema generation side, but much - # easier to do so here. - single_subschema = deep_copy(subschemas[single_idx]) - if get_json_schema_type(single_subschema) == json_type_str(schema['default']) - single_subschema['default'] = schema['default'] - end - resolved_subschema = resolve_schema(root_schema, subschemas[single_idx]) - - @logger.debug "Resolved as 'X or array of X' enum schema." - - return { '_resolved' => resolved_subschema, 'annotations' => 'single_or_array' } - end - end - end - - # Schema pattern: simple internally tagged enum with named fields. - # - # This a common pattern where we'll typically have enum variants that have named fields, and we - # use an internal tag (looks like a normal field next to the fields of the enum variant itself) - # where the tag value is the variant name. - # - # We want to generate an object schema where we expose the unique sum of all named fields from the - # various subschemas, and annotate each resulting unique property with the field value that makes - # it relevant. We do require that any overlapping fields between variants be identical, however. - # - # For example, buffer configuration allows configuring in-memory buffers or disk buffers. When - # using in-memory buffers, a property called `max_events` becomes relevant, so we want to be able - # to say that the `max_events` field is `relevant_when` the value of `type` (the adjacent tag) is - # `memory`. We do this for every property that _isn't_ the adjacent tag, but the adjacent tag - # _does_ get included in the resulting properties. - if enum_tagging == 'internal' - @logger.debug "Resolving enum subschemas to detect 'object'-ness..." - - # This transformation only makes sense when all subschemas are objects, so only resolve the ones - # that are objects, and only proceed if all of them were resolved. - resolved_subschemas = subschemas.filter_map do |subschema| - # TODO: the exact enum variant probably isn't an object? but probabilistic is... gotta handle that - resolved = resolve_schema(root_schema, subschema) - resolved if resolved_schema_type?(resolved) == 'object' - end - - if resolved_subschemas.count == subschemas.count - @logger.debug "Detected likely 'internally-tagged with named fields' enum schema, applying further validation..." - - unique_resolved_properties = {} - unique_tag_values = {} - - resolved_subschemas.each do |resolved_subschema| - @logger.debug "Resolved subschema: #{JSON.dump(resolved_subschema)}" - resolved_subschema_properties = resolved_subschema.dig('type', 'object', 'options') - - # Extract the tag property and figure out any necessary intersections, etc. - # - # Technically, a `const` value in JSON Schema can be an array or object, too... but like, we - # only ever use `const` for describing enum variants and what not, so this is my middle-ground - # approach to also allow for other constant-y types, but not objects/arrays which would - # just... not make sense. - # - # We also steal the title and/or description from the variant subschema and apply it to the - # tag's subschema, as we don't curry the title/description for a variant itself to the - # respective tag field used to indicate which variant is specified. - tag_subschema = resolved_subschema_properties.delete(enum_tag_field) - tag_subschema['title'] = resolved_subschema['title'] if !resolved_subschema['title'].nil? - tag_subschema['description'] = resolved_subschema['description'] if !resolved_subschema['description'].nil? - tag_value = nil - - %w[string number integer boolean].each do |allowed_type| - maybe_tag_values = tag_subschema.dig('type', allowed_type, 'enum') - maybe_tag_value = maybe_tag_values.keys.first unless maybe_tag_values.nil? - unless maybe_tag_value.nil? - tag_value = maybe_tag_value - break - end - end - - @logger.debug "Tag value of #{tag_value}, with original resolved schema: #{resolved_subschema}" - - if tag_value.nil? - @logger.error 'All enum subschemas representing an internally-tagged enum must have the tag field use a const value.' - @logger.error "Tag subschema: #{tag_subschema}" - exit 1 - end - - if unique_tag_values.key?(tag_value) - @logger.error "Found duplicate tag value '#{tag_value}' when resolving enum subschemas." - exit 1 - end - - unique_tag_values[tag_value] = tag_subschema - - # Now merge all of the properties from the given subschema, so long as the overlapping - # properties have the same schema. - resolved_subschema_properties.each do |property_name, property_schema| - existing_property = unique_resolved_properties[property_name] - resolved_property = if !existing_property.nil? - # The property is already being tracked, so just do a check to make sure the property from our - # current subschema matches the existing property, schema-wise, before we update it. - reduced_existing_property = get_reduced_resolved_schema(existing_property) - reduced_new_property = get_reduced_resolved_schema(property_schema) - - if reduced_existing_property != reduced_new_property - @logger.error "Had overlapping property '#{property_name}' from resolved enum subschema, but schemas differed:" - @logger.error "Existing property schema (reduced): #{to_pretty_json(reduced_existing_property)}" - @logger.error "New property schema (reduced): #{to_pretty_json(reduced_new_property)}" - exit 1 - end - - @logger.debug "Adding relevant tag to existing resolved property schema for '#{property_name}'." - - # The schemas match, so just update the list of "relevant when" values. - existing_property['relevant_when'].push(tag_value) - existing_property - else - @logger.debug "First time seeing resolved property schema for '#{property_name}'." - - # First time seeing this particular property. - property_schema['relevant_when'] = [tag_value] - property_schema - end - - unique_resolved_properties[property_name] = resolved_property - @logger.debug "Set unique resolved property '#{property_name}' to resolved schema: #{resolved_property}" - end - end - - # Now that we've gone through all of the non-tag field, possibly overlapped properties, go - # through and modify the properties so that we only keep the "relevant when" values if the - # list of those values does not match the full set of unique tag values. We don't want to show - # "relevant when" for fields that all variants share, basically. - unique_tags = unique_tag_values.keys - - unique_resolved_properties.transform_values! do |value| - # We check if a given property is relevant to all tag values by getting an intersection - # between `relevant_when` and the list of unique tag values, as well as asserting that the - # list lengths are identical. - relevant_when = value['relevant_when'] - if relevant_when.length == unique_tags.length && relevant_when & unique_tags == unique_tags - value.delete('relevant_when') - end - - # Add enough information from consumers to figure out _which_ field needs to have the given - # "relevant when" value. - if value.key?('relevant_when') - value['relevant_when'] = value['relevant_when'].map do |value| - "#{enum_tag_field} = #{JSON.dump(value)}" - end.to_a.join(' or ') - end - - value - end - - # Now we build our property for the tag field itself, and add that in before returning all of - # the unique resolved properties. - enum_values = unique_tag_values.transform_values do |tag_schema| - @logger.debug "Tag schema: #{tag_schema}" - get_rendered_description_from_schema(tag_schema) - end - - @logger.debug "Resolved enum values for tag property: #{enum_values}" - resolved_tag_property = { - 'required' => true, - 'type' => { - 'string' => { - 'enum' => enum_values - } - } - } - - tag_description = get_schema_metadata(schema, 'docs::enum_tag_description') - if tag_description.nil? - @logger.error "A unique tag description must be specified for enums which are internally tagged (i.e. `#[serde(tag = \"...\")/`). This can be specified via `#[configurable(metadata(docs::enum_tag_description = \"...\"))]`." - @logger.error "Schema being generated: #{JSON.pretty_generate(schema)}" - exit 1 - end - resolved_tag_property['description'] = tag_description - unique_resolved_properties[enum_tag_field] = resolved_tag_property - - @logger.debug "Resolved as 'internally-tagged with named fields' enum schema." - @logger.debug "Resolved properties for ITNF enum schema: #{unique_resolved_properties}" - - return { '_resolved' => { 'type' => { 'object' => { 'options' => unique_resolved_properties } } } } - end - end - - # Schema pattern: simple externally tagged enum with only unit variants. - # - # This a common pattern where basic enums that only have unit variants -- i.e. `enum { A, B, C }` - # -- end up being represented by a bunch of subschemas that are purely `const` values. - if enum_tagging == 'external' - tag_values = {} - - subschemas.each do |subschema| - # For each subschema, try and grab the value of the `const` property and use it as the key for - # storing this subschema. - # - # We take advantage of missing key index gets returning `nil` by checking below to make sure - # none of the keys are nil. If any of them _are_ nill, then we know not all variants had a - # `const` schema. - tag_values[subschema['const']] = subschema - end - - if tag_values.keys.all? { |tag| !tag.nil? && tag.is_a?(String) } - @logger.debug "Resolved as 'externally-tagged with only unit variants' enum schema." - - return { '_resolved' => { 'type' => { 'string' => { - 'enum' => tag_values.transform_values { |tag_schema| get_rendered_description_from_schema(tag_schema) } - } } } } - end - end - - # Schema pattern: untagged enum with narrowing constant variants and catch-all free-form variant. - # - # This a common pattern where an enum might represent a particular single value type, say a - # string, and it contains multiple variants where one is a "dynamic" variant and the others are - # "fixed", such that the dynamic variant can represent all possible string values _except_ for the - # string values defined by each fixed variant. - # - # An example of this is the `TimeZone` enum, where there is one variant `Local`, represented by - # `"local"`, and the other variant `Named` can represent any other possible timezone. - if enum_tagging == 'untagged' - type_def_kinds = [] - fixed_subschemas = 0 - freeform_subschemas = 0 - - subschemas.each do |subschema| - @logger.debug "Untagged subschema: #{subschema}" - schema_type = get_json_schema_type(subschema) - case schema_type - when nil, "all-of", "one-of" - # We don't handle these cases. - when "const" - # Track the type definition kind associated with the constant value. - type_def_kinds << docs_type_str(subschema['const']) - fixed_subschemas = fixed_subschemas + 1 - when "enum" - # Figure out the type definition kind for each enum value and track it. - type_def_kinds << subschema['enum'].map { |value| docs_type_str(value) } - fixed_subschemas = fixed_subschemas + 1 - else - # Just a regular schema type, so track it. - type_def_kinds << schema_type - freeform_subschemas = freeform_subschemas + 1 - end - end - - # If there's more than a single type definition in play, then it doesn't qualify. - unique_type_def_kinds = type_def_kinds.flatten.uniq - if unique_type_def_kinds.length == 1 && fixed_subschemas >= 1 && freeform_subschemas == 1 - @logger.debug "Resolved as 'untagged with narrowed free-form' enum schema." - - type_def_kind = unique_type_def_kinds[0] - - # TODO: It would be nice to forward along the fixed values so they could be enumerated in the - # documentation, and we could have a boilerplate blurb about "these values are fixed/reserved, - # but any other value than these can be passed", etc. - - return { '_resolved' => { 'type' => { type_def_kind => {} } }, 'annotations' => 'narrowed_free_form' } - end - end - - # Schema pattern: simple externally tagged enum with only non-unit variants. - # - # This pattern represents an enum where the default external tagging mode is used, which leads to - # a schema for each variant that looks like it's an object with a single property -- the name of - # which is the name of the variant itself -- and the schema of that property representing whatever - # the fields are for the variant. - # - # Contrasted with an externally tagged enum with only unit variants, this type of enum is treated - # like an object itself, where each possible variant is its own property, with whatever the - # resulting subschema for that variant may be. - if enum_tagging == 'external' - # With external tagging, and non-unit variants, each variant must be represented as an object schema. - if subschemas.all? { |subschema| get_json_schema_type(subschema) == "object" } - # Generate our overall object schema, where each variant is a property on this schema. The - # schema of that property should be the schema for the variant's tagging field. For example, - # a variant called `Foo` will have an object schema with a single, required field `foo`. We - # take the schema of that property `foo`, and set it as the schema for property `foo` on our - # new aggregated object schema. - # - # Additionally, since this is a "one of" schema, we can't make any of the properties on the - # aggregated object schema be required, since the whole point is that they're deserialized - # opportunistically. - aggregated_properties = {} - - subschemas.each { |subschema| - resolved_subschema = resolve_schema(root_schema, subschema) - - @logger.debug "ETNUV original subschema: #{subschema}" - @logger.debug "ETNUV resolved subschema: #{resolved_subschema}" - - resolved_properties = resolved_subschema.dig('type', 'object', 'options') - if resolved_properties.nil? || resolved_properties.keys.length != 1 - @logger.error "Encountered externally tagged enum schema with non-unit variants where the resulting \ - resolved schema for a given variant had more than one property!" - @logger.error "Original variant subschema: #{subschema}" - @logger.error "Resolved variant subschema: #{resolved_subschema}" - end - - # Generate a description from the overall subschema and apply it to the properly itself. We - # do this since it would be lost otherwise. - description = get_rendered_description_from_schema(subschema) - resolved_properties.each { |property_name, property_schema| - property_schema['description'] = description unless description.empty? - aggregated_properties[property_name] = property_schema - } - } - - @logger.debug "Resolved as 'externally-tagged with only non-unit variants' enum schema." - - return { '_resolved' => { 'type' => { 'object' => { 'options' => aggregated_properties } } } } - end - end - - # Fallback schema pattern: mixed-mode enums. - # - # These are enums that can basically be some combination of possible values: `Concurrency` is the - # canonical example as it can be set via `"none"`, `"adaptive"`, or an integer between 1 and... - # 2^64 or something like that. None of the subschemas overlap in any way. - # - # We just end up emitting a composite type output to cover each possibility, so the above would - # have the `string` type with an `enum` of `"none"` and `"adaptive"`, and the uint type for the - # integer side. This code mostly assumes the upstream schema is itself correct, in terms of not - # providing a schema that is too ambiguous to properly validate against an input document. - @logger.debug "Resolved as 'fallback mixed-mode' enum schema." - - @logger.debug "Tagging mode: #{enum_tagging}" - @logger.debug "Input subschemas: #{subschemas}" - - resolved_subschemas = subschemas.filter_map { |subschema| resolve_schema(root_schema, subschema) } - @logger.debug "Resolved fallback schemas: #{resolved_subschemas}" - - type_defs = resolved_subschemas.reduce { |acc, item| schema_aware_nested_merge(acc, item) } - - @logger.debug "Schema-aware merged result: #{type_defs}" - - { '_resolved' => { 'type' => type_defs['type'] }, 'annotations' => 'mixed_mode' } -end - -def apply_schema_default_value!(source_schema, resolved_schema) - @logger.debug "Applying schema default values." - - default_value = source_schema['default'] - unless default_value.nil? - # Make sure that the resolved schema actually has a type definition that matches the type of the - # given default value, since anything else would be indicative of a nasty bug in schema - # generation. - default_value_type = docs_type_str(default_value) - resolved_schema_type_field = get_json_schema_type_field_for_value(source_schema, resolved_schema, default_value) - if resolved_schema_type_field.nil? - @logger.error "Schema has default value declared that does not match type of resolved schema: \ - \ - Source schema: #{to_pretty_json(source_schema)} \ - Default value: #{to_pretty_json(default_value)} (type: #{default_value_type}) \ - Resolved schema: #{to_pretty_json(resolved_schema)}" - exit 1 - end - - case default_value_type - when 'object' - # For objects, we set the default values on a per-property basis by trying to extract the - # value for each property from the object set as the default value. This is because of how we - # generally render documentation for fields that are objects, where we want to show the - # default value alongside the field itself, rather than specifying at the top level all of the - # default values... - # - # In other words, instead of this: - # - # buffer: - # default value: { type = "memory", max_events = 500 } - # - # type: - # ... - # max_events: - # ... - # - # we want to get this: - # - # buffer: - # type: - # default value: "memory" - # max_events: - # default value: 500 - # - resolved_properties = resolved_schema_type_field['options'] - resolved_properties.each do |property_name, resolved_property| - @logger.debug "Trying to apply default value for resolved property '#{property_name}'..." - property_default_value = default_value[property_name] - if !property_default_value.nil? - source_property = find_nested_object_property_schema(source_schema, property_name) - if !source_property.nil? - # If we found the source schema for the property itself, use that to cleanly apply - # default values to the property. - source_property_with_default = deep_copy(source_property) - source_property_with_default['default'] = property_default_value - apply_schema_default_value!(source_property_with_default, resolved_property) - - resolved_property['required'] = false - else - # We don't have a source for the property itself, presumably because we're dealing with - # a complex subschema, so just go based off of the type of the default value itself. - property_type_field = get_json_schema_type_field_for_value(source_property, resolved_property, property_default_value) - if !property_type_field.nil? - property_type_field['default'] = property_default_value - resolved_property['required'] = false - end - end - end - end - else - # We're dealing with an array or normal scalar or whatever, so just apply the default directly. - resolved_schema_type_field['default'] = default_value - end - - @logger.debug "Applied default value(s) to schema." - end -end - -def apply_schema_metadata!(source_schema, resolved_schema) - # Handle marking string schemas as templateable, which shows a special blurb in the rendered - # documentation HTML that explains what this means and links to the template syntax, etc. - is_templateable = get_schema_metadata(source_schema, 'docs::templateable') == true - string_type_def = resolved_schema.dig('type', 'string') - if !string_type_def.nil? && is_templateable - @logger.debug "Schema is templateable." - string_type_def['syntax'] = 'template' - end - - # TODO: Handle the niche case where we have an object schema without any of its own fields -- aka a map - # of optional key/value pairs i.e. tags -- and it allows templateable values. - - # Handle adding any defined examples to the type definition. - # - # TODO: Since a resolved schema could be represented by _multiple_ input types, and we only take - # examples in the form of strings, we don't have a great way to discern which type definition - # should get the examples applied to it (i.e. for a number-or-enum-of-strings schema, do we - # apply the examples to the number type def or the enum type def?) so we simply... apply them to - # any type definition present in the resolved schema. - # - # We might be able to improve this in the future with a simply better heuristic, dunno. This might - # also work totally fine as-is! - examples = get_schema_metadata(source_schema, 'docs::examples') - if !examples.nil? - flattened_examples = [examples].flatten.map { |example| - if example.is_a?(Hash) - sort_hash_nested(example) - else - example - end - } - - @logger.debug "Schema has #{flattened_examples.length} example(s)." - resolved_schema['type'].each { |type_name, type_def| - # We need to recurse one more level if we're dealing with an array type definition, as we need - # to stick the examples on the type definition for the array's `items`. There might also be - # multiple type definitions under `items`, but we'll cross that bridge if/when we get to it. - case type_name - when 'array' - type_def['items']['type'].each { |subtype_name, subtype_def| - if subtype_name != 'array' - subtype_def['examples'] = flattened_examples - end - } - else - type_def['examples'] = flattened_examples - end - } - end - - # Apply any units that have been specified. - type_unit = get_schema_metadata(source_schema, 'docs::type_unit') - if !type_unit.nil? - schema_type = numeric_schema_type(resolved_schema) - if !schema_type.nil? - resolved_schema['type'][schema_type]['unit'] = type_unit.to_s - end - end - - # Modify the `syntax` of a string type definition if overridden via metadata. - syntax_override = get_schema_metadata(source_schema, 'docs::syntax_override') - if !syntax_override.nil? - if resolved_schema_type?(resolved_schema) != "string" - @logger.error "Non-string schemas should not use the `syntax_override` metadata attribute." - exit 1 - end - - resolved_schema['type']['string']['syntax'] = syntax_override.to_s - end -end - -# Reconciles the resolved schema, detecting and fixing any logical inconsistencies. -# -# This provides a mechanism to fix up any inconsistencies that are created during the resolution -# process that would otherwise be very complex to fix in the resolution codepath. Sometimes, -# inconsistencies are only present after resolving merged subschemas, and so on, and so this -# function serves as a spot to do such reconciliation, as it is called right before returning a -# resolved schema. -def reconcile_resolved_schema!(resolved_schema) - @logger.debug "Reconciling resolved schema..." - - # Only works if `type` is an object, which it won't be in some cases, such as a schema that maps - # to a cycle entrypoint, or is hidden, and so on. - if !resolved_schema['type'].is_a?(Hash) - @logger.debug "Schema was not a fully resolved schema; reconciliation not applicable." - return - end - - @logger.debug "Reconciling schema: #{to_pretty_json(resolved_schema)}" - - # If we're dealing with an object schema, run this for each of its properties. - object_properties = resolved_schema.dig('type', 'object', 'options') - if !object_properties.nil? - object_properties.values.each { |resolved_property| reconcile_resolved_schema!(resolved_property) } - else - # Look for required/default value inconsistencies. - # - # One example is the `lua` transform and the `version` field. It's marked as required by the - # version 2 configuration types, but it's optional for version 1, which allows the deserializer to - # only deserialize things as version 1 if `version` isn't set, avoiding a backwards-incompatible - # situation... but in this script, we only compare the subschemas in terms of their const-ness, - # and don't look at the `required` portion. - # - # This means that we can generate output for a field that says it has a default value of `null` - # but is a required field, which is a logical inconsistency in terms of the Cue schema where we - # import the generated output of this script: it doesn't allow setting a default value for a field - # if the field is required, and vice versa. - if resolved_schema['required'] - # For all schema type fields, see if they have a default value equal to `nil`. If so, remove - # the `default` field entirely. - resolved_schema['type'].keys.each { |type_field| - type_field_default_value = resolved_schema['type'][type_field].fetch('default', :does_not_exist) - if type_field_default_value.nil? - @logger.debug "Removing null `default` field for type field '#{type_field}'..." - - resolved_schema['type'][type_field].delete('default') - end - } - end - - # Look for merged string const values that need to become an enum. - # - # As part of our enum schema resolving, we have a fallback mode where we resolve each subschema - # and merge them together in a nested fashion, under the assumption that they don't overlap in - # an invalid way i.e. same field in two schemas but with differing/incompatible types. - # - # This works, but one area it falls down is where a field is a `const` in different subschemas, - # where even if the value is the same type for all overlaps of `const`, the normal nested merge - # would result in a last-write-wins for that field. We compensate for this by using a - # schema-aware nested merge, where if we're merging a field called `const`, we turn it into a - # array of the const data, which includes the const value itself and the description of the enum - # variant. - # - # The final step would be to change from `const` to `enum`, because Cue doesn't recognize the - # `const` type, regardless of whether it's a single value or a map of key/value pairs. We cannot - # do that in the merge, however, because there's no way to specify a new resulting key to use, - # only how the values should be merged. - # - # Thus, we handle that here by looking for any `const` field that has an array value, turning it - # into a map of const value to enum variant description. Since no normal schema would have a - # `const` value that was an array value to begin with, we can safely search for such instances - # and confidently know that the field can be renamed from `const` to `enum`. - resolved_schema['type'].keys.each { |type_field| - const_type_field = resolved_schema.dig('type', type_field, 'const') - if !const_type_field.nil? - @logger.debug "Converting `const` values to `enum` for type field '#{type_field}'..." - @logger.debug "Resolved schema: #{resolved_schema}" - - enum_values = if const_type_field.is_a?(Array) - const_type_field - .map { |const| [const['value'], const['description']] } - else - # If the value isn't already an array, we'll create the enum values map directly. - { const_type_field['value'] => const_type_field['description'] } - end - - @logger.debug "Reconciled enum values for const: #{enum_values}" - - resolved_schema['type'][type_field].delete('const') - resolved_schema['type'][type_field]['enum'] = enum_values - end - } - - # Push the schema description into the type field for string consts. - # - # As part of resolving const schemas, we need to use their descriptions when eventually - # converting them to an enum schema that is supported on the Cue side. This implies the const - # value becoming a key in a map, whose value is the description of the const schema. - # - # We do that here because it's simpler to not have to special case the addition of the - # description when resolving a const schema, as we do so uniformly as part of the final steps of - # resolving a schema in general, but before reconciliation is triggered. - resolved_schema['type'].keys.each { |type_field| - const_type_field = resolved_schema.dig('type', type_field, 'const') - if !const_type_field.nil? && !const_type_field.is_a?(Array) - @logger.debug "Adding schema description to `const` type field '#{type_field}'..." - - schema_description = resolved_schema['description'] - const_type_field['description'] = schema_description unless schema_description.nil? - end - } - end - - @logger.debug "Reconciled resolved schema." -end - -def get_rendered_description_from_schema(schema) - # Grab both the raw description and raw title, and if the title is empty, just use the - # description, otherwise concatenate the title and description with newlines so that there's a - # whitespace break between the title and description. - raw_description = schema.fetch('description', '') - raw_title = schema.fetch('title', '') - - description = raw_title.empty? ? raw_description : "#{raw_title}\n\n#{raw_description}" - description.strip -end - -def unwrap_resolved_schema(root_schema, schema_name, friendly_name) - @logger.info "[*] Resolving schema definition for #{friendly_name}..." - - # Try and resolve the schema, unwrapping it as an object schema which is a requirement/expectation - # of all component-level schemas. We additionally sort all of the object properties, which makes - # sure the docs are generated in alphabetical order. - resolved_schema = resolve_schema_by_name(root_schema, schema_name) - - unwrapped_resolved_schema = resolved_schema.dig('type', 'object', 'options') - if unwrapped_resolved_schema.nil? - @logger.error 'Configuration types must always resolve to an object schema.' - exit 1 - end - - return sort_hash_nested(unwrapped_resolved_schema) -end - -def render_and_import_schema(unwrapped_resolved_schema, friendly_name, config_map_path, cue_relative_path) - - # Set up the appropriate structure for the value based on the configuration map path. It defines - # the nested levels of the map where our resolved schema should go, as well as a means to generate - # an appropriate prefix for our temporary file. - data = {} - last = data - config_map_path.each do |segment| - last[segment] = {} if last[segment].nil? - - last = last[segment] - end - - last['configuration'] = unwrapped_resolved_schema - - config_map_path.prepend('config-schema-base') - tmp_file_prefix = config_map_path.join('-') - - final_json = to_pretty_json(data) - - # Write the resolved schema as JSON, which we'll then use to import into a Cue file. - json_output_file = write_to_temp_file(["config-schema-#{tmp_file_prefix}-", '.json'], final_json) - @logger.info "[βœ“] Wrote #{friendly_name} schema to '#{json_output_file}'. (#{final_json.length} bytes)" - - # Try importing it as Cue. - @logger.info "[*] Importing #{friendly_name} schema as Cue file..." - cue_output_file = "website/cue/reference/#{cue_relative_path}" - unless system(@cue_binary_path, 'import', '-f', '-o', cue_output_file, '-p', 'metadata', json_output_file) - @logger.error "[!] Failed to import #{friendly_name} schema as valid Cue." - exit 1 - end - @logger.info "[βœ“] Imported #{friendly_name} schema to '#{cue_output_file}'." -end - -def render_and_import_generated_component_schema(root_schema, schema_name, component_type) - friendly_name = "generated #{component_type} configuration" - unwrapped_resolved_schema = unwrap_resolved_schema(root_schema, schema_name, friendly_name) - render_and_import_schema( - unwrapped_resolved_schema, - friendly_name, - ["generated", "components", "#{component_type}s"], - "components/generated/#{component_type}s.cue" - ) -end - -def render_and_import_component_schema(root_schema, schema_name, component_type, component_name) - friendly_name = "'#{component_name}' #{component_type} configuration" - unwrapped_resolved_schema = unwrap_resolved_schema(root_schema, schema_name, friendly_name) - render_and_import_schema( - unwrapped_resolved_schema, - friendly_name, - ["generated", "components", "#{component_type}s", component_name], - "components/#{component_type}s/generated/#{component_name}.cue" - ) -end - -def render_and_import_generated_top_level_config_schema(root_schema) - top_level_config_schema = {} - - # Define logical groupings for top-level configuration fields - # These groups will be used to organize separate documentation pages - field_groups = { - # Pipeline component containers - 'sources' => 'pipeline_components', - 'transforms' => 'pipeline_components', - 'sinks' => 'pipeline_components', - 'enrichment_tables' => 'pipeline_components', - - # Individual feature pages - 'api' => 'api', - 'schema' => 'schema', - 'log_schema' => 'schema', - 'secret' => 'secrets', - - # Global options (everything else defaults to this) - } - - group_metadata = { - 'global_options' => { - 'title' => 'Global Options', - 'description' => 'Global configuration options that apply to Vector as a whole.', - 'order' => 1 - }, - 'pipeline_components' => { - 'title' => 'Pipeline Components', - 'description' => 'Configure sources, transforms, sinks, and enrichment tables for your observability pipeline.', - 'order' => 2 - }, - 'api' => { - 'title' => 'API', - 'description' => 'Configure Vector\'s observability API.', - 'order' => 3 - }, - 'schema' => { - 'title' => 'Schema', - 'description' => 'Configure Vector\'s internal schema system for type tracking and validation.', - 'order' => 4 - }, - 'secrets' => { - 'title' => 'Secrets', - 'description' => 'Configure secrets management for secure configuration.', - 'order' => 5 - } - } - - # Usage of #[serde(flatten)] creates multiple schemas in the `allOf` array: - # - One or more schemas contain ConfigBuilder's direct fields - # - One or more schemas contain flattened GlobalOptions fields - all_of_schemas = root_schema['allOf'] || [] - - if all_of_schemas.empty? - @logger.error "Could not find ConfigBuilder allOf schemas in root schema" - return - end - - # Collect all properties from all allOf schemas into a single hash. - # Since ConfigBuilder uses #[serde(flatten)], field names are unique across all schemas. - all_properties = all_of_schemas.reduce({}) do |acc, schema| - acc.merge(schema['properties'] || {}) - end - - @logger.info "[*] Found #{all_properties.keys.length} total properties across #{all_of_schemas.length} allOf schemas" - - # Process each property once - all_properties.each do |field_name, field_schema| - # Skip fields marked with docs::hidden - metadata = field_schema['_metadata'] || {} - if metadata['docs::hidden'] - @logger.info "[*] Skipping '#{field_name}' (marked as docs::hidden)" - next - end - - # Extract and resolve the field - @logger.info "[*] Extracting '#{field_name}' field from ConfigBuilder..." - resolved_field = resolve_schema(root_schema, field_schema) - - # Assign group metadata to organize the documentation - if field_groups.key?(field_name) - group_name = field_groups[field_name] - resolved_field['group'] = group_name - @logger.debug "Assigned '#{field_name}' to group '#{group_name}'" - else - # Default to global_options for any fields not explicitly grouped - resolved_field['group'] = 'global_options' - @logger.debug "Assigned '#{field_name}' to default group 'global_options'" - end - - top_level_config_schema[field_name] = resolved_field - @logger.info "[βœ“] Resolved '#{field_name}'" - end - - # Build the final data structure with both configuration and group metadata - friendly_name = "configuration" - config_map_path = ["generated", "configuration"] - cue_relative_path = "generated/configuration.cue" - - # Set up the structure for the value based on the configuration map path - data = {} - last = data - config_map_path.each do |segment| - last[segment] = {} if last[segment].nil? - last = last[segment] - end - - # Add both the configuration schema and the group metadata - last['configuration'] = top_level_config_schema - last['groups'] = group_metadata - - config_map_path.prepend('config-schema-base') - tmp_file_prefix = config_map_path.join('-') - final_json = to_pretty_json(data) - - # Write the resolved schema as JSON - json_output_file = write_to_temp_file(["config-schema-#{tmp_file_prefix}-", '.json'], final_json) - @logger.info "[βœ“] Wrote #{friendly_name} schema to '#{json_output_file}'. (#{final_json.length} bytes)" - - # Import it as Cue - @logger.info "[*] Importing #{friendly_name} schema as Cue file..." - cue_output_file = "website/cue/reference/#{cue_relative_path}" - unless system(@cue_binary_path, 'import', '-f', '-o', cue_output_file, '-p', 'metadata', json_output_file) - @logger.error "[!] Failed to import #{friendly_name} schema as valid Cue." - exit 1 - end - @logger.info "[βœ“] Imported #{friendly_name} schema to '#{cue_output_file}'." -end - -if ARGV.empty? - puts 'usage: extract-component-schema.rb ' - exit 1 -end - -# Ensure that Cue is present since we need it to import our intermediate JSON representation. -if @cue_binary_path.nil? - puts 'Failed to find \'cue\' binary on the current path. Install \'cue\' (or make it available on the current path) and try again.' - exit 1 -end - -schema_path = ARGV[0] -root_schema = JSON.parse(File.read(schema_path)) - -component_types = %w[source transform sink] - -# First off, we generate the component type configuration bases. These are the high-level -# configuration settings that are universal on a per-component type basis. -# -# For example, the "generated" configuration for a sink would be the inputs, buffer settings, healthcheck -# settings, and proxy settings... and then the configuration for a sink would be those, plus -# whatever the sink itself defines. -component_bases = root_schema['definitions'].filter_map do |key, definition| - component_base_type = get_schema_metadata(definition, 'docs::component_base_type') - { component_base_type => key } if component_types.include? component_base_type -end -.reduce { |acc, item| nested_merge(acc, item) } - -component_bases.each do |component_type, schema_name| - render_and_import_generated_component_schema(root_schema, schema_name, component_type) -end - -# Now we'll generate the base configuration for each component. -all_components = root_schema['definitions'].filter_map do |key, definition| - component_type = get_schema_metadata(definition, 'docs::component_type') - component_name = get_schema_metadata(definition, 'docs::component_name') - { component_type => { component_name => key } } if component_types.include? component_type -end -.reduce { |acc, item| nested_merge(acc, item) } - -all_components.each do |component_type, components| - components.each do |component_name, schema_name| - render_and_import_component_schema(root_schema, schema_name, component_type, component_name) - end -end - -# Finally, generate the top-level Vector configuration schema. We extract ALL top-level config fields directly from the -# ConfigBuilder struct (defined in src/config/builder.rs) by processing its allOf schemas. ConfigBuilder is the single -# source of truth for what's actually allowed at the top level of Vector's configuration file. -render_and_import_generated_top_level_config_schema(root_schema) diff --git a/scripts/generate-release-cue.rb b/scripts/generate-release-cue.rb deleted file mode 100755 index 80a35c0683015..0000000000000 --- a/scripts/generate-release-cue.rb +++ /dev/null @@ -1,464 +0,0 @@ -#!/usr/bin/env ruby - -# release-meta.rb -# -# SUMMARY -# -# A script that prepares the release .meta/releases/vX.X.X.toml file. -# Afterwards, the `make generate` command should be used to refresh -# the generated files against the new release metadata. - -# -# Setup -# - -require "json" -require "time" -require "optparse" -require 'pathname' -require_relative "util/commit" -require_relative "util/git_log_commit" -require_relative "util/printer" -require_relative "util/release" -require_relative "util/version" - -# Function to find the repository root by looking for .git directory -def find_repo_root - # Get the absolute path of the current script - script_path = File.expand_path(__FILE__) - dir = Pathname.new(script_path).dirname - - # Walk up the directory tree until we find .git or reach the filesystem root - loop do - return dir.to_s if File.exist?(File.join(dir, '.git')) - parent = dir.parent - raise "Could not find repository root (no .git directory found)" if parent == dir # Reached filesystem root - dir = parent - end -end - -# -# Constants -# - -ROOT = find_repo_root -RELEASE_REFERENCE_DIR = File.join(ROOT, "website", "cue", "reference", "releases") -CHANGELOG_DIR = File.join(ROOT, "changelog.d") -TYPES = ["chore", "docs", "feat", "fix", "enhancement", "perf", "revert"] -TYPES_THAT_REQUIRE_SCOPES = ["feat", "enhancement", "fix"] - -# Parse command-line options -options = {} -OptionParser.new do |opts| - opts.banner = "Usage: #{File.basename(__FILE__)} [options]" - - opts.on("--new-version VERSION", "Specify the new version (e.g., 1.2.3)") do |v| - options[:new_version] = v - end - opts.on("--[no-]interactive", "Enable/disable interactive prompts (default: true)") do |i| - options[:interactive] = i - end - opts.on_tail("-h", "--help", "Show this help message") do - puts opts - exit - end -end.parse! - -# -# Functions -# -# Sorted alphabetically. -# - -# Creates and updates the new release log file located at -# -# /.meta/releases/X.X.X.log -# -# This file is created from outstanding commits since the last release. -# It's meant to be a starting point. The resulting file should be reviewed -# and edited by a human before being turned into a cue file. -def create_log_file!(current_commits, new_version, interactive) - release_log_path = "#{RELEASE_REFERENCE_DIR}/#{new_version}.log" - - # Grab all existing commits - existing_commits = get_existing_commits! - - # Filter out duplicate commits - new_commits = current_commits.select do |current_commit| - !existing_commits.any? { |existing_commit| existing_commit.eql?(current_commit) } - end - - new_commit_lines = new_commits.collect { |c| c.to_git_log_commit.to_raw }.join("\n") - - if new_commits.any? - if File.exists?(release_log_path) - if interactive - words = <<~EOF - I found #{new_commits.length} new commits since you last ran this - command. So that I don't erase any other work in that file, please - manually add the following commit lines: - - #{new_commit_lines.split("\n").collect { |line| " #{line}" }.join("\n")} - - To: - - #{release_log_path} - - All done? Ready to proceed? - EOF - - if Util::Printer.get(words, ["y", "n"]) == "n" - Util::Printer.error!("Ok, re-run this command when you're ready.") - end - end - else - File.open(release_log_path, 'w+') do |file| - file.write(new_commit_lines) - end - - puts interactive - if interactive - words = <<~EOF - I've created a release log file here: - - #{release_log_path} - - Please review the commits and *adjust the commit messages as necessary*. - - All done? Ready to proceed? - EOF - - if Util::Printer.get(words, ["y", "n"]) == "n" - Util::Printer.error!("Ok, re-run this command when you're ready.") - end - end - end - end - - release_log_path -end - -def retire_changelog_entries!() - - Dir.glob("#{CHANGELOG_DIR}/*.md") do |fname| - if File.basename(fname) == "README.md" - next - end - system('git', 'rm', fname) - end -end - -def generate_changelog!(new_version) - - entries = "" - - Dir.glob("#{CHANGELOG_DIR}/*.md") do |fname| - - if File.basename(fname) == "README.md" - next - end - - if entries != "" - entries += ",\n" - end - - fragment_contents = File.open(fname) - - # add the GitHub username for any fragments - # that have an authors field at the end of the - # fragment. This is generally used for external - # contributor PRs. - lines = fragment_contents.to_a - last = lines.last - contributors = Array.new - - if last.start_with?("authors: ") - contributors = last[9..].split(" ").map(&:strip) - - # remove that line from the description - lines.pop() - end - - description = lines.join("").strip() - - # get the PR number of the changelog fragment. - # the fragment type is not used in the Vector release currently. - basename = File.basename(fname, ".md") - parts = basename.split(".") - - if parts.length() != 2 - Util::Printer.error!("Changelog fragment #{fname} is invalid (exactly two period delimiters required).") - end - - fragment_type = parts[1] - - # map the fragment type to Vector's semantic types - # https://github.com/vectordotdev/vector/blob/master/.github/semantic.yml#L13 - # the type "chore" isn't rendered in the changelog on the website currently, - # but we are mapping "breaking" and "deprecations" to that type, and both of - # these are handled in the upgrade guide separately. - - # NOTE: If the fragment types are altered, update both the 'changelog.d/README.md' and - # 'scripts/check_changelog_fragments.sh' accordingly. - type = "" - if fragment_type == "breaking" - type = "chore" - elsif fragment_type == "security" or fragment_type == "fix" - type = "fix" - elsif fragment_type == "deprecation" - type = "chore" - elsif fragment_type == "feature" - type = "feat" - elsif fragment_type == "enhancement" - type = "enhancement" - else - Util::Printer.error!("Changelog fragment #{fname} is invalid. Fragment type #{fragment_type} unrecognized.") - end - - # Note: `pr_numbers`, `scopes` and `breaking` are being omitted from the entries. - # These are currently not required for rendering in the website. - entry = "{\n" + - "type: #{type.to_json}\n" + - "description: \"\"\"\n" + - "#{description}\n" + - "\"\"\"\n" - - if contributors.length() > 0 - entry += "contributors: #{contributors.to_json}\n" - end - - entry += "}" - - entries += entry - end - - if entries != "" - retire_changelog_entries!() - end - - entries -end - -def create_release_file!(new_version) - release_log_path = "#{RELEASE_REFERENCE_DIR}/#{new_version}.log" - git_log = Vector::GitLogCommit.from_file!(release_log_path) - commits = Vector::Commit.from_git_log!(git_log) - - release_reference_path = "#{RELEASE_REFERENCE_DIR}/#{new_version}.cue" - - if commits.any? - commits.each(&:validate!) - cue_commits = commits.collect(&:to_cue_struct).join(",\n ") - - changelog_entries = generate_changelog!(new_version) - - if File.exists?(release_reference_path) - words = - <<~EOF - It looks like you already have a release file: - - #{release_reference_path} - - So that I don't overwrite your work, please copy these commits into - the release file: - - #{cue_commits} - - All done? Ready to proceed? - EOF - - if Util::Printer.get(words, ["y", "n"]) == "n" - Util::Printer.error!("Ok, re-run this command when you're ready.") - end - else - File.open(release_reference_path, 'w+') do |file| - file.write( - <<~EOF - package metadata - - releases: #{new_version.to_json}: { - date: #{Date.today.to_json} - codename: "" - - whats_next: [] - - changelog: [ - #{changelog_entries} - ] - - commits: [ - #{cue_commits} - ] - } - EOF - ) - end - - `cue fmt #{release_reference_path}` - - words = - <<~EOF - All done! I've create a release file at: - - #{release_reference_path} - - I recommend previewing the website changes with this release. - EOF - - Util::Printer.success(words) - end - - `cue fmt #{release_reference_path}` - - true - else - false - end -end - -def get_commits_since(last_version) - Vector::Commit.fetch_since(last_version) -end - -# Grabs all existing commits that are included in the `.meta/releases/*.toml` -# files. We grab _all_ commits to ensure we do not include duplicate commits -# in the new release. -def get_existing_commits! - releases = Vector::Release.all!(RELEASE_REFERENCE_DIR) - release_commits = releases.collect(&:commits).flatten - - release_log_paths = Dir.glob("#{RELEASE_REFERENCE_DIR}/*.log").to_a - - log_commits = - release_log_paths.collect do |release_log_path| - git_log = Vector::GitLogCommit.from_file!(release_log_path) - Vector::Commit.from_git_log!(git_log) - end.flatten - - release_commits + log_commits -end - -def get_new_version(last_version, current_commits) - next_version = - if current_commits.any? { |c| c.breaking_change? } - next_version = - if last_version.major == 0 - "0.#{last_version.minor + 1}.0" - else - "#{last_version.major + 1}.0.0" - end - - words = "It looks like the new commits contain breaking changes. " + - "Would you like to use the recommended version #{next_version} for " + - "this release?" - - if Util::Printer.get(words, ["y", "n"]) == "y" - next_version - else - nil - end - elsif current_commits.any? { |c| c.new_feature? } - next_version = "#{last_version.major}.#{last_version.minor + 1}.0" - - words = "It looks like this release contains commits with new features. " + - "Would you like to use the recommended version #{next_version} for " + - "this release?" - - if Util::Printer.get(words, ["y", "n"]) == "y" - next_version - else - nil - end - elsif current_commits.any? { |c| c.fix? } - next_version = "#{last_version.major}.#{last_version.minor}.#{last_version.patch + 1}" - - words = "It looks like this release contains commits with bug fixes. " + - "Would you like to use the recommended version #{next_version} for " + - "this release?" - - if Util::Printer.get(words, ["y", "n"]) == "y" - next_version - else - nil - end - end - - version_string = next_version || Util::Printer.get("What is the next version you are releasing? (current version is #{last_version})") - - version = - begin - Util::Version.new(version_string) - rescue ArgumentError => e - Util::Printer.invalid("It looks like the version you entered is invalid: #{e.message}") - get_new_version(last_version, current_commits) - end - - if last_version.bump_type(version).nil? - Util::Printer.invalid("The version you entered must be a single patch, minor, or major bump") - get_new_version(last_version, current_commits) - else - version - end -end - -def migrate_highlights(new_version) - Dir.glob("#{HIGHLIGHTS_ROOT}/*.md").to_a.each do |highlight_path| - content = File.read(highlight_path) - release_line = "\nrelease: \"nightly\"\n" - - if content.include?(release_line) - new_content = content.replace(release_line, "\nrelease: \"#{new_version}\"\n") - - File.open(highlight_path, 'w+') do |file| - file.write(new_content) - end - end - end -end - -# -# Execute -# -script_dir = File.expand_path(File.dirname(__FILE__)) -Dir.chdir script_dir - -Util::Printer.title("Creating release meta file...") - -all_tags = `git tag --list --sort=-v:refname`.lines.map(&:chomp) -valid_tag = all_tags.find do |t| - !t.start_with?('vdev-v') && - t.match(/^v\d+\.\d+\.\d+$/) -end - -if valid_tag.nil? - Util::Printer.error!("No valid semantic version tag found (e.g. v1.2.3)") - exit 1 -end - -last_tag = valid_tag -last_version = Util::Version.new(last_tag.gsub(/^v/, '')) -current_commits = get_commits_since(last_version) - -new_version_string = options[:new_version] -if new_version_string - begin - new_version = Util::Version.new(new_version_string) - if last_version.bump_type(new_version).nil? - Util::Printer.error!("The specified version '#{new_version_string}' must be a single patch, minor, or major bump from #{last_version}") - exit 1 - end - rescue ArgumentError => e - Util::Printer.error!("Invalid version specified: #{e.message}") - exit 1 - end -else - new_version = get_new_version(last_version, current_commits) -end - -log_file_path = create_log_file!(current_commits, new_version, options[":interactive"]) -create_release_file!(new_version) -File.delete(log_file_path) - -#Util::Printer.title("Migrating all nightly associated highlights to #{new_version}...") - -#migrate_highlights(new_version) diff --git a/scripts/package-deb.sh b/scripts/package-deb.sh index ede151f290b3b..6bbc86720e46e 100755 --- a/scripts/package-deb.sh +++ b/scripts/package-deb.sh @@ -13,6 +13,7 @@ set -x # $TARGET a target triple. ex: arm64-apple-darwin (no default) TARGET="${TARGET:?"You must specify a target triple, ex: arm64-apple-darwin"}" +PROFILE="${PROFILE:-release}" # # Local vars @@ -39,8 +40,8 @@ echo "TARGET: $TARGET" td="$(mktemp -d)" pushd "$td" tar -xvf "$ABSOLUTE_ARCHIVE_PATH" -mkdir -p "$PROJECT_ROOT/target/$TARGET/release" -mv "vector-$TARGET/bin/vector" "$PROJECT_ROOT/target/$TARGET/release" +mkdir -p "$PROJECT_ROOT/target/$TARGET/$PROFILE" +mv "vector-$TARGET/bin/vector" "$PROJECT_ROOT/target/$TARGET/$PROFILE" popd rm -rf "$td" @@ -71,7 +72,7 @@ cat LICENSE NOTICE >"$PROJECT_ROOT/target/debian-license.txt" # --no-build # because this step should follow a build -cargo deb --target "$TARGET" --deb-version "${PACKAGE_VERSION}-1" --variant "$TARGET" --no-build --no-strip +cargo deb --target "$TARGET" --deb-version "${PACKAGE_VERSION}-1" --variant "$TARGET" --no-build --no-strip --profile "$PROFILE" # Rename the resulting .deb file to remove TARGET from name. for file in target/"${TARGET}"/debian/*.deb; do diff --git a/scripts/package-rpm.sh b/scripts/package-rpm.sh index bedb4486731cd..c2d5453daab6e 100755 --- a/scripts/package-rpm.sh +++ b/scripts/package-rpm.sh @@ -69,6 +69,9 @@ case "$TARGET" in armv7-*-gnueabihf) STRIP_TOOL="arm-linux-gnueabihf-strip" ;; *) STRIP_TOOL="strip" ;; esac +# Fall back to the host's strip when building natively on the target arch +# (e.g., aarch64 native build doesn't have aarch64-linux-gnu-strip). +command -v "$STRIP_TOOL" >/dev/null 2>&1 || STRIP_TOOL="strip" # Perform the build. rpmbuild \ diff --git a/scripts/release-commit.rb b/scripts/release-commit.rb deleted file mode 100755 index 49e3495b57b83..0000000000000 --- a/scripts/release-commit.rb +++ /dev/null @@ -1,121 +0,0 @@ -#!/usr/bin/env ruby - -# release-commit.rb -# -# SUMMARY -# -# Commits and tags the pending release - -# -# Setup -# - -require "json" -require_relative "util/printer" -require_relative "util/release" - -# -# Constants -# - -ROOT_DIR = "." -RELEASE_REFERENCE_DIR = File.join(ROOT_DIR, "docs", "reference", "releases") - -# -# Functions -# - -def bump_cargo_version(version) - # Cargo.toml - content = File.read("#{ROOT_DIR}/Cargo.toml") - new_content = bump_version(content, version) - File.write("#{ROOT_DIR}/Cargo.toml", new_content) - - # Cargo.lock - content = File.read("#{ROOT_DIR}/Cargo.lock") - new_content = bump_version(content, version) - File.write("#{ROOT_DIR}/Cargo.lock", new_content) -end - -def bump_version(content, version) - content.sub( - /name = "vector"\nversion = "([a-z0-9.-]*)"\n/, - "name = \"vector\"\nversion = \"#{version}\"\n" - ) -end - -def release_exists?(release) - errors = `git rev-parse tags/v#{release.version} 2>&1 >/dev/null` - errors == "" -end - -# -# Execute -# - -release = Vector::Release.all!(RELEASE_REFERENCE_DIR).last - -if release_exists?(release) - Util::Printer.error!( - <<~EOF - It looks like release v#{release.version} has already been released. A tag for this release already exists. - - This command will only release the latest release. If you're trying to release from an older major or minor version, you must do so from that branch. - EOF - ) -else - Util::Printer.title("Committing and tagging release") - - bump_cargo_version(release.version) - - Util::Printer.success("Bumped the version in Cargo.toml & Cargo.lock to #{release.version}") - - branch_name = "#{release.version.major}.#{release.version.minor}" - - commands = - <<~EOF - git add #{ROOT_DIR} -A - git commit -sam 'chore: Prepare v#{release.version} release' || true - git tag -a v#{release.version} -m "v#{release.version}" - git branch v#{branch_name} 2>/dev/null || true - EOF - - commands.chomp! - - status = `git status --short`.chomp! - - words = - <<~EOF - We'll be releasing v#{release.version} with the following commands: - - #{commands} - - Your current `git status` is: - - #{status} - - Proceed to execute the above commands? - EOF - - if Util::Printer.get(words, ["y", "n"]) == "n" - Util::Printer.error!("Ok, I've aborted. Please re-run this command when you're ready.") - end - - commands.chomp.split("\n").each do |command| - system(command) - - if !$?.success? - Util::Printer.error!( - <<~EOF - Command failed! - - #{command} - - Produced the following error: - - #{$?.inspect} - EOF - ) - end - end -end diff --git a/scripts/release-s3.sh b/scripts/release-s3.sh index 9889fa556e4eb..8f6c51de08669 100755 --- a/scripts/release-s3.sh +++ b/scripts/release-s3.sh @@ -36,11 +36,27 @@ ls "$td_nightly" # # A helper function for verifying a published artifact. # +# Retries a content mismatch as well as a 404, since packages.timber.io is +# fronted by a CDN and an object we just overwrote via `aws s3 rm` + `cp` can +# serve stale bytes at the edge for a while. verify_artifact() { local URL="$1" local FILENAME="$2" + local attempts=7 + local delay=1 echo "Verifying $URL" - cmp <(wget -qO- --retry-on-http-error=404 --wait 10 --tries "$VERIFY_RETRIES" "$URL") "$FILENAME" + for ((attempt = 1; attempt <= attempts; attempt++)); do + if cmp <(wget -qO- --retry-on-http-error=404 --wait 10 --tries "$VERIFY_RETRIES" "$URL") "$FILENAME"; then + return 0 + fi + if (( attempt < attempts )); then + echo "Attempt $attempt/$attempts did not match (likely stale CDN cache); retrying in ${delay}s" + sleep "$delay" + delay=$((delay * 2)) + fi + done + echo "Verification of $URL failed after $attempts attempts" + return 1 } # @@ -59,15 +75,6 @@ if [[ "$CHANNEL" == "nightly" ]]; then aws s3 cp "$td_nightly" "s3://packages.timber.io/vector/nightly/latest" --recursive --sse --acl public-read echo "Uploaded archives" - echo "Redirecting old artifact names" - for file in $(aws s3api list-objects-v2 --bucket packages.timber.io --prefix "vector/$i/" --query 'Contents[*].Key' --output text | tr "\t" "\n" | grep '\-nightly'); do - file=$(basename "$file") - # vector-nightly-amd64.deb -> vector-amd64.deb - echo -n "" | aws s3 cp - "s3://packages.timber.io/vector/nightly/$DATE/${file/-nightly/}" --website-redirect "/vector/nightly/$DATE/$file" --acl public-read - echo -n "" | aws s3 cp - "s3://packages.timber.io/vector/nightly/latest/${file/-nightly/}" --website-redirect "/vector/nightly/latest/$file" --acl public-read - done - echo "Redirected old artifact names" - # Verify that the files exist and can be downloaded echo "Waiting for $VERIFY_TIMEOUT seconds before running the verifications" sleep "$VERIFY_TIMEOUT" diff --git a/scripts/run-integration-test.sh b/scripts/run-integration-test.sh index a1b22ce4d6c4c..2657116eeb2e6 100755 --- a/scripts/run-integration-test.sh +++ b/scripts/run-integration-test.sh @@ -34,6 +34,7 @@ Options: -v Increase verbosity; repeat for more (e.g. -vv or -vvv) -e One or more environments to run (repeatable or comma-separated). If provided, these are used as TEST_ENVIRONMENTS instead of auto-discovery. + -c Collect code coverage (outputs target/coverage/lcov.info) Notes: - All existing two-argument invocations remain compatible: @@ -45,7 +46,8 @@ USAGE # Parse options # Note: options must come before positional args (standard getopts behavior) TEST_ENV="" -while getopts ":hr:v:e:" opt; do +COVERAGE=false +while getopts ":hr:v:e:c" opt; do case "$opt" in h) usage @@ -64,6 +66,9 @@ while getopts ":hr:v:e:" opt; do e) TEST_ENV="$OPTARG" ;; + c) + COVERAGE=true + ;; \?) echo "ERROR: unknown option: -$OPTARG" >&2 usage @@ -113,6 +118,10 @@ else fi fi +# Remove stale combined coverage from a previous (possibly failed) attempt so +# retries via nick-fields/retry don't append to leftover data. +rm -f target/coverage/lcov-combined.info + for TEST_ENV in "${TEST_ENVIRONMENTS[@]}"; do # Execution flow for each environment: # 1. Clean up previous test output @@ -134,10 +143,27 @@ for TEST_ENV in "${TEST_ENVIRONMENTS[@]}"; do print_compose_logs_on_failure "$START_RET" if [[ "$START_RET" -eq 0 ]]; then - $vdev_cmd "${VERBOSITY}" "${TEST_TYPE}" test --retries "$RETRIES" "${TEST_NAME}" "${TEST_ENV}" + COVERAGE_FLAG="" + [[ "$COVERAGE" == "true" ]] && COVERAGE_FLAG="--coverage" + + $vdev_cmd "${VERBOSITY}" "${TEST_TYPE}" test --retries "$RETRIES" ${COVERAGE_FLAG} "${TEST_NAME}" "${TEST_ENV}" RET=$? print_compose_logs_on_failure "$RET" + # Normalize source paths in coverage report so they are relative to the repo root. + # The test runner container mounts source at /home/vector; strip that prefix so + # Datadog can resolve files against the repository root (e.g. SF:src/foo.rs). + # Append each environment's coverage to a combined file so multi-env services + # preserve all coverage data (not just the last environment). + if [[ "$COVERAGE" == "true" && "$RET" -eq 0 ]]; then + LCOV_FILE="target/coverage/lcov.info" + if [[ -f "$LCOV_FILE" ]]; then + sed -i 's|SF:/home/vector/|SF:|g' "$LCOV_FILE" + cat "$LCOV_FILE" >> target/coverage/lcov-combined.info + rm "$LCOV_FILE" + fi + fi + # Upload test results only if the vdev test step ran ./scripts/upload-test-results.sh else @@ -154,4 +180,9 @@ for TEST_ENV in "${TEST_ENVIRONMENTS[@]}"; do fi done +# Promote combined coverage file to the expected output path +if [[ "$COVERAGE" == "true" && -f target/coverage/lcov-combined.info ]]; then + mv target/coverage/lcov-combined.info target/coverage/lcov.info +fi + exit 0 diff --git a/scripts/util/commit.rb b/scripts/util/commit.rb deleted file mode 100644 index 8b4f5baba1ca1..0000000000000 --- a/scripts/util/commit.rb +++ /dev/null @@ -1,166 +0,0 @@ -require_relative "conventional_commit" -require_relative "git_log_commit" - -module Vector - class Commit - class << self - def fetch_since(last_version) - git_log = GitLogCommit.fetch_since!(last_version) - git_log.collect do |git_log_commit| - from_git_log_commit(git_log_commit) - end - end - - def fetch_since!(last_version) - git_log = GitLogCommit.fetch_since!(last_version) - git_log.collect do |git_log_commit| - from_git_log_commit!(git_log_commit) - end - end - - def from_git_log!(git_log) - git_log.collect do |git_log_commit| - from_git_log_commit!(git_log_commit) - end - end - - private - def from_git_log_commit(git_log_commit) - conventional_commit = ConventionalCommit.parse(git_log_commit.message) - hash = git_log_commit.to_h.merge(conventional_commit.to_h) - new(hash) - end - - def from_git_log_commit!(git_log_commit) - conventional_commit = ConventionalCommit.parse!(git_log_commit.message) - hash = git_log_commit.to_h.merge(conventional_commit.to_h) - new(hash) - end - end - - attr_reader :author, - :breaking_change, - :date, - :deletions_count, - :description, - :files_count, - :insertions_count, - :pr_number, - :scopes, - :sha, - :type - - def initialize(hash) - @author = hash.fetch("author") - @breaking_change = hash.fetch("breaking_change") - @date = hash.fetch("date") - @deletions_count = hash.fetch("deletions_count") - @description = hash.fetch("description") - @files_count = hash.fetch("files_count") - @insertions_count = hash.fetch("insertions_count") - @pr_number = hash.fetch("pr_number") - @scopes = hash.fetch("scopes") - @sha = hash.fetch("sha") - @type = hash.fetch("type") - end - - def eql?(other) - sha == other.sha || pr_number == other.pr_number - end - - def breaking_change? - breaking_change == true - end - - def fix? - type == "fix" - end - - def new_feature? - type == "feat" - end - - def to_cue_struct - "{" + - "sha: #{sha.to_json}, " + - "date: #{date.to_json}, " + - "description: #{description.to_json}, " + - "pr_number: #{pr_number.to_json}, " + - "scopes: #{scopes.to_json}, " + - "type: #{type.to_json}, " + - "breaking_change: #{breaking_change.to_json}, " + - "author: #{author.to_json}, " + - "files_count: #{files_count.to_json}, " + - "insertions_count: #{insertions_count.to_json}, " + - "deletions_count: #{deletions_count.to_json}}" - end - - def validate! - if !type.nil? && !TYPES.include?(type) - raise <<~EOF - The following commit has an invalid type! - - #{to_s} - - The type must be one of #{TYPES.inspect}. - - #{type.inspect} - - Please correct in the release /.meta file and retry. - EOF - end - - if TYPES_THAT_REQUIRE_SCOPES.include?(type) && scopes.empty? - raise <<~EOF - The following commit does not have a scope - - #{to_s} - - A scope is required for commits of type #{TYPES_THAT_REQUIRE_SCOPES.inspect}. - - #{description} - - Please correct in the release /.meta file and retry. - EOF - end - - true - end - - def to_git_log_commit - message = "" - - if type - message = "#{message}#{type.clone}" - end - - if scopes.any? - message = "#{message}(#{scopes.join(", ")})" - end - - if breaking_change? - message = "#{message}!" - end - - message = "#{message}: #{description}" - - if pr_number - message = "#{message} (##{pr_number})" - end - - GitLogCommit.new({ - "author" => author, - "date" => date, - "deletions_count" => deletions_count, - "files_count" => files_count, - "insertions_count" => insertions_count, - "message" => message, - "sha" => sha - }) - end - - def to_s - "#{sha} #{type}(#{scopes.join(", ")})#{breaking_change? ? "!" : ""}: #{description} (##{pr_number})" - end - end -end diff --git a/scripts/util/conventional_commit.rb b/scripts/util/conventional_commit.rb deleted file mode 100644 index 5944bb407f663..0000000000000 --- a/scripts/util/conventional_commit.rb +++ /dev/null @@ -1,97 +0,0 @@ -module Vector - class ConventionalCommit - class << self - def parse(message) - hash = parse_commit_message(message) - new(hash) - end - - def parse!(message) - hash = parse_commit_message!(message) - new(hash) - end - - private - def parse_commit_message(message) - begin - parse_commit_message!(message) - rescue Exception => e - if message.include?("Use `namespace` field in metric sources") - raise e - end - - { - "breaking_change" => nil, - "description" => message, - "pr_number" => nil, - "scopes" => [], - "type" => nil - } - end - end - - def parse_commit_message!(message) - match = message.match(/^(?[a-z]*)(\((?[a-z0-9_, ]*)\))?(?!)?: (?.*?)( \(#(?[0-9]*)\))?$/) - - if match.nil? - raise <<~EOF - Commit message does not conform to the conventional commit format. - - Unable to parse at all! - - #{message} - - Please correct in the release /.meta file and retry. - EOF - end - - attributes = - { - "type" => match[:type], - "breaking_change" => !match[:breaking_change].nil?, - "description" => match[:description] - } - - attributes["scopes"] = - if match[:scope] - match[:scope].split(",").collect(&:strip) - else - [] - end - - attributes["pr_number"] = - if match[:pr_number] - match[:pr_number].to_i - else - nil - end - - attributes - end - end - - attr_reader :breaking_change, - :description, - :pr_number, - :type, - :scopes - - def initialize(hash) - @breaking_change = hash.fetch("breaking_change") - @description = hash.fetch("description") - @pr_number = hash.fetch("pr_number") - @type = hash.fetch("type") - @scopes = hash.fetch("scopes") - end - - def to_h - { - "breaking_change" => breaking_change, - "description" => description, - "pr_number" => pr_number, - "type" => type, - "scopes" => scopes - } - end - end -end diff --git a/scripts/util/git_log_commit.rb b/scripts/util/git_log_commit.rb deleted file mode 100644 index fe8425209a86d..0000000000000 --- a/scripts/util/git_log_commit.rb +++ /dev/null @@ -1,122 +0,0 @@ -module Vector - class GitLogCommit - class << self - def fetch_since!(last_version) - range = "v#{last_version}..." - commit_log = `git log #{range} --cherry-pick --right-only --no-merges --pretty=format:'%H\t%s\t%aN\t%ad'`.chomp - commit_lines = commit_log.split("\n").reverse - - commit_lines.collect do |commit_line| - hash = parse_commit_line!(commit_line) - new(hash) - end - end - - def from_file!(path) - contents = File.read(path) - contents.split("\n").collect do |line| - hash = parse_commit_line!(line) - new(hash) - end - end - - private - # This is used for the `files_count`, `insertions_count`, and `deletions_count` - # attributes. It helps to communicate stats and the depth of changes in our - # release notes. - def get_commit_stats(sha) - `git show --shortstat --oneline #{sha}`.split("\n").last - end - - def parse_commit_line!(commit_line) - # Parse the full commit line - line_parts = commit_line.split("\t") - sha = line_parts.fetch(0) - message = line_parts.fetch(1) - author = line_parts.fetch(2) - date = Time.parse(line_parts.fetch(3)).utc - - attributes = - { - "sha" => sha, - "author" => author, - "date" => date, - "message" => message - } - - # Parse the stats - stats = get_commit_stats(attributes.fetch("sha")) - if /^\W*\p{Digit}+ files? changed,/.match(stats) - stats_attributes = parse_commit_stats!(stats) - attributes.merge!(stats_attributes) - end - - attributes - end - - # Parses the data from `#get_commit_stats`. - def parse_commit_stats!(stats) - attributes = {} - - stats.split(", ").each do |stats_part| - stats_part.strip! - - key = - case stats_part - when /insertions?/ - "insertions_count" - when /deletions?/ - "deletions_count" - when /files? changed/ - "files_count" - else - raise "Invalid commit stat: #{stats_part}" - end - - count = stats_part.match(/^(?[0-9]*) /)[:count].to_i - attributes[key] = count - end - - attributes["insertions_count"] ||= 0 - attributes["deletions_count"] ||= 0 - attributes["files_count"] ||= 0 - attributes - end - end - - attr_reader :author, - :date, - :deletions_count, - :files_count, - :insertions_count, - :message, - :raw, - :sha - - def initialize(hash) - @author = hash.fetch("author") - @date = hash.fetch("date") - @deletions_count = hash.fetch("deletions_count", 0) - @files_count = hash.fetch("files_count", 0) - @insertions_count = hash.fetch("insertions_count", 0) - @message = hash.fetch("message") - @sha = hash.fetch("sha") - end - - def to_h - { - "author" => author, - "date" => date, - "deletions_count" => deletions_count, - "files_count" => files_count, - "insertions_count" => insertions_count, - "message" => message, - "sha" => sha - } - end - - def to_raw - "#{sha}\t#{message}\t#{author}\t#{date.strftime("%a %b %d %H:%M:%S %Y %z")}" - end - end -end diff --git a/scripts/util/printer.rb b/scripts/util/printer.rb deleted file mode 100644 index 4f65f2d904359..0000000000000 --- a/scripts/util/printer.rb +++ /dev/null @@ -1,74 +0,0 @@ -require "paint" - -module Util - module Printer - PROMPT = "---> " - INDENT = " " - SEPARATOR = "-" * 80 - TITLE_PROMPT = "#### " - - extend self - - def error!(message) - Printer.say(message, color: :red) - exit(1) - end - - def Printer.get(words, choices = nil) - question = "#{words.strip}" - - if !choices.nil? - question += " (" + choices.join("/") + ")" - end - - Printer.say(question) - - print INDENT - - input = gets().chomp - - if choices && !choices.include?(input) - Printer.say("You must enter one of #{choices.join(", ")}", color: :red) - Printer.get(words, choices) - else - input - end - end - - def invalid(words) - Printer.say(words, color: :yellow) - end - - def say(words, color: nil, new: true, prompt: PROMPT) - prefix = new ? prompt : INDENT - - if color - words = Paint[prefix + words, color] - else - words = prefix + words - end - - puts words.gsub("\n", "\n#{INDENT}") - end - - def separate(color: nil) - string = SEPARATOR - - if color - string = Paint[string, color] - end - - puts "" - puts string - end - - def success(words) - Printer.say(words, color: :green) - end - - def title(words) - separate(color: :cyan) - Printer.say(words, color: :cyan, prompt: TITLE_PROMPT) - end - end -end diff --git a/scripts/util/release.rb b/scripts/util/release.rb deleted file mode 100644 index 112ee39e19536..0000000000000 --- a/scripts/util/release.rb +++ /dev/null @@ -1,39 +0,0 @@ -require_relative "commit" -require_relative "version" - -module Vector - class Release - class << self - def all!(dir) - release_meta_paths = Dir.glob("#{dir}/*.cue").to_a - - release_meta_paths. - collect do |release_meta_path| - urls_cue_path = File.join(File.dirname(release_meta_path), "..", "urls.cue") - release_json = `cue export #{urls_cue_path} #{release_meta_path}` - release_hash = JSON.parse(release_json) - name = release_hash.fetch("releases").keys.first - hash = release_hash.fetch("releases").values.first - new(hash.merge({"name" => name})) - end. - sort_by(&:version) - end - end - - attr_reader :codename, - :commits, - :date, - :name, - :version, - :whats_next - - def initialize(hash) - @codename = hash.fetch("codename", "") - @commits = hash.fetch("commits").collect { |commit_hash| Commit.new(commit_hash) } - @date = hash.fetch("date") - @name = hash.fetch("name") - @version = Util::Version.new(@name) - @whats_next = hash.fetch("whats_next", []) - end - end -end diff --git a/scripts/util/version.rb b/scripts/util/version.rb deleted file mode 100644 index afc69ab15483f..0000000000000 --- a/scripts/util/version.rb +++ /dev/null @@ -1,45 +0,0 @@ -module Util - class Version < Gem::Version - def bump_type(other_version) - # Return nil if the other version is not greater than the current version - if other_version <= self - return nil - end - - bumped_version = bump - next_major = segments.first + 1 - - if other_version.prerelease? - "pre" - elsif other_version < bumped_version - "patch" - elsif other_version == bumped_version - "minor" - elsif other_version.segments.first == next_major - "major" - else - nil - end - end - - def major - segments[0] - end - - def major_x - "#{segments[0]}.X" - end - - def minor - segments[1] - end - - def minor_x - "#{segments[0]}.#{segments[1]}.X" - end - - def patch - segments[2] - end - end -end diff --git a/scripts/verify-install.sh b/scripts/verify-install.sh index a73dccc7a52b4..ab9e6039f3f14 100755 --- a/scripts/verify-install.sh +++ b/scripts/verify-install.sh @@ -26,8 +26,10 @@ getent passwd vector || (echo "vector user missing" && exit 1) getent group vector || (echo "vector group missing" && exit 1) vector --version || (echo "vector --version failed" && exit 1) test -f /etc/default/vector || (echo "/etc/default/vector doesn't exist" && exit 1) -test -f /etc/vector/vector.yaml || (echo "/etc/vector/vector.yaml doesn't exist" && exit 1) +test ! -e /etc/vector/vector.yaml || (echo "/etc/vector/vector.yaml should not be installed by default" && exit 1) +test -f /usr/share/vector/examples/vector.yaml || (echo "/usr/share/vector/examples/vector.yaml doesn't exist" && exit 1) +mkdir -p /etc/vector echo "FOO=bar" > /etc/default/vector echo "foo: bar" > /etc/vector/vector.yaml @@ -37,6 +39,6 @@ getent passwd vector || (echo "vector user missing" && exit 1) getent group vector || (echo "vector group missing" && exit 1) vector --version || (echo "vector --version failed" && exit 1) grep -q "FOO=bar" "/etc/default/vector" || (echo "/etc/default/vector has incorrect contents" && exit 1) -grep -q "foo: bar" "/etc/vector/vector.yaml" || (echo "/etc/vector/vector.yaml has incorrect contents" && exit 1) +grep -q "foo: bar" "/etc/vector/vector.yaml" || (echo "user-provided /etc/vector/vector.yaml was not preserved on reinstall" && exit 1) dd-pkg lint "$package" diff --git a/src/api/grpc/mod.rs b/src/api/grpc/mod.rs new file mode 100644 index 0000000000000..f3df61dc08da2 --- /dev/null +++ b/src/api/grpc/mod.rs @@ -0,0 +1,3 @@ +mod service; + +pub use service::ObservabilityService; diff --git a/src/api/grpc/service.rs b/src/api/grpc/service.rs new file mode 100644 index 0000000000000..e1e0808619db9 --- /dev/null +++ b/src/api/grpc/service.rs @@ -0,0 +1,861 @@ +use std::collections::{HashMap, HashSet}; +use std::pin::Pin; +// std::sync::Mutex is intentional: the lock is never held across an .await point +// (only used in synchronous map updates inside IntervalStream closures), so the +// cheaper std mutex is correct here. tokio::sync::Mutex is only needed when the +// critical section itself contains .await. +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use futures::{StreamExt as FuturesStreamExt, stream}; +use rand::{Rng, SeedableRng as _, rngs::SmallRng}; +use tokio::select; +use tokio::sync::mpsc; +use tokio::time::{self, interval}; +use tokio_stream::{ + Stream, + wrappers::{IntervalStream, ReceiverStream}, +}; +use tonic::{Request, Response, Status}; +use vector_lib::tap::{ + controller::{TapController, TapPatterns, TapPayload}, + topology::WatchRx, +}; + +use crate::event::{Metric, MetricValue}; +use crate::metrics::Controller; +use crate::proto::observability::{ + self, Component as ProtoComponent, ComponentType, EventNotification, TappedEvent, *, +}; + +type BoxStream = Pin> + Send>>; + +/// Helper function to extract metric value as f64 +fn get_metric_value(metric: &Metric) -> Option { + match metric.value() { + MetricValue::Counter { value } => Some(*value), + MetricValue::Gauge { value } => Some(*value), + _ => None, + } +} + +/// Helper function to filter metrics by name and group by component_id tag +fn filter_and_group_metrics(metrics: &[Metric], metric_name: &str) -> HashMap { + let mut result = HashMap::new(); + + for metric in metrics.iter().filter(|m| m.name() == metric_name) { + if let Some(tags) = metric.tags() + && let Some(component_id) = tags.get("component_id") + && let Some(value) = get_metric_value(metric) + { + *result.entry(component_id.to_string()).or_insert(0.0) += value; + } + } + + result +} + +/// Filter metrics by name and group by (component_id, output) tag pair. +/// Used to populate per-output metrics in GetComponents responses. +fn filter_and_group_metrics_by_output( + metrics: &[Metric], + metric_name: &str, +) -> HashMap<(String, String), f64> { + let mut result = HashMap::new(); + + for metric in metrics.iter().filter(|m| m.name() == metric_name) { + if let Some(tags) = metric.tags() + && let Some(component_id) = tags.get("component_id") + && let Some(value) = get_metric_value(metric) + { + let output = tags.get("output").unwrap_or("_default").to_string(); + *result + .entry((component_id.to_string(), output)) + .or_insert(0.0) += value; + } + } + + result +} + +/// Extract all component metrics and group by component_id +fn extract_component_metrics(metrics: &[Metric]) -> HashMap { + let received_bytes = filter_and_group_metrics(metrics, "component_received_bytes_total"); + let received_events = filter_and_group_metrics(metrics, "component_received_events_total"); + let sent_bytes = filter_and_group_metrics(metrics, "component_sent_bytes_total"); + let sent_events = filter_and_group_metrics(metrics, "component_sent_events_total"); + + let mut all_component_ids = HashSet::new(); + all_component_ids.extend(received_bytes.keys().cloned()); + all_component_ids.extend(received_events.keys().cloned()); + all_component_ids.extend(sent_bytes.keys().cloned()); + all_component_ids.extend(sent_events.keys().cloned()); + + let mut result = HashMap::new(); + for component_id in all_component_ids { + result.insert( + component_id.clone(), + ComponentMetrics { + received_bytes_total: received_bytes.get(&component_id).map(|v| *v as i64), + received_events_total: received_events.get(&component_id).map(|v| *v as i64), + sent_bytes_total: sent_bytes.get(&component_id).map(|v| *v as i64), + sent_events_total: sent_events.get(&component_id).map(|v| *v as i64), + }, + ); + } + + result +} + +/// Helper function to calculate throughput by comparing current and previous values +fn calculate_throughput( + current: &HashMap, + previous: &HashMap, + interval_secs: f64, +) -> HashMap { + let mut result = HashMap::new(); + + for (component_id, current_value) in current { + if let Some(previous_value) = previous.get(component_id) { + let delta = current_value - previous_value; + let throughput = if interval_secs > 0.0 { + delta / interval_secs + } else { + 0.0 + }; + result.insert(component_id.clone(), throughput.max(0.0)); + } + } + + result +} + +/// Helper function to calculate throughput for per-output metrics by comparing +/// current and previous values keyed by `(component_id, output)`. +fn calculate_throughput_by_output( + current: &HashMap<(String, String), f64>, + previous: &HashMap<(String, String), f64>, + interval_secs: f64, +) -> HashMap<(String, String), f64> { + let mut result = HashMap::new(); + + for (key, current_value) in current { + if let Some(previous_value) = previous.get(key) { + let delta = current_value - previous_value; + let throughput = if interval_secs > 0.0 { + delta / interval_secs + } else { + 0.0 + }; + result.insert(key.clone(), throughput.max(0.0)); + } + } + + result +} + +/// Minimum allowed polling interval for metric streams (100 ms). +/// +/// Prevents clients from setting `interval_ms=1` and hammering `capture_metrics()` +/// at 1 kHz, which would cause high CPU load across concurrent subscriptions. +const MIN_INTERVAL_MS: i32 = 100; + +/// Validates `interval_ms` from a streaming request, returning the value as `u64` +/// or a gRPC error if it is out of range. +fn validate_interval_ms(interval_ms: i32) -> Result { + if interval_ms < MIN_INTERVAL_MS { + return Err(Status::invalid_argument(format!( + "interval_ms must be >= {MIN_INTERVAL_MS}" + ))); + } + Ok(interval_ms as u64) +} + +fn get_controller() -> Result<&'static Controller, Status> { + Controller::get().map_err(|_| Status::internal("Metrics system not initialized")) +} + +/// Builds a stream that emits per-component totals for `metric_name` every `duration`. +fn metric_totals_stream( + duration: Duration, + metric_name: &'static str, +) -> Result, Status> { + let controller = get_controller()?; + Ok(Box::pin( + tokio_stream::StreamExt::map(IntervalStream::new(interval(duration)), move |_| { + let metrics = controller.capture_metrics(); + let component_metrics = filter_and_group_metrics(&metrics, metric_name); + tokio_stream::iter( + component_metrics + .into_iter() + .map(|(component_id, total)| { + Ok(StreamComponentMetricsResponse { + component_id, + value: Some(stream_component_metrics_response::Value::Total( + TotalMetric { + value: total as i64, + output_totals: Default::default(), + }, + )), + }) + }) + .collect::>(), + ) + }) + .flatten(), + )) +} + +/// Builds a stream that emits per-component throughputs for `metric_name` every `duration`. +fn metric_throughput_stream( + duration: Duration, + metric_name: &'static str, +) -> Result, Status> { + let controller = get_controller()?; + let interval_secs = duration.as_secs_f64(); + let previous_values = Arc::new(Mutex::new(HashMap::new())); + Ok(Box::pin( + tokio_stream::StreamExt::map(IntervalStream::new(interval(duration)), move |_| { + let metrics = controller.capture_metrics(); + let current_values = filter_and_group_metrics(&metrics, metric_name); + let mut prev = match previous_values.lock() { + Ok(guard) => guard, + Err(poisoned) => { + error!("Mutex poisoned for metric throughput, recovering."); + poisoned.into_inner() + } + }; + let throughputs = calculate_throughput(¤t_values, &prev, interval_secs); + *prev = current_values; + tokio_stream::iter( + throughputs + .into_iter() + .map(|(component_id, throughput)| { + Ok(StreamComponentMetricsResponse { + component_id, + value: Some(stream_component_metrics_response::Value::Throughput( + ThroughputMetric { + value: throughput, + output_throughputs: Default::default(), + }, + )), + }) + }) + .collect::>(), + ) + }) + .flatten(), + )) +} + +/// Builds a stream that emits per-component sent_events totals with per-output breakdown. +fn sent_events_totals_stream( + duration: Duration, +) -> Result, Status> { + let controller = get_controller()?; + Ok(Box::pin( + tokio_stream::StreamExt::map(IntervalStream::new(interval(duration)), move |_| { + let metrics = controller.capture_metrics(); + let component_totals = + filter_and_group_metrics(&metrics, "component_sent_events_total"); + let by_output = + filter_and_group_metrics_by_output(&metrics, "component_sent_events_total"); + + // Group per-output values by component_id + let mut output_by_component: HashMap> = HashMap::new(); + for ((component_id, output), value) in by_output { + output_by_component + .entry(component_id) + .or_default() + .insert(output, value as i64); + } + + tokio_stream::iter( + component_totals + .into_iter() + .map(|(component_id, total)| { + let output_totals = output_by_component + .remove(&component_id) + .unwrap_or_default() + .into_iter() + .collect(); + Ok(StreamComponentMetricsResponse { + component_id, + value: Some(stream_component_metrics_response::Value::Total( + TotalMetric { + value: total as i64, + output_totals, + }, + )), + }) + }) + .collect::>(), + ) + }) + .flatten(), + )) +} + +/// Builds a stream that emits per-component sent_events throughputs with per-output breakdown. +fn sent_events_throughput_stream( + duration: Duration, +) -> Result, Status> { + let controller = get_controller()?; + let interval_secs = duration.as_secs_f64(); + let previous_totals = Arc::new(Mutex::new(HashMap::::new())); + let previous_outputs = Arc::new(Mutex::new(HashMap::<(String, String), f64>::new())); + Ok(Box::pin( + tokio_stream::StreamExt::map(IntervalStream::new(interval(duration)), move |_| { + let metrics = controller.capture_metrics(); + let current_totals = filter_and_group_metrics(&metrics, "component_sent_events_total"); + let current_outputs = + filter_and_group_metrics_by_output(&metrics, "component_sent_events_total"); + + let mut prev_t = match previous_totals.lock() { + Ok(guard) => guard, + Err(poisoned) => { + error!("Mutex poisoned for sent_events throughput totals, recovering."); + poisoned.into_inner() + } + }; + let mut prev_o = match previous_outputs.lock() { + Ok(guard) => guard, + Err(poisoned) => { + error!("Mutex poisoned for sent_events throughput outputs, recovering."); + poisoned.into_inner() + } + }; + + let throughputs = calculate_throughput(¤t_totals, &prev_t, interval_secs); + let output_throughputs_flat = + calculate_throughput_by_output(¤t_outputs, &prev_o, interval_secs); + + *prev_t = current_totals; + *prev_o = current_outputs; + + // Group per-output throughputs by component_id + let mut output_by_component: HashMap> = HashMap::new(); + for ((component_id, output), tp) in output_throughputs_flat { + output_by_component + .entry(component_id) + .or_default() + .insert(output, tp); + } + + tokio_stream::iter( + throughputs + .into_iter() + .map(|(component_id, throughput)| { + let output_throughputs = output_by_component + .remove(&component_id) + .unwrap_or_default() + .into_iter() + .collect(); + Ok(StreamComponentMetricsResponse { + component_id, + value: Some(stream_component_metrics_response::Value::Throughput( + ThroughputMetric { + value: throughput, + output_throughputs, + }, + )), + }) + }) + .collect::>(), + ) + }) + .flatten(), + )) +} + +/// Converts a component's output port names into proto `Output` messages, +/// populating `sent_events_total` from the per-output metric snapshot. +/// +/// `None` port means the default output (represented as `"_default"`). +fn ports_to_proto_outputs( + ports: &[Option<&str>], + component_id: &str, + sent_events_by_output: &HashMap<(String, String), f64>, +) -> Vec { + ports + .iter() + .map(|port| { + let output_id = port.unwrap_or("_default").to_string(); + let sent_events_total = sent_events_by_output + .get(&(component_id.to_string(), output_id.clone())) + .copied() + .unwrap_or(0.0) as i64; + Output { + output_id, + sent_events_total, + } + }) + .collect() +} + +/// gRPC observability service implementation. +pub struct ObservabilityService { + watch_rx: WatchRx, +} + +impl ObservabilityService { + pub const fn new(watch_rx: WatchRx) -> Self { + Self { watch_rx } + } +} + +#[tonic::async_trait] +impl observability::Service for ObservabilityService { + // ========== Simple Queries ========== + + async fn get_meta( + &self, + _request: Request, + ) -> Result, Status> { + let version = crate::get_version().to_string(); + let hostname = hostname::get() + .ok() + .and_then(|h| h.into_string().ok()) + .unwrap_or_else(|| "unknown".to_string()); + + Ok(Response::new(GetMetaResponse { version, hostname })) + } + + async fn get_allocation_tracing_status( + &self, + _request: Request, + ) -> Result, Status> { + #[cfg(feature = "allocation-tracing")] + let enabled = crate::internal_telemetry::allocations::is_allocation_tracing_enabled(); + #[cfg(not(feature = "allocation-tracing"))] + let enabled = false; + Ok(Response::new(GetAllocationTracingStatusResponse { + enabled, + })) + } + + async fn get_components( + &self, + request: Request, + ) -> Result, Status> { + let limit = request.into_inner().limit; + + // Get the current topology snapshot + let tap_resource = self.watch_rx.borrow().clone(); + + // Get metrics for all components + let controller = + Controller::get().map_err(|_| Status::internal("Metrics system not initialized"))?; + let metrics = controller.capture_metrics(); + let component_metrics_map = extract_component_metrics(&metrics); + let sent_events_by_output = + filter_and_group_metrics_by_output(&metrics, "component_sent_events_total"); + + let type_map = &tap_resource.type_names; + + // `tap_resource.outputs` and `tap_resource.inputs` are both full topology snapshots, + // maintained incrementally on RunningTopology. `source_keys`/`sink_keys` are diff-only + // (changed/added in the last reload) and must not be used for enumeration here. + // + // Component kind is determined by set membership: + // source = has outputs, no inputs + // transform = has outputs AND inputs + // sink = has inputs, no outputs + let output_ports = tap_resource.output_ports_by_component(); + + let mut components = Vec::new(); + + // Sources: present in `output_ports` but not in `inputs`. + // `output_ports` is keyed by component, so no duplicates. + for (key, ports) in &output_ports { + if tap_resource.inputs.contains_key(*key) { + continue; // transform or sink, handled below + } + let key_str = key.to_string(); + let on_type = type_map + .get(&key_str) + .cloned() + .unwrap_or_else(|| "unknown".to_string()); + components.push(ProtoComponent { + component_id: key_str.clone(), + component_type: ComponentType::Source as i32, + on_type, + outputs: ports_to_proto_outputs(ports, &key_str, &sent_events_by_output), + metrics: component_metrics_map.get(&key_str).cloned(), + }); + } + + // Transforms and sinks: present in `inputs` (full snapshot) + for component_key in tap_resource.inputs.keys() { + let key_str = component_key.to_string(); + let (component_type, outputs) = if let Some(ports) = output_ports.get(component_key) { + ( + ComponentType::Transform, + ports_to_proto_outputs(ports, &key_str, &sent_events_by_output), + ) + } else { + (ComponentType::Sink, vec![]) + }; + let on_type = type_map + .get(&key_str) + .cloned() + .unwrap_or_else(|| "unknown".to_string()); + components.push(ProtoComponent { + component_id: key_str.clone(), + component_type: component_type as i32, + on_type, + outputs, + metrics: component_metrics_map.get(&key_str).cloned(), + }); + } + + // Sort alphabetically for stable, deterministic ordering before applying limit + components.sort_unstable_by(|a, b| a.component_id.cmp(&b.component_id)); + if limit > 0 { + components.truncate(limit as usize); + } + + Ok(Response::new(GetComponentsResponse { components })) + } + + // ========== Streaming Metrics ========== + + type StreamHeartbeatStream = BoxStream; + + async fn stream_heartbeat( + &self, + request: Request, + ) -> Result, Status> { + let duration = + Duration::from_millis(validate_interval_ms(request.into_inner().interval_ms)?); + let stream = tokio_stream::StreamExt::map(IntervalStream::new(interval(duration)), |_| { + let utc = Some(prost_types::Timestamp { + seconds: chrono::Utc::now().timestamp(), + nanos: 0, + }); + Ok(StreamHeartbeatResponse { utc }) + }); + + Ok(Response::new(Box::pin(stream))) + } + + type StreamUptimeStream = BoxStream; + + async fn stream_uptime( + &self, + request: Request, + ) -> Result, Status> { + let duration = + Duration::from_millis(validate_interval_ms(request.into_inner().interval_ms)?); + let controller = get_controller()?; + let stream = + tokio_stream::StreamExt::map(IntervalStream::new(interval(duration)), move |_| { + let metrics = controller.capture_metrics(); + let uptime_seconds = metrics + .iter() + .find(|m| m.name() == "uptime_seconds") + .and_then(get_metric_value) + .unwrap_or(0.0) as i64; + + Ok(StreamUptimeResponse { uptime_seconds }) + }); + + Ok(Response::new(Box::pin(stream))) + } + + type StreamComponentAllocatedBytesStream = BoxStream; + + async fn stream_component_allocated_bytes( + &self, + request: Request, + ) -> Result, Status> { + let duration = + Duration::from_millis(validate_interval_ms(request.into_inner().interval_ms)?); + let controller = get_controller()?; + let stream = + tokio_stream::StreamExt::map(IntervalStream::new(interval(duration)), move |_| { + let metrics = controller.capture_metrics(); + let component_metrics = + filter_and_group_metrics(&metrics, "component_allocated_bytes"); + + tokio_stream::iter( + component_metrics + .into_iter() + .map(|(component_id, allocated_bytes)| { + Ok(StreamComponentAllocatedBytesResponse { + component_id, + allocated_bytes: allocated_bytes as i64, + }) + }) + .collect::>(), + ) + }) + .flatten(); + + Ok(Response::new(Box::pin(stream))) + } + + type StreamComponentMetricsStream = BoxStream; + + async fn stream_component_metrics( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + let duration = Duration::from_millis(validate_interval_ms(req.interval_ms)?); + let metric = MetricName::try_from(req.metric) + .map_err(|_| Status::invalid_argument("Unknown metric value"))?; + + let stream: BoxStream = match metric { + MetricName::Unspecified => { + return Err(Status::invalid_argument("metric must be specified")); + } + MetricName::ReceivedEventsTotal => { + metric_totals_stream(duration, "component_received_events_total")? + } + MetricName::SentEventsTotal => sent_events_totals_stream(duration)?, + MetricName::ReceivedBytesTotal => { + metric_totals_stream(duration, "component_received_bytes_total")? + } + MetricName::SentBytesTotal => { + metric_totals_stream(duration, "component_sent_bytes_total")? + } + MetricName::ErrorsTotal => metric_totals_stream(duration, "component_errors_total")?, + MetricName::ReceivedEventsThroughput => { + metric_throughput_stream(duration, "component_received_events_total")? + } + MetricName::SentEventsThroughput => sent_events_throughput_stream(duration)?, + MetricName::ReceivedBytesThroughput => { + metric_throughput_stream(duration, "component_received_bytes_total")? + } + MetricName::SentBytesThroughput => { + metric_throughput_stream(duration, "component_sent_bytes_total")? + } + }; + + Ok(Response::new(stream)) + } + + // ========== Event Tapping ========== + + type StreamOutputEventsStream = BoxStream; + + async fn stream_output_events( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + + // Validate before casting to prevent negative values from becoming large positive values + if req.limit <= 0 { + return Err(Status::invalid_argument( + "limit must be >= 1 (controls reservoir size and channel capacity)", + )); + } + + const MAX_TAP_LIMIT: i32 = 100_000; + if req.limit > MAX_TAP_LIMIT { + return Err(Status::invalid_argument(format!( + "limit must be <= {MAX_TAP_LIMIT}" + ))); + } + + let interval_ms = validate_interval_ms(req.interval_ms)?; + let limit = req.limit as usize; + + let patterns = TapPatterns { + for_outputs: req.outputs_patterns.into_iter().collect(), + for_inputs: req.inputs_patterns.into_iter().collect(), + }; + + // Channel for receiving tap payloads + let (tap_tx, tap_rx) = mpsc::channel(limit); + + // Channel for sending events to the client + let (event_tx, event_rx) = mpsc::channel::>(10); + + let watch_rx = self.watch_rx.clone(); + + crate::spawn_in_current_span(async move { + let _tap_controller = TapController::new(watch_rx, tap_tx, patterns); + let mut tap_rx = ReceiverStream::new(tap_rx); + let mut interval = time::interval(time::Duration::from_millis(interval_ms)); + let mut reservoir = Reservoir::new(limit); + + loop { + select! { + Some(tap_payload) = tokio_stream::StreamExt::next(&mut tap_rx) => { + if reservoir.handle_payload(tap_payload, &event_tx).await.is_err() { + break; + } + } + _ = interval.tick() => { + if event_tx.is_closed() || reservoir.flush(&event_tx).await.is_err() { + break; + } + } + } + } + }); + + let stream = FuturesStreamExt::flat_map(ReceiverStream::new(event_rx), |events| { + stream::iter(events.into_iter().map(Ok)) + }); + + Ok(Response::new(Box::pin(stream))) + } +} + +/// Reservoir sampler for tap events, batched and flushed on an interval. +struct Reservoir { + events: Vec<(usize, StreamOutputEventsResponse)>, + rng: SmallRng, + batch: usize, + limit: usize, +} + +impl Reservoir { + fn new(limit: usize) -> Self { + Self { + events: Vec::with_capacity(limit), + rng: SmallRng::from_rng(&mut rand::rng()), + batch: 0, + limit, + } + } + + /// Process a tap payload: notifications are forwarded immediately; data events + /// are reservoir-sampled (Algorithm R) for the next flush. + async fn handle_payload( + &mut self, + payload: TapPayload, + tx: &mpsc::Sender>, + ) -> Result<(), ()> { + for event in tap_payload_to_output_events(payload) { + if matches!( + event.event, + Some(stream_output_events_response::Event::Notification(_)) + ) { + tx.send(vec![event]).await.map_err(|err| { + debug!(message = "Couldn't send notification.", error = ?err); + })?; + } else { + // Reservoir sampling (Algorithm R). + // Draw from 0..=batch (inclusive) so that event i has + // exactly limit/(i+1) probability of entering the reservoir. + // Using 0..batch (exclusive) would guarantee replacement on + // the first post-fill event (100% instead of limit/(limit+1)). + if self.limit > self.events.len() { + self.events.push((self.batch, event)); + } else { + let idx = self.rng.random_range(0..=self.batch); + if idx < self.events.len() { + self.events[idx] = (self.batch, event); + } + } + self.batch += 1; + } + } + Ok(()) + } + + /// Flush sampled events to the client, sorted by arrival order. + async fn flush( + &mut self, + tx: &mpsc::Sender>, + ) -> Result<(), ()> { + if self.events.is_empty() { + return Ok(()); + } + self.batch = 0; + self.events.sort_by_key(|(batch, _)| *batch); + let events = self.events.drain(..).map(|(_, e)| e).collect(); + tx.send(events).await.map_err(|err| { + debug!(message = "Couldn't send events.", error = ?err); + }) + } +} + +/// Convert TapPayload to gRPC StreamOutputEventsResponse(s) +fn tap_payload_to_output_events(payload: TapPayload) -> Vec { + use crate::event::proto::{Event, EventWrapper}; + + match payload { + TapPayload::Log(output, log_array) => log_array + .into_iter() + .map(|log_event| { + // Convert Vector's internal LogEvent to proto Log (metadata is preserved in Log.metadata_full) + let proto_log: crate::event::proto::Log = log_event.into(); + let event_wrapper = Some(EventWrapper { + event: Some(Event::Log(proto_log)), + }); + + StreamOutputEventsResponse { + event: Some(stream_output_events_response::Event::TappedEvent( + TappedEvent { + component_id: output.output_id.component.id().to_string(), + component_type: output.component_type.to_string(), + component_kind: output.component_kind.to_string(), + event: event_wrapper, + }, + )), + } + }) + .collect(), + TapPayload::Metric(output, metric_array) => metric_array + .into_iter() + .map(|metric_event| { + // Convert Vector's internal Metric to proto Metric + let proto_metric: crate::event::proto::Metric = metric_event.into(); + let event_wrapper = Some(EventWrapper { + event: Some(Event::Metric(proto_metric)), + }); + + StreamOutputEventsResponse { + event: Some(stream_output_events_response::Event::TappedEvent( + TappedEvent { + component_id: output.output_id.component.id().to_string(), + component_type: output.component_type.to_string(), + component_kind: output.component_kind.to_string(), + event: event_wrapper, + }, + )), + } + }) + .collect(), + TapPayload::Trace(output, trace_array) => trace_array + .into_iter() + .map(|trace_event| { + // Convert Vector's internal TraceEvent to proto Trace + let proto_trace: crate::event::proto::Trace = trace_event.into(); + let event_wrapper = Some(EventWrapper { + event: Some(Event::Trace(proto_trace)), + }); + + StreamOutputEventsResponse { + event: Some(stream_output_events_response::Event::TappedEvent( + TappedEvent { + component_id: output.output_id.component.id().to_string(), + component_type: output.component_type.to_string(), + component_kind: output.component_kind.to_string(), + event: event_wrapper, + }, + )), + } + }) + .collect(), + TapPayload::Notification(notification) => { + vec![create_notification_event(notification.as_str())] + } + } +} + +fn create_notification_event(message: &str) -> StreamOutputEventsResponse { + StreamOutputEventsResponse { + event: Some(stream_output_events_response::Event::Notification( + EventNotification { + message: message.to_string(), + }, + )), + } +} diff --git a/src/api/grpc_server.rs b/src/api/grpc_server.rs new file mode 100644 index 0000000000000..f582b2f82981c --- /dev/null +++ b/src/api/grpc_server.rs @@ -0,0 +1,177 @@ +use std::{ + error::Error as StdError, + net::SocketAddr, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, +}; + +use axum::{ + Router, + extract::State, + http::{StatusCode, header}, + response::IntoResponse, + routing::get, +}; +use tokio::sync::oneshot; +use tonic::transport::Server as TonicServer; +use tonic_health::server::{HealthReporter, health_reporter}; +use vector_lib::tap::topology::WatchRx; + +use super::grpc::ObservabilityService; +use crate::{config::Config, proto::observability::Server as ObservabilityServer}; + +/// Shared flag backing the HTTP `/health` endpoint. Mirrors the gRPC +/// `HealthReporter` serving status so HTTP and gRPC probes agree. +type ServingState = Arc; + +/// gRPC API server for Vector observability. +pub struct GrpcServer { + _shutdown: oneshot::Sender<()>, + health_reporter: HealthReporter, + serving: ServingState, + addr: SocketAddr, +} + +impl GrpcServer { + /// Start the gRPC API server. + /// + /// This creates a new gRPC server listening on the configured address and spawns + /// it in the background. The server will shut down gracefully when this struct + /// is dropped. + /// + /// Returns an error if the server fails to bind to the configured address. + pub async fn start(config: &Config, watch_rx: WatchRx) -> crate::Result { + let addr = config.api.address.ok_or_else(|| { + crate::Error::from("API address not configured in config.api.address") + })?; + + // Bind the TCP listener first to ensure the port is available + // This will fail fast if the address is already in use + let listener = tokio::net::TcpListener::bind(addr).await.map_err(|e| { + crate::Error::from(format!("Failed to bind gRPC API server to {}: {}", addr, e)) + })?; + + let actual_addr = listener + .local_addr() + .map_err(|e| crate::Error::from(format!("Failed to get local address: {}", e)))?; + + info!("GRPC API server bound to {}.", actual_addr); + + let service = ObservabilityService::new(watch_rx); + + // Create the standard gRPC health service (grpc.health.v1.Health). + // The empty service ("") is registered as SERVING by default. + let (health_reporter, health_service) = health_reporter(); + + let serving: ServingState = Arc::new(AtomicBool::new(true)); + + let (_shutdown, rx) = oneshot::channel(); + + // Convert the tokio TcpListener into a std listener for hyper's Server. + let std_listener = listener + .into_std() + .map_err(|e| crate::Error::from(format!("Failed to convert TCP listener: {}", e)))?; + std_listener.set_nonblocking(true).map_err(|e| { + crate::Error::from(format!("Failed to set TCP listener non-blocking: {}", e)) + })?; + + let router_serving = Arc::clone(&serving); + + // Spawn the server with the already-bound listener + crate::spawn_in_current_span(async move { + // Build reflection service for tools like grpcurl + let reflection_service = tonic_reflection::server::Builder::configure() + .register_encoded_file_descriptor_set( + crate::proto::observability::FILE_DESCRIPTOR_SET, + ) + .register_encoded_file_descriptor_set(tonic_health::pb::FILE_DESCRIPTOR_SET) + .build() + .expect("Failed to build reflection service"); + + // Build the tonic router (gRPC services) and merge with the HTTP router + // so both protocols share the same port. `accept_http1(true)` lets plain + // HTTP/1.1 requests reach the merged axum routes. + let router = TonicServer::builder() + .accept_http1(true) + .add_service(health_service) + .add_service(ObservabilityServer::new(service)) + .add_service(reflection_service) + .into_router() + .merge(http_router(router_serving)); + + let result = hyper::Server::from_tcp(std_listener) + .expect("Failed to build HTTP server from TCP listener") + .serve(router.into_make_service()) + .with_graceful_shutdown(async { + rx.await.ok(); + info!("GRPC API server shutting down."); + }) + .await; + + if let Err(e) = result { + error!( + message = "GRPC server encountered an error.", + error = %e, + error_source = ?e.source(), + bind_addr = %actual_addr, + ); + } + }); + + info!("GRPC API server started on {}.", actual_addr); + + Ok(Self { + _shutdown, + health_reporter, + serving, + addr: actual_addr, + }) + } + + /// Signal that the server is no longer serving. + /// + /// Call this **before** draining the topology so that Kubernetes gRPC + /// readiness probes and HTTP `/health` probes fail early and the pod is + /// removed from endpoints before the process exits. + pub async fn set_not_serving(&mut self) { + self.serving.store(false, Ordering::Relaxed); + self.health_reporter + .set_service_status("", tonic_health::ServingStatus::NotServing) + .await; + } + + /// Get the address the server is listening on + pub const fn addr(&self) -> SocketAddr { + self.addr + } +} + +/// Axum router exposing `GET`/`HEAD /health`. +/// +/// Returns `200 {"ok":true}` while the server is serving and +/// `503 {"ok":false}` once [`GrpcServer::set_not_serving`] has been called. +/// Matches the response shape of the pre-gRPC GraphQL-era endpoint so +/// existing HTTP health probes (Kubernetes, load balancers) keep working. +fn http_router(state: ServingState) -> Router { + Router::new() + .route("/health", get(health_handler).head(health_handler)) + .with_state(state) +} + +async fn health_handler(State(state): State) -> impl IntoResponse { + if state.load(Ordering::Relaxed) { + ( + StatusCode::OK, + [(header::CONTENT_TYPE, "application/json")], + r#"{"ok":true}"#, + ) + } else { + ( + StatusCode::SERVICE_UNAVAILABLE, + [(header::CONTENT_TYPE, "application/json")], + r#"{"ok":false}"#, + ) + } +} diff --git a/src/api/handler.rs b/src/api/handler.rs deleted file mode 100644 index 72ecfce87c1dc..0000000000000 --- a/src/api/handler.rs +++ /dev/null @@ -1,23 +0,0 @@ -use std::sync::{ - Arc, - atomic::{self, AtomicBool}, -}; - -use serde_json::json; -use warp::{Rejection, Reply, reply::json}; - -// Health handler, responds with '{ ok: true }' when running and '{ ok: false}' -// when shutting down -pub(super) async fn health(running: Arc) -> Result { - if running.load(atomic::Ordering::Relaxed) { - Ok(warp::reply::with_status( - json(&json!({"ok": true})), - warp::http::StatusCode::OK, - )) - } else { - Ok(warp::reply::with_status( - json(&json!({"ok": false})), - warp::http::StatusCode::SERVICE_UNAVAILABLE, - )) - } -} diff --git a/src/api/mod.rs b/src/api/mod.rs index 947321277e728..f1992193e685a 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -1,9 +1,5 @@ #![allow(missing_docs)] -mod handler; -mod schema; -mod server; -#[cfg(all(test, feature = "vector-api-tests"))] -mod tests; +mod grpc; +mod grpc_server; -pub use schema::build_schema; -pub use server::Server; +pub use grpc_server::GrpcServer; diff --git a/src/api/schema/components/mod.rs b/src/api/schema/components/mod.rs deleted file mode 100644 index 69d509e449a50..0000000000000 --- a/src/api/schema/components/mod.rs +++ /dev/null @@ -1,577 +0,0 @@ -pub mod sink; -pub mod source; -pub mod state; -pub mod transform; - -use std::{ - cmp, - collections::{HashMap, HashSet}, - sync::LazyLock, -}; - -use async_graphql::{Enum, InputObject, Interface, Object, Subscription}; -use tokio_stream::{Stream, StreamExt, wrappers::BroadcastStream}; -use vector_lib::internal_event::DEFAULT_OUTPUT; - -use crate::{ - api::schema::{ - components::state::component_by_component_key, - filter::{self, filter_items}, - relay, sort, - }, - config::{ComponentKey, Config, get_transform_output_ids}, - filter_check, -}; - -#[allow(clippy::duplicated_attributes)] // False positive caused by `ty = "String"` -#[derive(Debug, Clone, Interface)] -#[graphql( - field(name = "component_id", ty = "String"), - field(name = "component_type", ty = "String") -)] -pub enum Component { - Source(source::Source), - Transform(transform::Transform), - Sink(sink::Sink), -} - -#[derive(Enum, Copy, Clone, Eq, PartialEq, Ord, PartialOrd)] -pub enum ComponentKind { - Source, - Transform, - Sink, -} - -impl Component { - const fn get_component_key(&self) -> &ComponentKey { - match self { - Component::Source(c) => &c.0.component_key, - Component::Transform(c) => &c.0.component_key, - Component::Sink(c) => &c.0.component_key, - } - } - - const fn get_component_kind(&self) -> ComponentKind { - match self { - Component::Source(_) => ComponentKind::Source, - Component::Transform(_) => ComponentKind::Transform, - Component::Sink(_) => ComponentKind::Sink, - } - } -} - -#[derive(Default, InputObject)] -pub struct ComponentsFilter { - component_id: Option>, - component_kind: Option>>, - or: Option>, -} - -impl filter::CustomFilter for ComponentsFilter { - fn matches(&self, component: &Component) -> bool { - filter_check!( - self.component_id.as_ref().map(|f| f - .iter() - .all(|f| f.filter_value(&component.get_component_key().to_string()))), - self.component_kind.as_ref().map(|f| f - .iter() - .all(|f| f.filter_value(component.get_component_kind()))) - ); - true - } - - fn or(&self) -> Option<&Vec> { - self.or.as_ref() - } -} - -#[derive(Enum, Copy, Clone, Eq, PartialEq)] -pub enum ComponentsSortFieldName { - ComponentKey, - ComponentKind, -} - -impl sort::SortableByField for Component { - fn sort(&self, rhs: &Self, field: &ComponentsSortFieldName) -> cmp::Ordering { - match field { - ComponentsSortFieldName::ComponentKey => { - Ord::cmp(&self.get_component_key(), &rhs.get_component_key()) - } - ComponentsSortFieldName::ComponentKind => { - Ord::cmp(&self.get_component_kind(), &rhs.get_component_kind()) - } - } - } -} - -#[derive(Default)] -pub struct ComponentsQuery; - -#[allow(clippy::too_many_arguments)] -#[Object] -impl ComponentsQuery { - /// Configured components (sources/transforms/sinks) - async fn components( - &self, - after: Option, - before: Option, - first: Option, - last: Option, - filter: Option, - sort: Option>>, - ) -> relay::ConnectionResult { - let filter = filter.unwrap_or_default(); - let mut components = filter_items(state::get_components().into_iter(), &filter); - - if let Some(sort_fields) = sort { - sort::by_fields(&mut components, &sort_fields); - } - - relay::query( - components.into_iter(), - relay::Params::new(after, before, first, last), - 10, - ) - .await - } - - /// Configured sources - async fn sources( - &self, - after: Option, - before: Option, - first: Option, - last: Option, - filter: Option, - sort: Option>>, - ) -> relay::ConnectionResult { - let filter = filter.unwrap_or_default(); - let mut sources = filter_items(state::get_sources().into_iter(), &filter); - - if let Some(sort_fields) = sort { - sort::by_fields(&mut sources, &sort_fields); - } - - relay::query( - sources.into_iter(), - relay::Params::new(after, before, first, last), - 10, - ) - .await - } - - /// Configured transforms - async fn transforms( - &self, - after: Option, - before: Option, - first: Option, - last: Option, - filter: Option, - sort: Option>>, - ) -> relay::ConnectionResult { - let filter = filter.unwrap_or_default(); - let mut transforms = filter_items(state::get_transforms().into_iter(), &filter); - - if let Some(sort_fields) = sort { - sort::by_fields(&mut transforms, &sort_fields); - } - - relay::query( - transforms.into_iter(), - relay::Params::new(after, before, first, last), - 10, - ) - .await - } - - /// Configured sinks - async fn sinks( - &self, - after: Option, - before: Option, - first: Option, - last: Option, - filter: Option, - sort: Option>>, - ) -> relay::ConnectionResult { - let filter = filter.unwrap_or_default(); - let mut sinks = filter_items(state::get_sinks().into_iter(), &filter); - - if let Some(sort_fields) = sort { - sort::by_fields(&mut sinks, &sort_fields); - } - - relay::query( - sinks.into_iter(), - relay::Params::new(after, before, first, last), - 10, - ) - .await - } - - /// Gets a configured component by component_key - async fn component_by_component_key(&self, component_id: String) -> Option { - let key = ComponentKey::from(component_id); - component_by_component_key(&key) - } -} - -#[derive(Clone, Debug)] -enum ComponentChanged { - Added(Component), - Removed(Component), -} - -static COMPONENT_CHANGED: LazyLock> = - LazyLock::new(|| { - let (tx, _) = tokio::sync::broadcast::channel(10); - tx - }); - -#[derive(Debug, Default)] -pub struct ComponentsSubscription; - -#[Subscription] -impl ComponentsSubscription { - /// Subscribes to all newly added components - async fn component_added(&self) -> impl Stream + use<> { - BroadcastStream::new(COMPONENT_CHANGED.subscribe()).filter_map(|c| match c { - Ok(ComponentChanged::Added(c)) => Some(c), - _ => None, - }) - } - - /// Subscribes to all removed components - async fn component_removed(&self) -> impl Stream + use<> { - BroadcastStream::new(COMPONENT_CHANGED.subscribe()).filter_map(|c| match c { - Ok(ComponentChanged::Removed(c)) => Some(c), - _ => None, - }) - } -} - -/// Update the 'global' configuration that will be consumed by component queries -pub fn update_config(config: &Config) { - let mut new_components = HashMap::new(); - - // Sources - let table_sources = config - .enrichment_tables() - .filter_map(|(k, e)| e.as_source(k)) - .collect::>(); - for (component_key, source) in config - .sources() - .chain(table_sources.iter().map(|(k, s)| (k, s))) - { - new_components.insert( - component_key.clone(), - Component::Source(source::Source(source::Data { - component_key: component_key.clone(), - component_type: source.inner.get_component_name().to_string(), - // TODO(#10745): This is obviously wrong, but there are a lot of assumptions in the - // API modules about `output_type` as it's a sortable field, etc. This is a stopgap - // until we decide how we want to change the rest of the usages. - output_type: source - .inner - .outputs(config.schema.log_namespace()) - .pop() - .unwrap() - .ty, - outputs: source - .inner - .outputs(config.schema.log_namespace()) - .into_iter() - .map(|output| output.port.unwrap_or_else(|| DEFAULT_OUTPUT.to_string())) - .collect(), - })), - ); - } - - // Transforms - for (component_key, transform) in config.transforms() { - new_components.insert( - component_key.clone(), - Component::Transform(transform::Transform(transform::Data { - component_key: component_key.clone(), - component_type: transform.inner.get_component_name().to_string(), - inputs: transform.inputs.clone(), - outputs: get_transform_output_ids( - transform.inner.as_ref(), - "".into(), - config.schema.log_namespace(), - ) - .map(|output| output.port.unwrap_or_else(|| DEFAULT_OUTPUT.to_string())) - .collect(), - })), - ); - } - - // Sinks - let table_sinks = config - .enrichment_tables() - .filter_map(|(k, e)| e.as_sink(k)) - .collect::>(); - for (component_key, sink) in config - .sinks() - .chain(table_sinks.iter().map(|(k, s)| (k, s))) - { - new_components.insert( - component_key.clone(), - Component::Sink(sink::Sink(sink::Data { - component_key: component_key.clone(), - component_type: sink.inner.get_component_name().to_string(), - inputs: sink.inputs.clone(), - })), - ); - } - - // Get the component_ids of existing components - let existing_component_keys = state::get_component_keys(); - let new_component_keys = new_components - .keys() - .cloned() - .collect::>(); - - // Publish all components that have been removed - existing_component_keys - .difference(&new_component_keys) - .for_each(|component_key| { - _ = COMPONENT_CHANGED.send(ComponentChanged::Removed( - state::component_by_component_key(component_key) - .expect("Couldn't get component by key"), - )); - }); - - // Publish all components that have been added - new_component_keys - .difference(&existing_component_keys) - .for_each(|component_key| { - _ = COMPONENT_CHANGED.send(ComponentChanged::Added( - new_components.get(component_key).unwrap().clone(), - )); - }); - - // Override the old component state - state::update(new_components); -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::{ - api::schema::sort, - config::{ComponentKey, DataType, OutputId}, - }; - - /// Generate component fixes for use with tests - fn component_fixtures() -> Vec { - vec![ - Component::Source(source::Source(source::Data { - component_key: ComponentKey::from("gen1"), - component_type: "demo_logs".to_string(), - output_type: DataType::Metric, - outputs: vec![], - })), - Component::Source(source::Source(source::Data { - component_key: ComponentKey::from("gen2"), - component_type: "demo_logs".to_string(), - output_type: DataType::Metric, - outputs: vec![], - })), - Component::Source(source::Source(source::Data { - component_key: ComponentKey::from("gen3"), - component_type: "demo_logs".to_string(), - output_type: DataType::Metric, - outputs: vec![], - })), - Component::Transform(transform::Transform(transform::Data { - component_key: ComponentKey::from("parse_json"), - component_type: "json".to_string(), - inputs: vec![OutputId::from("gen1"), OutputId::from("gen2")].into(), - outputs: vec![], - })), - Component::Sink(sink::Sink(sink::Data { - component_key: ComponentKey::from("devnull"), - component_type: "blackhole".to_string(), - inputs: vec![OutputId::from("gen3"), OutputId::from("parse_json")].into(), - })), - ] - } - - #[test] - fn components_filter_contains() { - let filter = ComponentsFilter { - component_id: Some(vec![filter::StringFilter { - contains: Some("gen".to_string()), - ..Default::default() - }]), - ..Default::default() - }; - - let components = filter_items(component_fixtures().into_iter(), &filter); - - assert_eq!(components.len(), 3); - } - - #[test] - fn components_filter_equals_or() { - let filter = ComponentsFilter { - component_id: Some(vec![filter::StringFilter { - equals: Some("gen1".to_string()), - ..Default::default() - }]), - or: Some(vec![ComponentsFilter { - component_id: Some(vec![filter::StringFilter { - equals: Some("devnull".to_string()), - ..Default::default() - }]), - ..Default::default() - }]), - ..Default::default() - }; - - let components = filter_items(component_fixtures().into_iter(), &filter); - - assert_eq!(components.len(), 2); - } - - #[test] - fn components_filter_and() { - let filter = ComponentsFilter { - component_id: Some(vec![filter::StringFilter { - equals: Some("gen1".to_string()), - ..Default::default() - }]), - component_kind: Some(vec![filter::EqualityFilter { - equals: Some(ComponentKind::Source), - not_equals: None, - }]), - ..Default::default() - }; - - let components = filter_items(component_fixtures().into_iter(), &filter); - - assert_eq!(components.len(), 1); - } - - #[test] - fn components_filter_and_or() { - let filter = ComponentsFilter { - component_id: Some(vec![filter::StringFilter { - equals: Some("gen1".to_string()), - ..Default::default() - }]), - component_kind: Some(vec![filter::EqualityFilter { - equals: Some(ComponentKind::Source), - not_equals: None, - }]), - or: Some(vec![ComponentsFilter { - component_kind: Some(vec![filter::EqualityFilter { - equals: Some(ComponentKind::Sink), - not_equals: None, - }]), - ..Default::default() - }]), - }; - - let components = filter_items(component_fixtures().into_iter(), &filter); - - assert_eq!(components.len(), 2); - } - - #[test] - fn components_sort_asc() { - let mut components = component_fixtures(); - let fields = vec![sort::SortField:: { - field: ComponentsSortFieldName::ComponentKey, - direction: sort::Direction::Asc, - }]; - sort::by_fields(&mut components, &fields); - - let expectations = ["devnull", "gen1", "gen2", "gen3", "parse_json"]; - - for (i, component_id) in expectations.iter().enumerate() { - assert_eq!(components[i].get_component_key().id(), *component_id); - } - } - - #[test] - fn components_sort_desc() { - let mut components = component_fixtures(); - let fields = vec![sort::SortField:: { - field: ComponentsSortFieldName::ComponentKey, - direction: sort::Direction::Desc, - }]; - sort::by_fields(&mut components, &fields); - - let expectations = ["parse_json", "gen3", "gen2", "gen1", "devnull"]; - - for (i, component_id) in expectations.iter().enumerate() { - assert_eq!(components[i].get_component_key().id(), *component_id); - } - } - - #[test] - fn components_sort_multi() { - let mut components = vec![ - Component::Sink(sink::Sink(sink::Data { - component_key: ComponentKey::from("a"), - component_type: "blackhole".to_string(), - inputs: vec![OutputId::from("gen3"), OutputId::from("parse_json")].into(), - })), - Component::Sink(sink::Sink(sink::Data { - component_key: ComponentKey::from("b"), - component_type: "blackhole".to_string(), - inputs: vec![OutputId::from("gen3"), OutputId::from("parse_json")].into(), - })), - Component::Transform(transform::Transform(transform::Data { - component_key: ComponentKey::from("c"), - component_type: "json".to_string(), - inputs: vec![OutputId::from("gen1"), OutputId::from("gen2")].into(), - outputs: vec![], - })), - Component::Source(source::Source(source::Data { - component_key: ComponentKey::from("e"), - component_type: "demo_logs".to_string(), - output_type: DataType::Metric, - outputs: vec![], - })), - Component::Source(source::Source(source::Data { - component_key: ComponentKey::from("d"), - component_type: "demo_logs".to_string(), - output_type: DataType::Metric, - outputs: vec![], - })), - Component::Source(source::Source(source::Data { - component_key: ComponentKey::from("g"), - component_type: "demo_logs".to_string(), - output_type: DataType::Metric, - outputs: vec![], - })), - Component::Source(source::Source(source::Data { - component_key: ComponentKey::from("f"), - component_type: "demo_logs".to_string(), - output_type: DataType::Metric, - outputs: vec![], - })), - ]; - - let fields = vec![ - sort::SortField:: { - field: ComponentsSortFieldName::ComponentKind, - direction: sort::Direction::Asc, - }, - sort::SortField:: { - field: ComponentsSortFieldName::ComponentKey, - direction: sort::Direction::Asc, - }, - ]; - sort::by_fields(&mut components, &fields); - - let expectations = ["d", "e", "f", "g", "c", "a", "b"]; - for (i, component_id) in expectations.iter().enumerate() { - assert_eq!(components[i].get_component_key().id(), *component_id); - } - } -} diff --git a/src/api/schema/components/sink.rs b/src/api/schema/components/sink.rs deleted file mode 100644 index 99bcca38cc634..0000000000000 --- a/src/api/schema/components/sink.rs +++ /dev/null @@ -1,202 +0,0 @@ -use std::cmp; - -use async_graphql::{Enum, InputObject, Object}; - -use super::{Component, source, state, transform}; -use crate::{ - api::schema::{ - filter, - metrics::{self, IntoSinkMetrics}, - sort, - }, - config::{ComponentKey, Inputs, OutputId}, - filter_check, -}; - -#[derive(Debug, Clone)] -pub struct Data { - pub component_key: ComponentKey, - pub component_type: String, - pub inputs: Inputs, -} - -#[derive(Debug, Clone)] -pub struct Sink(pub Data); - -impl Sink { - pub const fn get_component_key(&self) -> &ComponentKey { - &self.0.component_key - } - - pub const fn get_component_type(&self) -> &str { - self.0.component_type.as_str() - } -} - -#[derive(Default, InputObject)] -pub struct SinksFilter { - component_id: Option>, - component_type: Option>, - or: Option>, -} - -impl filter::CustomFilter for SinksFilter { - fn matches(&self, sink: &Sink) -> bool { - filter_check!( - self.component_id.as_ref().map(|f| f - .iter() - .all(|f| f.filter_value(&sink.get_component_key().to_string()))), - self.component_type - .as_ref() - .map(|f| f.iter().all(|f| f.filter_value(sink.get_component_type()))) - ); - true - } - - fn or(&self) -> Option<&Vec> { - self.or.as_ref() - } -} - -#[derive(Enum, Copy, Clone, Eq, PartialEq)] -pub enum SinksSortFieldName { - ComponentKey, - ComponentType, -} - -impl sort::SortableByField for Sink { - fn sort(&self, rhs: &Self, field: &SinksSortFieldName) -> cmp::Ordering { - match field { - SinksSortFieldName::ComponentKey => { - Ord::cmp(self.get_component_key(), rhs.get_component_key()) - } - SinksSortFieldName::ComponentType => { - Ord::cmp(self.get_component_type(), rhs.get_component_type()) - } - } - } -} - -#[Object] -impl Sink { - /// Sink component_id - pub async fn component_id(&self) -> &str { - self.get_component_key().id() - } - - /// Sink type - pub async fn component_type(&self) -> &str { - self.get_component_type() - } - - /// Source inputs - pub async fn sources(&self) -> Vec { - self.0 - .inputs - .iter() - .filter_map(|output_id| match state::component_by_output_id(output_id) { - Some(Component::Source(s)) => Some(s), - _ => None, - }) - .collect() - } - - /// Transform inputs - pub async fn transforms(&self) -> Vec { - self.0 - .inputs - .iter() - .filter_map(|output_id| match state::component_by_output_id(output_id) { - Some(Component::Transform(t)) => Some(t), - _ => None, - }) - .collect() - } - - /// Sink metrics - pub async fn metrics(&self) -> metrics::SinkMetrics { - metrics::by_component_key(self.get_component_key()) - .into_sink_metrics(self.get_component_type()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn sink_fixtures() -> Vec { - vec![ - Sink(Data { - component_key: ComponentKey::from("webserver"), - component_type: "http".to_string(), - inputs: Inputs::default(), - }), - Sink(Data { - component_key: ComponentKey::from("db"), - component_type: "clickhouse".to_string(), - inputs: Inputs::default(), - }), - Sink(Data { - component_key: ComponentKey::from("zip_drive"), - component_type: "file".to_string(), - inputs: Inputs::default(), - }), - ] - } - - #[test] - fn sort_component_id_asc() { - let mut sinks = sink_fixtures(); - let fields = vec![sort::SortField:: { - field: SinksSortFieldName::ComponentKey, - direction: sort::Direction::Asc, - }]; - sort::by_fields(&mut sinks, &fields); - - for (i, component_id) in ["db", "webserver", "zip_drive"].iter().enumerate() { - assert_eq!(sinks[i].get_component_key().to_string(), *component_id); - } - } - - #[test] - fn sort_component_id_desc() { - let mut sinks = sink_fixtures(); - let fields = vec![sort::SortField:: { - field: SinksSortFieldName::ComponentKey, - direction: sort::Direction::Desc, - }]; - sort::by_fields(&mut sinks, &fields); - - for (i, component_id) in ["zip_drive", "webserver", "db"].iter().enumerate() { - assert_eq!(sinks[i].get_component_key().to_string(), *component_id); - } - } - - #[test] - fn sort_component_type_asc() { - let mut sinks = sink_fixtures(); - let fields = vec![sort::SortField:: { - field: SinksSortFieldName::ComponentType, - direction: sort::Direction::Asc, - }]; - sort::by_fields(&mut sinks, &fields); - - for (i, component_id) in ["db", "zip_drive", "webserver"].iter().enumerate() { - assert_eq!(sinks[i].get_component_key().to_string(), *component_id); - } - } - - #[test] - fn sort_component_type_desc() { - let mut sinks = sink_fixtures(); - let fields = vec![sort::SortField:: { - field: SinksSortFieldName::ComponentType, - direction: sort::Direction::Desc, - }]; - sort::by_fields(&mut sinks, &fields); - - for (i, component_id) in ["webserver", "zip_drive", "db"].iter().enumerate() { - assert_eq!(sinks[i].get_component_key().to_string(), *component_id); - } - } -} diff --git a/src/api/schema/components/source.rs b/src/api/schema/components/source.rs deleted file mode 100644 index c2289c4b6a15b..0000000000000 --- a/src/api/schema/components/source.rs +++ /dev/null @@ -1,403 +0,0 @@ -use std::cmp; - -use async_graphql::{Enum, InputObject, Object}; - -use super::{Component, sink, state, transform}; -use crate::{ - api::schema::{ - filter, - metrics::{self, IntoSourceMetrics, Output, outputs_by_component_key}, - sort, - }, - config::{ComponentKey, DataType, OutputId}, - filter_check, -}; - -#[derive(Debug, Enum, Eq, PartialEq, Copy, Clone, Ord, PartialOrd)] -pub enum SourceOutputType { - Log, - Metric, - Trace, -} - -#[derive(Debug, Clone)] -pub struct Data { - pub component_key: ComponentKey, - pub component_type: String, - pub output_type: DataType, - pub outputs: Vec, -} - -#[derive(Enum, Copy, Clone, Eq, PartialEq)] -pub enum SourcesSortFieldName { - ComponentKey, - ComponentType, - OutputType, -} - -#[derive(Debug, Clone)] -pub struct Source(pub Data); - -impl Source { - #[allow(clippy::missing_const_for_fn)] // const cannot run destructor - pub fn get_component_key(&self) -> &ComponentKey { - &self.0.component_key - } - pub const fn get_component_type(&self) -> &str { - self.0.component_type.as_str() - } - pub fn get_output_types(&self) -> Vec { - [ - SourceOutputType::Log, - SourceOutputType::Metric, - SourceOutputType::Trace, - ] - .iter() - .copied() - .filter(|s| self.0.output_type.contains(s.into())) - .collect() - } - - pub fn get_outputs(&self) -> &[String] { - self.0.outputs.as_ref() - } -} - -impl From<&SourceOutputType> for DataType { - fn from(s: &SourceOutputType) -> Self { - match s { - SourceOutputType::Log => DataType::Log, - SourceOutputType::Metric => DataType::Metric, - SourceOutputType::Trace => DataType::Trace, - } - } -} - -impl sort::SortableByField for Source { - fn sort(&self, rhs: &Self, field: &SourcesSortFieldName) -> cmp::Ordering { - match field { - SourcesSortFieldName::ComponentKey => { - Ord::cmp(self.get_component_key(), rhs.get_component_key()) - } - SourcesSortFieldName::ComponentType => { - Ord::cmp(self.get_component_type(), rhs.get_component_type()) - } - SourcesSortFieldName::OutputType => { - Ord::cmp(&u8::from(self.0.output_type), &u8::from(rhs.0.output_type)) - } - } - } -} - -#[Object] -impl Source { - /// Source component_id - pub async fn component_id(&self) -> &str { - self.0.component_key.id() - } - - /// Source type - pub async fn component_type(&self) -> &str { - self.get_component_type() - } - - /// Source output type - pub async fn output_types(&self) -> Vec { - self.get_output_types() - } - - /// Source output streams - pub async fn outputs(&self) -> Vec { - outputs_by_component_key(self.get_component_key(), self.get_outputs()) - } - - /// Transform outputs - pub async fn transforms(&self) -> Vec { - state::filter_components(|(_component_key, components)| match components { - Component::Transform(t) - if t.0.inputs.contains(&OutputId::from(&self.0.component_key)) => - { - Some(t.clone()) - } - _ => None, - }) - } - - /// Sink outputs - pub async fn sinks(&self) -> Vec { - state::filter_components(|(_component_key, components)| match components { - Component::Sink(s) if s.0.inputs.contains(&OutputId::from(&self.0.component_key)) => { - Some(s.clone()) - } - _ => None, - }) - } - - /// Source metrics - pub async fn metrics(&self) -> metrics::SourceMetrics { - metrics::by_component_key(&self.0.component_key) - .into_source_metrics(self.get_component_type()) - } -} - -#[derive(Default, InputObject)] -pub(super) struct SourcesFilter { - component_id: Option>, - component_type: Option>, - output_type: Option>>, - or: Option>, -} - -impl filter::CustomFilter for SourcesFilter { - fn matches(&self, source: &Source) -> bool { - filter_check!( - self.component_id.as_ref().map(|f| f - .iter() - .all(|f| f.filter_value(&source.get_component_key().to_string()))), - self.component_type.as_ref().map(|f| f - .iter() - .all(|f| f.filter_value(source.get_component_type()))), - self.output_type - .as_ref() - .map(|f| f.iter().all(|f| f.filter_value(source.get_output_types()))) - ); - true - } - - fn or(&self) -> Option<&Vec> { - self.or.as_ref() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::config::DataType; - - /// Generate component fixes for use with tests - fn source_fixtures() -> Vec { - vec![ - Source(Data { - component_key: ComponentKey::from("gen1"), - component_type: "demo_logs".to_string(), - output_type: DataType::Log | DataType::Metric, - outputs: vec![], - }), - Source(Data { - component_key: ComponentKey::from("gen2"), - component_type: "demo_logs".to_string(), - output_type: DataType::Log, - outputs: vec![], - }), - Source(Data { - component_key: ComponentKey::from("gen3"), - component_type: "demo_logs".to_string(), - output_type: DataType::Metric, - outputs: vec![], - }), - ] - } - - #[test] - fn filter_output_type() { - struct Test { - component_id: &'static str, - output_types: Vec, - } - - let tests = vec![ - Test { - component_id: "gen1", - output_types: vec![SourceOutputType::Log, SourceOutputType::Metric], - }, - Test { - component_id: "gen2", - output_types: vec![SourceOutputType::Log], - }, - Test { - component_id: "gen3", - output_types: vec![SourceOutputType::Metric], - }, - ]; - - for t in tests { - let filter = SourcesFilter { - component_id: Some(vec![filter::StringFilter { - equals: Some(t.component_id.to_string()), - ..Default::default() - }]), - output_type: Some(vec![filter::ListFilter:: { - equals: Some(t.output_types), - not_equals: None, - contains: None, - not_contains: None, - }]), - ..Default::default() - }; - - let sources = filter::filter_items(source_fixtures().into_iter(), &filter); - assert_eq!(sources.len(), 1); - } - } - - #[test] - fn sort_component_id_desc() { - let mut sources = source_fixtures(); - let fields = vec![sort::SortField:: { - field: SourcesSortFieldName::ComponentKey, - direction: sort::Direction::Desc, - }]; - sort::by_fields(&mut sources, &fields); - - for (i, component_id) in ["gen3", "gen2", "gen1"].iter().enumerate() { - assert_eq!(sources[i].get_component_key().to_string(), *component_id); - } - } - - #[test] - fn sort_component_type_asc() { - let mut sources = vec![ - Source(Data { - component_key: ComponentKey::from("gen2"), - component_type: "file".to_string(), - output_type: DataType::Log | DataType::Metric, - outputs: vec![], - }), - Source(Data { - component_key: ComponentKey::from("gen3"), - component_type: "demo_logs".to_string(), - output_type: DataType::Log, - outputs: vec![], - }), - Source(Data { - component_key: ComponentKey::from("gen1"), - component_type: "docker_logs".to_string(), - output_type: DataType::Metric, - outputs: vec![], - }), - ]; - - let fields = vec![sort::SortField:: { - field: SourcesSortFieldName::ComponentType, - direction: sort::Direction::Asc, - }]; - sort::by_fields(&mut sources, &fields); - - for (i, component_id) in ["gen3", "gen1", "gen2"].iter().enumerate() { - assert_eq!(sources[i].get_component_key().to_string(), *component_id); - } - } - - #[test] - fn sort_component_type_desc() { - let mut sources = vec![ - Source(Data { - component_key: ComponentKey::from("gen3"), - component_type: "file".to_string(), - output_type: DataType::Log | DataType::Metric, - outputs: vec![], - }), - Source(Data { - component_key: ComponentKey::from("gen2"), - component_type: "demo_logs".to_string(), - output_type: DataType::Log, - outputs: vec![], - }), - Source(Data { - component_key: ComponentKey::from("gen1"), - component_type: "docker_logs".to_string(), - output_type: DataType::Metric, - outputs: vec![], - }), - ]; - - let fields = vec![sort::SortField:: { - field: SourcesSortFieldName::ComponentType, - direction: sort::Direction::Desc, - }]; - sort::by_fields(&mut sources, &fields); - - for (i, component_id) in ["gen3", "gen1", "gen2"].iter().enumerate() { - assert_eq!(sources[i].get_component_key().to_string(), *component_id); - } - } - - #[test] - fn sort_output_type_asc() { - let mut sources = vec![ - Source(Data { - component_key: ComponentKey::from("gen4"), - component_type: "demo_trace".to_string(), - output_type: DataType::Trace, - outputs: vec![], - }), - Source(Data { - component_key: ComponentKey::from("gen1"), - component_type: "demo_logs".to_string(), - output_type: DataType::Metric, - outputs: vec![], - }), - Source(Data { - component_key: ComponentKey::from("gen2"), - component_type: "file".to_string(), - output_type: DataType::Log, - outputs: vec![], - }), - Source(Data { - component_key: ComponentKey::from("gen3"), - component_type: "multiple_type".to_string(), - output_type: DataType::Log | DataType::Metric | DataType::Trace, - outputs: vec![], - }), - ]; - - let fields = vec![sort::SortField:: { - field: SourcesSortFieldName::OutputType, - direction: sort::Direction::Asc, - }]; - sort::by_fields(&mut sources, &fields); - - for (i, component_id) in ["gen2", "gen1", "gen4", "gen3"].iter().enumerate() { - assert_eq!(sources[i].get_component_key().to_string(), *component_id); - } - } - - #[test] - fn sort_output_type_desc() { - let mut sources = vec![ - Source(Data { - component_key: ComponentKey::from("gen4"), - component_type: "demo_trace".to_string(), - output_type: DataType::Trace, - outputs: vec![], - }), - Source(Data { - component_key: ComponentKey::from("gen1"), - component_type: "demo_logs".to_string(), - output_type: DataType::Metric, - outputs: vec![], - }), - Source(Data { - component_key: ComponentKey::from("gen2"), - component_type: "file".to_string(), - output_type: DataType::Log, - outputs: vec![], - }), - Source(Data { - component_key: ComponentKey::from("gen3"), - component_type: "multiple_type".to_string(), - output_type: DataType::Log | DataType::Metric | DataType::Trace, - outputs: vec![], - }), - ]; - - let fields = vec![sort::SortField:: { - field: SourcesSortFieldName::OutputType, - direction: sort::Direction::Desc, - }]; - sort::by_fields(&mut sources, &fields); - for (i, component_id) in ["gen3", "gen4", "gen1", "gen2"].iter().enumerate() { - assert_eq!(sources[i].get_component_key().to_string(), *component_id); - } - } -} diff --git a/src/api/schema/components/state.rs b/src/api/schema/components/state.rs deleted file mode 100644 index 1bb04894d81a9..0000000000000 --- a/src/api/schema/components/state.rs +++ /dev/null @@ -1,89 +0,0 @@ -use std::{ - collections::{HashMap, HashSet}, - sync::{Arc, LazyLock, RwLock}, -}; - -use super::{Component, sink, source, transform}; -use crate::config::{ComponentKey, OutputId}; - -pub const INVARIANT: &str = "Couldn't acquire lock on Vector components. Please report this."; - -pub static COMPONENTS: LazyLock>>> = - LazyLock::new(|| Arc::new(RwLock::new(HashMap::new()))); - -/// Filter components with the provided `map_func` -pub fn filter_components(map_func: impl Fn((&ComponentKey, &Component)) -> Option) -> Vec { - COMPONENTS - .read() - .expect(INVARIANT) - .iter() - .filter_map(map_func) - .collect() -} - -/// Returns all components -pub fn get_components() -> Vec { - filter_components(|(_component_key, components)| Some(components.clone())) -} - -/// Filters components, and returns a clone of sources -pub fn get_sources() -> Vec { - filter_components(|(_, components)| match components { - Component::Source(s) => Some(s.clone()), - _ => None, - }) -} - -/// Filters components, and returns a clone of transforms -pub fn get_transforms() -> Vec { - filter_components(|(_, components)| match components { - Component::Transform(t) => Some(t.clone()), - _ => None, - }) -} - -/// Filters components, and returns a clone of sinks -pub fn get_sinks() -> Vec { - filter_components(|(_, components)| match components { - Component::Sink(s) => Some(s.clone()), - _ => None, - }) -} - -/// Returns the current component component_keys as a HashSet -pub fn get_component_keys() -> HashSet { - COMPONENTS - .read() - .expect(INVARIANT) - .keys() - .cloned() - .collect::>() -} - -/// Gets a component by component_key -pub fn component_by_component_key(component_key: &ComponentKey) -> Option { - Some( - COMPONENTS - .read() - .expect(INVARIANT) - .get(component_key)? - .clone(), - ) -} - -/// Gets a component by output_id -pub fn component_by_output_id(output_id: &OutputId) -> Option { - filter_components(|(key, component)| { - if key == &output_id.component { - Some(component.clone()) - } else { - None - } - }) - .pop() -} - -/// Overwrites component state with new components. -pub fn update(new_components: HashMap) { - *COMPONENTS.write().expect(INVARIANT) = new_components -} diff --git a/src/api/schema/components/transform.rs b/src/api/schema/components/transform.rs deleted file mode 100644 index 7fe95238b7dda..0000000000000 --- a/src/api/schema/components/transform.rs +++ /dev/null @@ -1,223 +0,0 @@ -use std::cmp; - -use async_graphql::{Enum, InputObject, Object}; - -use super::{Component, sink, source, state}; -use crate::{ - api::schema::{ - filter, - metrics::{self, IntoTransformMetrics, Output, outputs_by_component_key}, - sort, - }, - config::{ComponentKey, Inputs, OutputId}, - filter_check, -}; - -#[derive(Debug, Clone)] -pub struct Data { - pub component_key: ComponentKey, - pub component_type: String, - pub inputs: Inputs, - pub outputs: Vec, -} - -#[derive(Debug, Clone)] -pub struct Transform(pub Data); - -impl Transform { - pub const fn get_component_key(&self) -> &ComponentKey { - &self.0.component_key - } - pub const fn get_component_type(&self) -> &str { - self.0.component_type.as_str() - } - pub fn get_outputs(&self) -> &[String] { - self.0.outputs.as_ref() - } -} - -#[derive(Enum, Copy, Clone, Eq, PartialEq)] -pub enum TransformsSortFieldName { - ComponentKey, - ComponentType, -} - -impl sort::SortableByField for Transform { - fn sort(&self, rhs: &Self, field: &TransformsSortFieldName) -> cmp::Ordering { - match field { - TransformsSortFieldName::ComponentKey => { - Ord::cmp(self.get_component_key(), rhs.get_component_key()) - } - TransformsSortFieldName::ComponentType => { - Ord::cmp(self.get_component_type(), rhs.get_component_type()) - } - } - } -} - -#[Object] -impl Transform { - /// Transform component_id - pub async fn component_id(&self) -> &str { - self.0.component_key.id() - } - - /// Transform type - pub async fn component_type(&self) -> &str { - self.get_component_type() - } - - /// Transform output streams - pub async fn outputs(&self) -> Vec { - outputs_by_component_key(self.get_component_key(), self.get_outputs()) - } - - /// Source inputs - pub async fn sources(&self) -> Vec { - self.0 - .inputs - .iter() - .filter_map(|output_id| match state::component_by_output_id(output_id) { - Some(Component::Source(s)) => Some(s), - _ => None, - }) - .collect() - } - - /// Transform outputs - pub async fn transforms(&self) -> Vec { - state::filter_components(|(_component_key, components)| match components { - Component::Transform(t) - if t.0.inputs.contains(&OutputId::from(&self.0.component_key)) => - { - Some(t.clone()) - } - _ => None, - }) - } - - /// Sink outputs - pub async fn sinks(&self) -> Vec { - state::filter_components(|(_component_key, components)| match components { - Component::Sink(s) if s.0.inputs.contains(&OutputId::from(&self.0.component_key)) => { - Some(s.clone()) - } - _ => None, - }) - } - - /// Transform metrics - pub async fn metrics(&self) -> metrics::TransformMetrics { - metrics::by_component_key(&self.0.component_key) - .into_transform_metrics(self.get_component_type()) - } -} - -#[derive(Default, InputObject)] -pub struct TransformsFilter { - component_id: Option>, - component_type: Option>, - or: Option>, -} - -impl filter::CustomFilter for TransformsFilter { - fn matches(&self, transform: &Transform) -> bool { - filter_check!( - self.component_id.as_ref().map(|f| f - .iter() - .all(|f| f.filter_value(&transform.get_component_key().to_string()))), - self.component_type.as_ref().map(|f| f - .iter() - .all(|f| f.filter_value(transform.get_component_type()))) - ); - true - } - - fn or(&self) -> Option<&Vec> { - self.or.as_ref() - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn transform_fixtures() -> Vec { - vec![ - Transform(Data { - component_key: ComponentKey::from("parse_json"), - component_type: "json".to_string(), - inputs: Inputs::default(), - outputs: vec![], - }), - Transform(Data { - component_key: ComponentKey::from("field_adder"), - component_type: "add_fields".to_string(), - inputs: Inputs::default(), - outputs: vec![], - }), - Transform(Data { - component_key: ComponentKey::from("append"), - component_type: "concat".to_string(), - inputs: Inputs::default(), - outputs: vec![], - }), - ] - } - - #[test] - fn sort_component_id_asc() { - let mut transforms = transform_fixtures(); - let fields = vec![sort::SortField:: { - field: TransformsSortFieldName::ComponentKey, - direction: sort::Direction::Asc, - }]; - sort::by_fields(&mut transforms, &fields); - - for (i, component_id) in ["append", "field_adder", "parse_json"].iter().enumerate() { - assert_eq!(transforms[i].get_component_key().to_string(), *component_id); - } - } - - #[test] - fn sort_component_id_desc() { - let mut transforms = transform_fixtures(); - let fields = vec![sort::SortField:: { - field: TransformsSortFieldName::ComponentKey, - direction: sort::Direction::Desc, - }]; - sort::by_fields(&mut transforms, &fields); - - for (i, component_id) in ["parse_json", "field_adder", "append"].iter().enumerate() { - assert_eq!(transforms[i].get_component_key().to_string(), *component_id); - } - } - - #[test] - fn sort_component_type_asc() { - let mut transforms = transform_fixtures(); - let fields = vec![sort::SortField:: { - field: TransformsSortFieldName::ComponentType, - direction: sort::Direction::Asc, - }]; - sort::by_fields(&mut transforms, &fields); - - for (i, component_id) in ["field_adder", "append", "parse_json"].iter().enumerate() { - assert_eq!(transforms[i].get_component_key().to_string(), *component_id); - } - } - - #[test] - fn sort_component_type_desc() { - let mut transforms = transform_fixtures(); - let fields = vec![sort::SortField:: { - field: TransformsSortFieldName::ComponentType, - direction: sort::Direction::Desc, - }]; - sort::by_fields(&mut transforms, &fields); - - for (i, component_id) in ["parse_json", "append", "field_adder"].iter().enumerate() { - assert_eq!(transforms[i].get_component_key().to_string(), *component_id); - } - } -} diff --git a/src/api/schema/events/encoding.rs b/src/api/schema/events/encoding.rs deleted file mode 100644 index 4b455b5e2a2fc..0000000000000 --- a/src/api/schema/events/encoding.rs +++ /dev/null @@ -1,9 +0,0 @@ -use async_graphql::Enum; - -#[derive(Enum, Copy, Clone, PartialEq, Eq)] -/// Encoding format for the event -pub enum EventEncodingType { - Json, - Yaml, - Logfmt, -} diff --git a/src/api/schema/events/log.rs b/src/api/schema/events/log.rs deleted file mode 100644 index a06a3ff5f5eab..0000000000000 --- a/src/api/schema/events/log.rs +++ /dev/null @@ -1,77 +0,0 @@ -use std::borrow::Cow; - -use async_graphql::Object; -use chrono::{DateTime, Utc}; -use vector_lib::{encode_logfmt, event, tap::topology::TapOutput}; -use vrl::event_path; - -use super::EventEncodingType; - -#[derive(Debug, Clone)] -pub struct Log { - output: TapOutput, - event: event::LogEvent, -} - -impl Log { - pub const fn new(output: TapOutput, event: event::LogEvent) -> Self { - Self { output, event } - } - - pub fn get_message(&self) -> Option> { - Some(self.event.get(event_path!("message"))?.to_string_lossy()) - } - - pub fn get_timestamp(&self) -> Option<&DateTime> { - self.event.get(event_path!("timestamp"))?.as_timestamp() - } -} - -#[Object] -/// Log event with fields for querying log data -impl Log { - /// Id of the component associated with the log event - async fn component_id(&self) -> &str { - self.output.output_id.component.id() - } - - /// Type of component associated with the log event - async fn component_type(&self) -> &str { - self.output.component_type.as_ref() - } - - /// Kind of component associated with the log event - async fn component_kind(&self) -> &str { - self.output.component_kind - } - - /// Log message - async fn message(&self) -> Option { - self.get_message().map(Into::into) - } - - /// Log timestamp - async fn timestamp(&self) -> Option<&DateTime> { - self.get_timestamp() - } - - /// Log event as an encoded string format - async fn string(&self, encoding: EventEncodingType) -> String { - match encoding { - EventEncodingType::Json => serde_json::to_string(&self.event) - .expect("JSON serialization of log event failed. Please report."), - EventEncodingType::Yaml => serde_yaml::to_string(&self.event) - .expect("YAML serialization of log event failed. Please report."), - EventEncodingType::Logfmt => encode_logfmt::encode_value(self.event.value()) - .expect("logfmt serialization of log event failed. Please report."), - } - } - - /// Get JSON field data on the log event, by field name - async fn json(&self, field: String) -> Option { - self.event.get(event_path!(field.as_str())).map(|field| { - serde_json::to_string(field) - .expect("JSON serialization of trace event field failed. Please report.") - }) - } -} diff --git a/src/api/schema/events/metric.rs b/src/api/schema/events/metric.rs deleted file mode 100644 index b9ae2a31a3d27..0000000000000 --- a/src/api/schema/events/metric.rs +++ /dev/null @@ -1,139 +0,0 @@ -use async_graphql::{Enum, Object}; -use chrono::{DateTime, Utc}; -use serde_json::Value; -use vector_lib::{encode_logfmt, event, tap::topology::TapOutput}; - -use super::EventEncodingType; - -#[derive(Debug, Clone)] -pub struct Metric { - output: TapOutput, - event: event::Metric, -} - -impl Metric { - pub const fn new(output: TapOutput, event: event::Metric) -> Self { - Self { output, event } - } -} - -#[derive(Copy, Clone, Debug, PartialEq, Eq, Enum)] -enum MetricKind { - /// Incremental metrics update previous values - Incremental, - /// Absolute metrics set the reference value for future updates - Absolute, -} - -impl From for MetricKind { - fn from(kind: event::MetricKind) -> Self { - match kind { - event::MetricKind::Incremental => Self::Incremental, - event::MetricKind::Absolute => Self::Absolute, - } - } -} - -struct MetricTag { - key: String, - value: String, -} - -#[Object] -impl MetricTag { - /// Metric tag key - async fn key(&self) -> &str { - self.key.as_ref() - } - - /// Metric tag value - async fn value(&self) -> &str { - self.value.as_ref() - } -} - -#[Object] -/// Metric event with fields for querying metric data -impl Metric { - /// Id of the component associated with the metric event - async fn component_id(&self) -> &str { - self.output.output_id.component.id() - } - - /// Type of component associated with the metric event - async fn component_type(&self) -> &str { - self.output.component_type.as_ref() - } - - /// Kind of component associated with the metric event - async fn component_kind(&self) -> &str { - self.output.component_kind - } - - /// Metric timestamp - async fn timestamp(&self) -> Option<&DateTime> { - self.event.data().timestamp() - } - - /// Metric name - async fn name(&self) -> &str { - self.event.name() - } - - /// Metric namespace - async fn namespace(&self) -> Option<&str> { - self.event.namespace() - } - - /// Metric kind - async fn kind(&self) -> MetricKind { - self.event.kind().into() - } - - /// Metric type - async fn value_type(&self) -> &str { - self.event.value().as_name() - } - - /// Metric value in human readable form - async fn value(&self) -> String { - self.event.value().to_string() - } - - /// Metric tags - async fn tags(&self) -> Option> { - self.event.tags().map(|tags| { - tags.iter_single() - .map(|(key, value)| MetricTag { - key: key.to_owned(), - value: value.to_owned(), - }) - .collect() - }) - } - - /// Metric event as an encoded string format - async fn string(&self, encoding: EventEncodingType) -> String { - match encoding { - EventEncodingType::Json => serde_json::to_string(&self.event) - .expect("JSON serialization of metric event failed. Please report."), - EventEncodingType::Yaml => serde_yaml::to_string(&self.event) - .expect("YAML serialization of metric event failed. Please report."), - EventEncodingType::Logfmt => { - let json = serde_json::to_value(&self.event) - .expect("logfmt serialization of metric event failed: conversion to serde Value failed. Please report."); - match json { - Value::Object(map) => encode_logfmt::encode_map( - &map.into_iter() - .map(|(k, v)| (event::KeyString::from(k), v)) - .collect(), - ) - .expect("logfmt serialization of metric event failed. Please report."), - _ => panic!( - "logfmt serialization of metric event failed: metric converted to unexpected serde Value. Please report." - ), - } - } - } - } -} diff --git a/src/api/schema/events/mod.rs b/src/api/schema/events/mod.rs deleted file mode 100644 index 9bebbdc194351..0000000000000 --- a/src/api/schema/events/mod.rs +++ /dev/null @@ -1,157 +0,0 @@ -pub mod encoding; -pub mod log; -pub mod metric; -pub mod output; -pub mod trace; - -use std::time::{SystemTime, UNIX_EPOCH}; - -use async_graphql::{Context, Subscription}; -use encoding::EventEncodingType; -use futures::{Stream, StreamExt, stream}; -use output::{OutputEventsPayload, from_tap_payload_to_output_events}; -use rand::{Rng, SeedableRng, rngs::SmallRng}; -use tokio::{select, sync::mpsc, time}; -use tokio_stream::wrappers::ReceiverStream; -use vector_lib::tap::{ - controller::{TapController, TapPatterns}, - topology::WatchRx, -}; - -#[derive(Debug, Default)] -pub struct EventsSubscription; - -#[Subscription] -impl EventsSubscription { - /// A stream of events emitted from matched component ID patterns - pub async fn output_events_by_component_id_patterns<'a>( - &'a self, - ctx: &'a Context<'a>, - outputs_patterns: Vec, - inputs_patterns: Option>, - #[graphql(default = 500)] interval: u32, - #[graphql(default = 100, validator(minimum = 1, maximum = 10_000))] limit: u32, - ) -> impl Stream> + 'a { - let watch_rx = ctx.data_unchecked::().clone(); - - let patterns = TapPatterns { - for_outputs: outputs_patterns.into_iter().collect(), - for_inputs: inputs_patterns.unwrap_or_default().into_iter().collect(), - }; - // Client input is confined to `u32` to provide sensible bounds. - create_events_stream(watch_rx, patterns, interval as u64, limit as usize) - } -} - -/// Creates an events stream based on component ids, and a provided interval. Will emit -/// control messages that bubble up the application if the sink goes away. The stream contains -/// all matching events; filtering should be done at the caller level. -pub(crate) fn create_events_stream( - watch_rx: WatchRx, - patterns: TapPatterns, - interval: u64, - limit: usize, -) -> impl Stream> { - // Channel for receiving individual tap payloads. Since we can process at most `limit` per - // interval, this is capped to the same value. - let (tap_tx, tap_rx) = mpsc::channel(limit); - let mut tap_rx = ReceiverStream::new(tap_rx) - .flat_map(|payload| stream::iter(from_tap_payload_to_output_events(payload))); - - // The resulting vector of `Event` sent to the client. Only one result set will be streamed - // back to the client at a time. This value is set higher than `1` to prevent blocking the event - // pipeline on slower client connections, but low enough to apply a modest cap on mem usage. - let (event_tx, event_rx) = mpsc::channel::>(10); - - tokio::spawn(async move { - // Create a tap controller. When this drops out of scope, clean up will be performed on the - // event handlers and topology observation that the tap controller provides. - let _tap_controller = TapController::new(watch_rx, tap_tx, patterns); - - // A tick interval to represent when to 'cut' the results back to the client. - let mut interval = time::interval(time::Duration::from_millis(interval)); - - // Temporary structure to hold sortable values of `Event`. - struct SortableOutputEventsPayload { - batch: usize, - payload: OutputEventsPayload, - } - - // Collect a vector of results, with a capacity of `limit`. As new `Event`s come in, - // they will be sampled and added to results. - let mut results = Vec::::with_capacity(limit); - - // Random number generator to allow for sampling. Speed trumps cryptographic security here. - // The RNG must be Send + Sync to use with the `select!` loop below, hence `SmallRng`. - let seed = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_nanos() as u64; - let mut rng = SmallRng::seed_from_u64(seed); - - // Keep a count of the batch size, which will be used as a seed for random eviction - // per the sampling strategy used below. - let mut batch = 0; - - loop { - select! { - // Process `TapPayload`s. A tap payload could contain log/metric events or a - // notification. Notifications are emitted immediately; events buffer until - // the next `interval`. - Some(payload) = tap_rx.next() => { - // Emit notifications immediately; these don't count as a 'batch'. - if let OutputEventsPayload::Notification(_) = payload { - // If an error occurs when sending, the subscription has likely gone - // away. Break the loop to terminate the thread. - if let Err(err) = event_tx.send(vec![payload]).await { - debug!(message = "Couldn't send notification.", error = ?err); - break; - } - } else { - // Wrap tap in a 'sortable' wrapper, using the batch as a key, to - // re-sort after random eviction. - let payload = SortableOutputEventsPayload { batch, payload }; - - // A simple implementation of "Algorithm R" per - // https://en.wikipedia.org/wiki/Reservoir_sampling. As we're unable to - // pluck the nth result, this is chosen over the more optimal "Algorithm L" - // since discarding results isn't an option. - if limit > results.len() { - results.push(payload); - } else { - let random_number = rng.random_range(0..batch); - if random_number < results.len() { - results[random_number] = payload; - } - } - // Increment the batch count, to be used for the next Algo R loop. - batch += 1; - } - } - _ = interval.tick() => { - // If there are any existing results after the interval tick, emit. - if !results.is_empty() { - // Reset the batch count, to adjust sampling probability for the next round. - batch = 0; - - // Since events will appear out of order per the random sampling - // strategy, drain the existing results and sort by timestamp. - results.sort_by_key(|r| r.batch); - let results = results.drain(..) - .map(|r| r.payload) - .collect(); - - // If we get an error here, it likely means that the subscription has - // gone has away. This is a valid/common situation. - if let Err(err) = event_tx.send(results).await { - debug!(message = "Couldn't send events.", error = ?err); - break; - } - } - } - } - } - }); - - ReceiverStream::new(event_rx) -} diff --git a/src/api/schema/events/output.rs b/src/api/schema/events/output.rs deleted file mode 100644 index de54bfefaad39..0000000000000 --- a/src/api/schema/events/output.rs +++ /dev/null @@ -1,70 +0,0 @@ -use async_graphql::{Object, Union}; -use vector_lib::tap::{controller::TapPayload, notification::Notification}; - -use crate::api::schema::events::{log::Log, metric::Metric, trace::Trace}; - -/// This wrapper struct hoists `message` up from [`Notification`] for a more -/// natural querying experience. While ideally [`Notification`] would be a -/// GraphQL interface with a common `message` field, an interface cannot be -/// directly nested into the union of [`super::OutputEventsPayload`]. -/// -/// The GraphQL specification forbids such a nesting: -/// -#[derive(Debug, Clone)] -pub struct EventNotification { - pub notification: Notification, -} - -#[Object] -/// A notification regarding events observation -impl EventNotification { - /// Notification details - async fn notification(&self) -> &Notification { - &self.notification - } - - /// The human-readable message associated with the notification - async fn message(&self) -> &str { - self.notification.as_str() - } -} - -#[derive(Union, Debug, Clone)] -#[allow(clippy::large_enum_variant)] -/// An event or a notification -pub enum OutputEventsPayload { - /// Log event - Log(Log), - - /// Metric event - Metric(Metric), - - // Notification - Notification(EventNotification), - - /// Trace event - Trace(Trace), -} - -/// Convert an `api::TapPayload` to the equivalent GraphQL type. -pub(crate) fn from_tap_payload_to_output_events(t: TapPayload) -> Vec { - match t { - TapPayload::Log(output, log_array) => log_array - .into_iter() - .map(|log| OutputEventsPayload::Log(Log::new(output.clone(), log))) - .collect(), - TapPayload::Metric(output, metric_array) => metric_array - .into_iter() - .map(|metric| OutputEventsPayload::Metric(Metric::new(output.clone(), metric))) - .collect(), - TapPayload::Notification(notification) => { - vec![OutputEventsPayload::Notification(EventNotification { - notification, - })] - } - TapPayload::Trace(output, trace_array) => trace_array - .into_iter() - .map(|trace| OutputEventsPayload::Trace(Trace::new(output.clone(), trace))) - .collect(), - } -} diff --git a/src/api/schema/events/trace.rs b/src/api/schema/events/trace.rs deleted file mode 100644 index 35b697b351a9e..0000000000000 --- a/src/api/schema/events/trace.rs +++ /dev/null @@ -1,56 +0,0 @@ -use async_graphql::Object; -use vector_lib::{encode_logfmt, event, tap::topology::TapOutput}; -use vrl::event_path; - -use super::EventEncodingType; - -#[derive(Debug, Clone)] -pub struct Trace { - output: TapOutput, - event: event::TraceEvent, -} - -impl Trace { - pub const fn new(output: TapOutput, event: event::TraceEvent) -> Self { - Self { output, event } - } -} - -#[Object] -/// Trace event with fields for querying trace data -impl Trace { - /// Id of the component associated with the trace event - async fn component_id(&self) -> &str { - self.output.output_id.component.id() - } - - /// Type of component associated with the trace event - async fn component_type(&self) -> &str { - self.output.component_type.as_ref() - } - - /// Kind of component associated with the trace event - async fn component_kind(&self) -> &str { - self.output.component_kind - } - - /// Trace event as an encoded string format - async fn string(&self, encoding: EventEncodingType) -> String { - match encoding { - EventEncodingType::Json => serde_json::to_string(&self.event) - .expect("JSON serialization of log event failed. Please report."), - EventEncodingType::Yaml => serde_yaml::to_string(&self.event) - .expect("YAML serialization of log event failed. Please report."), - EventEncodingType::Logfmt => encode_logfmt::encode_map(self.event.as_map()) - .expect("logfmt serialization of log event failed. Please report."), - } - } - - /// Get JSON field data on the trace event, by field name - async fn json(&self, field: String) -> Option { - self.event.get(event_path!(field.as_str())).map(|field| { - serde_json::to_string(field) - .expect("JSON serialization of log event field failed. Please report.") - }) - } -} diff --git a/src/api/schema/filter.rs b/src/api/schema/filter.rs deleted file mode 100644 index ac8f7f551d2ae..0000000000000 --- a/src/api/schema/filter.rs +++ /dev/null @@ -1,219 +0,0 @@ -use std::collections::BTreeSet; - -use async_graphql::{InputObject, InputType}; - -use super::components::{ComponentKind, source}; - -/// Takes an `&Option` and returns early if false -#[macro_export] -macro_rules! filter_check { - ($($match:expr_2021),+) => { - $( - if matches!($match, Some(t) if !t) { - return false; - } - )+ - } -} - -#[derive(Default, InputObject)] -/// Filter for String values -pub struct StringFilter { - pub equals: Option, - pub not_equals: Option, - pub contains: Option, - pub not_contains: Option, - pub starts_with: Option, - pub ends_with: Option, -} - -impl StringFilter { - pub fn filter_value(&self, value: &str) -> bool { - filter_check!( - // Equals - self.equals.as_ref().map(|s| value.eq(s)), - // Not equals - self.not_equals.as_ref().map(|s| !value.eq(s)), - // Contains - self.contains.as_ref().map(|s| value.contains(s)), - // Does not contain - self.not_contains.as_ref().map(|s| !value.contains(s)), - // Starts with - self.starts_with.as_ref().map(|s| value.starts_with(s)), - // Ends with - self.ends_with.as_ref().map(|s| value.ends_with(s)) - ); - true - } -} - -#[derive(InputObject)] -#[graphql(concrete(name = "SourceOutputTypeFilter", params(source::SourceOutputType)))] -// Filter for GraphQL lists -pub struct ListFilter { - pub equals: Option>, - pub not_equals: Option>, - pub contains: Option, - pub not_contains: Option, -} - -impl ListFilter { - pub fn filter_value(&self, value: Vec) -> bool { - let val = BTreeSet::from_iter(value.iter()); - filter_check!( - // Equals - self.equals - .as_ref() - .map(|s| BTreeSet::from_iter(s.iter()).eq(&val)), - // Not Equals - self.not_equals - .as_ref() - .map(|s| !BTreeSet::from_iter(s.iter()).eq(&val)), - // Contains - self.contains.as_ref().map(|s| val.contains(s)), - // Not Contains - self.not_contains.as_ref().map(|s| !val.contains(s)) - ); - true - } -} - -#[derive(InputObject)] -#[graphql(concrete(name = "ComponentKindFilter", params(ComponentKind)))] -pub struct EqualityFilter { - pub equals: Option, - pub not_equals: Option, -} - -impl EqualityFilter { - pub fn filter_value(&self, value: T) -> bool { - filter_check!( - // Equals - self.equals.as_ref().map(|s| value.eq(s)), - // Not equals - self.not_equals.as_ref().map(|s| !value.eq(s)) - ); - true - } -} - -/// CustomFilter trait to determine whether to include/exclude fields based on matches. -pub trait CustomFilter { - fn matches(&self, item: &T) -> bool; - fn or(&self) -> Option<&Vec> - where - Self: Sized; -} - -/// Returns true if a provided `Item` passes all 'AND' or 'OR' filter rules, recursively. -fn filter_item(item: &Item, f: &Filter) -> bool -where - Filter: CustomFilter, -{ - f.matches(item) - || f.or() - .map_or_else(|| false, |f| f.iter().any(|f| filter_item(item, f))) -} - -/// Filters items based on an implementation of `CustomFilter`. -pub fn filter_items(items: Iter, f: &Filter) -> Vec -where - Iter: Iterator, - Filter: CustomFilter, -{ - items.filter(|c| filter_item(c, f)).collect() -} - -#[cfg(test)] -mod test { - use super::StringFilter; - - #[test] - fn string_equals() { - let value = "test"; - - let sf = StringFilter { - equals: value.to_string().into(), - ..Default::default() - }; - - assert!(sf.filter_value(value)); - assert!(!sf.filter_value("not found")); - } - - #[test] - fn string_not_equals() { - let value = "value"; - let diff_value = "different value"; - - let sf = StringFilter { - not_equals: diff_value.to_string().into(), - ..Default::default() - }; - - assert!(sf.filter_value(value)); - assert!(!sf.filter_value(diff_value)); - } - - #[test] - fn string_contains() { - let sf = StringFilter { - contains: "234".to_string().into(), - ..Default::default() - }; - - assert!(sf.filter_value("12345")); - assert!(!sf.filter_value("xxx")); - } - - #[test] - fn string_not_contains() { - let contains = "xyz"; - - let sf = StringFilter { - not_contains: contains.to_string().into(), - ..Default::default() - }; - - assert!(sf.filter_value("abc")); - assert!(!sf.filter_value(contains)); - } - - #[test] - fn string_starts_with() { - let sf = StringFilter { - starts_with: "abc".to_string().into(), - ..Default::default() - }; - - assert!(sf.filter_value("abcdef")); - assert!(!sf.filter_value("xyz")); - } - - #[test] - fn string_ends_with() { - let sf = StringFilter { - ends_with: "456".to_string().into(), - ..Default::default() - }; - - assert!(sf.filter_value("123456")); - assert!(!sf.filter_value("123")); - } - - #[test] - fn string_multiple_all_match() { - let value = "123456"; - let sf = StringFilter { - equals: value.to_string().into(), - not_equals: "xyz".to_string().into(), - contains: "234".to_string().into(), - not_contains: "678".to_string().into(), - starts_with: "123".to_string().into(), - ends_with: "456".to_string().into(), - }; - - assert!(sf.filter_value(value)); - assert!(!sf.filter_value("should fail")); - } -} diff --git a/src/api/schema/gen.rs b/src/api/schema/gen.rs deleted file mode 100644 index c359f0db3b640..0000000000000 --- a/src/api/schema/gen.rs +++ /dev/null @@ -1,116 +0,0 @@ -use std::fs; - -use indoc::indoc; -use vector::api::build_schema; - -static INTROSPECTION_QUERY: &str = indoc! {r#" - query IntrospectionQuery { - __schema { - queryType { - name - } - mutationType { - name - } - subscriptionType { - name - } - types { - ...FullType - } - directives { - name - description - locations - args { - ...InputValue - } - } - } - } - fragment FullType on __Type { - kind - name - description - fields(includeDeprecated: true) { - name - description - args { - ...InputValue - } - type { - ...TypeRef - } - isDeprecated - deprecationReason - } - inputFields { - ...InputValue - } - interfaces { - ...TypeRef - } - enumValues(includeDeprecated: true) { - name - description - isDeprecated - deprecationReason - } - possibleTypes { - ...TypeRef - } - } - fragment InputValue on __InputValue { - name - description - type { - ...TypeRef - } - defaultValue - } - fragment TypeRef on __Type { - kind - name - ofType { - kind - name - ofType { - kind - name - ofType { - kind - name - ofType { - kind - name - ofType { - kind - name - ofType { - kind - name - ofType { - kind - name - } - } - } - } - } - } - } - } -"#}; - -#[tokio::main] -async fn main() { - let schema = build_schema().finish(); - let res = schema.execute(INTROSPECTION_QUERY).await; - let json = serde_json::to_string_pretty(&res).unwrap(); - - fs::write( - "lib/vector-api-client/graphql/schema.json", - format!("{json}\n"), - ) - .expect("Couldn't save schema file"); -} diff --git a/src/api/schema/health.rs b/src/api/schema/health.rs deleted file mode 100644 index 0ee3d1d897cb1..0000000000000 --- a/src/api/schema/health.rs +++ /dev/null @@ -1,43 +0,0 @@ -use async_graphql::{Object, SimpleObject, Subscription}; -use chrono::{DateTime, Utc}; -use tokio::time::Duration; -use tokio_stream::{Stream, StreamExt, wrappers::IntervalStream}; - -#[derive(SimpleObject)] -pub struct Heartbeat { - utc: DateTime, -} - -impl Heartbeat { - fn new() -> Self { - Heartbeat { utc: Utc::now() } - } -} - -#[derive(Default)] -pub(super) struct HealthQuery; - -#[Object] -impl HealthQuery { - /// Returns `true` to denote the GraphQL server is reachable - async fn health(&self) -> bool { - true - } -} - -#[derive(Default)] -pub struct HealthSubscription; - -#[Subscription] -impl HealthSubscription { - /// Heartbeat, containing the UTC timestamp of the last server-sent payload - async fn heartbeat( - &self, - #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream + use<> { - IntervalStream::new(tokio::time::interval(Duration::from_millis( - interval as u64, - ))) - .map(|_| Heartbeat::new()) - } -} diff --git a/src/api/schema/meta.rs b/src/api/schema/meta.rs deleted file mode 100644 index 76cbf2749c225..0000000000000 --- a/src/api/schema/meta.rs +++ /dev/null @@ -1,27 +0,0 @@ -use async_graphql::Object; - -#[derive(Default)] -pub struct Meta; - -#[Object] -impl Meta { - /// Vector version - async fn version_string(&self) -> String { - crate::get_version() - } - - /// Hostname - async fn hostname(&self) -> Option { - crate::get_hostname().ok() - } -} - -#[derive(Default)] -pub struct MetaQuery; - -#[Object] -impl MetaQuery { - async fn meta(&self) -> Meta { - Meta - } -} diff --git a/src/api/schema/metrics/allocated_bytes.rs b/src/api/schema/metrics/allocated_bytes.rs deleted file mode 100644 index 2f2f93ac833ad..0000000000000 --- a/src/api/schema/metrics/allocated_bytes.rs +++ /dev/null @@ -1,71 +0,0 @@ -use async_graphql::Object; -use chrono::{DateTime, Utc}; - -use crate::{ - config::ComponentKey, - event::{Metric, MetricValue}, -}; - -pub struct AllocatedBytes(Metric); - -impl AllocatedBytes { - pub const fn new(m: Metric) -> Self { - Self(m) - } -} - -#[Object] -impl AllocatedBytes { - /// Metric timestamp - pub async fn timestamp(&self) -> Option> { - self.0.timestamp() - } - - /// Allocated bytes - pub async fn allocated_bytes(&self) -> f64 { - match self.0.value() { - MetricValue::Gauge { value } => *value, - _ => 0.00, - } - } -} - -impl From for AllocatedBytes { - fn from(m: Metric) -> Self { - Self(m) - } -} - -pub struct ComponentAllocatedBytes { - component_key: ComponentKey, - metric: Metric, -} - -impl ComponentAllocatedBytes { - /// Returns a new `ComponentAllocatedBytes` struct, which is a GraphQL type. The - /// component id is hoisted for clear field resolution in the resulting payload - pub fn new(metric: Metric) -> Self { - let component_key = metric.tag_value("component_id").expect( - "Returned a metric without a `component_id`, which shouldn't happen. Please report.", - ); - let component_key = ComponentKey::from(component_key); - - Self { - component_key, - metric, - } - } -} - -#[Object] -impl ComponentAllocatedBytes { - /// Component id - async fn component_id(&self) -> &str { - self.component_key.id() - } - - /// Allocated bytes metric - async fn metric(&self) -> AllocatedBytes { - AllocatedBytes::new(self.metric.clone()) - } -} diff --git a/src/api/schema/metrics/errors.rs b/src/api/schema/metrics/errors.rs deleted file mode 100644 index acca7b74bdf26..0000000000000 --- a/src/api/schema/metrics/errors.rs +++ /dev/null @@ -1,71 +0,0 @@ -use async_graphql::Object; -use chrono::{DateTime, Utc}; - -use crate::{ - config::ComponentKey, - event::{Metric, MetricValue}, -}; - -pub struct ErrorsTotal(Metric); - -impl ErrorsTotal { - pub const fn new(m: Metric) -> Self { - Self(m) - } -} - -#[Object] -impl ErrorsTotal { - /// Metric timestamp - pub async fn timestamp(&self) -> Option> { - self.0.timestamp() - } - - /// Total error count - pub async fn errors_total(&self) -> f64 { - match self.0.value() { - MetricValue::Counter { value } => *value, - _ => 0.00, - } - } -} - -impl From for ErrorsTotal { - fn from(m: Metric) -> Self { - Self(m) - } -} - -pub struct ComponentErrorsTotal { - component_key: ComponentKey, - metric: Metric, -} - -impl ComponentErrorsTotal { - /// Returns a new `ComponentErrorsTotal` struct, which is a GraphQL type. The - /// component id is hoisted for clear field resolution in the resulting payload - pub fn new(metric: Metric) -> Self { - let component_key = metric.tag_value("component_id").expect( - "Returned a metric without a `component_id`, which shouldn't happen. Please report.", - ); - let component_key = ComponentKey::from(component_key); - - Self { - component_key, - metric, - } - } -} - -#[Object] -impl ComponentErrorsTotal { - /// Component id - async fn component_id(&self) -> &str { - self.component_key.id() - } - - /// Errors processed metric - async fn metric(&self) -> ErrorsTotal { - ErrorsTotal::new(self.metric.clone()) - } -} diff --git a/src/api/schema/metrics/filter.rs b/src/api/schema/metrics/filter.rs deleted file mode 100644 index c85379edeeff4..0000000000000 --- a/src/api/schema/metrics/filter.rs +++ /dev/null @@ -1,397 +0,0 @@ -use std::collections::{BTreeMap, HashSet}; - -use async_stream::stream; -use tokio::time::Duration; -use tokio_stream::{Stream, StreamExt}; - -use super::{ - OutputThroughput, ReceivedBytesTotal, ReceivedEventsTotal, SentBytesTotal, SentEventsTotal, - filter_output_metric, -}; -use crate::{ - config::ComponentKey, - event::{Metric, MetricValue}, - metrics::Controller, -}; - -fn get_controller() -> &'static Controller { - Controller::get().expect("Metrics system not initialized. Please report.") -} - -/// Sums an iterable of `&Metric`, by folding metric values. Convenience function typically -/// used to get aggregate metrics. -pub fn sum_metrics<'a, I: IntoIterator>(metrics: I) -> Option { - let mut iter = metrics.into_iter(); - let m = iter.next()?; - - Some(iter.fold( - m.clone(), - |mut m1, m2| { - if m1.update(m2) { m1 } else { m2.clone() } - }, - )) -} - -/// Sums an iterable of `Metric`, by folding metric values. Convenience function typically -/// used to get aggregate metrics. -fn sum_metrics_owned>(metrics: I) -> Option { - let mut iter = metrics.into_iter(); - let m = iter.next()?; - - Some(iter.fold(m, |mut m1, m2| if m1.update(&m2) { m1 } else { m2 })) -} - -pub trait MetricsFilter<'a> { - fn received_bytes_total(&self) -> Option; - fn received_events_total(&self) -> Option; - fn sent_bytes_total(&self) -> Option; - fn sent_events_total(&self) -> Option; -} - -impl MetricsFilter<'_> for Vec { - fn received_bytes_total(&self) -> Option { - let sum = sum_metrics( - self.iter() - .filter(|m| m.name() == "component_received_bytes_total"), - )?; - - Some(ReceivedBytesTotal::new(sum)) - } - - fn received_events_total(&self) -> Option { - let sum = sum_metrics( - self.iter() - .filter(|m| m.name() == "component_received_events_total"), - )?; - - Some(ReceivedEventsTotal::new(sum)) - } - - fn sent_bytes_total(&self) -> Option { - let sum = sum_metrics( - self.iter() - .filter(|m| m.name() == "component_sent_bytes_total"), - )?; - - Some(SentBytesTotal::new(sum)) - } - - fn sent_events_total(&self) -> Option { - let sum = sum_metrics( - self.iter() - .filter(|m| m.name() == "component_sent_events_total"), - )?; - - Some(SentEventsTotal::new(sum)) - } -} - -impl<'a> MetricsFilter<'a> for Vec<&'a Metric> { - fn received_bytes_total(&self) -> Option { - let sum = sum_metrics( - self.iter() - .filter(|m| m.name() == "component_received_bytes_total") - .copied(), - )?; - - Some(ReceivedBytesTotal::new(sum)) - } - - fn received_events_total(&self) -> Option { - let sum = sum_metrics( - self.iter() - .filter(|m| m.name() == "component_received_events_total") - .copied(), - )?; - - Some(ReceivedEventsTotal::new(sum)) - } - - fn sent_bytes_total(&self) -> Option { - let sum = sum_metrics( - self.iter() - .filter(|m| m.name() == "component_sent_bytes_total") - .copied(), - )?; - - Some(SentBytesTotal::new(sum)) - } - - fn sent_events_total(&self) -> Option { - let sum = sum_metrics( - self.iter() - .filter(|m| m.name() == "component_sent_events_total") - .copied(), - )?; - - Some(SentEventsTotal::new(sum)) - } -} - -/// Returns a stream of `Metric`s, collected at the provided millisecond interval. -pub fn get_metrics(interval: i32) -> impl Stream { - let controller = get_controller(); - let mut interval = tokio::time::interval(Duration::from_millis(interval as u64)); - - stream! { - loop { - interval.tick().await; - for m in controller.capture_metrics() { - yield m; - } - } - } -} - -pub fn get_all_metrics(interval: i32) -> impl Stream> { - let controller = get_controller(); - let mut interval = tokio::time::interval(Duration::from_millis(interval as u64)); - - stream! { - loop { - interval.tick().await; - yield controller.capture_metrics() - } - } -} - -/// Return [`Vec`] based on a component id tag. -pub fn by_component_key(component_key: &ComponentKey) -> Vec { - get_controller() - .capture_metrics() - .into_iter() - .filter_map(|m| { - m.tag_matches("component_id", component_key.id()) - .then_some(m) - }) - .collect() -} - -type MetricFilterFn = dyn Fn(&Metric) -> bool + Send + Sync; - -/// Returns a stream of `Vec`, where `metric_name` matches the name of the metric -/// (e.g. "component_sent_events_total"), and the value is derived from `MetricValue::Counter`. Uses a -/// local cache to match against the `component_id` of a metric, to return results only when -/// the value of a current iteration is greater than the previous. This is useful for the client -/// to be notified as metrics increase without returning 'empty' or identical results. -pub fn component_counter_metrics( - interval: i32, - filter_fn: &'static MetricFilterFn, -) -> impl Stream> { - let mut cache = BTreeMap::new(); - - component_to_filtered_metrics(interval, filter_fn).map(move |map| { - map.into_iter() - .filter_map(|(id, metrics)| { - let m = sum_metrics_owned(metrics)?; - match m.value() { - MetricValue::Counter { value } - if cache.insert(id, *value).unwrap_or(0.00) < *value => - { - Some(m) - } - _ => None, - } - }) - .collect() - }) -} - -/// Returns a stream of `Vec`, where `metric_name` matches the name of the metric -/// (e.g. "component_sent_events_total"), and the value is derived from `MetricValue::Gauge`. Uses a -/// local cache to match against the `component_id` of a metric, to return results only when -/// the value of a current iteration is greater than the previous. This is useful for the client -/// to be notified as metrics increase without returning 'empty' or identical results. -pub fn component_gauge_metrics( - interval: i32, - filter_fn: &'static MetricFilterFn, -) -> impl Stream> { - let mut cache = BTreeMap::new(); - - component_to_filtered_metrics(interval, filter_fn).map(move |map| { - map.into_iter() - .filter_map(|(id, metrics)| { - let m = sum_metrics_owned(metrics)?; - match m.value() { - MetricValue::Gauge { value } - if cache.insert(id, *value).unwrap_or(0.00) < *value => - { - Some(m) - } - _ => None, - } - }) - .collect() - }) -} - -/// Returns the throughput of a 'counter' metric, sampled over `interval` milliseconds -/// and filtered by the provided `filter_fn`. -pub fn counter_throughput( - interval: i32, - filter_fn: &'static MetricFilterFn, -) -> impl Stream { - let mut last = 0.00; - - get_metrics(interval) - .filter(filter_fn) - .filter_map(move |m| match m.value() { - MetricValue::Counter { value } if *value > last => { - let throughput = value - last; - last = *value; - Some((m, throughput)) - } - _ => None, - }) - // Ignore the first, since we only care about sampling between `interval` - .skip(1) -} - -/// Returns the throughput of a 'counter' metric, sampled over `interval` milliseconds -/// and filtered by the provided `filter_fn`, aggregated against each component. -pub fn component_counter_throughputs( - interval: i32, - filter_fn: &'static MetricFilterFn, -) -> impl Stream> { - let mut cache = BTreeMap::new(); - - component_to_filtered_metrics(interval, filter_fn) - .map(move |map| { - map.into_iter() - .filter_map(|(id, metrics)| { - let m = sum_metrics_owned(metrics)?; - match m.value() { - MetricValue::Counter { value } => { - let last = cache.insert(id, *value).unwrap_or(0.00); - let throughput = value - last; - Some((m, throughput)) - } - _ => None, - } - }) - .collect() - }) - // Ignore the first, since we only care about sampling between `interval` - .skip(1) -} - -/// Returns a stream of `Vec<(Metric, Vec)>`, where `Metric` is the -/// total `component_sent_events_total` metric for a component and `Vec` -/// is the `component_sent_events_total` metric split by output -pub fn component_sent_events_totals_metrics_with_outputs( - interval: i32, -) -> impl Stream)>> { - let mut cache = BTreeMap::new(); - - component_to_filtered_metrics(interval, &|m| m.name() == "component_sent_events_total").map( - move |map| { - map.into_iter() - .filter_map(|(id, metrics)| { - let outputs = metrics - .iter() - .filter_map(|m| m.tag_value("output")) - .collect::>(); - - let metric_by_outputs = outputs - .iter() - .filter_map(|output| { - let m = filter_output_metric(metrics.as_ref(), output.as_ref())?; - match m.value() { - MetricValue::Counter { value } - if cache - .insert(format!("{id}.{output}"), *value) - .unwrap_or(0.00) - < *value => - { - Some(m) - } - _ => None, - } - }) - .collect(); - - let sum = sum_metrics_owned(metrics)?; - match sum.value() { - MetricValue::Counter { value } - if cache.insert(id, *value).unwrap_or(0.00) < *value => - { - Some((sum, metric_by_outputs)) - } - _ => None, - } - }) - .collect() - }, - ) -} - -/// Returns the throughput of the 'component_sent_events_total' metric, sampled over `interval` milliseconds, -/// for each component. Within a particular component, throughput per output stream is also included. -pub fn component_sent_events_total_throughputs_with_outputs( - interval: i32, -) -> impl Stream)>> { - let mut cache = BTreeMap::new(); - - component_to_filtered_metrics(interval, &|m| m.name() == "component_sent_events_total") - .map(move |map| { - map.into_iter() - .filter_map(|(id, metrics)| { - let outputs = metrics - .iter() - .filter_map(|m| m.tag_value("output")) - .collect::>(); - - let throughput_by_outputs = outputs - .iter() - .filter_map(|output| { - let m = filter_output_metric(metrics.as_ref(), output.as_ref())?; - let throughput = throughput(&m, format!("{id}.{output}"), &mut cache)?; - Some(OutputThroughput::new(output.clone(), throughput as i64)) - }) - .collect::>(); - - let sum = sum_metrics_owned(metrics)?; - let total_throughput = throughput(&sum, id.clone(), &mut cache)?; - Some(( - ComponentKey::from(id), - total_throughput as i64, - throughput_by_outputs, - )) - }) - .collect() - }) - // Ignore the first, since we only care about sampling between `interval` - .skip(1) -} - -/// Returns a map of Component ID to list of metrics where metrics have been -/// filtered by `filter_fn` -fn component_to_filtered_metrics( - interval: i32, - filter_fn: &'static MetricFilterFn, -) -> impl Stream>> { - get_all_metrics(interval).map(move |m| { - m.into_iter() - .filter(filter_fn) - .filter_map(|m| m.tag_value("component_id").map(|id| (id, m))) - .fold( - BTreeMap::new(), - |mut map: BTreeMap>, (id, m)| { - map.entry(id).or_default().push(m); - map - }, - ) - }) -} - -/// Returns throughput based on a metric and provided `cache` of previous values -fn throughput(metric: &Metric, id: String, cache: &mut BTreeMap) -> Option { - match metric.value() { - MetricValue::Counter { value } => { - let last = cache.insert(id, *value).unwrap_or(0.00); - let throughput = value - last; - Some(throughput) - } - _ => None, - } -} diff --git a/src/api/schema/metrics/host.rs b/src/api/schema/metrics/host.rs deleted file mode 100644 index 08a8fbcb973be..0000000000000 --- a/src/api/schema/metrics/host.rs +++ /dev/null @@ -1,373 +0,0 @@ -use async_graphql::Object; -use cfg_if::cfg_if; - -use crate::{ - event::{Metric, MetricValue}, - sources::host_metrics, -}; - -pub struct MemoryMetrics(Vec); - -#[Object] -/// Host memory metrics -impl MemoryMetrics { - /// Total bytes - async fn total_bytes(&self) -> f64 { - filter_host_metric(&self.0, "memory_total_bytes") - } - - /// Free bytes - async fn free_bytes(&self) -> f64 { - filter_host_metric(&self.0, "memory_free_bytes") - } - - /// Available bytes - async fn available_bytes(&self) -> f64 { - filter_host_metric(&self.0, "memory_available_bytes") - } - - /// Active bytes (Linux/macOS only) - async fn active_bytes(&self) -> Option { - if cfg!(any(target_os = "linux", target_os = "macos")) { - Some(filter_host_metric(&self.0, "memory_active_bytes")) - } else { - None - } - } - - /// Buffers bytes (Linux only) - async fn buffers_bytes(&self) -> Option { - if cfg!(target_os = "linux") { - Some(filter_host_metric(&self.0, "memory_buffers_bytes")) - } else { - None - } - } - - /// Cached bytes (Linux only) - async fn cached_bytes(&self) -> Option { - if cfg!(target_os = "linux") { - Some(filter_host_metric(&self.0, "memory_cached_bytes")) - } else { - None - } - } - - /// Shared bytes (Linux only) - async fn shared_bytes(&self) -> Option { - if cfg!(target_os = "linux") { - Some(filter_host_metric(&self.0, "memory_shared_bytes")) - } else { - None - } - } - - /// Used bytes (Linux only) - async fn used_bytes(&self) -> Option { - if cfg!(target_os = "linux") { - Some(filter_host_metric(&self.0, "memory_used_bytes")) - } else { - None - } - } - - /// Inactive bytes (macOS only) - async fn inactive_bytes(&self) -> Option { - if cfg!(target_os = "macos") { - Some(filter_host_metric(&self.0, "memory_inactive_bytes")) - } else { - None - } - } - - /// Wired bytes (macOS only) - async fn wired_bytes(&self) -> Option { - if cfg!(target_os = "macos") { - Some(filter_host_metric(&self.0, "memory_wired_bytes")) - } else { - None - } - } -} - -pub struct SwapMetrics(Vec); - -#[Object] -impl SwapMetrics { - /// Swap free bytes - async fn free_bytes(&self) -> f64 { - filter_host_metric(&self.0, "memory_swap_free_bytes") - } - - /// Swap total bytes - async fn total_bytes(&self) -> f64 { - filter_host_metric(&self.0, "memory_swap_total_bytes") - } - - /// Swap used bytes - async fn used_bytes(&self) -> f64 { - filter_host_metric(&self.0, "memory_swap_used_bytes") - } - - /// Swapped in bytes total (not available on Windows) - async fn swapped_in_bytes_total(&self) -> Option { - if cfg!(not(windows)) { - Some(filter_host_metric(&self.0, "memory_swapped_in_bytes_total")) - } else { - None - } - } - - /// Swapped out bytes total (not available on Windows) - async fn swapped_out_bytes_total(&self) -> Option { - if cfg!(not(windows)) { - Some(filter_host_metric( - &self.0, - "memory_swapped_out_bytes_total", - )) - } else { - None - } - } -} - -pub struct CpuMetrics(Vec); - -#[Object] -impl CpuMetrics { - /// CPU seconds total - async fn cpu_seconds_total(&self) -> f64 { - filter_host_metric(&self.0, "cpu_seconds_total") - } -} - -pub struct LoadAverageMetrics(Vec); - -#[Object] -impl LoadAverageMetrics { - /// Load 1 average - async fn load1(&self) -> f64 { - filter_host_metric(&self.0, "load1") - } - - /// Load 5 average - async fn load5(&self) -> f64 { - filter_host_metric(&self.0, "load5") - } - - /// Load 15 average - async fn load15(&self) -> f64 { - filter_host_metric(&self.0, "load15") - } -} - -pub struct NetworkMetrics(Vec); - -#[Object] -impl NetworkMetrics { - /// Total bytes received - async fn receive_bytes_total(&self) -> f64 { - filter_host_metric(&self.0, "network_receive_bytes_total") - } - - /// Total errors received - async fn receive_errs_total(&self) -> f64 { - filter_host_metric(&self.0, "network_receive_errs_total") - } - - /// Total packets received - async fn receive_packets_total(&self) -> f64 { - filter_host_metric(&self.0, "network_receive_packets_total") - } - - /// Total bytes transmitted - async fn transmit_bytes_total(&self) -> f64 { - filter_host_metric(&self.0, "network_transmit_bytes_total") - } - - /// Total errors transmitted - async fn transmit_errs_total(&self) -> f64 { - filter_host_metric(&self.0, "network_transmit_errs_total") - } - - /// Total transmission packets dropped (Linux/Windows only) - async fn transmit_packets_drop_total(&self) -> Option { - if cfg!(any(target_os = "linux", windows)) { - Some(filter_host_metric( - &self.0, - "network_transmit_packets_drop_total", - )) - } else { - None - } - } - - /// Total transmission packets (Linux/Windows only) - async fn transmit_packets_total(&self) -> Option { - if cfg!(any(target_os = "linux", windows)) { - Some(filter_host_metric( - &self.0, - "network_transmit_packets_total", - )) - } else { - None - } - } -} - -pub struct FileSystemMetrics(Vec); - -#[Object] -impl FileSystemMetrics { - /// Free bytes - async fn free_bytes(&self) -> f64 { - filter_host_metric(&self.0, "filesystem_free_bytes") - } - - /// Total bytes - async fn total_bytes(&self) -> f64 { - filter_host_metric(&self.0, "filesystem_total_bytes") - } - - /// Used bytes - async fn used_bytes(&self) -> f64 { - filter_host_metric(&self.0, "filesystem_used_bytes") - } -} - -pub struct DiskMetrics(Vec); - -#[Object] -impl DiskMetrics { - /// Total bytes read - async fn read_bytes_total(&self) -> f64 { - filter_host_metric(&self.0, "disk_read_bytes_total") - } - - /// Total reads completed - async fn reads_completed_total(&self) -> f64 { - filter_host_metric(&self.0, "disk_reads_completed_total") - } - - /// Total bytes written - async fn written_bytes_total(&self) -> f64 { - filter_host_metric(&self.0, "disk_written_bytes_total") - } - - /// Total writes completed - async fn writes_completed_total(&self) -> f64 { - filter_host_metric(&self.0, "disk_writes_completed_total") - } -} - -cfg_if! { - if #[cfg(target_os = "linux")] { - pub struct TCPMetrics(Vec); - - #[Object] - impl TCPMetrics { - /// Total TCP connections - async fn tcp_conns_total(&self) -> f64 { - filter_host_metric(&self.0, "tcp_connections_total") - } - - /// Total bytes in the send queue across all connections. - async fn tcp_tx_queued_bytes_total(&self) -> f64 { - filter_host_metric(&self.0, "tcp_tx_queued_bytes_total") - } - - /// Total bytes in the receive queue across all connections. - async fn tcp_rx_queued_bytes_total(&self) -> f64 { - filter_host_metric(&self.0, "tcp_rx_queued_bytes_total") - } - } - } -} - -pub struct HostMetrics(host_metrics::HostMetrics); - -impl HostMetrics { - /// Primes the host metrics pump by passing through a new `HostMetrics` - pub fn new() -> Self { - let config = host_metrics::HostMetricsConfig::default(); - Self(host_metrics::HostMetrics::new(config)) - } -} - -#[Object] -/// Vector host metrics -impl HostMetrics { - /// Memory metrics - async fn memory(&self) -> MemoryMetrics { - let mut buffer = self.0.buffer(); - self.0.memory_metrics(&mut buffer).await; - MemoryMetrics(buffer.metrics) - } - - /// Swap metrics - async fn swap(&self) -> SwapMetrics { - let mut buffer = self.0.buffer(); - self.0.swap_metrics(&mut buffer).await; - SwapMetrics(buffer.metrics) - } - - /// CPU metrics - async fn cpu(&self) -> CpuMetrics { - let mut buffer = self.0.buffer(); - self.0.cpu_metrics(&mut buffer).await; - CpuMetrics(buffer.metrics) - } - - /// Load average metrics (*nix only) - async fn load_average(&self) -> Option { - if cfg!(unix) { - let mut buffer = self.0.buffer(); - self.0.loadavg_metrics(&mut buffer).await; - Some(LoadAverageMetrics(buffer.metrics)) - } else { - None - } - } - - /// Network metrics - async fn network(&self) -> NetworkMetrics { - let mut buffer = self.0.buffer(); - self.0.network_metrics(&mut buffer).await; - NetworkMetrics(buffer.metrics) - } - - /// Filesystem metrics - async fn filesystem(&self) -> FileSystemMetrics { - let mut buffer = self.0.buffer(); - self.0.filesystem_metrics(&mut buffer).await; - FileSystemMetrics(buffer.metrics) - } - - /// Disk metrics - async fn disk(&self) -> DiskMetrics { - let mut buffer = self.0.buffer(); - self.0.disk_metrics(&mut buffer).await; - DiskMetrics(buffer.metrics) - } - - #[cfg(target_os = "linux")] - /// TCP metrics - async fn tcp(&self) -> TCPMetrics { - let mut buffer = self.0.buffer(); - self.0.tcp_metrics(&mut buffer).await; - TCPMetrics(buffer.metrics) - } -} - -/// Filters a [`Vec`] by name, returning the inner `value` or 0.00 if not found -fn filter_host_metric(metrics: &[Metric], name: &str) -> f64 { - metrics - .iter() - .find(|m| matches!(m.namespace(), Some(n) if n == "host") && m.name() == name) - .map(|m| match m.value() { - MetricValue::Gauge { value } => *value, - MetricValue::Counter { value } => *value, - _ => 0.00, - }) - .unwrap_or_else(|| 0.00) -} diff --git a/src/api/schema/metrics/mod.rs b/src/api/schema/metrics/mod.rs deleted file mode 100644 index 056b3c27a7986..0000000000000 --- a/src/api/schema/metrics/mod.rs +++ /dev/null @@ -1,284 +0,0 @@ -mod allocated_bytes; -mod errors; -pub mod filter; -mod output; -mod received_bytes; -mod received_events; -mod sent_bytes; -mod sent_events; -mod sink; -pub mod source; -mod transform; -mod uptime; - -#[cfg(feature = "sources-host_metrics")] -mod host; - -pub use allocated_bytes::{AllocatedBytes, ComponentAllocatedBytes}; -use async_graphql::{Interface, Subscription}; -use chrono::{DateTime, Utc}; -pub use errors::{ComponentErrorsTotal, ErrorsTotal}; -pub use filter::*; -pub use output::*; -pub use received_bytes::{ - ComponentReceivedBytesThroughput, ComponentReceivedBytesTotal, ReceivedBytesTotal, -}; -pub use received_events::{ - ComponentReceivedEventsThroughput, ComponentReceivedEventsTotal, ReceivedEventsTotal, -}; -pub use sent_bytes::{ComponentSentBytesThroughput, ComponentSentBytesTotal, SentBytesTotal}; -pub use sent_events::{ComponentSentEventsThroughput, ComponentSentEventsTotal, SentEventsTotal}; -pub use sink::{IntoSinkMetrics, SinkMetrics}; -pub use source::{IntoSourceMetrics, SourceMetrics}; -use tokio_stream::{Stream, StreamExt}; -pub use transform::{IntoTransformMetrics, TransformMetrics}; -pub use uptime::Uptime; - -use crate::config::ComponentKey; - -#[derive(Interface)] -#[graphql(field(name = "timestamp", ty = "Option>"))] -pub enum MetricType { - Uptime(Uptime), -} - -#[cfg(feature = "sources-host_metrics")] -#[derive(Default)] -pub struct MetricsQuery; - -#[cfg(feature = "sources-host_metrics")] -#[async_graphql::Object] -impl MetricsQuery { - /// Vector host metrics - async fn host_metrics(&self) -> host::HostMetrics { - host::HostMetrics::new() - } -} - -#[derive(Default)] -pub struct MetricsSubscription; - -#[Subscription] -impl MetricsSubscription { - /// Metrics for how long the Vector instance has been running - async fn uptime( - &self, - #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream + use<> { - get_metrics(interval).filter_map(|m| match m.name() { - "uptime_seconds" => Some(Uptime::new(m)), - _ => None, - }) - } - - /// Total received events metrics - #[graphql(deprecation = "Use component_received_events_totals instead")] - async fn received_events_total( - &self, - #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream + use<> { - get_metrics(interval).filter_map(|m| match m.name() { - "component_received_events_total" => Some(ReceivedEventsTotal::new(m)), - _ => None, - }) - } - - /// Total received events throughput sampled over the provided millisecond `interval` - #[graphql(deprecation = "Use component_received_events_throughputs instead")] - async fn received_events_throughput( - &self, - #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream + use<> { - counter_throughput(interval, &|m| m.name() == "component_received_events_total") - .map(|(_, throughput)| throughput as i64) - } - - /// Total incoming component events throughput metrics over `interval` - async fn component_received_events_throughputs( - &self, - #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream> + use<> { - component_counter_throughputs(interval, &|m| m.name() == "component_received_events_total") - .map(|m| { - m.into_iter() - .map(|(m, throughput)| { - ComponentReceivedEventsThroughput::new( - ComponentKey::from(m.tag_value("component_id").unwrap()), - throughput as i64, - ) - }) - .collect() - }) - } - - /// Total received component event metrics over `interval` - async fn component_received_events_totals( - &self, - #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream> + use<> { - component_counter_metrics(interval, &|m| m.name() == "component_received_events_total").map( - |m| { - m.into_iter() - .map(ComponentReceivedEventsTotal::new) - .collect() - }, - ) - } - - /// Total sent events metrics - #[graphql(deprecation = "Use component_sent_events_totals instead")] - async fn sent_events_total( - &self, - #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream + use<> { - get_metrics(interval).filter_map(|m| match m.name() { - "component_sent_events_total" => Some(SentEventsTotal::new(m)), - _ => None, - }) - } - - /// Total outgoing events throughput sampled over the provided millisecond `interval` - #[graphql(deprecation = "Use component_sent_events_throughputs instead")] - async fn sent_events_throughput( - &self, - #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream + use<> { - counter_throughput(interval, &|m| m.name() == "component_sent_events_total") - .map(|(_, throughput)| throughput as i64) - } - - /// Total outgoing component event throughput metrics over `interval` - async fn component_sent_events_throughputs( - &self, - #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream> + use<> { - component_sent_events_total_throughputs_with_outputs(interval).map(|m| { - m.into_iter() - .map(|(key, total_throughput, outputs)| { - ComponentSentEventsThroughput::new(key, total_throughput, outputs) - }) - .collect() - }) - } - - /// Total outgoing component event metrics over `interval` - async fn component_sent_events_totals( - &self, - #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream> + use<> { - component_sent_events_totals_metrics_with_outputs(interval).map(|ms| { - ms.into_iter() - .map(|(m, m_by_outputs)| ComponentSentEventsTotal::new(m, m_by_outputs)) - .collect() - }) - } - - /// Component bytes received metrics over `interval`. - async fn component_received_bytes_totals( - &self, - #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream> + use<> { - component_counter_metrics(interval, &|m| m.name() == "component_received_bytes_total").map( - |m| { - m.into_iter() - .map(ComponentReceivedBytesTotal::new) - .collect() - }, - ) - } - - /// Component bytes received throughput over `interval` - async fn component_received_bytes_throughputs( - &self, - #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream> + use<> { - component_counter_throughputs(interval, &|m| m.name() == "component_received_bytes_total") - .map(|m| { - m.into_iter() - .map(|(m, throughput)| { - ComponentReceivedBytesThroughput::new( - ComponentKey::from(m.tag_value("component_id").unwrap()), - throughput as i64, - ) - }) - .collect() - }) - } - - /// Component bytes sent metrics over `interval`. - async fn component_sent_bytes_totals( - &self, - #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream> + use<> { - component_counter_metrics(interval, &|m| m.name() == "component_sent_bytes_total") - .map(|m| m.into_iter().map(ComponentSentBytesTotal::new).collect()) - } - - /// Component bytes sent throughput over `interval` - async fn component_sent_bytes_throughputs( - &self, - #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream> + use<> { - component_counter_throughputs(interval, &|m| m.name() == "component_sent_bytes_total").map( - |m| { - m.into_iter() - .map(|(m, throughput)| { - ComponentSentBytesThroughput::new( - ComponentKey::from(m.tag_value("component_id").unwrap()), - throughput as i64, - ) - }) - .collect() - }, - ) - } - - /// Total error metrics. - async fn errors_total( - &self, - #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream + use<> { - get_metrics(interval) - .filter(|m| m.name().ends_with("_errors_total")) - .map(ErrorsTotal::new) - } - - /// Allocated bytes metrics. - async fn allocated_bytes( - &self, - #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream + use<> { - get_metrics(interval) - .filter(|m| m.name() == "component_allocated_bytes") - .map(AllocatedBytes::new) - } - - /// Component allocation metrics - async fn component_allocated_bytes( - &self, - #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream> + use<> { - component_gauge_metrics(interval, &|m| m.name() == "component_allocated_bytes") - .map(|m| m.into_iter().map(ComponentAllocatedBytes::new).collect()) - } - - /// Component error metrics over `interval`. - async fn component_errors_totals( - &self, - #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream> + use<> { - component_counter_metrics(interval, &|m| m.name().ends_with("_errors_total")) - .map(|m| m.into_iter().map(ComponentErrorsTotal::new).collect()) - } - - /// All metrics. - async fn metrics( - &self, - #[graphql(default = 1000, validator(minimum = 10, maximum = 60_000))] interval: i32, - ) -> impl Stream + use<> { - get_metrics(interval).filter_map(|m| match m.name() { - "uptime_seconds" => Some(MetricType::Uptime(m.into())), - _ => None, - }) - } -} diff --git a/src/api/schema/metrics/output.rs b/src/api/schema/metrics/output.rs deleted file mode 100644 index e1fd0976e7b48..0000000000000 --- a/src/api/schema/metrics/output.rs +++ /dev/null @@ -1,88 +0,0 @@ -use async_graphql::Object; -use vector_lib::config::ComponentKey; - -use super::{SentEventsTotal, by_component_key, sum_metrics}; -use crate::event::Metric; - -#[derive(Debug, Clone)] -pub struct Output { - output_id: String, - sent_events_total: Option, -} - -impl Output { - pub const fn new(output_id: String, sent_events_total: Option) -> Self { - Self { - output_id, - sent_events_total, - } - } -} - -#[Object] -impl Output { - /// Id of the output stream - pub async fn output_id(&self) -> &str { - self.output_id.as_ref() - } - - /// Total sent events for the current output stream - pub async fn sent_events_total(&self) -> Option { - self.sent_events_total - .as_ref() - .map(|metric| SentEventsTotal::new(metric.clone())) - } -} - -#[derive(Debug, Clone)] -pub struct OutputThroughput { - output_id: String, - throughput: i64, -} - -impl OutputThroughput { - pub const fn new(output_id: String, throughput: i64) -> Self { - Self { - output_id, - throughput, - } - } -} - -#[Object] -impl OutputThroughput { - /// Id of the output stream - pub async fn output_id(&self) -> &str { - self.output_id.as_ref() - } - - /// Throughput for the output stream - pub async fn throughput(&self) -> i64 { - self.throughput - } -} - -pub fn outputs_by_component_key(component_key: &ComponentKey, outputs: &[String]) -> Vec { - let metrics = by_component_key(component_key) - .into_iter() - .filter(|m| m.name() == "component_sent_events_total") - .collect::>(); - - outputs - .iter() - .map(|output| { - Output::new( - output.clone(), - filter_output_metric(&metrics, output.as_ref()), - ) - }) - .collect::>() -} - -pub fn filter_output_metric(metrics: &[Metric], output_name: &str) -> Option { - sum_metrics( - metrics - .iter() - .filter(|m| m.tag_matches("output", output_name)), - ) -} diff --git a/src/api/schema/metrics/received_bytes.rs b/src/api/schema/metrics/received_bytes.rs deleted file mode 100644 index 516631559cbae..0000000000000 --- a/src/api/schema/metrics/received_bytes.rs +++ /dev/null @@ -1,108 +0,0 @@ -use async_graphql::Object; -use chrono::{DateTime, Utc}; - -use crate::{ - config::ComponentKey, - event::{Metric, MetricValue}, -}; - -pub struct ReceivedBytesTotal(Metric); - -impl ReceivedBytesTotal { - pub const fn new(m: Metric) -> Self { - Self(m) - } - - pub fn get_timestamp(&self) -> Option> { - self.0.timestamp() - } - - pub fn get_received_bytes_total(&self) -> f64 { - match self.0.value() { - MetricValue::Counter { value } => *value, - _ => 0.00, - } - } -} - -#[Object] -impl ReceivedBytesTotal { - /// Metric timestamp. - pub async fn timestamp(&self) -> Option> { - self.get_timestamp() - } - - /// Total number of bytes received. - pub async fn received_bytes_total(&self) -> f64 { - self.get_received_bytes_total() - } -} - -impl From for ReceivedBytesTotal { - fn from(m: Metric) -> Self { - Self(m) - } -} - -pub struct ComponentReceivedBytesTotal { - component_key: ComponentKey, - metric: Metric, -} - -impl ComponentReceivedBytesTotal { - /// Returns a new `ComponentReceivedBytesTotal`. - /// - /// Expects that the metric contains a tag for the component ID the metric is referenced to. - pub fn new(metric: Metric) -> Self { - let component_key = metric.tag_value("component_id").expect( - "Returned a metric without a `component_id`, which shouldn't happen. Please report.", - ); - let component_key = ComponentKey::from(component_key); - - Self { - component_key, - metric, - } - } -} - -#[Object] -impl ComponentReceivedBytesTotal { - /// Component ID. - async fn component_id(&self) -> &str { - self.component_key.id() - } - - /// Metric for total bytes received. - async fn metric(&self) -> ReceivedBytesTotal { - ReceivedBytesTotal::new(self.metric.clone()) - } -} - -pub struct ComponentReceivedBytesThroughput { - component_key: ComponentKey, - throughput: i64, -} - -impl ComponentReceivedBytesThroughput { - /// Returns a new `ComponentReceivedBytesThroughput` for the given component. - pub const fn new(component_key: ComponentKey, throughput: i64) -> Self { - Self { - component_key, - throughput, - } - } -} - -#[Object] -impl ComponentReceivedBytesThroughput { - /// Component ID. - async fn component_id(&self) -> &str { - self.component_key.id() - } - - /// Throughput of bytes sent. - async fn throughput(&self) -> i64 { - self.throughput - } -} diff --git a/src/api/schema/metrics/received_events.rs b/src/api/schema/metrics/received_events.rs deleted file mode 100644 index b313004a8ef72..0000000000000 --- a/src/api/schema/metrics/received_events.rs +++ /dev/null @@ -1,107 +0,0 @@ -use async_graphql::Object; -use chrono::{DateTime, Utc}; - -use crate::{ - config::ComponentKey, - event::{Metric, MetricValue}, -}; - -pub struct ReceivedEventsTotal(Metric); - -impl ReceivedEventsTotal { - pub const fn new(m: Metric) -> Self { - Self(m) - } - - pub fn get_timestamp(&self) -> Option> { - self.0.timestamp() - } - - pub fn get_received_events_total(&self) -> f64 { - match self.0.value() { - MetricValue::Counter { value } => *value, - _ => 0.00, - } - } -} - -#[Object] -impl ReceivedEventsTotal { - /// Metric timestamp - pub async fn timestamp(&self) -> Option> { - self.get_timestamp() - } - - /// Total incoming events - pub async fn received_events_total(&self) -> f64 { - self.get_received_events_total() - } -} - -impl From for ReceivedEventsTotal { - fn from(m: Metric) -> Self { - Self(m) - } -} - -pub struct ComponentReceivedEventsTotal { - component_key: ComponentKey, - metric: Metric, -} - -impl ComponentReceivedEventsTotal { - /// Returns a new `ComponentReceivedEventsTotal` struct, which is a GraphQL type. The - /// component id is hoisted for clear field resolution in the resulting payload. - pub fn new(metric: Metric) -> Self { - let component_key = metric.tag_value("component_id").expect( - "Returned a metric without a `component_id`, which shouldn't happen. Please report.", - ); - let component_key = ComponentKey::from(component_key); - - Self { - component_key, - metric, - } - } -} - -#[Object] -impl ComponentReceivedEventsTotal { - /// Component id - async fn component_id(&self) -> &str { - self.component_key.id() - } - - /// Total received events metric - async fn metric(&self) -> ReceivedEventsTotal { - ReceivedEventsTotal::new(self.metric.clone()) - } -} - -pub struct ComponentReceivedEventsThroughput { - component_key: ComponentKey, - throughput: i64, -} - -impl ComponentReceivedEventsThroughput { - /// Returns a new `ComponentReceivedEventsThroughput`, set to the provided id/throughput values. - pub const fn new(component_key: ComponentKey, throughput: i64) -> Self { - Self { - component_key, - throughput, - } - } -} - -#[Object] -impl ComponentReceivedEventsThroughput { - /// Component id - async fn component_id(&self) -> &str { - self.component_key.id() - } - - /// Received events throughput - async fn throughput(&self) -> i64 { - self.throughput - } -} diff --git a/src/api/schema/metrics/sent_bytes.rs b/src/api/schema/metrics/sent_bytes.rs deleted file mode 100644 index 02fb833a39adc..0000000000000 --- a/src/api/schema/metrics/sent_bytes.rs +++ /dev/null @@ -1,108 +0,0 @@ -use async_graphql::Object; -use chrono::{DateTime, Utc}; - -use crate::{ - config::ComponentKey, - event::{Metric, MetricValue}, -}; - -pub struct SentBytesTotal(Metric); - -impl SentBytesTotal { - pub const fn new(m: Metric) -> Self { - Self(m) - } - - pub fn get_timestamp(&self) -> Option> { - self.0.timestamp() - } - - pub fn get_sent_bytes_total(&self) -> f64 { - match self.0.value() { - MetricValue::Counter { value } => *value, - _ => 0.00, - } - } -} - -#[Object] -impl SentBytesTotal { - /// Metric timestamp. - pub async fn timestamp(&self) -> Option> { - self.get_timestamp() - } - - /// Total number of bytes sent. - pub async fn sent_bytes_total(&self) -> f64 { - self.get_sent_bytes_total() - } -} - -impl From for SentBytesTotal { - fn from(m: Metric) -> Self { - Self(m) - } -} - -pub struct ComponentSentBytesTotal { - component_key: ComponentKey, - metric: Metric, -} - -impl ComponentSentBytesTotal { - /// Returns a new `ComponentSentBytesTotal` for the given metric. - /// - /// Expects that the metric contains a tag for the component ID the metric is referenced to. - pub fn new(metric: Metric) -> Self { - let component_key = metric.tag_value("component_id").expect( - "Returned a metric without a `component_id`, which shouldn't happen. Please report.", - ); - let component_key = ComponentKey::from(component_key); - - Self { - component_key, - metric, - } - } -} - -#[Object] -impl ComponentSentBytesTotal { - /// Component ID. - async fn component_id(&self) -> &str { - self.component_key.id() - } - - /// Metric for total bytes sent. - async fn metric(&self) -> SentBytesTotal { - SentBytesTotal::new(self.metric.clone()) - } -} - -pub struct ComponentSentBytesThroughput { - component_key: ComponentKey, - throughput: i64, -} - -impl ComponentSentBytesThroughput { - /// Returns a new `ComponentSentBytesThroughput` for the given component. - pub const fn new(component_key: ComponentKey, throughput: i64) -> Self { - Self { - component_key, - throughput, - } - } -} - -#[Object] -impl ComponentSentBytesThroughput { - /// Component ID. - async fn component_id(&self) -> &str { - self.component_key.id() - } - - /// Throughput of bytes sent. - async fn throughput(&self) -> i64 { - self.throughput - } -} diff --git a/src/api/schema/metrics/sent_events.rs b/src/api/schema/metrics/sent_events.rs deleted file mode 100644 index 5085cebc05334..0000000000000 --- a/src/api/schema/metrics/sent_events.rs +++ /dev/null @@ -1,133 +0,0 @@ -use async_graphql::Object; -use chrono::{DateTime, Utc}; - -use super::{Output, OutputThroughput}; -use crate::{ - config::ComponentKey, - event::{Metric, MetricValue}, -}; - -pub struct SentEventsTotal(Metric); - -impl SentEventsTotal { - pub const fn new(m: Metric) -> Self { - Self(m) - } - - pub fn get_timestamp(&self) -> Option> { - self.0.timestamp() - } - - pub fn get_sent_events_total(&self) -> f64 { - match self.0.value() { - MetricValue::Counter { value } => *value, - _ => 0.00, - } - } -} - -#[Object] -impl SentEventsTotal { - /// Metric timestamp - pub async fn timestamp(&self) -> Option> { - self.get_timestamp() - } - - /// Total sent events - pub async fn sent_events_total(&self) -> f64 { - self.get_sent_events_total() - } -} - -impl From for SentEventsTotal { - fn from(m: Metric) -> Self { - Self(m) - } -} - -pub struct ComponentSentEventsTotal { - component_key: ComponentKey, - outputs: Vec, - metric: Metric, -} - -impl ComponentSentEventsTotal { - /// Returns a new `ComponentSentEventsTotal` struct, which is a GraphQL type. The - /// component id is hoisted for clear field resolution in the resulting payload. - pub fn new(metric: Metric, metric_by_outputs: Vec) -> Self { - let component_key = metric.tag_value("component_id").expect( - "Returned a metric without a `component_id`, which shouldn't happen. Please report.", - ); - let component_key = ComponentKey::from(component_key); - let outputs = metric_by_outputs - .iter() - .filter_map(|m| { - m.tag_value("output") - .map(|output_name| Output::new(output_name, Some(m.clone()))) - }) - .collect::>(); - - Self { - component_key, - outputs, - metric, - } - } -} - -#[Object] -impl ComponentSentEventsTotal { - /// Component id - async fn component_id(&self) -> &str { - self.component_key.id() - } - - /// Total outgoing events metric - async fn metric(&self) -> SentEventsTotal { - SentEventsTotal::new(self.metric.clone()) - } - - /// Output streams with outgoing events metrics - async fn outputs(&self) -> &Vec { - &self.outputs - } -} - -pub struct ComponentSentEventsThroughput { - component_key: ComponentKey, - throughput: i64, - outputs: Vec, -} - -impl ComponentSentEventsThroughput { - /// Returns a new `ComponentSentEventsThroughput`, set to the provided id/throughput values - pub const fn new( - component_key: ComponentKey, - throughput: i64, - outputs: Vec, - ) -> Self { - Self { - component_key, - throughput, - outputs, - } - } -} - -#[Object] -impl ComponentSentEventsThroughput { - /// Component id - async fn component_id(&self) -> &str { - self.component_key.id() - } - - /// Total events processed throughput - async fn throughput(&self) -> i64 { - self.throughput - } - - /// Output streams with throughputs - async fn outputs(&self) -> &Vec { - &self.outputs - } -} diff --git a/src/api/schema/metrics/sink/generic.rs b/src/api/schema/metrics/sink/generic.rs deleted file mode 100644 index 83c50bc6557b7..0000000000000 --- a/src/api/schema/metrics/sink/generic.rs +++ /dev/null @@ -1,33 +0,0 @@ -use async_graphql::Object; - -use crate::{ - api::schema::metrics::{self, MetricsFilter}, - event::Metric, -}; - -#[derive(Debug, Clone)] -pub struct GenericSinkMetrics(Vec); - -impl GenericSinkMetrics { - pub const fn new(metrics: Vec) -> Self { - Self(metrics) - } -} - -#[Object] -impl GenericSinkMetrics { - /// Total received events for the current sink - pub async fn received_events_total(&self) -> Option { - self.0.received_events_total() - } - - /// Total sent bytes for the current sink - pub async fn sent_bytes_total(&self) -> Option { - self.0.sent_bytes_total() - } - - /// Total sent events for the current sink - pub async fn sent_events_total(&self) -> Option { - self.0.sent_events_total() - } -} diff --git a/src/api/schema/metrics/sink/mod.rs b/src/api/schema/metrics/sink/mod.rs deleted file mode 100644 index c864270834cec..0000000000000 --- a/src/api/schema/metrics/sink/mod.rs +++ /dev/null @@ -1,26 +0,0 @@ -mod generic; - -use async_graphql::Interface; - -use super::{ReceivedEventsTotal, SentBytesTotal, SentEventsTotal}; -use crate::event::Metric; - -#[derive(Debug, Clone, Interface)] -#[graphql( - field(name = "received_events_total", ty = "Option"), - field(name = "sent_bytes_total", ty = "Option"), - field(name = "sent_events_total", ty = "Option") -)] -pub enum SinkMetrics { - GenericSinkMetrics(generic::GenericSinkMetrics), -} - -pub trait IntoSinkMetrics { - fn into_sink_metrics(self, component_type: &str) -> SinkMetrics; -} - -impl IntoSinkMetrics for Vec { - fn into_sink_metrics(self, _component_type: &str) -> SinkMetrics { - SinkMetrics::GenericSinkMetrics(generic::GenericSinkMetrics::new(self)) - } -} diff --git a/src/api/schema/metrics/source/file.rs b/src/api/schema/metrics/source/file.rs deleted file mode 100644 index 43c2a2751ceda..0000000000000 --- a/src/api/schema/metrics/source/file.rs +++ /dev/null @@ -1,351 +0,0 @@ -use std::{cmp::Ordering, collections::BTreeMap}; - -use async_graphql::{Enum, InputObject, Object}; - -use crate::{ - api::schema::{ - filter::{CustomFilter, StringFilter, filter_items}, - metrics::{self, MetricsFilter}, - relay, sort, - }, - event::Metric, - filter_check, -}; - -#[derive(Clone)] -pub struct FileSourceMetricFile<'a> { - name: String, - metrics: Vec<&'a Metric>, -} - -impl<'a> FileSourceMetricFile<'a> { - /// Returns a new FileSourceMetricFile from a (name, Vec<&Metric>) tuple - #[allow(clippy::missing_const_for_fn)] // const cannot run destructor - fn from_tuple((name, metrics): (String, Vec<&'a Metric>)) -> Self { - Self { name, metrics } - } - - pub const fn get_name(&self) -> &str { - self.name.as_str() - } -} - -#[Object] -impl FileSourceMetricFile<'_> { - /// File name - async fn name(&self) -> &str { - &*self.name - } - - /// Metric indicating bytes received for the current file - async fn received_bytes_total(&self) -> Option { - self.metrics.received_bytes_total() - } - - /// Metric indicating received events for the current file - async fn received_events_total(&self) -> Option { - self.metrics.received_events_total() - } - - /// Metric indicating outgoing events for the current file - async fn sent_events_total(&self) -> Option { - self.metrics.sent_events_total() - } -} - -#[derive(Debug, Clone)] -pub struct FileSourceMetrics(Vec); - -impl FileSourceMetrics { - pub const fn new(metrics: Vec) -> Self { - Self(metrics) - } - - pub fn get_files(&self) -> Vec> { - self.0 - .iter() - .filter_map(|m| m.tag_value("file").map(|file| (file, m))) - .fold( - BTreeMap::new(), - |mut map: BTreeMap>, (file, m)| { - map.entry(file).or_default().push(m); - map - }, - ) - .into_iter() - .map(FileSourceMetricFile::from_tuple) - .collect() - } -} - -#[derive(Enum, Copy, Clone, Eq, PartialEq)] -pub enum FileSourceMetricFilesSortFieldName { - Name, - ReceivedBytesTotal, - ReceivedEventsTotal, - SentEventsTotal, -} - -impl sort::SortableByField for FileSourceMetricFile<'_> { - fn sort(&self, rhs: &Self, field: &FileSourceMetricFilesSortFieldName) -> Ordering { - match field { - FileSourceMetricFilesSortFieldName::Name => Ord::cmp(&self.name, &rhs.name), - FileSourceMetricFilesSortFieldName::ReceivedBytesTotal => Ord::cmp( - &self - .metrics - .received_bytes_total() - .map(|m| m.get_received_bytes_total() as i64) - .unwrap_or(0), - &rhs.metrics - .received_bytes_total() - .map(|m| m.get_received_bytes_total() as i64) - .unwrap_or(0), - ), - FileSourceMetricFilesSortFieldName::ReceivedEventsTotal => Ord::cmp( - &self - .metrics - .received_events_total() - .map(|m| m.get_received_events_total() as i64) - .unwrap_or(0), - &rhs.metrics - .received_events_total() - .map(|m| m.get_received_events_total() as i64) - .unwrap_or(0), - ), - FileSourceMetricFilesSortFieldName::SentEventsTotal => Ord::cmp( - &self - .metrics - .sent_events_total() - .map(|m| m.get_sent_events_total() as i64) - .unwrap_or(0), - &rhs.metrics - .sent_events_total() - .map(|m| m.get_sent_events_total() as i64) - .unwrap_or(0), - ), - } - } -} - -#[derive(Default, InputObject)] -pub struct FileSourceMetricsFilesFilter { - name: Option>, - or: Option>, -} - -impl CustomFilter> for FileSourceMetricsFilesFilter { - fn matches(&self, file: &FileSourceMetricFile<'_>) -> bool { - filter_check!( - self.name - .as_ref() - .map(|f| f.iter().all(|f| f.filter_value(file.get_name()))) - ); - true - } - - fn or(&self) -> Option<&Vec> { - self.or.as_ref() - } -} - -#[allow(clippy::too_many_arguments)] -#[Object] -impl FileSourceMetrics { - /// File metrics - pub async fn files( - &self, - after: Option, - before: Option, - first: Option, - last: Option, - filter: Option, - sort: Option>>, - ) -> relay::ConnectionResult> { - let filter = filter.unwrap_or_default(); - let mut files = filter_items(self.get_files().into_iter(), &filter); - - if let Some(sort_fields) = sort { - sort::by_fields(&mut files, &sort_fields); - } - - relay::query( - files.into_iter(), - relay::Params::new(after, before, first, last), - 10, - ) - .await - } - - /// Total received bytes for the current file source - pub async fn received_bytes_total(&self) -> Option { - self.0.received_bytes_total() - } - - /// Total received events for the current file source - pub async fn received_events_total(&self) -> Option { - self.0.received_events_total() - } - - /// Total sent events for the current file source - pub async fn sent_events_total(&self) -> Option { - self.0.sent_events_total() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::{ - api::schema::sort::SortField, - event::{MetricKind, MetricValue}, - }; - - struct FileSourceMetricTest { - name: &'static str, - events_metric: Metric, - bytes_metric: Metric, - } - - impl FileSourceMetricTest { - fn new(name: &'static str, events_processed: f64, bytes_processed: f64) -> Self { - Self { - name, - events_metric: metric("component_sent_events_total", events_processed), - bytes_metric: metric("component_received_bytes_total", bytes_processed), - } - } - - fn get_metric(&self) -> FileSourceMetricFile<'_> { - FileSourceMetricFile::from_tuple(( - self.name.to_string(), - vec![&self.bytes_metric, &self.events_metric], - )) - } - } - - fn metric(name: &str, value: f64) -> Metric { - Metric::new( - name, - MetricKind::Incremental, - MetricValue::Counter { value }, - ) - } - - fn by_name(name: &'static str) -> FileSourceMetricTest { - FileSourceMetricTest::new(name, 0.00, 0.00) - } - - #[test] - fn sort_name_asc() { - let t1 = by_name("/path/to/file/2"); - let t2 = by_name("/path/to/file/3"); - let t3 = by_name("/path/to/file/1"); - - let mut files = vec![t1.get_metric(), t2.get_metric(), t3.get_metric()]; - let fields = vec![SortField:: { - field: FileSourceMetricFilesSortFieldName::Name, - direction: sort::Direction::Asc, - }]; - - sort::by_fields(&mut files, &fields); - - for (i, f) in ["1", "2", "3"].iter().enumerate() { - assert_eq!(files[i].name.as_str(), format!("/path/to/file/{f}")); - } - } - - #[test] - fn sort_name_desc() { - let t1 = by_name("/path/to/file/2"); - let t2 = by_name("/path/to/file/3"); - let t3 = by_name("/path/to/file/1"); - - let mut files = vec![t1.get_metric(), t2.get_metric(), t3.get_metric()]; - let fields = vec![SortField:: { - field: FileSourceMetricFilesSortFieldName::Name, - direction: sort::Direction::Desc, - }]; - - sort::by_fields(&mut files, &fields); - - for (i, f) in ["3", "2", "1"].iter().enumerate() { - assert_eq!(files[i].name.as_str(), format!("/path/to/file/{f}")); - } - } - - #[test] - fn processed_events_asc() { - let t1 = FileSourceMetricTest::new("a", 1000.00, 100.00); - let t2 = FileSourceMetricTest::new("b", 500.00, 300.00); - let t3 = FileSourceMetricTest::new("c", 250.00, 200.00); - - let mut files = vec![t1.get_metric(), t2.get_metric(), t3.get_metric()]; - let fields = vec![SortField:: { - field: FileSourceMetricFilesSortFieldName::SentEventsTotal, - direction: sort::Direction::Asc, - }]; - - sort::by_fields(&mut files, &fields); - - for (i, f) in ["c", "b", "a"].iter().enumerate() { - assert_eq!(&files[i].name, *f); - } - } - - #[test] - fn processed_events_desc() { - let t1 = FileSourceMetricTest::new("a", 1000.00, 100.00); - let t2 = FileSourceMetricTest::new("b", 500.00, 300.00); - let t3 = FileSourceMetricTest::new("c", 250.00, 200.00); - - let mut files = vec![t1.get_metric(), t2.get_metric(), t3.get_metric()]; - let fields = vec![SortField:: { - field: FileSourceMetricFilesSortFieldName::SentEventsTotal, - direction: sort::Direction::Desc, - }]; - - sort::by_fields(&mut files, &fields); - - for (i, f) in ["a", "b", "c"].iter().enumerate() { - assert_eq!(&files[i].name, *f); - } - } - - #[test] - fn received_bytes_asc() { - let t1 = FileSourceMetricTest::new("a", 1000.00, 100.00); - let t2 = FileSourceMetricTest::new("b", 500.00, 300.00); - let t3 = FileSourceMetricTest::new("c", 250.00, 200.00); - - let mut files = vec![t1.get_metric(), t2.get_metric(), t3.get_metric()]; - let fields = vec![SortField:: { - field: FileSourceMetricFilesSortFieldName::ReceivedBytesTotal, - direction: sort::Direction::Asc, - }]; - - sort::by_fields(&mut files, &fields); - - for (i, f) in ["a", "c", "b"].iter().enumerate() { - assert_eq!(&files[i].name, *f); - } - } - - #[test] - fn received_bytes_desc() { - let t1 = FileSourceMetricTest::new("a", 1000.00, 100.00); - let t2 = FileSourceMetricTest::new("b", 500.00, 300.00); - let t3 = FileSourceMetricTest::new("c", 250.00, 200.00); - - let mut files = vec![t1.get_metric(), t2.get_metric(), t3.get_metric()]; - let fields = vec![SortField:: { - field: FileSourceMetricFilesSortFieldName::ReceivedBytesTotal, - direction: sort::Direction::Desc, - }]; - - sort::by_fields(&mut files, &fields); - - for (i, f) in ["b", "c", "a"].iter().enumerate() { - assert_eq!(&files[i].name, *f); - } - } -} diff --git a/src/api/schema/metrics/source/generic.rs b/src/api/schema/metrics/source/generic.rs deleted file mode 100644 index 12435c1b36e90..0000000000000 --- a/src/api/schema/metrics/source/generic.rs +++ /dev/null @@ -1,33 +0,0 @@ -use async_graphql::Object; - -use crate::{ - api::schema::metrics::{self, MetricsFilter}, - event::Metric, -}; - -#[derive(Debug, Clone)] -pub struct GenericSourceMetrics(Vec); - -impl GenericSourceMetrics { - pub const fn new(metrics: Vec) -> Self { - Self(metrics) - } -} - -#[Object] -impl GenericSourceMetrics { - /// Total received bytes for the current source - pub async fn received_bytes_total(&self) -> Option { - self.0.received_bytes_total() - } - - /// Total received events for the current source - pub async fn received_events_total(&self) -> Option { - self.0.received_events_total() - } - - /// Total sent events for the current source - pub async fn sent_events_total(&self) -> Option { - self.0.sent_events_total() - } -} diff --git a/src/api/schema/metrics/source/mod.rs b/src/api/schema/metrics/source/mod.rs deleted file mode 100644 index fe5c40ff6c346..0000000000000 --- a/src/api/schema/metrics/source/mod.rs +++ /dev/null @@ -1,31 +0,0 @@ -pub mod file; -mod generic; - -use async_graphql::Interface; - -use super::{ReceivedBytesTotal, ReceivedEventsTotal, SentEventsTotal}; -use crate::event::Metric; - -#[derive(Debug, Clone, Interface)] -#[graphql( - field(name = "received_bytes_total", ty = "Option"), - field(name = "received_events_total", ty = "Option"), - field(name = "sent_events_total", ty = "Option") -)] -pub enum SourceMetrics { - GenericSourceMetrics(generic::GenericSourceMetrics), - FileSourceMetrics(file::FileSourceMetrics), -} - -pub trait IntoSourceMetrics { - fn into_source_metrics(self, component_type: &str) -> SourceMetrics; -} - -impl IntoSourceMetrics for Vec { - fn into_source_metrics(self, component_type: &str) -> SourceMetrics { - match component_type { - "file" => SourceMetrics::FileSourceMetrics(file::FileSourceMetrics::new(self)), - _ => SourceMetrics::GenericSourceMetrics(generic::GenericSourceMetrics::new(self)), - } - } -} diff --git a/src/api/schema/metrics/transform/generic.rs b/src/api/schema/metrics/transform/generic.rs deleted file mode 100644 index 854a845226cc1..0000000000000 --- a/src/api/schema/metrics/transform/generic.rs +++ /dev/null @@ -1,28 +0,0 @@ -use async_graphql::Object; - -use crate::{ - api::schema::metrics::{self, MetricsFilter}, - event::Metric, -}; - -#[derive(Debug, Clone)] -pub struct GenericTransformMetrics(Vec); - -impl GenericTransformMetrics { - pub const fn new(metrics: Vec) -> Self { - Self(metrics) - } -} - -#[Object] -impl GenericTransformMetrics { - /// Total received events for the current transform - pub async fn received_events_total(&self) -> Option { - self.0.received_events_total() - } - - /// Total sent events for the current transform - pub async fn sent_events_total(&self) -> Option { - self.0.sent_events_total() - } -} diff --git a/src/api/schema/metrics/transform/mod.rs b/src/api/schema/metrics/transform/mod.rs deleted file mode 100644 index 183a97907f592..0000000000000 --- a/src/api/schema/metrics/transform/mod.rs +++ /dev/null @@ -1,25 +0,0 @@ -mod generic; - -use async_graphql::Interface; - -use super::{ReceivedEventsTotal, SentEventsTotal}; -use crate::event::Metric; - -#[derive(Debug, Clone, Interface)] -#[graphql( - field(name = "received_events_total", ty = "Option"), - field(name = "sent_events_total", ty = "Option") -)] -pub enum TransformMetrics { - GenericTransformMetrics(generic::GenericTransformMetrics), -} - -pub trait IntoTransformMetrics { - fn into_transform_metrics(self, component_type: &str) -> TransformMetrics; -} - -impl IntoTransformMetrics for Vec { - fn into_transform_metrics(self, _component_type: &str) -> TransformMetrics { - TransformMetrics::GenericTransformMetrics(generic::GenericTransformMetrics::new(self)) - } -} diff --git a/src/api/schema/metrics/uptime.rs b/src/api/schema/metrics/uptime.rs deleted file mode 100644 index 7c02c223e0859..0000000000000 --- a/src/api/schema/metrics/uptime.rs +++ /dev/null @@ -1,34 +0,0 @@ -use async_graphql::Object; -use chrono::{DateTime, Utc}; - -use crate::event::{Metric, MetricValue}; - -pub struct Uptime(Metric); - -impl Uptime { - pub const fn new(m: Metric) -> Self { - Self(m) - } -} - -#[Object] -impl Uptime { - /// Metric timestamp - pub async fn timestamp(&self) -> Option> { - self.0.timestamp() - } - - /// Number of seconds the Vector instance has been alive - pub async fn seconds(&self) -> f64 { - match self.0.value() { - MetricValue::Gauge { value } => *value, - _ => 0.00, - } - } -} - -impl From for Uptime { - fn from(m: Metric) -> Self { - Self(m) - } -} diff --git a/src/api/schema/mod.rs b/src/api/schema/mod.rs deleted file mode 100644 index 58c22a03b687f..0000000000000 --- a/src/api/schema/mod.rs +++ /dev/null @@ -1,31 +0,0 @@ -pub mod components; -pub mod events; -pub mod filter; -mod health; -mod meta; -mod metrics; -mod relay; -pub mod sort; - -use async_graphql::{EmptyMutation, MergedObject, MergedSubscription, Schema, SchemaBuilder}; - -#[derive(MergedObject, Default)] -pub struct Query( - health::HealthQuery, - components::ComponentsQuery, - #[cfg(feature = "sources-host_metrics")] metrics::MetricsQuery, - meta::MetaQuery, -); - -#[derive(MergedSubscription, Default)] -pub struct Subscription( - health::HealthSubscription, - metrics::MetricsSubscription, - components::ComponentsSubscription, - events::EventsSubscription, -); - -/// Build a new GraphQL schema, comprised of Query, Mutation and Subscription types -pub fn build_schema() -> SchemaBuilder { - Schema::build(Query::default(), EmptyMutation, Subscription::default()) -} diff --git a/src/api/schema/relay.rs b/src/api/schema/relay.rs deleted file mode 100644 index 3124a2272a66e..0000000000000 --- a/src/api/schema/relay.rs +++ /dev/null @@ -1,168 +0,0 @@ -use std::convert::Infallible; - -use async_graphql::{ - Result, SimpleObject, - connection::{self, Connection, CursorType, Edge, EmptyFields}, -}; -use base64::prelude::{BASE64_URL_SAFE_NO_PAD, Engine as _}; - -/// Base64 invalid states, used by `Base64Cursor`. -pub enum Base64CursorError { - /// Invalid cursor. This can happen if the base64 string is valid, but its contents don't - /// conform to the `name:index` pattern. - Invalid, - /// Decoding error. If this happens, the string isn't valid base64. - DecodeError(base64::DecodeError), -} - -impl std::fmt::Display for Base64CursorError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "Invalid cursor") - } -} - -/// Base64 cursor implementation -pub struct Base64Cursor { - name: &'static str, - index: usize, -} - -impl Base64Cursor { - const fn new(index: usize) -> Self { - Self { - name: "Cursor", - index, - } - } - - /// Returns a base64 string representation of the cursor - fn encode(&self) -> String { - BASE64_URL_SAFE_NO_PAD.encode(format!("{}:{}", self.name, self.index)) - } - - /// Decodes a base64 string into a cursor result - fn decode(s: &str) -> Result { - let bytes = BASE64_URL_SAFE_NO_PAD - .decode(s) - .map_err(Base64CursorError::DecodeError)?; - - let cursor = String::from_utf8(bytes).map_err(|_| Base64CursorError::Invalid)?; - let index = cursor - .split(':') - .next_back() - .map(|s| s.parse::()) - .ok_or(Base64CursorError::Invalid)? - .map_err(|_| Base64CursorError::Invalid)?; - - Ok(Self::new(index)) - } - - /// Increment and return the index. Uses saturating_add to avoid overflow - /// issues. - const fn increment(&self) -> usize { - self.index.saturating_add(1) - } -} - -impl From for usize { - fn from(cursor: Base64Cursor) -> Self { - cursor.index - } -} - -/// Makes the `Base64Cursor` compatible with Relay connections -impl CursorType for Base64Cursor { - type Error = Base64CursorError; - - fn decode_cursor(s: &str) -> Result { - Base64Cursor::decode(s) - } - - fn encode_cursor(&self) -> String { - self.encode() - } -} - -/// Additional fields to attach to the connection -#[derive(SimpleObject)] -pub struct ConnectionFields { - /// Total result set count - total_count: usize, -} - -/// Relay connection result -pub type ConnectionResult = Result>; - -/// Relay-compliant connection parameters to page results by cursor/page size -pub struct Params { - after: Option, - before: Option, - first: Option, - last: Option, -} - -impl Params { - pub const fn new( - after: Option, - before: Option, - first: Option, - last: Option, - ) -> Self { - Self { - after, - before, - first, - last, - } - } -} - -/// Creates a new Relay-compliant connection. Iterator must implement `ExactSizeIterator` to -/// determine page position in the total result set. -pub async fn query>( - iter: I, - p: Params, - default_page_size: usize, -) -> ConnectionResult { - connection::query::<_, _, Base64Cursor, _, _, ConnectionFields, _, _, _, Infallible>( - p.after, - p.before, - p.first, - p.last, - |after, before, first, last| async move { - let iter_len = iter.len(); - - let (start, end) = { - let after = after.map(|a| a.increment()).unwrap_or(0); - let before: usize = before.map(|b| b.into()).unwrap_or(iter_len); - - // Calculate start/end based on the provided first/last. Note that async-graphql disallows - // providing both (returning an error), so we can safely assume we have, at most, one of - // first or last. - match (first, last) { - // First - (Some(first), _) => (after, (after.saturating_add(first)).min(before)), - // Last - (_, Some(last)) => ((before.saturating_sub(last)).max(after), before), - // Default page size - _ => (after, default_page_size.min(before)), - } - }; - - let mut connection = Connection::with_additional_fields( - start > 0, - end < iter_len, - ConnectionFields { - total_count: iter_len, - }, - ); - connection.edges.extend( - (start..end) - .zip(iter.skip(start)) - .map(|(cursor, node)| Edge::new(Base64Cursor::new(cursor), node)), - ); - Ok(connection) - }, - ) - .await -} diff --git a/src/api/schema/sort.rs b/src/api/schema/sort.rs deleted file mode 100644 index 8e40750c88a83..0000000000000 --- a/src/api/schema/sort.rs +++ /dev/null @@ -1,61 +0,0 @@ -use std::cmp::Ordering; - -use async_graphql::{Enum, InputObject, InputType}; -use itertools::{ - FoldWhile::{Continue, Done}, - Itertools, -}; - -use crate::api::schema::{ - components::{ - ComponentsSortFieldName, sink::SinksSortFieldName, source::SourcesSortFieldName, - transform::TransformsSortFieldName, - }, - metrics::source::file::FileSourceMetricFilesSortFieldName, -}; - -#[derive(Enum, Copy, Clone, Default, PartialEq, Eq)] -pub enum Direction { - #[default] - Asc, - Desc, -} - -#[derive(InputObject)] -#[graphql(concrete(name = "ComponentsSortField", params(ComponentsSortFieldName)))] -#[graphql(concrete(name = "SourcesSortField", params(SourcesSortFieldName)))] -#[graphql(concrete(name = "TransformsSortField", params(TransformsSortFieldName)))] -#[graphql(concrete(name = "SinksSortField", params(SinksSortFieldName)))] -#[graphql(concrete( - name = "FileSourceMetricFilesSortField", - params(FileSourceMetricFilesSortFieldName) -))] -pub struct SortField { - pub field: T, - #[graphql(default_with = "Direction::default()")] - pub direction: Direction, -} - -/// Defines a type as sortable by a given field -pub trait SortableByField { - fn sort(&self, rhs: &Self, field: &T) -> Ordering; -} - -/// Performs an in-place sort against a slice of `Sortable`, with the provided [`SortField`]s -pub fn by_fields(f: &mut [impl SortableByField], sort_fields: &[SortField]) { - f.sort_by(|a, b| { - sort_fields - .iter() - .fold_while(Ordering::Equal, |cmp, f| match cmp { - Ordering::Equal => { - let cmp = a.sort(b, &f.field); - Continue(match f.direction { - Direction::Desc => cmp.reverse(), - _ => cmp, - }) - } - _ => Done(cmp), - }) - .into_inner() - }); -} diff --git a/src/api/server.rs b/src/api/server.rs deleted file mode 100644 index 9ddbc407c7f22..0000000000000 --- a/src/api/server.rs +++ /dev/null @@ -1,204 +0,0 @@ -use std::{ - convert::Infallible, - net::SocketAddr, - sync::{Arc, atomic::AtomicBool}, -}; - -use async_graphql::{ - Data, Request, Schema, - http::{GraphQLPlaygroundConfig, WebSocketProtocols, playground_source}, -}; -use async_graphql_warp::{GraphQLResponse, GraphQLWebSocket, graphql_protocol}; -use hyper::{Server as HyperServer, server::conn::AddrIncoming, service::make_service_fn}; -use tokio::{runtime::Handle, sync::oneshot}; -use tower::ServiceBuilder; -use tracing::Span; -use vector_lib::tap::topology; -use warp::{Filter, Reply, filters::BoxedFilter, http::Response, ws::Ws}; - -use super::{handler, schema}; -use crate::{ - config::{self, api}, - http::build_http_trace_layer, - internal_events::{SocketBindError, SocketMode}, -}; - -pub struct Server { - _shutdown: oneshot::Sender<()>, - addr: SocketAddr, -} - -impl Server { - /// Start the API server. This creates the routes and spawns a Warp server. The server is - /// gracefully shut down when Self falls out of scope by way of the oneshot sender closing. - pub fn start( - config: &config::Config, - watch_rx: topology::WatchRx, - running: Arc, - handle: &Handle, - ) -> crate::Result { - let routes = make_routes(config.api, watch_rx, running); - - let (_shutdown, rx) = oneshot::channel(); - // warp uses `tokio::spawn` and so needs us to enter the runtime context. - let _guard = handle.enter(); - - let addr = config.api.address.expect("No socket address"); - let incoming = AddrIncoming::bind(&addr).inspect_err(|error| { - emit!(SocketBindError { - mode: SocketMode::Tcp, - error, - }); - })?; - - let span = Span::current(); - let make_svc = make_service_fn(move |_conn| { - let svc = ServiceBuilder::new() - .layer(build_http_trace_layer(span.clone())) - .service(warp::service(routes.clone())); - futures_util::future::ok::<_, Infallible>(svc) - }); - - let server = async move { - HyperServer::builder(incoming) - .serve(make_svc) - .with_graceful_shutdown(async { - rx.await.ok(); - }) - .await - .map_err(|err| { - error!("An error occurred: {:?}.", err); - }) - }; - - // Update component schema with the config before starting the server. - schema::components::update_config(config); - - // Spawn the server in the background. - handle.spawn(server); - - Ok(Self { _shutdown, addr }) - } - - /// Returns a copy of the SocketAddr that the server was started on. - pub const fn addr(&self) -> SocketAddr { - self.addr - } - - /// Update the configuration of a running server. While this instance method doesn't - /// directly involve `self`, it provides a neater API to expose an internal implementation - /// detail than exposing the function of the sub-mod directly. - pub fn update_config(&self, config: &config::Config) { - schema::components::update_config(config) - } -} - -fn make_routes( - api: api::Options, - watch_tx: topology::WatchRx, - running: Arc, -) -> BoxedFilter<(impl Reply,)> { - // Routes... - - // Health. - let health = warp::path("health") - .and(with_shared(running)) - .and_then(handler::health); - - // 404. - let not_found_graphql = warp::any().and_then(|| async { Err(warp::reject::not_found()) }); - let not_found = warp::any().and_then(|| async { Err(warp::reject::not_found()) }); - - // GraphQL subscription handler. Creates a Warp WebSocket handler and for each connection, - // parses the required headers for GraphQL and builds per-connection context based on the - // provided `WatchTx` channel sender. This allows GraphQL resolvers to subscribe to - // topology changes. - let graphql_subscription_handler = - warp::ws() - .and(graphql_protocol()) - .map(move |ws: Ws, protocol: WebSocketProtocols| { - let schema = schema::build_schema().finish(); - let watch_tx = watch_tx.clone(); - - let reply = ws.on_upgrade(move |socket| { - let mut data = Data::default(); - data.insert(watch_tx); - - GraphQLWebSocket::new(socket, schema, protocol) - .with_data(data) - .serve() - }); - - warp::reply::with_header( - reply, - "Sec-WebSocket-Protocol", - protocol.sec_websocket_protocol(), - ) - }); - - // Handle GraphQL queries. Headers will first be parsed to determine whether the query is - // a subscription and if so, an attempt will be made to upgrade the connection to WebSockets. - // All other queries will fall back to the default HTTP handler. - let graphql_handler = if api.graphql { - warp::path("graphql") - .and(graphql_subscription_handler.or( - async_graphql_warp::graphql(schema::build_schema().finish()).and_then( - |(schema, request): (Schema<_, _, _>, Request)| async move { - Ok::<_, Infallible>(GraphQLResponse::from(schema.execute(request).await)) - }, - ), - )) - .boxed() - } else { - not_found_graphql.boxed() - }; - - // Provide a playground for executing GraphQL queries/mutations/subscriptions. - let graphql_playground = if api.playground && api.graphql { - warp::path("playground") - .map(move || { - Response::builder() - .header("content-type", "text/html") - .body(playground_source( - GraphQLPlaygroundConfig::new("/graphql").subscription_endpoint("/graphql"), - )) - }) - .boxed() - } else { - not_found.boxed() - }; - - // Wire up the health + GraphQL endpoints. Provides a permissive CORS policy to allow for - // cross-origin interaction with the Vector API. - health - .or(graphql_handler) - .or(graphql_playground) - .or(not_found) - .with( - warp::cors() - .allow_any_origin() - .allow_headers(vec![ - "User-Agent", - "Sec-Fetch-Mode", - "Referer", - "Origin", - "Access-Control-Request-Method", - "Access-Control-Allow-Origin", - "Access-Control-Request-Headers", - "Content-Type", - "X-Apollo-Tracing", // for Apollo GraphQL clients - "Pragma", - "Host", - "Connection", - "Cache-Control", - ]) - .allow_methods(vec!["POST", "GET"]), - ) - .boxed() -} - -fn with_shared( - shared: Arc, -) -> impl Filter,), Error = Infallible> + Clone { - warp::any().map(move || Arc::::clone(&shared)) -} diff --git a/src/api/tests.rs b/src/api/tests.rs deleted file mode 100644 index 56794376b2a25..0000000000000 --- a/src/api/tests.rs +++ /dev/null @@ -1,589 +0,0 @@ -use std::{ - collections::{HashMap, HashSet}, - time::Duration, -}; - -use futures::StreamExt; -use tokio::sync::{mpsc, watch}; -use vector_lib::{ - config::ComponentKey, - fanout, - tap::{ - controller::{TapController, TapPatterns, TapPayload}, - notification::{InvalidMatch, Matched, NotMatched, Notification}, - topology::{TapOutput, TapResource}, - }, -}; - -use crate::{ - api::schema::events::{create_events_stream, log, metric, output::OutputEventsPayload}, - config::{Config, OutputId}, - event::{LogEvent, Metric, MetricKind, MetricValue}, - sinks::blackhole::BlackholeConfig, - sources::demo_logs::{DemoLogsConfig, OutputFormat}, - test_util::{start_topology, trace_init}, - transforms::{ - log_to_metric::{LogToMetricConfig, MetricConfig, MetricTypeConfig}, - remap::RemapConfig, - }, -}; - -#[tokio::test] -/// A tap sink should match a pattern, receive the correct notifications, -/// and receive events -async fn sink_events() { - let pattern_matched = "tes*"; - let pattern_not_matched = "xyz"; - let id = OutputId::from(&ComponentKey::from("test")); - - let (mut fanout, control_tx) = fanout::Fanout::new(); - let mut outputs = HashMap::new(); - outputs.insert( - TapOutput { - output_id: id.clone(), - component_kind: "source", - component_type: "demo".to_string(), - }, - control_tx, - ); - let tap_resource = TapResource { - outputs, - inputs: HashMap::new(), - source_keys: Vec::new(), - sink_keys: Vec::new(), - removals: HashSet::new(), - }; - - let (watch_tx, watch_rx) = watch::channel(TapResource::default()); - let (sink_tx, mut sink_rx) = mpsc::channel(10); - - let _controller = TapController::new( - watch_rx, - sink_tx, - TapPatterns::new( - HashSet::from([pattern_matched.to_string(), pattern_not_matched.to_string()]), - HashSet::new(), - ), - ); - - // Add the outputs to trigger a change event. - watch_tx.send(tap_resource).unwrap(); - - // First two events should contain a notification that one pattern matched, and - // one that didn't. - #[allow(clippy::mixed_read_write_in_expression)] - let notifications = vec![sink_rx.recv().await, sink_rx.recv().await]; - - for notification in notifications.into_iter() { - match notification { - Some(TapPayload::Notification(Notification::Matched(matched))) - if matched.pattern == pattern_matched => - { - continue; - } - Some(TapPayload::Notification(Notification::NotMatched(not_matched))) - if not_matched.pattern == pattern_not_matched => - { - continue; - } - _ => panic!("unexpected payload"), - } - } - - // Send some events down the wire. Waiting until the first notifications are in - // to ensure the event handler has been initialized. - let log_event = LogEvent::default(); - let metric_event = Metric::new( - id.to_string(), - MetricKind::Incremental, - MetricValue::Counter { value: 1.0 }, - ); - - fanout - .send(vec![metric_event].into(), None) - .await - .expect("should not fail"); - fanout - .send(vec![log_event].into(), None) - .await - .expect("should not fail"); - - // 3rd payload should be the metric event - assert!(matches!( - sink_rx.recv().await, - Some(TapPayload::Metric(output, _)) if output.output_id == id - )); - - // 4th payload should be the log event - assert!(matches!( - sink_rx.recv().await, - Some(TapPayload::Log(output, _)) if output.output_id == id - )); -} - -fn assert_notification(payload: OutputEventsPayload) -> Notification { - if let OutputEventsPayload::Notification(event_notification) = payload { - event_notification.notification - } else { - panic!("Expected payload to be a Notification") - } -} - -fn assert_log(payload: OutputEventsPayload) -> log::Log { - if let OutputEventsPayload::Log(log) = payload { - log - } else { - panic!("Expected payload to be a Log") - } -} - -fn assert_metric(payload: OutputEventsPayload) -> metric::Metric { - if let OutputEventsPayload::Metric(metric) = payload { - metric - } else { - panic!("Expected payload to be a Metric") - } -} - -#[tokio::test] -async fn integration_test_source_log() { - trace_init(); - - let mut config = Config::builder(); - config.add_source( - "in", - DemoLogsConfig { - interval: Duration::from_secs_f64(0.01), - count: 200, - format: OutputFormat::Json, - ..Default::default() - }, - ); - config.add_sink( - "out", - &["in"], - BlackholeConfig { - print_interval_secs: Duration::from_secs(1), - rate: None, - acknowledgements: Default::default(), - }, - ); - - let (topology, _) = start_topology(config.build().unwrap(), false).await; - - let source_tap_stream = create_events_stream( - topology.watch(), - TapPatterns::new(HashSet::from(["in".to_string()]), HashSet::new()), - 500, - 100, - ); - - let source_tap_events: Vec<_> = source_tap_stream.take(2).collect().await; - - assert_eq!( - assert_notification(source_tap_events[0][0].clone()), - Notification::Matched(Matched::new("in".to_string())) - ); - let _log = assert_log(source_tap_events[1][0].clone()); -} - -#[tokio::test] -async fn integration_test_source_metric() { - trace_init(); - - let mut config = Config::builder(); - config.add_source( - "in", - DemoLogsConfig { - interval: Duration::from_secs_f64(0.01), - count: 200, - format: OutputFormat::Shuffle { - sequence: false, - lines: vec!["1".to_string()], - }, - ..Default::default() - }, - ); - config.add_transform( - "to_metric", - &["in"], - LogToMetricConfig { - metrics: Some(vec![MetricConfig { - field: "message".try_into().expect("Fixed template string"), - name: None, - namespace: None, - tags: None, - metric: MetricTypeConfig::Gauge, - }]), - all_metrics: None, - }, - ); - config.add_sink( - "out", - &["to_metric"], - BlackholeConfig { - print_interval_secs: Duration::from_secs(1), - rate: None, - acknowledgements: Default::default(), - }, - ); - - let (topology, _) = start_topology(config.build().unwrap(), false).await; - - let source_tap_stream = create_events_stream( - topology.watch(), - TapPatterns::new(HashSet::from(["to_metric".to_string()]), HashSet::new()), - 500, - 100, - ); - - let source_tap_events: Vec<_> = source_tap_stream.take(2).collect().await; - - assert_eq!( - assert_notification(source_tap_events[0][0].clone()), - Notification::Matched(Matched::new("to_metric".to_string())) - ); - assert_metric(source_tap_events[1][0].clone()); -} - -#[tokio::test] -async fn integration_test_transform() { - trace_init(); - - let mut config = Config::builder(); - config.add_source( - "in", - DemoLogsConfig { - interval: Duration::from_secs_f64(0.01), - count: 200, - format: OutputFormat::Json, - ..Default::default() - }, - ); - config.add_transform( - "transform", - &["in"], - RemapConfig { - source: Some("".to_string()), - ..Default::default() - }, - ); - config.add_sink( - "out", - &["transform"], - BlackholeConfig { - print_interval_secs: Duration::from_secs(1), - rate: None, - acknowledgements: Default::default(), - }, - ); - - let (topology, _) = start_topology(config.build().unwrap(), false).await; - - let transform_tap_stream = create_events_stream( - topology.watch(), - TapPatterns::new(HashSet::from(["transform".to_string()]), HashSet::new()), - 500, - 100, - ); - - let transform_tap_events: Vec<_> = transform_tap_stream.take(2).collect().await; - - assert_eq!( - assert_notification(transform_tap_events[0][0].clone()), - Notification::Matched(Matched::new("transform".to_string())) - ); - let _log = assert_log(transform_tap_events[1][0].clone()); -} - -#[tokio::test] -async fn integration_test_transform_input() { - trace_init(); - - let mut config = Config::builder(); - config.add_source( - "in", - DemoLogsConfig { - interval: Duration::from_secs_f64(0.01), - count: 200, - format: OutputFormat::Shuffle { - sequence: false, - lines: vec!["test".to_string()], - }, - ..Default::default() - }, - ); - config.add_transform( - "transform", - &["in"], - RemapConfig { - source: Some(".message = \"new message\"".to_string()), - ..Default::default() - }, - ); - config.add_sink( - "out", - &["in"], - BlackholeConfig { - print_interval_secs: Duration::from_secs(1), - rate: None, - acknowledgements: Default::default(), - }, - ); - - let (topology, _) = start_topology(config.build().unwrap(), false).await; - - let tap_stream = create_events_stream( - topology.watch(), - TapPatterns::new( - HashSet::new(), - HashSet::from(["transform".to_string(), "in".to_string()]), - ), - 500, - 100, - ); - - let tap_events: Vec<_> = tap_stream.take(4).collect().await; - - let notifications = [ - assert_notification(tap_events[0][0].clone()), - assert_notification(tap_events[1][0].clone()), - assert_notification(tap_events[2][0].clone()), - ]; - assert!( - notifications - .iter() - .any(|n| *n == Notification::Matched(Matched::new("transform".to_string()))) - ); - // "in" is not matched since it corresponds to a source - assert!( - notifications - .iter() - .any(|n| *n == Notification::NotMatched(NotMatched::new("in".to_string()))) - ); - // "in" generates an invalid match notification to warn against an - // attempt to tap the input of a source - assert!(notifications.iter().any(|n| *n - == Notification::InvalidMatch(InvalidMatch::new("[tap] Warning: source inputs cannot be tapped. Input pattern 'in' matches sources [\"in\"]".to_string(), "in".to_string(), vec!["in".to_string()])))); - - assert_eq!( - assert_log(tap_events[3][0].clone()) - .get_message() - .unwrap_or_default(), - "test" - ); -} - -#[tokio::test] -async fn integration_test_sink() { - trace_init(); - - let mut config = Config::builder(); - config.add_source( - "in", - DemoLogsConfig { - interval: Duration::from_secs_f64(0.01), - count: 200, - format: OutputFormat::Shuffle { - sequence: false, - lines: vec!["test".to_string()], - }, - ..Default::default() - }, - ); - config.add_transform( - "transform", - &["in"], - RemapConfig { - source: Some(".message = \"new message\"".to_string()), - ..Default::default() - }, - ); - config.add_sink( - "out", - &["transform"], - BlackholeConfig { - print_interval_secs: Duration::from_secs(1), - rate: None, - acknowledgements: Default::default(), - }, - ); - - let (topology, _) = start_topology(config.build().unwrap(), false).await; - - let tap_stream = create_events_stream( - topology.watch(), - TapPatterns::new(HashSet::new(), HashSet::from(["out".to_string()])), - 500, - 100, - ); - - let tap_events: Vec<_> = tap_stream.take(2).collect().await; - - assert_eq!( - assert_notification(tap_events[0][0].clone()), - Notification::Matched(Matched::new("out".to_string())) - ); - assert_eq!( - assert_log(tap_events[1][0].clone()) - .get_message() - .unwrap_or_default(), - "new message" - ); -} - -#[tokio::test] -async fn integration_test_tap_non_default_output() { - trace_init(); - - let mut config = Config::builder(); - config.add_source( - "in", - DemoLogsConfig { - interval: Duration::from_secs_f64(0.01), - count: 200, - format: OutputFormat::Shuffle { - sequence: false, - lines: vec!["test2".to_string()], - }, - ..Default::default() - }, - ); - config.add_transform( - "transform", - &["in"], - RemapConfig { - source: Some("assert_eq!(.message, \"test1\")".to_string()), - drop_on_error: true, - reroute_dropped: true, - ..Default::default() - }, - ); - config.add_sink( - "out", - &["transform"], - BlackholeConfig { - print_interval_secs: Duration::from_secs(1), - rate: None, - acknowledgements: Default::default(), - }, - ); - - let (topology, _) = start_topology(config.build().unwrap(), false).await; - - let transform_tap_remap_dropped_stream = create_events_stream( - topology.watch(), - TapPatterns::new( - HashSet::from(["transform.dropped".to_string()]), - HashSet::new(), - ), - 500, - 100, - ); - - let transform_tap_events: Vec<_> = transform_tap_remap_dropped_stream.take(2).collect().await; - - assert_eq!( - assert_notification(transform_tap_events[0][0].clone()), - Notification::Matched(Matched::new("transform.dropped".to_string())) - ); - assert_eq!( - assert_log(transform_tap_events[1][0].clone()) - .get_message() - .unwrap_or_default(), - "test2" - ); -} - -#[tokio::test] -async fn integration_test_tap_multiple_outputs() { - trace_init(); - - let mut config = Config::builder(); - config.add_source( - "in-test1", - DemoLogsConfig { - interval: Duration::from_secs_f64(0.01), - count: 1, - format: OutputFormat::Shuffle { - sequence: false, - lines: vec!["test1".to_string()], - }, - ..Default::default() - }, - ); - config.add_source( - "in-test2", - DemoLogsConfig { - interval: Duration::from_secs_f64(0.01), - count: 1, - format: OutputFormat::Shuffle { - sequence: false, - lines: vec!["test2".to_string()], - }, - ..Default::default() - }, - ); - config.add_transform( - "transform", - &["in*"], - RemapConfig { - source: Some("assert_eq!(.message, \"test1\")".to_string()), - drop_on_error: true, - reroute_dropped: true, - ..Default::default() - }, - ); - config.add_sink( - "out", - &["transform"], - BlackholeConfig { - print_interval_secs: Duration::from_secs(1), - rate: None, - acknowledgements: Default::default(), - }, - ); - - let (topology, _) = start_topology(config.build().unwrap(), false).await; - - let mut transform_tap_all_outputs_stream = create_events_stream( - topology.watch(), - TapPatterns::new(HashSet::from(["transform*".to_string()]), HashSet::new()), - 500, - 100, - ); - - let transform_tap_notifications = transform_tap_all_outputs_stream.next().await.unwrap(); - assert_eq!( - assert_notification(transform_tap_notifications[0].clone()), - Notification::Matched(Matched::new("transform*".to_string())) - ); - - let mut default_output_found = false; - let mut dropped_output_found = false; - for _ in 0..2 { - if default_output_found && dropped_output_found { - break; - } - - match transform_tap_all_outputs_stream.next().await { - Some(tap_events) => { - if !default_output_found { - default_output_found = tap_events - .iter() - .map(|payload| assert_log(payload.clone())) - .any(|log| log.get_message().unwrap_or_default() == "test1"); - } - if !dropped_output_found { - dropped_output_found = tap_events - .iter() - .map(|payload| assert_log(payload.clone())) - .any(|log| log.get_message().unwrap_or_default() == "test2"); - } - } - None => break, - } - } - - assert!(default_output_found && dropped_output_found); -} diff --git a/src/app.rs b/src/app.rs index 9e8a415608a8f..580c06642a6e3 100644 --- a/src/app.rs +++ b/src/app.rs @@ -20,13 +20,17 @@ use tokio::{ use tokio_stream::wrappers::UnboundedReceiverStream; #[cfg(feature = "api")] -use crate::{api, internal_events::ApiStarted}; +use crate::api; +#[cfg(feature = "api")] +use crate::internal_events::ApiStarted; use crate::{ cli::{LogFormat, Opts, RootOpts, WatchConfigMethod, handle_config_errors}, config::{self, ComponentConfig, ComponentType, Config, ConfigPath}, extra_context::ExtraContext, heartbeat, - internal_events::{VectorConfigLoadError, VectorQuit, VectorStarted, VectorStopped}, + internal_events::{ + VectorConfigLoadError, VectorQuit, VectorStarted, VectorStopped, VectorStopping, + }, signal::{SignalHandler, SignalPair, SignalRx, SignalTo}, topology::{ ReloadOutcome, RunningTopology, SharedTopologyController, ShutdownErrorReceiver, @@ -129,28 +133,31 @@ impl ApplicationConfig { Ok(()) } - /// Configure the API server, if applicable + /// Configure the gRPC API server, if applicable #[cfg(feature = "api")] - pub fn setup_api(&self, handle: &Handle) -> Option { + pub fn setup_api(&self, handle: &Handle) -> Option { if self.api.enabled { - match api::Server::start( + // Start gRPC server + let api_server = handle.block_on(api::GrpcServer::start( self.topology.config(), self.topology.watch(), - std::sync::Arc::clone(&self.topology.running), - handle, - ) { - Ok(api_server) => { + )); + match api_server { + Ok(server) => { emit!(ApiStarted { - addr: self.api.address.unwrap(), - playground: self.api.playground, - graphql: self.api.graphql + addr: server.addr() }); - - Some(api_server) + Some(server) } Err(error) => { let error = error.to_string(); - error!(message = "An error occurred that Vector couldn't handle.", %error, internal_log_rate_limit = false); + error!( + message = "An error occurred that Vector couldn't handle.", + %error, + internal_log_rate_limit = false + ); + // Trigger shutdown because the API was explicitly enabled but failed to start + // This ensures users don't run Vector thinking top/tap will work when they won't _ = self .topology .abort_tx @@ -256,9 +263,12 @@ impl Application { signals, } = self; + #[cfg(feature = "api")] + let api_server = config.setup_api(handle); + let topology_controller = SharedTopologyController::new(TopologyController { #[cfg(feature = "api")] - api_server: config.setup_api(handle), + api_server, topology: config.topology, config_paths: config.config_paths.clone(), require_healthy: root_opts.require_healthy, @@ -482,16 +492,19 @@ impl FinishedApplication { } async fn stop(topology_controller: TopologyController, mut signal_rx: SignalRx) -> ExitStatus { - emit!(VectorStopped); + emit!(VectorStopping); tokio::select! { - _ = topology_controller.stop() => ExitStatus::from_raw({ - #[cfg(windows)] - { - exitcode::OK as u32 - } - #[cfg(unix)] - exitcode::OK - }), // Graceful shutdown finished + _ = topology_controller.stop() => { + emit!(VectorStopped); + ExitStatus::from_raw({ + #[cfg(windows)] + { + exitcode::OK as u32 + } + #[cfg(unix)] + exitcode::OK + }) + }, // Graceful shutdown finished _ = signal_rx.recv() => Self::quit(), } } diff --git a/src/azure/client.rs b/src/azure/client.rs new file mode 100644 index 0000000000000..b3ad5f708d7a6 --- /dev/null +++ b/src/azure/client.rs @@ -0,0 +1,726 @@ +//! Shared Azure client construction: authentication, connection-string handling, +//! and HTTP transport assembly for the new official Azure SDK (`azure_core` 1.0). +//! +//! Used by both Azure sinks (via re-exports in `crate::sinks::azure_common`) and +//! the `azure_blob` source. +#![allow(missing_docs)] + +use std::fs::File; +use std::io::Read; +use std::path::PathBuf; +use std::sync::Arc; + +#[cfg(test)] +use base64::prelude::*; + +use azure_core::credentials::{TokenCredential, TokenRequestOptions}; +use azure_core::http::{ClientMethodOptions, Url}; +use azure_core::{Error, error::ErrorKind}; + +use azure_identity::{ + AzureCliCredential, ClientAssertion, ClientAssertionCredential, ClientCertificateCredential, + ClientCertificateCredentialOptions, ClientSecretCredential, ManagedIdentityCredential, + ManagedIdentityCredentialOptions, UserAssignedId, WorkloadIdentityCredential, + WorkloadIdentityCredentialOptions, +}; + +use azure_storage_blob::{BlobContainerClient, BlobContainerClientOptions}; + +use vector_lib::{configurable::configurable_component, sensitive_string::SensitiveString}; + +use crate::azure::connection_string::{Auth, ParsedConnectionString}; +use crate::azure::shared_key_policy::SharedKeyAuthorizationPolicy; + +/// TLS configuration. +#[configurable_component] +#[configurable(metadata(docs::advanced))] +#[derive(Clone, Debug, Default)] +#[serde(deny_unknown_fields)] +pub struct AzureBlobTlsConfig { + /// Absolute path to an additional CA certificate file. + /// + /// The certificate must be in PEM (X.509) format. + #[serde(alias = "ca_path")] + #[configurable(metadata(docs::examples = "/path/to/certificate_authority.crt"))] + #[configurable(metadata(docs::human_name = "CA File Path"))] + pub ca_file: Option, +} + +/// Azure service principal authentication. +#[configurable_component] +#[derive(Clone, Debug, Eq, PartialEq)] +#[serde(deny_unknown_fields, untagged)] +pub enum AzureAuthentication { + #[configurable(metadata(docs::enum_tag_description = "The kind of Azure credential to use."))] + Specific(SpecificAzureCredential), + + /// Mock credential for testing β€” returns a static fake token + #[cfg(test)] + #[serde(skip)] + MockCredential, +} + +impl Default for AzureAuthentication { + // This should never be actually used. + // This is only needed when using Default::default() (such as unit tests), + // as serde requires `azure_credential_kind` to be specified. + fn default() -> Self { + Self::Specific(SpecificAzureCredential::ManagedIdentity { + user_assigned_managed_identity_id: None, + user_assigned_managed_identity_id_type: None, + }) + } +} + +#[configurable_component] +#[derive(Clone, Debug, Eq, PartialEq)] +#[serde(deny_unknown_fields, rename_all = "snake_case")] +#[derive(Default)] +/// User Assigned Managed Identity Types. +pub enum UserAssignedManagedIdentityIdType { + #[default] + /// Client ID + ClientId, + /// Object ID + ObjectId, + /// Resource ID + ResourceId, +} + +/// Specific Azure credential types. +#[configurable_component] +#[derive(Clone, Debug, Eq, PartialEq)] +#[serde( + tag = "azure_credential_kind", + rename_all = "snake_case", + deny_unknown_fields +)] +pub enum SpecificAzureCredential { + /// Use Azure CLI credentials + #[cfg(not(target_arch = "wasm32"))] + AzureCli {}, + + /// Use certificate credentials + ClientCertificateCredential { + /// The [Azure Tenant ID][azure_tenant_id]. + /// + /// [azure_tenant_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + #[configurable(metadata(docs::examples = "${AZURE_TENANT_ID:?err}"))] + azure_tenant_id: String, + + /// The [Azure Client ID][azure_client_id]. + /// + /// [azure_client_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + #[configurable(metadata(docs::examples = "${AZURE_CLIENT_ID:?err}"))] + azure_client_id: String, + + /// PKCS12 certificate with RSA private key. + #[configurable(metadata(docs::examples = "path/to/certificate.pfx"))] + #[configurable(metadata(docs::examples = "${AZURE_CLIENT_CERTIFICATE_PATH:?err}"))] + certificate_path: PathBuf, + + /// The password for the client certificate, if applicable. + #[configurable(metadata(docs::examples = "${AZURE_CLIENT_CERTIFICATE_PASSWORD}"))] + certificate_password: Option, + }, + + /// Use client ID/secret credentials + ClientSecretCredential { + /// The [Azure Tenant ID][azure_tenant_id]. + /// + /// [azure_tenant_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + #[configurable(metadata(docs::examples = "${AZURE_TENANT_ID:?err}"))] + azure_tenant_id: String, + + /// The [Azure Client ID][azure_client_id]. + /// + /// [azure_client_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + #[configurable(metadata(docs::examples = "${AZURE_CLIENT_ID:?err}"))] + azure_client_id: String, + + /// The [Azure Client Secret][azure_client_secret]. + /// + /// [azure_client_secret]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + #[configurable(metadata(docs::examples = "00-00~000000-0000000~0000000000000000000"))] + #[configurable(metadata(docs::examples = "${AZURE_CLIENT_SECRET:?err}"))] + azure_client_secret: SensitiveString, + }, + + /// Use Managed Identity credentials + ManagedIdentity { + /// The User Assigned Managed Identity to use. + #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + #[serde(default, skip_serializing_if = "Option::is_none")] + user_assigned_managed_identity_id: Option, + + /// The type of the User Assigned Managed Identity ID provided (Client ID, Object ID, + /// or Resource ID). Defaults to Client ID. + user_assigned_managed_identity_id_type: Option, + }, + + /// Use Managed Identity with Client Assertion credentials + ManagedIdentityClientAssertion { + /// The User Assigned Managed Identity to use for the managed identity. + #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + #[configurable(metadata( + docs::examples = "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/rg-vector/providers/Microsoft.ManagedIdentity/userAssignedIdentities/id-vector-uami" + ))] + #[serde(default, skip_serializing_if = "Option::is_none")] + user_assigned_managed_identity_id: Option, + + /// The type of the User Assigned Managed Identity ID provided (Client ID, Object ID, or Resource ID). Defaults to Client ID. + user_assigned_managed_identity_id_type: Option, + + /// The target Tenant ID to use. + #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + client_assertion_tenant_id: String, + + /// The target Client ID to use. + #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + client_assertion_client_id: String, + }, + + /// Use Workload Identity credentials + WorkloadIdentity { + /// The [Azure Tenant ID][azure_tenant_id]. Defaults to the value of the environment variable `AZURE_TENANT_ID`. + /// + /// [azure_tenant_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + #[configurable(metadata(docs::examples = "${AZURE_TENANT_ID}"))] + tenant_id: Option, + + /// The [Azure Client ID][azure_client_id]. Defaults to the value of the environment variable `AZURE_CLIENT_ID`. + /// + /// [azure_client_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal + #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] + #[configurable(metadata(docs::examples = "${AZURE_CLIENT_ID}"))] + client_id: Option, + + /// Path of a file containing a Kubernetes service account token. Defaults to the value of the environment variable `AZURE_FEDERATED_TOKEN_FILE`. + #[configurable(metadata( + docs::examples = "/var/run/secrets/azure/tokens/azure-identity-token" + ))] + #[configurable(metadata(docs::examples = "${AZURE_FEDERATED_TOKEN_FILE}"))] + token_file_path: Option, + }, +} + +#[derive(Debug)] +struct ManagedIdentityClientAssertion { + credential: Arc, + scope: String, +} + +#[cfg_attr(target_arch = "wasm32", async_trait::async_trait(?Send))] +#[cfg_attr(not(target_arch = "wasm32"), async_trait::async_trait)] +impl ClientAssertion for ManagedIdentityClientAssertion { + async fn secret(&self, options: Option>) -> azure_core::Result { + Ok(self + .credential + .get_token( + &[&self.scope], + Some(TokenRequestOptions { + method_options: options.unwrap_or_default(), + }), + ) + .await? + .token + .secret() + .to_string()) + } +} + +impl AzureAuthentication { + /// Returns the provider for the credentials based on the authentication mechanism chosen. + pub async fn credential(&self) -> azure_core::Result> { + match self { + Self::Specific(specific) => specific.credential().await, + + #[cfg(test)] + Self::MockCredential => Ok(Arc::new(MockTokenCredential) as Arc), + } + } +} + +impl SpecificAzureCredential { + /// Returns the provider for the credentials based on the specific credential type. + pub async fn credential(&self) -> azure_core::Result> { + let credential: Arc = match self { + #[cfg(not(target_arch = "wasm32"))] + Self::AzureCli {} => AzureCliCredential::new(None)?, + + // requires azure_identity feature 'client_certificate' + Self::ClientCertificateCredential { + azure_tenant_id, + azure_client_id, + certificate_path, + certificate_password, + } => { + let certificate_bytes: Vec = std::fs::read(certificate_path).map_err(|e| { + Error::with_message( + ErrorKind::Credential, + format!( + "Failed to read certificate file {}: {e}", + certificate_path.display() + ), + ) + })?; + + let mut options: ClientCertificateCredentialOptions = + ClientCertificateCredentialOptions::default(); + if let Some(password) = certificate_password { + options.password = Some(password.inner().to_string().into()); + } + + ClientCertificateCredential::new( + azure_tenant_id.clone(), + azure_client_id.clone(), + certificate_bytes.into(), + Some(options), + )? + } + + Self::ClientSecretCredential { + azure_tenant_id, + azure_client_id, + azure_client_secret, + } => { + if azure_tenant_id.is_empty() { + return Err(Error::with_message(ErrorKind::Credential, + "`auth.azure_tenant_id` is blank; either use `auth.azure_credential_kind`, or provide tenant ID, client ID, and secret.".to_string() + )); + } + if azure_client_id.is_empty() { + return Err(Error::with_message(ErrorKind::Credential, + "`auth.azure_client_id` is blank; either use `auth.azure_credential_kind`, or provide tenant ID, client ID, and secret.".to_string() + )); + } + if azure_client_secret.inner().is_empty() { + return Err(Error::with_message(ErrorKind::Credential, + "`auth.azure_client_secret` is blank; either use `auth.azure_credential_kind`, or provide tenant ID, client ID, and secret.".to_string() + )); + } + + let secret: String = azure_client_secret.inner().into(); + ClientSecretCredential::new( + &azure_tenant_id.clone(), + azure_client_id.clone(), + secret.into(), + None, + )? + } + + Self::ManagedIdentity { + user_assigned_managed_identity_id, + user_assigned_managed_identity_id_type, + } => { + let mut options = ManagedIdentityCredentialOptions::default(); + if let Some(id) = user_assigned_managed_identity_id { + options.user_assigned_id = match user_assigned_managed_identity_id_type + .as_ref() + .unwrap_or(&Default::default()) + { + UserAssignedManagedIdentityIdType::ClientId => { + Some(UserAssignedId::ClientId(id.clone())) + } + UserAssignedManagedIdentityIdType::ObjectId => { + Some(UserAssignedId::ObjectId(id.clone())) + } + UserAssignedManagedIdentityIdType::ResourceId => { + Some(UserAssignedId::ResourceId(id.clone())) + } + }; + } + ManagedIdentityCredential::new(Some(options))? + } + + Self::ManagedIdentityClientAssertion { + user_assigned_managed_identity_id, + user_assigned_managed_identity_id_type, + client_assertion_tenant_id, + client_assertion_client_id, + } => { + let mut options = ManagedIdentityCredentialOptions::default(); + if let Some(id) = user_assigned_managed_identity_id { + options.user_assigned_id = match user_assigned_managed_identity_id_type + .as_ref() + .unwrap_or(&Default::default()) + { + UserAssignedManagedIdentityIdType::ClientId => { + Some(UserAssignedId::ClientId(id.clone())) + } + UserAssignedManagedIdentityIdType::ObjectId => { + Some(UserAssignedId::ObjectId(id.clone())) + } + UserAssignedManagedIdentityIdType::ResourceId => { + Some(UserAssignedId::ResourceId(id.clone())) + } + }; + } + let msi: Arc = ManagedIdentityCredential::new(Some(options))?; + let assertion = ManagedIdentityClientAssertion { + credential: msi, + // Future: make this configurable for sovereign clouds? (no way to test...) + scope: "api://AzureADTokenExchange/.default".to_string(), + }; + + ClientAssertionCredential::new( + client_assertion_tenant_id.clone(), + client_assertion_client_id.clone(), + assertion, + None, + )? + } + + Self::WorkloadIdentity { + tenant_id, + client_id, + token_file_path, + } => { + let options = WorkloadIdentityCredentialOptions { + tenant_id: tenant_id.clone(), + client_id: client_id.clone(), + token_file_path: token_file_path.clone(), + ..Default::default() + }; + + WorkloadIdentityCredential::new(Some(options))? + } + }; + Ok(credential) + } +} + +/// An Azurite-supported storage service version, also accepted by the real Azure +/// service (which accepts all previous versions). +const STORAGE_SERVICE_VERSION: &str = "2025-11-05"; + +/// Resolve the authentication for a client from the parsed connection string and the +/// optional explicit authentication config. +/// +/// Returns the token credential to pass to the client constructor (if any) and pushes +/// the SharedKey signing policy into `client_options` when account-key auth is in use. +async fn resolve_credential( + parsed: &ParsedConnectionString, + auth: Option, + client_options: &mut azure_core::http::ClientOptions, +) -> crate::Result>> { + let mut credential: Option> = None; + + match (parsed.auth(), &auth) { + (Auth::None, None) => { + warn!("No authentication method provided, requests will be anonymous."); + } + (Auth::Sas { .. }, None) => { + info!("Using SAS token authentication."); + } + ( + Auth::SharedKey { + account_name, + account_key, + }, + None, + ) => { + info!("Using Shared Key authentication."); + + let policy = SharedKeyAuthorizationPolicy::new( + account_name, + account_key, + String::from(STORAGE_SERVICE_VERSION), + ) + .map_err(|e| format!("Failed to create SharedKey policy: {e}"))?; + client_options.per_call_policies.push(Arc::new(policy)); + } + (Auth::None, Some(AzureAuthentication::Specific(..))) => { + info!("Using Azure Authentication method."); + let credential_result: Arc = + auth.unwrap().credential().await.map_err(|e| { + Error::with_message( + ErrorKind::Credential, + format!("Failed to configure Azure Authentication: {e}"), + ) + })?; + credential = Some(credential_result); + } + (Auth::Sas { .. }, Some(AzureAuthentication::Specific(..))) => { + return Err(Box::new(Error::with_message( + ErrorKind::Credential, + "Cannot use both SAS token and another Azure Authentication method at the same time", + ))); + } + (Auth::SharedKey { .. }, Some(AzureAuthentication::Specific(..))) => { + return Err(Box::new(Error::with_message( + ErrorKind::Credential, + "Cannot use both Shared Key and another Azure Authentication method at the same time", + ))); + } + #[cfg(test)] + (Auth::None, Some(AzureAuthentication::MockCredential)) => { + warn!("Using mock token credential authentication."); + credential = Some(auth.unwrap().credential().await.unwrap()); + } + #[cfg(test)] + (_, Some(AzureAuthentication::MockCredential)) => { + return Err(Box::new(Error::with_message( + ErrorKind::Credential, + "Cannot use both connection string auth and mock credential at the same time", + ))); + } + } + + Ok(credential) +} + +/// Build the HTTP transport for an Azure client, honoring Vector's proxy configuration +/// and any additional CA certificate. +fn build_transport( + url: &Url, + proxy: &crate::config::ProxyConfig, + tls: Option<&AzureBlobTlsConfig>, +) -> crate::Result { + // Use reqwest v0.13 since Azure SDK only implements HttpClient for reqwest::Client v0.13 + let mut reqwest_builder = reqwest_13::ClientBuilder::new(); + let bypass_proxy = { + let host = url.host_str().unwrap_or(""); + let port = url.port(); + proxy.no_proxy.matches(host) + || port + .map(|p| proxy.no_proxy.matches(&format!("{}:{}", host, p))) + .unwrap_or(false) + }; + if bypass_proxy || !proxy.enabled { + // Ensure no proxy (and disable any potential system proxy auto-detection) + reqwest_builder = reqwest_builder.no_proxy(); + } else { + if let Some(http) = &proxy.http { + let p = reqwest_13::Proxy::http(http) + .map_err(|e| format!("Invalid HTTP proxy URL: {e}"))?; + // If credentials are embedded in the proxy URL, reqwest will handle them. + reqwest_builder = reqwest_builder.proxy(p); + } + if let Some(https) = &proxy.https { + let p = reqwest_13::Proxy::https(https) + .map_err(|e| format!("Invalid HTTPS proxy URL: {e}"))?; + // If credentials are embedded in the proxy URL, reqwest will handle them. + reqwest_builder = reqwest_builder.proxy(p); + } + } + + if let Some(AzureBlobTlsConfig { ca_file }) = tls + && let Some(ca_file) = ca_file + { + let mut buf = Vec::new(); + File::open(ca_file)?.read_to_end(&mut buf)?; + let cert = reqwest_13::Certificate::from_pem(&buf)?; + + warn!("Adding TLS root certificate from {}", ca_file.display()); + reqwest_builder = reqwest_builder.add_root_certificate(cert); + } + + Ok(azure_core::http::Transport::new(std::sync::Arc::new( + reqwest_builder + .build() + .map_err(|e| format!("Failed to build reqwest client: {e}"))?, + ))) +} + +/// Builds an Azure Blob Storage container client from a connection string, +/// optionally with an explicit authentication method. +/// +/// Supports both custom blob endpoints (e.g. Azurite) and standard Azure endpoints. +pub async fn build_client( + auth: Option, + connection_string: String, + container_name: String, + proxy: &crate::config::ProxyConfig, + tls: Option, +) -> crate::Result> { + // Parse connection string without legacy SDK + let parsed = ParsedConnectionString::parse(&connection_string) + .map_err(|e| format!("Invalid connection string: {e}"))?; + // Compose container URL (SAS appended if present) + let container_url = parsed + .container_url(&container_name) + .map_err(|e| format!("Failed to build container URL: {e}"))?; + let url = Url::parse(&container_url).map_err(|e| format!("Invalid container URL: {e}"))?; + + let mut options = BlobContainerClientOptions::default(); + let credential = resolve_credential(&parsed, auth, &mut options.client_options).await?; + options.client_options.transport = Some(build_transport(&url, proxy, tls.as_ref())?); + + let client = + BlobContainerClient::new(url, credential, Some(options)).map_err(|e| format!("{e}"))?; + Ok(Arc::new(client)) +} + +/// Builds an Azure Storage Queue client from a connection string, +/// optionally with an explicit authentication method. +/// +/// Supports both custom queue endpoints (e.g. Azurite) and standard Azure endpoints. +#[cfg(feature = "sources-azure_blob")] +pub async fn build_queue_client( + auth: Option, + connection_string: &str, + queue_name: &str, + proxy: &crate::config::ProxyConfig, + tls: Option, +) -> crate::Result> { + let parsed = ParsedConnectionString::parse(connection_string) + .map_err(|e| format!("Invalid connection string: {e}"))?; + // Compose queue URL (SAS appended if present) + let queue_url = parsed + .queue_url(queue_name) + .map_err(|e| format!("Failed to build queue URL: {e}"))?; + let url = Url::parse(&queue_url).map_err(|e| format!("Invalid queue URL: {e}"))?; + + let mut options = azure_storage_queue::QueueClientOptions { + // Pin an Azurite-supported service version; the real service accepts older + // versions, and the SharedKey policy signs with the same version. + version: String::from(STORAGE_SERVICE_VERSION), + ..Default::default() + }; + let credential = resolve_credential(&parsed, auth, &mut options.client_options).await?; + options.client_options.transport = Some(build_transport(&url, proxy, tls.as_ref())?); + + let client = azure_storage_queue::QueueClient::new(url, credential, Some(options)) + .map_err(|e| format!("{e}"))?; + Ok(Arc::new(client)) +} + +#[cfg(test)] +#[derive(Debug)] +struct MockTokenCredential; + +#[cfg(test)] +#[async_trait::async_trait] +impl TokenCredential for MockTokenCredential { + async fn get_token( + &self, + scopes: &[&str], + _options: Option>, + ) -> azure_core::Result { + let Some(scope) = scopes.first() else { + return Err(Error::with_message( + ErrorKind::Credential, + "no scopes were provided", + )); + }; + + // serde_json sometimes does and sometimes doesn't preserve order, be careful to sort + // the claims in alphabetical order to ensure a consistent base64 encoding for testing + let jwt = serde_json::json!({ + "aud": scope.strip_suffix("/.default").unwrap_or(*scope), + "exp": 2147483647, + "iat": 0, + "iss": "https://sts.windows.net/", + "nbf": 0, + }); + + // JWTs do not include standard base64 padding. + // this seemed cleaner than importing a new crates just for this function + let jwt_base64 = format!( + "e30.{}.", + BASE64_STANDARD + .encode(serde_json::to_string(&jwt).unwrap()) + .trim_end_matches("=") + ) + .to_string(); + + warn!( + "Using mock token credential, JWT: {}, base64: {}", + serde_json::to_string(&jwt).unwrap(), + jwt_base64 + ); + + Ok(azure_core::credentials::AccessToken::new( + jwt_base64, + azure_core::time::OffsetDateTime::now_utc() + std::time::Duration::from_secs(3600), + )) + } +} + +#[cfg(test)] +#[tokio::test] +async fn azure_mock_token_credential_test() { + let credential = MockTokenCredential; + let access_token = credential + .get_token(&["https://example.com/.default"], None) + .await + .expect("valid credential should return a token"); + assert_eq!( + access_token.token.secret(), + "e30.eyJhdWQiOiJodHRwczovL2V4YW1wbGUuY29tIiwiZXhwIjoyMTQ3NDgzNjQ3LCJpYXQiOjAsImlzcyI6Imh0dHBzOi8vc3RzLndpbmRvd3MubmV0LyIsIm5iZiI6MH0." + ); +} + +#[cfg(all(test, feature = "azure-blob-integration-tests"))] +mod azure_integration_tests { + use azure_storage_queue::models::{ + QueueClientReceiveMessagesOptions, QueueMessage, ReceivedMessages, + }; + + use super::*; + + /// End-to-end SharedKey signing spike for the queue client against Azurite: + /// create (zero-body PUT), send, receive (query-parameter canonicalization), + /// and delete (zero-body DELETE) must all authorize correctly. + #[tokio::test] + async fn azure_queue_client_shared_key_roundtrip() { + let address = std::env::var("AZURITE_ADDRESS").unwrap_or_else(|_| "localhost".to_string()); + let connection_string = format!( + "UseDevelopmentStorage=true;DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;QueueEndpoint=http://{address}:10001/devstoreaccount1;" + ); + let queue_name = format!("spike-{}", rand::random::()); + let client = build_queue_client( + None, + &connection_string, + &queue_name, + &crate::config::ProxyConfig::default(), + None, + ) + .await + .expect("Failed to build queue client"); + + // Zero-body PUT: exercises the Content-Length empty-string signing rule. + client.create(None).await.expect("Failed to create queue"); + + let message = QueueMessage { + message_text: Some("spike".to_string()), + }; + client + .send_message(message.try_into().expect("infallible"), None) + .await + .expect("Failed to send message"); + + // Query parameters: exercises canonicalized-resource query handling. + let received: ReceivedMessages = client + .receive_messages(Some(QueueClientReceiveMessagesOptions { + number_of_messages: Some(10), + visibility_timeout: Some(5), + ..Default::default() + })) + .await + .expect("Failed to receive messages") + .into_model() + .expect("Failed to parse received messages"); + + let items = received.items.unwrap_or_default(); + assert_eq!(items.len(), 1, "expected exactly one message"); + let msg = &items[0]; + assert_eq!(msg.message_text.as_deref(), Some("spike")); + + // Zero-body DELETE with pop-receipt query parameter. + client + .delete_message( + msg.message_id.as_deref().expect("message id present"), + msg.pop_receipt.as_deref().expect("pop receipt present"), + None, + ) + .await + .expect("Failed to delete message"); + } +} diff --git a/src/sinks/azure_common/connection_string.rs b/src/azure/connection_string.rs similarity index 79% rename from src/sinks/azure_common/connection_string.rs rename to src/azure/connection_string.rs index 391f9d196a9dd..c42047b3f77e7 100644 --- a/src/sinks/azure_common/connection_string.rs +++ b/src/azure/connection_string.rs @@ -45,6 +45,7 @@ Examples: "UseDevelopmentStorage=true;DefaultEndpointsProtocol=http;AccountName=devstoreaccount1" Container URL: */ +#![allow(missing_docs)] use std::collections::HashMap; @@ -95,6 +96,7 @@ pub struct ParsedConnectionString { pub default_endpoints_protocol: Option, pub endpoint_suffix: Option, pub blob_endpoint: Option, + pub queue_endpoint: Option, pub use_development_storage: bool, pub development_storage_proxy_uri: Option, } @@ -131,6 +133,7 @@ impl ParsedConnectionString { .map(|s| s.to_ascii_lowercase()), endpoint_suffix: map.get("endpointsuffix").cloned(), blob_endpoint: map.get("blobendpoint").cloned(), + queue_endpoint: map.get("queueendpoint").cloned(), use_development_storage: map .get("usedevelopmentstorage") .map(|v| v.eq_ignore_ascii_case("true")) @@ -234,6 +237,55 @@ impl ParsedConnectionString { )) } + /// Build the base Queue endpoint URL (no queue path). + /// + /// Resolution order mirrors [`Self::blob_account_endpoint`]: + /// 1. QueueEndpoint (as-is) + /// 2. Development storage synthesized URL: `{proto}://127.0.0.1:10001/{account}` + /// If DevelopmentStorageProxyUri is present, it will be used instead of 127.0.0.1:10001. + /// 3. Public cloud synthesized URL: `{proto}://{account}.queue.{suffix}` + pub fn queue_account_endpoint(&self) -> Result { + if let Some(explicit) = self.queue_endpoint.as_ref() { + return Ok(explicit.clone()); + } + + let account_name = self + .account_name + .as_ref() + .ok_or(ConnectionStringError::MissingAccountName)?; + + let proto = self.default_protocol(); + + if self.use_development_storage { + // If the proxy URI is provided, use it. Otherwise default to 127.0.0.1:10001 + let host = self + .development_storage_proxy_uri + .as_deref() + .map(|s| s.trim_end_matches('/').to_string()) + .unwrap_or_else(|| "127.0.0.1:10001".to_string()); + + let base = if host.starts_with("http://") || host.starts_with("https://") { + format!("{}/{}", trim_trailing_slash(&host), account_name) + } else { + format!("{proto}://{host}/{}", account_name) + }; + return Ok(base); + } + + // Public cloud-style base + let suffix = self.endpoint_suffix(); + Ok(format!("{proto}://{}.queue.{}", account_name, suffix)) + } + + /// Build a queue URL, optionally appending SAS if present. + pub fn queue_url(&self, queue: &str) -> Result { + let base = self.queue_account_endpoint()?; + Ok(append_query_segment( + &format!("{}/{}", trim_trailing_slash(&base), queue), + self.shared_access_signature.as_deref(), + )) + } + /// Build a blob URL, optionally appending SAS if present. pub fn blob_url(&self, container: &str, blob: &str) -> Result { // Build the base container URL without SAS, then append the blob path, @@ -289,6 +341,54 @@ fn encode_path_segment(seg: &str) -> String { mod tests { use super::*; + #[test] + fn queue_endpoint_public_cloud() { + let cs = "DefaultEndpointsProtocol=https;AccountName=myacct;AccountKey=base64==;EndpointSuffix=core.windows.net"; + let parsed = ParsedConnectionString::parse(cs).unwrap(); + assert_eq!( + parsed.queue_account_endpoint().unwrap(), + "https://myacct.queue.core.windows.net" + ); + assert_eq!( + parsed.queue_url("events").unwrap(), + "https://myacct.queue.core.windows.net/events" + ); + } + + #[test] + fn queue_endpoint_explicit_overrides_dev_storage() { + let cs = "UseDevelopmentStorage=true;DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=key==;QueueEndpoint=http://azurite:10001/devstoreaccount1;"; + let parsed = ParsedConnectionString::parse(cs).unwrap(); + assert_eq!( + parsed.queue_account_endpoint().unwrap(), + "http://azurite:10001/devstoreaccount1" + ); + assert_eq!( + parsed.queue_url("events").unwrap(), + "http://azurite:10001/devstoreaccount1/events" + ); + } + + #[test] + fn queue_endpoint_dev_storage_default() { + let cs = "UseDevelopmentStorage=true;AccountName=devstoreaccount1;AccountKey=key=="; + let parsed = ParsedConnectionString::parse(cs).unwrap(); + assert_eq!( + parsed.queue_account_endpoint().unwrap(), + "http://127.0.0.1:10001/devstoreaccount1" + ); + } + + #[test] + fn queue_url_appends_sas() { + let cs = "AccountName=myacct;SharedAccessSignature=sv=2020&sig=abc;"; + let parsed = ParsedConnectionString::parse(cs).unwrap(); + assert_eq!( + parsed.queue_url("events").unwrap(), + "https://myacct.queue.core.windows.net/events?sv=2020&sig=abc" + ); + } + #[test] fn parse_access_key_public_cloud() { let cs = "DefaultEndpointsProtocol=https;AccountName=myacct;AccountKey=base64==;EndpointSuffix=core.windows.net"; diff --git a/src/azure/mod.rs b/src/azure/mod.rs new file mode 100644 index 0000000000000..ccf2834c81b90 --- /dev/null +++ b/src/azure/mod.rs @@ -0,0 +1,8 @@ +//! Shared Azure client infrastructure used by both sources and sinks. + +/// Client construction: authentication, credentials, and HTTP transport assembly. +pub mod client; +/// Azure Storage connection-string parsing and endpoint resolution. +pub mod connection_string; +/// SharedKey request signing for the new Azure SDK pipeline. +pub mod shared_key_policy; diff --git a/src/sinks/azure_common/shared_key_policy.rs b/src/azure/shared_key_policy.rs similarity index 92% rename from src/sinks/azure_common/shared_key_policy.rs rename to src/azure/shared_key_policy.rs index f684f19a00a65..5d6c173dd4940 100644 --- a/src/sinks/azure_common/shared_key_policy.rs +++ b/src/azure/shared_key_policy.rs @@ -1,3 +1,6 @@ +//! SharedKey authorization as an `azure_core` pipeline policy. +#![allow(missing_docs)] + use std::{collections::BTreeMap, fmt::Write as _, sync::Arc}; use async_trait::async_trait; @@ -118,9 +121,18 @@ impl SharedKeyAuthorizationPolicy { } s.push('\n'); - // Content-Length (include value if present; keep "0") - if let Some(v) = header("Content-Length") { - s.push_str(v); + // Content-Length: must be the empty string when the content length of the + // request is zero (storage service versions 2015-02-21 and later). + // + // Generated SDK clients do not always set this header themselves β€” for + // body-carrying requests the HTTP transport adds it when sending β€” so fall + // back to the actual body length to sign the value the service will see. + let content_length = header("Content-Length") + .map(str::to_string) + .or_else(|| req.body().len().map(|l| l.to_string())) + .unwrap_or_default(); + if content_length != "0" { + s.push_str(&content_length); } s.push('\n'); diff --git a/src/common/datadog.rs b/src/common/datadog.rs index 9b84a2f8351dc..05638b474e940 100644 --- a/src/common/datadog.rs +++ b/src/common/datadog.rs @@ -105,7 +105,7 @@ fn compute_api_endpoint(endpoint: &str) -> String { // https://github.com/DataDog/datadog-agent/blob/cdcf0fc809b9ac1cd6e08057b4971c7dbb8dbe30/comp/forwarder/defaultforwarder/forwarder_health.go#L45-L47 // https://github.com/DataDog/datadog-agent/blob/cdcf0fc809b9ac1cd6e08057b4971c7dbb8dbe30/comp/forwarder/defaultforwarder/forwarder_health.go#L188-L190 static DOMAIN_REGEX: LazyLock = LazyLock::new(|| { - Regex::new(r"(?:[a-z]{2}\d\.)?(datadoghq\.[a-z]+|ddog-gov\.com)/*$") + Regex::new(r"((?:[a-z]{2}\d\.)?(?:datadoghq\.[a-z]+|ddog-gov\.com))/*$") .expect("Could not build Datadog domain regex") }); @@ -175,6 +175,27 @@ mod tests { ); } + #[test] + fn preserves_site_prefix_in_api_endpoint() { + for (prefix, tld) in [ + ("us3", "com"), + ("us5", "com"), + ("ap1", "com"), + ("eu1", "eu"), + ] { + assert_eq!( + compute_api_endpoint(&format!( + "https://http-intake.logs.{prefix}.datadoghq.{tld}" + )), + format!("https://api.{prefix}.datadoghq.{tld}") + ); + } + assert_eq!( + compute_api_endpoint("https://1-2-3-observability-pipelines.agent.us3.datadoghq.com"), + "https://api.us3.datadoghq.com" + ); + } + #[test] fn gets_correct_api_base_endpoint() { assert_eq!( @@ -189,5 +210,12 @@ mod tests { get_api_base_endpoint(Some("https://logs.datadoghq.eu"), DD_US_SITE), "https://api.datadoghq.eu" ); + assert_eq!( + get_api_base_endpoint( + Some("https://http-intake.logs.us3.datadoghq.com"), + DD_US_SITE + ), + "https://api.us3.datadoghq.com" + ); } } diff --git a/src/common/http/server_auth.rs b/src/common/http/server_auth.rs index 7d482796f99f3..57b6ded3fbb1d 100644 --- a/src/common/http/server_auth.rs +++ b/src/common/http/server_auth.rs @@ -12,6 +12,7 @@ use vector_config::configurable_component; use vector_lib::{ TimeZone, compile_vrl, event::{Event, LogEvent, VrlTarget}, + lookup::OwnedTargetPath, sensitive_string::SensitiveString, }; use vector_vrl_metrics::MetricsStorage; @@ -54,7 +55,7 @@ pub enum HttpServerAuthConfig { /// Custom authentication using VRL code. /// - /// Takes in request and validates it using VRL code. + /// Takes in request and validates it using VRL code. The VRL program must return a boolean. Custom { /// The VRL boolean expression. source: String, @@ -151,7 +152,9 @@ impl HttpServerAuthConfig { let mut config = CompileConfig::default(); config.set_custom(enrichment_tables.clone()); config.set_custom(metrics_storage.clone()); - config.set_read_only(); + // Lock the event body (.field) as read-only, but leave metadata (%field) writable + // so the VRL program can enrich authenticated events via %field = value. + config.set_read_only_path(OwnedTargetPath::event_root(), true); let CompilationResult { program, @@ -182,7 +185,9 @@ impl HttpServerAuthConfig { pub enum HttpServerAuthMatcher { /// Matcher for comparing exact value of Authorization header AuthHeader(HeaderValue, &'static str), - /// Matcher for running VRL script for requests, to allow for custom validation + /// Matcher for running VRL script for requests, to allow for custom validation. + /// Metadata (`%field`) writes in the program are extracted and returned to the caller + /// for injection into authenticated events. Vrl { /// Compiled VRL script program: Program, @@ -190,18 +195,19 @@ pub enum HttpServerAuthMatcher { } impl HttpServerAuthMatcher { - /// Compares passed headers to the matcher + /// Validates the request. Returns `Ok(Some(enrichment))` when auth passes and the VRL program + /// wrote `%field` values; returns `Ok(None)` when auth passes with no metadata enrichment. pub fn handle_auth( &self, address: Option<&SocketAddr>, headers: &HeaderMap, path: &str, - ) -> Result<(), ErrorMessage> { + ) -> Result, ErrorMessage> { match self { HttpServerAuthMatcher::AuthHeader(expected, err_message) => { if let Some(header) = headers.get(AUTHORIZATION) { if expected == header { - Ok(()) + Ok(None) } else { Err(ErrorMessage::new( StatusCode::UNAUTHORIZED, @@ -227,7 +233,7 @@ impl HttpServerAuthMatcher { headers: &HeaderMap, path: &str, program: &Program, - ) -> Result<(), ErrorMessage> { + ) -> Result, ErrorMessage> { let mut target = VrlTarget::new( Event::Log(LogEvent::from_map( ObjectMap::from([ @@ -263,16 +269,22 @@ impl HttpServerAuthMatcher { warn!("Handling auth failed: {}", e); ErrorMessage::new(StatusCode::UNAUTHORIZED, "Auth failed".to_owned()) })? { - vrl::core::Value::Boolean(result) => { - if result { - Ok(()) + vrl::core::Value::Boolean(true) => { + let enrichment = if let VrlTarget::LogEvent(_, metadata) = &target { + metadata + .value() + .as_object() + .filter(|m| !m.is_empty()) + .cloned() } else { - Err(ErrorMessage::new( - StatusCode::UNAUTHORIZED, - "Auth failed".to_owned(), - )) - } + None + }; + Ok(enrichment) } + vrl::core::Value::Boolean(false) => Err(ErrorMessage::new( + StatusCode::UNAUTHORIZED, + "Auth failed".to_owned(), + )), _ => Err(ErrorMessage::new( StatusCode::UNAUTHORIZED, "Invalid return value".to_owned(), @@ -643,4 +655,75 @@ mod tests { assert_eq!(401, error.code()); assert_eq!("Auth failed", error.message()); } + + // Backward-compat: existing `custom` scripts that don't write metadata still work and return + // Ok(None) β€” no enrichment, no change in behavior. + #[test] + fn custom_auth_matcher_returns_none_enrichment_when_no_metadata_written() { + let custom_auth = HttpServerAuthConfig::Custom { + source: r#".headers.authorization == "Bearer token""#.to_string(), + }; + + let matcher = custom_auth + .build(&Default::default(), &Default::default()) + .unwrap(); + + let mut headers = HeaderMap::new(); + headers.insert(AUTHORIZATION, HeaderValue::from_static("Bearer token")); + let (_guard, addr) = next_addr(); + let result = matcher.handle_auth(Some(&addr), &headers, "/"); + + assert!(result.is_ok()); + assert_eq!( + None, + result.unwrap(), + "no metadata written => no enrichment" + ); + } + + // Existing `custom` scripts that write metadata via `%field = value` now enrich events. + #[test] + fn custom_auth_matcher_returns_enrichment_when_metadata_written() { + let custom_auth = HttpServerAuthConfig::Custom { + source: indoc! {r#" + %tenant_id = "acme" + true + "#} + .to_string(), + }; + + let matcher = custom_auth + .build(&Default::default(), &Default::default()) + .unwrap(); + + let headers = HeaderMap::new(); + let (_guard, addr) = next_addr(); + let result = matcher.handle_auth(Some(&addr), &headers, "/"); + + assert!(result.is_ok()); + let enrichment = result.unwrap().expect("expected enrichment map"); + assert_eq!( + enrichment.get("tenant_id").cloned(), + Some(vrl::core::Value::from("acme")), + ); + } + + // Existing `custom` scripts still cannot mutate event body fields. + #[test] + fn custom_auth_build_fails_when_event_body_write_attempted() { + let custom_auth = HttpServerAuthConfig::Custom { + source: indoc! {r#" + .new_field = "value" + true + "#} + .to_string(), + }; + + assert!( + custom_auth + .build(&Default::default(), &Default::default()) + .is_err(), + "writing to event body (.field) must be rejected at compile time" + ); + } } diff --git a/src/config/api.rs b/src/config/api.rs index 5f99618c166f6..a421fe9f2e1b6 100644 --- a/src/config/api.rs +++ b/src/config/api.rs @@ -11,7 +11,7 @@ use vector_lib::configurable::configurable_component; #[derive(Clone, Copy, Debug, Eq, PartialEq)] #[serde(default, deny_unknown_fields)] pub struct Options { - /// Whether the GraphQL API is enabled for this Vector instance. + /// Whether the API is enabled for this Vector instance. #[serde(default = "default_enabled")] #[configurable(metadata(docs::common = true, docs::required = false))] pub enabled: bool, @@ -24,21 +24,6 @@ pub struct Options { #[configurable(metadata(docs::examples = "127.0.0.1:1234"))] #[configurable(metadata(docs::common = true, docs::required = false))] pub address: Option, - - /// Whether the [GraphQL Playground](https://github.com/graphql/graphql-playground) is enabled - /// for the API. The Playground is accessible via the `/playground` endpoint - /// of the address set using the `bind` parameter. Note that the `playground` - /// endpoint will only be enabled if the `graphql` endpoint is also enabled. - #[serde(default = "default_playground")] - #[configurable(metadata(docs::common = false, docs::required = false))] - pub playground: bool, - - /// Whether the endpoint for receiving and processing GraphQL queries is - /// enabled for the API. The endpoint is accessible via the `/graphql` - /// endpoint of the address set using the `bind` parameter. - #[serde(default = "default_graphql", skip_serializing_if = "is_true")] - #[configurable(metadata(docs::common = true, docs::required = false))] - pub graphql: bool, } impl_generate_config_from_default!(Options); @@ -47,19 +32,11 @@ impl Default for Options { fn default() -> Self { Self { enabled: default_enabled(), - playground: default_playground(), address: default_address(), - graphql: default_graphql(), } } } -// serde passes struct fields as reference -#[allow(clippy::trivially_copy_pass_by_ref)] -const fn is_true(value: &bool) -> bool { - *value -} - const fn default_enabled() -> bool { false } @@ -71,21 +48,13 @@ pub fn default_address() -> Option { Some(SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 8686)) } -/// Default GraphQL API address -pub fn default_graphql_url() -> Url { +/// Default gRPC API address for `vector top` and other API clients +pub fn default_grpc_url() -> Url { let addr = default_address().unwrap(); - Url::parse(&format!("http://{addr}/graphql")) + Url::parse(&format!("http://{addr}")) .expect("Couldn't parse default API URL. Please report this.") } -const fn default_playground() -> bool { - true -} - -const fn default_graphql() -> bool { - true -} - impl Options { pub fn merge(&mut self, other: Self) -> Result<(), String> { // Merge options @@ -108,8 +77,6 @@ impl Options { let options = Options { address, enabled: self.enabled | other.enabled, - playground: self.playground & other.playground, - graphql: self.graphql & other.graphql, }; *self = options; @@ -122,8 +89,6 @@ fn bool_merge() { let mut a = Options { enabled: true, address: None, - playground: false, - graphql: false, }; a.merge(Options::default()).unwrap(); @@ -133,8 +98,6 @@ fn bool_merge() { Options { enabled: true, address: default_address(), - playground: false, - graphql: false } ); } @@ -145,8 +108,6 @@ fn bind_merge() { let mut a = Options { enabled: true, address: Some(address), - playground: true, - graphql: true, }; a.merge(Options::default()).unwrap(); @@ -156,8 +117,6 @@ fn bind_merge() { Options { enabled: true, address: Some(address), - playground: true, - graphql: true, } ); } diff --git a/src/config/dot_graph.rs b/src/config/dot_graph.rs index 0d3d7ca1e4900..a006837cf672f 100644 --- a/src/config/dot_graph.rs +++ b/src/config/dot_graph.rs @@ -15,15 +15,52 @@ pub struct GraphConfig { /// They are added to the node as provided #[configurable(metadata( docs::additional_props_description = "A single graph node attribute in graphviz DOT language.", - docs::examples = "example_graph_options()" + docs::examples = "example_node_options()" ))] + #[serde(default)] pub node_attributes: HashMap, + + /// Edge attributes to add to the edges linked to this component's node in resulting graph + /// + /// They are added to the edge as provided + #[configurable(metadata( + docs::additional_props_description = "A collection of graph edge attributes in graphviz DOT language, related to a single input component.", + docs::examples = "example_edges_options()" + ))] + #[serde(default)] + pub edge_attributes: HashMap, } -fn example_graph_options() -> HashMap { +#[configurable_component] +#[configurable(metadata(docs::advanced))] +#[derive(Clone, Debug, Default, Eq, PartialEq)] +#[serde(deny_unknown_fields)] +/// A collection of graph edge attributes in graphviz DOT language, related to a single input +/// component. +pub struct EdgeAttributes( + #[configurable(metadata( + docs::additional_props_description = "A single graph edge attribute in graphviz DOT language.", + docs::examples = "example_edge_options()" + ))] + pub HashMap, +); + +fn example_node_options() -> HashMap { HashMap::<_, _>::from_iter([ ("name".to_string(), "Example Node".to_string()), ("color".to_string(), "red".to_string()), ("width".to_string(), "5.0".to_string()), ]) } + +fn example_edges_options() -> HashMap> { + HashMap::<_, _>::from_iter([("example_input".to_string(), example_edge_options())]) +} + +fn example_edge_options() -> HashMap { + HashMap::<_, _>::from_iter([ + ("label".to_string(), "Example Edge".to_string()), + ("color".to_string(), "red".to_string()), + ("width".to_string(), "5.0".to_string()), + ]) +} diff --git a/src/config/mod.rs b/src/config/mod.rs index dceade72d6eeb..4a24954926c13 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -446,11 +446,12 @@ impl TestDefinition { let TestOutput { extract_from, conditions, + expected_event_count, } = old; - (extract_from.to_vec(), conditions) + (extract_from.to_vec(), conditions, expected_event_count) }) - .filter_map(|(extract_from, conditions)| { + .filter_map(|(extract_from, conditions, expected_event_count)| { let mut outputs = Vec::new(); for from in extract_from { if no_outputs_from.contains(&from) { @@ -471,6 +472,7 @@ impl TestDefinition { Some(TestOutput { extract_from: outputs.into(), conditions, + expected_event_count, }) } }) @@ -525,6 +527,7 @@ impl TestDefinition { .collect::>() .into(), conditions: old.conditions, + expected_event_count: old.expected_event_count, }) .collect(); @@ -596,6 +599,15 @@ pub struct TestOutput { /// The conditions to run against the output to validate that they were transformed as expected. pub conditions: Option>, + + /// The expected number of events to be produced by the transform. + /// + /// If specified, the test will fail if the number of events emitted by the + /// transform does not match this value. This check is independent of + /// `conditions` -- the count is verified first, then each condition is + /// evaluated against the output events separately. This is useful for + /// transforms that may emit multiple events. + pub expected_event_count: Option, } #[cfg(all(test, feature = "sources-file", feature = "sinks-console"))] diff --git a/src/config/schema.rs b/src/config/schema.rs index 388765b29e1b9..a5b48945ce401 100644 --- a/src/config/schema.rs +++ b/src/config/schema.rs @@ -30,6 +30,8 @@ pub struct Options { /// Controls how metadata is stored in log events. /// + /// This feature is in beta and behavior may change. + /// /// When set to `false` (legacy mode), metadata fields like `host`, `timestamp`, and `source_type` /// are stored as top-level fields alongside your log data. /// @@ -38,6 +40,10 @@ pub struct Options { /// /// See the [Log Namespacing guide](/guides/level-up/log_namespace/) for detailed information /// about when to use Vector namespace mode and how to migrate from legacy mode. + #[configurable(metadata( + status = "beta", + docs::warnings = "Enabling log namespacing currently does not work when disk buffers are used. Avoid combining `schema.log_namespace = true` with disk buffers until [#18574](https://github.com/vectordotdev/vector/issues/18574) is resolved." + ))] pub log_namespace: Option, } diff --git a/src/config/unit_test/mod.rs b/src/config/unit_test/mod.rs index 4d8c1fa97ad65..5020c0b506629 100644 --- a/src/config/unit_test/mod.rs +++ b/src/config/unit_test/mod.rs @@ -293,14 +293,17 @@ impl UnitTestBuildMetadata { let mut template_sinks = IndexMap::new(); let mut test_result_rxs = Vec::new(); // Add sinks with checks - for (ids, checks) in outputs { + for (ids, built) in outputs { let (tx, rx) = oneshot::channel(); let sink_ids = ids.clone(); let sink_config = UnitTestSinkConfig { test_name: test_name.to_string(), transform_ids: ids.iter().map(|id| id.to_string()).collect(), result_tx: Arc::new(Mutex::new(Some(tx))), - check: UnitTestSinkCheck::Checks(checks), + check: UnitTestSinkCheck::Checks { + conditions: built.conditions, + expected_event_count: built.expected_event_count, + }, }; test_result_rxs.push(rx); @@ -567,10 +570,16 @@ fn build_and_validate_inputs( } } +#[derive(Default)] +pub(super) struct BuiltOutput { + pub(super) expected_event_count: Option, + pub(super) conditions: Vec>, +} + fn build_outputs( test_outputs: &[TestOutput], -) -> Result, Vec>>, Vec> { - let mut outputs: IndexMap, Vec>> = IndexMap::new(); +) -> Result, BuiltOutput>, Vec> { + let mut outputs: IndexMap, BuiltOutput> = IndexMap::new(); let mut errors = Vec::new(); for output in test_outputs { @@ -590,10 +599,47 @@ fn build_outputs( } } + let expected_event_count = output.expected_event_count; + if expected_event_count == Some(0) && !conditions.is_empty() { + errors.push(format!( + "output for {:?} has expected_event_count of 0 but also defines conditions; \ + conditions cannot be evaluated when no events are expected", + output.extract_from + )); + } outputs .entry(output.extract_from.clone().to_vec()) - .and_modify(|existing_conditions| existing_conditions.push(conditions.clone())) - .or_insert(vec![conditions.clone()]); + .and_modify(|existing| { + if let (Some(prev), Some(new)) = + (existing.expected_event_count, expected_event_count) + { + if prev != new { + errors.push(format!( + "conflicting expected_event_count for extract_from {:?}: {} vs {}", + output.extract_from, prev, new + )); + } + } else if existing.expected_event_count.is_none() { + existing.expected_event_count = expected_event_count; + } + existing.conditions.push(conditions.clone()); + }) + .or_insert_with(|| BuiltOutput { + expected_event_count, + conditions: vec![conditions.clone()], + }); + } + + // Post-merge validation: after merging entries that share the same + // extract_from, reject any that ended up with expected_event_count of 0 and + // non-empty conditions (which would pass vacuously against zero events). + for (extract_from, built) in &outputs { + if built.expected_event_count == Some(0) && built.conditions.iter().any(|c| !c.is_empty()) { + errors.push(format!( + "output for {extract_from:?} has expected_event_count of 0 but also defines conditions; \ + conditions cannot be evaluated when no events are expected", + )); + } } if errors.is_empty() { diff --git a/src/config/unit_test/tests.rs b/src/config/unit_test/tests.rs index 938286ac9ba9f..6881ba6f75901 100644 --- a/src/config/unit_test/tests.rs +++ b/src/config/unit_test/tests.rs @@ -1298,3 +1298,238 @@ async fn test_glob_input() { let mut tests = build_unit_tests(config).await.unwrap(); assert!(tests.remove(0).run().await.errors.is_empty()); } + +#[tokio::test] +async fn expected_event_count_correct() { + crate::test_util::trace_init(); + + let config: ConfigBuilder = crate::config::format::deserialize( + indoc! {r#" + transforms: + foo: + inputs: + - ignored + type: remap + source: | + . = [{"message": "one"}, {"message": "two"}, {"message": "three"}] + tests: + - name: event count check + inputs: + - insert_at: foo + value: doesnt matter + outputs: + - extract_from: foo + expected_event_count: 3 + conditions: + - type: vrl + source: | + assert!(exists(.message), "message field must exist") + "#}, + crate::config::Format::Yaml, + ) + .unwrap(); + + let mut tests = build_unit_tests(config).await.unwrap(); + assert!(tests.remove(0).run().await.errors.is_empty()); +} + +#[tokio::test] +async fn expected_event_count_wrong() { + crate::test_util::trace_init(); + + let config: ConfigBuilder = crate::config::format::deserialize( + indoc! {r#" + transforms: + foo: + inputs: + - ignored + type: remap + source: | + . = [{"message": "one"}, {"message": "two"}, {"message": "three"}] + tests: + - name: wrong event count + inputs: + - insert_at: foo + value: doesnt matter + outputs: + - extract_from: foo + expected_event_count: 2 + conditions: + - type: vrl + source: | + assert!(exists(.message), "message field must exist") + "#}, + crate::config::Format::Yaml, + ) + .unwrap(); + + let mut tests = build_unit_tests(config).await.unwrap(); + let errors = tests.remove(0).run().await.errors; + assert!(!errors.is_empty()); + assert!( + errors + .iter() + .any(|e| e.contains("expected 2 events") && e.contains("but received 3")), + "expected event count error, got: {errors:?}" + ); +} + +#[tokio::test] +async fn expected_event_count_zero_no_conditions() { + crate::test_util::trace_init(); + + let config: ConfigBuilder = crate::config::format::deserialize( + indoc! {r#" + transforms: + foo: + inputs: + - ignored + type: remap + source: | + if .message == "drop me" { + abort + } + tests: + - name: zero events expected + inputs: + - insert_at: foo + value: "drop me" + outputs: + - extract_from: foo + expected_event_count: 0 + "#}, + crate::config::Format::Yaml, + ) + .unwrap(); + + let mut tests = build_unit_tests(config).await.unwrap(); + assert!(tests.remove(0).run().await.errors.is_empty()); +} + +#[tokio::test] +async fn expected_event_count_zero_with_conditions_rejected() { + crate::test_util::trace_init(); + + let config: ConfigBuilder = crate::config::format::deserialize( + indoc! {r#" + transforms: + foo: + inputs: + - ignored + type: remap + source: | + if .message == "drop me" { + abort + } + tests: + - name: zero with conditions + inputs: + - insert_at: foo + value: "drop me" + outputs: + - extract_from: foo + expected_event_count: 0 + conditions: + - type: vrl + source: | + assert!(exists(.message), "unreachable") + "#}, + crate::config::Format::Yaml, + ) + .unwrap(); + + let errs = build_unit_tests(config).await.err().unwrap(); + assert!( + errs.iter() + .any(|e| e.contains("expected_event_count of 0") && e.contains("conditions")), + "expected config error about zero count with conditions, got: {errs:?}" + ); +} + +#[tokio::test] +async fn expected_event_count_conflicting_values() { + crate::test_util::trace_init(); + + let config: ConfigBuilder = crate::config::format::deserialize( + indoc! {r#" + transforms: + foo: + inputs: + - ignored + type: remap + source: | + . = [{"message": "one"}, {"message": "two"}] + tests: + - name: conflicting counts + inputs: + - insert_at: foo + value: doesnt matter + outputs: + - extract_from: foo + expected_event_count: 2 + conditions: + - type: vrl + source: | + assert!(exists(.message), "ok") + - extract_from: foo + expected_event_count: 5 + conditions: + - type: vrl + source: | + assert!(exists(.message), "ok") + "#}, + crate::config::Format::Yaml, + ) + .unwrap(); + + let errs = build_unit_tests(config).await.err().unwrap(); + assert!( + errs.iter() + .any(|e| e.contains("conflicting expected_event_count")), + "expected conflicting count error, got: {errs:?}" + ); +} + +#[tokio::test] +async fn expected_event_count_zero_split_with_conditions_rejected() { + crate::test_util::trace_init(); + + // Splitting expected_event_count: 0 and conditions across two output + // entries that share the same extract_from must still be rejected after + // merge, otherwise the conditions would pass vacuously against zero events. + let config: ConfigBuilder = crate::config::format::deserialize( + indoc! {r#" + transforms: + foo: + inputs: + - ignored + type: remap + source: | + if .message == "drop me" { + abort + } + tests: + - name: split zero with conditions + inputs: + - insert_at: foo + value: "drop me" + outputs: + - extract_from: foo + expected_event_count: 0 + - extract_from: foo + conditions: + - type: vrl + source: | + assert!(false, "should never be evaluated") + "#}, + crate::config::Format::Yaml, + ) + .unwrap(); + + let errs = build_unit_tests(config).await.err().unwrap(); + assert!( + errs.iter() + .any(|e| e.contains("expected_event_count of 0") && e.contains("conditions")), + "expected config error about zero count with conditions after merge, got: {errs:?}" + ); +} diff --git a/src/config/unit_test/unit_test_components.rs b/src/config/unit_test/unit_test_components.rs index 02808fb083ee7..b3f09098237da 100644 --- a/src/config/unit_test/unit_test_components.rs +++ b/src/config/unit_test/unit_test_components.rs @@ -117,7 +117,10 @@ impl SourceConfig for UnitTestStreamSourceConfig { #[derive(Clone, Default)] pub enum UnitTestSinkCheck { /// Check all events that are received against the list of conditions. - Checks(Vec>), + Checks { + conditions: Vec>, + expected_event_count: Option, + }, /// Check that no events were received. NoOutputs, @@ -203,8 +206,21 @@ impl StreamSink for UnitTestSink { } match self.check { - UnitTestSinkCheck::Checks(checks) => { - if output_events.is_empty() { + UnitTestSinkCheck::Checks { + conditions: checks, + expected_event_count, + } => { + if let Some(expected) = expected_event_count { + let actual = output_events.len(); + if actual != expected { + result.test_errors.push(format!( + "expected {} events from transforms {:?}, but received {}", + expected, self.transform_ids, actual + )); + } + } + + if output_events.is_empty() && expected_event_count != Some(0) { result .test_errors .push(format!("checks for transforms {:?} failed: no events received. Topology may be disconnected or transform is missing inputs.", self.transform_ids)); diff --git a/src/docker.rs b/src/docker.rs index cd13a4c4c54ce..381ffddac0219 100644 --- a/src/docker.rs +++ b/src/docker.rs @@ -4,12 +4,11 @@ use std::{collections::HashMap, env, path::PathBuf}; use bollard::{ API_DEFAULT_VERSION, Docker, errors::Error as DockerError, - models::HostConfig, + models::{ContainerCreateBody, HostConfig}, query_parameters::{ CreateContainerOptionsBuilder, CreateImageOptionsBuilder, ListImagesOptionsBuilder, RemoveContainerOptions, StartContainerOptions, StopContainerOptions, }, - secret::ContainerCreateBody, }; use futures::StreamExt; use http::uri::Uri; @@ -130,7 +129,7 @@ async fn pull_image(docker: &Docker, image: &str, tag: &str) { .create_image(options, None, None) .for_each(|item| async move { let info = item.unwrap(); - if let Some(error) = info.error { + if let Some(error) = info.error_detail { panic!("{error:?}"); } }) diff --git a/src/enrichment_tables/geoip.rs b/src/enrichment_tables/geoip.rs index 782dbe8179afa..6716b4eaf2216 100644 --- a/src/enrichment_tables/geoip.rs +++ b/src/enrichment_tables/geoip.rs @@ -118,9 +118,15 @@ pub struct Geoip { fn lookup_value<'de, A: Deserialize<'de>>( dbreader: &'de Reader>, address: IpAddr, -) -> crate::Result> { +) -> crate::Result> { let result = dbreader.lookup(address)?; - Ok(result.decode::()?) + match result.decode::()? { + Some(data) => { + let network = result.network()?.to_string(); + Ok(Some((data, network))) + } + None => Ok(None), + } } impl Geoip { @@ -174,7 +180,7 @@ impl Geoip { match self.dbkind { DatabaseKind::Asn | DatabaseKind::Isp => { - let data = lookup_value::(&self.dbreader, ip).ok()??; + let (data, network) = lookup_value::(&self.dbreader, ip).ok()??; add_field!("autonomous_system_number", data.autonomous_system_number); add_field!( @@ -183,9 +189,11 @@ impl Geoip { ); add_field!("isp", data.isp); add_field!("organization", data.organization); + add_field!("network", Some(network)); } DatabaseKind::City => { - let data: City = lookup_value::(&self.dbreader, ip).ok()??; + let (data, network): (City, String) = + lookup_value::(&self.dbreader, ip).ok()??; add_field!("city_name", self.take_translation(&data.city.names)); @@ -223,14 +231,16 @@ impl Geoip { subdivision.and_then(|subdivision| subdivision.iso_code) ); add_field!("postal_code", data.postal.code); + add_field!("network", Some(network)); } DatabaseKind::ConnectionType => { - let data = lookup_value::(&self.dbreader, ip).ok()??; + let (data, network) = lookup_value::(&self.dbreader, ip).ok()??; add_field!("connection_type", data.connection_type); + add_field!("network", Some(network)); } DatabaseKind::AnonymousIp => { - let data = lookup_value::(&self.dbreader, ip).ok()??; + let (data, network) = lookup_value::(&self.dbreader, ip).ok()??; add_field!("is_anonymous", data.is_anonymous); add_field!("is_anonymous_vpn", data.is_anonymous_vpn); @@ -238,6 +248,7 @@ impl Geoip { add_field!("is_public_proxy", data.is_public_proxy); add_field!("is_residential_proxy", data.is_residential_proxy); add_field!("is_tor_exit_node", data.is_tor_exit_node); + add_field!("network", Some(network)); } } @@ -367,6 +378,7 @@ mod tests { expected.insert("longitude".into(), Value::from(-1.25)); expected.insert("postal_code".into(), "OX1".into()); expected.insert("metro_code".into(), Value::Null); + expected.insert("network".into(), "2.125.160.216/29".into()); assert_eq!(values, expected); } @@ -403,6 +415,7 @@ mod tests { expected.insert("longitude".into(), Value::from(90.5)); expected.insert("postal_code".into(), Value::Null); expected.insert("metro_code".into(), Value::Null); + expected.insert("network".into(), "67.43.156.0/24".into()); assert_eq!(values, expected); } @@ -426,6 +439,7 @@ mod tests { ); expected.insert("isp".into(), "Verizon Business".into()); expected.insert("organization".into(), "Verizon Business".into()); + expected.insert("network".into(), "208.192.0.0/10".into()); assert_eq!(values, expected); } @@ -442,6 +456,7 @@ mod tests { ); expected.insert("isp".into(), Value::Null); expected.insert("organization".into(), Value::Null); + expected.insert("network".into(), "2600:7000::/24".into()); assert_eq!(values, expected); } @@ -463,6 +478,7 @@ mod tests { let mut expected = ObjectMap::new(); expected.insert("connection_type".into(), "Corporate".into()); + expected.insert("network".into(), "201.243.200.0/24".into()); assert_eq!(values, expected); } @@ -494,6 +510,7 @@ mod tests { expected.insert("is_tor_exit_node".into(), true.into()); expected.insert("is_public_proxy".into(), Value::Null); expected.insert("is_residential_proxy".into(), Value::Null); + expected.insert("network".into(), "101.99.92.179/32".into()); assert_eq!(values, expected); } diff --git a/src/enrichment_tables/memory/internal_events.rs b/src/enrichment_tables/memory/internal_events.rs index 0799c6662a6af..dfd7b8d6f4c78 100644 --- a/src/enrichment_tables/memory/internal_events.rs +++ b/src/enrichment_tables/memory/internal_events.rs @@ -1,6 +1,8 @@ -use metrics::{counter, gauge}; use vector_lib::{ - NamedInternalEvent, configurable::configurable_component, internal_event::InternalEvent, + NamedInternalEvent, + configurable::configurable_component, + counter, gauge, + internal_event::{CounterName, GaugeName, InternalEvent}, }; /// Configuration of internal metrics for enrichment memory table. @@ -26,12 +28,12 @@ impl InternalEvent for MemoryEnrichmentTableRead<'_> { fn emit(self) { if self.include_key_metric_tag { counter!( - "memory_enrichment_table_reads_total", + CounterName::MemoryEnrichmentTableReadsTotal, "key" => self.key.to_owned() ) .increment(1); } else { - counter!("memory_enrichment_table_reads_total",).increment(1); + counter!(CounterName::MemoryEnrichmentTableReadsTotal,).increment(1); } } } @@ -46,12 +48,12 @@ impl InternalEvent for MemoryEnrichmentTableInserted<'_> { fn emit(self) { if self.include_key_metric_tag { counter!( - "memory_enrichment_table_insertions_total", + CounterName::MemoryEnrichmentTableInsertionsTotal, "key" => self.key.to_owned() ) .increment(1); } else { - counter!("memory_enrichment_table_insertions_total",).increment(1); + counter!(CounterName::MemoryEnrichmentTableInsertionsTotal,).increment(1); } } } @@ -64,9 +66,9 @@ pub(crate) struct MemoryEnrichmentTableFlushed { impl InternalEvent for MemoryEnrichmentTableFlushed { fn emit(self) { - counter!("memory_enrichment_table_flushes_total",).increment(1); - gauge!("memory_enrichment_table_objects_count",).set(self.new_objects_count as f64); - gauge!("memory_enrichment_table_byte_size",).set(self.new_byte_size as f64); + counter!(CounterName::MemoryEnrichmentTableFlushesTotal,).increment(1); + gauge!(GaugeName::MemoryEnrichmentTableObjectsCount,).set(self.new_objects_count as f64); + gauge!(GaugeName::MemoryEnrichmentTableByteSize,).set(self.new_byte_size as f64); } } @@ -80,12 +82,12 @@ impl InternalEvent for MemoryEnrichmentTableTtlExpired<'_> { fn emit(self) { if self.include_key_metric_tag { counter!( - "memory_enrichment_table_ttl_expirations", + CounterName::MemoryEnrichmentTableTtlExpirations, "key" => self.key.to_owned() ) .increment(1); } else { - counter!("memory_enrichment_table_ttl_expirations",).increment(1); + counter!(CounterName::MemoryEnrichmentTableTtlExpirations,).increment(1); } } } @@ -100,12 +102,12 @@ impl InternalEvent for MemoryEnrichmentTableReadFailed<'_> { fn emit(self) { if self.include_key_metric_tag { counter!( - "memory_enrichment_table_failed_reads", + CounterName::MemoryEnrichmentTableFailedReads, "key" => self.key.to_owned() ) .increment(1); } else { - counter!("memory_enrichment_table_failed_reads",).increment(1); + counter!(CounterName::MemoryEnrichmentTableFailedReads,).increment(1); } } } @@ -120,12 +122,12 @@ impl InternalEvent for MemoryEnrichmentTableInsertFailed<'_> { fn emit(self) { if self.include_key_metric_tag { counter!( - "memory_enrichment_table_failed_insertions", + CounterName::MemoryEnrichmentTableFailedInsertions, "key" => self.key.to_owned() ) .increment(1); } else { - counter!("memory_enrichment_table_failed_insertions",).increment(1); + counter!(CounterName::MemoryEnrichmentTableFailedInsertions,).increment(1); } } } diff --git a/src/gcp.rs b/src/gcp.rs index 9a0ff365a69ee..89fec86770e0b 100644 --- a/src/gcp.rs +++ b/src/gcp.rs @@ -194,7 +194,7 @@ impl GcpAuthenticator { pub fn spawn_regenerate_token(&self) -> watch::Receiver<()> { let (sender, receiver) = watch::channel(()); - tokio::spawn(self.clone().token_regenerator(sender)); + crate::spawn_in_current_span(self.clone().token_regenerator(sender)); receiver } diff --git a/src/graph.rs b/src/graph.rs index cfdcfffb3c6f5..241833b32f59c 100644 --- a/src/graph.rs +++ b/src/graph.rs @@ -1,9 +1,17 @@ -use std::{collections::HashMap, fmt::Write as _, path::PathBuf}; +use std::{ + collections::{HashMap, HashSet}, + fmt::Write as _, + path::PathBuf, +}; use clap::Parser; use itertools::Itertools; +use vector_lib::{config::OutputId, id::ComponentKey}; -use crate::config; +use crate::config::{ + self, + dot_graph::{EdgeAttributes, GraphConfig}, +}; #[derive(Parser, Debug)] #[command(rename_all = "kebab-case")] @@ -94,6 +102,16 @@ fn node_attributes_to_string(attributes: &HashMap, default_shape attrs.iter().map(|(k, v)| format!("{k}=\"{v}\"")).join(" ") } +fn edge_attributes_to_string(attributes: &EdgeAttributes, default_label: Option<&str>) -> String { + let mut attrs = attributes.0.clone(); + if let Some(default_label) = default_label + && !attrs.contains_key("label") + { + attrs.insert("label".to_string(), default_label.to_string()); + } + attrs.iter().map(|(k, v)| format!("{k}=\"{v}\"")).join(" ") +} + pub(crate) fn cmd(opts: &Opts) -> exitcode::ExitCode { let paths = opts.paths_with_formats(); let paths = match config::process_paths(&paths) { @@ -122,6 +140,43 @@ pub(crate) fn cmd(opts: &Opts) -> exitcode::ExitCode { fn render_dot(config: config::Config) -> exitcode::ExitCode { let mut dot = String::from("digraph {\n"); + let mut written_tables = HashSet::::new(); + + for (id, table) in config + .enrichment_tables + .iter() + .filter_map(|(key, table)| table.as_source(key)) + { + writeln!( + dot, + " \"{}\" [{}]", + id, + node_attributes_to_string(&table.graph.node_attributes, "cylinder") + ) + .expect("write to String never fails"); + written_tables.insert(id); + } + + for (id, table) in config + .enrichment_tables + .iter() + .filter_map(|(key, table)| table.as_sink(key)) + { + if !written_tables.contains(&id) { + writeln!( + dot, + " \"{}\" [{}]", + id, + node_attributes_to_string(&table.graph.node_attributes, "cylinder") + ) + .expect("write to String never fails"); + } + + for input in table.inputs.iter() { + render_dot_edge(&mut dot, &id, input, &table.graph); + } + } + for (id, source) in config.sources() { writeln!( dot, @@ -142,16 +197,7 @@ fn render_dot(config: config::Config) -> exitcode::ExitCode { .expect("write to String never fails"); for input in transform.inputs.iter() { - if let Some(port) = &input.port { - writeln!( - dot, - " \"{}\" -> \"{}\" [label=\"{}\"]", - input.component, id, port - ) - .expect("write to String never fails"); - } else { - writeln!(dot, " \"{input}\" -> \"{id}\"").expect("write to String never fails"); - } + render_dot_edge(&mut dot, id, input, &transform.graph); } } @@ -165,16 +211,7 @@ fn render_dot(config: config::Config) -> exitcode::ExitCode { .expect("write to String never fails"); for input in &sink.inputs { - if let Some(port) = &input.port { - writeln!( - dot, - " \"{}\" -> \"{}\" [label=\"{}\"]", - input.component, id, port - ) - .expect("write to String never fails"); - } else { - writeln!(dot, " \"{input}\" -> \"{id}\"").expect("write to String never fails"); - } + render_dot_edge(&mut dot, id, input, &sink.graph); } } @@ -188,9 +225,67 @@ fn render_dot(config: config::Config) -> exitcode::ExitCode { exitcode::OK } +fn render_dot_edge(into: &mut String, id: &ComponentKey, input: &OutputId, graph: &GraphConfig) { + let edge_attributes = graph + .edge_attributes + .get(&input.to_string()) + .or_else(|| graph.edge_attributes.get(&input.component.to_string())); + if let Some(port) = &input.port { + writeln!( + into, + " \"{}\" -> \"{id}\" [{}]", + input.component, + edge_attributes_to_string( + edge_attributes.unwrap_or(&EdgeAttributes::default()), + Some(port) + ) + ) + .expect("write to String never fails"); + } else if let Some(edge_attributes) = edge_attributes { + writeln!( + into, + " \"{input}\" -> \"{id}\" [{}]", + edge_attributes_to_string(edge_attributes, None) + ) + .expect("write to String never fails"); + } else { + writeln!(into, " \"{input}\" -> \"{id}\"").expect("write to String never fails"); + } +} + fn render_mermaid(config: config::Config) -> exitcode::ExitCode { let mut mermaid = String::from("flowchart TD;\n"); + writeln!(mermaid, "\n %% Enrichment tables").unwrap(); + let mut written_tables = HashSet::::new(); + + for (id, _) in config + .enrichment_tables + .iter() + .filter_map(|(key, table)| table.as_source(key)) + { + writeln!(mermaid, " {id}[({id})]").unwrap(); + written_tables.insert(id); + } + + for (id, table) in config + .enrichment_tables + .iter() + .filter_map(|(key, table)| table.as_sink(key)) + { + if !written_tables.contains(&id) { + writeln!(mermaid, " {id}[({id})]").unwrap(); + } + + for input in table.inputs.iter() { + if let Some(port) = &input.port { + writeln!(mermaid, " {0} -->|{port}| {id}", input.component).unwrap(); + } else { + writeln!(mermaid, " {0} --> {id}", input.component).unwrap(); + } + } + } + writeln!(mermaid, "\n %% Sources").unwrap(); for (id, _) in config.sources() { writeln!(mermaid, " {id}[/{id}/]").unwrap(); diff --git a/src/internal_events/adaptive_concurrency.rs b/src/internal_events/adaptive_concurrency.rs index f815f7d4e2b4b..b43ec8452e20b 100644 --- a/src/internal_events/adaptive_concurrency.rs +++ b/src/internal_events/adaptive_concurrency.rs @@ -1,6 +1,7 @@ use std::time::Duration; -use metrics::{Histogram, histogram}; +use metrics::Histogram; +use vector_lib::{histogram, internal_event::HistogramName}; #[derive(Clone, Copy)] pub struct AdaptiveConcurrencyLimitData { @@ -17,10 +18,10 @@ registered_event! { // These are histograms, as they may have a number of different // values over each reporting interval, and each of those values // is valuable for diagnosis. - limit: Histogram = histogram!("adaptive_concurrency_limit"), - reached_limit: Histogram = histogram!("adaptive_concurrency_reached_limit"), - back_pressure: Histogram = histogram!("adaptive_concurrency_back_pressure"), - past_rtt_mean: Histogram = histogram!("adaptive_concurrency_past_rtt_mean"), + limit: Histogram = histogram!(HistogramName::AdaptiveConcurrencyLimit), + reached_limit: Histogram = histogram!(HistogramName::AdaptiveConcurrencyReachedLimit), + back_pressure: Histogram = histogram!(HistogramName::AdaptiveConcurrencyBackPressure), + past_rtt_mean: Histogram = histogram!(HistogramName::AdaptiveConcurrencyPastRttMean), } fn emit(&self, data: AdaptiveConcurrencyLimitData) { @@ -36,7 +37,7 @@ registered_event! { registered_event! { AdaptiveConcurrencyInFlight => { - in_flight: Histogram = histogram!("adaptive_concurrency_in_flight"), + in_flight: Histogram = histogram!(HistogramName::AdaptiveConcurrencyInFlight), } fn emit(&self, in_flight: u64) { @@ -46,7 +47,7 @@ registered_event! { registered_event! { AdaptiveConcurrencyObservedRtt => { - observed_rtt: Histogram = histogram!("adaptive_concurrency_observed_rtt"), + observed_rtt: Histogram = histogram!(HistogramName::AdaptiveConcurrencyObservedRtt), } fn emit(&self, rtt: Duration) { @@ -56,7 +57,7 @@ registered_event! { registered_event! { AdaptiveConcurrencyAveragedRtt => { - averaged_rtt: Histogram = histogram!("adaptive_concurrency_averaged_rtt"), + averaged_rtt: Histogram = histogram!(HistogramName::AdaptiveConcurrencyAveragedRtt), } fn emit(&self, rtt: Duration) { diff --git a/src/internal_events/aggregate.rs b/src/internal_events/aggregate.rs index 1b70c4350f614..96f48f5de013c 100644 --- a/src/internal_events/aggregate.rs +++ b/src/internal_events/aggregate.rs @@ -1,13 +1,14 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::InternalEvent; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent}, +}; #[derive(Debug, NamedInternalEvent)] pub struct AggregateEventRecorded; impl InternalEvent for AggregateEventRecorded { fn emit(self) { - counter!("aggregate_events_recorded_total").increment(1); + counter!(CounterName::AggregateEventsRecordedTotal).increment(1); } } @@ -16,7 +17,7 @@ pub struct AggregateFlushed; impl InternalEvent for AggregateFlushed { fn emit(self) { - counter!("aggregate_flushes_total").increment(1); + counter!(CounterName::AggregateFlushesTotal).increment(1); } } @@ -25,6 +26,6 @@ pub struct AggregateUpdateFailed; impl InternalEvent for AggregateUpdateFailed { fn emit(self) { - counter!("aggregate_failed_updates").increment(1); + counter!(CounterName::AggregateFailedUpdates).increment(1); } } diff --git a/src/internal_events/amqp.rs b/src/internal_events/amqp.rs index 45acdf201305a..e7864acc44e98 100644 --- a/src/internal_events/amqp.rs +++ b/src/internal_events/amqp.rs @@ -1,8 +1,9 @@ #[cfg(feature = "sources-amqp")] pub mod source { - use metrics::counter; - use vector_lib::NamedInternalEvent; - use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; + use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, + }; #[derive(Debug, NamedInternalEvent)] pub struct AmqpBytesReceived { @@ -18,7 +19,7 @@ pub mod source { protocol = %self.protocol, ); counter!( - "component_received_bytes_total", + CounterName::ComponentReceivedBytesTotal, "protocol" => self.protocol, ) .increment(self.byte_size as u64); @@ -38,7 +39,7 @@ pub mod source { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::REQUEST_FAILED, "stage" => error_stage::RECEIVING, ) @@ -59,7 +60,7 @@ pub mod source { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::ACKNOWLEDGMENT_FAILED, "stage" => error_stage::RECEIVING, ) @@ -80,7 +81,7 @@ pub mod source { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::COMMAND_FAILED, "stage" => error_stage::RECEIVING, ) diff --git a/src/internal_events/apache_metrics.rs b/src/internal_events/apache_metrics.rs index dd5e9c34e4cf5..544dda88ba5a6 100644 --- a/src/internal_events/apache_metrics.rs +++ b/src/internal_events/apache_metrics.rs @@ -1,7 +1,6 @@ -use metrics::counter; use vector_lib::{ - NamedInternalEvent, - internal_event::{InternalEvent, error_stage, error_type}, + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, json_size::JsonSize, }; @@ -19,12 +18,12 @@ impl InternalEvent for ApacheMetricsEventsReceived<'_> { fn emit(self) { trace!(message = "Events received.", count = %self.count, byte_size = %self.byte_size, endpoint = %self.endpoint); counter!( - "component_received_events_total", + CounterName::ComponentReceivedEventsTotal, "endpoint" => self.endpoint.to_owned(), ) .increment(self.count as u64); counter!( - "component_received_event_bytes_total", + CounterName::ComponentReceivedEventBytesTotal, "endpoint" => self.endpoint.to_owned(), ) .increment(self.byte_size.get() as u64); @@ -47,7 +46,7 @@ impl InternalEvent for ApacheMetricsParseError<'_> { endpoint = %self.endpoint, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "stage" => error_stage::PROCESSING, "error_type" => error_type::PARSER_FAILED, "endpoint" => self.endpoint.to_owned(), diff --git a/src/internal_events/api.rs b/src/internal_events/api.rs index 4894fc1de33e6..b9196989471c8 100644 --- a/src/internal_events/api.rs +++ b/src/internal_events/api.rs @@ -1,27 +1,21 @@ use std::net::SocketAddr; -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::InternalEvent; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent}, +}; #[derive(Debug, NamedInternalEvent)] pub struct ApiStarted { pub addr: SocketAddr, - pub playground: bool, - pub graphql: bool, } impl InternalEvent for ApiStarted { fn emit(self) { - let playground = &*format!("http://{}:{}/playground", self.addr.ip(), self.addr.port()); - let graphql = &*format!("http://{}:{}/graphql", self.addr.ip(), self.addr.port()); info!( - message="API server running.", - address = ?self.addr, - playground = %if self.playground { playground } else { "off" }, - graphql = %if self.graphql { graphql } else { "off" } - + message = "API server running.", + address = %self.addr, ); - counter!("api_started_total").increment(1); + counter!(CounterName::ApiStartedTotal).increment(1); } } diff --git a/src/internal_events/aws.rs b/src/internal_events/aws.rs index ff4da6c01e56b..d4c91776d7d55 100644 --- a/src/internal_events/aws.rs +++ b/src/internal_events/aws.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::InternalEvent; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent}, +}; #[derive(NamedInternalEvent)] pub struct AwsBytesSent { @@ -21,7 +22,7 @@ impl InternalEvent for AwsBytesSent { region = ?self.region, ); counter!( - "component_sent_bytes_total", + CounterName::ComponentSentBytesTotal, "protocol" => "https", "region" => region, ) diff --git a/src/internal_events/aws_cloudwatch_logs.rs b/src/internal_events/aws_cloudwatch_logs.rs index 6325d6c71b25d..b4964f25d84fe 100644 --- a/src/internal_events/aws_cloudwatch_logs.rs +++ b/src/internal_events/aws_cloudwatch_logs.rs @@ -1,8 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, error_stage, error_type, }; +use vector_lib::{NamedInternalEvent, counter}; #[derive(Debug, NamedInternalEvent)] pub struct AwsCloudwatchLogsMessageSizeError { @@ -22,7 +21,7 @@ impl InternalEvent for AwsCloudwatchLogsMessageSizeError { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "message_too_long", "error_type" => error_type::ENCODER_FAILED, "stage" => error_stage::PROCESSING, diff --git a/src/internal_events/aws_ec2_metadata.rs b/src/internal_events/aws_ec2_metadata.rs index 43c28aee61272..c59364c75348c 100644 --- a/src/internal_events/aws_ec2_metadata.rs +++ b/src/internal_events/aws_ec2_metadata.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; #[derive(Debug, NamedInternalEvent)] pub struct AwsEc2MetadataRefreshSuccessful; @@ -8,7 +9,7 @@ pub struct AwsEc2MetadataRefreshSuccessful; impl InternalEvent for AwsEc2MetadataRefreshSuccessful { fn emit(self) { debug!(message = "AWS EC2 metadata refreshed."); - counter!("metadata_refresh_successful_total").increment(1); + counter!(CounterName::MetadataRefreshSuccessfulTotal).increment(1); } } @@ -26,12 +27,12 @@ impl InternalEvent for AwsEc2MetadataRefreshError { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::REQUEST_FAILED, "stage" => error_stage::PROCESSING, ) .increment(1); // deprecated - counter!("metadata_refresh_failed_total").increment(1); + counter!(CounterName::MetadataRefreshFailedTotal).increment(1); } } diff --git a/src/internal_events/aws_ecs_metrics.rs b/src/internal_events/aws_ecs_metrics.rs index 86b0d32012713..59127134e2666 100644 --- a/src/internal_events/aws_ecs_metrics.rs +++ b/src/internal_events/aws_ecs_metrics.rs @@ -1,9 +1,8 @@ use std::borrow::Cow; -use metrics::counter; use vector_lib::{ - NamedInternalEvent, - internal_event::{InternalEvent, error_stage, error_type}, + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, json_size::JsonSize, }; @@ -24,12 +23,12 @@ impl InternalEvent for AwsEcsMetricsEventsReceived<'_> { endpoint = %self.endpoint, ); counter!( - "component_received_events_total", + CounterName::ComponentReceivedEventsTotal, "endpoint" => self.endpoint.to_string(), ) .increment(self.count as u64); counter!( - "component_received_event_bytes_total", + CounterName::ComponentReceivedEventBytesTotal, "endpoint" => self.endpoint.to_string(), ) .increment(self.byte_size.get() as u64); @@ -57,9 +56,9 @@ impl InternalEvent for AwsEcsMetricsParseError<'_> { endpoint = %self.endpoint, "Failed to parse response.", ); - counter!("parse_errors_total").increment(1); + counter!(CounterName::ParseErrorsTotal).increment(1); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "stage" => error_stage::PROCESSING, "error_type" => error_type::PARSER_FAILED, "endpoint" => self.endpoint.to_string(), diff --git a/src/internal_events/aws_kinesis.rs b/src/internal_events/aws_kinesis.rs index 9fa2fb4315118..f8a7dcff49de2 100644 --- a/src/internal_events/aws_kinesis.rs +++ b/src/internal_events/aws_kinesis.rs @@ -1,8 +1,8 @@ -/// Used in both `aws_kinesis_streams` and `aws_kinesis_firehose` sinks -use metrics::counter; use vector_lib::NamedInternalEvent; +/// Used in both `aws_kinesis_streams` and `aws_kinesis_firehose` sinks +use vector_lib::counter; use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, error_stage, error_type, }; #[derive(Debug, NamedInternalEvent)] @@ -22,7 +22,7 @@ impl InternalEvent for AwsKinesisStreamNoPartitionKeyError<'_> { ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, ) diff --git a/src/internal_events/aws_kinesis_firehose.rs b/src/internal_events/aws_kinesis_firehose.rs index 6cb4ae43ea51f..58f976839debd 100644 --- a/src/internal_events/aws_kinesis_firehose.rs +++ b/src/internal_events/aws_kinesis_firehose.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; use super::prelude::{http_error_code, io_error_code}; use crate::sources::aws_kinesis_firehose::Compression; @@ -49,7 +50,7 @@ impl InternalEvent for AwsKinesisFirehoseRequestError<'_> { request_id = %self.request_id.unwrap_or(""), ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "stage" => error_stage::RECEIVING, "error_type" => error_type::REQUEST_FAILED, "error_code" => self.error_code, @@ -75,7 +76,7 @@ impl InternalEvent for AwsKinesisFirehoseAutomaticRecordDecodeError { compression = %self.compression, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "stage" => error_stage::PROCESSING, "error_type" => error_type::PARSER_FAILED, "error_code" => io_error_code(&self.error), diff --git a/src/internal_events/aws_sqs.rs b/src/internal_events/aws_sqs.rs index 4bdcc8ef48e6c..c04cb21b117fb 100644 --- a/src/internal_events/aws_sqs.rs +++ b/src/internal_events/aws_sqs.rs @@ -1,11 +1,13 @@ #![allow(dead_code)] // TODO requires optional feature compilation -use metrics::counter; #[cfg(feature = "sources-aws_s3")] pub use s3::*; +use vector_lib::counter; #[cfg(any(feature = "sources-aws_s3", feature = "sources-aws_sqs"))] -use vector_lib::internal_event::{error_stage, error_type}; -use vector_lib::{NamedInternalEvent, internal_event::InternalEvent}; +use vector_lib::{ + NamedInternalEvent, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; #[cfg(feature = "sources-aws_s3")] mod s3 { @@ -15,7 +17,8 @@ mod s3 { BatchResultErrorEntry, DeleteMessageBatchRequestEntry, DeleteMessageBatchResultEntry, SendMessageBatchRequestEntry, SendMessageBatchResultEntry, }; - use metrics::histogram; + use vector_lib::histogram; + use vector_lib::internal_event::HistogramName; use super::*; use crate::sources::aws_s3::sqs::ProcessingError; @@ -34,7 +37,7 @@ mod s3 { duration_ms = %self.duration.as_millis(), ); histogram!( - "s3_object_processing_succeeded_duration_seconds", + HistogramName::S3ObjectProcessingSucceededDurationSeconds, "bucket" => self.bucket.to_owned(), ) .record(self.duration); @@ -55,7 +58,7 @@ mod s3 { duration_ms = %self.duration.as_millis(), ); histogram!( - "s3_object_processing_failed_duration_seconds", + HistogramName::S3ObjectProcessingFailedDurationSeconds, "bucket" => self.bucket.to_owned(), ) .record(self.duration); @@ -79,7 +82,7 @@ mod s3 { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "failed_processing_sqs_message", "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, @@ -100,7 +103,8 @@ mod s3 { .map(|x| x.id.as_str()) .collect::>() .join(", ")); - counter!("sqs_message_delete_succeeded_total").increment(self.message_ids.len() as u64); + counter!(CounterName::SqsMessageDeleteSucceededTotal) + .increment(self.message_ids.len() as u64); } } @@ -122,7 +126,7 @@ mod s3 { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "failed_deleting_some_sqs_messages", "error_type" => error_type::ACKNOWLEDGMENT_FAILED, "stage" => error_stage::PROCESSING, @@ -151,7 +155,7 @@ mod s3 { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "failed_deleting_all_sqs_messages", "error_type" => error_type::ACKNOWLEDGMENT_FAILED, "stage" => error_stage::PROCESSING, @@ -172,7 +176,8 @@ mod s3 { .map(|x| x.id.as_str()) .collect::>() .join(", ")); - counter!("sqs_message_defer_succeeded_total").increment(self.message_ids.len() as u64); + counter!(CounterName::SqsMessageDeferSucceededTotal) + .increment(self.message_ids.len() as u64); } } @@ -194,7 +199,7 @@ mod s3 { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "failed_deferring_some_sqs_messages", "error_type" => error_type::ACKNOWLEDGMENT_FAILED, "stage" => error_stage::PROCESSING, @@ -223,7 +228,7 @@ mod s3 { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "failed_deferring_all_sqs_messages", "error_type" => error_type::ACKNOWLEDGMENT_FAILED, "stage" => error_stage::PROCESSING, @@ -248,7 +253,7 @@ impl InternalEvent for SqsMessageReceiveError<'_, E> { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "failed_fetching_sqs_events", "error_type" => error_type::REQUEST_FAILED, "stage" => error_stage::RECEIVING, @@ -265,8 +270,8 @@ pub struct SqsMessageReceiveSucceeded { impl InternalEvent for SqsMessageReceiveSucceeded { fn emit(self) { trace!(message = "Received SQS messages.", count = %self.count); - counter!("sqs_message_receive_succeeded_total").increment(1); - counter!("sqs_message_received_messages_total").increment(self.count as u64); + counter!(CounterName::SqsMessageReceiveSucceededTotal).increment(1); + counter!(CounterName::SqsMessageReceivedMessagesTotal).increment(self.count as u64); } } @@ -278,7 +283,7 @@ pub struct SqsMessageProcessingSucceeded<'a> { impl InternalEvent for SqsMessageProcessingSucceeded<'_> { fn emit(self) { trace!(message = "Processed SQS message successfully.", message_id = %self.message_id); - counter!("sqs_message_processing_succeeded_total").increment(1); + counter!(CounterName::SqsMessageProcessingSucceededTotal).increment(1); } } @@ -300,7 +305,7 @@ impl InternalEvent for SqsMessageDeleteError<'_, E> { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::WRITER_FAILED, "stage" => error_stage::PROCESSING, ) @@ -322,7 +327,7 @@ impl InternalEvent for SqsS3EventRecordInvalidEventIgnored<'_> { fn emit(self) { warn!(message = "Ignored S3 record in SQS message for an event that was not ObjectCreated.", bucket = %self.bucket, key = %self.key, kind = %self.kind, name = %self.name); - counter!("sqs_s3_event_record_ignored_total", "ignore_type" => "invalid_event_kind") + counter!(CounterName::SqsS3EventRecordIgnoredTotal, "ignore_type" => "invalid_event_kind") .increment(1); } } diff --git a/src/internal_events/azure_queue.rs b/src/internal_events/azure_queue.rs new file mode 100644 index 0000000000000..89cd9a081b3b9 --- /dev/null +++ b/src/internal_events/azure_queue.rs @@ -0,0 +1,219 @@ +#[cfg(feature = "sources-azure_blob")] +pub use azure_blob::*; +use vector_lib::counter; +use vector_lib::internal_event::{CounterName, InternalEvent, error_stage, error_type}; + +#[cfg(feature = "sources-azure_blob")] +mod azure_blob { + use super::*; + use crate::event::Event; + use crate::sources::azure_blob::queue::ProcessingError; + + #[derive(Debug, NamedInternalEvent)] + pub struct QueueMessageProcessingError<'a> { + pub message_id: &'a str, + pub error: &'a ProcessingError, + } + + impl<'a> InternalEvent for QueueMessageProcessingError<'a> { + fn emit(self) { + error!( + message = "Failed to process Queue message.", + message_id = %self.message_id, + error = %self.error, + error_code = "failed_processing_azure_queue_message", + error_type = error_type::PARSER_FAILED, + stage = error_stage::PROCESSING, + internal_log_rate_limit = true, + ); + counter!( + CounterName::ComponentErrorsTotal, + "error_code" => "failed_processing_azure_queue_message", + "error_type" => error_type::PARSER_FAILED, + "stage" => error_stage::PROCESSING, + ) + .increment(1); + } + } + + #[derive(Debug, NamedInternalEvent)] + pub struct InvalidRowEventTypeError<'a> { + pub event: &'a Event, + } + + impl<'a> InternalEvent for InvalidRowEventTypeError<'a> { + fn emit(self) { + error!( + message = "Expected Azure rows as Log Events.", + event = ?self.event, + error_code = "invalid_azure_row_event", + error_type = error_type::CONDITION_FAILED, + stage = error_stage::PROCESSING, + ); + counter!( + CounterName::ComponentErrorsTotal, + "error_code" => "invalid_azure_row_event", + "error_type" => error_type::CONDITION_FAILED, + "stage" => error_stage::PROCESSING, + ) + .increment(1); + } + } +} + +#[cfg(feature = "sources-azure_blob")] +#[derive(Debug, NamedInternalEvent)] +pub struct QueueMessageReceiveError<'a, E> { + pub error: &'a E, +} + +#[cfg(feature = "sources-azure_blob")] +impl<'a, E: std::fmt::Display> InternalEvent for QueueMessageReceiveError<'a, E> { + fn emit(self) { + error!( + message = "Failed to fetch Azure Queue events.", + error = %self.error, + error_code = "failed_fetching_azure_queue_events", + error_type = error_type::REQUEST_FAILED, + stage = error_stage::RECEIVING, + ); + counter!( + CounterName::ComponentErrorsTotal, + "error_code" => "failed_fetching_azure_queue_events", + "error_type" => error_type::REQUEST_FAILED, + "stage" => error_stage::RECEIVING, + ) + .increment(1); + } +} + +#[cfg(feature = "sources-azure_blob")] +#[derive(Debug, NamedInternalEvent)] +pub struct QueueMessageDeleteError<'a, E> { + pub error: &'a E, +} + +#[cfg(feature = "sources-azure_blob")] +impl<'a, E: std::fmt::Display> InternalEvent for QueueMessageDeleteError<'a, E> { + fn emit(self) { + error!( + message = "Failed deleting message.", + error = %self.error, + error_code = "failed_deleting_azure_queue_event", + error_type = error_type::ACKNOWLEDGMENT_FAILED, + stage = error_stage::PROCESSING, + ); + counter!( + CounterName::ComponentErrorsTotal, + "error_code" => "failed_deleting_azure_queue_event", + "error_type" => error_type::ACKNOWLEDGMENT_FAILED, + "stage" => error_stage::PROCESSING, + ) + .increment(1); + } +} + +#[cfg(feature = "sources-azure_blob")] +#[derive(Debug, NamedInternalEvent)] +pub struct QueueStorageInvalidEventIgnored<'a> { + pub container: &'a str, + pub subject: &'a str, + pub event_type: &'a str, +} + +#[cfg(feature = "sources-azure_blob")] +impl<'a> InternalEvent for QueueStorageInvalidEventIgnored<'a> { + fn emit(self) { + trace!( + message = "Ignoring event because of wrong event type.", + container = %self.container, + subject = %self.subject, + event_type = %self.event_type + ); + counter!( + CounterName::AzureQueueEventIgnoredTotal, + "ignore_type" => "invalid_event_type" + ) + .increment(1); + } +} + +#[cfg(feature = "sources-azure_blob")] +#[derive(Debug, NamedInternalEvent)] +pub struct QueueStorageMismatchingContainerName<'a> { + pub container: &'a str, + pub configured_container: &'a str, +} + +#[cfg(feature = "sources-azure_blob")] +impl<'a> InternalEvent for QueueStorageMismatchingContainerName<'a> { + fn emit(self) { + warn!( + message = "Ignoring event because of wrong container name.", + configured_container = %self.configured_container, + container = %self.container, + ); + counter!( + CounterName::AzureQueueEventIgnoredTotal, + "ignore_type" => "mismatching_container_name" + ) + .increment(1); + } +} + +#[cfg(feature = "sources-azure_blob")] +#[derive(Debug, NamedInternalEvent)] +pub struct QueueMessageProcessingSucceeded {} + +#[cfg(feature = "sources-azure_blob")] +impl InternalEvent for QueueMessageProcessingSucceeded { + fn emit(self) { + trace!(message = "Processed azure queue message successfully."); + counter!(CounterName::AzureQueueMessageProcessingSucceededTotal).increment(1); + } +} + +#[cfg(feature = "sources-azure_blob")] +#[derive(Debug, NamedInternalEvent)] +pub struct QueueMessageProcessingErrored {} + +#[cfg(feature = "sources-azure_blob")] +impl InternalEvent for QueueMessageProcessingErrored { + fn emit(self) { + warn!(message = "Batch event had a transient error in delivery."); + counter!(CounterName::AzureQueueMessageProcessingErroredTotal).increment(1); + } +} + +#[cfg(feature = "sources-azure_blob")] +#[derive(Debug, NamedInternalEvent)] +pub struct QueueMessageProcessingRejected {} + +#[cfg(feature = "sources-azure_blob")] +impl InternalEvent for QueueMessageProcessingRejected { + fn emit(self) { + warn!(message = "Batch event had a permanent failure or rejection."); + counter!(CounterName::AzureQueueMessageProcessingRejectedTotal).increment(1); + } +} + +#[cfg(feature = "sources-azure_blob")] +#[derive(Debug, NamedInternalEvent)] +pub struct BlobDoesntExist<'a> { + pub nonexistent_blob_name: &'a str, +} + +#[cfg(feature = "sources-azure_blob")] +impl<'a> InternalEvent for BlobDoesntExist<'a> { + fn emit(self) { + warn!( + message = "Ignoring event because blob doesn't exist in storage.", + blob_name = self.nonexistent_blob_name + ); + counter!( + CounterName::AzureQueueEventIgnoredTotal, + "ignore_type" => "blob_doesnt_exist" + ) + .increment(1); + } +} diff --git a/src/internal_events/batch.rs b/src/internal_events/batch.rs index 8b7a4decfa770..bd553c1e9863d 100644 --- a/src/internal_events/batch.rs +++ b/src/internal_events/batch.rs @@ -1,8 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, error_stage, error_type, }; +use vector_lib::{NamedInternalEvent, counter}; #[derive(Debug, NamedInternalEvent)] pub struct LargeEventDroppedError { @@ -21,7 +20,7 @@ impl InternalEvent for LargeEventDroppedError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "oversized", "error_type" => error_type::CONDITION_FAILED, "stage" => error_stage::SENDING, diff --git a/src/internal_events/common.rs b/src/internal_events/common.rs index 69877e3db9ef5..5a6f9c50ede93 100644 --- a/src/internal_events/common.rs +++ b/src/internal_events/common.rs @@ -1,11 +1,12 @@ use std::time::Instant; -use metrics::{counter, histogram}; use vector_lib::NamedInternalEvent; pub use vector_lib::internal_event::EventsReceived; use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, HistogramName, InternalEvent, UNINTENTIONAL, error_stage, + error_type, }; +use vector_lib::{counter, histogram}; #[derive(Debug, NamedInternalEvent)] pub struct EndpointBytesReceived<'a> { @@ -23,7 +24,7 @@ impl InternalEvent for EndpointBytesReceived<'_> { endpoint = %self.endpoint, ); counter!( - "component_received_bytes_total", + CounterName::ComponentReceivedBytesTotal, "protocol" => self.protocol.to_owned(), "endpoint" => self.endpoint.to_owned(), ) @@ -47,7 +48,7 @@ impl InternalEvent for EndpointBytesSent<'_> { endpoint = %self.endpoint ); counter!( - "component_sent_bytes_total", + CounterName::ComponentSentBytesTotal, "protocol" => self.protocol.to_string(), "endpoint" => self.endpoint.to_string() ) @@ -70,7 +71,7 @@ impl InternalEvent for SocketOutgoingConnectionError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "failed_connecting", "error_type" => error_type::CONNECTION_FAILED, "stage" => error_stage::SENDING, @@ -95,7 +96,7 @@ impl InternalEvent for StreamClosedError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => STREAM_CLOSED, "error_type" => error_type::WRITER_FAILED, "stage" => error_stage::SENDING, @@ -117,8 +118,8 @@ pub struct CollectionCompleted { impl InternalEvent for CollectionCompleted { fn emit(self) { debug!(message = "Collection completed."); - counter!("collect_completed_total").increment(1); - histogram!("collect_duration_seconds").record(self.end - self.start); + counter!(CounterName::CollectCompletedTotal).increment(1); + histogram!(HistogramName::CollectDurationSeconds).record(self.end - self.start); } } @@ -139,7 +140,7 @@ impl InternalEvent for SinkRequestBuildError { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::ENCODER_FAILED, "stage" => error_stage::PROCESSING, ) diff --git a/src/internal_events/conditions.rs b/src/internal_events/conditions.rs index 96f7ae0ae0d50..e89f63b6a9c54 100644 --- a/src/internal_events/conditions.rs +++ b/src/internal_events/conditions.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; #[derive(Debug, Copy, Clone, NamedInternalEvent)] pub struct VrlConditionExecutionError<'a> { @@ -16,7 +17,7 @@ impl InternalEvent for VrlConditionExecutionError<'_> { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::SCRIPT_FAILED, "stage" => error_stage::PROCESSING, ) diff --git a/src/internal_events/datadog_agent.rs b/src/internal_events/datadog_agent.rs index 9ee5296407b24..84a4a8535c0c7 100644 --- a/src/internal_events/datadog_agent.rs +++ b/src/internal_events/datadog_agent.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; #[derive(Debug, NamedInternalEvent)] pub struct DatadogAgentJsonParseError<'a> { @@ -16,7 +17,7 @@ impl InternalEvent for DatadogAgentJsonParseError<'_> { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, ) diff --git a/src/internal_events/datadog_metrics.rs b/src/internal_events/datadog_metrics.rs index 5daa3ab87fa79..00e7d50a8a7d7 100644 --- a/src/internal_events/datadog_metrics.rs +++ b/src/internal_events/datadog_metrics.rs @@ -1,8 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, error_stage, error_type, }; +use vector_lib::{NamedInternalEvent, counter}; #[derive(Debug, NamedInternalEvent)] pub struct DatadogMetricsEncodingError<'a> { @@ -21,7 +20,7 @@ impl InternalEvent for DatadogMetricsEncodingError<'_> { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => self.error_code, "error_type" => error_type::ENCODER_FAILED, "stage" => error_stage::PROCESSING, diff --git a/src/internal_events/datadog_traces.rs b/src/internal_events/datadog_traces.rs index 06b48552fb840..016002681f81b 100644 --- a/src/internal_events/datadog_traces.rs +++ b/src/internal_events/datadog_traces.rs @@ -1,8 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, error_stage, error_type, }; +use vector_lib::{NamedInternalEvent, counter}; #[derive(Debug, NamedInternalEvent)] pub struct DatadogTracesEncodingError { @@ -22,7 +21,7 @@ impl InternalEvent for DatadogTracesEncodingError { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::ENCODER_FAILED, "stage" => error_stage::PROCESSING, ) @@ -51,7 +50,7 @@ impl InternalEvent for DatadogTracesAPMStatsError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::WRITER_FAILED, "stage" => error_stage::SENDING, ) diff --git a/src/internal_events/dedupe.rs b/src/internal_events/dedupe.rs index eb7511578cdd7..a1c36a3c4fa2b 100644 --- a/src/internal_events/dedupe.rs +++ b/src/internal_events/dedupe.rs @@ -1,5 +1,7 @@ -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{ComponentEventsDropped, INTENTIONAL, InternalEvent}; +use vector_lib::{ + NamedInternalEvent, + internal_event::{ComponentEventsDropped, INTENTIONAL, InternalEvent}, +}; #[derive(Debug, NamedInternalEvent)] pub struct DedupeEventsDropped { diff --git a/src/internal_events/demo_logs.rs b/src/internal_events/demo_logs.rs index 406a9856fe6e9..3958f41421ca7 100644 --- a/src/internal_events/demo_logs.rs +++ b/src/internal_events/demo_logs.rs @@ -1,5 +1,4 @@ -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::InternalEvent; +use vector_lib::{NamedInternalEvent, internal_event::InternalEvent}; #[derive(Debug, NamedInternalEvent)] pub struct DemoLogsEventProcessed; diff --git a/src/internal_events/dnstap.rs b/src/internal_events/dnstap.rs index 2fc20b95848d1..1ef67ee3001e0 100644 --- a/src/internal_events/dnstap.rs +++ b/src/internal_events/dnstap.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; #[derive(Debug, NamedInternalEvent)] pub(crate) struct DnstapParseError { @@ -16,7 +17,7 @@ impl InternalEvent for DnstapParseError { error_type = error_type::PARSER_FAILED, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "stage" => error_stage::PROCESSING, "error_type" => error_type::PARSER_FAILED, ) diff --git a/src/internal_events/docker_logs.rs b/src/internal_events/docker_logs.rs index bb44e112e05b8..fe24d41b6350b 100644 --- a/src/internal_events/docker_logs.rs +++ b/src/internal_events/docker_logs.rs @@ -1,9 +1,8 @@ use bollard::errors::Error; use chrono::ParseError; -use metrics::counter; use vector_lib::{ - NamedInternalEvent, - internal_event::{InternalEvent, error_stage, error_type}, + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, json_size::JsonSize, }; @@ -23,11 +22,11 @@ impl InternalEvent for DockerLogsEventsReceived<'_> { container_id = %self.container_id ); counter!( - "component_received_events_total", "container_name" => self.container_name.to_owned() + CounterName::ComponentReceivedEventsTotal, "container_name" => self.container_name.to_owned() ) .increment(1); counter!( - "component_received_event_bytes_total", "container_name" => self.container_name.to_owned() + CounterName::ComponentReceivedEventBytesTotal, "container_name" => self.container_name.to_owned() ).increment(self.byte_size.get() as u64); } } @@ -45,7 +44,7 @@ impl InternalEvent for DockerLogsContainerEventReceived<'_> { container_id = %self.container_id, action = %self.action, ); - counter!("container_processed_events_total").increment(1); + counter!(CounterName::ContainerProcessedEventsTotal).increment(1); } } @@ -60,7 +59,7 @@ impl InternalEvent for DockerLogsContainerWatch<'_> { message = "Started watching for container logs.", container_id = %self.container_id, ); - counter!("containers_watched_total").increment(1); + counter!(CounterName::ContainersWatchedTotal).increment(1); } } @@ -75,7 +74,7 @@ impl InternalEvent for DockerLogsContainerUnwatch<'_> { message = "Stopped watching for container logs.", container_id = %self.container_id, ); - counter!("containers_unwatched_total").increment(1); + counter!(CounterName::ContainersUnwatchedTotal).increment(1); } } @@ -95,7 +94,7 @@ impl InternalEvent for DockerLogsCommunicationError<'_> { container_id = ?self.container_id ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::CONNECTION_FAILED, "stage" => error_stage::RECEIVING, ) @@ -119,7 +118,7 @@ impl InternalEvent for DockerLogsContainerMetadataFetchError<'_> { container_id = ?self.container_id ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::REQUEST_FAILED, "stage" => error_stage::RECEIVING, "container_id" => self.container_id.to_owned(), @@ -144,7 +143,7 @@ impl InternalEvent for DockerLogsTimestampParseError<'_> { container_id = ?self.container_id ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, "container_id" => self.container_id.to_owned(), @@ -169,7 +168,7 @@ impl InternalEvent for DockerLogsLoggingDriverUnsupportedError<'_> { container_id = ?self.container_id, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::CONFIGURATION_FAILED, "stage" => error_stage::RECEIVING, "container_id" => self.container_id.to_owned(), diff --git a/src/internal_events/doris.rs b/src/internal_events/doris.rs index 0a2430869b7c5..15dffd26418b4 100644 --- a/src/internal_events/doris.rs +++ b/src/internal_events/doris.rs @@ -1,5 +1,7 @@ -use metrics::counter; -use vector_lib::{NamedInternalEvent, internal_event::InternalEvent}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent}, +}; /// Emitted when rows are successfully loaded into Doris. #[derive(Debug, NamedInternalEvent)] @@ -17,10 +19,10 @@ impl InternalEvent for DorisRowsLoaded { ); // Record the number of rows loaded - counter!("doris_rows_loaded_total").increment(self.loaded_rows as u64); + counter!(CounterName::DorisRowsLoadedTotal).increment(self.loaded_rows as u64); // Record the number of bytes loaded - counter!("doris_bytes_loaded_total").increment(self.load_bytes as u64); + counter!(CounterName::DorisBytesLoadedTotal).increment(self.load_bytes as u64); } } @@ -37,6 +39,6 @@ impl InternalEvent for DorisRowsFiltered { filtered_rows = %self.filtered_rows ); - counter!("doris_rows_filtered_total").increment(self.filtered_rows as u64); + counter!(CounterName::DorisRowsFilteredTotal).increment(self.filtered_rows as u64); } } diff --git a/src/internal_events/encoding_transcode.rs b/src/internal_events/encoding_transcode.rs index 76ad81792a63a..ec6280f1c703d 100644 --- a/src/internal_events/encoding_transcode.rs +++ b/src/internal_events/encoding_transcode.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::InternalEvent; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent}, +}; #[derive(Debug, NamedInternalEvent)] pub struct DecoderBomRemoval { @@ -13,7 +14,7 @@ impl InternalEvent for DecoderBomRemoval { message = "Removing initial BOM bytes from the final output while decoding to utf8.", from_encoding = %self.from_encoding ); - counter!("decoder_bom_removals_total").increment(1); + counter!(CounterName::DecoderBomRemovalsTotal).increment(1); } } @@ -30,7 +31,7 @@ impl InternalEvent for DecoderMalformedReplacement { ); // NOT the actual number of replacements in the output: there's no easy // way to get that from the lib we use here (encoding_rs) - counter!("decoder_malformed_replacement_warnings_total").increment(1); + counter!(CounterName::DecoderMalformedReplacementWarningsTotal).increment(1); } } @@ -47,6 +48,6 @@ impl InternalEvent for EncoderUnmappableReplacement { ); // NOT the actual number of replacements in the output: there's no easy // way to get that from the lib we use here (encoding_rs) - counter!("encoder_unmappable_replacement_warnings_total").increment(1); + counter!(CounterName::EncoderUnmappableReplacementWarningsTotal).increment(1); } } diff --git a/src/internal_events/eventstoredb_metrics.rs b/src/internal_events/eventstoredb_metrics.rs index c72b9c7dbeb7c..c7db7fad354a5 100644 --- a/src/internal_events/eventstoredb_metrics.rs +++ b/src/internal_events/eventstoredb_metrics.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; #[derive(Debug, NamedInternalEvent)] pub struct EventStoreDbMetricsHttpError { @@ -16,7 +17,7 @@ impl InternalEvent for EventStoreDbMetricsHttpError { error_type = error_type::REQUEST_FAILED, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "stage" => error_stage::RECEIVING, "error_type" => error_type::REQUEST_FAILED, ) @@ -38,7 +39,7 @@ impl InternalEvent for EventStoreDbStatsParsingError { error_type = error_type::PARSER_FAILED, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "stage" => error_stage::PROCESSING, "error_type" => error_type::PARSER_FAILED, ) diff --git a/src/internal_events/exec.rs b/src/internal_events/exec.rs index abe226897d5d3..21edc68f9df2a 100644 --- a/src/internal_events/exec.rs +++ b/src/internal_events/exec.rs @@ -1,11 +1,11 @@ use std::time::Duration; -use metrics::{counter, histogram}; use tokio::time::error::Elapsed; use vector_lib::{ - NamedInternalEvent, + NamedInternalEvent, counter, histogram, internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, HistogramName, InternalEvent, UNINTENTIONAL, + error_stage, error_type, }, json_size::JsonSize, }; @@ -28,12 +28,12 @@ impl InternalEvent for ExecEventsReceived<'_> { command = %self.command, ); counter!( - "component_received_events_total", + CounterName::ComponentReceivedEventsTotal, "command" => self.command.to_owned(), ) .increment(self.count as u64); counter!( - "component_received_event_bytes_total", + CounterName::ComponentReceivedEventBytesTotal, "command" => self.command.to_owned(), ) .increment(self.byte_size.get() as u64); @@ -57,7 +57,7 @@ impl InternalEvent for ExecFailedError<'_> { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "command" => self.command.to_owned(), "error_type" => error_type::COMMAND_FAILED, "error_code" => io_error_code(&self.error), @@ -85,7 +85,7 @@ impl InternalEvent for ExecTimeoutError<'_> { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "command" => self.command.to_owned(), "error_type" => error_type::TIMED_OUT, "stage" => error_stage::RECEIVING, @@ -120,14 +120,14 @@ impl InternalEvent for ExecCommandExecuted<'_> { elapsed_millis = %self.exec_duration.as_millis(), ); counter!( - "command_executed_total", + CounterName::CommandExecutedTotal, "command" => self.command.to_owned(), "exit_status" => exit_status.clone(), ) .increment(1); histogram!( - "command_execution_duration_seconds", + HistogramName::CommandExecutionDurationSeconds, "exit_status" => exit_status, "command" => self.command.to_owned(), ) @@ -196,7 +196,7 @@ impl InternalEvent for ExecFailedToSignalChildError<'_> { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "command" => format!("{:?}", self.command.as_std()), "error_code" => self.error.to_error_code(), "error_type" => error_type::COMMAND_FAILED, @@ -218,7 +218,7 @@ impl InternalEvent for ExecChannelClosedError { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::COMMAND_FAILED, "stage" => error_stage::RECEIVING, ) diff --git a/src/internal_events/expansion.rs b/src/internal_events/expansion.rs index 7b2633dd64f97..7f5b48914c69c 100644 --- a/src/internal_events/expansion.rs +++ b/src/internal_events/expansion.rs @@ -1,8 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, error_stage, error_type, }; +use vector_lib::{NamedInternalEvent, counter}; #[derive(NamedInternalEvent)] pub struct PairExpansionError<'a> { @@ -25,7 +24,7 @@ impl InternalEvent for PairExpansionError<'_> { ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, ) diff --git a/src/internal_events/file.rs b/src/internal_events/file.rs index 0e85c67150621..f1299ae751694 100644 --- a/src/internal_events/file.rs +++ b/src/internal_events/file.rs @@ -2,12 +2,13 @@ use std::borrow::Cow; -use metrics::{counter, gauge}; use vector_lib::{ NamedInternalEvent, configurable::configurable_component, + counter, gauge, internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, GaugeName, InternalEvent, UNINTENTIONAL, error_stage, + error_type, }, }; @@ -34,7 +35,7 @@ pub struct FileOpen { impl InternalEvent for FileOpen { fn emit(self) { - gauge!("open_files").set(self.count as f64); + gauge!(GaugeName::OpenFiles).set(self.count as f64); } } @@ -55,13 +56,13 @@ impl InternalEvent for FileBytesSent<'_> { ); if self.include_file_metric_tag { counter!( - "component_sent_bytes_total", + CounterName::ComponentSentBytesTotal, "protocol" => "file", "file" => self.file.clone().into_owned(), ) } else { counter!( - "component_sent_bytes_total", + CounterName::ComponentSentBytesTotal, "protocol" => "file", ) } @@ -89,7 +90,7 @@ impl InternalEvent for FileIoError<'_, P> { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => self.code, "error_type" => error_type::IO_FAILED, "stage" => error_stage::SENDING, @@ -110,11 +111,12 @@ mod source { use std::{io::Error, path::Path, time::Duration}; use bytes::BytesMut; - use metrics::counter; use vector_lib::{ - NamedInternalEvent, emit, + NamedInternalEvent, counter, emit, file_source_common::internal_events::FileSourceInternalEvents, - internal_event::{ComponentEventsDropped, INTENTIONAL, error_stage, error_type}, + internal_event::{ + ComponentEventsDropped, CounterName, INTENTIONAL, error_stage, error_type, + }, json_size::JsonSize, }; @@ -137,13 +139,13 @@ mod source { ); if self.include_file_metric_tag { counter!( - "component_received_bytes_total", + CounterName::ComponentReceivedBytesTotal, "protocol" => "file", "file" => self.file.to_owned() ) } else { counter!( - "component_received_bytes_total", + CounterName::ComponentReceivedBytesTotal, "protocol" => "file", ) } @@ -169,18 +171,18 @@ mod source { ); if self.include_file_metric_tag { counter!( - "component_received_events_total", + CounterName::ComponentReceivedEventsTotal, "file" => self.file.to_owned(), ) .increment(self.count as u64); counter!( - "component_received_event_bytes_total", + CounterName::ComponentReceivedEventBytesTotal, "file" => self.file.to_owned(), ) .increment(self.byte_size.get() as u64); } else { - counter!("component_received_events_total").increment(self.count as u64); - counter!("component_received_event_bytes_total") + counter!(CounterName::ComponentReceivedEventsTotal).increment(self.count as u64); + counter!(CounterName::ComponentReceivedEventBytesTotal) .increment(self.byte_size.get() as u64); } } @@ -200,11 +202,11 @@ mod source { ); if self.include_file_metric_tag { counter!( - "checksum_errors_total", + CounterName::ChecksumErrorsTotal, "file" => self.file.to_string_lossy().into_owned(), ) } else { - counter!("checksum_errors_total") + counter!(CounterName::ChecksumErrorsTotal) } .increment(1); } @@ -229,7 +231,7 @@ mod source { ); if self.include_file_metric_tag { counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "reading_fingerprint", "error_type" => error_type::READER_FAILED, "stage" => error_stage::RECEIVING, @@ -237,7 +239,7 @@ mod source { ) } else { counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "reading_fingerprint", "error_type" => error_type::READER_FAILED, "stage" => error_stage::RECEIVING, @@ -268,7 +270,7 @@ mod source { ); if self.include_file_metric_tag { counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "file" => self.file.to_string_lossy().into_owned(), "error_code" => DELETION_FAILED, "error_type" => error_type::COMMAND_FAILED, @@ -276,7 +278,7 @@ mod source { ) } else { counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => DELETION_FAILED, "error_type" => error_type::COMMAND_FAILED, "stage" => error_stage::RECEIVING, @@ -300,11 +302,11 @@ mod source { ); if self.include_file_metric_tag { counter!( - "files_deleted_total", + CounterName::FilesDeletedTotal, "file" => self.file.to_string_lossy().into_owned(), ) } else { - counter!("files_deleted_total") + counter!(CounterName::FilesDeletedTotal) } .increment(1); } @@ -327,13 +329,13 @@ mod source { ); if self.include_file_metric_tag { counter!( - "files_unwatched_total", + CounterName::FilesUnwatchedTotal, "file" => self.file.to_string_lossy().into_owned(), "reached_eof" => reached_eof, ) } else { counter!( - "files_unwatched_total", + CounterName::FilesUnwatchedTotal, "reached_eof" => reached_eof, ) } @@ -360,7 +362,7 @@ mod source { ); if self.include_file_metric_tag { counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "watching", "error_type" => error_type::COMMAND_FAILED, "stage" => error_stage::RECEIVING, @@ -368,7 +370,7 @@ mod source { ) } else { counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "watching", "error_type" => error_type::COMMAND_FAILED, "stage" => error_stage::RECEIVING, @@ -394,11 +396,11 @@ mod source { ); if self.include_file_metric_tag { counter!( - "files_resumed_total", + CounterName::FilesResumedTotal, "file" => self.file.to_string_lossy().into_owned(), ) } else { - counter!("files_resumed_total") + counter!(CounterName::FilesResumedTotal) } .increment(1); } @@ -418,11 +420,11 @@ mod source { ); if self.include_file_metric_tag { counter!( - "files_added_total", + CounterName::FilesAddedTotal, "file" => self.file.to_string_lossy().into_owned(), ) } else { - counter!("files_added_total") + counter!(CounterName::FilesAddedTotal) } .increment(1); } @@ -441,7 +443,7 @@ mod source { count = %self.count, duration_ms = self.duration.as_millis() as u64, ); - counter!("checkpoints_total").increment(self.count as u64); + counter!(CounterName::CheckpointsTotal).increment(self.count as u64); } } @@ -460,7 +462,7 @@ mod source { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "writing_checkpoints", "error_type" => error_type::WRITER_FAILED, "stage" => error_stage::RECEIVING, @@ -486,7 +488,7 @@ mod source { path = %self.path.display(), ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "globbing", "error_type" => error_type::READER_FAILED, "stage" => error_stage::RECEIVING, @@ -513,7 +515,7 @@ mod source { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "reading_line_from_file", "error_type" => error_type::CONDITION_FAILED, "stage" => error_stage::RECEIVING, diff --git a/src/internal_events/file_descriptor.rs b/src/internal_events/file_descriptor.rs index 7882ed4d95b12..8db4e7895729e 100644 --- a/src/internal_events/file_descriptor.rs +++ b/src/internal_events/file_descriptor.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; #[derive(Debug, NamedInternalEvent)] pub struct FileDescriptorReadError { @@ -19,7 +20,7 @@ where stage = error_stage::RECEIVING ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::CONNECTION_FAILED, "stage" => error_stage::RECEIVING, ) diff --git a/src/internal_events/fluent.rs b/src/internal_events/fluent.rs index 6d6bb408edc5f..f47b927d5cbe2 100644 --- a/src/internal_events/fluent.rs +++ b/src/internal_events/fluent.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; use crate::sources::fluent::DecodeError; @@ -31,7 +32,7 @@ impl InternalEvent for FluentMessageDecodeError<'_> { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, ) diff --git a/src/internal_events/gcp_pubsub.rs b/src/internal_events/gcp_pubsub.rs index 4066d71936529..752546cfa7d09 100644 --- a/src/internal_events/gcp_pubsub.rs +++ b/src/internal_events/gcp_pubsub.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; #[derive(NamedInternalEvent)] pub struct GcpPubsubConnectError { @@ -18,7 +19,7 @@ impl InternalEvent for GcpPubsubConnectError { ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "failed_connecting", "error_type" => error_type::CONNECTION_FAILED, "stage" => error_stage::RECEIVING, @@ -43,7 +44,7 @@ impl InternalEvent for GcpPubsubStreamingPullError { ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "failed_streaming_pull", "error_type" => error_type::REQUEST_FAILED, "stage" => error_stage::RECEIVING, @@ -68,7 +69,7 @@ impl InternalEvent for GcpPubsubReceiveError { ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "failed_fetching_events", "error_type" => error_type::REQUEST_FAILED, "stage" => error_stage::RECEIVING, diff --git a/src/internal_events/grpc.rs b/src/internal_events/grpc.rs index 882b1dbfaf33b..66da41e10b797 100644 --- a/src/internal_events/grpc.rs +++ b/src/internal_events/grpc.rs @@ -1,10 +1,11 @@ use std::time::Duration; use http::response::Response; -use metrics::{counter, histogram}; use tonic::Code; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, histogram, + internal_event::{CounterName, HistogramName, InternalEvent, error_stage, error_type}, +}; const GRPC_STATUS_LABEL: &str = "grpc_status"; @@ -13,7 +14,7 @@ pub struct GrpcServerRequestReceived; impl InternalEvent for GrpcServerRequestReceived { fn emit(self) { - counter!("grpc_server_messages_received_total").increment(1); + counter!(CounterName::GrpcServerMessagesReceivedTotal).increment(1); } } @@ -34,8 +35,8 @@ impl InternalEvent for GrpcServerResponseSent<'_, B> { let grpc_code = grpc_code_to_name(grpc_code); let labels = &[(GRPC_STATUS_LABEL, grpc_code)]; - counter!("grpc_server_messages_sent_total", labels).increment(1); - histogram!("grpc_server_handler_duration_seconds", labels).record(self.latency); + counter!(CounterName::GrpcServerMessagesSentTotal, labels).increment(1); + histogram!(HistogramName::GrpcServerHandlerDurationSeconds, labels).record(self.latency); } } @@ -53,7 +54,7 @@ impl InternalEvent for GrpcInvalidCompressionSchemeError<'_> { stage = error_stage::RECEIVING ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::REQUEST_FAILED, "stage" => error_stage::RECEIVING, ) @@ -78,7 +79,7 @@ where stage = error_stage::RECEIVING ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::REQUEST_FAILED, "stage" => error_stage::RECEIVING, ) diff --git a/src/internal_events/heartbeat.rs b/src/internal_events/heartbeat.rs index 658a24fcb8762..16191669e7317 100644 --- a/src/internal_events/heartbeat.rs +++ b/src/internal_events/heartbeat.rs @@ -1,8 +1,9 @@ use std::time::Instant; -use metrics::gauge; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::InternalEvent; +use vector_lib::{ + NamedInternalEvent, gauge, + internal_event::{GaugeName, InternalEvent}, +}; use crate::built_info; @@ -14,9 +15,9 @@ pub struct Heartbeat { impl InternalEvent for Heartbeat { fn emit(self) { trace!(target: "vector", message = "Beep."); - gauge!("uptime_seconds").set(self.since.elapsed().as_secs() as f64); + gauge!(GaugeName::UptimeSeconds).set(self.since.elapsed().as_secs() as f64); gauge!( - "build_info", + GaugeName::BuildInfo, "debug" => built_info::DEBUG, "version" => built_info::PKG_VERSION, "rust_version" => built_info::RUST_VERSION, diff --git a/src/internal_events/host_metrics.rs b/src/internal_events/host_metrics.rs index d60771644571e..7662ca5e808a4 100644 --- a/src/internal_events/host_metrics.rs +++ b/src/internal_events/host_metrics.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; #[derive(Debug, NamedInternalEvent)] pub struct HostMetricsScrapeError { @@ -16,7 +17,7 @@ impl InternalEvent for HostMetricsScrapeError { ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::READER_FAILED, "stage" => error_stage::RECEIVING, ) @@ -40,7 +41,7 @@ impl InternalEvent for HostMetricsScrapeDetailError { ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::READER_FAILED, "stage" => error_stage::RECEIVING, ) @@ -66,7 +67,7 @@ impl InternalEvent for HostMetricsScrapeFilesystemError { ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::READER_FAILED, "stage" => error_stage::RECEIVING, ) diff --git a/src/internal_events/http.rs b/src/internal_events/http.rs index a0d8f5d2a92d3..4aad267d7683b 100644 --- a/src/internal_events/http.rs +++ b/src/internal_events/http.rs @@ -1,10 +1,9 @@ use std::{error::Error, time::Duration}; use http::Response; -use metrics::{counter, histogram}; use vector_lib::{ - NamedInternalEvent, - internal_event::{InternalEvent, error_stage, error_type}, + NamedInternalEvent, counter, histogram, + internal_event::{CounterName, HistogramName, InternalEvent, error_stage, error_type}, json_size::JsonSize, }; @@ -16,7 +15,7 @@ pub struct HttpServerRequestReceived; impl InternalEvent for HttpServerRequestReceived { fn emit(self) { debug!(message = "Received HTTP request."); - counter!("http_server_requests_received_total").increment(1); + counter!(CounterName::HttpServerRequestsReceivedTotal).increment(1); } } @@ -32,8 +31,8 @@ impl InternalEvent for HttpServerResponseSent<'_, B> { HTTP_STATUS_LABEL, self.response.status().as_u16().to_string(), )]; - counter!("http_server_responses_sent_total", labels).increment(1); - histogram!("http_server_handler_duration_seconds", labels).record(self.latency); + counter!(CounterName::HttpServerResponsesSentTotal, labels).increment(1); + histogram!(HistogramName::HttpServerHandlerDurationSeconds, labels).record(self.latency); } } @@ -53,7 +52,7 @@ impl InternalEvent for HttpBytesReceived<'_> { protocol = %self.protocol ); counter!( - "component_received_bytes_total", + CounterName::ComponentReceivedBytesTotal, "http_path" => self.http_path.to_string(), "protocol" => self.protocol, ) @@ -79,15 +78,15 @@ impl InternalEvent for HttpEventsReceived<'_> { protocol = %self.protocol, ); - histogram!("component_received_events_count").record(self.count as f64); + histogram!(HistogramName::ComponentReceivedEventsCount).record(self.count as f64); counter!( - "component_received_events_total", + CounterName::ComponentReceivedEventsTotal, "http_path" => self.http_path.to_string(), "protocol" => self.protocol, ) .increment(self.count as u64); counter!( - "component_received_event_bytes_total", + CounterName::ComponentReceivedEventBytesTotal, "http_path" => self.http_path.to_string(), "protocol" => self.protocol, ) @@ -126,7 +125,7 @@ impl InternalEvent for HttpBadRequest<'_> { http_code = %self.code, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => self.error_code, "error_type" => error_type::REQUEST_FAILED, "error_stage" => error_stage::RECEIVING, @@ -152,7 +151,7 @@ impl InternalEvent for HttpDecompressError<'_> { encoding = %self.encoding ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "failed_decompressing_payload", "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::RECEIVING, @@ -174,7 +173,7 @@ impl InternalEvent for HttpInternalError<'_> { stage = error_stage::RECEIVING ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::CONNECTION_FAILED, "stage" => error_stage::RECEIVING, ) diff --git a/src/internal_events/http_client.rs b/src/internal_events/http_client.rs index d1e4c7549d15a..38e6fe94f8d7e 100644 --- a/src/internal_events/http_client.rs +++ b/src/internal_events/http_client.rs @@ -2,12 +2,13 @@ use std::time::Duration; use http::{ Request, Response, - header::{self, HeaderMap, HeaderValue}, + header::{self, HeaderMap, HeaderName, HeaderValue}, }; use hyper::{Error, body::HttpBody}; -use metrics::{counter, histogram}; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, histogram, + internal_event::{CounterName, HistogramName, InternalEvent, error_stage, error_type}, +}; #[derive(Debug, NamedInternalEvent)] pub struct AboutToSendHttpRequest<'a, T> { @@ -16,13 +17,21 @@ pub struct AboutToSendHttpRequest<'a, T> { fn remove_sensitive(headers: &HeaderMap) -> HeaderMap { let mut headers = headers.clone(); - for name in &[ + let sensitive: &[HeaderName] = &[ header::AUTHORIZATION, header::PROXY_AUTHORIZATION, + header::PROXY_AUTHENTICATE, + header::WWW_AUTHENTICATE, header::COOKIE, header::SET_COOKIE, - ] { - if let Some(value) = headers.get_mut(name) { + HeaderName::from_static("cookie2"), + HeaderName::from_static("dd-api-key"), + HeaderName::from_static("x-honeycomb-team"), + HeaderName::from_static("x-api-key"), + HeaderName::from_static("api-key"), + ]; + for (name, value) in headers.iter_mut() { + if sensitive.contains(name) { value.set_sensitive(true); } } @@ -39,7 +48,7 @@ impl InternalEvent for AboutToSendHttpRequest<'_, T> { headers = ?remove_sensitive(self.request.headers()), body = %FormatBody(self.request.body()), ); - counter!("http_client_requests_sent_total", "method" => self.request.method().to_string()) + counter!(CounterName::HttpClientRequestsSentTotal, "method" => self.request.method().to_string()) .increment(1); } } @@ -60,13 +69,13 @@ impl InternalEvent for GotHttpResponse<'_, T> { body = %FormatBody(self.response.body()), ); counter!( - "http_client_responses_total", + CounterName::HttpClientResponsesTotal, "status" => self.response.status().as_u16().to_string(), ) .increment(1); - histogram!("http_client_rtt_seconds").record(self.roundtrip); + histogram!(HistogramName::HttpClientRttSeconds).record(self.roundtrip); histogram!( - "http_client_response_rtt_seconds", + HistogramName::HttpClientResponseRttSeconds, "status" => self.response.status().as_u16().to_string(), ) .record(self.roundtrip); @@ -87,9 +96,10 @@ impl InternalEvent for GotHttpWarning<'_> { error_type = error_type::REQUEST_FAILED, stage = error_stage::PROCESSING, ); - counter!("http_client_errors_total", "error_kind" => self.error.to_string()).increment(1); - histogram!("http_client_rtt_seconds").record(self.roundtrip); - histogram!("http_client_error_rtt_seconds", "error_kind" => self.error.to_string()) + counter!(CounterName::HttpClientErrorsTotal, "error_kind" => self.error.to_string()) + .increment(1); + histogram!(HistogramName::HttpClientRttSeconds).record(self.roundtrip); + histogram!(HistogramName::HttpClientErrorRttSeconds, "error_kind" => self.error.to_string()) .record(self.roundtrip); } } @@ -112,3 +122,78 @@ impl std::fmt::Display for FormatBody<'_, B> { } } } + +#[cfg(test)] +mod tests { + use http::header::{self, HeaderMap, HeaderName, HeaderValue}; + + use super::remove_sensitive; + + fn is_sensitive(map: &HeaderMap, name: &HeaderName) -> Vec { + map.get_all(name) + .iter() + .map(HeaderValue::is_sensitive) + .collect() + } + + #[test] + fn marks_single_sensitive_header() { + let mut headers = HeaderMap::new(); + headers.insert( + header::AUTHORIZATION, + HeaderValue::from_static("Bearer token"), + ); + let result = remove_sensitive(&headers); + assert!( + is_sensitive(&result, &header::AUTHORIZATION) + .iter() + .all(|&s| s) + ); + } + + #[test] + fn marks_all_duplicate_sensitive_headers() { + let x_api_key: HeaderName = HeaderName::from_static("x-api-key"); + let mut headers = HeaderMap::new(); + headers.insert(x_api_key.clone(), HeaderValue::from_static("key-one")); + headers.append(x_api_key.clone(), HeaderValue::from_static("key-two")); + headers.append(x_api_key.clone(), HeaderValue::from_static("key-three")); + + let result = remove_sensitive(&headers); + let sensitive_flags = is_sensitive(&result, &x_api_key); + assert_eq!(sensitive_flags.len(), 3); + assert!( + sensitive_flags.iter().all(|&s| s), + "not all duplicate x-api-key values were marked sensitive: {sensitive_flags:?}" + ); + } + + #[test] + fn header_name_matching_is_case_insensitive() { + // HeaderName normalizes to lowercase, so mixed-case variants are identical. + let mut headers = HeaderMap::new(); + headers.insert( + HeaderName::from_static("x-api-key"), + HeaderValue::from_static("secret"), + ); + let result = remove_sensitive(&headers); + // Lookup with the mixed-case form resolves to the same normalized name. + let mixed_case = HeaderName::from_bytes(b"X-Api-Key").unwrap(); + assert!(is_sensitive(&result, &mixed_case).iter().all(|&s| s)); + } + + #[test] + fn does_not_mark_non_sensitive_headers() { + let mut headers = HeaderMap::new(); + headers.insert( + header::CONTENT_TYPE, + HeaderValue::from_static("application/json"), + ); + let result = remove_sensitive(&headers); + assert!( + is_sensitive(&result, &header::CONTENT_TYPE) + .iter() + .all(|&s| !s) + ); + } +} diff --git a/src/internal_events/http_client_source.rs b/src/internal_events/http_client_source.rs index ff67535c0c14b..0ba52d722dfee 100644 --- a/src/internal_events/http_client_source.rs +++ b/src/internal_events/http_client_source.rs @@ -1,9 +1,8 @@ #![allow(dead_code)] // TODO requires optional feature compilation -use metrics::counter; use vector_lib::{ - NamedInternalEvent, - internal_event::{InternalEvent, error_stage, error_type}, + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, json_size::JsonSize, }; @@ -25,12 +24,12 @@ impl InternalEvent for HttpClientEventsReceived { url = %self.url, ); counter!( - "component_received_events_total", + CounterName::ComponentReceivedEventsTotal, "uri" => self.url.clone(), ) .increment(self.count as u64); counter!( - "component_received_event_bytes_total", + CounterName::ComponentReceivedEventBytesTotal, "uri" => self.url.clone(), ) .increment(self.byte_size.get() as u64); @@ -53,7 +52,7 @@ impl InternalEvent for HttpClientHttpResponseError { error_code = %http_error_code(self.code.as_u16()), ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "url" => self.url, "stage" => error_stage::RECEIVING, "error_type" => error_type::REQUEST_FAILED, @@ -79,7 +78,7 @@ impl InternalEvent for HttpClientHttpError { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "url" => self.url, "error_type" => error_type::REQUEST_FAILED, "stage" => error_stage::RECEIVING, diff --git a/src/internal_events/influxdb.rs b/src/internal_events/influxdb.rs index 22b34c3dc532c..a142642140f6f 100644 --- a/src/internal_events/influxdb.rs +++ b/src/internal_events/influxdb.rs @@ -1,8 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, error_stage, error_type, }; +use vector_lib::{NamedInternalEvent, counter}; #[derive(Debug, NamedInternalEvent)] pub struct InfluxdbEncodingError { @@ -20,7 +19,7 @@ impl InternalEvent for InfluxdbEncodingError { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::ENCODER_FAILED, "stage" => error_stage::PROCESSING, ) diff --git a/src/internal_events/internal_logs.rs b/src/internal_events/internal_logs.rs index 63c520723c9c7..2f0674d380edd 100644 --- a/src/internal_events/internal_logs.rs +++ b/src/internal_events/internal_logs.rs @@ -1,5 +1,8 @@ -use metrics::counter; -use vector_lib::{NamedInternalEvent, internal_event::InternalEvent, json_size::JsonSize}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent}, + json_size::JsonSize, +}; #[derive(Debug, NamedInternalEvent)] pub struct InternalLogsBytesReceived { @@ -10,7 +13,7 @@ impl InternalEvent for InternalLogsBytesReceived { fn emit(self) { // MUST NOT emit logs here to avoid an infinite log loop counter!( - "component_received_bytes_total", + CounterName::ComponentReceivedBytesTotal, "protocol" => "internal", ) .increment(self.byte_size as u64); @@ -26,7 +29,8 @@ pub struct InternalLogsEventsReceived { impl InternalEvent for InternalLogsEventsReceived { fn emit(self) { // MUST NOT emit logs here to avoid an infinite log loop - counter!("component_received_events_total").increment(self.count as u64); - counter!("component_received_event_bytes_total").increment(self.byte_size.get() as u64); + counter!(CounterName::ComponentReceivedEventsTotal).increment(self.count as u64); + counter!(CounterName::ComponentReceivedEventBytesTotal) + .increment(self.byte_size.get() as u64); } } diff --git a/src/internal_events/journald.rs b/src/internal_events/journald.rs index 0debba0de5d29..02f6300cb6905 100644 --- a/src/internal_events/journald.rs +++ b/src/internal_events/journald.rs @@ -1,8 +1,8 @@ -use metrics::counter; use vector_lib::{ NamedInternalEvent, codecs::decoding::BoxedFramingError, - internal_event::{InternalEvent, error_stage, error_type}, + counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, }; #[derive(Debug, NamedInternalEvent)] @@ -21,7 +21,7 @@ impl InternalEvent for JournaldInvalidRecordError { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "stage" => error_stage::PROCESSING, "error_type" => error_type::PARSER_FAILED, ) @@ -43,7 +43,7 @@ impl InternalEvent for JournaldStartJournalctlError { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "stage" => error_stage::RECEIVING, "error_type" => error_type::COMMAND_FAILED, ) @@ -65,7 +65,7 @@ impl InternalEvent for JournaldReadError { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "stage" => error_stage::PROCESSING, "error_type" => error_type::READER_FAILED, ) @@ -89,7 +89,7 @@ impl InternalEvent for JournaldCheckpointSetError { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "stage" => error_stage::PROCESSING, "error_type" => error_type::IO_FAILED, ) @@ -113,7 +113,7 @@ impl InternalEvent for JournaldCheckpointFileOpenError { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "stage" => error_stage::RECEIVING, "error_type" => error_type::IO_FAILED, ) diff --git a/src/internal_events/kafka.rs b/src/internal_events/kafka.rs index 87ad581e1c453..1cbd2ba3ce8cb 100644 --- a/src/internal_events/kafka.rs +++ b/src/internal_events/kafka.rs @@ -1,9 +1,8 @@ #![allow(dead_code)] // TODO requires optional feature compilation -use metrics::{counter, gauge}; use vector_lib::{ - NamedInternalEvent, - internal_event::{InternalEvent, error_stage, error_type}, + NamedInternalEvent, counter, gauge, + internal_event::{CounterName, GaugeName, InternalEvent, error_stage, error_type}, json_size::JsonSize, }; use vrl::path::OwnedTargetPath; @@ -26,7 +25,7 @@ impl InternalEvent for KafkaBytesReceived<'_> { partition = %self.partition, ); counter!( - "component_received_bytes_total", + CounterName::ComponentReceivedBytesTotal, "protocol" => self.protocol, "topic" => self.topic.to_string(), "partition" => self.partition.to_string(), @@ -53,13 +52,13 @@ impl InternalEvent for KafkaEventsReceived<'_> { partition = %self.partition, ); counter!( - "component_received_events_total", + CounterName::ComponentReceivedEventsTotal, "topic" => self.topic.to_string(), "partition" => self.partition.to_string(), ) .increment(self.count as u64); counter!( - "component_received_event_bytes_total", + CounterName::ComponentReceivedEventBytesTotal, "topic" => self.topic.to_string(), "partition" => self.partition.to_string(), ) @@ -82,7 +81,7 @@ impl InternalEvent for KafkaOffsetUpdateError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "kafka_offset_update", "error_type" => error_type::READER_FAILED, "stage" => error_stage::SENDING, @@ -106,7 +105,7 @@ impl InternalEvent for KafkaReadError { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "reading_message", "error_type" => error_type::READER_FAILED, "stage" => error_stage::RECEIVING, @@ -123,24 +122,24 @@ pub struct KafkaStatisticsReceived<'a> { impl InternalEvent for KafkaStatisticsReceived<'_> { fn emit(self) { - gauge!("kafka_queue_messages").set(self.statistics.msg_cnt as f64); - gauge!("kafka_queue_messages_bytes").set(self.statistics.msg_size as f64); - counter!("kafka_requests_total").absolute(self.statistics.tx as u64); - counter!("kafka_requests_bytes_total").absolute(self.statistics.tx_bytes as u64); - counter!("kafka_responses_total").absolute(self.statistics.rx as u64); - counter!("kafka_responses_bytes_total").absolute(self.statistics.rx_bytes as u64); - counter!("kafka_produced_messages_total").absolute(self.statistics.txmsgs as u64); - counter!("kafka_produced_messages_bytes_total") + gauge!(GaugeName::KafkaQueueMessages).set(self.statistics.msg_cnt as f64); + gauge!(GaugeName::KafkaQueueMessagesBytes).set(self.statistics.msg_size as f64); + counter!(CounterName::KafkaRequestsTotal).absolute(self.statistics.tx as u64); + counter!(CounterName::KafkaRequestsBytesTotal).absolute(self.statistics.tx_bytes as u64); + counter!(CounterName::KafkaResponsesTotal).absolute(self.statistics.rx as u64); + counter!(CounterName::KafkaResponsesBytesTotal).absolute(self.statistics.rx_bytes as u64); + counter!(CounterName::KafkaProducedMessagesTotal).absolute(self.statistics.txmsgs as u64); + counter!(CounterName::KafkaProducedMessagesBytesTotal) .absolute(self.statistics.txmsg_bytes as u64); - counter!("kafka_consumed_messages_total").absolute(self.statistics.rxmsgs as u64); - counter!("kafka_consumed_messages_bytes_total") + counter!(CounterName::KafkaConsumedMessagesTotal).absolute(self.statistics.rxmsgs as u64); + counter!(CounterName::KafkaConsumedMessagesBytesTotal) .absolute(self.statistics.rxmsg_bytes as u64); if self.expose_lag_metrics { for (topic_id, topic) in &self.statistics.topics { for (partition_id, partition) in &topic.partitions { gauge!( - "kafka_consumer_lag", + GaugeName::KafkaConsumerLag, "topic_id" => topic_id.clone(), "partition_id" => partition_id.to_string(), ) @@ -166,7 +165,7 @@ impl InternalEvent for KafkaHeaderExtractionError<'_> { header_field = self.header_field.to_string(), ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "extracting_header", "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::RECEIVING, diff --git a/src/internal_events/kubernetes_logs.rs b/src/internal_events/kubernetes_logs.rs index c6edb03724169..15b4ca4e61ee0 100644 --- a/src/internal_events/kubernetes_logs.rs +++ b/src/internal_events/kubernetes_logs.rs @@ -1,8 +1,8 @@ -use metrics::counter; use vector_lib::{ - NamedInternalEvent, + NamedInternalEvent, counter, internal_event::{ - ComponentEventsDropped, INTENTIONAL, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, INTENTIONAL, InternalEvent, UNINTENTIONAL, + error_stage, error_type, }, json_size::JsonSize, }; @@ -37,21 +37,21 @@ impl InternalEvent for KubernetesLogsEventsReceived<'_> { let pod_namespace = pod_info.namespace; counter!( - "component_received_events_total", + CounterName::ComponentReceivedEventsTotal, "pod_name" => pod_name.clone(), "pod_namespace" => pod_namespace.clone(), ) .increment(1); counter!( - "component_received_event_bytes_total", + CounterName::ComponentReceivedEventBytesTotal, "pod_name" => pod_name, "pod_namespace" => pod_namespace, ) .increment(self.byte_size.get() as u64); } None => { - counter!("component_received_events_total").increment(1); - counter!("component_received_event_bytes_total") + counter!(CounterName::ComponentReceivedEventsTotal).increment(1); + counter!(CounterName::ComponentReceivedEventBytesTotal) .increment(self.byte_size.get() as u64); } } @@ -75,7 +75,7 @@ impl InternalEvent for KubernetesLogsEventAnnotationError<'_> { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => ANNOTATION_FAILED, "error_type" => error_type::READER_FAILED, "stage" => error_stage::PROCESSING, @@ -99,13 +99,13 @@ impl InternalEvent for KubernetesLogsEventNamespaceAnnotationError<'_> { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => ANNOTATION_FAILED, "error_type" => error_type::READER_FAILED, "stage" => error_stage::PROCESSING, ) .increment(1); - counter!("k8s_event_namespace_annotation_failures_total").increment(1); + counter!(CounterName::K8sEventNamespaceAnnotationFailuresTotal).increment(1); } } @@ -124,13 +124,13 @@ impl InternalEvent for KubernetesLogsEventNodeAnnotationError<'_> { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => ANNOTATION_FAILED, "error_type" => error_type::READER_FAILED, "stage" => error_stage::PROCESSING, ) .increment(1); - counter!("k8s_event_node_annotation_failures_total").increment(1); + counter!(CounterName::K8sEventNodeAnnotationFailuresTotal).increment(1); } } @@ -145,7 +145,7 @@ impl InternalEvent for KubernetesLogsFormatPickerEdgeCase { message = "Encountered format picker edge case.", what = %self.what, ); - counter!("k8s_format_picker_edge_cases_total").increment(1); + counter!(CounterName::K8sFormatPickerEdgeCasesTotal).increment(1); } } @@ -163,12 +163,12 @@ impl InternalEvent for KubernetesLogsDockerFormatParseError<'_> { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, ) .increment(1); - counter!("k8s_docker_format_parse_failures_total").increment(1); + counter!(CounterName::K8sDockerFormatParseFailuresTotal).increment(1); } } @@ -191,7 +191,7 @@ impl InternalEvent for KubernetesLifecycleError { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => KUBERNETES_LIFECYCLE, "error_type" => error_type::READER_FAILED, "stage" => error_stage::PROCESSING, @@ -222,7 +222,7 @@ impl InternalEvent for KubernetesMergedLineTooBigError<'_> { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "reading_line_from_kubernetes_log", "error_type" => error_type::CONDITION_FAILED, "stage" => error_stage::RECEIVING, diff --git a/src/internal_events/log_to_metric.rs b/src/internal_events/log_to_metric.rs index 9fbb64dba0622..e8e3da5b898b7 100644 --- a/src/internal_events/log_to_metric.rs +++ b/src/internal_events/log_to_metric.rs @@ -1,9 +1,10 @@ use std::num::ParseFloatError; -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{ + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, error_stage, error_type, + }, }; #[derive(NamedInternalEvent)] @@ -22,7 +23,7 @@ impl InternalEvent for LogToMetricFieldNullError<'_> { null_field = %self.field ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "field_null", "error_type" => error_type::CONDITION_FAILED, "stage" => error_stage::PROCESSING, @@ -52,7 +53,7 @@ impl InternalEvent for LogToMetricParseFloatError<'_> { stage = error_stage::PROCESSING ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "failed_parsing_float", "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, @@ -83,7 +84,7 @@ impl InternalEvent for MetricMetadataInvalidFieldValueError<'_> { stage = error_stage::PROCESSING ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "invalid_field_value", "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, @@ -112,7 +113,7 @@ impl InternalEvent for MetricMetadataParseError<'_> { stage = error_stage::PROCESSING ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => format!("failed_parsing_{}", self.kind), "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, @@ -137,7 +138,7 @@ impl InternalEvent for MetricMetadataMetricDetailsNotFoundError { stage = error_stage::PROCESSING ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "missing_metric_details", "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, diff --git a/src/internal_events/logplex.rs b/src/internal_events/logplex.rs index b28588c5c2d1e..4a45df6349b65 100644 --- a/src/internal_events/logplex.rs +++ b/src/internal_events/logplex.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; use super::prelude::io_error_code; @@ -37,7 +38,7 @@ impl InternalEvent for HerokuLogplexRequestReadError { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::READER_FAILED, "error_code" => io_error_code(&self.error), "stage" => error_stage::PROCESSING, diff --git a/src/internal_events/loki.rs b/src/internal_events/loki.rs index 523efd7e4f59b..3a5935e1a372a 100644 --- a/src/internal_events/loki.rs +++ b/src/internal_events/loki.rs @@ -1,8 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; use vector_lib::internal_event::{ - ComponentEventsDropped, INTENTIONAL, InternalEvent, error_stage, error_type, + ComponentEventsDropped, CounterName, INTENTIONAL, InternalEvent, error_stage, error_type, }; +use vector_lib::{NamedInternalEvent, counter}; #[derive(Debug, NamedInternalEvent)] pub struct LokiEventUnlabeledError; @@ -17,7 +16,7 @@ impl InternalEvent for LokiEventUnlabeledError { ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "unlabeled_event", "error_type" => error_type::CONDITION_FAILED, "stage" => error_stage::PROCESSING, @@ -48,7 +47,7 @@ impl InternalEvent for LokiOutOfOrderEventDroppedError { }); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "out_of_order", "error_type" => error_type::CONDITION_FAILED, "stage" => error_stage::PROCESSING, @@ -69,7 +68,7 @@ impl InternalEvent for LokiOutOfOrderEventRewritten { count = self.count, reason = "out_of_order", ); - counter!("rewritten_timestamp_events_total").increment(self.count as u64); + counter!(CounterName::RewrittenTimestampEventsTotal).increment(self.count as u64); } } @@ -90,7 +89,7 @@ impl InternalEvent for LokiTimestampNonParsableEventsDropped { emit!(ComponentEventsDropped:: { count: 1, reason }); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "non-parsable_timestamp", "error_type" => error_type::CONDITION_FAILED, "stage" => error_stage::PROCESSING, diff --git a/src/internal_events/lua.rs b/src/internal_events/lua.rs index feb9d04212b88..3a8779d9cd4b1 100644 --- a/src/internal_events/lua.rs +++ b/src/internal_events/lua.rs @@ -1,8 +1,9 @@ -use metrics::{counter, gauge}; use vector_lib::NamedInternalEvent; use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, GaugeName, InternalEvent, UNINTENTIONAL, error_stage, + error_type, }; +use vector_lib::{counter, gauge}; use crate::transforms::lua::v2::BuildError; @@ -13,7 +14,7 @@ pub struct LuaGcTriggered { impl InternalEvent for LuaGcTriggered { fn emit(self) { - gauge!("lua_memory_used_bytes").set(self.used_memory as f64); + gauge!(GaugeName::LuaMemoryUsedBytes).set(self.used_memory as f64); } } @@ -32,7 +33,7 @@ impl InternalEvent for LuaScriptError { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => mlua_error_code(&self.error), "error_type" => error_type::SCRIPT_FAILED, "stage" => error_stage::PROCESSING, @@ -62,7 +63,7 @@ impl InternalEvent for LuaBuildError { internal_log_rate_limit = false, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => lua_build_error_code(&self.error), "error_type" => error_type::SCRIPT_FAILED, "stage" => error_stage:: PROCESSING, diff --git a/src/internal_events/metric_to_log.rs b/src/internal_events/metric_to_log.rs index a536a1b24df24..a11ed286be17a 100644 --- a/src/internal_events/metric_to_log.rs +++ b/src/internal_events/metric_to_log.rs @@ -1,9 +1,8 @@ -use metrics::counter; use serde_json::Error; -use vector_lib::NamedInternalEvent; use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, error_stage, error_type, }; +use vector_lib::{NamedInternalEvent, counter}; #[derive(Debug, NamedInternalEvent)] pub struct MetricToLogSerializeError { @@ -20,7 +19,7 @@ impl InternalEvent for MetricToLogSerializeError { stage = error_stage::PROCESSING ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::ENCODER_FAILED, "stage" => error_stage::PROCESSING, ) diff --git a/src/internal_events/mod.rs b/src/internal_events/mod.rs index 02b7460856f2d..1a908102e1eb9 100644 --- a/src/internal_events/mod.rs +++ b/src/internal_events/mod.rs @@ -40,7 +40,7 @@ mod datadog_traces; mod dedupe; #[cfg(feature = "sources-demo_logs")] mod demo_logs; -#[cfg(feature = "sources-dnstap")] +#[cfg(all(unix, feature = "sources-dnstap"))] mod dnstap; #[cfg(feature = "sources-docker_logs")] mod docker_logs; @@ -154,6 +154,9 @@ mod windows_event_log; #[cfg(windows)] mod windows; +#[cfg(feature = "sources-azure_blob")] +mod azure_queue; + #[cfg(any(feature = "transforms-log_to_metric", feature = "sinks-loki"))] mod expansion; #[cfg(feature = "sources-mongodb_metrics")] @@ -184,6 +187,8 @@ pub(crate) use self::aws_kinesis::*; pub(crate) use self::aws_kinesis_firehose::*; #[cfg(any(feature = "sources-aws_s3", feature = "sources-aws_sqs",))] pub(crate) use self::aws_sqs::*; +#[cfg(feature = "sources-azure_blob")] +pub(crate) use self::azure_queue::*; #[cfg(feature = "sources-datadog_agent")] pub(crate) use self::datadog_agent::*; #[cfg(feature = "sinks-datadog_metrics")] @@ -194,7 +199,7 @@ pub(crate) use self::datadog_traces::*; pub(crate) use self::dedupe::*; #[cfg(feature = "sources-demo_logs")] pub(crate) use self::demo_logs::*; -#[cfg(feature = "sources-dnstap")] +#[cfg(all(unix, feature = "sources-dnstap"))] pub(crate) use self::dnstap::*; #[cfg(feature = "sources-docker_logs")] pub(crate) use self::docker_logs::*; diff --git a/src/internal_events/mongodb_metrics.rs b/src/internal_events/mongodb_metrics.rs index 3cad1e4ef97f9..b0023085f025f 100644 --- a/src/internal_events/mongodb_metrics.rs +++ b/src/internal_events/mongodb_metrics.rs @@ -1,8 +1,7 @@ -use metrics::counter; use mongodb::{bson, error::Error as MongoError}; use vector_lib::{ - NamedInternalEvent, - internal_event::{InternalEvent, error_stage, error_type}, + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, json_size::JsonSize, }; @@ -23,12 +22,12 @@ impl InternalEvent for MongoDbMetricsEventsReceived<'_> { endpoint = self.endpoint, ); counter!( - "component_received_events_total", + CounterName::ComponentReceivedEventsTotal, "endpoint" => self.endpoint.to_owned(), ) .increment(self.count as u64); counter!( - "component_received_event_bytes_total", + CounterName::ComponentReceivedEventBytesTotal, "endpoint" => self.endpoint.to_owned(), ) .increment(self.byte_size.get() as u64); @@ -51,7 +50,7 @@ impl InternalEvent for MongoDbMetricsRequestError<'_> { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::REQUEST_FAILED, "stage" => error_stage::RECEIVING, ) @@ -75,7 +74,7 @@ impl InternalEvent for MongoDbMetricsBsonParseError<'_> { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::RECEIVING, "endpoint" => self.endpoint.to_owned(), diff --git a/src/internal_events/mqtt.rs b/src/internal_events/mqtt.rs index 1e406a9494f1f..8d416c4c748ee 100644 --- a/src/internal_events/mqtt.rs +++ b/src/internal_events/mqtt.rs @@ -1,9 +1,10 @@ use std::fmt::Debug; -use metrics::counter; use rumqttc::ConnectionError; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; #[derive(Debug, NamedInternalEvent)] pub struct MqttConnectionError { @@ -20,7 +21,7 @@ impl InternalEvent for MqttConnectionError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "mqtt_connection_error", "error_type" => error_type::WRITER_FAILED, "stage" => error_stage::SENDING, diff --git a/src/internal_events/nginx_metrics.rs b/src/internal_events/nginx_metrics.rs index a3384d4ef0faf..82cd7ea3878b7 100644 --- a/src/internal_events/nginx_metrics.rs +++ b/src/internal_events/nginx_metrics.rs @@ -1,7 +1,6 @@ -use metrics::counter; use vector_lib::{ - NamedInternalEvent, - internal_event::{InternalEvent, error_stage, error_type}, + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, json_size::JsonSize, }; @@ -23,12 +22,12 @@ impl InternalEvent for NginxMetricsEventsReceived<'_> { endpoint = self.endpoint, ); counter!( - "component_received_events_total", + CounterName::ComponentReceivedEventsTotal, "endpoint" => self.endpoint.to_owned(), ) .increment(self.count as u64); counter!( - "component_received_event_bytes_total", + CounterName::ComponentReceivedEventBytesTotal, "endpoint" => self.endpoint.to_owned(), ) .increment(self.byte_size.get() as u64); @@ -51,7 +50,7 @@ impl InternalEvent for NginxMetricsRequestError<'_> { stage = error_stage::RECEIVING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "endpoint" => self.endpoint.to_owned(), "error_type" => error_type::REQUEST_FAILED, "stage" => error_stage::RECEIVING, @@ -76,7 +75,7 @@ impl InternalEvent for NginxMetricsStubStatusParseError<'_> { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "endpoint" => self.endpoint.to_owned(), "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, diff --git a/src/internal_events/open.rs b/src/internal_events/open.rs index e48e0ac337820..58e9c9d40b3f8 100644 --- a/src/internal_events/open.rs +++ b/src/internal_events/open.rs @@ -6,9 +6,10 @@ use std::{ }, }; -use metrics::gauge; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::InternalEvent; +use vector_lib::{ + NamedInternalEvent, gauge, + internal_event::{GaugeName, InternalEvent}, +}; #[derive(Debug, NamedInternalEvent)] pub struct ConnectionOpen { @@ -17,7 +18,7 @@ pub struct ConnectionOpen { impl InternalEvent for ConnectionOpen { fn emit(self) { - gauge!("open_connections").set(self.count as f64); + gauge!(GaugeName::OpenConnections).set(self.count as f64); } } @@ -28,7 +29,7 @@ pub struct EndpointsActive { impl InternalEvent for EndpointsActive { fn emit(self) { - gauge!("active_endpoints").set(self.count as f64); + gauge!(GaugeName::ActiveEndpoints).set(self.count as f64); } } diff --git a/src/internal_events/parser.rs b/src/internal_events/parser.rs index 85a3e1383afa3..2610ae5458958 100644 --- a/src/internal_events/parser.rs +++ b/src/internal_events/parser.rs @@ -2,10 +2,11 @@ use std::borrow::Cow; -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{ + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, error_stage, error_type, + }, }; fn truncate_string_at(s: &str, maxlen: usize) -> Cow<'_, str> { @@ -36,7 +37,7 @@ impl InternalEvent for ParserMatchError<'_> { field = &truncate_string_at(&String::from_utf8_lossy(self.value), 60)[..] ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "no_match_found", "error_type" => error_type::CONDITION_FAILED, "stage" => error_stage::PROCESSING, @@ -66,7 +67,7 @@ impl InternalEvent for ParserMissingFieldError<'_, DROP_ stage = error_stage::PROCESSING ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "field_not_found", "error_type" => error_type::CONDITION_FAILED, "stage" => error_stage::PROCESSING, @@ -97,7 +98,7 @@ impl InternalEvent for ParserConversionError<'_> { stage = error_stage::PROCESSING ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "type_conversion", "error_type" => error_type::CONVERSION_FAILED, "stage" => error_stage::PROCESSING, diff --git a/src/internal_events/postgresql_metrics.rs b/src/internal_events/postgresql_metrics.rs index d6a58f968ab7f..62fa4d18a722d 100644 --- a/src/internal_events/postgresql_metrics.rs +++ b/src/internal_events/postgresql_metrics.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; #[derive(Debug, NamedInternalEvent)] pub struct PostgresqlMetricsCollectError<'a> { @@ -18,7 +19,7 @@ impl InternalEvent for PostgresqlMetricsCollectError<'_> { endpoint = %self.endpoint, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::REQUEST_FAILED, "stage" => error_stage::RECEIVING, ) diff --git a/src/internal_events/process.rs b/src/internal_events/process.rs index cc3609737aea9..19701c6feaca8 100644 --- a/src/internal_events/process.rs +++ b/src/internal_events/process.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; use crate::{built_info, config}; @@ -17,7 +18,7 @@ impl InternalEvent for VectorStarted { arch = built_info::TARGET_ARCH, revision = built_info::VECTOR_BUILD_DESC.unwrap_or(""), ); - counter!("started_total").increment(1); + counter!(CounterName::StartedTotal).increment(1); } } @@ -34,7 +35,19 @@ impl InternalEvent for VectorReloaded<'_> { path = ?self.config_paths, internal_log_rate_limit = false, ); - counter!("reloaded_total").increment(1); + counter!(CounterName::ReloadedTotal).increment(1); + } +} + +#[derive(Debug, NamedInternalEvent)] +pub struct VectorStopping; + +impl InternalEvent for VectorStopping { + fn emit(self) { + info!( + target: "vector", + message = "Vector is stopping.", + ); } } @@ -47,7 +60,7 @@ impl InternalEvent for VectorStopped { target: "vector", message = "Vector has stopped.", ); - counter!("stopped_total").increment(1); + counter!(CounterName::StoppedTotal).increment(1); } } @@ -60,7 +73,7 @@ impl InternalEvent for VectorQuit { target: "vector", message = "Vector has quit.", ); - counter!("quit_total").increment(1); + counter!(CounterName::QuitTotal).increment(1); } } @@ -80,7 +93,7 @@ impl InternalEvent for VectorReloadError { internal_log_rate_limit = false, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "reload", "error_type" => error_type::CONFIGURATION_FAILED, "stage" => error_stage::PROCESSING, @@ -103,7 +116,7 @@ impl InternalEvent for VectorConfigLoadError { internal_log_rate_limit = false, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "config_load", "error_type" => error_type::CONFIGURATION_FAILED, "stage" => error_stage::PROCESSING, @@ -125,7 +138,7 @@ impl InternalEvent for VectorRecoveryError { internal_log_rate_limit = false, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "recovery", "error_type" => error_type::CONFIGURATION_FAILED, "stage" => error_stage::PROCESSING, diff --git a/src/internal_events/prometheus.rs b/src/internal_events/prometheus.rs index 27b28fb5ffafa..3ddd3c555f728 100644 --- a/src/internal_events/prometheus.rs +++ b/src/internal_events/prometheus.rs @@ -3,13 +3,14 @@ #[cfg(feature = "sources-prometheus-scrape")] use std::borrow::Cow; -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, -}; #[cfg(feature = "sources-prometheus-scrape")] use vector_lib::prometheus::parser::ParserError; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{ + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, error_stage, error_type, + }, +}; #[cfg(feature = "sources-prometheus-scrape")] #[derive(Debug, NamedInternalEvent)] @@ -34,7 +35,7 @@ impl InternalEvent for PrometheusParseError<'_> { url = %self.url ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, "url" => self.url.to_string(), @@ -57,7 +58,7 @@ impl InternalEvent for PrometheusRemoteWriteParseError { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, ) @@ -77,7 +78,7 @@ impl InternalEvent for PrometheusNormalizationError { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::CONVERSION_FAILED, "stage" => error_stage::PROCESSING, ) diff --git a/src/internal_events/pulsar.rs b/src/internal_events/pulsar.rs index a2212ef0d5971..0e2e11b061f3e 100644 --- a/src/internal_events/pulsar.rs +++ b/src/internal_events/pulsar.rs @@ -2,11 +2,10 @@ #[cfg(feature = "sources-pulsar")] use metrics::Counter; -use metrics::counter; -use vector_lib::NamedInternalEvent; use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, error_stage, error_type, }; +use vector_lib::{NamedInternalEvent, counter}; #[derive(Debug, NamedInternalEvent)] pub struct PulsarSendingError { @@ -24,7 +23,7 @@ impl InternalEvent for PulsarSendingError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::REQUEST_FAILED, "stage" => error_stage::SENDING, ) @@ -51,7 +50,7 @@ impl InternalEvent for PulsarPropertyExtractionError { property_field = %self.property_field, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "extracting_property", "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, @@ -77,21 +76,21 @@ pub struct PulsarErrorEventData { registered_event!( PulsarErrorEvent => { ack_errors: Counter = counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "acknowledge_message", "error_type" => error_type::ACKNOWLEDGMENT_FAILED, "stage" => error_stage::RECEIVING, ), nack_errors: Counter = counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "negative_acknowledge_message", "error_type" => error_type::ACKNOWLEDGMENT_FAILED, "stage" => error_stage::RECEIVING, ), read_errors: Counter = counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "reading_message", "error_type" => error_type::READER_FAILED, "stage" => error_stage::RECEIVING, diff --git a/src/internal_events/redis.rs b/src/internal_events/redis.rs index 45d76f3670dcd..87b9bc6d7b2f8 100644 --- a/src/internal_events/redis.rs +++ b/src/internal_events/redis.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; #[derive(Debug, NamedInternalEvent)] pub struct RedisReceiveEventError { @@ -25,7 +26,7 @@ impl InternalEvent for RedisReceiveEventError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => self.error_code, "error_type" => error_type::READER_FAILED, "stage" => error_stage::RECEIVING, diff --git a/src/internal_events/reduce.rs b/src/internal_events/reduce.rs index 8917c5aad0a5a..0962c6332a439 100644 --- a/src/internal_events/reduce.rs +++ b/src/internal_events/reduce.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; use vrl::{path::PathParseError, value::KeyString}; #[derive(Debug, NamedInternalEvent)] @@ -8,7 +9,7 @@ pub struct ReduceStaleEventFlushed; impl InternalEvent for ReduceStaleEventFlushed { fn emit(self) { - counter!("stale_events_flushed_total").increment(1); + counter!(CounterName::StaleEventsFlushedTotal).increment(1); } } @@ -28,7 +29,7 @@ impl InternalEvent for ReduceAddEventError { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, ) diff --git a/src/internal_events/remap.rs b/src/internal_events/remap.rs index 989a6aa064065..d0e6500d69341 100644 --- a/src/internal_events/remap.rs +++ b/src/internal_events/remap.rs @@ -1,8 +1,8 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; use vector_lib::internal_event::{ - ComponentEventsDropped, INTENTIONAL, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, INTENTIONAL, InternalEvent, UNINTENTIONAL, error_stage, + error_type, }; +use vector_lib::{NamedInternalEvent, counter}; #[derive(Debug, NamedInternalEvent)] pub struct RemapMappingError { @@ -21,7 +21,7 @@ impl InternalEvent for RemapMappingError { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::CONVERSION_FAILED, "stage" => error_stage::PROCESSING, ) diff --git a/src/internal_events/sample.rs b/src/internal_events/sample.rs index 3722f9530b668..825175dc42555 100644 --- a/src/internal_events/sample.rs +++ b/src/internal_events/sample.rs @@ -1,5 +1,7 @@ -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{ComponentEventsDropped, INTENTIONAL, InternalEvent}; +use vector_lib::{ + NamedInternalEvent, + internal_event::{ComponentEventsDropped, INTENTIONAL, InternalEvent}, +}; #[derive(Debug, NamedInternalEvent)] pub struct SampleEventDiscarded; diff --git a/src/internal_events/sematext_metrics.rs b/src/internal_events/sematext_metrics.rs index 65d23c53e00ef..cb03742a5d3d1 100644 --- a/src/internal_events/sematext_metrics.rs +++ b/src/internal_events/sematext_metrics.rs @@ -1,7 +1,8 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{ + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, error_stage, error_type, + }, }; use crate::event::metric::Metric; @@ -23,7 +24,7 @@ impl InternalEvent for SematextMetricsInvalidMetricError<'_> { kind = ?self.metric.kind(), ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "invalid_metric", "error_type" => error_type::ENCODER_FAILED, "stage" => error_stage::PROCESSING, @@ -49,7 +50,7 @@ impl InternalEvent for SematextMetricsEncodeEventError stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::ENCODER_FAILED, "stage" => error_stage::PROCESSING, ) diff --git a/src/internal_events/socket.rs b/src/internal_events/socket.rs index f0b58ce33bbd7..d38788f46767c 100644 --- a/src/internal_events/socket.rs +++ b/src/internal_events/socket.rs @@ -1,10 +1,10 @@ use std::net::Ipv4Addr; -use metrics::{counter, histogram}; use vector_lib::{ - NamedInternalEvent, + NamedInternalEvent, counter, histogram, internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, HistogramName, InternalEvent, UNINTENTIONAL, + error_stage, error_type, }, json_size::JsonSize, }; @@ -42,11 +42,11 @@ impl InternalEvent for SocketBytesReceived { %protocol, ); counter!( - "component_received_bytes_total", + CounterName::ComponentReceivedBytesTotal, "protocol" => protocol, ) .increment(self.byte_size as u64); - histogram!("component_received_bytes").record(self.byte_size as f64); + histogram!(HistogramName::ComponentReceivedBytes).record(self.byte_size as f64); } } @@ -66,10 +66,12 @@ impl InternalEvent for SocketEventsReceived { byte_size = self.byte_size.get(), %mode, ); - counter!("component_received_events_total", "mode" => mode).increment(self.count as u64); - counter!("component_received_event_bytes_total", "mode" => mode) + counter!(CounterName::ComponentReceivedEventsTotal, "mode" => mode) + .increment(self.count as u64); + counter!(CounterName::ComponentReceivedEventBytesTotal, "mode" => mode) .increment(self.byte_size.get() as u64); - histogram!("component_received_bytes", "mode" => mode).record(self.byte_size.get() as f64); + histogram!(HistogramName::ComponentReceivedBytes, "mode" => mode) + .record(self.byte_size.get() as f64); } } @@ -88,7 +90,7 @@ impl InternalEvent for SocketBytesSent { %protocol, ); counter!( - "component_sent_bytes_total", + CounterName::ComponentSentBytesTotal, "protocol" => protocol, ) .increment(self.byte_size as u64); @@ -105,8 +107,9 @@ pub struct SocketEventsSent { impl InternalEvent for SocketEventsSent { fn emit(self) { trace!(message = "Events sent.", count = %self.count, byte_size = %self.byte_size.get()); - counter!("component_sent_events_total", "mode" => self.mode.as_str()).increment(self.count); - counter!("component_sent_event_bytes_total", "mode" => self.mode.as_str()) + counter!(CounterName::ComponentSentEventsTotal, "mode" => self.mode.as_str()) + .increment(self.count); + counter!(CounterName::ComponentSentEventBytesTotal, "mode" => self.mode.as_str()) .increment(self.byte_size.get() as u64); } } @@ -129,7 +132,7 @@ impl InternalEvent for SocketBindError { %mode, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "socket_bind", "error_type" => error_type::IO_FAILED, "stage" => error_stage::INITIALIZING, @@ -164,7 +167,7 @@ impl InternalEvent for SocketMulticastGroupJoinError { %interface, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "socket_multicast_group_join", "error_type" => error_type::IO_FAILED, "stage" => error_stage::INITIALIZING, @@ -194,7 +197,7 @@ impl InternalEvent for SocketReceiveError { %mode, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "socket_receive", "error_type" => error_type::READER_FAILED, "stage" => error_stage::RECEIVING, @@ -223,7 +226,7 @@ impl InternalEvent for SocketSendError { %mode, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "socket_send", "error_type" => error_type::WRITER_FAILED, "stage" => error_stage::SENDING, diff --git a/src/internal_events/splunk_hec.rs b/src/internal_events/splunk_hec.rs index ea090684ed85d..bd3fde83c2318 100644 --- a/src/internal_events/splunk_hec.rs +++ b/src/internal_events/splunk_hec.rs @@ -7,11 +7,13 @@ pub use self::source::*; #[cfg(feature = "sinks-splunk_hec")] mod sink { - use metrics::{counter, gauge}; use serde_json::Error; - use vector_lib::NamedInternalEvent; - use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, + use vector_lib::{ + NamedInternalEvent, counter, gauge, + internal_event::{ + ComponentEventsDropped, CounterName, GaugeName, InternalEvent, UNINTENTIONAL, + error_stage, error_type, + }, }; use crate::{ @@ -35,7 +37,7 @@ mod sink { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "serializing_json", "error_type" => error_type::ENCODER_FAILED, "stage" => error_stage::PROCESSING, @@ -63,13 +65,13 @@ mod sink { kind = ?self.kind, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::INVALID_METRIC, "stage" => error_stage::PROCESSING, ) .increment(1); counter!( - "component_discarded_events_total", + CounterName::ComponentDiscardedEventsTotal, "error_type" => error_type::INVALID_METRIC, "stage" => error_stage::PROCESSING, ) @@ -92,7 +94,7 @@ mod sink { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "invalid_response", "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::SENDING, @@ -117,7 +119,7 @@ mod sink { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "indexer_ack_failed", "error_type" => error_type::ACKNOWLEDGMENT_FAILED, "stage" => error_stage::SENDING, @@ -141,7 +143,7 @@ mod sink { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "indexer_ack_unavailable", "error_type" => error_type::ACKNOWLEDGMENT_FAILED, "stage" => error_stage::SENDING, @@ -155,7 +157,7 @@ mod sink { impl InternalEvent for SplunkIndexerAcknowledgementAckAdded { fn emit(self) { - gauge!("splunk_pending_acks").increment(1.0); + gauge!(GaugeName::SplunkPendingAcks).increment(1.0); } } @@ -166,7 +168,7 @@ mod sink { impl InternalEvent for SplunkIndexerAcknowledgementAcksRemoved { fn emit(self) { - gauge!("splunk_pending_acks").decrement(self.count); + gauge!(GaugeName::SplunkPendingAcks).decrement(self.count); } } @@ -197,9 +199,10 @@ mod sink { #[cfg(feature = "sources-splunk_hec")] mod source { - use metrics::counter; - use vector_lib::NamedInternalEvent; - use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; + use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, + }; use crate::sources::splunk_hec::ApiError; @@ -218,7 +221,7 @@ mod source { stage = error_stage::PROCESSING ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "invalid_request_body", "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, @@ -234,18 +237,36 @@ mod source { impl InternalEvent for SplunkHecRequestError { fn emit(self) { - error!( - message = "Error processing request.", - error = ?self.error, - error_type = error_type::REQUEST_FAILED, - stage = error_stage::RECEIVING - ); - counter!( - "component_errors_total", - "error_type" => error_type::REQUEST_FAILED, - "stage" => error_stage::RECEIVING, - ) - .increment(1); + match self.error { + ApiError::InvalidAuthorization | ApiError::MissingAuthorization => { + error!( + message = "Unauthenticated request.", + error = ?self.error, + error_type = error_type::AUTHENTICATION_FAILED, + stage = error_stage::RECEIVING + ); + counter!( + CounterName::ComponentErrorsTotal, + "error_type" => error_type::AUTHENTICATION_FAILED, + "stage" => error_stage::RECEIVING, + ) + .increment(1); + } + _ => { + error!( + message = "Error processing request.", + error = ?self.error, + error_type = error_type::REQUEST_FAILED, + stage = error_stage::RECEIVING + ); + counter!( + CounterName::ComponentErrorsTotal, + "error_type" => error_type::REQUEST_FAILED, + "stage" => error_stage::RECEIVING, + ) + .increment(1); + } + } } } } diff --git a/src/internal_events/statsd_sink.rs b/src/internal_events/statsd_sink.rs index 7f0ee81f48ddc..858084124b562 100644 --- a/src/internal_events/statsd_sink.rs +++ b/src/internal_events/statsd_sink.rs @@ -1,8 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, error_stage, error_type, }; +use vector_lib::{NamedInternalEvent, counter}; use crate::event::metric::{MetricKind, MetricValue}; @@ -24,7 +23,7 @@ impl InternalEvent for StatsdInvalidMetricError<'_> { kind = ?self.kind, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "invalid_metric", "error_type" => error_type::ENCODER_FAILED, "stage" => error_stage::PROCESSING, diff --git a/src/internal_events/tag_cardinality_limit.rs b/src/internal_events/tag_cardinality_limit.rs index 83a82acd342ae..8b9cd25c6d64c 100644 --- a/src/internal_events/tag_cardinality_limit.rs +++ b/src/internal_events/tag_cardinality_limit.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{ComponentEventsDropped, INTENTIONAL, InternalEvent}; +use vector_lib::{ + NamedInternalEvent, counter, gauge, + internal_event::{ComponentEventsDropped, CounterName, GaugeName, INTENTIONAL, InternalEvent}, +}; #[derive(NamedInternalEvent)] pub struct TagCardinalityLimitRejectingEvent<'a> { @@ -20,13 +21,13 @@ impl InternalEvent for TagCardinalityLimitRejectingEvent<'_> { ); if self.include_extended_tags { counter!( - "tag_value_limit_exceeded_total", + CounterName::TagValueLimitExceededTotal, "metric_name" => self.metric_name.to_string(), "tag_key" => self.tag_key.to_string(), ) .increment(1); } else { - counter!("tag_value_limit_exceeded_total").increment(1); + counter!(CounterName::TagValueLimitExceededTotal).increment(1); } emit!(ComponentEventsDropped:: { @@ -54,13 +55,13 @@ impl InternalEvent for TagCardinalityLimitRejectingTag<'_> { ); if self.include_extended_tags { counter!( - "tag_value_limit_exceeded_total", + CounterName::TagValueLimitExceededTotal, "metric_name" => self.metric_name.to_string(), "tag_key" => self.tag_key.to_string(), ) .increment(1); } else { - counter!("tag_value_limit_exceeded_total").increment(1); + counter!(CounterName::TagValueLimitExceededTotal).increment(1); } } } @@ -76,6 +77,29 @@ impl InternalEvent for TagCardinalityValueLimitReached<'_> { message = "Value_limit reached for key. New values for this key will be rejected.", key = %self.key, ); - counter!("value_limit_reached_total").increment(1); + counter!(CounterName::ValueLimitReachedTotal).increment(1); + } +} + +#[derive(NamedInternalEvent)] +pub struct TagCardinalityLimitUntracked; + +impl InternalEvent for TagCardinalityLimitUntracked { + fn emit(self) { + debug!( + message = "Max tracked keys limit reached; forwarding one or more metric tags without cardinality checks." + ); + counter!(CounterName::TagCardinalityUntrackedEventsTotal).increment(1); + } +} + +#[derive(NamedInternalEvent)] +pub struct TagCardinalityTrackedKeys { + pub count: usize, +} + +impl InternalEvent for TagCardinalityTrackedKeys { + fn emit(self) { + gauge!(GaugeName::TagCardinalityTrackedKeys).set(self.count as f64); } } diff --git a/src/internal_events/tcp.rs b/src/internal_events/tcp.rs index a121eef1e0327..698c0dfab525e 100644 --- a/src/internal_events/tcp.rs +++ b/src/internal_events/tcp.rs @@ -1,8 +1,9 @@ use std::net::SocketAddr; -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; use crate::{internal_events::SocketOutgoingConnectionError, tls::TlsError}; @@ -18,7 +19,7 @@ impl InternalEvent for TcpSocketConnectionEstablished { } else { debug!(message = "Connected.", peer_addr = "unknown"); } - counter!("connection_established_total", "mode" => "tcp").increment(1); + counter!(CounterName::ConnectionEstablishedTotal, "mode" => "tcp").increment(1); } } @@ -41,7 +42,23 @@ pub struct TcpSocketConnectionShutdown; impl InternalEvent for TcpSocketConnectionShutdown { fn emit(self) { warn!(message = "Received EOF from the server, shutdown."); - counter!("connection_shutdown_total", "mode" => "tcp").increment(1); + counter!(CounterName::ConnectionShutdownTotal, "mode" => "tcp").increment(1); + } +} + +/// Emitted once per accepted TCP source connection, after the per-connection +/// task exits β€” regardless of cause. This includes pre-loop exits (TLS +/// handshake failure, shutdown signal arriving during handshake) as well as +/// every read/ack loop exit (graceful peer EOF, decoder failure, downstream +/// closed, ack write failure, shutdown signal, tripwire, max connection +/// duration). Pairs exactly with `ConnectionOpen`. +#[derive(Debug, NamedInternalEvent)] +pub struct TcpSourceConnectionClosed; + +impl InternalEvent for TcpSourceConnectionClosed { + fn emit(self) { + debug!(message = "Connection closed."); + counter!(CounterName::ConnectionShutdownTotal, "mode" => "tcp").increment(1); } } @@ -63,7 +80,7 @@ impl InternalEvent for TcpSocketError<'_, E> { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::CONNECTION_FAILED, "stage" => error_stage::PROCESSING, ) @@ -100,7 +117,7 @@ impl InternalEvent for TcpSocketTlsConnectionError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "connection_failed", "error_type" => error_type::WRITER_FAILED, "stage" => error_stage::SENDING, @@ -127,7 +144,7 @@ impl InternalEvent for TcpSendAckError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "ack_failed", "error_type" => error_type::WRITER_FAILED, "stage" => error_stage::SENDING, @@ -152,8 +169,99 @@ impl InternalEvent for TcpBytesReceived { peer_addr = %self.peer_addr, ); counter!( - "component_received_bytes_total", "protocol" => "tcp" + CounterName::ComponentReceivedBytesTotal, "protocol" => "tcp" ) .increment(self.byte_size as u64); } } + +#[cfg(test)] +mod tests { + use std::io; + + use serial_test::serial; + use vector_lib::event::MetricValue; + use vector_lib::internal_event::InternalEvent; + use vector_lib::metrics::Controller; + + use super::{TcpSendAckError, TcpSourceConnectionClosed}; + + /// Returns the current value of a counter matching `name` and all `tags`. + /// Counters that have not yet been touched aren't in the snapshot and + /// register as 0.0 here. + fn counter_value(name: &str, tags: &[(&str, &str)]) -> f64 { + Controller::get() + .expect("metrics controller initialized") + .capture_metrics() + .into_iter() + .find(|m| { + m.name() == name + && tags + .iter() + .all(|(k, v)| m.tags().is_some_and(|t| t.get(k) == Some(*v))) + }) + .map(|m| match m.value() { + MetricValue::Counter { value } => *value, + other => panic!("expected counter for {name}, got {other:?}"), + }) + .unwrap_or(0.0) + } + + /// `TcpSourceConnectionClosed` MUST bump `connection_shutdown_total{mode="tcp"}` + /// once per emit. Pins the contract that this event is the sole owner of the + /// connection-close counter on the source side. + #[test] + #[serial] + fn tcp_source_connection_closed_increments_shutdown_total() { + crate::test_util::trace_init(); + let before = counter_value("connection_shutdown_total", &[("mode", "tcp")]); + + TcpSourceConnectionClosed.emit(); + + let after = counter_value("connection_shutdown_total", &[("mode", "tcp")]); + assert_eq!(after - before, 1.0); + } + + /// `TcpSendAckError` is an `*Error` event and per the instrumentation spec MUST + /// only emit on real errors β€” bumping `component_errors_total` with the + /// `ack_failed` error_code. + #[test] + #[serial] + fn tcp_send_ack_error_emit_always_increments_component_errors_total() { + crate::test_util::trace_init(); + let errors_before = counter_value( + "component_errors_total", + &[ + ("error_code", "ack_failed"), + ("error_type", "writer_failed"), + ("stage", "sending"), + ("mode", "tcp"), + ], + ); + let shutdown_before = counter_value("connection_shutdown_total", &[("mode", "tcp")]); + + TcpSendAckError { + error: io::Error::from(io::ErrorKind::ConnectionReset), + } + .emit(); + + assert_eq!( + counter_value( + "component_errors_total", + &[ + ("error_code", "ack_failed"), + ("error_type", "writer_failed"), + ("stage", "sending"), + ("mode", "tcp"), + ], + ) - errors_before, + 1.0, + ); + assert_eq!( + counter_value("connection_shutdown_total", &[("mode", "tcp")]), + shutdown_before, + "TcpSendAckError must not bump the connection-close counter β€” \ + that is TcpSourceConnectionClosed's responsibility.", + ); + } +} diff --git a/src/internal_events/template.rs b/src/internal_events/template.rs index 147418042a0c6..db43081e2a22a 100644 --- a/src/internal_events/template.rs +++ b/src/internal_events/template.rs @@ -1,7 +1,8 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{ + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, error_stage, error_type, + }, }; #[derive(NamedInternalEvent)] @@ -29,7 +30,7 @@ impl InternalEvent for TemplateRenderingError<'_> { ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::TEMPLATE_FAILED, "stage" => error_stage::PROCESSING, ) diff --git a/src/internal_events/throttle.rs b/src/internal_events/throttle.rs index 274fedd7741b9..3757bbae40535 100644 --- a/src/internal_events/throttle.rs +++ b/src/internal_events/throttle.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{ComponentEventsDropped, INTENTIONAL, InternalEvent}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{ComponentEventsDropped, CounterName, INTENTIONAL, InternalEvent}, +}; #[derive(Debug, NamedInternalEvent)] pub(crate) struct ThrottleEventDiscarded { @@ -23,7 +24,7 @@ impl InternalEvent for ThrottleEventDiscarded { // if we should change the specification wording? Sort of a similar situation to the // `error_code` tag for the component errors metric, where it's meant to be optional and // only specified when relevant. - counter!("events_discarded_total", "key" => self.key).increment(1); // Deprecated. + counter!(CounterName::EventsDiscardedTotal, "key" => self.key).increment(1); // Deprecated. } emit!(ComponentEventsDropped:: { diff --git a/src/internal_events/udp.rs b/src/internal_events/udp.rs index 0602d4fdb3329..31e5f5c6533a4 100644 --- a/src/internal_events/udp.rs +++ b/src/internal_events/udp.rs @@ -1,7 +1,8 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{ + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, error_stage, error_type, + }, }; use crate::internal_events::SocketOutgoingConnectionError; @@ -14,7 +15,7 @@ pub struct UdpSocketConnectionEstablished; impl InternalEvent for UdpSocketConnectionEstablished { fn emit(self) { debug!(message = "Connected."); - counter!("connection_established_total", "mode" => "udp").increment(1); + counter!(CounterName::ConnectionEstablishedTotal, "mode" => "udp").increment(1); } } @@ -51,13 +52,13 @@ impl InternalEvent for UdpSendIncompleteError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::WRITER_FAILED, "stage" => error_stage::SENDING, ) .increment(1); // deprecated - counter!("connection_send_errors_total", "mode" => "udp").increment(1); + counter!(CounterName::ConnectionSendErrorsTotal, "mode" => "udp").increment(1); emit!(ComponentEventsDropped:: { count: 1, reason }); } @@ -80,7 +81,7 @@ impl InternalEvent for UdpChunkingError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::WRITER_FAILED, "stage" => error_stage::SENDING, ) diff --git a/src/internal_events/unix.rs b/src/internal_events/unix.rs index 6fab3348dc4d7..fc98ff7feacbd 100644 --- a/src/internal_events/unix.rs +++ b/src/internal_events/unix.rs @@ -2,11 +2,10 @@ use std::{io::Error, path::Path}; -use metrics::counter; -use vector_lib::NamedInternalEvent; use vector_lib::internal_event::{ - ComponentEventsDropped, InternalEvent, UNINTENTIONAL, error_stage, error_type, + ComponentEventsDropped, CounterName, InternalEvent, UNINTENTIONAL, error_stage, error_type, }; +use vector_lib::{NamedInternalEvent, counter}; use crate::internal_events::SocketOutgoingConnectionError; @@ -18,7 +17,7 @@ pub struct UnixSocketConnectionEstablished<'a> { impl InternalEvent for UnixSocketConnectionEstablished<'_> { fn emit(self) { debug!(message = "Connected.", path = ?self.path); - counter!("connection_established_total", "mode" => "unix").increment(1); + counter!(CounterName::ConnectionEstablishedTotal, "mode" => "unix").increment(1); } } @@ -59,7 +58,7 @@ impl InternalEvent for UnixSocketError<'_, E> { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::CONNECTION_FAILED, "stage" => error_stage::PROCESSING, ) @@ -84,7 +83,7 @@ impl InternalEvent for UnixSocketSendError<'_, E> { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::WRITER_FAILED, "stage" => error_stage::SENDING, ) @@ -112,7 +111,7 @@ impl InternalEvent for UnixSendIncompleteError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_type" => error_type::WRITER_FAILED, "stage" => error_stage::SENDING, ) @@ -139,7 +138,7 @@ impl InternalEvent for UnixSocketFileDeleteError<'_> { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "delete_socket_file", "error_type" => error_type::WRITER_FAILED, "stage" => error_stage::PROCESSING, diff --git a/src/internal_events/websocket.rs b/src/internal_events/websocket.rs index 960aec1916133..b842407ee0313 100644 --- a/src/internal_events/websocket.rs +++ b/src/internal_events/websocket.rs @@ -5,14 +5,15 @@ use std::{ fmt::{Debug, Display, Formatter, Result}, }; -use metrics::{counter, histogram}; use tokio_tungstenite::tungstenite::error::Error as TungsteniteError; use vector_common::{ internal_event::{error_stage, error_type}, json_size::JsonSize, }; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::InternalEvent; +use vector_lib::{ + NamedInternalEvent, counter, histogram, + internal_event::{CounterName, HistogramName, InternalEvent}, +}; pub const PROTOCOL: &str = "websocket"; @@ -22,7 +23,7 @@ pub struct WebSocketConnectionEstablished; impl InternalEvent for WebSocketConnectionEstablished { fn emit(self) { debug!(message = "Connected."); - counter!("connection_established_total").increment(1); + counter!(CounterName::ConnectionEstablishedTotal).increment(1); } } @@ -41,7 +42,7 @@ impl InternalEvent for WebSocketConnectionFailedError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "protocol" => PROTOCOL, "error_code" => "websocket_connection_failed", "error_type" => error_type::CONNECTION_FAILED, @@ -57,7 +58,7 @@ pub struct WebSocketConnectionShutdown; impl InternalEvent for WebSocketConnectionShutdown { fn emit(self) { warn!(message = "Closed by the server."); - counter!("connection_shutdown_total").increment(1); + counter!(CounterName::ConnectionShutdownTotal).increment(1); } } @@ -76,7 +77,7 @@ impl InternalEvent for WebSocketConnectionError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "protocol" => PROTOCOL, "error_code" => "websocket_connection_error", "error_type" => error_type::WRITER_FAILED, @@ -121,7 +122,7 @@ impl InternalEvent for WebSocketBytesReceived<'_> { kind = %self.kind ); let counter = counter!( - "component_received_bytes_total", + CounterName::ComponentReceivedBytesTotal, "url" => self.url.to_string(), "protocol" => self.protocol, "kind" => self.kind.to_string() @@ -150,17 +151,17 @@ impl InternalEvent for WebSocketMessageReceived<'_> { kind = %self.kind ); - let histogram = histogram!("component_received_events_count"); + let histogram = histogram!(HistogramName::ComponentReceivedEventsCount); histogram.record(self.count as f64); let counter = counter!( - "component_received_events_total", + CounterName::ComponentReceivedEventsTotal, "uri" => self.url.to_string(), "protocol" => PROTOCOL, "kind" => self.kind.to_string() ); counter.increment(self.count as u64); let counter = counter!( - "component_received_event_bytes_total", + CounterName::ComponentReceivedEventBytesTotal, "url" => self.url.to_string(), "protocol" => PROTOCOL, "kind" => self.kind.to_string() @@ -184,7 +185,7 @@ impl InternalEvent for WebSocketReceiveError<'_> { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "protocol" => PROTOCOL, "error_code" => "websocket_receive_error", "error_type" => error_type::CONNECTION_FAILED, @@ -209,7 +210,7 @@ impl InternalEvent for WebSocketSendError<'_> { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "protocol" => PROTOCOL, "error_code" => "websocket_send_error", "error_type" => error_type::CONNECTION_FAILED, diff --git a/src/internal_events/websocket_server.rs b/src/internal_events/websocket_server.rs index fd064c41f2f8a..bdace6b079da5 100644 --- a/src/internal_events/websocket_server.rs +++ b/src/internal_events/websocket_server.rs @@ -1,8 +1,9 @@ use std::{error::Error, fmt::Debug}; -use metrics::{counter, gauge}; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, gauge, + internal_event::{CounterName, GaugeName, InternalEvent, error_stage, error_type}, +}; #[derive(Debug, NamedInternalEvent)] pub struct WebSocketListenerConnectionEstablished { @@ -18,8 +19,8 @@ impl InternalEvent for WebSocketListenerConnectionEstablished { self.client_count ) ); - counter!("connection_established_total", &self.extra_tags).increment(1); - gauge!("active_clients", &self.extra_tags).set(self.client_count as f64); + counter!(CounterName::ConnectionEstablishedTotal, &self.extra_tags).increment(1); + gauge!(GaugeName::ActiveClients, &self.extra_tags).set(self.client_count as f64); } } @@ -49,7 +50,7 @@ impl InternalEvent for WebSocketListenerConnectionFailedError { ]); // Tags required by `component_errors_total` are dynamically added above. // ## skip check-validity-events ## - counter!("component_errors_total", &all_tags).increment(1); + counter!(CounterName::ComponentErrorsTotal, &all_tags).increment(1); } } @@ -67,8 +68,8 @@ impl InternalEvent for WebSocketListenerConnectionShutdown { self.client_count ) ); - counter!("connection_shutdown_total", &self.extra_tags).increment(1); - gauge!("active_clients", &self.extra_tags).set(self.client_count as f64); + counter!(CounterName::ConnectionShutdownTotal, &self.extra_tags).increment(1); + gauge!(GaugeName::ActiveClients, &self.extra_tags).set(self.client_count as f64); } } @@ -87,7 +88,7 @@ impl InternalEvent for WebSocketListenerSendError { stage = error_stage::SENDING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "ws_server_connection_error", "error_type" => error_type::WRITER_FAILED, "stage" => error_stage::SENDING, @@ -104,8 +105,8 @@ pub struct WebSocketListenerMessageSent { impl InternalEvent for WebSocketListenerMessageSent { fn emit(self) { - counter!("websocket_messages_sent_total", &self.extra_tags).increment(1); - counter!("websocket_bytes_sent_total", &self.extra_tags) + counter!(CounterName::WebsocketMessagesSentTotal, &self.extra_tags).increment(1); + counter!(CounterName::WebsocketBytesSentTotal, &self.extra_tags) .increment(self.message_size as u64); } } diff --git a/src/internal_events/windows.rs b/src/internal_events/windows.rs index 7b36e2fa62466..5ea6bae62aa25 100644 --- a/src/internal_events/windows.rs +++ b/src/internal_events/windows.rs @@ -1,6 +1,7 @@ -use metrics::counter; -use vector_lib::NamedInternalEvent; -use vector_lib::internal_event::{InternalEvent, error_stage, error_type}; +use vector_lib::{ + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, +}; #[derive(Debug, NamedInternalEvent)] pub struct WindowsServiceStart<'a> { @@ -16,7 +17,7 @@ impl InternalEvent for WindowsServiceStart<'_> { "Started Windows Service.", ); counter!( - "windows_service_start_total", + CounterName::WindowsServiceStartTotal, "already_started" => self.already_started.to_string(), ) .increment(1); @@ -37,7 +38,7 @@ impl InternalEvent for WindowsServiceStop<'_> { "Stopped Windows Service.", ); counter!( - "windows_service_stop_total", + CounterName::WindowsServiceStopTotal, "already_stopped" => self.already_stopped.to_string(), ) .increment(1); @@ -55,7 +56,7 @@ impl InternalEvent for WindowsServiceRestart<'_> { name = ?self.name, "Restarted Windows Service." ); - counter!("windows_service_restart_total").increment(1) + counter!(CounterName::WindowsServiceRestartTotal).increment(1) } } @@ -70,7 +71,7 @@ impl InternalEvent for WindowsServiceInstall<'_> { name = ?self.name, "Installed Windows Service.", ); - counter!("windows_service_install_total").increment(1); + counter!(CounterName::WindowsServiceInstallTotal).increment(1); } } @@ -85,7 +86,7 @@ impl InternalEvent for WindowsServiceUninstall<'_> { name = ?self.name, "Uninstalled Windows Service.", ); - counter!("windows_service_uninstall_total").increment(1); + counter!(CounterName::WindowsServiceUninstallTotal).increment(1); } } @@ -104,7 +105,7 @@ impl InternalEvent for WindowsServiceDoesNotExistError<'_> { stage = error_stage::PROCESSING, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "service_missing", "error_type" => error_type::CONDITION_FAILED, "stage" => error_stage::PROCESSING, diff --git a/src/internal_events/windows_event_log.rs b/src/internal_events/windows_event_log.rs index ae5363e2c52be..7ec356ba253ff 100644 --- a/src/internal_events/windows_event_log.rs +++ b/src/internal_events/windows_event_log.rs @@ -1,8 +1,7 @@ -use metrics::counter; use tracing::error; use vector_lib::{ - NamedInternalEvent, - internal_event::{InternalEvent, error_stage, error_type}, + NamedInternalEvent, counter, + internal_event::{CounterName, InternalEvent, error_stage, error_type}, }; #[derive(Debug, NamedInternalEvent)] @@ -25,7 +24,7 @@ impl InternalEvent for WindowsEventLogParseError { internal_log_rate_limit = true, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "parse_failed", "error_type" => error_type::PARSER_FAILED, "stage" => error_stage::PROCESSING, @@ -54,7 +53,7 @@ impl InternalEvent for WindowsEventLogQueryError { internal_log_rate_limit = true, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "query_failed", "error_type" => error_type::REQUEST_FAILED, "stage" => error_stage::RECEIVING, @@ -81,7 +80,7 @@ impl InternalEvent for WindowsEventLogBookmarkError { internal_log_rate_limit = true, ); counter!( - "component_errors_total", + CounterName::ComponentErrorsTotal, "error_code" => "bookmark_failed", "error_type" => error_type::REQUEST_FAILED, "stage" => error_stage::PROCESSING, diff --git a/src/internal_telemetry/allocations/mod.rs b/src/internal_telemetry/allocations/mod.rs index a2680256ec167..15ad82b3665fa 100644 --- a/src/internal_telemetry/allocations/mod.rs +++ b/src/internal_telemetry/allocations/mod.rs @@ -11,8 +11,11 @@ use std::{ }; use arr_macro::arr; -use metrics::{counter, gauge}; use rand_distr::num_traits::ToPrimitive; +use vector_common::{ + counter, gauge, + internal_event::{CounterName, GaugeName}, +}; use self::allocator::Tracer; pub(crate) use self::allocator::{ @@ -144,26 +147,26 @@ pub fn init_allocation_tracing() { let group_info = group.lock().unwrap(); if allocations_diff > 0 { counter!( - "component_allocated_bytes_total", "component_kind" => group_info.component_kind.clone(), + CounterName::ComponentAllocatedBytesTotal, "component_kind" => group_info.component_kind.clone(), "component_type" => group_info.component_type.clone(), "component_id" => group_info.component_id.clone()).increment(allocations_diff); } if deallocations_diff > 0 { counter!( - "component_deallocated_bytes_total", "component_kind" => group_info.component_kind.clone(), + CounterName::ComponentDeallocatedBytesTotal, "component_kind" => group_info.component_kind.clone(), "component_type" => group_info.component_type.clone(), "component_id" => group_info.component_id.clone()).increment(deallocations_diff); } if mem_used_diff > 0 { gauge!( - "component_allocated_bytes", "component_type" => group_info.component_type.clone(), + GaugeName::ComponentAllocatedBytes, "component_type" => group_info.component_type.clone(), "component_id" => group_info.component_id.clone(), "component_kind" => group_info.component_kind.clone()) .increment(mem_used_diff.to_f64().expect("failed to convert mem_used from int to float")); } if mem_used_diff < 0 { gauge!( - "component_allocated_bytes", "component_type" => group_info.component_type.clone(), + GaugeName::ComponentAllocatedBytes, "component_type" => group_info.component_type.clone(), "component_id" => group_info.component_id.clone(), "component_kind" => group_info.component_kind.clone()) .decrement(-mem_used_diff.to_f64().expect("failed to convert mem_used from int to float")); diff --git a/src/kafka.rs b/src/kafka.rs index ee652cc4f3f90..0c9315288e313 100644 --- a/src/kafka.rs +++ b/src/kafka.rs @@ -19,12 +19,11 @@ enum KafkaError { /// Supported compression types for Kafka. #[configurable_component] -#[derive(Clone, Copy, Debug, Derivative)] -#[derivative(Default)] +#[derive(Clone, Copy, Debug, Default)] #[serde(rename_all = "lowercase")] pub enum KafkaCompression { /// No compression. - #[derivative(Default)] + #[default] None, /// Gzip. diff --git a/src/lib.rs b/src/lib.rs index d4d4ab1eeb38a..170ca0b162e12 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -35,11 +35,15 @@ pub use indoc::indoc; // re-export codecs for convenience pub use vector_lib::codecs; -#[cfg(all(feature = "tikv-jemallocator", not(feature = "allocation-tracing")))] +#[cfg(all( + unix, + feature = "tikv-jemallocator", + not(feature = "allocation-tracing") +))] #[global_allocator] static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; -#[cfg(all(feature = "tikv-jemallocator", feature = "allocation-tracing"))] +#[cfg(all(unix, feature = "tikv-jemallocator", feature = "allocation-tracing"))] #[global_allocator] static ALLOC: self::internal_telemetry::allocations::Allocator = self::internal_telemetry::allocations::get_grouped_tracing_allocator( @@ -74,6 +78,12 @@ pub mod app; pub mod async_read; #[cfg(feature = "aws-config")] pub mod aws; +#[cfg(any( + feature = "sinks-azure_blob", + feature = "sinks-azure_logs_ingestion", + feature = "sources-azure_blob" +))] +pub mod azure; pub mod common; pub mod completion; mod convert_config; @@ -237,6 +247,8 @@ pub fn get_hostname() -> std::io::Result { }) } +pub(crate) use vector_lib::spawn_in_current_span; + /// Spawn a task with the given name. The name is only used if /// built with [`tokio_unstable`][tokio_unstable]. /// diff --git a/src/net.rs b/src/net.rs index f62abfa32974a..ed6e46d8bfb95 100644 --- a/src/net.rs +++ b/src/net.rs @@ -50,3 +50,129 @@ where pub fn set_keepalive(socket: &TcpStream, ttl: Duration) -> io::Result<()> { SockRef::from(socket).set_tcp_keepalive(&TcpKeepalive::new().with_time(ttl)) } + +// SSL_R_PROTOCOL_IS_SHUTDOWN from openssl/include/openssl/sslerr.h. Stable across +// OpenSSL 1.1.1 and 3.x. Not re-exported by the `openssl-sys` crate so we pin it here. +const SSL_R_PROTOCOL_IS_SHUTDOWN: std::ffi::c_int = 207; + +/// Returns true when an `io::Error` represents a peer-initiated, graceful TLS +/// shutdown (close_notify), rather than a real I/O failure. +/// +/// Two cases are recognized: +/// - `SSL_ERROR_ZERO_RETURN`: the peer sent `close_notify` and we observed it +/// during this I/O call. +/// - `SSL_R_PROTOCOL_IS_SHUTDOWN`: a subsequent write after the session was +/// already shut down ("ssl session has been shut down"). +pub fn is_graceful_tls_shutdown(err: &io::Error) -> bool { + let Some(ssl) = err + .get_ref() + .and_then(|inner| inner.downcast_ref::()) + else { + return false; + }; + if ssl.code() == openssl::ssl::ErrorCode::ZERO_RETURN { + return true; + } + ssl.ssl_error().is_some_and(|stack| { + stack + .errors() + .iter() + .any(|e| e.reason_code() == SSL_R_PROTOCOL_IS_SHUTDOWN) + }) +} + +#[cfg(test)] +mod tests { + use std::pin::Pin; + + use openssl::ssl::{SslAcceptor, SslConnector, SslFiletype, SslMethod, SslVerifyMode}; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::net::TcpListener; + use tokio_openssl::SslStream; + + use crate::tls::{TEST_PEM_CA_PATH, TEST_PEM_CRT_PATH, TEST_PEM_KEY_PATH}; + + use super::{TcpStream, io, is_graceful_tls_shutdown}; + + #[test] + fn plain_io_errors_are_not_graceful() { + for err in [ + io::Error::from(io::ErrorKind::BrokenPipe), + io::Error::from(io::ErrorKind::ConnectionReset), + io::Error::from(io::ErrorKind::UnexpectedEof), + io::Error::other("not an ssl error"), + ] { + assert!( + !is_graceful_tls_shutdown(&err), + "expected non-graceful, got graceful for {err:?}", + ); + } + } + + // Drives a real TLS handshake between two local sockets and completes a + // bidirectional SSL shutdown. A subsequent write surfaces a `std::io::Error` + // wrapping an `openssl::ssl::Error` from the same code path production hits, + // validating that the helper correctly identifies it as a graceful shutdown + // β€” without having to synthesize an `openssl::ssl::Error` (whose fields are + // crate-private). Bidirectional shutdown is what reliably elicits + // SSL_R_PROTOCOL_IS_SHUTDOWN; a half-closed session would still permit + // writes per RFC 5246. + #[tokio::test] + async fn detects_graceful_shutdown_from_real_ssl_stream() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + + let server = tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + let mut acceptor = SslAcceptor::mozilla_intermediate(SslMethod::tls()).unwrap(); + acceptor + .set_private_key_file(TEST_PEM_KEY_PATH, SslFiletype::PEM) + .unwrap(); + acceptor + .set_certificate_chain_file(TEST_PEM_CRT_PATH) + .unwrap(); + let acceptor = acceptor.build(); + let ssl = openssl::ssl::Ssl::new(acceptor.context()).unwrap(); + let mut tls = SslStream::new(ssl, stream).unwrap(); + Pin::new(&mut tls).accept().await.unwrap(); + // Cleanly close the SSL session β€” sends close_notify and waits for the peer's. + Pin::new(&mut tls).shutdown().await.unwrap(); + }); + + let mut connector = SslConnector::builder(SslMethod::tls()).unwrap(); + connector.set_ca_file(TEST_PEM_CA_PATH).unwrap(); + connector.set_verify(SslVerifyMode::NONE); + let ssl = connector + .build() + .configure() + .unwrap() + .into_ssl("localhost") + .unwrap(); + let stream = TcpStream::connect(addr).await.unwrap(); + let mut tls = SslStream::new(ssl, stream).unwrap(); + Pin::new(&mut tls).connect().await.unwrap(); + + // Drain the server's close_notify so our SSL state observes the peer shutdown. + let mut buf = [0u8; 1]; + let n = tls.read(&mut buf).await.unwrap(); + assert_eq!(n, 0, "expected EOF from peer's close_notify"); + + // Complete the bidirectional SSL shutdown locally. Once both sides are + // shut down, OpenSSL marks the session as SHUTDOWN and any further write + // returns SSL_R_PROTOCOL_IS_SHUTDOWN ("ssl session has been shut down"). + Pin::new(&mut tls).shutdown().await.unwrap(); + + let err = tls + .write_all(b"too late") + .await + .expect_err("write after bidirectional shutdown should fail"); + + assert!( + is_graceful_tls_shutdown(&err), + "expected graceful shutdown detection, got: {err:?} (inner: {:?})", + err.get_ref(), + ); + + server.await.unwrap(); + } +} diff --git a/src/proto/mod.rs b/src/proto/mod.rs index bf576c8f7e542..75796939fb368 100644 --- a/src/proto/mod.rs +++ b/src/proto/mod.rs @@ -1,9 +1,9 @@ -#[cfg(any(feature = "sources-vector", feature = "sinks-vector"))] -use crate::event::proto as event; - #[cfg(any(feature = "sources-vector", feature = "sinks-vector"))] pub mod vector; +#[cfg(feature = "api")] +pub mod observability; + #[cfg(feature = "sinks-datadog_metrics")] pub mod fds { use std::sync::OnceLock; diff --git a/src/proto/observability.rs b/src/proto/observability.rs new file mode 100644 index 0000000000000..f62f9adec9f97 --- /dev/null +++ b/src/proto/observability.rs @@ -0,0 +1,13 @@ +#![allow(clippy::clone_on_ref_ptr)] +#![allow(warnings, clippy::pedantic, clippy::nursery)] + +tonic::include_proto!("vector.observability.v1"); + +pub use observability_service_client::ObservabilityServiceClient as Client; +pub use observability_service_server::{ + ObservabilityService as Service, ObservabilityServiceServer as Server, +}; + +/// File descriptor set for gRPC reflection +pub const FILE_DESCRIPTOR_SET: &[u8] = + include_bytes!(concat!(env!("OUT_DIR"), "/protobuf-fds.bin")); diff --git a/src/secrets/exec.rs b/src/secrets/exec.rs index 081c752c2b36a..2a6e8c464065f 100644 --- a/src/secrets/exec.rs +++ b/src/secrets/exec.rs @@ -179,7 +179,7 @@ async fn query_backend( .ok_or("unable to acquire stdout")?; let query = serde_json::to_vec(&query)?; - tokio::spawn(async move { stdin.write_all(&query).await }); + crate::spawn_in_current_span(async move { stdin.write_all(&query).await }); let timeout = time::sleep(time::Duration::from_secs(timeout)); tokio::pin!(timeout); diff --git a/src/service.rs b/src/service.rs index 5f384b888490d..974ac1ddeaf70 100644 --- a/src/service.rs +++ b/src/service.rs @@ -101,6 +101,18 @@ impl InstallOpts { } } +#[derive(Parser, Debug)] +#[command(rename_all = "kebab-case")] +struct UninstallOpts { + /// The name of the service. + #[arg(long)] + name: Option, + + /// How long to wait for the service to stop before uninstalling, in seconds. + #[arg(default_value = "10", long)] + stop_timeout: u32, +} + #[derive(Parser, Debug)] #[command(rename_all = "kebab-case")] struct RestartOpts { @@ -113,6 +125,18 @@ struct RestartOpts { stop_timeout: u32, } +#[derive(Parser, Debug)] +#[command(rename_all = "kebab-case")] +struct StopOpts { + /// The name of the service. + #[arg(long)] + name: Option, + + /// How long to wait for the service to stop, in seconds. + #[arg(default_value = "10", long)] + stop_timeout: u32, +} + impl RestartOpts { fn service_info(&self) -> ServiceInfo { let mut default_service = ServiceInfo::default(); @@ -123,6 +147,26 @@ impl RestartOpts { } } +impl UninstallOpts { + fn service_info(&self) -> ServiceInfo { + let mut default_service = ServiceInfo::default(); + let service_name = self.name.as_deref().unwrap_or(DEFAULT_SERVICE_NAME); + + default_service.name = OsString::from(service_name); + default_service + } +} + +impl StopOpts { + fn service_info(&self) -> ServiceInfo { + let mut default_service = ServiceInfo::default(); + let service_name = self.name.as_deref().unwrap_or(DEFAULT_SERVICE_NAME); + + default_service.name = OsString::from(service_name); + default_service + } +} + #[derive(Parser, Debug)] #[command(rename_all = "kebab-case")] struct StandardOpts { @@ -147,11 +191,11 @@ enum SubCommand { /// Install the service. Install(InstallOpts), /// Uninstall the service. - Uninstall(StandardOpts), + Uninstall(UninstallOpts), /// Start the service. Start(StandardOpts), /// Stop the service. - Stop(StandardOpts), + Stop(StopOpts), /// Restart the service. Restart(RestartOpts), } @@ -182,9 +226,9 @@ impl Default for ServiceInfo { #[derive(Debug, Clone, PartialEq)] enum ControlAction { Install, - Uninstall, + Uninstall { stop_timeout: Duration }, Start, - Stop, + Stop { stop_timeout: Duration }, Restart { stop_timeout: Duration }, } @@ -196,10 +240,17 @@ pub fn cmd(opts: &Opts) -> exitcode::ExitCode { control_service(&opts.service_info(), ControlAction::Install) } SubCommand::Uninstall(opts) => { - control_service(&opts.service_info(), ControlAction::Uninstall) + let stop_timeout = Duration::from_secs(opts.stop_timeout as u64); + control_service( + &opts.service_info(), + ControlAction::Uninstall { stop_timeout }, + ) } SubCommand::Start(opts) => control_service(&opts.service_info(), ControlAction::Start), - SubCommand::Stop(opts) => control_service(&opts.service_info(), ControlAction::Stop), + SubCommand::Stop(opts) => { + let stop_timeout = Duration::from_secs(opts.stop_timeout as u64); + control_service(&opts.service_info(), ControlAction::Stop { stop_timeout }) + } SubCommand::Restart(opts) => { let stop_timeout = Duration::from_secs(opts.stop_timeout as u64); control_service( @@ -233,17 +284,17 @@ fn control_service(service: &ServiceInfo, action: ControlAction) -> exitcode::Ex &service_definition, vector_windows::service_control::ControlAction::Install, ), - ControlAction::Uninstall => vector_windows::service_control::control( + ControlAction::Uninstall { stop_timeout } => vector_windows::service_control::control( &service_definition, - vector_windows::service_control::ControlAction::Uninstall, + vector_windows::service_control::ControlAction::Uninstall { stop_timeout }, ), ControlAction::Start => vector_windows::service_control::control( &service_definition, vector_windows::service_control::ControlAction::Start, ), - ControlAction::Stop => vector_windows::service_control::control( + ControlAction::Stop { stop_timeout } => vector_windows::service_control::control( &service_definition, - vector_windows::service_control::ControlAction::Stop, + vector_windows::service_control::ControlAction::Stop { stop_timeout }, ), ControlAction::Restart { stop_timeout } => vector_windows::service_control::control( &service_definition, diff --git a/src/sinks/appsignal/config.rs b/src/sinks/appsignal/config.rs index 9e8c000f995a3..fea94af05b132 100644 --- a/src/sinks/appsignal/config.rs +++ b/src/sinks/appsignal/config.rs @@ -21,7 +21,7 @@ use crate::{ prelude::{SinkConfig, SinkContext}, util::{ BatchConfig, Compression, ServiceBuilderExt, SinkBatchSettings, TowerRequestConfig, - http::HttpStatusRetryLogic, + http::{HttpStatusRetryLogic, RetryStrategy}, }, }, }; @@ -67,6 +67,10 @@ pub(super) struct AppsignalConfig { skip_serializing_if = "crate::serde::is_default" )] acknowledgements: AcknowledgementsConfig, + + #[configurable(derived)] + #[serde(default)] + retry_strategy: RetryStrategy, } pub(super) fn default_endpoint() -> String { @@ -99,7 +103,10 @@ impl AppsignalConfig { let request_opts = self.request; let request_settings = request_opts.into_settings(); - let retry_logic = HttpStatusRetryLogic::new(|req: &AppsignalResponse| req.http_status); + let retry_logic = HttpStatusRetryLogic::new( + |req: &AppsignalResponse| req.http_status, + self.retry_strategy.clone(), + ); let service = ServiceBuilder::new() .settings(request_settings, retry_logic) diff --git a/src/sinks/aws_cloudwatch_logs/sink.rs b/src/sinks/aws_cloudwatch_logs/sink.rs index da272b2fafd28..7b47a5b34ff8f 100644 --- a/src/sinks/aws_cloudwatch_logs/sink.rs +++ b/src/sinks/aws_cloudwatch_logs/sink.rs @@ -49,7 +49,7 @@ where let age_range = start..end; future::ready(age_range.contains(&req.timestamp)) }) - .batched_partitioned(CloudwatchPartitioner, || { + .batched_partitioned(CloudwatchPartitioner, batcher_settings.timeout, |_| { batcher_settings.as_byte_size_config() }) .map(|(key, events)| { diff --git a/src/sinks/aws_s3/config.rs b/src/sinks/aws_s3/config.rs index 08ba92245d65c..6853c57360ac4 100644 --- a/src/sinks/aws_s3/config.rs +++ b/src/sinks/aws_s3/config.rs @@ -1,9 +1,13 @@ use aws_sdk_s3::Client as S3Client; use tower::ServiceBuilder; +#[cfg(feature = "codecs-parquet")] +use vector_lib::codecs::BatchEncoder; +#[cfg(feature = "codecs-parquet")] +use vector_lib::codecs::encoding::{BatchSerializerConfig, format::ParquetSerializerConfig}; use vector_lib::{ TimeZone, codecs::{ - TextSerializerConfig, + EncoderKind, TextSerializerConfig, encoding::{Framer, FramingConfig}, }, configurable::configurable_component, @@ -33,6 +37,21 @@ use crate::{ tls::TlsConfig, }; +/// Batch encoding configuration for the `aws_s3` sink. +#[cfg(feature = "codecs-parquet")] +#[configurable_component] +#[derive(Clone, Debug)] +#[serde(tag = "codec", rename_all = "snake_case")] +#[configurable(metadata( + docs::enum_tag_description = "The codec to use for batch encoding events." +))] +pub enum S3BatchEncoding { + /// Encodes events in [Apache Parquet][apache_parquet] columnar format. + /// + /// [apache_parquet]: https://parquet.apache.org/ + Parquet(ParquetSerializerConfig), +} + /// Configuration for the `aws_s3` sink. #[configurable_component(sink( "aws_s3", @@ -105,6 +124,16 @@ pub struct S3SinkConfig { #[serde(flatten)] pub encoding: EncodingConfigWithFraming, + /// Batch encoding configuration for columnar formats. + /// + /// When set, events are encoded together as a batch in a columnar format (Parquet) + /// instead of the standard per-event framing-based encoding. The columnar format handles + /// its own internal compression, so the top-level `compression` setting is bypassed. + #[cfg(feature = "codecs-parquet")] + #[configurable(derived)] + #[serde(default)] + pub batch_encoding: Option, + /// Compression configuration. /// /// All compression algorithms use the default compression level unless otherwise specified. @@ -176,6 +205,8 @@ impl GenerateConfig for S3SinkConfig { options: S3Options::default(), region: RegionOrEndpoint::default(), encoding: (None::, TextSerializerConfig::default()).into(), + #[cfg(feature = "codecs-parquet")] + batch_encoding: None, compression: Compression::gzip_default(), batch: BatchConfig::default(), request: TowerRequestConfig::default(), @@ -201,6 +232,12 @@ impl SinkConfig for S3SinkConfig { } fn input(&self) -> Input { + #[cfg(feature = "codecs-parquet")] + if let Some(batch_encoding) = &self.batch_encoding { + let S3BatchEncoding::Parquet(parquet_config) = batch_encoding; + let resolved = BatchSerializerConfig::Parquet(parquet_config.clone()); + return Input::new(resolved.input_type()); + } Input::new(self.encoding.config().1.input_type()) } @@ -246,8 +283,57 @@ impl S3SinkConfig { let partitioner = S3KeyPartitioner::new(key_prefix, ssekms_key_id, None); let transformer = self.encoding.transformer(); + + // When batch_encoding is configured (e.g., Parquet), use batch mode + // with internal compression and appropriate file extension. + #[cfg(feature = "codecs-parquet")] + if let Some(batch_encoding) = &self.batch_encoding { + let S3BatchEncoding::Parquet(parquet_config) = batch_encoding; + let resolved_batch_config = BatchSerializerConfig::Parquet(parquet_config.clone()); + + let batch_serializer = resolved_batch_config.build_batch_serializer()?; + let batch_encoder = BatchEncoder::new(batch_serializer); + + // Auto-detect Content-Type from batch format. Users can still + // override via `options.content_type`; we only set it when unset. + let mut api_options = self.options.clone(); + if api_options.content_type.is_none() { + api_options.content_type = batch_encoder.content_type().map(|s| s.to_string()); + } + + let encoder = EncoderKind::Batch(batch_encoder); + + let filename_extension = self.filename_extension.clone().or_else(|| { + Some( + match batch_encoding { + S3BatchEncoding::Parquet(_) => "parquet", + } + .to_string(), + ) + }); + + if self.compression != Compression::None { + warn!("Top level compression setting ignored when batch_encoding set to parquet.") + } + + let request_options = S3RequestOptions { + bucket: self.bucket.clone(), + api_options, + filename_extension, + filename_time_format: self.filename_time_format.clone(), + filename_append_uuid: self.filename_append_uuid, + encoder: (transformer, encoder), + // Batch formats handle their own compression internally + compression: Compression::None, + filename_tz_offset: offset, + }; + + let sink = S3Sink::new(service, request_options, partitioner, batch_settings); + return Ok(VectorSink::from_event_streamsink(sink)); + } + let (framer, serializer) = self.encoding.build(SinkType::MessageBased)?; - let encoder = Encoder::::new(framer, serializer); + let encoder = EncoderKind::Framed(Box::new(Encoder::::new(framer, serializer))); let request_options = S3RequestOptions { bucket: self.bucket.clone(), @@ -289,4 +375,231 @@ mod tests { fn generate_config() { crate::test_util::test_generate_config::(); } + + /// Correct TOML shape: `batch_encoding.codec = "parquet"` with `schema_mode = "auto_infer"`. + #[cfg(feature = "codecs-parquet")] + #[test] + fn parquet_batch_encoding_correct_toml_shape() { + let config: S3SinkConfig = toml::from_str( + r#" + bucket = "test-bucket" + compression = "none" + + [encoding] + codec = "text" + + [batch_encoding] + schema_mode = "auto_infer" + codec = "parquet" + + [batch_encoding.compression] + algorithm = "snappy" + + "#, + ) + .expect("correct batch_encoding shape should parse"); + + let batch_enc = config + .batch_encoding + .expect("batch_encoding should be Some"); + let super::S3BatchEncoding::Parquet(ref p) = batch_enc; + use vector_lib::codecs::encoding::format::{ParquetCompression, ParquetSchemaMode}; + assert_eq!(p.schema_mode, ParquetSchemaMode::AutoInfer); + assert_eq!(p.compression, ParquetCompression::Snappy); + } + + /// Content-Type must be auto-detected as `application/vnd.apache.parquet` + /// when `batch_encoding` is set and `content_type` is not explicitly provided. + #[cfg(feature = "codecs-parquet")] + #[test] + fn parquet_content_type_auto_detected() { + use vector_lib::codecs::encoding::format::{ + ParquetCompression, ParquetSchemaMode, ParquetSerializerConfig, + }; + + use crate::sinks::s3_common::config::S3Options; + use crate::sinks::util::{BatchConfig, BulkSizeBasedDefaultBatchSettings, Compression}; + use vector_lib::codecs::TextSerializerConfig; + use vector_lib::codecs::encoding::{BatchSerializerConfig, FramingConfig}; + + let parquet_config = ParquetSerializerConfig { + schema_mode: ParquetSchemaMode::AutoInfer, + compression: ParquetCompression::Snappy, + ..Default::default() + }; + + let config = S3SinkConfig { + bucket: "test".to_string(), + key_prefix: super::default_key_prefix(), + filename_time_format: super::default_filename_time_format(), + filename_append_uuid: true, + filename_extension: None, + options: S3Options::default(), + region: crate::aws::RegionOrEndpoint::with_both("us-east-1", "http://localhost:4566"), + encoding: (None::, TextSerializerConfig::default()).into(), + batch_encoding: Some(super::S3BatchEncoding::Parquet(parquet_config)), + compression: Compression::None, + batch: BatchConfig::::default(), + request: Default::default(), + tls: Default::default(), + auth: Default::default(), + acknowledgements: Default::default(), + timezone: Default::default(), + force_path_style: true, + retry_strategy: Default::default(), + }; + + let super::S3BatchEncoding::Parquet(p) = config.batch_encoding.as_ref().unwrap(); + let batch_config = BatchSerializerConfig::Parquet(p.clone()); + let batch_serializer = batch_config.build_batch_serializer().unwrap(); + let batch_encoder = vector_lib::codecs::BatchEncoder::new(batch_serializer); + + let mut api_options = config.options.clone(); + if api_options.content_type.is_none() { + api_options.content_type = batch_encoder.content_type().map(|s| s.to_string()); + } + + assert_eq!( + api_options.content_type.as_deref(), + Some("application/vnd.apache.parquet"), + "Content-Type must be auto-detected for Parquet" + ); + } + + /// When user explicitly sets `content_type`, the auto-detection must not override it. + #[cfg(feature = "codecs-parquet")] + #[test] + fn parquet_content_type_user_override_preserved() { + let config: S3SinkConfig = toml::from_str( + r#" + bucket = "test-bucket" + compression = "none" + content_type = "application/octet-stream" + + [encoding] + codec = "text" + + [batch_encoding] + codec = "parquet" + schema_mode = "auto_infer" + + [batch_encoding.compression] + algorithm = "gzip" + level = 9 + "#, + ) + .unwrap(); + + let super::S3BatchEncoding::Parquet(p) = config.batch_encoding.as_ref().unwrap(); + let batch_config = vector_lib::codecs::encoding::BatchSerializerConfig::Parquet(p.clone()); + let batch_serializer = batch_config.build_batch_serializer().unwrap(); + let batch_encoder = vector_lib::codecs::BatchEncoder::new(batch_serializer); + + let mut api_options = config.options.clone(); + if api_options.content_type.is_none() { + api_options.content_type = batch_encoder.content_type().map(|s| s.to_string()); + } + + assert_eq!( + api_options.content_type.as_deref(), + Some("application/octet-stream"), + "User-specified Content-Type must not be overridden" + ); + } + + /// Codecs other than `parquet` must be rejected at parse time, since + /// `S3BatchEncoding` only exposes the `parquet` variant. + #[cfg(feature = "codecs-parquet")] + #[test] + fn parquet_batch_encoding_rejects_unsupported_codec() { + let err = serde_yaml::from_str::( + r#" + bucket: test-bucket + compression: none + encoding: + codec: text + batch_encoding: + codec: arrow_stream + "#, + ) + .unwrap_err(); + + assert!( + err.to_string().contains("arrow_stream"), + "expected error to mention the offending codec, got: {err}" + ); + } + + /// Explicit filename_extension overrides the `.parquet` default. + #[cfg(feature = "codecs-parquet")] + #[test] + fn parquet_filename_extension_user_override() { + let config: S3SinkConfig = toml::from_str( + r#" + bucket = "test-bucket" + compression = "none" + filename_extension = "pq" + + [encoding] + codec = "text" + + [batch_encoding] + codec = "parquet" + schema_mode = "auto_infer" + "#, + ) + .unwrap(); + + assert_eq!(config.filename_extension.as_deref(), Some("pq")); + } + + /// `schema_mode` defaults to `relaxed` when not specified. + #[cfg(feature = "codecs-parquet")] + #[test] + fn parquet_schema_mode_defaults_to_relaxed() { + use vector_lib::codecs::encoding::format::ParquetSchemaMode; + + let config: S3SinkConfig = toml::from_str( + r#" + bucket = "test-bucket" + compression = "none" + + [encoding] + codec = "text" + + [batch_encoding] + codec = "parquet" + "#, + ) + .unwrap(); + + let super::S3BatchEncoding::Parquet(p) = config.batch_encoding.unwrap(); + assert_eq!(p.schema_mode, ParquetSchemaMode::Relaxed); + } + + /// Explicit `schema_mode = "strict"` is correctly parsed. + #[cfg(feature = "codecs-parquet")] + #[test] + fn parquet_schema_mode_strict_parsed() { + use vector_lib::codecs::encoding::format::ParquetSchemaMode; + + let config: S3SinkConfig = toml::from_str( + r#" + bucket = "test-bucket" + compression = "none" + + [encoding] + codec = "text" + + [batch_encoding] + codec = "parquet" + schema_mode = "strict" + schema_file = "tmp/something.schema" + "#, + ) + .unwrap(); + + let super::S3BatchEncoding::Parquet(p) = config.batch_encoding.unwrap(); + assert_eq!(p.schema_mode, ParquetSchemaMode::Strict); + } } diff --git a/src/sinks/aws_s3/integration_tests.rs b/src/sinks/aws_s3/integration_tests.rs index 31907d70b5c0f..3a7e156208387 100644 --- a/src/sinks/aws_s3/integration_tests.rs +++ b/src/sinks/aws_s3/integration_tests.rs @@ -1,10 +1,14 @@ #![cfg(all(test, feature = "aws-s3-integration-tests"))] use std::{ + collections::HashSet, io::{BufRead, BufReader}, + num::NonZeroU64, time::Duration, }; +#[cfg(feature = "codecs-parquet")] +use super::config::S3BatchEncoding; use aws_sdk_s3::{ Client as S3Client, operation::{create_bucket::CreateBucketError, get_object::GetObjectOutput}, @@ -18,10 +22,12 @@ use bytes::Buf; use flate2::read::MultiGzDecoder; use futures::{Stream, stream}; use similar_asserts::assert_eq; +use tempfile::TempDir; use tokio_stream::StreamExt; use vector_lib::{ + buffers::{BufferConfig, BufferType, WhenFull}, codecs::{TextSerializerConfig, encoding::FramingConfig}, - config::proxy::ProxyConfig, + config::{ComponentKey, proxy::ProxyConfig}, event::{BatchNotifier, BatchStatus, BatchStatusReceiver, Event, EventArray, LogEvent}, }; @@ -29,18 +35,20 @@ use super::S3SinkConfig; use crate::{ aws::{AwsAuthentication, RegionOrEndpoint, create_client}, common::s3::S3ClientBuilder, - config::SinkContext, + config::{Config, SinkContext}, sinks::{ aws_s3::config::default_filename_time_format, s3_common::config::{S3Options, S3ServerSideEncryption}, util::{BatchConfig, Compression, TowerRequestConfig}, }, test_util::{ + self, components::{ AWS_SINK_TAGS, COMPONENT_ERROR_TAGS, run_and_assert_sink_compliance, run_and_assert_sink_error, }, - random_lines_with_stream, random_string, + mock::basic_source, + random_lines_with_stream, random_string, start_topology, }, }; @@ -56,8 +64,10 @@ async fn s3_insert_message_into_with_flat_key_prefix() { create_bucket(&bucket, false).await; - let mut config = config(&bucket, 1000000); - config.key_prefix = "test-prefix".to_string(); + let config = S3SinkConfig { + key_prefix: "test-prefix".to_string(), + ..config(&bucket, 1000000, 5.0) + }; let prefix = config.key_prefix.clone(); let service = config.create_service(&cx.globals.proxy).await.unwrap(); let sink = config.build_processor(service, cx).unwrap(); @@ -90,8 +100,10 @@ async fn s3_insert_message_into_with_folder_key_prefix() { create_bucket(&bucket, false).await; - let mut config = config(&bucket, 1000000); - config.key_prefix = "test-prefix/".to_string(); + let config = S3SinkConfig { + key_prefix: "test-prefix/".to_string(), + ..config(&bucket, 1000000, 5.0) + }; let prefix = config.key_prefix.clone(); let service = config.create_service(&cx.globals.proxy).await.unwrap(); let sink = config.build_processor(service, cx).unwrap(); @@ -124,11 +136,16 @@ async fn s3_insert_message_into_with_ssekms_key_id() { create_bucket(&bucket, false).await; - let mut config = config(&bucket, 1000000); - config.key_prefix = "test-prefix".to_string(); + let config = S3SinkConfig { + key_prefix: "test-prefix".to_string(), + options: S3Options { + server_side_encryption: Some(S3ServerSideEncryption::AwsKms), + ssekms_key_id: Some("alias/aws/s3".to_string()), + ..S3Options::default() + }, + ..config(&bucket, 1000000, 5.0) + }; let prefix = config.key_prefix.clone(); - config.options.server_side_encryption = Some(S3ServerSideEncryption::AwsKms); - config.options.ssekms_key_id = Some("alias/aws/s3".to_string()); let service = config.create_service(&cx.globals.proxy).await.unwrap(); let sink = config.build_processor(service, cx).unwrap(); @@ -165,7 +182,7 @@ async fn s3_rotate_files_after_the_buffer_size_is_reached() { key_prefix: format!("{}/{}", random_string(10), "{{i}}"), filename_time_format: "waitsforfullbatch".into(), filename_append_uuid: false, - ..config(&bucket, 10) + ..config(&bucket, 10, 5.0) }; let prefix = config.key_prefix.clone(); let service = config.create_service(&cx.globals.proxy).await.unwrap(); @@ -223,7 +240,7 @@ async fn s3_gzip() { let config = S3SinkConfig { compression: Compression::gzip_default(), filename_time_format: "%s%f".into(), - ..config(&bucket, batch_size) + ..config(&bucket, batch_size, 5.0) }; let prefix = config.key_prefix.clone(); @@ -268,7 +285,7 @@ async fn s3_zstd() { let config = S3SinkConfig { compression: Compression::zstd_default(), filename_time_format: "%s%f".into(), - ..config(&bucket, batch_size) + ..config(&bucket, batch_size, 5.0) }; let prefix = config.key_prefix.clone(); @@ -332,7 +349,7 @@ async fn s3_insert_message_into_object_lock() { .await .unwrap(); - let config = config(&bucket, 1000000); + let config = config(&bucket, 1000000, 5.0); let prefix = config.key_prefix.clone(); let service = config.create_service(&cx.globals.proxy).await.unwrap(); let sink = config.build_processor(service, cx).unwrap(); @@ -362,9 +379,10 @@ async fn acknowledges_failures() { create_bucket(&bucket, false).await; - let mut config = config(&bucket, 1); - // Break the bucket name - config.bucket = format!("BREAK{}IT", config.bucket); + let config = S3SinkConfig { + bucket: format!("BREAK{}IT", &bucket), // Break the bucket name + ..config(&bucket, 1, 5.0) + }; let prefix = config.key_prefix.clone(); let service = config.create_service(&cx.globals.proxy).await.unwrap(); let sink = config.build_processor(service, cx).unwrap(); @@ -383,7 +401,7 @@ async fn s3_healthchecks() { create_bucket(&bucket, false).await; - let config = config(&bucket, 1); + let config = config(&bucket, 1, 5.0); let service = config .create_service(&ProxyConfig::from_env()) .await @@ -397,7 +415,7 @@ async fn s3_healthchecks() { #[tokio::test] async fn s3_healthchecks_invalid_bucket() { - let config = config("s3_healthchecks_invalid_bucket", 1); + let config = config("s3_healthchecks_invalid_bucket", 1, 5.0); let service = config .create_service(&ProxyConfig::from_env()) .await @@ -419,31 +437,7 @@ async fn s3_flush_on_exhaustion() { create_bucket(&bucket, false).await; // batch size of ten events, timeout of ten seconds - let config = { - let mut batch = BatchConfig::default(); - batch.max_events = Some(10); - batch.timeout_secs = Some(10.0); - - S3SinkConfig { - bucket: bucket.to_string(), - key_prefix: random_string(10) + "/date=%F", - filename_time_format: default_filename_time_format(), - filename_append_uuid: true, - filename_extension: None, - options: S3Options::default(), - region: RegionOrEndpoint::with_both("us-east-1", s3_address()), - encoding: (None::, TextSerializerConfig::default()).into(), - compression: Compression::None, - batch, - request: TowerRequestConfig::default(), - tls: Default::default(), - auth: Default::default(), - acknowledgements: Default::default(), - timezone: Default::default(), - force_path_style: true, - retry_strategy: Default::default(), - } - }; + let config = config(&bucket, 10, 10.0); let prefix = config.key_prefix.clone(); let service = config.create_service(&cx.globals.proxy).await.unwrap(); let sink = config.build_processor(service, cx).unwrap(); @@ -490,6 +484,211 @@ async fn s3_flush_on_exhaustion() { assert_eq!(lines, response_lines); // if all events are received, and lines.len() < batch size, then a flush was performed. } +#[cfg(feature = "codecs-parquet")] +#[tokio::test] +async fn s3_parquet_insert_message() { + use vector_lib::codecs::encoding::format::{ + ParquetCompression, ParquetSchemaMode, ParquetSerializerConfig, + }; + + let cx = SinkContext::default(); + let bucket = uuid::Uuid::new_v4().to_string(); + create_bucket(&bucket, false).await; + + let parquet_config = ParquetSerializerConfig { + schema_mode: ParquetSchemaMode::AutoInfer, + compression: ParquetCompression::Snappy, + ..Default::default() + }; + + let config = S3SinkConfig { + batch_encoding: Some(S3BatchEncoding::Parquet(parquet_config)), + ..config(&bucket, 100, 5.0) + }; + + let prefix = config.key_prefix.clone(); + let service = config.create_service(&cx.globals.proxy).await.unwrap(); + let sink = config.build_processor(service, cx).unwrap(); + + let (batch_notifier, receiver) = BatchNotifier::new_with_receiver(); + let events: Vec = (0..10) + .map(|i| { + let mut log = LogEvent::from(format!("message_{}", i)); + log.insert("host", format!("host_{}", i % 3)); + Event::from(log).with_batch_notifier(&batch_notifier) + }) + .collect(); + + drop(batch_notifier); + run_and_assert_sink_compliance(sink, stream::iter(events), &AWS_SINK_TAGS).await; + assert_eq!(receiver.await, BatchStatus::Delivered); + + let keys = get_keys(&bucket, prefix).await; + assert_eq!(keys.len(), 1); + + let key = keys[0].clone(); + assert!( + key.ends_with(".parquet"), + "Expected .parquet extension, got: {}", + key + ); + + // Download and validate Parquet file + let obj = get_object(&bucket, key).await; + assert_eq!(obj.content_encoding, None); + + let body = obj.body.collect().await.unwrap().into_bytes(); + assert!(body.len() >= 4, "Output too short to be valid Parquet"); + assert_eq!(&body[..4], b"PAR1", "Missing Parquet magic bytes"); + + // Verify we can read rows from the Parquet file + use bytes::Bytes; + use parquet::file::reader::{FileReader, SerializedFileReader}; + use parquet::record::reader::RowIter; + + let reader = + SerializedFileReader::new(Bytes::copy_from_slice(&body)).expect("Invalid Parquet file"); + let row_count = RowIter::from_file_into(Box::new(reader)).count(); + assert_eq!(row_count, 10, "Expected 10 rows in Parquet file"); + + // Verify schema has our columns + let reader = + SerializedFileReader::new(Bytes::copy_from_slice(&body)).expect("Invalid Parquet file"); + let schema = reader.metadata().file_metadata().schema_descr(); + let columns: Vec = schema + .columns() + .iter() + .map(|c| c.name().to_string()) + .collect(); + assert!(columns.contains(&"message".to_string())); + assert!(columns.contains(&"host".to_string())); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn s3_disk_buffer_reload_delivers_all_events() { + test_util::trace_init(); + + let bucket = uuid::Uuid::new_v4().to_string(); + create_bucket(&bucket, false).await; + + let data_dir = TempDir::new().expect("Could not create tempdir"); + + // batch.timeout_secs is deliberately large (300 s) so that the test would + // hang if the reload waited for the batch timer instead of cancelling it. + let s3_config = config(&bucket, 10, 300.0); + let prefix = s3_config.key_prefix.clone(); + + // Build topology + let (mut source_tx, source_config) = basic_source(); + + let mut old_config = Config::builder(); + old_config.global.data_dir = Some(data_dir.path().to_path_buf()); + old_config.add_source("in", source_config); + old_config.add_sink("out", &["in"], s3_config); + + let sink_key = ComponentKey::from("out"); + old_config.sinks[&sink_key].buffer = BufferConfig::Single(BufferType::DiskV2 { + max_size: NonZeroU64::new(268435488).unwrap(), + when_full: WhenFull::Block, + }); + + // Clone config before building so we can create the reload config. + let mut new_config = old_config.clone(); + new_config.sinks[&sink_key].buffer = BufferConfig::Single(BufferType::DiskV2 { + max_size: NonZeroU64::new(536870912).unwrap(), + when_full: WhenFull::Block, + }); + + // 1. Start topology with initial disk buffer config. + let (mut topology, _crash) = start_topology(old_config.build().unwrap(), false).await; + + // 2. Send first batch of events (enough to trigger batch.max_events flush). + let (pre_lines, pre_events, pre_receiver) = make_events_batch(100, 10); + for event in pre_events.collect::>().await { + source_tx.send_event(event).await.unwrap(); + } + + // 3. Wait for events to appear in S3. + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + loop { + let keys = get_keys(&bucket, prefix.clone()).await; + let count: usize = futures::future::join_all( + keys.into_iter() + .map(|key| async { get_lines(get_object(&bucket, key).await).await.len() }), + ) + .await + .into_iter() + .sum(); + if count >= pre_lines.len() { + break; + } + assert!( + tokio::time::Instant::now() < deadline, + "Timed out waiting for pre-reload events in S3 (found {count}/{})", + pre_lines.len() + ); + tokio::time::sleep(Duration::from_millis(500)).await; + } + assert_eq!(pre_receiver.await, BatchStatus::Delivered); + + // 4. Reload config with a different disk buffer max_size. + // Simulate what the `-w` file watcher does: mark the changed sink so its + // buffer is rebuilt rather than reused. + topology.extend_reload_set(HashSet::from_iter(vec![sink_key])); + + let reload_result = tokio::time::timeout( + Duration::from_secs(5), + topology.reload_config_and_respawn(new_config.build().unwrap(), Default::default()), + ) + .await; + + assert!( + reload_result.is_ok(), + "Reload timed out: disk buffer config change should not stall the reload" + ); + reload_result.unwrap().unwrap(); + + // Give the new sink a moment to initialise. + tokio::time::sleep(Duration::from_secs(1)).await; + + // 5. Send more events post-reload. + let (post_lines, post_events, post_receiver) = make_events_batch(100, 10); + for event in post_events.collect::>().await { + source_tx.send_event(event).await.unwrap(); + } + + // 6. Assert all events (pre- and post-reload) are present in S3. + let mut all_expected: Vec = pre_lines; + all_expected.extend(post_lines); + + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + let mut all_actual: Vec; + loop { + all_actual = Vec::new(); + let keys = get_keys(&bucket, prefix.clone()).await; + for key in keys { + all_actual.extend(get_lines(get_object(&bucket, key).await).await); + } + if all_actual.len() >= all_expected.len() { + break; + } + assert!( + tokio::time::Instant::now() < deadline, + "Timed out waiting for all events in S3 (found {}/{})", + all_actual.len(), + all_expected.len() + ); + tokio::time::sleep(Duration::from_millis(500)).await; + } + assert_eq!(post_receiver.await, BatchStatus::Delivered); + + all_expected.sort(); + all_actual.sort(); + assert_eq!(all_expected, all_actual); + + topology.stop().await; +} + async fn client() -> S3Client { let auth = AwsAuthentication::test_auth(); let region = RegionOrEndpoint::with_both("us-east-1", s3_address()); @@ -512,10 +711,10 @@ async fn client() -> S3Client { .unwrap() } -fn config(bucket: &str, batch_size: usize) -> S3SinkConfig { +fn config(bucket: &str, batch_size: usize, timeout_secs: f64) -> S3SinkConfig { let mut batch = BatchConfig::default(); batch.max_events = Some(batch_size); - batch.timeout_secs = Some(5.0); + batch.timeout_secs = Some(timeout_secs); S3SinkConfig { bucket: bucket.to_string(), @@ -526,6 +725,8 @@ fn config(bucket: &str, batch_size: usize) -> S3SinkConfig { options: S3Options::default(), region: RegionOrEndpoint::with_both("us-east-1", s3_address()), encoding: (None::, TextSerializerConfig::default()).into(), + #[cfg(feature = "codecs-parquet")] + batch_encoding: None, compression: Compression::None, batch, request: TowerRequestConfig::default(), diff --git a/src/sinks/aws_s3/mod.rs b/src/sinks/aws_s3/mod.rs index 3027de1bb6287..4589746175558 100644 --- a/src/sinks/aws_s3/mod.rs +++ b/src/sinks/aws_s3/mod.rs @@ -3,4 +3,6 @@ mod sink; mod integration_tests; +#[cfg(feature = "codecs-parquet")] +pub use config::S3BatchEncoding; pub use config::S3SinkConfig; diff --git a/src/sinks/aws_s3/sink.rs b/src/sinks/aws_s3/sink.rs index 26d47cdb7039c..442e0beb544b6 100644 --- a/src/sinks/aws_s3/sink.rs +++ b/src/sinks/aws_s3/sink.rs @@ -3,10 +3,10 @@ use std::io; use bytes::Bytes; use chrono::{FixedOffset, Utc}; use uuid::Uuid; -use vector_lib::{codecs::encoding::Framer, event::Finalizable, request_metadata::RequestMetadata}; +use vector_lib::{codecs::EncoderKind, event::Finalizable, request_metadata::RequestMetadata}; use crate::{ - codecs::{Encoder, Transformer}, + codecs::Transformer, event::Event, sinks::{ s3_common::{ @@ -28,7 +28,7 @@ pub struct S3RequestOptions { pub filename_append_uuid: bool, pub filename_extension: Option, pub api_options: S3Options, - pub encoder: (Transformer, Encoder), + pub encoder: (Transformer, EncoderKind), pub compression: Compression, pub filename_tz_offset: Option, } @@ -36,7 +36,7 @@ pub struct S3RequestOptions { impl RequestBuilder<(S3PartitionKey, Vec)> for S3RequestOptions { type Metadata = S3Metadata; type Events = Vec; - type Encoder = (Transformer, Encoder); + type Encoder = (Transformer, EncoderKind); type Payload = Bytes; type Request = S3Request; type Error = io::Error; // TODO: this is ugly. diff --git a/src/sinks/axiom/config.rs b/src/sinks/axiom/config.rs index 0349fc02831b8..0930a997abc12 100644 --- a/src/sinks/axiom/config.rs +++ b/src/sinks/axiom/config.rs @@ -15,7 +15,8 @@ use crate::{ Healthcheck, VectorSink, http::config::{HttpMethod, HttpSinkConfig}, util::{ - BatchConfig, Compression, RealtimeSizeBasedDefaultBatchSettings, http::RequestConfig, + BatchConfig, Compression, RealtimeSizeBasedDefaultBatchSettings, + http::{RequestConfig, RetryStrategy}, }, }, tls::TlsConfig, @@ -124,6 +125,10 @@ pub struct AxiomConfig { skip_serializing_if = "crate::serde::is_default" )] pub acknowledgements: AcknowledgementsConfig, + + #[configurable(derived)] + #[serde(default)] + pub retry_strategy: RetryStrategy, } impl GenerateConfig for AxiomConfig { @@ -170,7 +175,6 @@ impl SinkConfig for AxiomConfig { request, acknowledgements: self.acknowledgements, batch: self.batch, - headers: None, encoding: EncodingConfigWithFraming::new( Some(FramingConfig::NewlineDelimited), SerializerConfig::Json(JsonSerializerConfig { @@ -181,6 +185,7 @@ impl SinkConfig for AxiomConfig { ), payload_prefix: "".into(), // Always newline delimited JSON payload_suffix: "".into(), // Always newline delimited JSON + retry_strategy: self.retry_strategy.clone(), }; http_sink_config.build(cx).await diff --git a/src/sinks/azure_blob/config.rs b/src/sinks/azure_blob/config.rs index db0521b4fa137..1311e09edd4f7 100644 --- a/src/sinks/azure_blob/config.rs +++ b/src/sinks/azure_blob/config.rs @@ -16,7 +16,8 @@ use crate::{ sinks::{ Healthcheck, VectorSink, azure_common::{ - self, config::AzureBlobRetryLogic, service::AzureBlobService, sink::AzureBlobSink, + self, config::AzureAuthentication, config::AzureBlobRetryLogic, + config::AzureBlobTlsConfig, service::AzureBlobService, sink::AzureBlobSink, }, util::{ BatchConfig, BulkSizeBasedDefaultBatchSettings, Compression, ServiceBuilderExt, @@ -41,6 +42,10 @@ impl TowerRequestConfigDefaults for AzureBlobTowerRequestConfigDefaults { #[derive(Clone, Debug)] #[serde(deny_unknown_fields)] pub struct AzureBlobSinkConfig { + #[configurable(derived)] + #[serde(default)] + pub auth: Option, + /// The Azure Blob Storage Account connection string. /// /// Authentication with an access key or shared access signature (SAS) @@ -55,13 +60,33 @@ pub struct AzureBlobSinkConfig { /// | Allowed services | Blob | /// | Allowed resource types | Container & Object | /// | Allowed permissions | Read & Create | + #[configurable(metadata( + docs::warnings = "Access keys and SAS tokens can be used to gain unauthorized access to Azure Blob Storage \ + resources. Numerous security breaches have occurred due to leaked connection strings. It is important to keep \ + connection strings secure and not expose them in logs, error messages, or version control systems." + ))] #[configurable(metadata( docs::examples = "DefaultEndpointsProtocol=https;AccountName=mylogstorage;AccountKey=storageaccountkeybase64encoded;EndpointSuffix=core.windows.net" ))] #[configurable(metadata( docs::examples = "BlobEndpoint=https://mylogstorage.blob.core.windows.net/;SharedAccessSignature=generatedsastoken" ))] - pub connection_string: SensitiveString, + #[configurable(metadata(docs::examples = "AccountName=mylogstorage"))] + pub connection_string: Option, + + /// The Azure Blob Storage Account name. + /// + /// If provided, this will be used instead of the `connection_string`. + /// This is useful for authenticating with an Azure credential. + #[configurable(metadata(docs::examples = "mylogstorage"))] + pub(super) account_name: Option, + + /// The Azure Blob Storage endpoint. + /// + /// If provided, this will be used instead of the `connection_string`. + /// This is useful for authenticating with an Azure credential. + #[configurable(metadata(docs::examples = "https://mylogstorage.blob.core.windows.net/"))] + pub(super) blob_endpoint: Option, /// The Azure Blob Storage Account container name. #[configurable(metadata(docs::examples = "my-logs"))] @@ -138,6 +163,10 @@ pub struct AzureBlobSinkConfig { skip_serializing_if = "crate::serde::is_default" )] pub(super) acknowledgements: AcknowledgementsConfig, + + #[configurable(derived)] + #[serde(default)] + pub tls: Option, } pub fn default_blob_prefix() -> Template { @@ -147,7 +176,10 @@ pub fn default_blob_prefix() -> Template { impl GenerateConfig for AzureBlobSinkConfig { fn generate_config() -> toml::Value { toml::Value::try_from(Self { - connection_string: String::from("DefaultEndpointsProtocol=https;AccountName=some-account-name;AccountKey=some-account-key;").into(), + auth: None, + connection_string: Some(String::from("DefaultEndpointsProtocol=https;AccountName=some-account-name;AccountKey=some-account-key;").into()), + account_name: None, + blob_endpoint: None, container_name: String::from("logs"), blob_prefix: default_blob_prefix(), blob_time_format: Some(String::from("%s")), @@ -157,6 +189,7 @@ impl GenerateConfig for AzureBlobSinkConfig { batch: BatchConfig::default(), request: TowerRequestConfig::default(), acknowledgements: Default::default(), + tls: None, }) .unwrap() } @@ -166,11 +199,56 @@ impl GenerateConfig for AzureBlobSinkConfig { #[typetag::serde(name = "azure_blob")] impl SinkConfig for AzureBlobSinkConfig { async fn build(&self, cx: SinkContext) -> Result<(VectorSink, Healthcheck)> { + let connection_string: String = match ( + &self.connection_string, + &self.account_name, + &self.blob_endpoint, + ) { + (Some(connstr), None, None) => connstr.inner().into(), + (None, Some(account_name), None) => { + if self.auth.is_none() { + return Err( + "`auth` configuration must be provided when using `account_name`".into(), + ); + } + format!("AccountName={}", account_name) + } + (None, None, Some(blob_endpoint)) => { + if self.auth.is_none() { + return Err( + "`auth` configuration must be provided when using `blob_endpoint`".into(), + ); + } + // BlobEndpoint must always end in a trailing slash + let blob_endpoint = if blob_endpoint.ends_with('/') { + blob_endpoint.clone() + } else { + format!("{}/", blob_endpoint) + }; + format!("BlobEndpoint={}", blob_endpoint) + } + (None, None, None) => { + return Err("One of `connection_string`, `account_name`, or `blob_endpoint` must be provided".into()); + } + (Some(_), Some(_), _) => { + return Err("Cannot provide both `connection_string` and `account_name`".into()); + } + (Some(_), _, Some(_)) => { + return Err("Cannot provide both `connection_string` and `blob_endpoint`".into()); + } + (_, Some(_), Some(_)) => { + return Err("Cannot provide both `account_name` and `blob_endpoint`".into()); + } + }; + let client = azure_common::config::build_client( - self.connection_string.clone().into(), + self.auth.clone(), + connection_string.clone(), self.container_name.clone(), cx.proxy(), - )?; + self.tls.clone(), + ) + .await?; let healthcheck = azure_common::config::build_healthcheck( self.container_name.clone(), diff --git a/src/sinks/azure_blob/integration_tests.rs b/src/sinks/azure_blob/integration_tests.rs index 63943312fe66d..9703db841a062 100644 --- a/src/sinks/azure_blob/integration_tests.rs +++ b/src/sinks/azure_blob/integration_tests.rs @@ -1,6 +1,8 @@ use std::io::{BufRead, BufReader}; +use std::sync::Arc; use azure_core::http::StatusCode; +use azure_storage_blob::BlobContainerClient; use bytes::{Buf, BytesMut}; use flate2::read::GzDecoder; @@ -24,17 +26,24 @@ use crate::{ components::{SINK_TAGS, assert_sink_compliance}, random_events_with_stream, random_lines, random_lines_with_stream, random_string, }, + tls, }; #[tokio::test] async fn azure_blob_healthcheck_passed() { let config = AzureBlobSinkConfig::new_emulator().await; - let client = azure_common::config::build_client( - config.connection_string.clone().into(), - config.container_name.clone(), - &crate::config::ProxyConfig::default(), - ) - .expect("Failed to create client"); + let client = config.build_test_client().await; + + azure_common::config::build_healthcheck(config.container_name, client) + .expect("Failed to build healthcheck") + .await + .expect("Failed to pass healthcheck"); +} + +#[tokio::test] +async fn azure_blob_healthcheck_passed_with_oauth() { + let config = AzureBlobSinkConfig::new_emulator_with_oauth().await; + let client = config.build_test_client().await; azure_common::config::build_healthcheck(config.container_name, client) .expect("Failed to build healthcheck") @@ -49,12 +58,7 @@ async fn azure_blob_healthcheck_unknown_container() { container_name: String::from("other-container-name"), ..config }; - let client = azure_common::config::build_client( - config.connection_string.clone().into(), - config.container_name.clone(), - &crate::config::ProxyConfig::default(), - ) - .expect("Failed to create client"); + let client = config.build_test_client().await; assert_eq!( azure_common::config::build_healthcheck(config.container_name, client) @@ -66,10 +70,8 @@ async fn azure_blob_healthcheck_unknown_container() { ); } -#[tokio::test] -async fn azure_blob_insert_lines_into_blob() { +async fn assert_insert_lines_into_blob(config: AzureBlobSinkConfig) { let blob_prefix = format!("lines/into/blob/{}", random_string(10)); - let config = AzureBlobSinkConfig::new_emulator().await; let config = AzureBlobSinkConfig { blob_prefix: blob_prefix.clone().try_into().unwrap(), ..config @@ -88,9 +90,17 @@ async fn azure_blob_insert_lines_into_blob() { } #[tokio::test] -async fn azure_blob_insert_json_into_blob() { +async fn azure_blob_insert_lines_into_blob() { + assert_insert_lines_into_blob(AzureBlobSinkConfig::new_emulator().await).await; +} + +#[tokio::test] +async fn azure_blob_insert_lines_into_blob_with_oauth() { + assert_insert_lines_into_blob(AzureBlobSinkConfig::new_emulator_with_oauth().await).await; +} + +async fn assert_insert_json_into_blob(config: AzureBlobSinkConfig) { let blob_prefix = format!("json/into/blob/{}", random_string(10)); - let config = AzureBlobSinkConfig::new_emulator().await; let config = AzureBlobSinkConfig { blob_prefix: blob_prefix.clone().try_into().unwrap(), encoding: ( @@ -117,6 +127,16 @@ async fn azure_blob_insert_json_into_blob() { assert_eq!(expected, blob_lines); } +#[tokio::test] +async fn azure_blob_insert_json_into_blob() { + assert_insert_json_into_blob(AzureBlobSinkConfig::new_emulator().await).await; +} + +#[tokio::test] +async fn azure_blob_insert_json_into_blob_with_oauth() { + assert_insert_json_into_blob(AzureBlobSinkConfig::new_emulator_with_oauth().await).await; +} + #[ignore] #[tokio::test] // This test fails to get the posted blob with "header not found content-length". @@ -177,14 +197,12 @@ async fn azure_blob_insert_json_into_blob_gzip() { assert_eq!(expected, blob_lines); } -#[tokio::test] -async fn azure_blob_rotate_files_after_the_buffer_size_is_reached() { +async fn assert_rotate_files_after_the_buffer_size_is_reached(mut config: AzureBlobSinkConfig) { let groups = 3; let (lines, size, input) = random_lines_with_stream_with_group_key(100, 30, groups); let size_per_group = (size / groups) + 10; let blob_prefix = format!("lines-rotate/into/blob/{}", random_string(10)); - let mut config = AzureBlobSinkConfig::new_emulator().await; config.batch.max_bytes = Some(size_per_group); let config = AzureBlobSinkConfig { @@ -211,52 +229,104 @@ async fn azure_blob_rotate_files_after_the_buffer_size_is_reached() { } } +#[tokio::test] +async fn azure_blob_rotate_files_after_the_buffer_size_is_reached() { + assert_rotate_files_after_the_buffer_size_is_reached(AzureBlobSinkConfig::new_emulator().await) + .await; +} + +#[tokio::test] +async fn azure_blob_rotate_files_after_the_buffer_size_is_reached_with_oauth() { + assert_rotate_files_after_the_buffer_size_is_reached( + AzureBlobSinkConfig::new_emulator_with_oauth().await, + ) + .await; +} + impl AzureBlobSinkConfig { pub async fn new_emulator() -> AzureBlobSinkConfig { - let address = std::env::var("AZURE_ADDRESS").unwrap_or_else(|_| "localhost".into()); + let address = std::env::var("AZURITE_ADDRESS").unwrap_or_else(|_| "localhost".into()); let config = AzureBlobSinkConfig { - connection_string: format!("UseDevelopmentStorage=true;DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://{address}:10000/devstoreaccount1;QueueEndpoint=http://{address}:10001/devstoreaccount1;TableEndpoint=http://{address}:10002/devstoreaccount1;").into(), - container_name: "logs".to_string(), - blob_prefix: Default::default(), - blob_time_format: None, - blob_append_uuid: None, - encoding: (None::, TextSerializerConfig::default()).into(), - compression: Compression::None, - batch: Default::default(), - request: TowerRequestConfig::default(), - acknowledgements: Default::default(), - }; + auth: None, + connection_string: Some(format!("UseDevelopmentStorage=true;DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://{address}:10000/devstoreaccount1;QueueEndpoint=http://{address}:10001/devstoreaccount1;TableEndpoint=http://{address}:10002/devstoreaccount1;").into()), + account_name: None, + blob_endpoint: None, + container_name: "logs".to_string(), + blob_prefix: Default::default(), + blob_time_format: None, + blob_append_uuid: None, + encoding: (None::, TextSerializerConfig::default()).into(), + compression: Compression::None, + batch: Default::default(), + request: TowerRequestConfig::default(), + acknowledgements: Default::default(), + tls: None, + }; config.ensure_container().await; config } - fn to_sink(&self) -> VectorSink { - let client = azure_common::config::build_client( - self.connection_string.clone().into(), + pub async fn new_emulator_with_oauth() -> AzureBlobSinkConfig { + let address = std::env::var("AZURITE_OAUTH_ADDRESS").unwrap_or_else(|_| "localhost".into()); + let config = AzureBlobSinkConfig { + auth: Some(azure_common::config::AzureAuthentication::MockCredential), + connection_string: Some(format!("DefaultEndpointsProtocol=https;AccountName=devstoreaccount1;BlobEndpoint=https://{address}:14430/devstoreaccount1;QueueEndpoint=https://{address}:14431/devstoreaccount1;TableEndpoint=https://{address}:14432/devstoreaccount1;").into()), + account_name: None, + blob_endpoint: None, + container_name: "logs".to_string(), + blob_prefix: Default::default(), + blob_time_format: None, + blob_append_uuid: None, + encoding: (None::, TextSerializerConfig::default()).into(), + compression: Compression::None, + batch: Default::default(), + request: TowerRequestConfig::default(), + acknowledgements: Default::default(), + tls: Some(azure_common::config::AzureBlobTlsConfig { + ca_file: Some(tls::TEST_PEM_CA_PATH.into()), + }), + }; + + config.ensure_container().await; + + config + } + + async fn build_test_client(&self) -> Arc { + azure_common::config::build_client( + self.auth.clone(), + self.connection_string + .clone() + .expect("failed to unwrap connection_string") + .inner() + .to_string(), self.container_name.clone(), &crate::config::ProxyConfig::default(), + self.tls.clone(), ) - .expect("Failed to create client"); + .await + .expect("Failed to create client") + } + async fn to_sink(&self) -> VectorSink { + let client = self.build_test_client().await; self.build_processor(client).expect("Failed to create sink") } async fn run_assert(&self, input: impl Stream + Send) { // `to_sink` needs to be inside the assertion check - assert_sink_compliance(&SINK_TAGS, async move { self.to_sink().run(input).await }) - .await - .expect("Running sink failed"); + assert_sink_compliance( + &SINK_TAGS, + async move { self.to_sink().await.run(input).await }, + ) + .await + .expect("Running sink failed"); } pub async fn list_blobs(&self, prefix: String) -> Vec { - let client = azure_common::config::build_client( - self.connection_string.clone().into(), - self.container_name.clone(), - &crate::config::ProxyConfig::default(), - ) - .unwrap(); + let client = self.build_test_client().await; // Iterate pager results and collect blob names. Filter by prefix server-side. let mut pager = client @@ -265,7 +335,7 @@ impl AzureBlobSinkConfig { let mut names = Vec::new(); while let Some(result) = pager.next().await { let item = result.expect("Failed to fetch blobs"); - if let Some(name) = item.name.and_then(|bn| bn.content) + if let Some(name) = item.name && name.starts_with(&prefix) { names.push(name); @@ -276,12 +346,7 @@ impl AzureBlobSinkConfig { } pub async fn get_blob(&self, blob: String) -> (Option, Option, Vec) { - let client = azure_common::config::build_client( - self.connection_string.clone().into(), - self.container_name.clone(), - &crate::config::ProxyConfig::default(), - ) - .unwrap(); + let client = self.build_test_client().await; let blob_client = client.blob_client(&blob); @@ -314,7 +379,7 @@ impl AzureBlobSinkConfig { .await .expect("Failed to download blob"); let body_bytes = downloaded - .into_body() + .body .collect() .await .expect("Failed to read blob body"); @@ -337,13 +402,8 @@ impl AzureBlobSinkConfig { } async fn ensure_container(&self) { - let client = azure_common::config::build_client( - self.connection_string.clone().into(), - self.container_name.clone(), - &crate::config::ProxyConfig::default(), - ) - .unwrap(); - let result = client.create_container(None).await; + let client = self.build_test_client().await; + let result = client.create(None).await; let response = match result { Ok(_) => Ok(()), diff --git a/src/sinks/azure_blob/test.rs b/src/sinks/azure_blob/test.rs index b7f1a9689229f..2dd6aaf69bdd4 100644 --- a/src/sinks/azure_blob/test.rs +++ b/src/sinks/azure_blob/test.rs @@ -14,6 +14,8 @@ use super::{config::AzureBlobSinkConfig, request_builder::AzureBlobRequestOption use crate::{ codecs::{Encoder, EncodingConfigWithFraming}, event::{Event, LogEvent}, + sinks::azure_common::config::{AzureAuthentication, SpecificAzureCredential}, + sinks::prelude::*, sinks::util::{ Compression, request_builder::{EncodeResult, RequestBuilder}, @@ -22,7 +24,10 @@ use crate::{ fn default_config(encoding: EncodingConfigWithFraming) -> AzureBlobSinkConfig { AzureBlobSinkConfig { + auth: Default::default(), connection_string: Default::default(), + account_name: Default::default(), + blob_endpoint: Default::default(), container_name: Default::default(), blob_prefix: Default::default(), blob_time_format: Default::default(), @@ -32,6 +37,7 @@ fn default_config(encoding: EncodingConfigWithFraming) -> AzureBlobSinkConfig { batch: Default::default(), request: Default::default(), acknowledgements: Default::default(), + tls: Default::default(), } } @@ -234,3 +240,317 @@ fn azure_blob_build_request_with_uuid() { assert_eq!(request.content_encoding, None); assert_eq!(request.content_type, "text/plain"); } + +#[tokio::test] +async fn azure_blob_build_config_with_null_auth() { + let config: Result = toml::from_str::( + r#" + connection_string = "AccountName=mylogstorage" + container_name = "my-logs" + + [encoding] + codec = "json" + + [auth] + "#, + ); + + match config { + Ok(_) => panic!("Config parsing should have failed due to invalid auth config"), + Err(e) => { + let err_str = e.to_string(); + assert!( + err_str.contains("data did not match any variant of untagged enum"), + "Config parsing did not complain about invalid auth config: {}", + err_str + ); + } + } +} + +#[tokio::test] +async fn azure_blob_build_config_with_client_id_and_secret() { + let config: AzureBlobSinkConfig = toml::from_str::( + r#" + connection_string = "AccountName=mylogstorage" + container_name = "my-logs" + + [encoding] + codec = "json" + + [auth] + azure_credential_kind = "client_secret_credential" + azure_tenant_id = "00000000-0000-0000-0000-000000000000" + azure_client_id = "mock-client-id" + azure_client_secret = "mock-client-secret" + "#, + ) + .unwrap_or_else(|error| panic!("Config parsing failed: {error:?}")); + + assert!(&config.auth.is_some()); + + match &config.auth.clone().unwrap() { + AzureAuthentication::Specific(SpecificAzureCredential::ClientSecretCredential { + azure_tenant_id, + azure_client_id, + azure_client_secret, + }) => { + assert_eq!(azure_tenant_id, "00000000-0000-0000-0000-000000000000"); + assert_eq!(azure_client_id, "mock-client-id"); + let secret: String = azure_client_secret.inner().into(); + assert_eq!(secret, "mock-client-secret"); + } + _ => panic!("Expected Specific(ClientSecretCredential) variant"), + } + + let cx = SinkContext::default(); + let _sink = config + .build(cx) + .await + .unwrap_or_else(|error| panic!("Failed to build sink: {error:?}")); +} + +#[tokio::test] +async fn azure_blob_build_config_with_client_certificate() { + let config: AzureBlobSinkConfig = toml::from_str::( + r#" + connection_string = "AccountName=mylogstorage" + container_name = "my-logs" + + [encoding] + codec = "json" + + [auth] + azure_credential_kind = "client_certificate_credential" + azure_tenant_id = "00000000-0000-0000-0000-000000000000" + azure_client_id = "mock-client-id" + certificate_path = "tests/data/ClientCertificateAuth.pfx" + certificate_password = "MockPassword123" + "#, + ) + .unwrap_or_else(|error| panic!("Config parsing failed: {error:?}")); + + assert!(&config.auth.is_some()); + + match &config.auth.clone().unwrap() { + AzureAuthentication::Specific(SpecificAzureCredential::ClientCertificateCredential { + .. + }) => { + // Expected variant + } + _ => panic!("Expected Specific(ClientCertificateCredential) variant"), + } + + let cx = SinkContext::default(); + let _sink = config + .build(cx) + .await + .unwrap_or_else(|error| panic!("Failed to build sink: {error:?}")); +} + +#[tokio::test] +async fn azure_blob_build_config_with_account_name() { + let config: AzureBlobSinkConfig = toml::from_str::( + r#" + account_name = "mylogstorage" + container_name = "my-logs" + + [encoding] + codec = "json" + + [auth] + azure_credential_kind = "client_secret_credential" + azure_tenant_id = "00000000-0000-0000-0000-000000000000" + azure_client_id = "mock-client-id" + azure_client_secret = "mock-client-secret" + "#, + ) + .unwrap_or_else(|error| panic!("Config parsing failed: {error:?}")); + + let cx = SinkContext::default(); + let _ = config + .build(cx) + .await + .unwrap_or_else(|error| panic!("Failed to build sink: {error:?}")); +} + +#[tokio::test] +async fn azure_blob_build_config_with_account_name_with_no_auth() { + let config: AzureBlobSinkConfig = toml::from_str::( + r#" + account_name = "mylogstorage" + container_name = "my-logs" + + [encoding] + codec = "json" + "#, + ) + .unwrap_or_else(|error| panic!("Config parsing failed: {error:?}")); + + let cx = SinkContext::default(); + let sink = config.build(cx).await; + match sink { + Ok(_) => panic!("Config build should have errored due to missing `auth`"), + Err(e) => { + let err_str = e.to_string(); + assert!( + err_str.contains("`auth` configuration must be provided"), + "Config build did not complain about missing `auth`: {}", + err_str + ); + } + } +} + +#[tokio::test] +async fn azure_blob_build_config_with_blob_endpoint() { + let config: AzureBlobSinkConfig = toml::from_str::( + r#" + blob_endpoint = "https://localhost:10000/devstoreaccount1" + container_name = "my-logs" + + [encoding] + codec = "json" + + [auth] + azure_credential_kind = "client_secret_credential" + azure_tenant_id = "00000000-0000-0000-0000-000000000000" + azure_client_id = "mock-client-id" + azure_client_secret = "mock-client-secret" + "#, + ) + .unwrap_or_else(|error| panic!("Config parsing failed: {error:?}")); + + let cx = SinkContext::default(); + let _ = config + .build(cx) + .await + .unwrap_or_else(|error| panic!("Failed to build sink: {error:?}")); +} + +#[tokio::test] +async fn azure_blob_build_config_with_blob_endpoint_with_no_auth() { + let config: AzureBlobSinkConfig = toml::from_str::( + r#" + blob_endpoint = "https://localhost:10000/devstoreaccount1" + container_name = "my-logs" + + [encoding] + codec = "json" + "#, + ) + .unwrap_or_else(|error| panic!("Config parsing failed: {error:?}")); + + let cx = SinkContext::default(); + let sink = config.build(cx).await; + match sink { + Ok(_) => panic!("Config build should have errored due to missing `auth`"), + Err(e) => { + let err_str = e.to_string(); + assert!( + err_str.contains("`auth` configuration must be provided"), + "Config build did not complain about missing `auth`: {}", + err_str + ); + } + } +} + +#[tokio::test] +async fn azure_blob_build_config_with_conflicting_connection_string_and_account_name() { + let config: AzureBlobSinkConfig = toml::from_str::( + r#" + connection_string = "AccountName=mylogstorage" + account_name = "mylogstorage" + container_name = "my-logs" + + [encoding] + codec = "json" + "#, + ) + .unwrap_or_else(|error| panic!("Config parsing failed: {error:?}")); + + let cx = SinkContext::default(); + let sink = config.build(cx).await; + match sink { + Ok(_) => panic!( + "Config build should have errored due to conflicting connection_string and account_name" + ), + Err(e) => { + let err_str = e.to_string(); + assert!( + err_str.contains("`connection_string` and `account_name`"), + "Config build did not complain about conflicting connection_string and account_name: {}", + err_str + ); + } + } +} + +#[tokio::test] +async fn azure_blob_build_config_with_conflicting_connection_string_and_client_id_and_secret() { + let config: AzureBlobSinkConfig = toml::from_str::( + r#" + connection_string = "AccountName=mylogstorage;AccountKey=mockkey" + container_name = "my-logs" + + [encoding] + codec = "json" + + [auth] + azure_credential_kind = "client_secret_credential" + azure_tenant_id = "00000000-0000-0000-0000-000000000000" + azure_client_id = "mock-client-id" + azure_client_secret = "mock-client-secret" + "#, + ) + .unwrap_or_else(|error| panic!("Config parsing failed: {error:?}")); + + assert!(&config.auth.is_some()); + + let cx = SinkContext::default(); + let sink = config.build(cx).await; + match sink { + Ok(_) => { + panic!("Config build should have errored due to conflicting Shared Key and Client ID") + } + Err(e) => { + let err_str = e.to_string(); + assert!( + err_str + .contains("Cannot use both Shared Key and another Azure Authentication method"), + "Config build did not complain about conflicting Shared Key and Client ID: {}", + err_str + ); + } + } +} + +#[tokio::test] +async fn azure_blob_build_config_with_custom_ca_certificate() { + let config: AzureBlobSinkConfig = toml::from_str::( + r#" + account_name = "mylogstorage" + container_name = "my-logs" + + [encoding] + codec = "json" + + [tls] + ca_file = "tests/data/ca/certs/ca.cert.pem" + + [auth] + azure_credential_kind = "client_secret_credential" + azure_tenant_id = "00000000-0000-0000-0000-000000000000" + azure_client_id = "mock-client-id" + azure_client_secret = "mock-client-secret" + "#, + ) + .unwrap_or_else(|error| panic!("Config parsing failed: {error:?}")); + + let cx = SinkContext::default(); + let _ = config + .build(cx) + .await + .unwrap_or_else(|error| panic!("Failed to build sink: {error:?}")); +} diff --git a/src/sinks/azure_common/config.rs b/src/sinks/azure_common/config.rs index 139608d9847ab..38cf6f15c9de6 100644 --- a/src/sinks/azure_common/config.rs +++ b/src/sinks/azure_common/config.rs @@ -1,13 +1,8 @@ use std::sync::Arc; use azure_core::error::Error as AzureCoreError; - -use crate::sinks::azure_common::connection_string::{Auth, ParsedConnectionString}; -use crate::sinks::azure_common::shared_key_policy::SharedKeyAuthorizationPolicy; -use azure_core::http::Url; -use azure_storage_blob::{BlobContainerClient, BlobContainerClientOptions}; - use azure_core::http::StatusCode; +use azure_storage_blob::BlobContainerClient; use bytes::Bytes; use futures::FutureExt; use snafu::Snafu; @@ -22,6 +17,13 @@ use crate::{ sinks::{Healthcheck, util::retries::RetryLogic}, }; +// Shared Azure client machinery lives in `crate::azure`; re-exported here to keep +// existing sink import paths stable. +pub use crate::azure::client::{ + AzureAuthentication, AzureBlobTlsConfig, SpecificAzureCredential, + UserAssignedManagedIdentityIdType, build_client, +}; + #[derive(Debug, Clone)] pub struct AzureBlobRequest { pub blob_data: Bytes, @@ -125,78 +127,3 @@ pub fn build_healthcheck( Ok(healthcheck.boxed()) } - -pub fn build_client( - connection_string: String, - container_name: String, - proxy: &crate::config::ProxyConfig, -) -> crate::Result> { - // Parse connection string without legacy SDK - let parsed = ParsedConnectionString::parse(&connection_string) - .map_err(|e| format!("Invalid connection string: {e}"))?; - // Compose container URL (SAS appended if present) - let container_url = parsed - .container_url(&container_name) - .map_err(|e| format!("Failed to build container URL: {e}"))?; - let url = Url::parse(&container_url).map_err(|e| format!("Invalid container URL: {e}"))?; - - // Prepare options; attach Shared Key policy if needed - let mut options = BlobContainerClientOptions::default(); - match parsed.auth() { - Auth::Sas { .. } | Auth::None => { - // No extra policy; SAS is in the URL already (or anonymous) - } - Auth::SharedKey { - account_name, - account_key, - } => { - let policy = SharedKeyAuthorizationPolicy::new( - account_name, - account_key, - // Use an Azurite-supported storage service version - String::from("2025-11-05"), - ) - .map_err(|e| format!("Failed to create SharedKey policy: {e}"))?; - options - .client_options - .per_call_policies - .push(Arc::new(policy)); - } - } - - // Use reqwest v0.12 since Azure SDK only implements HttpClient for reqwest::Client v0.12 - let mut reqwest_builder = reqwest_12::ClientBuilder::new(); - let bypass_proxy = { - let host = url.host_str().unwrap_or(""); - let port = url.port(); - proxy.no_proxy.matches(host) - || port - .map(|p| proxy.no_proxy.matches(&format!("{}:{}", host, p))) - .unwrap_or(false) - }; - if bypass_proxy || !proxy.enabled { - // Ensure no proxy (and disable any potential system proxy auto-detection) - reqwest_builder = reqwest_builder.no_proxy(); - } else { - if let Some(http) = &proxy.http { - let p = reqwest_12::Proxy::http(http) - .map_err(|e| format!("Invalid HTTP proxy URL: {e}"))?; - // If credentials are embedded in the proxy URL, reqwest will handle them. - reqwest_builder = reqwest_builder.proxy(p); - } - if let Some(https) = &proxy.https { - let p = reqwest_12::Proxy::https(https) - .map_err(|e| format!("Invalid HTTPS proxy URL: {e}"))?; - // If credentials are embedded in the proxy URL, reqwest will handle them. - reqwest_builder = reqwest_builder.proxy(p); - } - } - options.client_options.transport = Some(azure_core::http::Transport::new(std::sync::Arc::new( - reqwest_builder - .build() - .map_err(|e| format!("Failed to build reqwest client: {e}"))?, - ))); - let client = - BlobContainerClient::from_url(url, None, Some(options)).map_err(|e| format!("{e}"))?; - Ok(Arc::new(client)) -} diff --git a/src/sinks/azure_common/mod.rs b/src/sinks/azure_common/mod.rs index 368c88164fd8d..4d1c931977f03 100644 --- a/src/sinks/azure_common/mod.rs +++ b/src/sinks/azure_common/mod.rs @@ -1,5 +1,3 @@ pub mod config; -pub mod connection_string; pub mod service; -pub mod shared_key_policy; pub mod sink; diff --git a/src/sinks/azure_common/service.rs b/src/sinks/azure_common/service.rs index 1f1d322156f78..31668ab85ba70 100644 --- a/src/sinks/azure_common/service.rs +++ b/src/sinks/azure_common/service.rs @@ -47,13 +47,12 @@ impl Service for AzureBlobService { blob_content_type: Some(request.content_type.to_string()), blob_content_encoding: request.content_encoding.map(|e| e.to_string()), ..Default::default() - }; + } + .if_not_exists(); let result = blob_client .upload( RequestContent::from(request.blob_data.to_vec()), - false, - byte_size as u64, Some(upload_options), ) .instrument(info_span!("request").or_current()) diff --git a/src/sinks/azure_common/sink.rs b/src/sinks/azure_common/sink.rs index 1309ae3ba0d66..de567272cf8cd 100644 --- a/src/sinks/azure_common/sink.rs +++ b/src/sinks/azure_common/sink.rs @@ -47,7 +47,9 @@ where let request_builder = self.request_builder; input - .batched_partitioned(partitioner, || settings.as_byte_size_config()) + .batched_partitioned(partitioner, settings.timeout, |_| { + settings.as_byte_size_config() + }) .filter_map(|(key, batch)| async move { // We don't need to emit an error here if the event is dropped since this will occur if the template // couldn't be rendered during the partitioning. A `TemplateRenderingError` is already emitted when diff --git a/src/sinks/azure_logs_ingestion/config.rs b/src/sinks/azure_logs_ingestion/config.rs index 4e876a18a26bd..89da496045d9a 100644 --- a/src/sinks/azure_logs_ingestion/config.rs +++ b/src/sinks/azure_logs_ingestion/config.rs @@ -1,22 +1,19 @@ use std::sync::Arc; -use azure_core::credentials::{TokenCredential, TokenRequestOptions}; -use azure_core::http::ClientMethodOptions; -use azure_core::{Error, error::ErrorKind}; - -use azure_identity::{ - AzureCliCredential, ClientAssertion, ClientAssertionCredential, ClientSecretCredential, - ManagedIdentityCredential, ManagedIdentityCredentialOptions, UserAssignedId, - WorkloadIdentityCredential, -}; -use vector_lib::{configurable::configurable_component, schema, sensitive_string::SensitiveString}; +use azure_core::credentials::TokenCredential; + +use vector_lib::{configurable::configurable_component, schema}; use vrl::value::Kind; use crate::{ http::{HttpClient, get_http_scheme_from_uri}, sinks::{ + azure_common::config::AzureAuthentication, prelude::*, - util::{RealtimeSizeBasedDefaultBatchSettings, UriSerde, http::HttpStatusRetryLogic}, + util::{ + RealtimeSizeBasedDefaultBatchSettings, UriSerde, + http::{HttpStatusRetryLogic, RetryStrategy}, + }, }, }; @@ -65,7 +62,6 @@ pub struct AzureLogsIngestionConfig { pub stream_name: String, #[configurable(derived)] - #[serde(default)] pub auth: AzureAuthentication, /// [Token scope][token_scope] for dedicated Azure regions. @@ -109,6 +105,10 @@ pub struct AzureLogsIngestionConfig { skip_serializing_if = "crate::serde::is_default" )] pub acknowledgements: AcknowledgementsConfig, + + #[configurable(derived)] + #[serde(default)] + pub retry_strategy: RetryStrategy, } impl Default for AzureLogsIngestionConfig { @@ -125,195 +125,11 @@ impl Default for AzureLogsIngestionConfig { request: Default::default(), tls: None, acknowledgements: Default::default(), + retry_strategy: Default::default(), } } } -/// Configuration of the authentication strategy for interacting with Azure services. -#[configurable_component] -#[derive(Clone, Debug, Derivative, Eq, PartialEq)] -#[derivative(Default)] -#[serde(deny_unknown_fields, untagged)] -pub enum AzureAuthentication { - /// Use client credentials - #[derivative(Default)] - ClientSecretCredential { - /// The [Azure Tenant ID][azure_tenant_id]. - /// - /// [azure_tenant_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal - #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] - azure_tenant_id: String, - - /// The [Azure Client ID][azure_client_id]. - /// - /// [azure_client_id]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal - #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] - azure_client_id: String, - - /// The [Azure Client Secret][azure_client_secret]. - /// - /// [azure_client_secret]: https://learn.microsoft.com/entra/identity-platform/howto-create-service-principal-portal - #[configurable(metadata(docs::examples = "00-00~000000-0000000~0000000000000000000"))] - azure_client_secret: SensitiveString, - }, - - /// Use credentials from environment variables - #[configurable(metadata(docs::enum_tag_description = "The kind of Azure credential to use."))] - Specific(SpecificAzureCredential), -} - -/// Specific Azure credential types. -#[configurable_component] -#[derive(Clone, Debug, Eq, PartialEq)] -#[serde( - tag = "azure_credential_kind", - rename_all = "snake_case", - deny_unknown_fields -)] -pub enum SpecificAzureCredential { - /// Use Azure CLI credentials - #[cfg(not(target_arch = "wasm32"))] - AzureCli {}, - - /// Use Managed Identity credentials - ManagedIdentity { - /// The User Assigned Managed Identity (Client ID) to use. - #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] - #[serde(default, skip_serializing_if = "Option::is_none")] - user_assigned_managed_identity_id: Option, - }, - - /// Use Managed Identity with Client Assertion credentials - ManagedIdentityClientAssertion { - /// The User Assigned Managed Identity (Client ID) to use for the managed identity. - #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] - #[serde(default, skip_serializing_if = "Option::is_none")] - user_assigned_managed_identity_id: Option, - - /// The target Tenant ID to use. - #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] - client_assertion_tenant_id: String, - - /// The target Client ID to use. - #[configurable(metadata(docs::examples = "00000000-0000-0000-0000-000000000000"))] - client_assertion_client_id: String, - }, - - /// Use Workload Identity credentials - WorkloadIdentity {}, -} - -#[derive(Debug)] -struct ManagedIdentityClientAssertion { - credential: Arc, - scope: String, -} - -#[cfg_attr(target_arch = "wasm32", async_trait::async_trait(?Send))] -#[cfg_attr(not(target_arch = "wasm32"), async_trait::async_trait)] -impl ClientAssertion for ManagedIdentityClientAssertion { - async fn secret(&self, options: Option>) -> azure_core::Result { - Ok(self - .credential - .get_token( - &[&self.scope], - Some(TokenRequestOptions { - method_options: options.unwrap_or_default(), - }), - ) - .await? - .token - .secret() - .to_string()) - } -} - -impl AzureAuthentication { - /// Returns the provider for the credentials based on the authentication mechanism chosen. - pub async fn credential(&self) -> azure_core::Result> { - match self { - Self::ClientSecretCredential { - azure_tenant_id, - azure_client_id, - azure_client_secret, - } => { - if azure_tenant_id.is_empty() { - return Err(Error::with_message(ErrorKind::Credential, - "`auth.azure_tenant_id` is blank; either use `auth.azure_credential_kind`, or provide tenant ID, client ID, and secret.".to_string() - )); - } - if azure_client_id.is_empty() { - return Err(Error::with_message(ErrorKind::Credential, - "`auth.azure_client_id` is blank; either use `auth.azure_credential_kind`, or provide tenant ID, client ID, and secret.".to_string() - )); - } - if azure_client_secret.inner().is_empty() { - return Err(Error::with_message(ErrorKind::Credential, - "`auth.azure_client_secret` is blank; either use `auth.azure_credential_kind`, or provide tenant ID, client ID, and secret.".to_string() - )); - } - let secret: String = azure_client_secret.inner().into(); - let credential: Arc = ClientSecretCredential::new( - &azure_tenant_id.clone(), - azure_client_id.clone(), - secret.into(), - None, - )?; - Ok(credential) - } - - Self::Specific(specific) => specific.credential().await, - } - } -} - -impl SpecificAzureCredential { - /// Returns the provider for the credentials based on the specific credential type. - pub async fn credential(&self) -> azure_core::Result> { - let credential: Arc = match self { - #[cfg(not(target_arch = "wasm32"))] - Self::AzureCli {} => AzureCliCredential::new(None)?, - - Self::ManagedIdentity { - user_assigned_managed_identity_id, - } => { - let mut options = ManagedIdentityCredentialOptions::default(); - if let Some(id) = user_assigned_managed_identity_id { - options.user_assigned_id = Some(UserAssignedId::ClientId(id.clone())); - } - ManagedIdentityCredential::new(Some(options))? - } - - Self::ManagedIdentityClientAssertion { - user_assigned_managed_identity_id, - client_assertion_tenant_id, - client_assertion_client_id, - } => { - let mut options = ManagedIdentityCredentialOptions::default(); - if let Some(id) = user_assigned_managed_identity_id { - options.user_assigned_id = Some(UserAssignedId::ClientId(id.clone())); - } - let msi: Arc = ManagedIdentityCredential::new(Some(options))?; - let assertion = ManagedIdentityClientAssertion { - credential: msi, - // Future: make this configurable for sovereign clouds? (no way to test...) - scope: "api://AzureADTokenExchange/.default".to_string(), - }; - - ClientAssertionCredential::new( - client_assertion_tenant_id.clone(), - client_assertion_client_id.clone(), - assertion, - None, - )? - } - - Self::WorkloadIdentity {} => WorkloadIdentityCredential::new(None)?, - }; - Ok(credential) - } -} - impl AzureLogsIngestionConfig { #[allow(clippy::too_many_arguments)] pub(super) async fn build_inner( @@ -348,8 +164,10 @@ impl AzureLogsIngestionConfig { )?; let healthcheck = service.healthcheck(); - let retry_logic = - HttpStatusRetryLogic::new(|res: &AzureLogsIngestionResponse| res.http_status); + let retry_logic = HttpStatusRetryLogic::new( + |res: &AzureLogsIngestionResponse| res.http_status, + self.retry_strategy.clone(), + ); let request_settings = self.request.into_settings(); let service = ServiceBuilder::new() .settings(request_settings, retry_logic) diff --git a/src/sinks/azure_logs_ingestion/tests.rs b/src/sinks/azure_logs_ingestion/tests.rs index 26eda91338836..46095071bd28c 100644 --- a/src/sinks/azure_logs_ingestion/tests.rs +++ b/src/sinks/azure_logs_ingestion/tests.rs @@ -8,6 +8,8 @@ use vector_lib::config::log_schema; use azure_core::credentials::{AccessToken, TokenCredential}; use azure_core::time::OffsetDateTime; +use crate::sinks::azure_common::config::{AzureAuthentication, SpecificAzureCredential}; + use super::config::AzureLogsIngestionConfig; use crate::{ @@ -26,50 +28,22 @@ fn generate_config() { #[tokio::test] async fn basic_config_error_with_no_auth() { - let config: AzureLogsIngestionConfig = toml::from_str::( - r#" + let config: Result = + toml::from_str::( + r#" endpoint = "https://my-dce-5kyl.eastus-1.ingest.monitor.azure.com" dcr_immutable_id = "dcr-00000000000000000000000000000000" stream_name = "Custom-UnitTest" "#, - ) - .expect("Config parsing failed"); - - assert_eq!( - config.endpoint, - "https://my-dce-5kyl.eastus-1.ingest.monitor.azure.com" - ); - assert_eq!( - config.dcr_immutable_id, - "dcr-00000000000000000000000000000000" - ); - assert_eq!(config.stream_name, "Custom-UnitTest"); - assert_eq!(config.token_scope, "https://monitor.azure.com/.default"); - assert_eq!(config.timestamp_field, "TimeGenerated"); - - match &config.auth { - crate::sinks::azure_logs_ingestion::config::AzureAuthentication::ClientSecretCredential { - azure_tenant_id, - azure_client_id, - azure_client_secret, - } => { - assert_eq!(azure_tenant_id, ""); - assert_eq!(azure_client_id, ""); - let secret: String = azure_client_secret.inner().into(); - assert_eq!(secret, ""); - } - _ => panic!("Expected ClientSecretCredential variant"), - } + ); - let cx = SinkContext::default(); - let sink = config.build(cx).await; - match sink { - Ok(_) => panic!("Config build should have errored due to missing auth info"), + match config { + Ok(_) => panic!("Config parsing should have failed due to missing auth config"), Err(e) => { let err_str = e.to_string(); assert!( - err_str.contains("`auth.azure_tenant_id` is blank"), - "Config build did not complain about azure_tenant_id being blank: {}", + err_str.contains("missing field `auth`"), + "Config parsing did not complain about missing auth field: {}", err_str ); } @@ -85,6 +59,7 @@ fn basic_config_with_client_credentials() { stream_name = "Custom-UnitTest" [auth] + azure_credential_kind = "client_secret_credential" azure_tenant_id = "00000000-0000-0000-0000-000000000000" azure_client_id = "mock-client-id" azure_client_secret = "mock-client-secret" @@ -105,17 +80,17 @@ fn basic_config_with_client_credentials() { assert_eq!(config.timestamp_field, "TimeGenerated"); match &config.auth { - crate::sinks::azure_logs_ingestion::config::AzureAuthentication::ClientSecretCredential { + AzureAuthentication::Specific(SpecificAzureCredential::ClientSecretCredential { azure_tenant_id, azure_client_id, azure_client_secret, - } => { + }) => { assert_eq!(azure_tenant_id, "00000000-0000-0000-0000-000000000000"); assert_eq!(azure_client_id, "mock-client-id"); let secret: String = azure_client_secret.inner().into(); assert_eq!(secret, "mock-client-secret"); } - _ => panic!("Expected ClientSecretCredential variant"), + _ => panic!("Expected Specific(ClientSecretCredential) variant"), } } @@ -146,11 +121,7 @@ fn basic_config_with_managed_identity() { assert_eq!(config.timestamp_field, "TimeGenerated"); match &config.auth { - crate::sinks::azure_logs_ingestion::config::AzureAuthentication::Specific( - crate::sinks::azure_logs_ingestion::config::SpecificAzureCredential::ManagedIdentity { - .. - }, - ) => { + AzureAuthentication::Specific(SpecificAzureCredential::ManagedIdentity { .. }) => { // Expected variant } _ => panic!("Expected Specific(ManagedIdentity) variant"), @@ -182,6 +153,7 @@ async fn correct_request() { stream_name = "Custom-UnitTest" [auth] + azure_credential_kind = "client_secret_credential" azure_tenant_id = "00000000-0000-0000-0000-000000000000" azure_client_id = "mock-client-id" azure_client_secret = "mock-client-secret" @@ -292,6 +264,7 @@ async fn mock_healthcheck_with_400_response() { stream_name = "Custom-UnitTest" [auth] + azure_credential_kind = "client_secret_credential" azure_tenant_id = "00000000-0000-0000-0000-000000000000" azure_client_id = "mock-client-id" azure_client_secret = "mock-client-secret" @@ -361,6 +334,7 @@ async fn mock_healthcheck_with_403_response() { stream_name = "Custom-UnitTest" [auth] + azure_credential_kind = "client_secret_credential" azure_tenant_id = "00000000-0000-0000-0000-000000000000" azure_client_id = "mock-client-id" azure_client_secret = "mock-client-secret" diff --git a/src/sinks/azure_monitor_logs/config.rs b/src/sinks/azure_monitor_logs/config.rs index dc60d179a2ff5..6d68aed7efd75 100644 --- a/src/sinks/azure_monitor_logs/config.rs +++ b/src/sinks/azure_monitor_logs/config.rs @@ -16,7 +16,10 @@ use crate::{ http::{HttpClient, get_http_scheme_from_uri}, sinks::{ prelude::*, - util::{RealtimeSizeBasedDefaultBatchSettings, UriSerde, http::HttpStatusRetryLogic}, + util::{ + RealtimeSizeBasedDefaultBatchSettings, UriSerde, + http::{HttpStatusRetryLogic, RetryStrategy}, + }, }, }; @@ -113,6 +116,10 @@ pub struct AzureMonitorLogsConfig { skip_serializing_if = "crate::serde::is_default" )] pub acknowledgements: AcknowledgementsConfig, + + #[configurable(derived)] + #[serde(default)] + pub retry_strategy: RetryStrategy, } impl Default for AzureMonitorLogsConfig { @@ -129,6 +136,7 @@ impl Default for AzureMonitorLogsConfig { time_generated_key: None, tls: None, acknowledgements: Default::default(), + retry_strategy: Default::default(), } } } @@ -181,8 +189,10 @@ impl AzureMonitorLogsConfig { )?; let healthcheck = service.healthcheck(); - let retry_logic = - HttpStatusRetryLogic::new(|res: &AzureMonitorLogsResponse| res.http_status); + let retry_logic = HttpStatusRetryLogic::new( + |res: &AzureMonitorLogsResponse| res.http_status, + self.retry_strategy.clone(), + ); let request_settings = self.request.into_settings(); let service = ServiceBuilder::new() .settings(request_settings, retry_logic) diff --git a/src/sinks/blackhole/sink.rs b/src/sinks/blackhole/sink.rs index ff0744913cc8b..8796c7425f691 100644 --- a/src/sinks/blackhole/sink.rs +++ b/src/sinks/blackhole/sink.rs @@ -57,7 +57,7 @@ impl StreamSink for BlackholeSink { if self.config.print_interval_secs.as_secs() > 0 { let interval_dur = self.config.print_interval_secs; - tokio::spawn(async move { + crate::spawn_in_current_span(async move { let mut print_interval = interval(interval_dur); loop { select! { diff --git a/src/sinks/clickhouse/arrow/parser.rs b/src/sinks/clickhouse/arrow/parser.rs index ddafd0d0dc373..94286c09759ca 100644 --- a/src/sinks/clickhouse/arrow/parser.rs +++ b/src/sinks/clickhouse/arrow/parser.rs @@ -42,6 +42,9 @@ pub enum ClickHouseType { String, FixedString(u32), + // UUID type + Uuid, + // Date/time types Date, DateTime, @@ -93,7 +96,13 @@ impl ClickHouseType { }), // String types - Self::String | Self::FixedString(_) => Ok(DataType::Utf8), + // Note: UUID is mapped to Utf8 for two reasons: + // 1. Vector has no native UUID type β€” UUIDs are represented as strings in the event model + // 2. ClickHouse does not support UUID in Arrow format: + // https://github.com/ClickHouse/ClickHouse/blob/master/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp + // https://github.com/ClickHouse/ClickHouse/blob/master/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp + // ClickHouse handles the String β†’ UUID cast implicitly on insert. + Self::String | Self::FixedString(_) | Self::Uuid => Ok(DataType::Utf8), // Date/time types Self::Date => Ok(DataType::Date32), @@ -230,6 +239,7 @@ fn ch_type(input: &str) -> IResult<&str, ClickHouseType> { // String types "String" => Ok((rest, ClickHouseType::String)), + "UUID" => Ok((rest, ClickHouseType::Uuid)), "FixedString" => parens(parse_u32) .map(ClickHouseType::FixedString) .parse(rest), @@ -465,6 +475,19 @@ mod tests { assert!(matches!(data_type, DataType::Map(_, _))); } + #[test] + fn test_uuid_type_mapping() { + assert_eq!(convert_type("UUID").unwrap(), (DataType::Utf8, false)); + assert_eq!( + convert_type("Nullable(UUID)").unwrap(), + (DataType::Utf8, true) + ); + assert_eq!( + convert_type("LowCardinality(Nullable(UUID))").unwrap(), + (DataType::Utf8, true) + ); + } + #[test] fn test_unknown_type_fails() { let result = convert_type("UnknownType"); diff --git a/src/sinks/clickhouse/config.rs b/src/sinks/clickhouse/config.rs index 827b684f2234b..53050dca831ea 100644 --- a/src/sinks/clickhouse/config.rs +++ b/src/sinks/clickhouse/config.rs @@ -4,8 +4,8 @@ use std::fmt; use http::{Request, StatusCode, Uri}; use hyper::Body; +use vector_lib::codecs::encoding::ArrowStreamSerializerConfig; use vector_lib::codecs::encoding::format::SchemaProvider; -use vector_lib::codecs::encoding::{ArrowStreamSerializerConfig, BatchSerializerConfig}; use super::{ request_builder::ClickhouseRequestBuilder, @@ -26,12 +26,11 @@ use crate::{ /// /// [formats]: https://clickhouse.com/docs/en/interfaces/formats #[configurable_component] -#[derive(Clone, Copy, Debug, Derivative, Eq, PartialEq, Hash)] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Hash)] #[serde(rename_all = "snake_case")] -#[derivative(Default)] #[allow(clippy::enum_variant_names)] pub enum Format { - #[derivative(Default)] + #[default] /// JSONEachRow. JsonEachRow, @@ -46,6 +45,23 @@ pub enum Format { ArrowStream, } +/// Batch encoding configuration for the `clickhouse` sink. +#[configurable_component] +#[derive(Clone, Debug)] +#[serde(tag = "codec", rename_all = "snake_case")] +#[configurable(metadata( + docs::enum_tag_description = "The codec to use for batch encoding events." +))] +pub enum ClickhouseBatchEncoding { + /// Encodes events in [Apache Arrow][apache_arrow] IPC streaming format. + /// + /// This is the streaming variant of the Arrow IPC format, which writes + /// a continuous stream of record batches. + /// + /// [apache_arrow]: https://arrow.apache.org/ + ArrowStream(ArrowStreamSerializerConfig), +} + impl fmt::Display for Format { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { @@ -107,7 +123,7 @@ pub struct ClickhouseConfig { /// This is mutually exclusive with per-event encoding based on the `format` field. #[configurable(derived)] #[serde(default)] - pub batch_encoding: Option, + pub batch_encoding: Option, #[configurable(derived)] #[serde(default)] @@ -280,7 +296,8 @@ impl ClickhouseConfig { }; if let Some(batch_encoding) = &self.batch_encoding { - use vector_lib::codecs::{BatchEncoder, BatchSerializer}; + use vector_lib::codecs::BatchEncoder; + use vector_lib::codecs::encoding::BatchSerializerConfig; // Validate that batch_encoding is only compatible with ArrowStream format if self.format != Format::ArrowStream { @@ -291,9 +308,8 @@ impl ClickhouseConfig { .into()); } - let mut arrow_config = match batch_encoding { - BatchSerializerConfig::ArrowStream(config) => config.clone(), - }; + let ClickhouseBatchEncoding::ArrowStream(arrow_config) = batch_encoding; + let mut arrow_config = arrow_config.clone(); self.resolve_arrow_schema( client, @@ -305,8 +321,7 @@ impl ClickhouseConfig { .await?; let resolved_batch_config = BatchSerializerConfig::ArrowStream(arrow_config); - let arrow_serializer = resolved_batch_config.build()?; - let batch_serializer = BatchSerializer::Arrow(arrow_serializer); + let batch_serializer = resolved_batch_config.build_batch_serializer()?; let encoder = EncoderKind::Batch(BatchEncoder::new(batch_serializer)); return Ok((Format::ArrowStream, encoder)); @@ -426,10 +441,31 @@ mod tests { ); } + /// Codecs other than `arrow_stream` must be rejected at parse time, since + /// `ClickhouseBatchEncoding` only exposes the `arrow_stream` variant. + #[cfg(feature = "codecs-parquet")] + #[test] + fn batch_encoding_rejects_unsupported_codec() { + let err = serde_yaml::from_str::( + r#" + endpoint: http://localhost:8123 + table: test_table + batch_encoding: + codec: parquet + "#, + ) + .unwrap_err(); + + assert!( + err.to_string().contains("parquet"), + "expected error to mention the offending codec, got: {err}" + ); + } + /// Helper to create a minimal ClickhouseConfig for testing fn create_test_config( format: Format, - batch_encoding: Option, + batch_encoding: Option, ) -> ClickhouseConfig { ClickhouseConfig { endpoint: "http://localhost:8123".parse::().unwrap().into(), @@ -462,7 +498,7 @@ mod tests { for (format, format_name) in incompatible_formats { let config = create_test_config( format, - Some(BatchSerializerConfig::ArrowStream( + Some(ClickhouseBatchEncoding::ArrowStream( ArrowStreamSerializerConfig::default(), )), ); diff --git a/src/sinks/clickhouse/integration_tests.rs b/src/sinks/clickhouse/integration_tests.rs index f86aeba915db9..2e27a20340dfa 100644 --- a/src/sinks/clickhouse/integration_tests.rs +++ b/src/sinks/clickhouse/integration_tests.rs @@ -18,7 +18,7 @@ use serde::Deserialize; use serde_json::Value; use tokio::time::{Duration, timeout}; use vector_lib::{ - codecs::encoding::{ArrowStreamSerializerConfig, BatchSerializerConfig}, + codecs::encoding::ArrowStreamSerializerConfig, event::{BatchNotifier, BatchStatus, BatchStatusReceiver, Event, LogEvent}, lookup::PathPrefix, }; @@ -28,7 +28,7 @@ use crate::{ codecs::{TimestampFormat, Transformer}, config::{SinkConfig, SinkContext, log_schema}, sinks::{ - clickhouse::config::ClickhouseConfig, + clickhouse::config::{ClickhouseBatchEncoding, ClickhouseConfig}, util::{BatchConfig, Compression, TowerRequestConfig}, }, test_util::{ @@ -502,7 +502,7 @@ async fn insert_events_arrow_format() { table: table.clone().try_into().unwrap(), compression: Compression::None, format: crate::sinks::clickhouse::config::Format::ArrowStream, - batch_encoding: Some(BatchSerializerConfig::ArrowStream(Default::default())), + batch_encoding: Some(ClickhouseBatchEncoding::ArrowStream(Default::default())), batch, request: TowerRequestConfig { retry_attempts: 1, @@ -565,7 +565,7 @@ async fn insert_events_arrow_with_schema_fetching() { client .create_table( &table, - "host String, timestamp DateTime64(3), message String, id Int64, name String, score Float64, active Bool", + "host String, timestamp DateTime64(3), message String, id Int64, name String, score Float64, active Bool, request_id UUID", ) .await; @@ -574,7 +574,7 @@ async fn insert_events_arrow_with_schema_fetching() { table: table.clone().try_into().unwrap(), compression: Compression::None, format: crate::sinks::clickhouse::config::Format::ArrowStream, - batch_encoding: Some(BatchSerializerConfig::ArrowStream(Default::default())), + batch_encoding: Some(ClickhouseBatchEncoding::ArrowStream(Default::default())), batch, request: TowerRequestConfig { retry_attempts: 1, @@ -595,6 +595,10 @@ async fn insert_events_arrow_with_schema_fetching() { event.insert("name", format!("user_{}", i)); event.insert("score", 95.5 + i as f64); event.insert("active", i % 2 == 0); + event.insert( + "request_id", + format!("550e8400-e29b-41d4-a716-44665544000{}", i), + ); events.push(event.into()); } @@ -604,7 +608,7 @@ async fn insert_events_arrow_with_schema_fetching() { assert_eq!(3, output.rows); // Verify all fields exist and have the correct types - for row in output.data.iter() { + for (i, row) in output.data.iter().enumerate() { // Check standard Vector fields exist assert!(row.get("host").and_then(|v| v.as_str()).is_some()); assert!(row.get("message").and_then(|v| v.as_str()).is_some()); @@ -620,6 +624,16 @@ async fn insert_events_arrow_with_schema_fetching() { assert!(row.get("name").and_then(|v| v.as_str()).is_some()); assert!(row.get("score").and_then(|v| v.as_f64()).is_some()); assert!(row.get("active").and_then(|v| v.as_bool()).is_some()); + + // Check UUID field + let request_id = row + .get("request_id") + .and_then(|v| v.as_str()) + .expect("request_id should be present"); + assert_eq!( + request_id, + format!("550e8400-e29b-41d4-a716-44665544000{}", i) + ); } } @@ -643,7 +657,7 @@ async fn test_complex_types() { table: table.clone().try_into().unwrap(), compression: Compression::None, format: crate::sinks::clickhouse::config::Format::ArrowStream, - batch_encoding: Some(BatchSerializerConfig::ArrowStream(arrow_config)), + batch_encoding: Some(ClickhouseBatchEncoding::ArrowStream(arrow_config)), batch, request: TowerRequestConfig { retry_attempts: 1, @@ -1217,7 +1231,7 @@ async fn test_missing_required_field_emits_null_constraint_error() { table: table.clone().try_into().unwrap(), compression: Compression::None, format: crate::sinks::clickhouse::config::Format::ArrowStream, - batch_encoding: Some(BatchSerializerConfig::ArrowStream(Default::default())), + batch_encoding: Some(ClickhouseBatchEncoding::ArrowStream(Default::default())), batch, request: TowerRequestConfig { retry_attempts: 1, @@ -1309,7 +1323,7 @@ async fn arrow_schema_excludes_non_insertable_columns() { table: table.clone().try_into().unwrap(), compression: Compression::None, format: crate::sinks::clickhouse::config::Format::ArrowStream, - batch_encoding: Some(BatchSerializerConfig::ArrowStream( + batch_encoding: Some(ClickhouseBatchEncoding::ArrowStream( ArrowStreamSerializerConfig::default(), )), batch, diff --git a/src/sinks/clickhouse/sink.rs b/src/sinks/clickhouse/sink.rs index e86f484d89024..ea2f6d434097e 100644 --- a/src/sinks/clickhouse/sink.rs +++ b/src/sinks/clickhouse/sink.rs @@ -43,7 +43,8 @@ where input .batched_partitioned( KeyPartitioner::new(self.database, self.table, self.format), - || batch_settings.as_byte_size_config(), + batch_settings.timeout, + |_| batch_settings.as_byte_size_config(), ) .filter_map(|(key, batch)| async move { key.map(move |k| (k, batch)) }) .request_builder( diff --git a/src/sinks/console/config.rs b/src/sinks/console/config.rs index 9175cb3d0b516..c7a8717c0a933 100644 --- a/src/sinks/console/config.rs +++ b/src/sinks/console/config.rs @@ -18,14 +18,13 @@ use crate::{ /// /// [standard_streams]: https://en.wikipedia.org/wiki/Standard_streams #[configurable_component] -#[derive(Clone, Debug, Derivative)] -#[derivative(Default)] +#[derive(Clone, Debug, Default)] #[serde(rename_all = "lowercase")] pub enum Target { /// Write output to [STDOUT][stdout]. /// /// [stdout]: https://en.wikipedia.org/wiki/Standard_streams#Standard_output_(stdout) - #[derivative(Default)] + #[default] Stdout, /// Write output to [STDERR][stderr]. diff --git a/src/sinks/databricks_zerobus/config.rs b/src/sinks/databricks_zerobus/config.rs new file mode 100644 index 0000000000000..1df2f62a6f1c3 --- /dev/null +++ b/src/sinks/databricks_zerobus/config.rs @@ -0,0 +1,421 @@ +//! Configuration for the Zerobus sink. + +use vector_lib::configurable::configurable_component; +use vector_lib::sensitive_string::SensitiveString; + +use crate::config::{AcknowledgementsConfig, GenerateConfig, Input, SinkConfig, SinkContext}; +use crate::sinks::{ + prelude::*, + util::{BatchConfig, RealtimeSizeBasedDefaultBatchSettings}, +}; + +use super::{error::ZerobusSinkError, service::ZerobusService, sink::ZerobusSink}; + +/// Authentication configuration for Databricks. +#[configurable_component] +#[derive(Clone, Debug)] +#[serde(tag = "strategy", rename_all = "snake_case")] +#[configurable(metadata( + docs::enum_tag_description = "The authentication strategy to use for Databricks." +))] +pub enum DatabricksAuthentication { + /// Authenticate using OAuth 2.0 client credentials. + #[serde(rename = "oauth")] + OAuth { + /// OAuth 2.0 client ID. + #[configurable(metadata(docs::examples = "${DATABRICKS_CLIENT_ID}"))] + #[configurable(metadata(docs::examples = "abc123..."))] + client_id: SensitiveString, + + /// OAuth 2.0 client secret. + #[configurable(metadata(docs::examples = "${DATABRICKS_CLIENT_SECRET}"))] + #[configurable(metadata(docs::examples = "secret123..."))] + client_secret: SensitiveString, + }, +} + +impl DatabricksAuthentication { + /// Extract the client ID and client secret as string references. + pub fn credentials(&self) -> (&str, &str) { + match self { + DatabricksAuthentication::OAuth { + client_id, + client_secret, + } => (client_id.inner(), client_secret.inner()), + } + } +} + +/// Zerobus stream configuration options. +/// +/// This is a thin wrapper around the SDK's `StreamConfigurationOptions` with Vector-specific +/// configuration attributes and custom defaults suitable for Vector's use case. +#[configurable_component] +#[derive(Clone, Debug)] +#[serde(deny_unknown_fields)] +pub struct ZerobusStreamOptions { + /// Timeout in milliseconds for flush operations. + #[serde(default = "default_flush_timeout_ms")] + #[configurable(metadata(docs::examples = 30000))] + pub flush_timeout_ms: u64, + + /// Timeout in milliseconds for server acknowledgements. + #[serde(default = "default_server_ack_timeout_ms")] + #[configurable(metadata(docs::examples = 60000))] + pub server_lack_of_ack_timeout_ms: u64, +} + +impl Default for ZerobusStreamOptions { + fn default() -> Self { + Self { + flush_timeout_ms: default_flush_timeout_ms(), + server_lack_of_ack_timeout_ms: default_server_ack_timeout_ms(), + } + } +} + +/// Configuration for the Databricks Zerobus sink. +#[configurable_component(sink( + "databricks_zerobus", + "Stream observability data to Databricks Unity Catalog via Zerobus." +))] +#[derive(Clone, Debug)] +#[serde(deny_unknown_fields)] +pub struct ZerobusSinkConfig { + /// The Zerobus ingestion endpoint URL. + /// + /// This should be the full URL to the Zerobus ingestion service. + #[configurable(metadata(docs::examples = "https://ingest.dev.databricks.com"))] + #[configurable(metadata(docs::examples = "https://ingest.prod.databricks.com"))] + pub ingestion_endpoint: String, + + /// The Unity Catalog table name to write to. + /// + /// This should be in the format `catalog.schema.table`. + #[configurable(metadata(docs::examples = "logging_platform.my_team.logs"))] + #[configurable(metadata(docs::examples = "main.default.vector_logs"))] + pub table_name: String, + + /// The Unity Catalog endpoint URL. + /// + /// This is used for authentication and table metadata. + #[configurable(metadata( + docs::examples = "https://dbc-e2f0eb31-2b0e.staging.cloud.databricks.com" + ))] + #[configurable(metadata(docs::examples = "https://your-workspace.cloud.databricks.com"))] + pub unity_catalog_endpoint: String, + + /// Databricks authentication configuration. + #[configurable(derived)] + pub auth: DatabricksAuthentication, + + #[configurable(derived)] + #[serde(default)] + pub stream_options: ZerobusStreamOptions, + + #[configurable(derived)] + #[serde(default)] + pub batch: BatchConfig, + + #[configurable(derived)] + #[serde(default)] + pub request: TowerRequestConfig, + + #[configurable(derived)] + #[serde( + default, + deserialize_with = "crate::serde::bool_or_struct", + skip_serializing_if = "crate::serde::is_default" + )] + pub acknowledgements: AcknowledgementsConfig, +} + +impl GenerateConfig for ZerobusSinkConfig { + fn generate_config() -> toml::Value { + toml::Value::try_from(Self { + ingestion_endpoint: "https://ingest.dev.databricks.com".to_string(), + table_name: "catalog.schema.table".to_string(), + unity_catalog_endpoint: "https://your-workspace.cloud.databricks.com".to_string(), + auth: DatabricksAuthentication::OAuth { + client_id: SensitiveString::from("${DATABRICKS_CLIENT_ID}".to_string()), + client_secret: SensitiveString::from("${DATABRICKS_CLIENT_SECRET}".to_string()), + }, + stream_options: ZerobusStreamOptions::default(), + batch: BatchConfig::default(), + request: TowerRequestConfig::default(), + acknowledgements: AcknowledgementsConfig::default(), + }) + .unwrap() + } +} + +#[async_trait::async_trait] +#[typetag::serde(name = "databricks_zerobus")] +impl SinkConfig for ZerobusSinkConfig { + async fn build(&self, cx: SinkContext) -> crate::Result<(VectorSink, Healthcheck)> { + self.validate()?; + + let service = ZerobusService::new(self.clone(), cx.proxy()).await?; + let healthcheck_service = service.clone(); + + let request_limits = self.request.into_settings(); + + let sink = ZerobusSink::new(service, request_limits, self.batch)?; + + let healthcheck = async move { + healthcheck_service + .ensure_stream() + .await + .map_err(|e| e.into()) + }; + + Ok(( + VectorSink::from_event_streamsink(sink), + Box::pin(healthcheck), + )) + } + + fn input(&self) -> Input { + Input::log() + } + + fn acknowledgements(&self) -> &AcknowledgementsConfig { + &self.acknowledgements + } +} + +impl ZerobusSinkConfig { + pub fn validate(&self) -> Result<(), ZerobusSinkError> { + if self.ingestion_endpoint.is_empty() { + return Err(ZerobusSinkError::ConfigError { + message: "ingestion_endpoint cannot be empty".to_string(), + }); + } + + if self.table_name.is_empty() { + return Err(ZerobusSinkError::ConfigError { + message: "table_name cannot be empty".to_string(), + }); + } + + let parts: Vec<&str> = self.table_name.split('.').collect(); + if parts.len() != 3 || parts.iter().any(|p| p.is_empty()) { + return Err(ZerobusSinkError::ConfigError { + message: "table_name must be in format 'catalog.schema.table' (exactly 3 non-empty parts)" + .to_string(), + }); + } + + if self.unity_catalog_endpoint.is_empty() { + return Err(ZerobusSinkError::ConfigError { + message: "unity_catalog_endpoint cannot be empty".to_string(), + }); + } + + // Validate authentication credentials + match &self.auth { + DatabricksAuthentication::OAuth { + client_id, + client_secret, + } => { + if client_id.inner().is_empty() { + return Err(ZerobusSinkError::ConfigError { + message: "OAuth client_id cannot be empty".to_string(), + }); + } + if client_secret.inner().is_empty() { + return Err(ZerobusSinkError::ConfigError { + message: "OAuth client_secret cannot be empty".to_string(), + }); + } + } + } + + if let Some(max_bytes) = self.batch.max_bytes { + // Zerobus SDK limits max bytes to 10MB. This cap is a conservative safety limit: + // it's measured against Vector's pre-serialization sizing, not the protobuf bytes + // the SDK actually sends. Vector's pre-serialization size is generally larger than + // the SDK's protobuf-encoded size, so enforcing the 10MB cap here ensures the SDK's + // 10MB limit cannot be exceeded at runtime. + if max_bytes > 10_000_000 { + return Err(ZerobusSinkError::ConfigError { + message: "max_bytes must be less than or equal to 10MB".to_string(), + }); + } + } + + Ok(()) + } +} + +// Default value functions +const fn default_flush_timeout_ms() -> u64 { + 30000 +} + +const fn default_server_ack_timeout_ms() -> u64 { + 60000 +} + +#[cfg(test)] +mod tests { + use super::*; + use vector_lib::sensitive_string::SensitiveString; + + fn create_test_config() -> ZerobusSinkConfig { + ZerobusSinkConfig { + ingestion_endpoint: "https://test.databricks.com".to_string(), + table_name: "test.default.logs".to_string(), + unity_catalog_endpoint: "https://test-workspace.databricks.com".to_string(), + auth: DatabricksAuthentication::OAuth { + client_id: SensitiveString::from("test-client-id".to_string()), + client_secret: SensitiveString::from("test-client-secret".to_string()), + }, + stream_options: ZerobusStreamOptions::default(), + batch: Default::default(), + request: Default::default(), + acknowledgements: Default::default(), + } + } + + #[test] + fn test_config_validation_success() { + let config = create_test_config(); + assert!(config.validate().is_ok()); + } + + #[test] + fn test_config_validation_empty_endpoint() { + let mut config = create_test_config(); + config.ingestion_endpoint = "".to_string(); + + let result = config.validate(); + assert!(result.is_err()); + + if let Err(crate::sinks::databricks_zerobus::error::ZerobusSinkError::ConfigError { + message, + }) = result + { + assert!(message.contains("ingestion_endpoint cannot be empty")); + } else { + panic!("Expected ConfigError for empty ingestion_endpoint"); + } + } + + #[test] + fn test_config_validation_empty_table_name() { + let mut config = create_test_config(); + config.table_name = "".to_string(); + + let result = config.validate(); + assert!(result.is_err()); + + if let Err(crate::sinks::databricks_zerobus::error::ZerobusSinkError::ConfigError { + message, + }) = result + { + assert!(message.contains("table_name cannot be empty")); + } else { + panic!("Expected ConfigError for empty table_name"); + } + } + + #[test] + fn test_config_validation_invalid_table_name() { + let mut config = create_test_config(); + config.table_name = "invalid_table".to_string(); // Missing dots + + let result = config.validate(); + assert!(result.is_err()); + + if let Err(crate::sinks::databricks_zerobus::error::ZerobusSinkError::ConfigError { + message, + }) = result + { + assert!(message.contains("catalog.schema.table")); + } else { + panic!("Expected ConfigError for invalid table_name format"); + } + } + + #[test] + fn test_config_validation_table_name_empty_segments() { + for bad in [ + "catalog..table", + ".schema.table", + "catalog.schema.", + "..", + "catalog.schema.table.extra", + ] { + let mut config = create_test_config(); + config.table_name = bad.to_string(); + let result = config.validate(); + assert!(result.is_err(), "expected error for table_name={bad:?}"); + if let Err(crate::sinks::databricks_zerobus::error::ZerobusSinkError::ConfigError { + message, + }) = result + { + assert!(message.contains("catalog.schema.table")); + } else { + panic!("Expected ConfigError for table_name={bad:?}"); + } + } + } + + #[test] + fn test_config_validation_empty_unity_catalog_endpoint() { + let mut config = create_test_config(); + config.unity_catalog_endpoint = "".to_string(); + + let result = config.validate(); + assert!(result.is_err()); + + if let Err(crate::sinks::databricks_zerobus::error::ZerobusSinkError::ConfigError { + message, + }) = result + { + assert!(message.contains("unity_catalog_endpoint cannot be empty")); + } else { + panic!("Expected ConfigError for empty unity_catalog_endpoint"); + } + } + + #[test] + fn test_config_validation_empty_oauth_credentials() { + let mut config = create_test_config(); + config.auth = DatabricksAuthentication::OAuth { + client_id: SensitiveString::from("".to_string()), + client_secret: SensitiveString::from("test-secret".to_string()), + }; + + let result = config.validate(); + assert!(result.is_err()); + + if let Err(crate::sinks::databricks_zerobus::error::ZerobusSinkError::ConfigError { + message, + }) = result + { + assert!(message.contains("OAuth client_id cannot be empty")); + } else { + panic!("Expected ConfigError for empty OAuth client_id"); + } + } + + /// When `batch.max_bytes` is `None` (user omitted the field or set it to `null`), + /// `into_batcher_settings()` must merge it against + /// `RealtimeSizeBasedDefaultBatchSettings::MAX_BYTES` (10MB) β€” never unbounded. + /// This guarantees the Zerobus SDK's 10MB limit cannot be exceeded at runtime + /// even without an explicit user cap. + #[test] + fn test_batch_max_bytes_none_defaults_to_10mb() { + let mut config = create_test_config(); + config.batch.max_bytes = None; + + let settings = config + .batch + .into_batcher_settings() + .expect("batch settings should build"); + + assert_eq!(settings.size_limit, 10_000_000); + } +} diff --git a/src/sinks/databricks_zerobus/error.rs b/src/sinks/databricks_zerobus/error.rs new file mode 100644 index 0000000000000..1e9a630415724 --- /dev/null +++ b/src/sinks/databricks_zerobus/error.rs @@ -0,0 +1,207 @@ +//! Error types for the Zerobus sink. + +use databricks_zerobus_ingest_sdk::ZerobusError; +use snafu::Snafu; +use vector_lib::event::EventStatus; + +/// Errors that can occur when using the Zerobus sink. +#[derive(Debug, Snafu)] +#[allow(clippy::enum_variant_names)] +pub enum ZerobusSinkError { + /// Configuration validation failed. + #[snafu(display("Configuration error: {}", message))] + ConfigError { message: String }, + + /// Event encoding failed. + #[snafu(display("Encoding error: {}", message))] + EncodingError { message: String }, + + /// Zerobus SDK error. + #[snafu(display("Zerobus error: {}", source))] + ZerobusError { source: ZerobusError }, + + /// Stream initialization failed. + #[snafu(display("Stream initialization failed: {}", source))] + StreamInitError { source: ZerobusError }, + + /// Record ingestion failed. + #[snafu(display("Record ingestion failed: {}", source))] + IngestionError { source: ZerobusError }, + + /// The SDK returned no offset to wait on, so server acceptance of the + /// batch cannot be confirmed. Treated as non-retryable. + #[snafu(display("Zerobus ingest returned no offset; cannot confirm server acceptance"))] + MissingAckOffset, + + /// The shared stream was closed concurrently (by shutdown or retry-driven + /// replacement) before this ingest could run. Retryable: the next attempt + /// will create a fresh stream via `get_or_create_stream`. + #[snafu(display("Zerobus stream was closed concurrently"))] + StreamClosed, + + /// Resolving the table schema from Unity Catalog failed. `retryable` + /// distinguishes transient failures (network, 5xx, 408, 429) from + /// permanent ones (404, 401, 403, ...). + #[snafu(display("Schema resolution failed: {}", message))] + SchemaError { message: String, retryable: bool }, +} + +impl ZerobusSinkError { + /// Whether this error should be retried. + pub fn is_retryable(&self) -> bool { + match self { + Self::ZerobusError { source } + | Self::StreamInitError { source } + | Self::IngestionError { source } => source.is_retryable(), + Self::StreamClosed => true, + Self::SchemaError { retryable, .. } => *retryable, + Self::ConfigError { .. } | Self::EncodingError { .. } | Self::MissingAckOffset => false, + } + } + + /// Event status to apply to a batch's finalizers when this error occurs. + pub fn event_status(&self) -> EventStatus { + if self.is_retryable() { + EventStatus::Errored + } else { + EventStatus::Rejected + } + } +} + +impl From for ZerobusSinkError { + fn from(error: ZerobusError) -> Self { + ZerobusSinkError::ZerobusError { source: error } + } +} + +impl From for EventStatus { + fn from(error: ZerobusSinkError) -> Self { + error.event_status() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::sinks::databricks_zerobus::service::ZerobusRetryLogic; + use crate::sinks::util::retries::RetryLogic; + + fn retryable_error() -> ZerobusError { + // ChannelCreationError is always retryable + ZerobusError::ChannelCreationError("connection reset".to_string()) + } + + fn non_retryable_error() -> ZerobusError { + // InvalidArgument is never retryable + ZerobusError::InvalidArgument("bad field".to_string()) + } + + #[test] + fn retryable_ingestion_error_maps_to_errored() { + let error = ZerobusSinkError::IngestionError { + source: retryable_error(), + }; + assert_eq!(EventStatus::from(error), EventStatus::Errored); + } + + #[test] + fn non_retryable_ingestion_error_maps_to_rejected() { + let error = ZerobusSinkError::IngestionError { + source: non_retryable_error(), + }; + assert_eq!(EventStatus::from(error), EventStatus::Rejected); + } + + #[test] + fn retryable_stream_init_error_maps_to_errored() { + let error = ZerobusSinkError::StreamInitError { + source: retryable_error(), + }; + assert_eq!(EventStatus::from(error), EventStatus::Errored); + } + + #[test] + fn non_retryable_stream_init_error_maps_to_rejected() { + let error = ZerobusSinkError::StreamInitError { + source: non_retryable_error(), + }; + assert_eq!(EventStatus::from(error), EventStatus::Rejected); + } + + #[test] + fn config_error_maps_to_rejected() { + let error = ZerobusSinkError::ConfigError { + message: "bad config".to_string(), + }; + assert_eq!(EventStatus::from(error), EventStatus::Rejected); + } + + #[test] + fn encoding_error_maps_to_rejected() { + let error = ZerobusSinkError::EncodingError { + message: "encode failed".to_string(), + }; + assert_eq!(EventStatus::from(error), EventStatus::Rejected); + } + + #[test] + fn retry_logic_retryable_errors() { + let logic = ZerobusRetryLogic; + + let error = ZerobusSinkError::IngestionError { + source: retryable_error(), + }; + assert!(logic.is_retriable_error(&error)); + + let error = ZerobusSinkError::StreamInitError { + source: retryable_error(), + }; + assert!(logic.is_retriable_error(&error)); + + let error = ZerobusSinkError::ZerobusError { + source: retryable_error(), + }; + assert!(logic.is_retriable_error(&error)); + } + + #[test] + fn retry_logic_non_retryable_errors() { + let logic = ZerobusRetryLogic; + + let error = ZerobusSinkError::IngestionError { + source: non_retryable_error(), + }; + assert!(!logic.is_retriable_error(&error)); + + let error = ZerobusSinkError::ConfigError { + message: "bad".to_string(), + }; + assert!(!logic.is_retriable_error(&error)); + + let error = ZerobusSinkError::EncodingError { + message: "bad".to_string(), + }; + assert!(!logic.is_retriable_error(&error)); + } + + #[test] + fn retryable_schema_error_maps_to_errored() { + let error = ZerobusSinkError::SchemaError { + message: "UC 503".to_string(), + retryable: true, + }; + assert!(ZerobusRetryLogic.is_retriable_error(&error)); + assert_eq!(EventStatus::from(error), EventStatus::Errored); + } + + #[test] + fn non_retryable_schema_error_maps_to_rejected() { + let error = ZerobusSinkError::SchemaError { + message: "UC 404".to_string(), + retryable: false, + }; + assert!(!ZerobusRetryLogic.is_retriable_error(&error)); + assert_eq!(EventStatus::from(error), EventStatus::Rejected); + } +} diff --git a/src/sinks/databricks_zerobus/mod.rs b/src/sinks/databricks_zerobus/mod.rs new file mode 100644 index 0000000000000..19419b3216e4d --- /dev/null +++ b/src/sinks/databricks_zerobus/mod.rs @@ -0,0 +1,12 @@ +//! The Zerobus sink. +//! +//! This sink streams observability data to Databricks Unity Catalog tables +//! via the Zerobus/Shinkansen ingestion service. + +mod config; +mod error; +mod service; +mod sink; +mod unity_catalog_schema; + +pub use config::ZerobusSinkConfig; diff --git a/src/sinks/databricks_zerobus/service.rs b/src/sinks/databricks_zerobus/service.rs new file mode 100644 index 0000000000000..076488e55bf49 --- /dev/null +++ b/src/sinks/databricks_zerobus/service.rs @@ -0,0 +1,1024 @@ +//! Zerobus service wrapper for Vector sink integration. + +use crate::config::ProxyConfig; +use crate::event::Event; +use crate::http::HttpClient; +use crate::sinks::util::retries::RetryLogic; +use crate::tls::TlsSettings; +use databricks_zerobus_ingest_sdk::{ConnectorFactory, ProxyConnector, ZerobusSdk, ZerobusStream}; +use futures::future::BoxFuture; +use std::sync::Arc; +use tokio::sync::{Mutex, OnceCell, RwLock}; +use tower::{Layer, Service}; +use tracing::warn; +use vector_lib::codecs::encoding::{ + BatchEncoder, BatchOutput, BatchSerializerConfig, ProtoBatchSerializerConfig, +}; +use vector_lib::finalization::{EventFinalizers, Finalizable}; +use vector_lib::request_metadata::{GroupedCountByteSize, MetaDescriptive, RequestMetadata}; +use vector_lib::stream::DriverResponse; + +use super::{config::ZerobusSinkConfig, error::ZerobusSinkError, unity_catalog_schema}; + +/// Build a connector factory that routes Zerobus gRPC traffic through +/// Vector's configured proxy, honoring `no_proxy` rules. +/// +/// The Zerobus endpoint is always HTTPS gRPC, so the `https` proxy is +/// preferred; the `http` proxy is used as a fallback if only that is set. +/// The returned factory fully replaces the SDK's default env-var proxy +/// detection β€” Vector's `ProxyConfig` has already merged the process +/// environment at a higher layer and is the single source of truth. +/// +/// When proxying is disabled or no proxy URL is configured, returns a +/// factory that unconditionally yields `None`, forcing direct connections. +/// Returns an error if the configured proxy URL is malformed, so the +/// problem surfaces at sink startup rather than per-connection. +fn build_connector_factory(proxy: &ProxyConfig) -> Result { + let proxy_url = if proxy.enabled { + proxy.https.clone().or_else(|| proxy.http.clone()) + } else { + None + }; + let Some(proxy_url) = proxy_url else { + return Ok(Arc::new(|_host: &str| None)); + }; + // Validate the proxy URL once up-front so a malformed value surfaces at + // sink startup rather than per-connection. + ProxyConnector::new(&proxy_url).map_err(|e| ZerobusSinkError::ConfigError { + message: format!("Invalid proxy URL '{}': {}", proxy_url, e), + })?; + let no_proxy = proxy.no_proxy.clone(); + Ok(Arc::new(move |host: &str| { + if no_proxy.matches(host) { + return None; + } + ProxyConnector::new(&proxy_url).ok() + })) +} + +/// Request type for the Zerobus service. +/// +/// Carries the *unencoded* batch β€” encoding happens inside `Service::call` so +/// that schema-fetch failures flow through the Tower retry layer. Events live +/// behind an `Arc` because Tower's retry policy clones the request before +/// every call (not just on retry), and a deep clone of `Vec` per call +/// would be wasteful. +#[derive(Clone)] +pub struct ZerobusRequest { + pub events: Arc>, + pub metadata: RequestMetadata, + pub finalizers: EventFinalizers, +} + +/// Response type for the Zerobus service. +/// +/// Carries the final `EventStatus` so the driver can mark finalizers correctly: +/// `Delivered` on success, `Errored` when the retry budget was exhausted on a +/// transient failure (asking the source / disk buffer to replay), and `Err` +/// from `Service::call` reserved for permanent failures (driver maps to +/// `Rejected`). +#[derive(Debug)] +pub struct ZerobusResponse { + pub events_byte_size: GroupedCountByteSize, + pub status: vector_lib::event::EventStatus, +} + +impl ZerobusResponse { + const fn delivered(events_byte_size: GroupedCountByteSize) -> Self { + Self { + events_byte_size, + status: vector_lib::event::EventStatus::Delivered, + } + } + + /// Synthesize a response signalling a transient failure that exhausted the + /// retry budget. Carries a telemetry-aware zero `events_byte_size` because + /// the driver only consumes `events_sent()` on the `Delivered` path. + fn errored() -> Self { + Self { + events_byte_size: vector_lib::config::telemetry().create_request_count_byte_size(), + status: vector_lib::event::EventStatus::Errored, + } + } +} + +impl DriverResponse for ZerobusResponse { + fn event_status(&self) -> vector_lib::event::EventStatus { + self.status + } + + fn events_sent(&self) -> &GroupedCountByteSize { + &self.events_byte_size + } +} + +impl Finalizable for ZerobusRequest { + fn take_finalizers(&mut self) -> EventFinalizers { + std::mem::take(&mut self.finalizers) + } +} + +impl MetaDescriptive for ZerobusRequest { + fn get_metadata(&self) -> &RequestMetadata { + &self.metadata + } + + fn metadata_mut(&mut self) -> &mut RequestMetadata { + &mut self.metadata + } +} + +/// The active stream. +/// +/// The SDK's `ZerobusStream::close()` requires `&mut self`, but ingests need +/// shared access to call `&self` methods concurrently. We resolve this with an +/// `RwLock`: ingests hold a read guard across `ingest_records_offset`, and +/// `close()` takes the write guard, pulls the stream out of the `Option`, and +/// awaits its SDK-level close on the owned value. Any holder of an `Arc` can +/// invoke `close()`, so the graceful path always runs β€” there is no +/// `try_unwrap`/`get_mut` race. +enum ActiveStream { + Proto(RwLock>>), + /// Test-only variant that returns a pre-configured error on ingest. + #[cfg(test)] + Mock(MockStream), +} + +impl ActiveStream { + fn proto(stream: ZerobusStream) -> Self { + ActiveStream::Proto(RwLock::new(Some(Box::new(stream)))) + } + + /// Gracefully flush and close the underlying SDK stream. + /// + /// Waits for any in-flight ingests (read-lock holders) to complete, then + /// pulls the stream out of the slot and runs the SDK's awaitable `close()` + /// on the owned value (released-lock so further ingests fail fast with + /// `StreamClosed` rather than blocking). + /// + /// Idempotent: a second call after the stream has been taken is a no-op. + /// The SDK's own `Drop` is also a no-op once close has run. + async fn close(&self) { + let result = match self { + ActiveStream::Proto(lock) => { + let taken = lock.write().await.take(); + match taken { + Some(mut stream) => stream.close().await, + None => return, + } + } + #[cfg(test)] + ActiveStream::Mock(m) => { + m.closed.store(true, std::sync::atomic::Ordering::Relaxed); + Ok(()) + } + }; + if let Err(e) = result { + warn!(message = "Failed to close Zerobus stream.", error = %e); + } + } +} + +/// A mock stream that returns a configurable error on the next ingest call. +#[cfg(test)] +pub struct MockStream { + /// When `Some`, the next ingest returns this error; when `None`, ingest succeeds. + next_error: std::sync::Mutex>, + /// Shared flag set to `true` when `ActiveStream::close()` is called. + closed: Arc, + /// Optional gate: when set, each ingest call signals `started` and then + /// waits to acquire a `release` permit before returning. Lets tests + /// deterministically force two ingests to overlap (each holding an `Arc` + /// clone of the `ActiveStream`) before they fail. + gate: Option, +} + +#[cfg(test)] +struct MockGate { + started: Arc, + release: Arc, +} + +#[cfg(test)] +#[derive(Clone)] +pub struct MockGateHandle { + started: Arc, + release: Arc, +} + +#[cfg(test)] +impl MockGateHandle { + /// Wait until `n` ingests have entered the gated region. + pub async fn wait_for_started(&self, n: u32) { + let permit = self.started.acquire_many(n).await.unwrap(); + permit.forget(); + } + + /// Release `n` queued ingests so they can return their result. + pub fn release(&self, n: u32) { + self.release.add_permits(n as usize); + } +} + +#[cfg(test)] +impl MockStream { + pub fn succeeding() -> Self { + Self { + next_error: std::sync::Mutex::new(None), + closed: Arc::new(std::sync::atomic::AtomicBool::new(false)), + gate: None, + } + } + + pub fn failing(error: databricks_zerobus_ingest_sdk::ZerobusError) -> Self { + Self { + next_error: std::sync::Mutex::new(Some(error)), + closed: Arc::new(std::sync::atomic::AtomicBool::new(false)), + gate: None, + } + } + + /// Install a gate so ingests block until the test releases them. + /// Returns a handle the test uses to coordinate. + pub fn with_gate(mut self) -> (Self, MockGateHandle) { + let started = Arc::new(tokio::sync::Semaphore::new(0)); + let release = Arc::new(tokio::sync::Semaphore::new(0)); + self.gate = Some(MockGate { + started: Arc::clone(&started), + release: Arc::clone(&release), + }); + (self, MockGateHandle { started, release }) + } + + /// Returns a shared handle to the closed flag for test assertions. + pub fn closed_flag(&self) -> Arc { + Arc::clone(&self.closed) + } + + /// Set the error that will be returned on the next ingest call. + pub fn set_next_error(&self, error: databricks_zerobus_ingest_sdk::ZerobusError) { + *self.next_error.lock().unwrap() = Some(error); + } + + async fn try_ingest(&self) -> Result<(), databricks_zerobus_ingest_sdk::ZerobusError> { + if let Some(gate) = &self.gate { + gate.started.add_permits(1); + // Acquire and immediately forget β€” we don't need to release the + // permit on drop, the test's `release()` call hands them out. + gate.release.acquire().await.unwrap().forget(); + } + match self.next_error.lock().unwrap().take() { + Some(e) => Err(e), + None => Ok(()), + } + } +} + +/// Schema and encoding state derived from the Unity Catalog table. +pub(super) struct ResolvedSchema { + encoder: BatchEncoder, + /// SDK-typed (prost-types 0.14) descriptor β€” held in this form so each + /// stream rebuild avoids re-encoding from the prost-reflect 0.13 form. + descriptor_proto: Arc, +} + +/// Service for handling Zerobus requests. +pub struct ZerobusService { + sdk: Arc, + config: Arc, + http_client: HttpClient, + stream: Arc>>>, + schema: Arc>, +} + +impl ZerobusService { + pub async fn new( + config: ZerobusSinkConfig, + proxy: &ProxyConfig, + ) -> Result { + let mut builder = ZerobusSdk::builder() + .endpoint(&config.ingestion_endpoint) + .unity_catalog_url(&config.unity_catalog_endpoint); + builder = builder.connector_factory(build_connector_factory(proxy)?); + let sdk = builder.build().map_err(|e| ZerobusSinkError::ConfigError { + message: format!("Failed to create Zerobus SDK: {}", e), + })?; + + let http_client = HttpClient::new(TlsSettings::default(), proxy).map_err(|e| { + ZerobusSinkError::ConfigError { + message: format!("Failed to create HTTP client: {}", e), + } + })?; + + Ok(Self { + sdk: Arc::new(sdk), + config: Arc::new(config), + http_client, + stream: Arc::new(Mutex::new(None)), + schema: Arc::new(OnceCell::new()), + }) + } + + /// Resolve the protobuf message descriptor from Unity Catalog. + /// + /// Returns both the prost-reflect `MessageDescriptor` (used by the proto + /// batch encoder) and the SDK-typed `DescriptorProto` (used to construct + /// Zerobus streams). Returning both avoids re-encoding the descriptor + /// every time a stream is rebuilt after a retryable failure. + async fn resolve_descriptor( + config: &ZerobusSinkConfig, + http_client: &HttpClient, + ) -> Result< + ( + prost_reflect::MessageDescriptor, + prost_types_014::DescriptorProto, + ), + ZerobusSinkError, + > { + let (client_id, client_secret) = config.auth.credentials(); + + let table_schema = unity_catalog_schema::fetch_table_schema( + &config.unity_catalog_endpoint, + &config.table_name, + client_id, + client_secret, + http_client, + ) + .await?; + + unity_catalog_schema::generate_descriptor_from_schema(&table_schema) + } + + /// Resolve the schema on first use; cache the result. + pub(super) async fn ensure_schema(&self) -> Result<&ResolvedSchema, ZerobusSinkError> { + self.schema + .get_or_try_init(|| async { + let (descriptor, sdk_descriptor_proto) = + Self::resolve_descriptor(&self.config, &self.http_client).await?; + let descriptor_proto = Arc::new(sdk_descriptor_proto); + + let batch_serializer = + BatchSerializerConfig::ProtoBatch(ProtoBatchSerializerConfig { + descriptor: Some(descriptor), + }) + .build_batch_serializer() + .map_err(|e| ZerobusSinkError::ConfigError { + message: format!("Failed to build batch serializer: {}", e), + })?; + + Ok(ResolvedSchema { + encoder: BatchEncoder::new(batch_serializer), + descriptor_proto, + }) + }) + .await + } + + pub(super) fn encode_records( + schema: &ResolvedSchema, + events: &[Event], + ) -> Result>, ZerobusSinkError> { + match schema + .encoder + .encode_batch(events) + .map_err(|e| ZerobusSinkError::EncodingError { + message: format!("Failed to encode batch: {}", e), + })? { + BatchOutput::Records(records) => Ok(records), + #[cfg(feature = "codecs-arrow")] + BatchOutput::Arrow(_) => Err(ZerobusSinkError::EncodingError { + message: "The Databricks Zerobus sink only supports proto-batch output.".into(), + }), + } + } + + /// Ensure we have an active stream, creating one if necessary. + /// + /// Also used as the healthcheck: resolving the schema verifies the table + /// and credentials against Unity Catalog, and creating the stream verifies + /// connectivity to the Zerobus endpoint. + pub async fn ensure_stream(&self) -> Result<(), ZerobusSinkError> { + let schema = self.ensure_schema().await?; + self.get_or_create_stream(schema).await.map(|_| ()) + } + + /// Return an `Arc` handle to the active stream, creating one if needed. + /// + /// The lock is held only while checking/creating the stream; callers can + /// then use the returned `Arc` without holding the lock. + async fn get_or_create_stream( + &self, + schema: &ResolvedSchema, + ) -> Result, ZerobusSinkError> { + let mut stream_guard = self.stream.lock().await; + + if stream_guard.is_none() { + let (client_id, client_secret) = self.config.auth.credentials(); + let (client_id, client_secret) = (client_id.to_string(), client_secret.to_string()); + + let stream_options = &self.config.stream_options; + let stream = self + .sdk + .stream_builder() + .table(self.config.table_name.clone()) + .oauth(client_id, client_secret) + .compiled_proto((*schema.descriptor_proto).clone()) + .server_lack_of_ack_timeout_ms(stream_options.server_lack_of_ack_timeout_ms) + .flush_timeout_ms(stream_options.flush_timeout_ms) + .build() + .await + .map_err(|e| ZerobusSinkError::StreamInitError { source: e })?; + + *stream_guard = Some(Arc::new(ActiveStream::proto(stream))); + } + + Ok(Arc::clone(stream_guard.as_ref().unwrap())) + } + + /// Gracefully close and remove the active stream. + /// + /// `ActiveStream::close()` takes `&self`, so this works regardless of how + /// many `Arc` clones are still in flight: the inner write lock waits for + /// any concurrent ingests to release their read guards before the SDK + /// flush + close runs. The slot lock is released before close starts so + /// concurrent `get_or_create_stream` calls aren't blocked on the SDK + /// shutdown path. + pub async fn close_stream(&self) { + let stream = self.stream.lock().await.take(); + if let Some(stream) = stream { + stream.close().await; + } + } + + /// Send encoded records to an already-resolved stream. + /// + /// On retryable errors the active stream is removed from the slot so that + /// the next attempt (driven by Tower retry) creates a fresh one. + async fn ingest( + &self, + stream: Arc, + records: Vec>, + events_byte_size: GroupedCountByteSize, + ) -> Result { + // Slot lock is not held here β€” concurrent ingests acquire read guards + // on the inner `RwLock` and run truly in parallel. + let result = match stream.as_ref() { + ActiveStream::Proto(lock) => { + let guard = lock.read().await; + let Some(s) = guard.as_ref() else { + return Err(ZerobusSinkError::StreamClosed); + }; + match s.ingest_records_offset(records).await { + Ok(Some(offset)) => s.wait_for_offset(offset).await.map(|_| ()), + Ok(None) => { + return Err(ZerobusSinkError::MissingAckOffset); + } + Err(e) => Err(e), + } + } + #[cfg(test)] + ActiveStream::Mock(mock) => mock.try_ingest().await, + }; + + match result { + Ok(()) => Ok(ZerobusResponse::delivered(events_byte_size)), + Err(e) => { + if e.is_retryable() { + // Clear the slot so the next attempt creates a fresh stream, + // but only if it still points to the same stream that failed β€” + // a concurrent task may have already replaced it. + { + let mut guard = self.stream.lock().await; + if guard.as_ref().is_some_and(|s| Arc::ptr_eq(s, &stream)) { + guard.take(); + } + } + // `close()` takes `&self`, so we can always run the graceful + // path here regardless of how many other `Arc` clones are in + // flight. The write lock will wait for any concurrent ingests + // holding read guards to drain before flushing. + stream.close().await; + } + Err(ZerobusSinkError::IngestionError { source: e }) + } + } + } +} + +impl Service for ZerobusService { + type Response = ZerobusResponse; + type Error = ZerobusSinkError; + type Future = BoxFuture<'static, Result>; + + fn poll_ready( + &mut self, + _cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + std::task::Poll::Ready(Ok(())) + } + + fn call(&mut self, mut request: ZerobusRequest) -> Self::Future { + let service = self.clone(); + let events_byte_size = + std::mem::take(request.metadata_mut()).into_events_estimated_json_encoded_byte_size(); + + Box::pin(async move { + let schema = service.ensure_schema().await?; + let records = Self::encode_records(schema, &request.events)?; + let stream = service.get_or_create_stream(schema).await?; + service.ingest(stream, records, events_byte_size).await + }) + } +} + +impl Clone for ZerobusService { + fn clone(&self) -> Self { + Self { + sdk: Arc::clone(&self.sdk), + config: Arc::clone(&self.config), + http_client: self.http_client.clone(), + stream: Arc::clone(&self.stream), + schema: Arc::clone(&self.schema), + } + } +} + +/// Retry logic for the Zerobus service. +/// +/// For SDK errors (`ZerobusError`), delegates to the SDK's `is_retryable()` which +/// correctly marks transient errors (stream closed, channel issues) as retriable +/// and permanent errors (invalid table name, invalid argument, invalid endpoint) +/// as non-retriable. +#[derive(Debug, Default, Clone)] +pub struct ZerobusRetryLogic; + +#[cfg(test)] +impl ZerobusService { + /// Create a service with a mock stream already installed for testing. + pub async fn new_with_mock( + config: ZerobusSinkConfig, + mock: MockStream, + ) -> Result { + config.validate()?; + + let sdk = ZerobusSdk::builder() + .endpoint(&config.ingestion_endpoint) + .unity_catalog_url(&config.unity_catalog_endpoint) + .build() + .map_err(|e| ZerobusSinkError::ConfigError { + message: format!("Failed to create Zerobus SDK: {}", e), + })?; + + let http_client = HttpClient::new(TlsSettings::default(), &ProxyConfig::default()) + .map_err(|e| ZerobusSinkError::ConfigError { + message: format!("Failed to create HTTP client: {}", e), + })?; + + Ok(Self { + sdk: Arc::new(sdk), + config: Arc::new(config), + http_client, + stream: Arc::new(Mutex::new(Some(Arc::new(ActiveStream::Mock(mock))))), + schema: Arc::new(OnceCell::new()), + }) + } + + /// Returns true if the service currently has an active stream. + pub async fn has_active_stream(&self) -> bool { + self.stream.lock().await.is_some() + } +} + +impl RetryLogic for ZerobusRetryLogic { + type Error = ZerobusSinkError; + type Request = ZerobusRequest; + type Response = ZerobusResponse; + + fn is_retriable_error(&self, error: &Self::Error) -> bool { + error.is_retryable() + } +} + +/// Tower layer that converts retry-budget-exhausted retryable errors into a +/// successful `ZerobusResponse` carrying `EventStatus::Errored`. +/// +/// Wraps the retry layer from the outside. When the retry layer returns: +/// - `Ok(resp)` β€” pass through unchanged. +/// - `Err(e)` where `e.is_retryable()` β€” convert to `Ok(ZerobusResponse::errored())` +/// so the driver marks finalizers `Errored` (transient β€” source / disk +/// buffer may replay) rather than `Rejected` (permanent drop). +/// - `Err(e)` permanent β€” propagate so the driver maps to `Rejected`. +/// +/// Without this layer the driver maps every `Err` from `Service::call` to +/// `EventStatus::Rejected`, which would drop transient-but-exhausted failures +/// as if they were permanent. +#[derive(Clone, Debug, Default)] +pub struct RetryableErrorAsErroredLayer; + +impl Layer for RetryableErrorAsErroredLayer { + type Service = RetryableErrorAsErrored; + + fn layer(&self, inner: S) -> Self::Service { + RetryableErrorAsErrored { inner } + } +} + +#[derive(Clone, Debug)] +pub struct RetryableErrorAsErrored { + inner: S, +} + +impl Service for RetryableErrorAsErrored +where + S: Service, + S::Future: Send + 'static, +{ + type Response = ZerobusResponse; + type Error = crate::Error; + type Future = BoxFuture<'static, Result>; + + fn poll_ready( + &mut self, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + self.inner.poll_ready(cx) + } + + fn call(&mut self, req: ZerobusRequest) -> Self::Future { + let fut = self.inner.call(req); + Box::pin(async move { + match fut.await { + Ok(resp) => Ok(resp), + Err(e) => { + // The Tower stack boxes errors above us (retry, timeout, + // adaptive-concurrency). Downcast to inspect retryability; + // anything that isn't a `ZerobusSinkError` (e.g. a timeout + // `Elapsed`) is conservatively treated as transient. + let retryable = match e.downcast_ref::() { + Some(zb) => zb.is_retryable(), + None => true, + }; + if retryable { + warn!( + message = "Zerobus retry budget exhausted on transient error; signaling Errored so source or buffer may replay.", + error = %e, + ); + Ok(ZerobusResponse::errored()) + } else { + Err(e) + } + } + } + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::sinks::databricks_zerobus::config::{ + DatabricksAuthentication, ZerobusStreamOptions, + }; + use databricks_zerobus_ingest_sdk::ZerobusError; + use vector_lib::sensitive_string::SensitiveString; + + fn test_config() -> ZerobusSinkConfig { + ZerobusSinkConfig { + ingestion_endpoint: "https://127.0.0.1:1".to_string(), + table_name: "test.default.logs".to_string(), + unity_catalog_endpoint: "https://127.0.0.1:1".to_string(), + auth: DatabricksAuthentication::OAuth { + client_id: SensitiveString::from("id".to_string()), + client_secret: SensitiveString::from("secret".to_string()), + }, + stream_options: ZerobusStreamOptions::default(), + batch: Default::default(), + request: Default::default(), + acknowledgements: Default::default(), + } + } + + fn dummy_records() -> Vec> { + vec![vec![1, 2, 3]] + } + + async fn current_stream(service: &ZerobusService) -> Arc { + Arc::clone(service.stream.lock().await.as_ref().unwrap()) + } + + #[tokio::test] + async fn ingest_succeeds_with_mock_stream() { + let service = ZerobusService::new_with_mock(test_config(), MockStream::succeeding()) + .await + .unwrap(); + + let stream = current_stream(&service).await; + let result = service + .ingest( + stream, + dummy_records(), + GroupedCountByteSize::new_untagged(), + ) + .await; + + assert!(result.is_ok()); + assert!(service.has_active_stream().await); + } + + #[tokio::test] + async fn retryable_error_clears_stream() { + let mock = MockStream::failing(ZerobusError::ChannelCreationError( + "connection reset".to_string(), + )); + let service = ZerobusService::new_with_mock(test_config(), mock) + .await + .unwrap(); + + assert!(service.has_active_stream().await); + + let stream = current_stream(&service).await; + let err = service + .ingest( + stream, + dummy_records(), + GroupedCountByteSize::new_untagged(), + ) + .await + .unwrap_err(); + + assert!(matches!(err, ZerobusSinkError::IngestionError { .. })); + assert!(ZerobusRetryLogic.is_retriable_error(&err)); + // Stream must have been cleared for the next retry. + assert!(!service.has_active_stream().await); + } + + #[tokio::test] + async fn non_retryable_error_keeps_stream() { + let mock = MockStream::failing(ZerobusError::InvalidArgument("bad field".to_string())); + let service = ZerobusService::new_with_mock(test_config(), mock) + .await + .unwrap(); + + assert!(service.has_active_stream().await); + + let stream = current_stream(&service).await; + let err = service + .ingest( + stream, + dummy_records(), + GroupedCountByteSize::new_untagged(), + ) + .await + .unwrap_err(); + + assert!(matches!(err, ZerobusSinkError::IngestionError { .. })); + assert!(!ZerobusRetryLogic.is_retriable_error(&err)); + // Stream should NOT be cleared for non-retryable errors. + assert!(service.has_active_stream().await); + } + + #[tokio::test] + async fn stream_recovers_after_retryable_failure() { + // Simulate: success β†’ retryable failure β†’ success again. + let mock = MockStream::succeeding(); + let service = ZerobusService::new_with_mock(test_config(), mock) + .await + .unwrap(); + + // First ingest succeeds. + let stream = current_stream(&service).await; + assert!( + service + .ingest( + stream, + dummy_records(), + GroupedCountByteSize::new_untagged() + ) + .await + .is_ok() + ); + assert!(service.has_active_stream().await); + + // Inject a retryable error for the next call. + { + let guard = service.stream.lock().await; + if let Some(arc) = guard.as_ref() + && let ActiveStream::Mock(mock) = arc.as_ref() + { + mock.set_next_error(ZerobusError::ChannelCreationError("reset".to_string())); + } + } + + // Second ingest fails and clears the stream. + let stream = current_stream(&service).await; + let err = service + .ingest( + stream, + dummy_records(), + GroupedCountByteSize::new_untagged(), + ) + .await + .unwrap_err(); + assert!(ZerobusRetryLogic.is_retriable_error(&err)); + assert!(!service.has_active_stream().await); + + // Simulate Tower retry: re-inject a fresh mock stream + // (in production, ensure_stream() would create a new real stream). + *service.stream.lock().await = Some(Arc::new(ActiveStream::Mock(MockStream::succeeding()))); + + // Third ingest succeeds on the new stream. + let stream = current_stream(&service).await; + assert!( + service + .ingest( + stream, + dummy_records(), + GroupedCountByteSize::new_untagged() + ) + .await + .is_ok() + ); + assert!(service.has_active_stream().await); + } + + #[tokio::test] + async fn close_stream_calls_close_on_active_stream() { + let mock = MockStream::succeeding(); + let closed = mock.closed_flag(); + + let service = ZerobusService::new_with_mock(test_config(), mock) + .await + .unwrap(); + + assert!(service.has_active_stream().await); + assert!(!closed.load(std::sync::atomic::Ordering::Relaxed)); + + service.close_stream().await; + + assert!(!service.has_active_stream().await); + assert!(closed.load(std::sync::atomic::Ordering::Relaxed)); + } + + /// Regression test for the "silent abort-only Drop" issue: when two + /// ingests are in flight (each holding an `Arc`) and one + /// fails retryably, the failing task must still run the graceful close + /// path. Under the previous design `Arc::get_mut` returned `None` here + /// because the second task held a clone, so close was skipped and the + /// stream fell to abort-only Drop. + #[tokio::test] + async fn retryable_failure_with_concurrent_ingest_still_closes() { + let (mock, gate) = MockStream::failing(ZerobusError::ChannelCreationError( + "connection reset".to_string(), + )) + .with_gate(); + let closed = mock.closed_flag(); + + let service = ZerobusService::new_with_mock(test_config(), mock) + .await + .unwrap(); + + // Spawn two concurrent ingests. Each clones the same stream `Arc`, + // then blocks in the gate. + let s1 = service.clone(); + let t1 = tokio::spawn(async move { + let stream = current_stream(&s1).await; + s1.ingest( + stream, + dummy_records(), + GroupedCountByteSize::new_untagged(), + ) + .await + }); + let s2 = service.clone(); + let t2 = tokio::spawn(async move { + let stream = current_stream(&s2).await; + s2.ingest( + stream, + dummy_records(), + GroupedCountByteSize::new_untagged(), + ) + .await + }); + + // Wait until both ingests are inside the gate (both `Arc`s alive). + gate.wait_for_started(2).await; + + // Release both. The failing one will go through the retry-cleanup + // path while the other still holds an `Arc`. Under the old design + // `Arc::get_mut` would return `None` and close would be skipped. + gate.release(2); + + let r1 = t1.await.unwrap(); + let r2 = t2.await.unwrap(); + + // At least one task observed the retryable error (the mock only + // produces a single error, but ordering between tasks is undefined). + assert!(r1.is_err() || r2.is_err()); + + // The graceful close path must have run despite concurrent `Arc`s. + assert!( + closed.load(std::sync::atomic::Ordering::Relaxed), + "graceful close did not run; stream would have leaked under old design" + ); + // And the slot was cleared so the next ingest creates a fresh stream. + assert!(!service.has_active_stream().await); + } + + fn dummy_request() -> ZerobusRequest { + ZerobusRequest { + events: Arc::new(vec![]), + metadata: RequestMetadata::default(), + finalizers: EventFinalizers::default(), + } + } + + #[tokio::test] + async fn retryable_err_after_exhaustion_becomes_ok_errored() { + use tower::ServiceExt; + let inner = tower::service_fn(|_req: ZerobusRequest| async move { + let err: crate::Error = Box::new(ZerobusSinkError::SchemaError { + message: "UC 503".to_string(), + retryable: true, + }); + Err::(err) + }); + let mut svc = RetryableErrorAsErrored { inner }; + let resp = svc + .ready() + .await + .unwrap() + .call(dummy_request()) + .await + .unwrap(); + assert_eq!(resp.status, vector_lib::event::EventStatus::Errored); + } + + #[tokio::test] + async fn non_retryable_err_propagates() { + use tower::ServiceExt; + let inner = tower::service_fn(|_req: ZerobusRequest| async move { + let err: crate::Error = Box::new(ZerobusSinkError::EncodingError { + message: "bad".to_string(), + }); + Err::(err) + }); + let mut svc = RetryableErrorAsErrored { inner }; + let err = svc + .ready() + .await + .unwrap() + .call(dummy_request()) + .await + .unwrap_err(); + let zb = err.downcast_ref::().unwrap(); + assert!(matches!(zb, ZerobusSinkError::EncodingError { .. })); + } + + #[tokio::test] + async fn unknown_err_treated_as_transient() { + use tower::ServiceExt; + // Simulate a Tower-layer error that isn't a ZerobusSinkError (e.g. + // timeout `Elapsed`): conservatively becomes Errored, not Rejected. + #[derive(Debug)] + struct Other; + impl std::fmt::Display for Other { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "other") + } + } + impl std::error::Error for Other {} + + let inner = tower::service_fn(|_req: ZerobusRequest| async move { + let err: crate::Error = Box::new(Other); + Err::(err) + }); + let mut svc = RetryableErrorAsErrored { inner }; + let resp = svc + .ready() + .await + .unwrap() + .call(dummy_request()) + .await + .unwrap(); + assert_eq!(resp.status, vector_lib::event::EventStatus::Errored); + } + + #[tokio::test] + async fn ok_response_passes_through() { + use tower::ServiceExt; + let inner = tower::service_fn(|_req: ZerobusRequest| async move { + Ok::<_, crate::Error>(ZerobusResponse::delivered( + GroupedCountByteSize::new_untagged(), + )) + }); + let mut svc = RetryableErrorAsErrored { inner }; + let resp = svc + .ready() + .await + .unwrap() + .call(dummy_request()) + .await + .unwrap(); + assert_eq!(resp.status, vector_lib::event::EventStatus::Delivered); + } +} diff --git a/src/sinks/databricks_zerobus/sink.rs b/src/sinks/databricks_zerobus/sink.rs new file mode 100644 index 0000000000000..037984fd1d4d4 --- /dev/null +++ b/src/sinks/databricks_zerobus/sink.rs @@ -0,0 +1,80 @@ +//! The main Zerobus sink implementation. + +use std::num::NonZeroUsize; +use std::sync::Arc; + +use futures::StreamExt; +use futures::stream::BoxStream; + +use vector_lib::finalization::Finalizable; + +use crate::sinks::prelude::*; +use crate::sinks::util::metadata::RequestMetadataBuilder; +use crate::sinks::util::{RealtimeSizeBasedDefaultBatchSettings, TowerRequestSettings}; + +use super::service::{ + RetryableErrorAsErroredLayer, ZerobusRequest, ZerobusRetryLogic, ZerobusService, +}; + +/// The main Zerobus sink. +pub struct ZerobusSink { + service: ZerobusService, + request_limits: TowerRequestSettings, + batch_settings: BatcherSettings, +} + +impl ZerobusSink { + pub fn new( + service: ZerobusService, + request_limits: TowerRequestSettings, + batch_config: BatchConfig, + ) -> Result { + let batch_settings = batch_config.into_batcher_settings()?; + + Ok(Self { + service, + request_limits, + batch_settings, + }) + } + + async fn run_inner(self: Box, input: BoxStream<'_, Event>) -> Result<(), ()> { + let result = { + let tower_service = ServiceBuilder::new() + .layer(RetryableErrorAsErroredLayer) + .settings(self.request_limits, ZerobusRetryLogic) + .service(self.service.clone()); + + input + .batched(self.batch_settings.as_byte_size_config()) + .map(|mut events| { + let finalizers = events.take_finalizers(); + // The encoded request size isn't known until `Service::call` + // encodes the batch, and nothing in the zerobus path reads + // `RequestMetadata::request_encoded_size`, so a placeholder + // is fine here. + let metadata = RequestMetadataBuilder::from_events(&events) + .with_request_size(NonZeroUsize::MIN); + ZerobusRequest { + events: Arc::new(events), + metadata, + finalizers, + } + }) + .into_driver(tower_service) + .run() + .await + }; + + self.service.close_stream().await; + + result + } +} + +#[async_trait::async_trait] +impl StreamSink for ZerobusSink { + async fn run(self: Box, input: BoxStream<'_, Event>) -> Result<(), ()> { + self.run_inner(input).await + } +} diff --git a/src/sinks/databricks_zerobus/tests/fixtures/nested_structs_complete_schema.json b/src/sinks/databricks_zerobus/tests/fixtures/nested_structs_complete_schema.json new file mode 100644 index 0000000000000..f30f749d13230 --- /dev/null +++ b/src/sinks/databricks_zerobus/tests/fixtures/nested_structs_complete_schema.json @@ -0,0 +1,127 @@ +{ + "name": "nested_structs_table", + "catalog_name": "test_catalog", + "schema_name": "test_schema", + "columns": [ + { + "name": "field_001", + "type_text": "bigint", + "type_name": "LONG", + "position": 0, + "nullable": true, + "type_json": "{\"name\":\"field_001\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}" + }, + { + "name": "field_002", + "type_text": "string", + "type_name": "STRING", + "position": 1, + "nullable": true, + "type_json": "{\"name\":\"field_002\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}" + }, + { + "name": "field_003", + "type_text": "string", + "type_name": "STRING", + "position": 2, + "nullable": true, + "type_json": "{\"name\":\"field_003\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}" + }, + { + "name": "field_004", + "type_text": "string", + "type_name": "STRING", + "position": 3, + "nullable": true, + "type_json": "{\"name\":\"field_004\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}" + }, + { + "name": "field_005", + "type_text": "string", + "type_name": "STRING", + "position": 4, + "nullable": true, + "type_json": "{\"name\":\"field_005\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}" + }, + { + "name": "field_006", + "type_text": "bigint", + "type_name": "LONG", + "position": 5, + "nullable": true, + "type_json": "{\"name\":\"field_006\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}" + }, + { + "name": "field_007", + "type_text": "bigint", + "type_name": "LONG", + "position": 6, + "nullable": true, + "type_json": "{\"name\":\"field_007\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}" + }, + { + "name": "field_008", + "type_text": "struct,field_012:struct,field_014:struct,field_016:struct>", + "type_name": "STRUCT", + "position": 7, + "nullable": true, + "type_json": "{\"name\":\"field_008\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"field_009\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"field_010\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"field_011\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"field_012\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"field_013\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"field_014\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"field_015\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"field_016\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"field_017\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}" + }, + { + "name": "field_018", + "type_text": "struct", + "type_name": "STRUCT", + "position": 8, + "nullable": true, + "type_json": "{\"name\":\"field_018\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"field_019\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"field_020\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}" + }, + { + "name": "field_021", + "type_text": "struct", + "type_name": "STRUCT", + "position": 9, + "nullable": true, + "type_json": "{\"name\":\"field_021\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"field_022\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"field_023\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"field_020\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"field_024\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"field_025\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"field_026\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}" + }, + { + "name": "field_027", + "type_text": "array", + "type_name": "ARRAY", + "position": 10, + "nullable": true, + "type_json": "{\"name\":\"field_027\",\"type\":{\"type\":\"array\",\"elementType\":\"long\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}}" + }, + { + "name": "field_028", + "type_text": "boolean", + "type_name": "BOOLEAN", + "position": 11, + "nullable": true, + "type_json": "{\"name\":\"field_028\",\"type\":\"boolean\",\"nullable\":true,\"metadata\":{}}" + }, + { + "name": "field_029", + "type_text": "string", + "type_name": "STRING", + "position": 12, + "nullable": true, + "type_json": "{\"name\":\"field_029\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}" + }, + { + "name": "field_030", + "type_text": "string", + "type_name": "STRING", + "position": 13, + "nullable": true, + "type_json": "{\"name\":\"field_030\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}" + }, + { + "name": "field_031", + "type_text": "boolean", + "type_name": "BOOLEAN", + "position": 14, + "nullable": true, + "type_json": "{\"name\":\"field_031\",\"type\":\"boolean\",\"nullable\":true,\"metadata\":{}}" + } + ] +} diff --git a/src/sinks/databricks_zerobus/unity_catalog_schema.rs b/src/sinks/databricks_zerobus/unity_catalog_schema.rs new file mode 100644 index 0000000000000..343604512242d --- /dev/null +++ b/src/sinks/databricks_zerobus/unity_catalog_schema.rs @@ -0,0 +1,530 @@ +//! Unity Catalog schema fetching and protobuf descriptor generation. +//! +//! The UC-to-protobuf conversion is delegated to +//! [`databricks_zerobus_ingest_sdk::schema`]; this module only wraps it with +//! the HTTP fetching + descriptor-pool assembly that the sink needs. + +use bytes::Buf; +use databricks_zerobus_ingest_sdk::schema::descriptor_from_uc_schema; +use http::{Request, StatusCode, Uri}; +use http_body::Body as HttpBody; +use hyper::Body; +use percent_encoding::{NON_ALPHANUMERIC, percent_encode}; +use prost_reflect::prost_types; +use serde::Deserialize; + +use super::error::ZerobusSinkError; +use crate::http::HttpClient; + +/// Whether a Unity Catalog HTTP response status should be retried. +/// +/// Delegates to the canonical Vector HTTP retry policy +/// [`crate::sinks::util::http::RetryStrategy::Default`] so this sink stays in +/// lock-step with other HTTP-based sinks: 5xx (except 501 Not Implemented), +/// 408 (Request Timeout), and 429 (Too Many Requests) are transient; 4xx +/// otherwise (404, 401, 403, ...) and 501 are permanent. +fn status_is_retryable(status: StatusCode) -> bool { + use crate::sinks::util::{http::RetryStrategy, retries::RetryAction}; + matches!( + RetryStrategy::Default.retry_action::<()>(status), + RetryAction::Retry(_) | RetryAction::RetryPartial(_) + ) +} + +// Alias the SDK types under the names the rest of the sink already uses. +#[cfg(test)] +use databricks_zerobus_ingest_sdk::schema::UcColumn as UnityCatalogColumn; +pub use databricks_zerobus_ingest_sdk::schema::UcTableSchema as UnityCatalogTableSchema; + +/// OAuth token response from Databricks +#[derive(Debug, Deserialize)] +struct OAuthTokenResponse { + access_token: String, +} + +/// Fetch table schema from Unity Catalog API +pub async fn fetch_table_schema( + unity_catalog_endpoint: &str, + table_name: &str, + client_id: &str, + client_secret: &str, + http_client: &HttpClient, +) -> Result { + let token = get_oauth_token( + http_client, + unity_catalog_endpoint, + client_id, + client_secret, + ) + .await?; + + // Fetch table schema. + // Encode each segment of the fully-qualified table name (catalog.schema.table) + // so that reserved URI characters in quoted Unity Catalog identifiers (spaces, + // #, /, etc.) don't break URI parsing or hit the wrong endpoint. + let encoded_table_name: String = table_name + .split('.') + .map(|seg| percent_encode(seg.as_bytes(), NON_ALPHANUMERIC).to_string()) + .collect::>() + .join("."); + let url = format!( + "{}/api/2.1/unity-catalog/tables/{}", + unity_catalog_endpoint.trim_end_matches('/'), + encoded_table_name + ); + + let uri: Uri = url.parse().map_err(|e| ZerobusSinkError::ConfigError { + message: format!("Invalid Unity Catalog endpoint URL: {}", e), + })?; + + let request = Request::get(uri) + .header("Authorization", format!("Bearer {}", token)) + .header("Content-Type", "application/json") + .body(Body::empty()) + .map_err(|e| ZerobusSinkError::ConfigError { + message: format!("Failed to build request: {}", e), + })?; + + let response = http_client + .send(request) + .await + .map_err(|e| ZerobusSinkError::SchemaError { + message: format!("Failed to fetch table schema: {}", e), + retryable: true, + })?; + + let status = response.status(); + if !status.is_success() { + let body_bytes = response + .into_body() + .collect() + .await + .map(|c| c.to_bytes()) + .unwrap_or_default(); + let error_text = String::from_utf8_lossy(&body_bytes); + return Err(ZerobusSinkError::SchemaError { + message: format!( + "Unity Catalog API returned error {}: {}", + status, error_text + ), + retryable: status_is_retryable(status), + }); + } + + let body_bytes = response + .into_body() + .collect() + .await + .map(|c| c.to_bytes()) + .map_err(|e| ZerobusSinkError::SchemaError { + message: format!("Failed to read response body: {}", e), + retryable: true, + })?; + + let schema: UnityCatalogTableSchema = + serde_json::from_reader(body_bytes.reader()).map_err(|e| { + ZerobusSinkError::ConfigError { + message: format!("Failed to parse table schema response: {}", e), + } + })?; + + Ok(schema) +} + +/// Get OAuth token from Databricks +async fn get_oauth_token( + http_client: &HttpClient, + unity_catalog_endpoint: &str, + client_id: &str, + client_secret: &str, +) -> Result { + let token_url = format!( + "{}/oidc/v1/token", + unity_catalog_endpoint.trim_end_matches('/') + ); + + let uri: Uri = token_url + .parse() + .map_err(|e| ZerobusSinkError::ConfigError { + message: format!("Invalid token endpoint URL: {}", e), + })?; + + // Build form-encoded body + let form_body = format!( + "grant_type=client_credentials&client_id={}&client_secret={}&scope=all-apis", + percent_encode(client_id.as_bytes(), NON_ALPHANUMERIC), + percent_encode(client_secret.as_bytes(), NON_ALPHANUMERIC) + ); + + let request = Request::post(uri) + .header("Content-Type", "application/x-www-form-urlencoded") + .body(Body::from(form_body)) + .map_err(|e| ZerobusSinkError::ConfigError { + message: format!("Failed to build OAuth request: {}", e), + })?; + + let response = http_client + .send(request) + .await + .map_err(|e| ZerobusSinkError::SchemaError { + message: format!("Failed to get OAuth token: {}", e), + retryable: true, + })?; + + let status = response.status(); + if !status.is_success() { + let body_bytes = response + .into_body() + .collect() + .await + .map(|c| c.to_bytes()) + .unwrap_or_default(); + let error_text = String::from_utf8_lossy(&body_bytes); + return Err(ZerobusSinkError::SchemaError { + message: format!("OAuth token request failed {}: {}", status, error_text), + retryable: status_is_retryable(status), + }); + } + + let body_bytes = response + .into_body() + .collect() + .await + .map(|c| c.to_bytes()) + .map_err(|e| ZerobusSinkError::SchemaError { + message: format!("Failed to read OAuth response body: {}", e), + retryable: true, + })?; + + let token_response: OAuthTokenResponse = + serde_json::from_reader(body_bytes.reader()).map_err(|e| { + ZerobusSinkError::ConfigError { + message: format!("Failed to parse OAuth token response: {}", e), + } + })?; + + Ok(token_response.access_token) +} + +/// Format a protobuf MessageDescriptor as a .proto file string for logging +fn format_descriptor_as_proto(descriptor: &prost_reflect::MessageDescriptor) -> String { + let mut output = String::new(); + format_message_as_proto(descriptor, &mut output, 0); + output +} + +/// Recursively format a message and its nested types +fn format_message_as_proto( + descriptor: &prost_reflect::MessageDescriptor, + output: &mut String, + indent_level: usize, +) { + let indent = " ".repeat(indent_level); + + // Write message header + output.push_str(&format!("{}message {} {{\n", indent, descriptor.name())); + + // Write fields + for field in descriptor.fields() { + let field_indent = " ".repeat(indent_level + 1); + let field_type = format_field_type(&field); + let field_number = field.number(); + output.push_str(&format!( + "{}{}{} = {};\n", + field_indent, + field_type, + field.name(), + field_number + )); + } + + output.push_str(&format!("{}}}\n", indent)); + + // Write nested message types + for nested in descriptor.child_messages() { + output.push('\n'); + format_message_as_proto(&nested, output, indent_level); + } +} + +/// Format a field's type declaration +fn format_field_type(field: &prost_reflect::FieldDescriptor) -> String { + use prost_reflect::Kind; + + if field.is_map() { + // Map fields: map field_name + if let Kind::Message(map_entry) = field.kind() { + let key_field = map_entry.fields().find(|f| f.name() == "key").unwrap(); + let value_field = map_entry.fields().find(|f| f.name() == "value").unwrap(); + let key_type = format_scalar_type(&key_field); + let value_type = format_scalar_type(&value_field); + return format!("map<{}, {}> ", key_type, value_type); + } + } + + let base_type = match field.kind() { + Kind::Message(msg) => msg.name().to_string(), + kind => format_kind_type(&kind), + }; + + if field.is_list() { + format!("repeated {} ", base_type) + } else { + format!("{} ", base_type) + } +} + +/// Format a scalar field type (for map keys/values) +fn format_scalar_type(field: &prost_reflect::FieldDescriptor) -> String { + match field.kind() { + prost_reflect::Kind::Message(msg) => msg.name().to_string(), + kind => format_kind_type(&kind), + } +} + +/// Map Kind enum to proto type string +fn format_kind_type(kind: &prost_reflect::Kind) -> String { + use prost_reflect::Kind; + match kind { + Kind::Double => "double".into(), + Kind::Float => "float".into(), + Kind::Int32 => "int32".into(), + Kind::Int64 => "int64".into(), + Kind::Uint32 => "uint32".into(), + Kind::Uint64 => "uint64".into(), + Kind::Sint32 => "sint32".into(), + Kind::Sint64 => "sint64".into(), + Kind::Fixed32 => "fixed32".into(), + Kind::Fixed64 => "fixed64".into(), + Kind::Sfixed32 => "sfixed32".into(), + Kind::Sfixed64 => "sfixed64".into(), + Kind::Bool => "bool".into(), + Kind::String => "string".into(), + Kind::Bytes => "bytes".into(), + Kind::Message(msg) => msg.name().to_string(), + Kind::Enum(e) => e.name().to_string(), + } +} + +/// Generate a protobuf message descriptor from a Unity Catalog table schema. +/// +/// The core UC-type β†’ protobuf conversion lives in +/// [`databricks_zerobus_ingest_sdk::schema::descriptor_from_uc_schema`]; this +/// wrapper adds the `FileDescriptorProto` / `DescriptorPool` plumbing that +/// Vector needs to get a `prost_reflect::MessageDescriptor` usable for +/// dynamic message encoding. +/// +/// Returns both the `MessageDescriptor` (for dynamic encoding via prost-reflect) +/// and the SDK-typed `DescriptorProto` (prost-types 0.14, used when constructing +/// the Zerobus stream). Returning both lets us avoid re-encoding the descriptor +/// on every stream rebuild. +pub fn generate_descriptor_from_schema( + schema: &UnityCatalogTableSchema, +) -> Result< + ( + prost_reflect::MessageDescriptor, + prost_types_014::DescriptorProto, + ), + ZerobusSinkError, +> { + let sdk_message_proto = + descriptor_from_uc_schema(schema).map_err(|e| ZerobusSinkError::ConfigError { + message: format!("Failed to convert Unity Catalog schema to protobuf: {}", e), + })?; + + // The SDK returns a prost-types 0.14 DescriptorProto, but prost-reflect (used + // below to build the descriptor pool) is on prost-types 0.13. Re-encode through + // the protobuf wire format to bridge the two versions. + let message_name = sdk_message_proto.name().to_string(); + let encoded = prost_014::Message::encode_to_vec(&sdk_message_proto); + let message_proto = + ::decode( + encoded.as_slice(), + ) + .map_err(|e| ZerobusSinkError::ConfigError { + message: format!("Failed to re-decode SDK DescriptorProto: {}", e), + })?; + let package_name = sanitize_package_name(&schema.catalog_name); + + let file_proto = prost_types::FileDescriptorProto { + name: Some(format!("{}.proto", message_name)), + package: Some(package_name.clone()), + message_type: vec![message_proto], + ..Default::default() + }; + + let file_descriptor_set = prost_types::FileDescriptorSet { + file: vec![file_proto], + }; + + let pool = prost_reflect::DescriptorPool::from_file_descriptor_set(file_descriptor_set) + .map_err(|e| ZerobusSinkError::ConfigError { + message: format!("Failed to build descriptor pool: {}", e), + })?; + + let full_message_name = format!("{}.{}", package_name, message_name); + let message_descriptor = pool + .get_message_by_name(&full_message_name) + .ok_or_else(|| ZerobusSinkError::ConfigError { + message: format!("Failed to get message descriptor for {}", full_message_name), + })?; + + if tracing::enabled!(tracing::Level::INFO) { + let proto_schema = format_descriptor_as_proto(&message_descriptor); + info!( + "Inferred protobuf schema from Unity Catalog table {}.{}.{}:\n{}", + schema.catalog_name, schema.schema_name, schema.name, proto_schema + ); + } + + Ok((message_descriptor, sdk_message_proto)) +} + +/// Default prefix for package name segments that start with a non-letter. +const PACKAGE_SEGMENT_PREFIX: char = 'p'; + +/// Sanitize a string for use as a protobuf package name. +/// +/// Package identifiers allow `[a-zA-Z][a-zA-Z0-9_]*` segments separated by `.`. +/// Invalid characters are replaced with `_` and each segment is ensured to start +/// with a letter. +fn sanitize_package_name(name: &str) -> String { + name.split('.') + .map(|segment| { + let mut s: String = segment + .chars() + .map(|c| { + if c.is_ascii_alphanumeric() || c == '_' { + c + } else { + '_' + } + }) + .collect(); + if s.is_empty() || !s.starts_with(|c: char| c.is_ascii_alphabetic()) { + s.insert(0, PACKAGE_SEGMENT_PREFIX); + } + s + }) + .collect::>() + .join(".") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn status_is_retryable_matches_canonical_policy() { + // Transient β€” must retry. + assert!(status_is_retryable(StatusCode::INTERNAL_SERVER_ERROR)); + assert!(status_is_retryable(StatusCode::BAD_GATEWAY)); + assert!(status_is_retryable(StatusCode::SERVICE_UNAVAILABLE)); + assert!(status_is_retryable(StatusCode::GATEWAY_TIMEOUT)); + assert!(status_is_retryable(StatusCode::REQUEST_TIMEOUT)); + assert!(status_is_retryable(StatusCode::TOO_MANY_REQUESTS)); + // Permanent β€” must not retry. 501 in particular: the server doesn't + // support the requested functionality; retry won't change that. + assert!(!status_is_retryable(StatusCode::NOT_IMPLEMENTED)); + assert!(!status_is_retryable(StatusCode::NOT_FOUND)); + assert!(!status_is_retryable(StatusCode::UNAUTHORIZED)); + assert!(!status_is_retryable(StatusCode::FORBIDDEN)); + assert!(!status_is_retryable(StatusCode::BAD_REQUEST)); + } + + /// Smoke test: the wrapper calls into the SDK and builds a usable + /// `MessageDescriptor` via the descriptor pool. + #[test] + fn test_generate_descriptor_simple_schema() { + let schema = UnityCatalogTableSchema { + name: "test_table".to_string(), + catalog_name: "test_catalog".to_string(), + schema_name: "test_schema".to_string(), + columns: vec![ + UnityCatalogColumn { + name: "id".to_string(), + type_text: "bigint".to_string(), + type_name: "BIGINT".to_string(), + position: 1, + nullable: false, + type_json: "{}".to_string(), + }, + UnityCatalogColumn { + name: "body".to_string(), + type_text: "string".to_string(), + type_name: "STRING".to_string(), + position: 2, + nullable: true, + type_json: "{}".to_string(), + }, + ], + }; + + let (descriptor, _sdk_proto) = + generate_descriptor_from_schema(&schema).expect("descriptor should be generated"); + assert_eq!(descriptor.fields().len(), 2); + assert!(descriptor.get_field_by_name("id").is_some()); + assert!(descriptor.get_field_by_name("body").is_some()); + } + + /// Snapshot test for the proto-text formatter used in info logging. + /// The UCβ†’proto conversion itself is covered by the SDK's own tests; + /// this guards the local `format_descriptor_as_proto` rendering. + #[test] + fn test_proto_schema_snapshot() { + let json = include_str!("tests/fixtures/nested_structs_complete_schema.json"); + let schema: UnityCatalogTableSchema = + serde_json::from_str(json).expect("Failed to parse nested_structs_complete schema"); + + let (descriptor, _sdk_proto) = + generate_descriptor_from_schema(&schema).expect("Failed to generate descriptor"); + + let proto_text = format_descriptor_as_proto(&descriptor); + + assert!( + proto_text.contains("message TestSchemaNestedStructsTable"), + "Proto should have main message definition" + ); + assert!( + proto_text.contains("string field_003"), + "Proto should have field_003 (string)" + ); + assert!( + proto_text.contains("int64 field_007"), + "Proto should have field_007 (int64)" + ); + assert!( + proto_text.contains("repeated int64 field_027"), + "Proto should have field_027 as repeated int64" + ); + assert!( + proto_text.contains("message Field018"), + "Proto should have Field018 nested message" + ); + assert!( + proto_text.contains("message Field021"), + "Proto should have Field021 nested message" + ); + assert!( + proto_text.contains("message Field008"), + "Proto should have Field008 nested message" + ); + } + + #[test] + fn test_sanitize_package_name_non_ascii() { + // Non-ASCII alphanumeric characters (e.g. accented letters, CJK) are + // valid for `char::is_alphanumeric` but not for protobuf identifiers, + // so they must be replaced with `_`. + assert_eq!(sanitize_package_name("cafΓ©"), "caf_"); + assert_eq!(sanitize_package_name("ζ—₯本.tbl"), "p__.tbl"); + assert_eq!(sanitize_package_name("naΓ―ve.schema"), "na_ve.schema"); + } + + #[test] + fn test_sanitize_package_name_ascii_preserved() { + assert_eq!(sanitize_package_name("main.default_v2"), "main.default_v2"); + assert_eq!(sanitize_package_name("1abc"), "p1abc"); + assert_eq!(sanitize_package_name("_x"), "p_x"); + } +} diff --git a/src/sinks/datadog/events/config.rs b/src/sinks/datadog/events/config.rs index 2bfdc66315c3e..f9b0f9361bba8 100644 --- a/src/sinks/datadog/events/config.rs +++ b/src/sinks/datadog/events/config.rs @@ -14,7 +14,10 @@ use crate::{ sinks::{ Healthcheck, VectorSink, datadog::{DatadogCommonConfig, LocalDatadogCommonConfig}, - util::{ServiceBuilderExt, TowerRequestConfig, http::HttpStatusRetryLogic}, + util::{ + ServiceBuilderExt, TowerRequestConfig, + http::{HttpStatusRetryLogic, RetryStrategy}, + }, }, tls::MaybeTlsSettings, }; @@ -33,6 +36,10 @@ pub struct DatadogEventsConfig { #[configurable(derived)] #[serde(default)] pub request: TowerRequestConfig, + + #[configurable(derived)] + #[serde(default)] + pub retry_strategy: RetryStrategy, } impl GenerateConfig for DatadogEventsConfig { @@ -64,7 +71,10 @@ impl DatadogEventsConfig { let request_opts = self.request; let request_settings = request_opts.into_settings(); - let retry_logic = HttpStatusRetryLogic::new(|req: &DatadogEventsResponse| req.http_status); + let retry_logic = HttpStatusRetryLogic::new( + |req: &DatadogEventsResponse| req.http_status, + self.retry_strategy.clone(), + ); let service = ServiceBuilder::new() .settings(request_settings, retry_logic) diff --git a/src/sinks/datadog/events/tests.rs b/src/sinks/datadog/events/tests.rs index a363307f6c4e7..c800d09b562f9 100644 --- a/src/sinks/datadog/events/tests.rs +++ b/src/sinks/datadog/events/tests.rs @@ -1,11 +1,7 @@ use std::sync::Arc; use bytes::Bytes; -use futures::{ - StreamExt, - channel::mpsc::{Receiver, TryRecvError}, - stream::Stream, -}; +use futures::{StreamExt, channel::mpsc::Receiver, stream::Stream}; use hyper::StatusCode; use indoc::indoc; use similar_asserts::assert_eq; @@ -105,7 +101,7 @@ async fn smoke() { async fn handles_failure() { let (_expected, mut rx) = start_test(StatusCode::FORBIDDEN, BatchStatus::Rejected).await; - assert!(matches!(rx.try_next(), Err(TryRecvError { .. }))); + assert!(rx.try_recv().is_err()); } #[tokio::test] diff --git a/src/sinks/datadog/logs/sink.rs b/src/sinks/datadog/logs/sink.rs index 862d6973aea5f..5fb5e2189fa10 100644 --- a/src/sinks/datadog/logs/sink.rs +++ b/src/sinks/datadog/logs/sink.rs @@ -2,6 +2,7 @@ use std::{collections::VecDeque, fmt::Debug, io, sync::Arc}; use itertools::Itertools; use snafu::Snafu; +use tracing::Instrument; use vector_lib::{ event::{ObjectMap, Value}, internal_event::{ComponentEventsDropped, UNINTENTIONAL}, @@ -386,19 +387,26 @@ where conforms_as_agent: self.conforms_as_agent, }); - let input = input.batched_partitioned(partitioner, || { + let input = input.batched_partitioned(partitioner, batch_settings.timeout, |_| { batch_settings.as_item_size_config(HttpJsonBatchSizer) }); input .concurrent_map(default_request_builder_concurrency_limit(), move |input| { let builder = Arc::clone(&builder); - Box::pin(async move { - let (api_key, events) = input; - let api_key = api_key.unwrap_or_else(|| Arc::clone(&builder.default_api_key)); + // `concurrent_map` spawns this future on a detached task. The closure itself runs + // within `run_inner`'s span, so `in_current_span` captures the sink span here and + // re-enters it on the spawned task to preserve the sink's automatic component tags. + Box::pin( + async move { + let (api_key, events) = input; + let api_key = + api_key.unwrap_or_else(|| Arc::clone(&builder.default_api_key)); - builder.build_request(events, api_key) - }) + builder.build_request(events, api_key) + } + .in_current_span(), + ) }) .filter_map(|request| async move { match request { diff --git a/src/sinks/datadog/logs/tests.rs b/src/sinks/datadog/logs/tests.rs index 3867ababb48c5..d394a71e0b3b6 100644 --- a/src/sinks/datadog/logs/tests.rs +++ b/src/sinks/datadog/logs/tests.rs @@ -4,10 +4,7 @@ use std::sync::Arc; use bytes::Bytes; use chrono::Utc; -use futures::{ - StreamExt, - channel::mpsc::{Receiver, TryRecvError}, -}; +use futures::{StreamExt, channel::mpsc::Receiver}; use http::request::Parts; use indoc::indoc; use vector_lib::{ @@ -195,9 +192,9 @@ async fn telemetry() { async fn handles_failure_v1() { let (_expected, mut rx) = start_test_error(ApiStatus::BadRequestv1, BatchStatus::Rejected).await; - let res = rx.try_next(); + let res = rx.try_recv(); - assert!(matches!(res, Err(TryRecvError { .. }))); + assert!(res.is_err()); } #[tokio::test] @@ -209,9 +206,9 @@ async fn handles_failure_v1() { async fn handles_failure_v2() { let (_expected, mut rx) = start_test_error(ApiStatus::BadRequestv2, BatchStatus::Rejected).await; - let res = rx.try_next(); + let res = rx.try_recv(); - assert!(matches!(res, Err(TryRecvError { .. }))); + assert!(res.is_err()); } #[tokio::test] diff --git a/src/sinks/datadog/metrics/config.rs b/src/sinks/datadog/metrics/config.rs index 6d004e694e0ea..a5fdccede6a14 100644 --- a/src/sinks/datadog/metrics/config.rs +++ b/src/sinks/datadog/metrics/config.rs @@ -1,9 +1,9 @@ -use std::sync::OnceLock; - use http::Uri; use snafu::ResultExt; use tower::ServiceBuilder; -use vector_lib::{config::proxy::ProxyConfig, configurable::configurable_component}; +use vector_lib::{ + config::proxy::ProxyConfig, configurable::configurable_component, stream::BatcherSettings, +}; use super::{ request_builder::DatadogMetricsRequestBuilder, @@ -21,18 +21,13 @@ use crate::{ }, tls::{MaybeTlsSettings, TlsEnableableConfig}, }; - #[derive(Clone, Copy, Debug, Default)] pub struct DatadogMetricsDefaultBatchSettings; -// This default is centered around "series" data, which should be the lion's share of what we -// process. Given that a single series, when encoded, is in the 150-300 byte range, we can fit a -// lot of these into a single request, something like 150-200K series. Simply to be a little more -// conservative, though, we use 100K here. This will also get a little more tricky when it comes to -// distributions and sketches, but we're going to have to implement incremental encoding to handle -// "we've exceeded our maximum payload size, split this batch" scenarios anyways. impl SinkBatchSettings for DatadogMetricsDefaultBatchSettings { const MAX_EVENTS: Option = Some(100_000); + // No default byte cap here; the appropriate limit (v1: 60 MiB, v2: 5 MiB) is applied at + // sink build time based on the active series API version. const MAX_BYTES: Option = None; const TIMEOUT_SECS: f64 = 2.0; } @@ -41,9 +36,21 @@ pub(super) const SERIES_V1_PATH: &str = "/api/v1/series"; pub(super) const SERIES_V2_PATH: &str = "/api/v2/series"; pub(super) const SKETCHES_PATH: &str = "/api/beta/sketches"; -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +/// The API version to use when submitting series metrics to Datadog. +#[configurable_component] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +#[serde(rename_all = "snake_case")] pub enum SeriesApiVersion { + /// Use the v1 series endpoint (`/api/v1/series`). + /// + /// This is a legacy endpoint. Prefer `v2` unless you have a specific reason to use v1. + #[configurable(deprecated)] V1, + + /// Use the v2 series endpoint (`/api/v2/series`). + /// + /// This is the recommended and default endpoint. + #[default] V2, } @@ -54,15 +61,6 @@ impl SeriesApiVersion { Self::V2 => SERIES_V2_PATH, } } - fn get_api_version() -> Self { - static API_VERSION: OnceLock = OnceLock::new(); - *API_VERSION.get_or_init(|| { - match std::env::var("VECTOR_TEMP_USE_DD_METRICS_SERIES_V2_API") { - Ok(_) => Self::V2, - Err(_) => Self::V1, - } - }) - } } /// Various metric type-specific API types. @@ -89,19 +87,8 @@ impl DatadogMetricsEndpoint { } } - // Gets whether or not this is a series endpoint. - pub const fn is_series(self) -> bool { - matches!(self, Self::Series { .. }) - } - - // Creates an instance of the `Series` variant with the default API version. - pub fn series() -> Self { - Self::Series(SeriesApiVersion::get_api_version()) - } - pub(super) const fn payload_limits(self) -> DatadogMetricsPayloadLimits { // from https://docs.datadoghq.com/api/latest/metrics/#submit-metrics - let (uncompressed, compressed) = match self { // Sketches use the same payload size limits as v1 series DatadogMetricsEndpoint::Series(SeriesApiVersion::V1) @@ -120,6 +107,32 @@ impl DatadogMetricsEndpoint { compressed, } } + + /// Returns the compression scheme used for this endpoint. + pub(super) const fn compression(self) -> DatadogMetricsCompression { + match self { + Self::Series(SeriesApiVersion::V1) => DatadogMetricsCompression::Zlib, + _ => DatadogMetricsCompression::Zstd, + } + } +} + +/// Selects the compressor for a given Datadog metrics endpoint. +#[derive(Clone, Copy, Debug)] +pub(super) enum DatadogMetricsCompression { + /// zlib (deflate) β€” used by Series v1. + Zlib, + /// zstd β€” used by Series v2 and Sketches. + Zstd, +} + +impl DatadogMetricsCompression { + pub(super) const fn content_encoding(self) -> &'static str { + match self { + Self::Zstd => "zstd", + Self::Zlib => "deflate", + } + } } /// Maps Datadog metric endpoints to their actual URI. @@ -162,6 +175,13 @@ pub struct DatadogMetricsConfig { #[serde(default)] pub default_namespace: Option, + /// Controls which Datadog series API endpoint is used to submit metrics. + /// + /// Defaults to `v2` (`/api/v2/series`). Set to `v1` (`/api/v1/series`) only if you need to + /// fall back to the legacy endpoint. + #[serde(default)] + pub series_api_version: SeriesApiVersion, + #[configurable(derived)] #[serde(default)] pub batch: BatchConfig, @@ -222,7 +242,7 @@ impl DatadogMetricsConfig { ) -> crate::Result { let base_uri = self.get_base_agent_endpoint(dd_common); - let series_endpoint = build_uri(&base_uri, SeriesApiVersion::get_api_version().get_path())?; + let series_endpoint = build_uri(&base_uri, self.series_api_version.get_path())?; let sketches_endpoint = build_uri(&base_uri, SKETCHES_PATH)?; Ok(DatadogMetricsEndpointConfiguration::new( @@ -253,7 +273,8 @@ impl DatadogMetricsConfig { dd_common: &DatadogCommonConfig, client: HttpClient, ) -> crate::Result { - let batcher_settings = self.batch.into_batcher_settings()?; + let (batcher_settings, sketches_batcher_settings) = + resolve_endpoint_batch_settings(self.batch, self.series_api_version)?; // TODO: revisit our concurrency and batching defaults let request_limits = self.request.into_settings(); @@ -269,10 +290,18 @@ impl DatadogMetricsConfig { let request_builder = DatadogMetricsRequestBuilder::new( endpoint_configuration, self.default_namespace.clone(), - )?; + self.series_api_version, + ); let protocol = self.get_protocol(dd_common); - let sink = DatadogMetricsSink::new(service, request_builder, batcher_settings, protocol); + let sink = DatadogMetricsSink::new( + service, + request_builder, + batcher_settings, + sketches_batcher_settings, + protocol, + self.series_api_version, + ); Ok(VectorSink::from_event_streamsink(sink)) } @@ -287,6 +316,28 @@ impl DatadogMetricsConfig { } } +/// Returns `(series_settings, sketches_settings)`. +/// +/// When the user has not set an explicit `max_bytes`, each endpoint is capped to its own +/// uncompressed payload limit (5 MiB for Series v2, 60 MiB for Sketches). When an explicit +/// limit is configured, both endpoints share it. +fn resolve_endpoint_batch_settings( + batch: BatchConfig, + series_version: SeriesApiVersion, +) -> crate::Result<(BatcherSettings, BatcherSettings)> { + let mut series = batch.into_batcher_settings()?; + let mut sketches = series; + if series.size_limit == usize::MAX { + series.size_limit = DatadogMetricsEndpoint::Series(series_version) + .payload_limits() + .uncompressed; + sketches.size_limit = DatadogMetricsEndpoint::Sketches + .payload_limits() + .uncompressed; + } + Ok((series, sketches)) +} + fn build_uri(host: &str, endpoint: &str) -> crate::Result { let result = format!("{host}{endpoint}") .parse::() @@ -302,4 +353,36 @@ mod tests { fn generate_config() { crate::test_util::test_generate_config::(); } + + // When max_bytes is unset, each endpoint gets its own API payload limit. + #[test] + fn default_batch_config_uses_endpoint_specific_size_limits() { + let (series, sketches) = + resolve_endpoint_batch_settings(BatchConfig::default(), SeriesApiVersion::V2).unwrap(); + + assert_eq!(series.size_limit, 5_242_880); // 5 MiB β€” Series v2 limit + assert_eq!(sketches.size_limit, 62_914_560); // 60 MiB β€” Sketches limit + } + + #[test] + fn v1_batch_config_uses_v1_size_limit() { + let (series, sketches) = + resolve_endpoint_batch_settings(BatchConfig::default(), SeriesApiVersion::V1).unwrap(); + + assert_eq!(series.size_limit, 62_914_560); // 60 MiB β€” Series v1 limit + assert_eq!(sketches.size_limit, 62_914_560); // 60 MiB β€” Sketches limit + } + + // When the user sets max_bytes, both endpoints share that limit unchanged. + #[test] + fn explicit_max_bytes_applies_to_both_endpoints() { + let mut config = BatchConfig::::default(); + config.max_bytes = Some(1_000_000); + + let (series, sketches) = + resolve_endpoint_batch_settings(config, SeriesApiVersion::V2).unwrap(); + + assert_eq!(series.size_limit, 1_000_000); + assert_eq!(sketches.size_limit, 1_000_000); + } } diff --git a/src/sinks/datadog/metrics/encoder.rs b/src/sinks/datadog/metrics/encoder.rs index 617e69c1e5e6b..a626244123861 100644 --- a/src/sinks/datadog/metrics/encoder.rs +++ b/src/sinks/datadog/metrics/encoder.rs @@ -16,7 +16,12 @@ use vector_lib::{ request_metadata::GroupedCountByteSize, }; -use super::config::{DatadogMetricsEndpoint, SeriesApiVersion}; +use vector_common::constants::{ + ZLIB_FRAME_OVERHEAD, ZLIB_STORED_BLOCK_OVERHEAD, ZLIB_STORED_BLOCK_SIZE, + ZSTD_SMALL_INPUT_THRESHOLD, +}; + +use super::config::{DatadogMetricsCompression, DatadogMetricsEndpoint, SeriesApiVersion}; use crate::{ common::datadog::{ DatadogMetricType, DatadogPoint, DatadogSeriesMetric, DatadogSeriesMetricMetadata, @@ -47,23 +52,6 @@ mod ddmetric_proto { include!(concat!(env!("OUT_DIR"), "/datadog.agentpayload.rs")); } -#[derive(Debug, Snafu)] -pub enum CreateError { - #[snafu(display("Invalid compressed/uncompressed payload size limits were given"))] - InvalidLimits, -} - -impl CreateError { - /// Gets the telemetry-friendly string version of this error. - /// - /// The value will be a short string with only lowercase letters and underscores. - pub const fn as_error_type(&self) -> &'static str { - match self { - Self::InvalidLimits => "invalid_payload_limits", - } - } -} - #[derive(Debug, Snafu)] pub enum EncoderError { #[snafu(display( @@ -133,6 +121,16 @@ impl FinishError { struct EncoderState { writer: Compressor, written: usize, + /// Upper bound on uncompressed bytes sitting in the compressor's internal buffer (written but + /// not yet flushed to `writer.get_ref()`). All compressors may buffer internally: zstd holds + /// up to 128 KB per block, zlib's BufWriter holds up to 4 KB. Since `get_ref().len()` only + /// reflects bytes that have been flushed through all layers, we track this bound to avoid + /// underestimating the compressed payload size. + /// + /// Increases by `n` on each write. Resets to `n` when a new compressed block is detected in + /// `writer.get_ref()` (the triggering write may straddle the block boundary, so `n` is a safe + /// upper bound on what remains buffered after the flush). + buffered_bound: usize, buf: Vec, processed: Vec, byte_size: GroupedCountByteSize, @@ -141,8 +139,9 @@ struct EncoderState { impl Default for EncoderState { fn default() -> Self { Self { - writer: get_compressor(), + writer: Compression::zlib_default().into(), written: 0, + buffered_bound: 0, buf: Vec::with_capacity(1024), processed: Vec::new(), byte_size: telemetry().create_request_count_byte_size(), @@ -164,45 +163,62 @@ pub struct DatadogMetricsEncoder { impl DatadogMetricsEncoder { /// Creates a new `DatadogMetricsEncoder` for the given endpoint. - pub fn new( - endpoint: DatadogMetricsEndpoint, - default_namespace: Option, - ) -> Result { + pub fn new(endpoint: DatadogMetricsEndpoint, default_namespace: Option) -> Self { let payload_limits = endpoint.payload_limits(); - Self::with_payload_limits( + + Self { endpoint, - default_namespace, - payload_limits.uncompressed, - payload_limits.compressed, - ) + default_namespace: default_namespace.map(Arc::from), + uncompressed_limit: payload_limits.uncompressed, + compressed_limit: payload_limits.compressed, + state: EncoderState { + writer: endpoint.compression().compressor(), + ..Default::default() + }, + log_schema: log_schema(), + origin_product_value: *ORIGIN_PRODUCT_VALUE, + } } +} +#[cfg(test)] +impl DatadogMetricsEncoder { /// Creates a new `DatadogMetricsEncoder` for the given endpoint, with specific payload limits. + /// + /// Only available in tests; production code always uses the API-defined limits via `new`. pub fn with_payload_limits( endpoint: DatadogMetricsEndpoint, default_namespace: Option, uncompressed_limit: usize, compressed_limit: usize, - ) -> Result { - let (uncompressed_limit, compressed_limit) = - validate_payload_size_limits(endpoint, uncompressed_limit, compressed_limit) - .ok_or(CreateError::InvalidLimits)?; - - Ok(Self { + ) -> Self { + Self { endpoint, default_namespace: default_namespace.map(Arc::from), uncompressed_limit, compressed_limit, - state: EncoderState::default(), + state: EncoderState { + writer: endpoint.compression().compressor(), + ..Default::default() + }, log_schema: log_schema(), origin_product_value: *ORIGIN_PRODUCT_VALUE, - }) + } + } + + /// Returns the current `buffered_bound` value for white-box testing of zstd block-flush reset. + fn buffered_bound(&self) -> usize { + self.state.buffered_bound } } impl DatadogMetricsEncoder { fn reset_state(&mut self) -> EncoderState { - mem::take(&mut self.state) + let new_state = EncoderState { + writer: self.endpoint.compression().compressor(), + ..Default::default() + }; + mem::replace(&mut self.state, new_state) } fn encode_single_metric(&mut self, metric: Metric) -> Result, EncoderError> { @@ -342,24 +358,36 @@ impl DatadogMetricsEncoder { // compressor might have internal state around checksums, etc, that can't be similarly // rolled back. // - // Our strategy is thus: simply take the encoded-but-decompressed size and see if it would - // fit within the compressed limit. In `get_endpoint_payload_size_limits`, we calculate the - // expected maximum overhead of zlib based on all input data being incompressible, which - // zlib ensures will be the worst case as it can figure out whether a compressed or - // uncompressed block would take up more space _before_ choosing which strategy to go with. + // Strategy: split the estimate into two parts: + // 1. Bytes already flushed to the output buffer (`get_ref().len()`) β€” exact compressed size. + // 2. Bytes still in the compressor's internal buffer plus this new metric β€” estimated via + // max_compressed_size(buffered_bound + n) (worst-case upper bound). // - // Thus, simply put, we've already accounted for the uncertainty by making our check here - // assume the worst case while our limits assume the worst case _overhead_. Maybe our - // numbers are technically off in the end, but `finish` catches that for us, too. - let compressed_len = self.state.writer.get_ref().len(); - let max_compressed_metric_len = n + max_compressed_overhead_len(n); - if compressed_len + max_compressed_metric_len > self.compressed_limit { + // All compressors may buffer data internally before flushing to the output: zstd buffers + // up to 128 KB per block, zlib's BufWriter holds up to 4 KB. `get_ref().len()` only + // reflects bytes that have been flushed through all layers. We track `buffered_bound` β€” + // an upper bound on uncompressed bytes written but not yet visible in `get_ref()` β€” and + // include it in the estimate for all compressor types. + let compression = self.endpoint.compression(); + let flushed_compressed = self.state.writer.get_ref().len(); + if flushed_compressed + compression.max_compressed_size(self.state.buffered_bound + n) + > self.compressed_limit + { return Ok(false); } // We should be safe to write our holding buffer to the compressor and store the metric. + // + // Update buffered_bound: if a new block appeared in the output (flushed_compressed grew), + // reset to n β€” the triggering write may straddle the block boundary, so n is a safe upper + // bound on what remains buffered. Otherwise accumulate. self.state.writer.write_all(&self.state.buf)?; self.state.written += n; + if self.state.writer.get_ref().len() > flushed_compressed { + self.state.buffered_bound = n; + } else { + self.state.buffered_bound += n; + } Ok(true) } @@ -383,7 +411,10 @@ impl DatadogMetricsEncoder { // Make sure we've written our header already. if self.state.written == 0 { match write_payload_header(self.endpoint, &mut self.state.writer) { - Ok(n) => self.state.written += n, + Ok(n) => { + self.state.written += n; + self.state.buffered_bound += n; + } Err(_) => return Ok(Some(metric)), } } @@ -874,75 +905,41 @@ fn generate_series_metrics( }]) } -fn get_compressor() -> Compressor { - // We use the "zlib default" compressor because it's all Datadog supports, and adding it - // generically to `Compression` would make things a little weird because of the conversion trait - // implementations that are also only none vs gzip. - Compression::zlib_default().into() -} - -const fn max_uncompressed_header_len() -> usize { - SERIES_PAYLOAD_HEADER.len() + SERIES_PAYLOAD_FOOTER.len() -} - -// Datadog ingest APIs accept zlib, which is what we're accounting for here. By default, zlib -// has a 2 byte header and 4 byte CRC trailer. [1] -// -// [1] https://www.zlib.net/zlib_tech.html -const ZLIB_HEADER_TRAILER: usize = 6; - -const fn max_compression_overhead_len(compressed_limit: usize) -> usize { - // We calculate the overhead as the zlib header/trailer plus the worst case overhead of - // compressing `compressed_limit` bytes, such that we assume all of the data we write may not be - // compressed at all. - ZLIB_HEADER_TRAILER + max_compressed_overhead_len(compressed_limit) -} - -const fn max_compressed_overhead_len(len: usize) -> usize { - // Datadog ingest APIs accept zlib, which is what we're accounting for here. - // - // Deflate, the underlying compression algorithm, has a technique to ensure that input data - // can't be encoded in such a way where it's expanded by a meaningful amount. This technique - // allows storing blocks of uncompressed data with only 5 bytes of overhead per block. - // Technically, the blocks can be up to 65KB in Deflate, but modern zlib implementations use - // block sizes of 16KB. [1][2] - // - // We calculate the overhead of compressing a given `len` bytes as the worst case of that many - // bytes being written to the compressor and being unable to be compressed at all - // - // [1] https://www.zlib.net/zlib_tech.html - // [2] https://www.bolet.org/~pornin/deflate-flush-fr.html - const STORED_BLOCK_SIZE: usize = 16384; - (1 + len.saturating_sub(ZLIB_HEADER_TRAILER) / STORED_BLOCK_SIZE) * 5 -} - -const fn validate_payload_size_limits( - endpoint: DatadogMetricsEndpoint, - uncompressed_limit: usize, - compressed_limit: usize, -) -> Option<(usize, usize)> { - if endpoint.is_series() { - // For series, we need to make sure the uncompressed limit can account for the header/footer - // we would add that wraps the encoded metrics up in the expected JSON object. This does - // imply that adding 1 to this limit would be allowed, and obviously we can't encode a - // series metric in a single byte, but this is just a simple sanity check, not an exhaustive - // search of the absolute bare minimum size. - let header_len = max_uncompressed_header_len(); - if uncompressed_limit <= header_len { - return None; +impl DatadogMetricsCompression { + fn compressor(self) -> Compressor { + match self { + Self::Zstd => Compression::zstd_default().into(), + Self::Zlib => Compression::zlib_default().into(), } } - // Get the maximum possible overhead of the compression container, based on the incoming - // _uncompressed_ limit. We use the uncompressed limit because we're calculating the maximum - // overhead in the case that, theoretically, none of the input data was compressible. This - // possibility is essentially impossible, but serves as a proper worst-case-scenario check. - let max_compression_overhead = max_compression_overhead_len(uncompressed_limit); - if compressed_limit <= max_compression_overhead { - return None; + /// Returns the worst-case compressed size of `n` uncompressed bytes. + /// + /// For zlib (deflate), the worst case occurs when data is entirely incompressible and stored in + /// uncompressed blocks (5 bytes overhead per 16 KB block, as per the DEFLATE spec). + /// + /// For zstd, this uses the same formula as `ZSTD_compressBound` from the zstd C library. + const fn max_compressed_size(self, n: usize) -> usize { + match self { + Self::Zlib => { + // Deflate stores incompressible data in uncompressed blocks, each with fixed + // overhead. We subtract the zlib frame from the block count since those bytes + // are not stored-block data. + n + (1 + n.saturating_sub(ZLIB_FRAME_OVERHEAD) / ZLIB_STORED_BLOCK_SIZE) + * ZLIB_STORED_BLOCK_OVERHEAD + } + Self::Zstd => { + // zstd_safe::compress_bound is not const, so we use the same formula it uses + // internally: srcSize + (srcSize >> 8) + small correction for inputs < 128 KB. + n + (n >> 8) + + if n < ZSTD_SMALL_INPUT_THRESHOLD { + (ZSTD_SMALL_INPUT_THRESHOLD - n) >> 11 + } else { + 0 + } + } + } } - - Some((uncompressed_limit, compressed_limit)) } fn write_payload_header( @@ -983,11 +980,8 @@ fn write_payload_footer( #[cfg(test)] mod tests { - use std::{ - io::{self, copy}, - num::NonZeroU32, - sync::Arc, - }; + use std::io::{self, Write as _}; + use std::{num::NonZeroU32, sync::Arc}; use bytes::{BufMut, Bytes, BytesMut}; use chrono::{DateTime, TimeZone, Timelike, Utc}; @@ -1010,19 +1004,30 @@ mod tests { use super::{ DatadogMetricsEncoder, EncoderError, ddmetric_proto, encode_proto_key_and_message, - encode_tags, encode_timestamp, generate_series_metrics, get_compressor, - get_sketch_payload_sketches_field_number, max_compression_overhead_len, - max_uncompressed_header_len, series_to_proto_message, sketch_to_proto_message, - validate_payload_size_limits, write_payload_footer, write_payload_header, + encode_tags, encode_timestamp, generate_series_metrics, + get_sketch_payload_sketches_field_number, series_to_proto_message, sketch_to_proto_message, + write_payload_footer, write_payload_header, }; use crate::{ common::datadog::DatadogMetricType, - sinks::datadog::metrics::{ - config::{DatadogMetricsEndpoint, SeriesApiVersion}, - encoder::{DEFAULT_DD_ORIGIN_PRODUCT_VALUE, ORIGIN_PRODUCT_VALUE}, + sinks::{ + datadog::metrics::{ + config::{DatadogMetricsCompression, DatadogMetricsEndpoint, SeriesApiVersion}, + encoder::{DEFAULT_DD_ORIGIN_PRODUCT_VALUE, ORIGIN_PRODUCT_VALUE}, + }, + util::{Compression, Compressor}, }, }; + const fn max_uncompressed_header_len(endpoint: DatadogMetricsEndpoint) -> usize { + match endpoint { + DatadogMetricsEndpoint::Series(SeriesApiVersion::V1) => { + super::SERIES_PAYLOAD_HEADER.len() + super::SERIES_PAYLOAD_FOOTER.len() + } + _ => 0, + } + } + fn get_simple_counter() -> Metric { let value = MetricValue::Counter { value: 3.14 }; Metric::new("basic_counter", MetricKind::Incremental, value).with_timestamp(Some(ts())) @@ -1048,8 +1053,8 @@ mod tests { .with_timestamp(Some(ts())) } - fn get_compressed_empty_series_payload() -> Bytes { - let mut compressor = get_compressor(); + fn get_compressed_empty_series_v1_payload() -> Bytes { + let mut compressor = Compressor::from(Compression::zlib_default()); _ = write_payload_header( DatadogMetricsEndpoint::Series(SeriesApiVersion::V1), @@ -1066,14 +1071,104 @@ mod tests { } fn get_compressed_empty_sketches_payload() -> Bytes { - get_compressor().finish().expect("should not fail").freeze() + Compressor::from(Compression::zstd_default()) + .finish() + .expect("should not fail") + .freeze() } - fn decompress_payload(payload: Bytes) -> io::Result { + fn get_compressed_empty_series_v2_payload() -> Bytes { + Compressor::from(Compression::zstd_default()) + .finish() + .expect("should not fail") + .freeze() + } + + fn decompress_zlib_payload(payload: Bytes) -> io::Result { let mut decompressor = ZlibDecoder::new(&payload[..]); let mut decompressed = BytesMut::new().writer(); - let result = copy(&mut decompressor, &mut decompressed); - result.map(|_| decompressed.into_inner().freeze()) + io::copy(&mut decompressor, &mut decompressed)?; + Ok(decompressed.into_inner().freeze()) + } + + fn decompress_zstd_payload(payload: Bytes) -> io::Result { + let decompressed = zstd::decode_all(&payload[..])?; + Ok(Bytes::from(decompressed)) + } + + /// Returns the number of bytes added to the compressor's output buffer after writing `n` + /// bytes of high-entropy data. Measures only the *incremental* bytes, not the frame overhead + /// that `finish()` would append (Adler-32 / empty final block for zlib, end frame for zstd). + /// + /// This mirrors how `try_compress_buffer` uses `max_compressed_size`: it checks how many + /// more compressed bytes would be produced, against the current running output length. + /// Compresses `n` bytes of high-entropy (worst-case for compression) data and returns the + /// total output size after `finish()`. + fn total_compressed_len(compression: DatadogMetricsCompression, n: usize) -> usize { + // Xorshift64 β€” period 2^64-1, passes BigCrush, produces statistically random bytes + // that neither zlib nor zstd can compress significantly. + let mut state = 0xdeadbeef_cafebabe_u64; + let data: Vec = (0..n) + .map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state as u8 + }) + .collect(); + let mut compressor = compression.compressor(); + compressor.write_all(&data).expect("write should succeed"); + compressor.finish().expect("finish should succeed").len() + } + + /// Validates that `max_compressed_size(n)` is a true upper bound on the compressed bytes + /// attributable to `n` uncompressed bytes, for both zlib and zstd. + /// + /// We measure `total_compressed_len(n) - total_compressed_len(0)` to strip the fixed frame + /// overhead (header + trailer) written regardless of input size, isolating the bytes + /// contributed by the data itself. + #[test] + fn max_compressed_size_is_upper_bound() { + // zlib stored-block boundary: 16 384 bytes; zstd block boundary: 131 072 bytes. + let test_sizes = [ + 0, 1, 100, 1_000, 16_383, 16_384, 16_385, 32_767, 32_768, 131_071, 131_072, 131_073, + 500_000, + ]; + + let zlib_frame = total_compressed_len(DatadogMetricsCompression::Zlib, 0); + let zstd_frame = total_compressed_len(DatadogMetricsCompression::Zstd, 0); + + // The formula must not overestimate by more than 1% of input + 64 bytes (a small + // constant that covers the zstd correction term for very small inputs). + let max_slack = |n: usize| n / 100 + 64; + + for &n in &test_sizes { + let actual_zlib = total_compressed_len(DatadogMetricsCompression::Zlib, n) - zlib_frame; + let max_zlib = DatadogMetricsCompression::Zlib.max_compressed_size(n); + assert!( + actual_zlib <= max_zlib, + "zlib n={n}: formula underestimates: actual={actual_zlib} > max={max_zlib}" + ); + assert!( + max_zlib - actual_zlib <= max_slack(n), + "zlib n={n}: formula overestimates: slack={} > {}", + max_zlib - actual_zlib, + max_slack(n) + ); + + let actual_zstd = total_compressed_len(DatadogMetricsCompression::Zstd, n) - zstd_frame; + let max_zstd = DatadogMetricsCompression::Zstd.max_compressed_size(n); + assert!( + actual_zstd <= max_zstd, + "zstd n={n}: formula underestimates: actual={actual_zstd} > max={max_zstd}" + ); + assert!( + max_zstd - actual_zstd <= max_slack(n), + "zstd n={n}: formula overestimates: slack={} > {}", + max_zstd - actual_zstd, + max_slack(n) + ); + } } fn ts() -> DateTime { @@ -1149,8 +1244,7 @@ mod tests { #[test] fn incorrect_metric_for_endpoint_causes_error() { // Series metrics can't go to the sketches endpoint. - let mut sketch_encoder = DatadogMetricsEncoder::new(DatadogMetricsEndpoint::Sketches, None) - .expect("default payload size limits should be valid"); + let mut sketch_encoder = DatadogMetricsEncoder::new(DatadogMetricsEndpoint::Sketches, None); let series_result = sketch_encoder.try_encode(get_simple_counter()); assert!(matches!( series_result.err(), @@ -1159,8 +1253,7 @@ mod tests { // And sketches can't go to the series endpoint. let mut series_v1_encoder = - DatadogMetricsEncoder::new(DatadogMetricsEndpoint::Series(SeriesApiVersion::V1), None) - .expect("default payload size limits should be valid"); + DatadogMetricsEncoder::new(DatadogMetricsEndpoint::Series(SeriesApiVersion::V1), None); let sketch_result = series_v1_encoder.try_encode(get_simple_sketch()); assert!(matches!( sketch_result.err(), @@ -1168,8 +1261,7 @@ mod tests { )); let mut series_v2_encoder = - DatadogMetricsEncoder::new(DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), None) - .expect("default payload size limits should be valid"); + DatadogMetricsEncoder::new(DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), None); let sketch_result = series_v2_encoder.try_encode(get_simple_sketch()); assert!(matches!( sketch_result.err(), @@ -1380,8 +1472,7 @@ mod tests { // This is a simple test where we ensure that a single metric, with the default limits, can // be encoded without hitting any errors. let mut encoder = - DatadogMetricsEncoder::new(DatadogMetricsEndpoint::Series(SeriesApiVersion::V1), None) - .expect("default payload size limits should be valid"); + DatadogMetricsEncoder::new(DatadogMetricsEndpoint::Series(SeriesApiVersion::V1), None); let counter = get_simple_counter(); let expected = counter.clone(); @@ -1404,8 +1495,7 @@ mod tests { // This is a simple test where we ensure that a single metric, with the default limits, can // be encoded without hitting any errors. let mut encoder = - DatadogMetricsEncoder::new(DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), None) - .expect("default payload size limits should be valid"); + DatadogMetricsEncoder::new(DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), None); let counter = get_simple_counter(); let expected = counter.clone(); @@ -1427,8 +1517,7 @@ mod tests { fn encode_single_sketch_metric_with_default_limits() { // This is a simple test where we ensure that a single metric, with the default limits, can // be encoded without hitting any errors. - let mut encoder = DatadogMetricsEncoder::new(DatadogMetricsEndpoint::Sketches, None) - .expect("default payload size limits should be valid"); + let mut encoder = DatadogMetricsEncoder::new(DatadogMetricsEndpoint::Sketches, None); let sketch = get_simple_sketch(); let expected = sketch.clone(); @@ -1450,8 +1539,7 @@ mod tests { fn encode_empty_sketch() { // This is a simple test where we ensure that a single metric, with the default limits, can // be encoded without hitting any errors. - let mut encoder = DatadogMetricsEncoder::new(DatadogMetricsEndpoint::Sketches, None) - .expect("default payload size limits should be valid"); + let mut encoder = DatadogMetricsEncoder::new(DatadogMetricsEndpoint::Sketches, None); let sketch = Metric::new( "empty", MetricKind::Incremental, @@ -1512,74 +1600,73 @@ mod tests { } #[test] - fn payload_size_limits_series() { - // Get the maximum length of the header/trailer data. - let header_len = max_uncompressed_header_len(); + fn default_payload_limits_are_endpoint_aware() { + let v1 = DatadogMetricsEndpoint::Series(SeriesApiVersion::V1).payload_limits(); + assert_eq!(v1.uncompressed, 62_914_560); + assert_eq!(v1.compressed, 3_200_000); + + let v2 = DatadogMetricsEndpoint::Series(SeriesApiVersion::V2).payload_limits(); + assert_eq!(v2.uncompressed, 5_242_880); + assert_eq!(v2.compressed, 512_000); + + let sketches = DatadogMetricsEndpoint::Sketches.payload_limits(); + assert_eq!(sketches.uncompressed, 62_914_560); + assert_eq!(sketches.compressed, 3_200_000); + } - // This is too small. - let result = validate_payload_size_limits( - DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), - header_len, - usize::MAX, - ); - assert_eq!(result, None); + #[test] + fn v2_series_default_limits_split_large_batches() { + // Simulate a large send and validate that default V2 limits split payloads into multiple + // requests, while still making forward progress each pass. + let mut pending = vec![get_simple_counter(); 120_000]; + let mut encoded_batches = 0; + let mut encoded_metrics = 0; + + while !pending.is_empty() { + let mut encoder = DatadogMetricsEncoder::new( + DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), + None, + ); - // This is just right. - let result = validate_payload_size_limits( - DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), - header_len + 1, - usize::MAX, - ); - assert_eq!(result, Some((header_len + 1, usize::MAX))); + let mut next_pending = Vec::new(); + let mut hit_limit = false; + for metric in pending.drain(..) { + match encoder.try_encode(metric.clone()) { + Ok(None) => {} + Ok(Some(returned_metric)) => { + hit_limit = true; + next_pending.push(returned_metric); + } + Err(error) => panic!("unexpected encoding error: {error}"), + } + } - // Get the maximum compressed overhead length, based on our input uncompressed size. This - // represents the worst case overhead based on the input data (of length usize::MAX, in this - // case) being entirely incompressible. - let compression_overhead_len = max_compression_overhead_len(usize::MAX); + let finish_result = encoder.finish(); + assert!(finish_result.is_ok()); + let (_payload, processed) = finish_result.unwrap(); + assert!( + !processed.is_empty(), + "encoder should always make progress for a non-empty batch" + ); - // This is too small. - let result = validate_payload_size_limits( - DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), - usize::MAX, - compression_overhead_len, - ); - assert_eq!(result, None); + encoded_metrics += processed.len(); + encoded_batches += 1; - // This is just right. - let result = validate_payload_size_limits( - DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), - usize::MAX, - compression_overhead_len + 1, - ); - assert_eq!(result, Some((usize::MAX, compression_overhead_len + 1))); - } + if hit_limit { + assert!( + !next_pending.is_empty(), + "hitting limits should leave metrics to process in the next batch" + ); + } - #[test] - fn payload_size_limits_sketches() { - // There's no lower bound on uncompressed size for the sketches payload. - let result = validate_payload_size_limits(DatadogMetricsEndpoint::Sketches, 0, usize::MAX); - assert_eq!(result, Some((0, usize::MAX))); - - // Get the maximum compressed overhead length, based on our input uncompressed size. This - // represents the worst case overhead based on the input data (of length usize::MAX, in this - // case) being entirely incompressible. - let compression_overhead_len = max_compression_overhead_len(usize::MAX); - - // This is too small. - let result = validate_payload_size_limits( - DatadogMetricsEndpoint::Sketches, - usize::MAX, - compression_overhead_len, - ); - assert_eq!(result, None); + pending = next_pending; + } - // This is just right. - let result = validate_payload_size_limits( - DatadogMetricsEndpoint::Sketches, - usize::MAX, - compression_overhead_len + 1, + assert_eq!(encoded_metrics, 120_000); + assert!( + encoded_batches > 1, + "expected multiple batches for V2 default limits" ); - assert_eq!(result, Some((usize::MAX, compression_overhead_len + 1))); } #[test] @@ -1587,14 +1674,14 @@ mod tests { // We manually create the encoder with an arbitrarily low "uncompressed" limit but high // "compressed" limit to exercise the codepath that should avoid encoding a metric when the // uncompressed payload would exceed the limit. - let header_len = max_uncompressed_header_len(); + let header_len = + max_uncompressed_header_len(DatadogMetricsEndpoint::Series(SeriesApiVersion::V1)); let mut encoder = DatadogMetricsEncoder::with_payload_limits( DatadogMetricsEndpoint::Series(SeriesApiVersion::V1), None, header_len + 1, usize::MAX, - ) - .expect("payload size limits should be valid"); + ); // Trying to encode a metric that would cause us to exceed our uncompressed limits will // _not_ return an error from `try_encode`, but instead will simply return back the metric @@ -1613,11 +1700,11 @@ mod tests { let (payload, processed) = result.unwrap(); assert_eq!( payload.uncompressed_byte_size, - max_uncompressed_header_len() + max_uncompressed_header_len(DatadogMetricsEndpoint::Series(SeriesApiVersion::V1)) ); assert_eq!( payload.into_payload(), - get_compressed_empty_series_payload() + get_compressed_empty_series_v1_payload() ); assert_eq!(processed.len(), 0); } @@ -1632,8 +1719,7 @@ mod tests { None, 1, usize::MAX, - ) - .expect("payload size limits should be valid"); + ); // Trying to encode a metric that would cause us to exceed our uncompressed limits will // _not_ return an error from `try_encode`, but instead will simply return back the metric @@ -1669,8 +1755,7 @@ mod tests { None, uncompressed_limit, compressed_limit, - ) - .expect("payload size limits should be valid"); + ); // Trying to encode a metric that would cause us to exceed our compressed limits will // _not_ return an error from `try_encode`, but instead will simply return back the metric @@ -1689,11 +1774,11 @@ mod tests { let (payload, processed) = result.unwrap(); assert_eq!( payload.uncompressed_byte_size, - max_uncompressed_header_len() + max_uncompressed_header_len(DatadogMetricsEndpoint::Series(SeriesApiVersion::V1)) ); assert_eq!( payload.into_payload(), - get_compressed_empty_series_payload() + get_compressed_empty_series_v1_payload() ); assert_eq!(processed.len(), 0); } @@ -1704,14 +1789,13 @@ mod tests { // "uncompressed" limit to exercise the codepath that should avoid encoding a metric when the // compressed payload would exceed the limit. let uncompressed_limit = 128; - let compressed_limit = 16; + let compressed_limit = 32; let mut encoder = DatadogMetricsEncoder::with_payload_limits( DatadogMetricsEndpoint::Sketches, None, uncompressed_limit, compressed_limit, - ) - .expect("payload size limits should be valid"); + ); // Trying to encode a metric that would cause us to exceed our compressed limits will // _not_ return an error from `try_encode`, but instead will simply return back the metric @@ -1736,6 +1820,284 @@ mod tests { assert_eq!(processed.len(), 0); } + #[test] + fn encode_series_v2_breaks_out_when_limit_reached_compressed() { + // We manually create the encoder with an arbitrarily low "compressed" limit but high + // "uncompressed" limit to exercise the codepath that should avoid encoding a metric when the + // compressed payload would exceed the limit. + let uncompressed_limit = 128; + let compressed_limit = 32; + let mut encoder = DatadogMetricsEncoder::with_payload_limits( + DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), + None, + uncompressed_limit, + compressed_limit, + ); + + // Trying to encode a metric that would cause us to exceed our compressed limits will + // _not_ return an error from `try_encode`, but instead will simply return back the metric + // as it could not be added. + let counter = get_simple_counter(); + let result = encoder.try_encode(counter.clone()); + assert!(result.is_ok()); + assert_eq!(result.unwrap(), Some(counter)); + + // And similarly, since we didn't actually encode a metric, we _should_ be able to finish + // this payload, but it will be empty (effectively, the header/footer will exist) and no + // processed metrics should be returned. + let result = encoder.finish(); + assert!(result.is_ok()); + + let (payload, processed) = result.unwrap(); + assert_eq!(payload.uncompressed_byte_size, 0); + assert_eq!( + payload.into_payload(), + get_compressed_empty_series_v2_payload() + ); + assert_eq!(processed.len(), 0); + } + + #[test] + fn zstd_v2_payload_never_exceeds_512kb_with_incompressible_data() { + // End-to-end regression test using the real 512 KB compressed limit. + // + // Metric names are generated with a xorshift64 PRNG producing random printable ASCII + // (6.5 bits of entropy per byte), making them effectively incompressible for zstd. + // This makes the capacity estimate tight, so the test validates both directions: + // + // Safety (upper bound): payload ≀ 512 KB. + // Without the fix, the encoder ignores zstd's internal 128 KB buffer. When the + // encoder finally stops, finish() flushes that hidden buffer on top of the already + // ~511 KB payload β†’ ~639 KB β†’ TooLarge. + // + // Utilization (lower bound): payload > 95% of 512 KB. + // With incompressible data, actual_compressed β‰ˆ max_cs(uncompressed), so the + // estimate is tight. The ~2.5% gap comes from: (1) compressible proto framing + // (field tags, timestamps, metadata) that zstd compresses better than max_cs + // predicts, (2) the max_cs overhead term (~0.4%), and (3) one-metric stopping + // granularity (~1%). + + // xorshift64 PRNG: 5000 random printable ASCII chars per metric name (0x21..0x7E, + // 93 values β‰ˆ 6.5 bits/byte). Long names ensure the random portion dominates the + // compressible proto framing, maximizing utilization. + const PRINTABLE_START: u8 = 0x21; + const PRINTABLE_END: u8 = 0x7E; + const PRINTABLE_LEN: u64 = (PRINTABLE_END - PRINTABLE_START + 1) as u64; // 93 + let mut xor_state = 0xdeadbeef_cafebabe_u64; + let mut next_name = || -> String { + std::iter::once('m') + .chain((0..4999).map(|_| { + xor_state ^= xor_state << 13; + xor_state ^= xor_state >> 7; + xor_state ^= xor_state << 17; + (PRINTABLE_START + (xor_state % PRINTABLE_LEN) as u8) as char + })) + .collect() + }; + + let mut encoder = + DatadogMetricsEncoder::new(DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), None); + + let mut accepted = 0usize; + loop { + let metric = Metric::new( + next_name(), + MetricKind::Incremental, + MetricValue::Counter { + value: (accepted + 1) as f64, + }, + ) + .with_timestamp(Some(ts())); + + match encoder.try_encode(metric) { + Ok(None) => accepted += 1, + Ok(Some(_)) => break, + Err(e) => panic!("unexpected encoding error: {e}"), + } + } + + assert!(accepted > 0, "at least one metric must be accepted"); + + let compressed_limit = DatadogMetricsEndpoint::Series(SeriesApiVersion::V2) + .payload_limits() + .compressed; + + let (payload, _) = encoder.finish().unwrap_or_else(|e| { + panic!( + "finish() returned an error after {accepted} metrics β€” \ + the capacity estimate failed to prevent overflow: {e:?}" + ) + }); + let payload_len = payload.into_payload().len(); + + // Safety: the hard limit must never be exceeded. + assert!( + payload_len <= compressed_limit, + "payload ({payload_len} bytes) exceeded the {compressed_limit}-byte compressed limit" + ); + + // Utilization: the encoder should use at least 95% of the available capacity. + let min_utilization = compressed_limit * 95 / 100; + assert!( + payload_len > min_utilization, + "payload ({payload_len} bytes) is below 95% of the {compressed_limit}-byte limit \ + ({min_utilization} bytes) β€” estimate may be over-conservative" + ); + } + + #[test] + fn compressed_limit_is_respected_regardless_of_compressor_internal_buffering() { + // Regression test for zstd's internal buffering hiding the compressed-size check. + // + // zstd buffers up to 128 KB internally before flushing a block to the output Vec. + // The old capacity check used `get_ref().len()` alone as "compressed bytes so far", which + // returns 0 until zstd's first 128 KB block completes. This caused the encoder to accept + // metrics indefinitely β€” finish() would then return TooLarge, triggering expensive + // re-encoding. + // + // The fix splits the estimate: exact compressed size for flushed blocks, plus + // max_compressed_size for the unflushed portion (bytes still in zstd's internal buffer). + // This is accurate for flushed data and bounded for unflushed data. + // + // At compressed_limit=512, no zstd block will flush (needs 128 KB of input), so + // get_ref().len() stays 0 throughout. The old code would accept all 100 metrics; + // the new code stops after a handful. + let compressed_limit = 512; + let mut encoder = DatadogMetricsEncoder::with_payload_limits( + DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), + None, + 1_000_000, + compressed_limit, + ); + + let mut accepted = 0; + for i in 0..100 { + let metric = Metric::new( + format!("counter_{i:0>20}"), + MetricKind::Incremental, + MetricValue::Counter { + value: (i + 1) as f64, + }, + ) + .with_timestamp(Some(ts())); + match encoder.try_encode(metric) { + Ok(None) => accepted += 1, + Ok(Some(_)) => break, + Err(e) => panic!("unexpected encoding error: {e}"), + } + } + + assert!(accepted > 0, "encoder should accept at least one metric"); + assert!( + accepted < 10, + "encoder accepted too many metrics β€” compressed limit was likely not enforced (accepted={accepted})" + ); + + let result = encoder.finish(); + assert!( + result.is_ok(), + "finish() must not return TooLarge: {:?}", + result.err() + ); + let (payload, _) = result.unwrap(); + assert!( + payload.into_payload().len() <= compressed_limit, + "payload exceeded compressed_limit" + ); + } + + #[test] + fn zstd_buffered_bound_resets_to_last_metric_size_after_block_flush() { + // White-box test: directly verifies that buffered_bound resets to exactly n (the last + // metric's encoded size) when a zstd block flush occurs, not to 0 or some other value. + // + // buffered_bound is an upper bound on bytes in zstd's internal buffer. On each write it + // accumulates (+= n). When a flush is detected (get_ref().len() grows), it resets to n β€” + // meaning only the triggering write could straddle the block boundary. + // + // If it reset to 0 instead, subsequent capacity checks would degenerate to + // flushed_compressed + max_cs(n) + // which vastly underestimates for any data written after the flush, re-introducing the + // original blind spot. If it failed to reset at all, the encoder would become + // over-conservative and reject valid metrics after the first flush. + // + // Strategy: + // 1. Measure a single metric's encoded size by inspecting buffered_bound after one write. + // 2. Feed metrics into a second encoder (with unlimited limits) until buffered_bound + // *decreases*, which signals a block flush. Assert the post-flush value equals + // exactly one metric's encoded size. + + let make_metric = |i: usize| { + // Fixed-width name (600-char zero-padded) gives a constant per-metric encoded size. + // Value is (i + 1) to ensure it is never 0.0: proto3 omits default (zero) values, + // which would make the first metric smaller than the rest. + Metric::new( + format!("counter_{i:0>600}"), + MetricKind::Incremental, + MetricValue::Counter { + value: (i + 1) as f64, + }, + ) + .with_timestamp(Some(ts())) + }; + + // Step 1: measure a single metric's encoded size. + let metric_size = { + let mut probe = DatadogMetricsEncoder::with_payload_limits( + DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), + None, + usize::MAX, + usize::MAX, + ); + assert!( + probe.try_encode(make_metric(0)).unwrap().is_none(), + "first metric must be accepted" + ); + probe.buffered_bound() + }; + assert!(metric_size > 0, "encoded metric must be non-empty"); + + // Step 2: encode metrics until buffered_bound decreases (= flush detected). + let mut encoder = DatadogMetricsEncoder::with_payload_limits( + DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), + None, + usize::MAX, + usize::MAX, + ); + + let mut prev_buffered = 0usize; + let mut flush_seen = false; + + for i in 0..1000 { + let metric = make_metric(i); + match encoder.try_encode(metric) { + Ok(None) => {} + Ok(Some(_)) => panic!("unexpected rejection at i={i} with unlimited limits"), + Err(e) => panic!("unexpected error at i={i}: {e}"), + } + + let curr = encoder.buffered_bound(); + + if curr < prev_buffered { + // A block flush just occurred: buffered_bound must reset to exactly n. + assert_eq!( + curr, metric_size, + "after block flush, buffered_bound should reset to one metric's encoded size \ + ({metric_size} bytes) but got {curr}" + ); + flush_seen = true; + break; + } + + prev_buffered = curr; + } + + assert!( + flush_seen, + "no zstd block flush detected after 1000 metrics β€” increase loop bound or metric size" + ); + } + fn arb_counter_metric() -> impl Strategy { let name = string_regex("[a-zA-Z][a-zA-Z0-9_]{8,96}").expect("regex should not be invalid"); let value = ARB_POSITIVE_F64; @@ -1756,9 +2118,9 @@ mod tests { proptest! { #[test] - fn encoding_check_for_payload_limit_edge_cases( - uncompressed_limit in 0..64_000_000usize, - compressed_limit in 0..10_000_000usize, + fn encoding_check_for_payload_limit_edge_cases_v1( + uncompressed_limit in 1..64_000_000usize, + compressed_limit in 1..10_000_000usize, metric in arb_counter_metric(), ) { // We simply try to encode a single metric into an encoder, and make sure that when we @@ -1767,25 +2129,51 @@ mod tests { // // We check this with targeted unit tests as well but this is some cheap insurance to // show that we're hopefully not missing any particular corner cases. - let result = DatadogMetricsEncoder::with_payload_limits( + let mut encoder = DatadogMetricsEncoder::with_payload_limits( + DatadogMetricsEndpoint::Series(SeriesApiVersion::V1), + None, + uncompressed_limit, + compressed_limit, + ); + _ = encoder.try_encode(metric); + + if let Ok((payload, _processed)) = encoder.finish() { + let payload = payload.into_payload(); + prop_assert!(payload.len() <= compressed_limit); + + // V1 uses zlib/deflate. + let result = decompress_zlib_payload(payload); + prop_assert!(result.is_ok()); + + let decompressed = result.unwrap(); + prop_assert!(decompressed.len() <= uncompressed_limit); + } + } + + #[test] + fn encoding_check_for_payload_limit_edge_cases_v2( + uncompressed_limit in 1..10_000_000usize, + compressed_limit in 1..1_000_000usize, + metric in arb_counter_metric(), + ) { + let mut encoder = DatadogMetricsEncoder::with_payload_limits( DatadogMetricsEndpoint::Series(SeriesApiVersion::V2), None, uncompressed_limit, compressed_limit, ); - if let Ok(mut encoder) = result { - _ = encoder.try_encode(metric); + _ = encoder.try_encode(metric); - if let Ok((payload, _processed)) = encoder.finish() { - let payload = payload.into_payload(); - prop_assert!(payload.len() <= compressed_limit); + if let Ok((payload, _processed)) = encoder.finish() { + let payload = payload.into_payload(); + prop_assert!(payload.len() <= compressed_limit); - let result = decompress_payload(payload); - prop_assert!(result.is_ok()); + // V2 uses zstd. + let result = decompress_zstd_payload(payload); + prop_assert!(result.is_ok()); - let decompressed = result.unwrap(); - prop_assert!(decompressed.len() <= uncompressed_limit); - } + let decompressed = result.unwrap(); + prop_assert!(decompressed.len() <= uncompressed_limit); } } } diff --git a/src/sinks/datadog/metrics/integration_tests.rs b/src/sinks/datadog/metrics/integration_tests.rs index e20ef106f7ee2..176c2153d88e0 100644 --- a/src/sinks/datadog/metrics/integration_tests.rs +++ b/src/sinks/datadog/metrics/integration_tests.rs @@ -29,6 +29,7 @@ use crate::{ DATA_VOLUME_SINK_TAGS, SINK_TAGS, assert_data_volume_sink_compliance, assert_sink_compliance, }, + compression::is_zstd, map_event_batch_stream, }, }; @@ -142,10 +143,14 @@ async fn start_test(events: Vec) -> (Vec, Receiver<(http::request: } fn decompress_payload(payload: Vec) -> std::io::Result> { - let mut decompressor = ZlibDecoder::new(&payload[..]); - let mut decompressed = Vec::new(); - let result = std::io::copy(&mut decompressor, &mut decompressed); - result.map(|_| decompressed) + if is_zstd(&payload) { + zstd::decode_all(&payload[..]) + } else { + let mut decompressor = ZlibDecoder::new(&payload[..]); + let mut decompressed = Vec::new(); + std::io::copy(&mut decompressor, &mut decompressed)?; + Ok(decompressed) + } } #[tokio::test] @@ -321,7 +326,9 @@ fn validate_protobuf_set_gauge_rate(request: &(Parts, Bytes)) { ddmetric_proto::metric_payload::MetricType::Gauge ); assert_eq!(gauge.points[0].value, 5678.0); - assert_eq!(gauge.interval, 10); + // interval_ms is dropped during normalization (incremental β†’ absolute conversion + // via into_absolute() sets interval_ms: None), so interval is always 0 for gauges. + assert_eq!(gauge.interval, 0); } // validate counter w interval = rate diff --git a/src/sinks/datadog/metrics/request_builder.rs b/src/sinks/datadog/metrics/request_builder.rs index 32f3a7e2e5ae5..ba8a7723dbf07 100644 --- a/src/sinks/datadog/metrics/request_builder.rs +++ b/src/sinks/datadog/metrics/request_builder.rs @@ -8,20 +8,14 @@ use vector_lib::{ }; use super::{ - config::{DatadogMetricsEndpoint, DatadogMetricsEndpointConfiguration}, - encoder::{CreateError, DatadogMetricsEncoder, EncoderError, FinishError}, + config::{DatadogMetricsEndpoint, DatadogMetricsEndpointConfiguration, SeriesApiVersion}, + encoder::{DatadogMetricsEncoder, EncoderError, FinishError}, service::DatadogMetricsRequest, }; use crate::sinks::util::{IncrementalRequestBuilder, metadata::RequestMetadataBuilder}; #[derive(Debug, Snafu)] pub enum RequestBuilderError { - #[snafu( - context(false), - display("Failed to build the request builder: {source}") - )] - FailedToBuild { source: CreateError }, - #[snafu(context(false), display("Failed to encode metric: {source}"))] FailedToEncode { source: EncoderError }, @@ -40,7 +34,6 @@ impl RequestBuilderError { /// many events were dropped as a result. pub fn into_parts(self) -> (String, &'static str, u64) { match self { - Self::FailedToBuild { source } => (source.to_string(), source.as_error_type(), 0), // Encoding errors always happen at the per-metric level, so we could only ever drop a // single metric/event at a time. Self::FailedToEncode { source } => (source.to_string(), source.as_error_type(), 1), @@ -80,18 +73,19 @@ impl DatadogMetricsRequestBuilder { pub fn new( endpoint_configuration: DatadogMetricsEndpointConfiguration, default_namespace: Option, - ) -> Result { - Ok(Self { + series_api_version: SeriesApiVersion, + ) -> Self { + Self { endpoint_configuration, series_encoder: DatadogMetricsEncoder::new( - DatadogMetricsEndpoint::series(), + DatadogMetricsEndpoint::Series(series_api_version), default_namespace.clone(), - )?, + ), sketches_encoder: DatadogMetricsEncoder::new( DatadogMetricsEndpoint::Sketches, default_namespace, - )?, - }) + ), + } } const fn get_encoder( @@ -246,6 +240,7 @@ impl IncrementalRequestBuilder<((Option>, DatadogMetricsEndpoint), Vec< payload, uri, content_type: ddmetrics_metadata.endpoint.content_type(), + content_encoding: ddmetrics_metadata.endpoint.compression().content_encoding(), finalizers: ddmetrics_metadata.finalizers, metadata: request_metadata, } diff --git a/src/sinks/datadog/metrics/service.rs b/src/sinks/datadog/metrics/service.rs index e40c7cd91c406..024493ed88fc2 100644 --- a/src/sinks/datadog/metrics/service.rs +++ b/src/sinks/datadog/metrics/service.rs @@ -44,6 +44,7 @@ pub struct DatadogMetricsRequest { pub payload: Bytes, pub uri: Uri, pub content_type: &'static str, + pub content_encoding: &'static str, pub finalizers: EventFinalizers, pub metadata: RequestMetadata, } @@ -63,11 +64,6 @@ impl DatadogMetricsRequest { HeaderValue::from_str(&key).expect("API key should be only valid ASCII characters") }, ); - // Requests to the metrics endpoints can be compressed, and there's almost no reason to - // _not_ compress them given tha t metric data, when encoded, is very repetitive. Thus, - // here and through the sink code, we always compress requests. Datadog also only supports - // zlib (DEFLATE) compression, which is why it's hard-coded here vs being set via the common - // `Compression` value that most sinks utilize. let request = Request::post(self.uri) .header("DD-API-KEY", api_key) // TODO: The Datadog Agent sends this header to indicate the version of the Go library @@ -82,7 +78,7 @@ impl DatadogMetricsRequest { // this header. .header("DD-Agent-Payload", "4.87.0") .header(CONTENT_TYPE, self.content_type) - .header(CONTENT_ENCODING, "deflate"); + .header(CONTENT_ENCODING, self.content_encoding); request.body(Body::from(self.payload)) } diff --git a/src/sinks/datadog/metrics/sink.rs b/src/sinks/datadog/metrics/sink.rs index 231aca75485ee..a366cc81150c8 100644 --- a/src/sinks/datadog/metrics/sink.rs +++ b/src/sinks/datadog/metrics/sink.rs @@ -8,6 +8,7 @@ use futures_util::{ stream::{self, BoxStream}, }; use tower::Service; +use tracing::Instrument; use vector_lib::{ event::{Event, Metric, MetricValue}, partition::Partitioner, @@ -16,8 +17,10 @@ use vector_lib::{ }; use super::{ - config::DatadogMetricsEndpoint, normalizer::DatadogMetricsNormalizer, - request_builder::DatadogMetricsRequestBuilder, service::DatadogMetricsRequest, + config::{DatadogMetricsEndpoint, SeriesApiVersion}, + normalizer::DatadogMetricsNormalizer, + request_builder::DatadogMetricsRequestBuilder, + service::DatadogMetricsRequest, }; use crate::{ internal_events::DatadogMetricsEncodingError, @@ -33,21 +36,24 @@ use crate::{ /// Generally speaking, all "basic" metrics -- counter, gauge, set, aggregated summary-- are sent to /// the Series API, while distributions, aggregated histograms, and sketches (hehe) are sent to the /// Sketches API. -struct DatadogMetricsTypePartitioner; +struct DatadogMetricsTypePartitioner { + series_api_version: SeriesApiVersion, +} impl Partitioner for DatadogMetricsTypePartitioner { type Item = Metric; type Key = (Option>, DatadogMetricsEndpoint); fn partition(&self, item: &Self::Item) -> Self::Key { + let series = DatadogMetricsEndpoint::Series(self.series_api_version); let endpoint = match item.data().value() { - MetricValue::Counter { .. } => DatadogMetricsEndpoint::series(), - MetricValue::Gauge { .. } => DatadogMetricsEndpoint::series(), - MetricValue::Set { .. } => DatadogMetricsEndpoint::series(), + MetricValue::Counter { .. } => series, + MetricValue::Gauge { .. } => series, + MetricValue::Set { .. } => series, MetricValue::Distribution { .. } => DatadogMetricsEndpoint::Sketches, MetricValue::AggregatedHistogram { .. } => DatadogMetricsEndpoint::Sketches, // NOTE: AggregatedSummary will be split into counters and gauges during normalization - MetricValue::AggregatedSummary { .. } => DatadogMetricsEndpoint::series(), + MetricValue::AggregatedSummary { .. } => series, MetricValue::Sketch { .. } => DatadogMetricsEndpoint::Sketches, }; (item.metadata().datadog_api_key(), endpoint) @@ -57,8 +63,10 @@ impl Partitioner for DatadogMetricsTypePartitioner { pub(crate) struct DatadogMetricsSink { service: S, request_builder: DatadogMetricsRequestBuilder, - batch_settings: BatcherSettings, + series_batch_settings: BatcherSettings, + sketches_batch_settings: BatcherSettings, protocol: String, + series_api_version: SeriesApiVersion, } impl DatadogMetricsSink @@ -72,20 +80,28 @@ where pub const fn new( service: S, request_builder: DatadogMetricsRequestBuilder, - batch_settings: BatcherSettings, + series_batch_settings: BatcherSettings, + sketches_batch_settings: BatcherSettings, protocol: String, + series_api_version: SeriesApiVersion, ) -> Self { DatadogMetricsSink { service, request_builder, - batch_settings, + series_batch_settings, + sketches_batch_settings, protocol, + series_api_version, } } async fn run_inner(self: Box, input: BoxStream<'_, Event>) -> Result<(), ()> { let mut splitter: MetricSplitter = MetricSplitter::default(); - let batch_settings = self.batch_settings; + let series_batch_settings = self.series_batch_settings; + let sketches_batch_settings = self.sketches_batch_settings; + let partitioner = DatadogMetricsTypePartitioner { + series_api_version: self.series_api_version, + }; input // Convert `Event` to `Metric` so we don't have to deal with constant conversions. @@ -99,10 +115,20 @@ where // what quantiles to generate, etc. .normalized_with_default::() // We batch metrics by their endpoint: series endpoint for counters, gauge, and sets vs sketch endpoint for - // distributions, aggregated histograms, and sketches. - .batched_partitioned(DatadogMetricsTypePartitioner, || { - batch_settings.as_byte_size_config() - }) + // distributions, aggregated histograms, and sketches. Each endpoint uses its own byte size + // limit: 5 MiB for Series v2 and 60 MiB for Sketches. + .batched_partitioned( + partitioner, + series_batch_settings.timeout, + |(_api_key, endpoint)| match endpoint { + DatadogMetricsEndpoint::Series(_) => { + series_batch_settings.as_byte_size_config() + } + DatadogMetricsEndpoint::Sketches => { + sketches_batch_settings.as_byte_size_config() + } + }, + ) // Aggregate counters with identical timestamps, otherwise identical counters (same // series and same timestamp, when rounded to whole seconds) will be dropped in a // last-write-wins situation when they hit the DD metrics intake. @@ -111,11 +137,18 @@ where .concurrent_map( default_request_builder_concurrency_limit(), |((api_key, endpoint), metrics)| { - Box::pin(async move { - let collapsed_metrics = - sort_and_collapse_counters_by_series_and_timestamp(metrics); - ((api_key, endpoint), collapsed_metrics) - }) + // `concurrent_map` spawns this future on a detached task. The closure itself + // runs within `run_inner`'s span, so `in_current_span` captures the sink span + // here and re-enters it on the spawned task to preserve the sink's automatic + // component tags on any internal metrics/logs emitted during aggregation. + Box::pin( + async move { + let collapsed_metrics = + sort_and_collapse_counters_by_series_and_timestamp(metrics); + ((api_key, endpoint), collapsed_metrics) + } + .in_current_span(), + ) }, ) // We build our requests "incrementally", which means that for a single batch of metrics, we might generate @@ -233,10 +266,8 @@ mod tests { use chrono::{DateTime, Utc}; use proptest::prelude::*; - use vector_lib::{ - event::{Metric, MetricKind, MetricValue}, - metric_tags, - }; + use vector_lib::event::{Metric, MetricKind, MetricValue}; + use vector_lib::metric_tags; use super::sort_and_collapse_counters_by_series_and_timestamp; diff --git a/src/sinks/datadog/traces/config.rs b/src/sinks/datadog/traces/config.rs index 7ace1b75f9817..00682514dadc1 100644 --- a/src/sinks/datadog/traces/config.rs +++ b/src/sinks/datadog/traces/config.rs @@ -178,7 +178,7 @@ impl DatadogTracesConfig { // Send the APM stats payloads independently of the sink framework. // This is necessary to comply with what the APM stats backend of Datadog expects with // respect to receiving stats payloads. - tokio::spawn(flush_apm_stats_thread( + crate::spawn_in_current_span(flush_apm_stats_thread( tripwire, client, compression, diff --git a/src/sinks/datadog/traces/sink.rs b/src/sinks/datadog/traces/sink.rs index 8f48ab0acc63b..77f931fb931ab 100644 --- a/src/sinks/datadog/traces/sink.rs +++ b/src/sinks/datadog/traces/sink.rs @@ -108,7 +108,9 @@ where let batch_settings = self.batch_settings; input - .batched_partitioned(EventPartitioner, || batch_settings.as_byte_size_config()) + .batched_partitioned(EventPartitioner, batch_settings.timeout, |_| { + batch_settings.as_byte_size_config() + }) .incremental_request_builder(self.request_builder) .flat_map(stream::iter) .filter_map(|request| async move { diff --git a/src/sinks/doris/sink.rs b/src/sinks/doris/sink.rs index ac9a77f09742e..c87e76557c653 100644 --- a/src/sinks/doris/sink.rs +++ b/src/sinks/doris/sink.rs @@ -34,7 +34,9 @@ where let batch_settings = self.batch_settings; let key_partitioner = DorisKeyPartitioner::new(self.database, self.table); input - .batched_partitioned(key_partitioner, || batch_settings.as_byte_size_config()) + .batched_partitioned(key_partitioner, batch_settings.timeout, |_| { + batch_settings.as_byte_size_config() + }) .filter_map(|(key, batch)| async move { key.map(move |k| (k, batch)) }) .request_builder( default_request_builder_concurrency_limit(), diff --git a/src/sinks/elasticsearch/mod.rs b/src/sinks/elasticsearch/mod.rs index fddae4eb3caf9..1466c415dbd1b 100644 --- a/src/sinks/elasticsearch/mod.rs +++ b/src/sinks/elasticsearch/mod.rs @@ -80,7 +80,7 @@ pub enum ElasticsearchMode { /// Bulk API actions. #[configurable_component] -#[derive(Clone, Copy, Debug, Derivative, Eq, Hash, PartialEq)] +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] #[serde(deny_unknown_fields, rename_all = "snake_case")] pub enum BulkAction { /// The `index` action. @@ -127,7 +127,7 @@ impl TryFrom<&str> for BulkAction { /// Elasticsearch version types. #[configurable_component] -#[derive(Clone, Copy, Debug, Derivative, Eq, Hash, PartialEq)] +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] #[serde(deny_unknown_fields, rename_all = "snake_case")] pub enum VersionType { /// The `internal` type. diff --git a/src/sinks/file/mod.rs b/src/sinks/file/mod.rs index e3af256f6501b..f6c0934fa2d9e 100644 --- a/src/sinks/file/mod.rs +++ b/src/sinks/file/mod.rs @@ -57,6 +57,9 @@ pub struct FileSinkConfig { docs::examples = "/tmp/application-{{ application_id }}-%Y-%m-%d.log" ))] #[configurable(metadata(docs::examples = "/tmp/vector-%Y-%m-%d.log.zst"))] + #[configurable(metadata( + docs::warnings = "The rendered path can resolve to any location on the filesystem. Vector will write to it if the process has permission." + ))] pub path: Template, /// The amount of time that a file can be idle and stay open. diff --git a/src/sinks/gcp/stackdriver/logs/config.rs b/src/sinks/gcp/stackdriver/logs/config.rs index 94be249e1dfff..addb45163df76 100644 --- a/src/sinks/gcp/stackdriver/logs/config.rs +++ b/src/sinks/gcp/stackdriver/logs/config.rs @@ -21,7 +21,7 @@ use crate::{ prelude::*, util::{ BoxedRawValue, RealtimeSizeBasedDefaultBatchSettings, - http::{HttpService, http_response_retry_logic}, + http::{HttpService, RetryStrategy, http_response_retry_logic}, service::TowerRequestConfigDefaults, }, }, @@ -106,6 +106,10 @@ pub(super) struct StackdriverConfig { skip_serializing_if = "crate::serde::is_default" )] acknowledgements: AcknowledgementsConfig, + + #[configurable(derived)] + #[serde(default)] + pub retry_strategy: RetryStrategy, } pub(super) fn default_endpoint() -> String { @@ -162,8 +166,7 @@ pub(super) enum StackdriverLogName { /// Label Configuration. #[configurable_component] -#[derive(Clone, Debug, Derivative)] -#[derivative(Default)] +#[derive(Clone, Debug, Default)] pub(super) struct StackdriverLabelConfig { /// The value of this field is used to retrieve the associated labels from the `jsonPayload` /// and extract their values to set as LogEntry labels. @@ -270,7 +273,10 @@ impl SinkConfig for StackdriverConfig { let service = HttpService::new(client.clone(), stackdriver_logs_service_request_builder); let service = ServiceBuilder::new() - .settings(request_limits, http_response_retry_logic()) + .settings( + request_limits, + http_response_retry_logic(self.retry_strategy.clone()), + ) .service(service); let sink = StackdriverLogsSink::new(service, batch_settings, request_builder); diff --git a/src/sinks/gcp/stackdriver/metrics/config.rs b/src/sinks/gcp/stackdriver/metrics/config.rs index 2398a18e9beb9..ea2cc9d6b5293 100644 --- a/src/sinks/gcp/stackdriver/metrics/config.rs +++ b/src/sinks/gcp/stackdriver/metrics/config.rs @@ -15,7 +15,8 @@ use crate::{ prelude::*, util::{ http::{ - HttpRequest, HttpService, HttpServiceRequestBuilder, http_response_retry_logic, + HttpRequest, HttpService, HttpServiceRequestBuilder, RetryStrategy, + http_response_retry_logic, }, service::TowerRequestConfigDefaults, }, @@ -77,6 +78,10 @@ pub struct StackdriverConfig { skip_serializing_if = "crate::serde::is_default" )] pub(super) acknowledgements: AcknowledgementsConfig, + + #[configurable(derived)] + #[serde(default)] + pub retry_strategy: RetryStrategy, } fn default_metric_namespace_value() -> String { @@ -126,7 +131,10 @@ impl SinkConfig for StackdriverConfig { let service = HttpService::new(client, stackdriver_metrics_service_request_builder); let service = ServiceBuilder::new() - .settings(request_limits, http_response_retry_logic()) + .settings( + request_limits, + http_response_retry_logic(self.retry_strategy.clone()), + ) .service(service); let sink = StackdriverMetricsSink::new(service, batch_settings, request_builder); diff --git a/src/sinks/gcp_chronicle/compression.rs b/src/sinks/gcp_chronicle/compression.rs index 3002d934f18dd..b58478eb6d04f 100644 --- a/src/sinks/gcp_chronicle/compression.rs +++ b/src/sinks/gcp_chronicle/compression.rs @@ -20,11 +20,10 @@ use crate::sinks::util::{ }; /// Compression configuration. -#[derive(Clone, Copy, Debug, Derivative, Eq, PartialEq)] -#[derivative(Default)] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] pub enum ChronicleCompression { /// No compression. - #[derivative(Default)] + #[default] None, /// [Gzip][gzip] compression. diff --git a/src/sinks/gcp_chronicle/sink.rs b/src/sinks/gcp_chronicle/sink.rs index 8f37872ee4f72..bd98487b2ed96 100644 --- a/src/sinks/gcp_chronicle/sink.rs +++ b/src/sinks/gcp_chronicle/sink.rs @@ -48,7 +48,9 @@ where let request_builder = self.request_builder; input - .batched_partitioned(partitioner, || settings.as_byte_size_config()) + .batched_partitioned(partitioner, settings.timeout, |_| { + settings.as_byte_size_config() + }) .filter_map(|(key, batch)| async move { // A `TemplateRenderingError` will have been emitted by `KeyPartitioner` if the key here is `None`, // thus no further `EventsDropped` event needs emitting at this stage. diff --git a/src/sinks/gcs_common/config.rs b/src/sinks/gcs_common/config.rs index 9f148190a72b3..dcebb6bea9e77 100644 --- a/src/sinks/gcs_common/config.rs +++ b/src/sinks/gcs_common/config.rs @@ -26,8 +26,7 @@ pub fn default_endpoint() -> String { /// /// [predefined_acls]: https://cloud.google.com/storage/docs/access-control/lists#predefined-acl #[configurable_component] -#[derive(Clone, Copy, Debug, Derivative)] -#[derivative(Default)] +#[derive(Clone, Copy, Debug, Default)] #[serde(rename_all = "kebab-case")] pub enum GcsPredefinedAcl { /// Bucket/object can be read by authenticated users. @@ -65,7 +64,7 @@ pub enum GcsPredefinedAcl { /// part of the project team is granted the `READER` permission. /// /// This is the default. - #[derivative(Default)] + #[default] ProjectPrivate, /// Bucket/object can be read publicly. @@ -81,14 +80,13 @@ pub enum GcsPredefinedAcl { /// /// [storage_classes]: https://cloud.google.com/storage/docs/storage-classes #[configurable_component] -#[derive(Clone, Copy, Debug, Derivative, PartialEq, Eq)] -#[derivative(Default)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] #[serde(rename_all = "SCREAMING_SNAKE_CASE")] pub enum GcsStorageClass { /// Standard storage. /// /// This is the default. - #[derivative(Default)] + #[default] Standard, /// Nearline storage. diff --git a/src/sinks/gcs_common/sink.rs b/src/sinks/gcs_common/sink.rs index c545aaa6b196b..b2e4010ba301a 100644 --- a/src/sinks/gcs_common/sink.rs +++ b/src/sinks/gcs_common/sink.rs @@ -50,7 +50,9 @@ where let request_builder = self.request_builder; input - .batched_partitioned(partitioner, || settings.as_byte_size_config()) + .batched_partitioned(partitioner, settings.timeout, |_| { + settings.as_byte_size_config() + }) .filter_map(|(key, batch)| async move { // A `TemplateRenderingError` will have been emitted by `KeyPartitioner` if the key here is `None`, // thus no further `EventsDropped` event needs emitting at this stage. diff --git a/src/sinks/greptimedb/logs/config.rs b/src/sinks/greptimedb/logs/config.rs index 317827453d939..d667d4ce64f68 100644 --- a/src/sinks/greptimedb/logs/config.rs +++ b/src/sinks/greptimedb/logs/config.rs @@ -28,10 +28,6 @@ fn extra_params_examples() -> HashMap { HashMap::<_, _>::from_iter([("source".to_owned(), "vector".to_owned())]) } -fn extra_headers_examples() -> HashMap { - HashMap::new() -} - /// Configuration for the `greptimedb_logs` sink. #[configurable_component(sink("greptimedb_logs", "Ingest logs data into GreptimeDB."))] #[derive(Clone, Debug, Default, Derivative)] @@ -108,7 +104,6 @@ pub struct GreptimeDBLogsConfig { #[configurable(metadata( docs::additional_props_description = "Extra header key-value pairs." ))] - #[configurable(metadata(docs::examples = "extra_headers_examples()"))] pub extra_headers: Option>, #[configurable(derived)] diff --git a/src/sinks/greptimedb/logs/sink.rs b/src/sinks/greptimedb/logs/sink.rs index c61ce3d76dfc3..a2b5b68acdb81 100644 --- a/src/sinks/greptimedb/logs/sink.rs +++ b/src/sinks/greptimedb/logs/sink.rs @@ -52,7 +52,8 @@ where self.logs_sink_setting.pipeline_name, self.logs_sink_setting.pipeline_version, ), - || batcher_settings.as_byte_size_config(), + batcher_settings.timeout, + |_| batcher_settings.as_byte_size_config(), ) .filter_map(|(key, batch)| async move { key.map(move |k| (k, batch)) }) .request_builder( diff --git a/src/sinks/greptimedb/metrics/config.rs b/src/sinks/greptimedb/metrics/config.rs index a4003ae4292a0..8f91392325698 100644 --- a/src/sinks/greptimedb/metrics/config.rs +++ b/src/sinks/greptimedb/metrics/config.rs @@ -2,7 +2,7 @@ use vector_lib::{configurable::configurable_component, sensitive_string::Sensiti use crate::sinks::{ greptimedb::{ - GreptimeDBDefaultBatchSettings, default_dbname, + GreptimeDBDefaultBatchSettings, GrpcCompression, default_dbname, metrics::{ request::GreptimeDBGrpcRetryLogic, request_builder::RequestBuilderOptions, @@ -18,7 +18,7 @@ use crate::sinks::{ #[configurable(metadata( deprecated = "The `greptimedb` sink has been renamed. Please use `greptimedb_metrics` instead." ))] -#[derive(Clone, Debug, Derivative)] +#[derive(Clone, Debug, Default)] pub struct GreptimeDBConfig(GreptimeDBMetricsConfig); impl GenerateConfig for GreptimeDBConfig { @@ -85,11 +85,9 @@ pub struct GreptimeDBMetricsConfig { #[configurable(metadata(docs::examples = "password"))] #[serde(default)] pub password: Option, - /// Set gRPC compression encoding for the request - /// Default to none, `gzip` or `zstd` is supported. - #[configurable(metadata(docs::examples = "gzip"))] + /// Set gRPC compression encoding for the request. #[serde(default)] - pub grpc_compression: Option, + pub grpc_compression: GrpcCompression, #[configurable(derived)] #[serde(default)] diff --git a/src/sinks/greptimedb/metrics/request.rs b/src/sinks/greptimedb/metrics/request.rs index eedfc6f26f2d8..68767844412dc 100644 --- a/src/sinks/greptimedb/metrics/request.rs +++ b/src/sinks/greptimedb/metrics/request.rs @@ -1,6 +1,7 @@ use std::num::NonZeroUsize; -use greptimedb_ingester::{Error as GreptimeError, api::v1::*}; +use greptimedb_ingester::Error as GreptimeError; +use greptimedb_ingester::api::v1::RowInsertRequests; use vector_lib::event::Metric; use crate::sinks::{ diff --git a/src/sinks/greptimedb/metrics/request_builder.rs b/src/sinks/greptimedb/metrics/request_builder.rs index 1980d67e6a78c..e596ae06e0cee 100644 --- a/src/sinks/greptimedb/metrics/request_builder.rs +++ b/src/sinks/greptimedb/metrics/request_builder.rs @@ -1,5 +1,8 @@ use chrono::Utc; -use greptimedb_ingester::{api::v1::*, helpers::values::*}; +use greptimedb_ingester::api::v1::{ + ColumnDataType, ColumnSchema, Row, RowInsertRequest, Rows, SemanticType, Value, +}; +use greptimedb_ingester::helpers::values::{f64_value, string_value, timestamp_millisecond_value}; use vector_lib::{ event::{ Metric, MetricValue, @@ -238,6 +241,7 @@ fn tag_column(name: &str) -> ColumnSchema { #[cfg(test)] mod tests { + use greptimedb_ingester::api::v1::value; use similar_asserts::assert_eq; use super::*; diff --git a/src/sinks/greptimedb/metrics/service.rs b/src/sinks/greptimedb/metrics/service.rs index 77ac61593302a..e3bdc4a07b1b6 100644 --- a/src/sinks/greptimedb/metrics/service.rs +++ b/src/sinks/greptimedb/metrics/service.rs @@ -1,16 +1,22 @@ use std::{sync::Arc, task::Poll}; use greptimedb_ingester::{ - Client, ClientBuilder, Compression, Database, Error as GreptimeError, - api::v1::{auth_header::AuthScheme, *}, - channel_manager::*, + Error as GreptimeError, GrpcCompression as IngesterGrpcCompression, + api::v1::Basic, + api::v1::auth_header::AuthScheme, + channel_manager::{ChannelConfig, ChannelManager, ClientTlsOption}, + client::Client, + database::Database, }; use vector_lib::sensitive_string::SensitiveString; use crate::sinks::{ - greptimedb::metrics::{ - config::GreptimeDBMetricsConfig, - request::{GreptimeDBGrpcBatchOutput, GreptimeDBGrpcRequest}, + greptimedb::{ + GrpcCompression, + metrics::{ + config::GreptimeDBMetricsConfig, + request::{GreptimeDBGrpcBatchOutput, GreptimeDBGrpcRequest}, + }, }, prelude::*, }; @@ -22,49 +28,63 @@ pub struct GreptimeDBGrpcService { } fn new_client_from_config(config: &GreptimeDBGrpcServiceConfig) -> crate::Result { - let mut builder = ClientBuilder::default().peers(vec![&config.endpoint]); + let send_compression_encoding = match config.compression { + GrpcCompression::None => None, + GrpcCompression::Gzip => Some(IngesterGrpcCompression::Gzip), + GrpcCompression::Zstd => Some(IngesterGrpcCompression::Zstd), + }; + + let mut channel_config = ChannelConfig { + send_compression_encoding, + ..Default::default() + }; + + let channel_manager = if let Some(tls_config) = &config.tls { + let TlsConfig { + verify_certificate, + verify_hostname, + alpn_protocols, + ca_file, + crt_file, + key_file, + key_pass, + server_name, + } = tls_config; + + if verify_certificate.is_some() + || verify_hostname.is_some() + || alpn_protocols.is_some() + || key_pass.is_some() + || server_name.is_some() + { + warn!( + message = "TlsConfig: verify_certificate, verify_hostname, alpn_protocols, key_pass and server_name are not supported by greptimedb client at the moment." + ); + } - if let Some(compression) = config.compression.as_ref() { - let compression = match compression.as_str() { - "gzip" => Compression::Gzip, - "zstd" => Compression::Zstd, + // The greptimedb ingester requires all three TLS paths (ca_file, crt_file, + // key_file) to be set. Refuse a partial config rather than downgrading to plaintext. + match (ca_file, crt_file, key_file) { + (Some(ca), Some(crt), Some(key)) => { + channel_config.client_tls = Some(ClientTlsOption { + server_ca_cert_path: ca.to_string_lossy().into_owned(), + client_cert_path: crt.to_string_lossy().into_owned(), + client_key_path: key.to_string_lossy().into_owned(), + }); + ChannelManager::with_tls_config(channel_config).map_err(Box::new)? + } _ => { - warn!(message = "Unknown gRPC compression type: {compression}, disabled."); - Compression::None + return Err( + "GreptimeDB TLS requires ca_file, crt_file, and key_file to all be set.".into(), + ); } - }; - builder = builder.compression(compression); - } - - if let Some(tls_config) = &config.tls { - let channel_config = ChannelConfig { - client_tls: Some(try_from_tls_config(tls_config)?), - ..Default::default() - }; - - builder = builder - .channel_manager(ChannelManager::with_tls_config(channel_config).map_err(Box::new)?); - } - - Ok(builder.build()) -} - -fn try_from_tls_config(tls_config: &TlsConfig) -> crate::Result { - if tls_config.key_pass.is_some() - || tls_config.alpn_protocols.is_some() - || tls_config.verify_certificate.is_some() - || tls_config.verify_hostname.is_some() - { - warn!( - message = "TlsConfig: key_pass, alpn_protocols, verify_certificate and verify_hostname are not supported by greptimedb client at the moment." - ); - } + } + } else { + ChannelManager::with_config(channel_config) + }; + let client = Client::with_manager_and_urls(channel_manager, vec![&config.endpoint]); - Ok(ClientTlsOption { - server_ca_cert_path: tls_config.ca_file.clone(), - client_cert_path: tls_config.crt_file.clone(), - client_key_path: tls_config.key_file.clone(), - }) + Ok(client) } impl GreptimeDBGrpcService { @@ -103,7 +123,7 @@ impl Service for GreptimeDBGrpcService { Box::pin(async move { let metadata = req.metadata; - let result = client.row_insert(req.items).await?; + let result = client.insert(req.items).await?; Ok(GreptimeDBGrpcBatchOutput { _item_count: result, @@ -119,7 +139,7 @@ pub(super) struct GreptimeDBGrpcServiceConfig { dbname: String, username: Option, password: Option, - compression: Option, + compression: GrpcCompression, tls: Option, } @@ -130,7 +150,7 @@ impl From<&GreptimeDBMetricsConfig> for GreptimeDBGrpcServiceConfig { dbname: val.dbname.clone(), username: val.username.clone(), password: val.password.clone(), - compression: val.grpc_compression.clone(), + compression: val.grpc_compression, tls: val.tls.clone(), } } diff --git a/src/sinks/greptimedb/mod.rs b/src/sinks/greptimedb/mod.rs index 8116681652dc2..5369992e21c46 100644 --- a/src/sinks/greptimedb/mod.rs +++ b/src/sinks/greptimedb/mod.rs @@ -4,6 +4,20 @@ use crate::sinks::prelude::*; mod logs; mod metrics; +/// Compression algorithm for gRPC requests to GreptimeDB. +#[configurable_component] +#[derive(Clone, Copy, Debug, Default)] +#[serde(rename_all = "lowercase")] +enum GrpcCompression { + /// No compression. + #[default] + None, + /// Gzip compression. + Gzip, + /// Zstandard compression. + Zstd, +} + fn default_dbname() -> String { greptimedb_ingester::DEFAULT_SCHEMA_NAME.to_string() } diff --git a/src/sinks/honeycomb/config.rs b/src/sinks/honeycomb/config.rs index 05a56cd9df368..f179248e1697b 100644 --- a/src/sinks/honeycomb/config.rs +++ b/src/sinks/honeycomb/config.rs @@ -16,7 +16,7 @@ use crate::{ prelude::*, util::{ BatchConfig, BoxedRawValue, - http::{HttpService, http_response_retry_logic}, + http::{HttpService, RetryStrategy, http_response_retry_logic}, }, }, }; @@ -71,6 +71,10 @@ pub struct HoneycombConfig { skip_serializing_if = "crate::serde::is_default" )] acknowledgements: AcknowledgementsConfig, + + #[configurable(derived)] + #[serde(default)] + pub retry_strategy: RetryStrategy, } fn default_endpoint() -> String { @@ -124,7 +128,10 @@ impl SinkConfig for HoneycombConfig { let request_limits = self.request.into_settings(); let service = ServiceBuilder::new() - .settings(request_limits, http_response_retry_logic()) + .settings( + request_limits, + http_response_retry_logic(self.retry_strategy.clone()), + ) .service(service); let sink = HoneycombSink::new(service, batch_settings, request_builder); diff --git a/src/sinks/http/config.rs b/src/sinks/http/config.rs index 2c9f257fae5e1..2b6abebe0d402 100644 --- a/src/sinks/http/config.rs +++ b/src/sinks/http/config.rs @@ -30,7 +30,10 @@ use crate::{ prelude::*, util::{ RealtimeSizeBasedDefaultBatchSettings, UriSerde, - http::{HttpService, OrderedHeaderName, RequestConfig, http_response_retry_logic}, + http::{ + HttpService, OrderedHeaderName, RequestConfig, RetryStrategy, + http_response_retry_logic, + }, }, }, }; @@ -57,13 +60,6 @@ pub struct HttpSinkConfig { #[configurable(derived)] pub auth: Option, - /// A list of custom headers to add to each request. - #[configurable(deprecated = "This option has been deprecated, use `request.headers` instead.")] - #[configurable(metadata( - docs::additional_props_description = "An HTTP request header and it's value." - ))] - pub headers: Option>, - #[configurable(derived)] #[serde(default)] pub compression: Compression, @@ -107,6 +103,10 @@ pub struct HttpSinkConfig { skip_serializing_if = "crate::serde::is_default" )] pub acknowledgements: AcknowledgementsConfig, + + #[configurable(derived)] + #[serde(default)] + pub retry_strategy: RetryStrategy, } /// HTTP method. @@ -115,9 +115,8 @@ pub struct HttpSinkConfig { /// /// [rfc9110]: https://datatracker.ietf.org/doc/html/rfc9110#section-9.1 #[configurable_component] -#[derive(Clone, Copy, Debug, Derivative, Eq, PartialEq)] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] #[serde(rename_all = "snake_case")] -#[derivative(Default)] pub enum HttpMethod { /// GET. Get, @@ -126,7 +125,7 @@ pub enum HttpMethod { Head, /// POST. - #[derivative(Default)] + #[default] Post, /// PUT. @@ -243,8 +242,7 @@ impl SinkConfig for HttpSinkConfig { let encoder = self.build_encoder()?; let transformer = self.encoding.transformer(); - let mut request = self.request.clone(); - request.add_old_option(self.headers.clone()); + let request = self.request.clone(); validate_headers(&request.headers, self.auth.is_some())?; let (static_headers, template_headers) = request.split_headers(); @@ -339,7 +337,10 @@ impl SinkConfig for HttpSinkConfig { let request_limits = self.request.tower.into_settings(); let service = ServiceBuilder::new() - .settings(request_limits, http_response_retry_logic()) + .settings( + request_limits, + http_response_retry_logic(self.retry_strategy.clone()), + ) .service(service); let sink = HttpSink::new( @@ -407,7 +408,6 @@ mod tests { Transformer::default(), ), auth: None, - headers: None, compression: Compression::default(), batch: BatchConfig::default(), request: RequestConfig::default(), @@ -415,6 +415,7 @@ mod tests { acknowledgements: AcknowledgementsConfig::default(), payload_prefix: String::new(), payload_suffix: String::new(), + retry_strategy: RetryStrategy::default(), }; let external_resource = ExternalResource::new( diff --git a/src/sinks/http/sink.rs b/src/sinks/http/sink.rs index 982a6515aac93..0c9a14d269daf 100644 --- a/src/sinks/http/sink.rs +++ b/src/sinks/http/sink.rs @@ -43,9 +43,11 @@ where }; input // Batch the input stream with size calculation based on the configured codec - .batched_partitioned(KeyPartitioner::new(self.uri, self.headers), || { - self.batch_settings.as_item_size_config(batch_sizer.clone()) - }) + .batched_partitioned( + KeyPartitioner::new(self.uri, self.headers), + self.batch_settings.timeout, + |_| self.batch_settings.as_item_size_config(batch_sizer.clone()), + ) .filter_map(|(key, batch)| async move { key.map(move |k| (k, batch)) }) // Build requests with default concurrency limit. .request_builder( diff --git a/src/sinks/http/tests.rs b/src/sinks/http/tests.rs index 7182d608a865a..2b4b6942c5c7b 100644 --- a/src/sinks/http/tests.rs +++ b/src/sinks/http/tests.rs @@ -59,7 +59,6 @@ fn default_cfg(encoding: EncodingConfigWithFraming) -> HttpSinkConfig { uri: Default::default(), method: Default::default(), auth: Default::default(), - headers: Default::default(), compression: Default::default(), encoding, payload_prefix: Default::default(), @@ -68,6 +67,7 @@ fn default_cfg(encoding: EncodingConfigWithFraming) -> HttpSinkConfig { request: Default::default(), tls: Default::default(), acknowledgements: Default::default(), + retry_strategy: Default::default(), } } @@ -446,6 +446,105 @@ async fn retries_on_temporary_error() { .await; } +#[tokio::test] +async fn custom_retry_retries_only_configured_status_code() { + components::assert_sink_compliance(&HTTP_SINK_TAGS, async { + const NUM_LINES: usize = 1; + const NUM_FAILURES: usize = 2; + const CUSTOM_RETRY_CONFIG: &str = r#" + request.retry_attempts = 2 + request.retry_initial_backoff_secs = 1 + request.retry_max_duration_secs = 1 + retry_strategy.type = "custom" + retry_strategy.status_codes = [408, 425, 429, 503] + "#; + + let (in_addr, sink) = build_sink(CUSTOM_RETRY_CONFIG).await; + + let counter = Arc::new(atomic::AtomicUsize::new(0)); + let in_counter = Arc::clone(&counter); + let (rx, trigger, server) = build_test_server_generic(in_addr, move || { + let count = in_counter.fetch_add(1, atomic::Ordering::Relaxed); + if count < NUM_FAILURES { + Response::builder() + .status(StatusCode::SERVICE_UNAVAILABLE) + .body(Body::empty()) + .unwrap_or_else(|_| unreachable!()) + } else { + Response::new(Body::empty()) + } + }); + + let (batch, mut receiver) = BatchNotifier::new_with_receiver(); + let (input_lines, events) = random_lines_with_stream(100, NUM_LINES, Some(batch)); + let pump = sink.run(events); + + tokio::spawn(server); + + pump.await.unwrap(); + drop(trigger); + + assert_eq!(receiver.try_recv(), Ok(BatchStatus::Delivered)); + + let output_lines = get_received_gzip(rx, |parts| { + assert_eq!(Method::POST, parts.method); + assert_eq!("/frames", parts.uri.path()); + }) + .await; + + let tries = counter.load(atomic::Ordering::Relaxed); + assert_eq!(tries, NUM_FAILURES + 1); + assert_eq!(NUM_LINES, output_lines.len()); + assert_eq!(input_lines, output_lines); + }) + .await; +} + +#[tokio::test] +async fn custom_retry_does_not_retry_unconfigured_status_code() { + components::assert_sink_error(&COMPONENT_ERROR_TAGS, async { + const NUM_LINES: usize = 1; + const CUSTOM_RETRY_CONFIG: &str = r#" + request.retry_attempts = 2 + request.retry_initial_backoff_secs = 1 + request.retry_max_duration_secs = 1 + retry_strategy.type = "custom" + retry_strategy.status_codes = [408, 425, 429, 503] + "#; + + let (in_addr, sink) = build_sink(CUSTOM_RETRY_CONFIG).await; + + let counter = Arc::new(atomic::AtomicUsize::new(0)); + let in_counter = Arc::clone(&counter); + let (rx, trigger, server) = build_test_server_generic(in_addr, move || { + in_counter.fetch_add(1, atomic::Ordering::Relaxed); + Response::builder() + .status(StatusCode::BAD_REQUEST) + .body(Body::empty()) + .unwrap_or_else(|_| unreachable!()) + }); + + let (batch, mut receiver) = BatchNotifier::new_with_receiver(); + let (_input_lines, events) = random_lines_with_stream(100, NUM_LINES, Some(batch)); + let pump = sink.run(events); + + tokio::spawn(server); + + pump.await.unwrap(); + drop(trigger); + + assert_eq!(receiver.try_recv(), Ok(BatchStatus::Rejected)); + assert_eq!(counter.load(atomic::Ordering::Relaxed), 1); + + let output_lines = get_received_gzip(rx, |_| { + unreachable!("There should be no successful requests") + }) + .await; + assert!(output_lines.is_empty()); + }) + .await; +} + #[tokio::test] async fn fails_on_permanent_error() { components::assert_sink_error(&COMPONENT_ERROR_TAGS, async { diff --git a/src/sinks/kafka/config.rs b/src/sinks/kafka/config.rs index 67af212957d5b..b1e00a3e0c32c 100644 --- a/src/sinks/kafka/config.rs +++ b/src/sinks/kafka/config.rs @@ -288,8 +288,7 @@ impl SinkConfig for KafkaSinkConfig { fn input(&self) -> Input { let requirements = Requirement::empty().optional_meaning("timestamp", Kind::timestamp()); - Input::new(self.encoding.config().input_type() & (DataType::Log | DataType::Metric)) - .with_schema_requirement(requirements) + Input::new(self.encoding.config().input_type()).with_schema_requirement(requirements) } fn acknowledgements(&self) -> &AcknowledgementsConfig { diff --git a/src/sinks/kafka/request_builder.rs b/src/sinks/kafka/request_builder.rs index 23c67a1f40bb0..feb280bbcd864 100644 --- a/src/sinks/kafka/request_builder.rs +++ b/src/sinks/kafka/request_builder.rs @@ -67,28 +67,36 @@ impl RequestBuilder<(String, Event)> for KafkaRequestBuilder { fn get_key(event: &Event, key_field: Option<&OwnedTargetPath>) -> Option { key_field.and_then(|key_field| match event { Event::Log(log) => log.get(key_field).map(|value| value.coerce_to_bytes()), + Event::Trace(trace) => trace.get(key_field).map(|value| value.coerce_to_bytes()), Event::Metric(metric) => metric .tags() .and_then(|tags| tags.get(key_field.to_string().as_str())) .map(|value| value.to_owned().into()), - _ => None, }) } fn get_timestamp_millis(event: &Event) -> Option { match &event { Event::Log(log) => log.get_timestamp().and_then(|v| v.as_timestamp()).copied(), + Event::Trace(trace) => trace + .as_ref() + .get_timestamp() + .and_then(|v| v.as_timestamp()) + .copied(), Event::Metric(metric) => metric.timestamp(), - _ => None, } .map(|ts| ts.timestamp_millis()) } fn get_headers(event: &Event, headers_key: Option<&OwnedTargetPath>) -> Option { headers_key.and_then(|headers_key| { - if let Event::Log(log) = event - && let Some(headers) = log.get(headers_key) - { + let headers = match event { + Event::Log(log) => log.get(headers_key), + Event::Trace(trace) => trace.get(headers_key), + Event::Metric(_) => None, + }; + + if let Some(headers) = headers { match headers { Value::Object(headers_map) => { let mut owned_headers = OwnedHeaders::new_with_capacity(headers_map.len()); @@ -123,7 +131,7 @@ mod tests { use rdkafka::message::Headers; use super::*; - use crate::event::{LogEvent, ObjectMap}; + use crate::event::{LogEvent, ObjectMap, TraceEvent}; #[test] fn kafka_get_headers() { @@ -141,4 +149,21 @@ mod tests { assert_eq!(headers.get(1).key, "b-key"); assert_eq!(headers.get(1).value.unwrap(), "b-value".as_bytes()); } + + #[test] + fn kafka_get_headers_trace() { + let headers_key = OwnedTargetPath::try_from("headers".to_string()).unwrap(); + let mut header_values = ObjectMap::new(); + header_values.insert("a-key".into(), Value::Bytes(Bytes::from("a-value"))); + header_values.insert("b-key".into(), Value::Bytes(Bytes::from("b-value"))); + + let mut event = Event::Trace(TraceEvent::from(LogEvent::from("hello"))); + event.as_mut_trace().insert(&headers_key, header_values); + + let headers = get_headers(&event, Some(&headers_key)).unwrap(); + assert_eq!(headers.get(0).key, "a-key"); + assert_eq!(headers.get(0).value.unwrap(), "a-value".as_bytes()); + assert_eq!(headers.get(1).key, "b-key"); + assert_eq!(headers.get(1).value.unwrap(), "b-value".as_bytes()); + } } diff --git a/src/sinks/kafka/tests.rs b/src/sinks/kafka/tests.rs index c3286ee71ce47..845b17e44061b 100644 --- a/src/sinks/kafka/tests.rs +++ b/src/sinks/kafka/tests.rs @@ -6,14 +6,14 @@ mod integration_test { use std::{collections::HashMap, future::ready, thread, time::Duration}; use bytes::Bytes; - use futures::StreamExt; + use futures::{StreamExt, stream}; use rdkafka::{ Message, Offset, TopicPartitionList, consumer::{BaseConsumer, Consumer}, message::Headers, }; use vector_lib::{ - codecs::TextSerializerConfig, + codecs::{JsonSerializerConfig, TextSerializerConfig}, config::{Tags, Telemetry, init_telemetry}, event::{BatchNotifier, BatchStatus}, lookup::lookup_v2::ConfigTargetPath, @@ -21,7 +21,7 @@ mod integration_test { use super::super::{config::KafkaSinkConfig, sink::KafkaSink, *}; use crate::{ - event::{ObjectMap, Value}, + event::{ObjectMap, TraceEvent, Value}, kafka::{KafkaAuthConfig, KafkaCompression, KafkaSaslConfig}, sinks::prelude::*, test_util::{ @@ -29,7 +29,7 @@ mod integration_test { DATA_VOLUME_SINK_TAGS, SINK_TAGS, assert_data_volume_sink_compliance, assert_sink_compliance, }, - random_lines_with_stream, random_string, wait_for, + map_event_batch_stream, random_lines_with_stream, random_string, wait_for, }, tls::{TEST_PEM_INTERMEDIATE_CA_PATH, TlsConfig, TlsEnableableConfig}, }; @@ -311,6 +311,127 @@ mod integration_test { .await; } + #[tokio::test] + async fn kafka_happy_path_trace_events() { + crate::test_util::trace_init(); + + assert_sink_compliance(&SINK_TAGS, async move { + let topic_prefix = format!("test-trace-{}", random_string(10)); + let topic = format!("{}-{}", topic_prefix, chrono::Utc::now().format("%Y%m%d")); + let key_field = ConfigTargetPath::try_from("trace_key".to_string()).unwrap(); + let headers_key = ConfigTargetPath::try_from("trace_headers".to_string()).unwrap(); + let trace_key = "trace-partition-key"; + let header_key = "trace-header-key"; + let header_value = "trace-header-value"; + + let config = KafkaSinkConfig { + bootstrap_servers: kafka_address(9091), + topic: Template::try_from(format!("{topic_prefix}-%Y%m%d")).unwrap(), + healthcheck_topic: None, + key_field: Some(key_field.clone()), + encoding: JsonSerializerConfig::default().into(), + batch: BatchConfig::default(), + compression: KafkaCompression::None, + auth: KafkaAuthConfig::default(), + socket_timeout_ms: Duration::from_millis(60000), + message_timeout_ms: Duration::from_millis(300000), + rate_limit_duration_secs: 1, + rate_limit_num: i64::MAX as u64, + librdkafka_options: HashMap::new(), + headers_key: Some(headers_key.clone()), + acknowledgements: Default::default(), + }; + + let num_events = 100; + let mut expected_messages = Vec::with_capacity(num_events); + + let (batch, receiver) = BatchNotifier::new_with_receiver(); + let mut events = Vec::with_capacity(num_events); + for i in 0..num_events { + let message = format!("trace-message-{i}"); + expected_messages.push(message.clone()); + + let mut trace = TraceEvent::default(); + trace.insert("message", message); + trace.insert("trace_key", trace_key); + trace.insert("timestamp", chrono::Utc::now()); + + let mut trace_headers = ObjectMap::new(); + trace_headers.insert(header_key.into(), Value::Bytes(Bytes::from(header_value))); + trace.insert(&headers_key, trace_headers); + + events.push(Event::Trace(trace.with_batch_notifier(&batch))); + } + + let sink = KafkaSink::new(config).unwrap(); + let sink = VectorSink::from_event_streamsink(sink); + let stream = map_event_batch_stream(stream::iter(events), Some(batch)); + sink.run(stream).await.unwrap(); + assert_eq!(receiver.await, BatchStatus::Delivered); + + // Read back everything from the beginning. + let mut client_config = rdkafka::ClientConfig::new(); + client_config.set("bootstrap.servers", kafka_address(9091).as_str()); + client_config.set("group.id", random_string(10)); + client_config.set("enable.partition.eof", "true"); + let mut tpl = TopicPartitionList::new(); + tpl.add_partition(&topic, 0) + .set_offset(Offset::Beginning) + .unwrap(); + let consumer: BaseConsumer = client_config.create().unwrap(); + consumer.assign(&tpl).unwrap(); + + wait_for( + || match consumer.fetch_watermarks(&topic, 0, Duration::from_secs(3)) { + Ok((_low, high)) => ready(high >= num_events as i64), + Err(err) => { + println!("retrying due to error fetching watermarks: {err}"); + ready(false) + } + }, + ) + .await; + + let (low, high) = consumer + .fetch_watermarks(&topic, 0, Duration::from_secs(3)) + .unwrap(); + assert_eq!((0, num_events as i64), (low, high)); + + let mut failures = 0; + let mut observed_messages = Vec::new(); + while failures < 100 { + match consumer.poll(Duration::from_secs(3)) { + Some(Ok(msg)) => { + let payload: &str = msg.payload_view().unwrap().unwrap(); + let payload_json: serde_json::Value = + serde_json::from_str(payload).unwrap(); + observed_messages + .push(payload_json["message"].as_str().unwrap().to_owned()); + + let key = msg.key().unwrap(); + assert_eq!(key, trace_key.as_bytes()); + + let timestamp = msg.timestamp().to_millis(); + assert!(timestamp.is_some()); + + let header = msg.headers().unwrap().get(0); + assert_eq!(header.key, header_key); + assert_eq!(header.value.unwrap(), header_value.as_bytes()); + } + None if observed_messages.len() >= num_events => break, + _ => { + failures += 1; + thread::sleep(Duration::from_millis(50)); + } + } + } + + assert_eq!(observed_messages.len(), num_events); + assert_eq!(observed_messages, expected_messages); + }) + .await; + } + async fn kafka_happy_path( server: String, sasl: Option, diff --git a/src/sinks/keep/config.rs b/src/sinks/keep/config.rs index 7b81281253672..d3810976aa3c1 100644 --- a/src/sinks/keep/config.rs +++ b/src/sinks/keep/config.rs @@ -16,7 +16,7 @@ use crate::{ prelude::*, util::{ BatchConfig, BoxedRawValue, - http::{HttpService, http_response_retry_logic}, + http::{HttpService, RetryStrategy, http_response_retry_logic}, }, }, }; @@ -59,6 +59,10 @@ pub struct KeepConfig { skip_serializing_if = "crate::serde::is_default" )] acknowledgements: AcknowledgementsConfig, + + #[configurable(derived)] + #[serde(default)] + pub retry_strategy: RetryStrategy, } fn default_endpoint() -> String { @@ -111,7 +115,10 @@ impl SinkConfig for KeepConfig { let request_limits = self.request.into_settings(); let service = ServiceBuilder::new() - .settings(request_limits, http_response_retry_logic()) + .settings( + request_limits, + http_response_retry_logic(self.retry_strategy.clone()), + ) .service(service); let sink = KeepSink::new(service, batch_settings, request_builder); diff --git a/src/sinks/loki/config.rs b/src/sinks/loki/config.rs index 06f827b353bab..16f25dc3d554b 100644 --- a/src/sinks/loki/config.rs +++ b/src/sinks/loki/config.rs @@ -171,8 +171,7 @@ impl SinkBatchSettings for LokiDefaultBatchSettings { /// any necessary sorting/reordering. If you're using an earlier version, then you must use `Drop` /// or `RewriteTimestamp` depending on which option makes the most sense for your use case. #[configurable_component] -#[derive(Copy, Clone, Debug, Derivative)] -#[derivative(Default)] +#[derive(Copy, Clone, Debug, Default)] #[serde(rename_all = "snake_case")] pub enum OutOfOrderAction { /// Accept the event. @@ -180,7 +179,7 @@ pub enum OutOfOrderAction { /// The event is not dropped and is sent without modification. /// /// Requires Loki 2.4.0 or newer. - #[derivative(Default)] + #[default] Accept, /// Rewrite the timestamp of the event to the timestamp of the latest event seen by the sink. diff --git a/src/sinks/loki/sink.rs b/src/sinks/loki/sink.rs index 55a74dcbcfc0f..1bff5845f3618 100644 --- a/src/sinks/loki/sink.rs +++ b/src/sinks/loki/sink.rs @@ -513,7 +513,9 @@ impl LokiSink { .map(|event| encoder.encode_event(event)) .filter_map(|event| async { event }) .map(|record| filter.filter_record(record)) - .batched_partitioned(RecordPartitioner, || batch_settings.as_byte_size_config()) + .batched_partitioned(RecordPartitioner, batch_settings.timeout, |_| { + batch_settings.as_byte_size_config() + }) .filter_map(|(partition, batch)| async { if let Some(partition) = partition { let mut count: usize = 0; diff --git a/src/sinks/mezmo.rs b/src/sinks/mezmo.rs index d24142993623b..e83c2ab949b7f 100644 --- a/src/sinks/mezmo.rs +++ b/src/sinks/mezmo.rs @@ -524,7 +524,7 @@ mod tests { async fn smoke_fails() { let (_hosts, _partitions, mut rx) = smoke_start(StatusCode::FORBIDDEN, BatchStatus::Rejected).await; - assert!(matches!(rx.try_next(), Err(mpsc::TryRecvError { .. }))); + assert!(rx.try_recv().is_err()); } #[tokio::test] diff --git a/src/sinks/mod.rs b/src/sinks/mod.rs index 3a9c3c049a0fa..95967c645aa16 100644 --- a/src/sinks/mod.rs +++ b/src/sinks/mod.rs @@ -26,7 +26,7 @@ pub mod aws_s_s; pub mod axiom; #[cfg(feature = "sinks-azure_blob")] pub mod azure_blob; -#[cfg(feature = "sinks-azure_blob")] +#[cfg(any(feature = "sinks-azure_blob", feature = "sinks-azure_logs_ingestion",))] pub mod azure_common; #[cfg(feature = "sinks-azure_logs_ingestion")] pub mod azure_logs_ingestion; @@ -40,6 +40,8 @@ pub mod clickhouse; pub mod console; #[cfg(feature = "sinks-databend")] pub mod databend; +#[cfg(feature = "sinks-databricks-zerobus")] +pub mod databricks_zerobus; #[cfg(any( feature = "sinks-datadog_events", feature = "sinks-datadog_logs", diff --git a/src/sinks/mqtt/config.rs b/src/sinks/mqtt/config.rs index 4b6e589d3beb5..af0e4a3aaf645 100644 --- a/src/sinks/mqtt/config.rs +++ b/src/sinks/mqtt/config.rs @@ -53,13 +53,12 @@ pub struct MqttSinkConfig { /// Supported Quality of Service types for MQTT. #[configurable_component] -#[derive(Clone, Copy, Debug, Derivative)] -#[derivative(Default)] +#[derive(Clone, Copy, Debug, Default)] #[serde(rename_all = "lowercase")] #[allow(clippy::enum_variant_names)] pub enum MqttQoS { /// AtLeastOnce. - #[derivative(Default)] + #[default] AtLeastOnce, /// AtMostOnce. diff --git a/src/sinks/mqtt/sink.rs b/src/sinks/mqtt/sink.rs index cc69c8d6b9ff8..e9feb7bb0144e 100644 --- a/src/sinks/mqtt/sink.rs +++ b/src/sinks/mqtt/sink.rs @@ -60,7 +60,7 @@ impl MqttSink { let (client, mut connection) = self.connector.connect(); // This is necessary to keep the mqtt event loop moving forward. - tokio::spawn(async move { + crate::spawn_in_current_span(async move { loop { // If an error is returned here there is currently no way to tie this back // to the event that was posted which means we can't accurately provide diff --git a/src/sinks/new_relic/config.rs b/src/sinks/new_relic/config.rs index f0f6c1d7695a3..fa1e6622d6d61 100644 --- a/src/sinks/new_relic/config.rs +++ b/src/sinks/new_relic/config.rs @@ -12,12 +12,11 @@ use crate::{http::HttpClient, sinks::prelude::*}; /// New Relic region. #[configurable_component] -#[derive(Clone, Copy, Debug, Derivative, Eq, PartialEq)] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] #[serde(rename_all = "snake_case")] -#[derivative(Default)] pub enum NewRelicRegion { /// US region. - #[derivative(Default)] + #[default] Us, /// EU region. @@ -26,12 +25,11 @@ pub enum NewRelicRegion { /// New Relic API endpoint. #[configurable_component] -#[derive(Clone, Copy, Derivative, Debug, Eq, PartialEq)] +#[derive(Clone, Copy, Default, Debug, Eq, PartialEq)] #[serde(rename_all = "snake_case")] -#[derivative(Default)] pub enum NewRelicApi { /// Events API. - #[derivative(Default)] + #[default] Events, /// Metrics API. diff --git a/src/sinks/opendal_common.rs b/src/sinks/opendal_common.rs index 4396002ae64b3..53e23e33ae8cc 100644 --- a/src/sinks/opendal_common.rs +++ b/src/sinks/opendal_common.rs @@ -78,7 +78,9 @@ where let request_builder = self.request_builder; input - .batched_partitioned(partitioner, || settings.as_byte_size_config()) + .batched_partitioned(partitioner, settings.timeout, |_| { + settings.as_byte_size_config() + }) .filter_map(|(key, batch)| async move { // We don't need to emit an error here if the event is dropped since this will occur if the template // couldn't be rendered during the partitioning. A `TemplateRenderingError` is already emitted when diff --git a/src/sinks/opentelemetry/mod.rs b/src/sinks/opentelemetry/mod.rs index 88963f8603cde..3759c962ff78b 100644 --- a/src/sinks/opentelemetry/mod.rs +++ b/src/sinks/opentelemetry/mod.rs @@ -49,7 +49,6 @@ impl Default for Protocol { uri: Default::default(), method: HttpMethod::Post, auth: Default::default(), - headers: Default::default(), compression: Default::default(), payload_prefix: Default::default(), payload_suffix: Default::default(), @@ -57,6 +56,7 @@ impl Default for Protocol { request: Default::default(), tls: Default::default(), acknowledgements: Default::default(), + retry_strategy: Default::default(), }) } } @@ -78,7 +78,10 @@ impl GenerateConfig for OpenTelemetryConfig { impl SinkConfig for OpenTelemetryConfig { async fn build(&self, cx: SinkContext) -> crate::Result<(VectorSink, Healthcheck)> { match &self.protocol { - Protocol::Http(config) => config.build(cx).await, + Protocol::Http(config) => { + warn_on_invalid_otlp_batching(config); + config.build(cx).await + } } } @@ -95,6 +98,21 @@ impl SinkConfig for OpenTelemetryConfig { } } +fn warn_on_invalid_otlp_batching(config: &HttpSinkConfig) { + let (_, serializer) = config.encoding.config(); + let is_json = matches!(serializer, SerializerConfig::Json(_)); + let batches_more_than_one = !matches!(config.batch.max_events, Some(1)); + if is_json && batches_more_than_one { + tracing::warn!( + message = "`opentelemetry` sink is configured with `encoding.codec = json` and \ + `batch.max_events` greater than 1. This produces invalid OTLP request \ + bodies that receivers reject with HTTP 400. Use `encoding.codec = otlp` \ + (recommended) or set `batch.max_events = 1`. See \ + https://github.com/vectordotdev/vector/issues/22054.", + ); + } +} + #[cfg(test)] mod test { #[test] diff --git a/src/sinks/prometheus/exporter.rs b/src/sinks/prometheus/exporter.rs index 6072bc8274ea0..8ed514b638fa0 100644 --- a/src/sinks/prometheus/exporter.rs +++ b/src/sinks/prometheus/exporter.rs @@ -466,7 +466,7 @@ impl PrometheusExporter { let tls = MaybeTlsSettings::from_config(tls.as_ref(), true)?; let listener = tls.bind(&address).await?; - tokio::spawn(async move { + crate::spawn_in_current_span(async move { info!(message = "Building HTTP server.", address = %address); Server::builder(hyper::server::accept::from_stream(listener.accept_stream())) diff --git a/src/sinks/prometheus/remote_write/config.rs b/src/sinks/prometheus/remote_write/config.rs index c1b420d6b216d..c9f214953003c 100644 --- a/src/sinks/prometheus/remote_write/config.rs +++ b/src/sinks/prometheus/remote_write/config.rs @@ -17,7 +17,7 @@ use crate::{ prometheus::PrometheusRemoteWriteAuth, util::{ auth::Auth, - http::{OrderedHeaderName, http_response_retry_logic}, + http::{OrderedHeaderName, RetryStrategy, http_response_retry_logic}, service::TowerRequestConfig, }, }, @@ -129,6 +129,10 @@ pub struct RemoteWriteConfig { #[serde(default = "default_compression")] #[derivative(Default(value = "default_compression()"))] pub compression: Compression, + + #[configurable(derived)] + #[serde(default)] + pub retry_strategy: RetryStrategy, } const fn default_compression() -> Compression { @@ -251,7 +255,10 @@ impl SinkConfig for RemoteWriteConfig { headers: validated_headers, }; let service = ServiceBuilder::new() - .settings(request_settings, http_response_retry_logic()) + .settings( + request_settings, + http_response_retry_logic(self.retry_strategy.clone()), + ) .service(service); let sink = RemoteWriteSink { diff --git a/src/sinks/prometheus/remote_write/sink.rs b/src/sinks/prometheus/remote_write/sink.rs index 1b5f82ad76916..858f535a0d98e 100644 --- a/src/sinks/prometheus/remote_write/sink.rs +++ b/src/sinks/prometheus/remote_write/sink.rs @@ -182,10 +182,14 @@ where .filter_map(move |event| { future::ready(make_remote_write_event(tenant_id.as_ref(), event)) }) - .batched_partitioned(PrometheusTenantIdPartitioner, || { - batch_settings - .as_reducer_config(ByteSizeOfItemSize, EventCollection::new(self.aggregate)) - }) + .batched_partitioned( + PrometheusTenantIdPartitioner, + batch_settings.timeout, + |_| { + batch_settings + .as_reducer_config(ByteSizeOfItemSize, EventCollection::new(self.aggregate)) + }, + ) .request_builder(default_request_builder_concurrency_limit(), request_builder) .filter_map(|request| async move { match request { diff --git a/src/sinks/pulsar/config.rs b/src/sinks/pulsar/config.rs index b006b6212d417..e149c9c5acfe5 100644 --- a/src/sinks/pulsar/config.rs +++ b/src/sinks/pulsar/config.rs @@ -162,12 +162,11 @@ pub struct OAuth2Config { /// Supported compression types for Pulsar. #[configurable_component] -#[derive(Clone, Copy, Debug, Derivative)] -#[derivative(Default)] +#[derive(Clone, Copy, Debug, Default)] #[serde(rename_all = "lowercase")] pub enum PulsarCompression { /// No compression. - #[derivative(Default)] + #[default] None, /// LZ4. @@ -310,6 +309,7 @@ impl PulsarSinkConfig { .map_or(default_retry_options.keep_alive, |secs| { Duration::from_secs(secs) }), + connection_max_idle: Default::default(), } }); @@ -338,6 +338,9 @@ impl PulsarSinkConfig { batch_size: self.batch.max_events, batch_byte_size: self.batch.max_bytes, compression: None, + batch_timeout: Default::default(), + block_queue_if_full: Default::default(), + routing_policy: Default::default(), }; match &self.compression { diff --git a/src/sinks/redis/config.rs b/src/sinks/redis/config.rs index 17939e679da7a..69dfbdf44e6a3 100644 --- a/src/sinks/redis/config.rs +++ b/src/sinks/redis/config.rs @@ -22,8 +22,7 @@ impl TowerRequestConfigDefaults for RedisTowerRequestConfigDefaults { /// Redis data type to store messages in. #[configurable_component] -#[derive(Clone, Copy, Debug, Derivative)] -#[derivative(Default)] +#[derive(Clone, Copy, Debug, Default)] #[serde(rename_all = "lowercase")] pub enum DataTypeConfig { /// The Redis `list` type. @@ -31,7 +30,7 @@ pub enum DataTypeConfig { /// This resembles a deque, where messages can be popped and pushed from either end. /// /// This is the default. - #[derivative(Default)] + #[default] List, /// The Redis `sorted set` type. @@ -48,7 +47,7 @@ pub enum DataTypeConfig { /// List-specific options. #[configurable_component] -#[derive(Clone, Copy, Debug, Derivative, Eq, PartialEq)] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] #[serde(rename_all = "lowercase")] pub struct ListOption { /// The method to use for pushing messages into a `list`. @@ -57,8 +56,7 @@ pub struct ListOption { /// Method for pushing messages into a `list`. #[configurable_component] -#[derive(Clone, Copy, Debug, Derivative, Eq, PartialEq)] -#[derivative(Default)] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] #[serde(rename_all = "lowercase")] pub enum ListMethod { /// Use the `rpush` method. @@ -66,7 +64,7 @@ pub enum ListMethod { /// This pushes messages onto the tail of the list. /// /// This is the default. - #[derivative(Default)] + #[default] RPush, /// Use the `lpush` method. @@ -77,7 +75,7 @@ pub enum ListMethod { /// Sorted Set-specific options #[configurable_component] -#[derive(Clone, Debug, Derivative, Eq, PartialEq)] +#[derive(Clone, Debug, Default, Eq, PartialEq)] #[serde(rename_all = "lowercase")] pub struct SortedSetOption { /// The method to use for pushing messages into a `sorted set`. @@ -96,8 +94,7 @@ pub struct SortedSetOption { /// Method for pushing messages into a `sorted set`. #[configurable_component] -#[derive(Clone, Copy, Debug, Derivative, Eq, PartialEq)] -#[derivative(Default)] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] #[serde(rename_all = "lowercase")] pub enum SortedSetMethod { /// Use the `zadd` method. @@ -105,7 +102,7 @@ pub enum SortedSetMethod { /// This adds messages onto a queue with a score. /// /// This is the default. - #[derivative(Default)] + #[default] ZAdd, } @@ -281,14 +278,13 @@ impl From for SentinelNodeConnectionInfo { /// How/if TLS should be established. #[configurable_component] -#[derive(Clone, Copy, Debug, Derivative, Eq, PartialEq)] -#[derivative(Default)] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] #[serde(rename_all = "lowercase")] pub enum MaybeTlsMode { /// Don't use TLS. /// /// This is the default. - #[derivative(Default)] + #[default] None, /// Enable TLS with certificate verification. @@ -311,8 +307,7 @@ impl From for Option { /// Connection independent information used to establish a connection /// to a redis instance sentinel owns. #[configurable_component] -#[derive(Clone, Debug, Derivative)] -#[derivative(Default)] +#[derive(Clone, Debug, Default)] pub struct RedisConnectionSettings { /// The database number to use. Usually `0`. pub db: i64, @@ -340,13 +335,12 @@ impl From for RedisConnectionInfo { /// The communication protocol to use with the redis server. #[configurable_component] -#[derive(Clone, Copy, Debug, Derivative, Eq, PartialEq)] -#[derivative(Default)] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] pub enum RedisProtocolVersion { /// Use RESP2. /// /// This is the default. - #[derivative(Default)] + #[default] RESP2, /// Use RESP3. diff --git a/src/sinks/redis/sink.rs b/src/sinks/redis/sink.rs index 274d2b103cf86..b3719d01e514a 100644 --- a/src/sinks/redis/sink.rs +++ b/src/sinks/redis/sink.rs @@ -139,7 +139,7 @@ impl RedisConnection { Ok(Self::Sentinel { connection_send: conn_tx, connection_recv: conn_rx, - repair_task: Arc::new(tokio::spawn(async move { + repair_task: Arc::new(crate::spawn_in_current_span(async move { Self::repair_connection_manager_task( sentinel, service_name, diff --git a/src/sinks/s3_common/config.rs b/src/sinks/s3_common/config.rs index aeb7d4b93113b..f78c649c6f667 100644 --- a/src/sinks/s3_common/config.rs +++ b/src/sinks/s3_common/config.rs @@ -141,12 +141,11 @@ fn example_tags() -> HashMap { /// /// [aws_docs]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/storage-class-intro.html #[configurable_component] -#[derive(Clone, Copy, Debug, Derivative, PartialEq, Eq)] -#[derivative(Default)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] #[serde(rename_all = "SCREAMING_SNAKE_CASE")] pub enum S3StorageClass { /// Standard Redundancy. - #[derivative(Default)] + #[default] Standard, /// Reduced Redundancy. @@ -228,8 +227,7 @@ impl From for ServerSideEncryption { /// /// [canned_acl]: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl #[configurable_component] -#[derive(Clone, Copy, Debug, Derivative)] -#[derivative(Default)] +#[derive(Clone, Copy, Debug, Default)] #[serde(rename_all = "kebab-case")] pub enum S3CannedAcl { /// Bucket/object are private. @@ -238,7 +236,7 @@ pub enum S3CannedAcl { /// access. /// /// This is the default. - #[derivative(Default)] + #[default] Private, /// Bucket/object can be read publicly. diff --git a/src/sinks/s3_common/sink.rs b/src/sinks/s3_common/sink.rs index 729a67153f641..5cad7ba89b6ee 100644 --- a/src/sinks/s3_common/sink.rs +++ b/src/sinks/s3_common/sink.rs @@ -47,7 +47,9 @@ where let request_builder = self.request_builder; input - .batched_partitioned(partitioner, || settings.as_byte_size_config()) + .batched_partitioned(partitioner, settings.timeout, |_| { + settings.as_byte_size_config() + }) .filter_map(|(key, batch)| async move { key.map(move |k| (k, batch)) }) .request_builder(default_request_builder_concurrency_limit(), request_builder) .filter_map(|request| async move { diff --git a/src/sinks/splunk_hec/common/service.rs b/src/sinks/splunk_hec/common/service.rs index 095da254d832b..3b1c68c876561 100644 --- a/src/sinks/splunk_hec/common/service.rs +++ b/src/sinks/splunk_hec/common/service.rs @@ -58,7 +58,7 @@ where let max_pending_acks = indexer_acknowledgements.max_pending_acks.get(); let tx = if let Some(ack_client) = ack_client { let (tx, rx) = mpsc::channel(128); - tokio::spawn(run_acknowledgements( + crate::spawn_in_current_span(run_acknowledgements( rx, ack_client, Arc::clone(&http_request_builder), diff --git a/src/sinks/splunk_hec/logs/sink.rs b/src/sinks/splunk_hec/logs/sink.rs index fb6742586130e..d80c44838c915 100644 --- a/src/sinks/splunk_hec/logs/sink.rs +++ b/src/sinks/splunk_hec/logs/sink.rs @@ -86,7 +86,8 @@ where } else { EventPartitioner::new(None, None, None, None) }, - || batch_settings.as_byte_size_config(), + batch_settings.timeout, + |_| batch_settings.as_byte_size_config(), ) .request_builder( default_request_builder_concurrency_limit(), diff --git a/src/sinks/splunk_hec/metrics/sink.rs b/src/sinks/splunk_hec/metrics/sink.rs index bb61a4ef1ecfb..57e05cc3953a4 100644 --- a/src/sinks/splunk_hec/metrics/sink.rs +++ b/src/sinks/splunk_hec/metrics/sink.rs @@ -53,7 +53,9 @@ where default_namespace, )) }) - .batched_partitioned(EventPartitioner, || batch_settings.as_byte_size_config()) + .batched_partitioned(EventPartitioner, batch_settings.timeout, |_| { + batch_settings.as_byte_size_config() + }) .request_builder( default_request_builder_concurrency_limit(), self.request_builder, diff --git a/src/sinks/util/adaptive_concurrency/tests.rs b/src/sinks/util/adaptive_concurrency/tests.rs index bce42b64ca156..740477d6de144 100644 --- a/src/sinks/util/adaptive_concurrency/tests.rs +++ b/src/sinks/util/adaptive_concurrency/tests.rs @@ -50,11 +50,10 @@ use crate::{ /// Request handling action when the request limit has been exceeded. #[configurable_component] -#[derive(Clone, Copy, Debug, Derivative)] -#[derivative(Default)] +#[derive(Clone, Copy, Debug, Default)] #[serde(rename_all = "lowercase")] enum Action { - #[derivative(Default)] + #[default] /// Additional requests will return with an error. Defer, diff --git a/src/sinks/util/buffer/compression.rs b/src/sinks/util/buffer/compression.rs index c990025c853d4..38e87108f7f9a 100644 --- a/src/sinks/util/buffer/compression.rs +++ b/src/sinks/util/buffer/compression.rs @@ -16,11 +16,10 @@ use vector_lib::configurable::{ use crate::sinks::util::zstd::ZstdCompressionLevel; /// Compression configuration. -#[derive(Copy, Clone, Debug, Derivative, Eq, PartialEq)] -#[derivative(Default)] +#[derive(Copy, Clone, Debug, Default, Eq, PartialEq)] pub enum Compression { /// No compression. - #[derivative(Default)] + #[default] None, /// [Gzip][gzip] compression. diff --git a/src/sinks/util/buffer/metrics/normalize.rs b/src/sinks/util/buffer/metrics/normalize.rs index c244506508f18..45026c0cc5e81 100644 --- a/src/sinks/util/buffer/metrics/normalize.rs +++ b/src/sinks/util/buffer/metrics/normalize.rs @@ -3,6 +3,7 @@ use std::{ time::{Duration, Instant}, }; +use indexmap::IndexMap; use lru::LruCache; use serde_with::serde_as; use snafu::Snafu; @@ -375,14 +376,96 @@ pub struct MetricSetSettings { pub time_to_live: Option, } -/// Dual-limit cache using standard LRU with optional capacity and TTL policies. +/// Inner storage for `MetricSet`. /// -/// This implementation uses the standard LRU crate with optional enforcement of both -/// memory and entry count limits via CapacityPolicy, plus optional TTL via TtlPolicy. +/// Uses `IndexMap` when no capacity eviction policy is configured β€” avoiding the +/// per-access LRU bookkeeping (pointer chasing in a doubly-linked list) that +/// `LruCache::get_mut` performs unconditionally. `LruCache` is used only when a +/// capacity policy is set, so that LRU eviction order is maintained correctly. +#[derive(Clone, Debug)] +enum MetricSetInner { + /// Unbounded storage with no eviction. Hash-map lookup only, no LRU overhead. + Unbounded(IndexMap), + /// Bounded storage with LRU eviction semantics. + Bounded(LruCache), +} + +impl MetricSetInner { + fn len(&self) -> usize { + match self { + Self::Unbounded(m) => m.len(), + Self::Bounded(m) => m.len(), + } + } + + fn is_empty(&self) -> bool { + match self { + Self::Unbounded(m) => m.is_empty(), + Self::Bounded(m) => m.is_empty(), + } + } + + /// Returns a mutable reference to the entry. + /// + /// For `Unbounded` this is a plain hash-map lookup. + /// For `Bounded` this also promotes the entry to the MRU end of the LRU list. + fn get_mut(&mut self, key: &MetricSeries) -> Option<&mut MetricEntry> { + match self { + Self::Unbounded(m) => m.get_mut(key), + Self::Bounded(m) => m.get_mut(key), + } + } + + /// Inserts or replaces an entry, returning the previous value if any. + fn put(&mut self, key: MetricSeries, value: MetricEntry) -> Option { + match self { + Self::Unbounded(m) => m.insert(key, value), + Self::Bounded(m) => m.put(key, value), + } + } + + /// Removes an entry by key, returning it if present. + fn pop(&mut self, key: &MetricSeries) -> Option { + match self { + // swap_remove is O(1) vs shift_remove's O(n); insertion order is not required here. + Self::Unbounded(m) => m.swap_remove(key), + Self::Bounded(m) => m.pop(key), + } + } + + fn iter(&self) -> MetricSetIter<'_> { + match self { + Self::Unbounded(m) => MetricSetIter::Unbounded(m.iter()), + Self::Bounded(m) => MetricSetIter::Bounded(m.iter()), + } + } +} + +enum MetricSetIter<'a> { + Unbounded(indexmap::map::Iter<'a, MetricSeries, MetricEntry>), + Bounded(lru::Iter<'a, MetricSeries, MetricEntry>), +} + +impl<'a> Iterator for MetricSetIter<'a> { + type Item = (&'a MetricSeries, &'a MetricEntry); + + fn next(&mut self) -> Option { + match self { + Self::Unbounded(it) => it.next(), + Self::Bounded(it) => it.next(), + } + } +} + +/// Dual-limit cache for metric normalization with optional capacity and TTL policies. +/// +/// Uses `IndexMap` internally when no capacity eviction policy is configured, avoiding +/// the per-access LRU pointer-manipulation overhead of `LruCache`. Switches to +/// `LruCache` only when a `max_bytes` or `max_events` capacity policy is set, so that +/// LRU eviction ordering is preserved for those cases. #[derive(Clone, Debug)] pub struct MetricSet { - /// LRU cache for storing metric entries - inner: LruCache, + inner: MetricSetInner, /// Optional capacity policy for memory and/or entry count limits capacity_policy: Option, /// Optional TTL policy for time-based expiration @@ -411,10 +494,15 @@ impl MetricSet { capacity_policy: Option, ttl_policy: Option, ) -> Self { - // Always use an unbounded cache since we manually track limits - // This ensures our capacity policy can properly track memory for all evicted entries + // Use LruCache only when a capacity policy requires LRU eviction ordering. + // Without a capacity policy, IndexMap avoids the per-access LRU overhead. + let inner = if capacity_policy.is_some() { + MetricSetInner::Bounded(LruCache::unbounded()) + } else { + MetricSetInner::Unbounded(IndexMap::default()) + }; Self { - inner: LruCache::unbounded(), + inner, capacity_policy, ttl_policy, } @@ -463,9 +551,15 @@ impl MetricSet { return; // No capacity limits configured }; + // A capacity policy is only set when inner is Bounded; this should always be true. + let MetricSetInner::Bounded(ref mut lru) = self.inner else { + debug_assert!(false, "capacity policy set but inner is not Bounded"); + return; + }; + // Keep evicting until we're within limits - while capacity_policy.needs_eviction(self.inner.len()) { - if let Some((series, entry)) = self.inner.pop_lru() { + while capacity_policy.needs_eviction(lru.len()) { + if let Some((series, entry)) = lru.pop_lru() { capacity_policy.free_item(&series, &entry); } else { break; // No more entries to evict @@ -497,14 +591,13 @@ impl MetricSet { return; // No TTL policy, nothing to do }; - let mut expired_keys = Vec::new(); - // Collect expired keys using the provided timestamp - for (series, entry) in self.inner.iter() { - if entry.is_expired(ttl, now) { - expired_keys.push(series.clone()); - } - } + let expired_keys: Vec = self + .inner + .iter() + .filter(|(_, e)| e.is_expired(ttl, now)) + .map(|(s, _)| s.clone()) + .collect(); // Remove expired entries and update memory tracking (if max_bytes is set) for series in expired_keys { @@ -549,11 +642,19 @@ impl MetricSet { pub fn into_metrics(mut self) -> Vec { // Clean up expired entries first (using current time) self.cleanup_expired(Instant::now()); - let mut metrics = Vec::new(); - while let Some((series, entry)) = self.inner.pop_lru() { - metrics.push(entry.into_metric(series)); + match self.inner { + MetricSetInner::Unbounded(m) => m + .into_iter() + .map(|(series, entry)| entry.into_metric(series)) + .collect(), + MetricSetInner::Bounded(mut m) => { + let mut metrics = Vec::with_capacity(m.len()); + while let Some((series, entry)) = m.pop_lru() { + metrics.push(entry.into_metric(series)); + } + metrics + } } - metrics } /// Either pass the metric through as-is if absolute, or convert it @@ -705,3 +806,56 @@ impl Default for MetricSet { Self::new(MetricSetSettings::default()) } } + +#[cfg(test)] +mod tests { + use vector_lib::event::metric::{MetricKind, MetricValue}; + + use super::*; + + fn counter(name: &str, value: f64, kind: MetricKind) -> Metric { + Metric::new(name, kind, MetricValue::Counter { value }) + } + + // Verifies that the default (no capacity policy) path uses IndexMap and that + // make_absolute / into_metrics behave correctly across multiple updates. + #[test] + fn unbounded_incremental_to_absolute_accumulates() { + let mut set = MetricSet::default(); + assert!(matches!(set.inner, MetricSetInner::Unbounded(_))); + + // First incremental: stored as reference, emitted as absolute 1.0 + let out = set.make_absolute(counter("hits", 1.0, MetricKind::Incremental)); + assert_eq!(out.unwrap().value(), &MetricValue::Counter { value: 1.0 }); + + // Second incremental: accumulated with previous, emitted as absolute 3.0 + let out = set.make_absolute(counter("hits", 2.0, MetricKind::Incremental)); + assert_eq!(out.unwrap().value(), &MetricValue::Counter { value: 3.0 }); + + // into_metrics drains the set and returns all tracked series + let metrics = set.into_metrics(); + assert_eq!(metrics.len(), 1); + assert_eq!(metrics[0].name(), "hits"); + } + + #[test] + fn unbounded_absolute_passes_through() { + let mut set = MetricSet::default(); + + let out = set.make_absolute(counter("rps", 42.0, MetricKind::Absolute)); + assert_eq!(out.unwrap().value(), &MetricValue::Counter { value: 42.0 }); + + // Absolute metrics are not stored in the set + assert!(set.is_empty()); + } + + // Verifies that capacity policy switches to the LruCache (Bounded) path. + #[test] + fn bounded_path_selected_when_capacity_policy_set() { + let set = MetricSet::new(MetricSetSettings { + max_events: Some(10), + ..Default::default() + }); + assert!(matches!(set.inner, MetricSetInner::Bounded(_))); + } +} diff --git a/src/sinks/util/builder.rs b/src/sinks/util/builder.rs index 6d2b2a68753f5..725e20d974782 100644 --- a/src/sinks/util/builder.rs +++ b/src/sinks/util/builder.rs @@ -46,9 +46,13 @@ pub trait SinkBuilderExt: Stream { /// The stream will yield batches of events, with their partition key, when either a batch fills /// up or times out. [`Partitioner`] operates on a per-event basis, and has access to the event /// itself, and so can access any and all fields of an event. + /// + /// The `settings` closure receives the partition key for each new partition, allowing callers + /// to vary the batch configuration (e.g. byte size limit) per partition. fn batched_partitioned( self, partitioner: P, + timeout: Duration, settings: F, ) -> PartitionedBatcher, C, F, B> where @@ -57,9 +61,9 @@ pub trait SinkBuilderExt: Stream { P::Key: Eq + Hash + Clone, P::Item: ByteSizeOf, C: BatchConfig, - F: Fn() -> C + Send, + F: Fn(&P::Key) -> C + Send, { - PartitionedBatcher::new(self, partitioner, settings) + PartitionedBatcher::new(self, partitioner, timeout, settings) } /// Batches the stream based on the given batch settings and item size calculator. diff --git a/src/sinks/util/encoding.rs b/src/sinks/util/encoding.rs index 4cc49b00f358b..b07e439bd0c88 100644 --- a/src/sinks/util/encoding.rs +++ b/src/sinks/util/encoding.rs @@ -99,7 +99,6 @@ impl Encoder for (Transformer, vector_lib::codecs::Encoder<()>) { } } -#[cfg(feature = "codecs-arrow")] impl Encoder> for (Transformer, vector_lib::codecs::BatchEncoder) { fn encode_input( &self, @@ -107,6 +106,7 @@ impl Encoder> for (Transformer, vector_lib::codecs::BatchEncoder) { writer: &mut dyn io::Write, ) -> io::Result<(usize, GroupedCountByteSize)> { use tokio_util::codec::Encoder as _; + use vector_lib::internal_event::{ComponentEventsDropped, UNINTENTIONAL}; let mut encoder = self.1.clone(); let mut byte_size = telemetry().create_request_count_byte_size(); @@ -122,7 +122,22 @@ impl Encoder> for (Transformer, vector_lib::codecs::BatchEncoder) { let mut bytes = BytesMut::new(); encoder .encode(transformed_events, &mut bytes) - .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?; + .map_err(|error| { + // Codec error paths emit their own internal event + // (e.g. SchemaGenerationError, EncoderNullConstraintError, + // EncoderRecordBatchError) which logs the error and increments + // component_errors_total. We only emit the drop count here to + // avoid double-counting. + // n_events is the pre-filter count; Parquet filters non-log + // events before encoding, but that only happens if a sink is + // misconfigured to send non-log events into a log-only encoder, + // so the overcount is not a practical concern. + emit!(ComponentEventsDropped:: { + count: n_events, + reason: "Failed to batch encode events.", + }); + io::Error::new(io::ErrorKind::InvalidData, error) + })?; write_all(writer, n_events, &bytes)?; Ok((bytes.len(), byte_size)) @@ -140,7 +155,6 @@ impl Encoder> for (Transformer, vector_lib::codecs::EncoderKind) { vector_lib::codecs::EncoderKind::Framed(encoder) => { (self.0.clone(), *encoder.clone()).encode_input(events, writer) } - #[cfg(feature = "codecs-arrow")] vector_lib::codecs::EncoderKind::Batch(encoder) => { (self.0.clone(), encoder.clone()).encode_input(events, writer) } @@ -202,6 +216,7 @@ mod tests { use std::{collections::BTreeMap, env, path::PathBuf}; use bytes::{BufMut, Bytes}; + use cfg_if::cfg_if; use vector_lib::{ codecs::{ CharacterDelimitedEncoder, JsonSerializerConfig, LengthDelimitedEncoder, @@ -214,6 +229,17 @@ mod tests { }; use vrl::value::{KeyString, Value}; + cfg_if! { + if #[cfg(feature = "codecs-arrow")] { + use arrow::datatypes::{DataType, Field, Schema as ArrowSchema}; + use vector_lib::codecs::{ + BatchEncoder, + encoding::{ArrowStreamSerializer, ArrowStreamSerializerConfig, BatchSerializer}, + }; + use vector_lib::event_test_util::{clear_recorded_events, contains_name_once}; + } + } + use super::*; #[test] @@ -558,4 +584,35 @@ mod tests { assert_eq!(CountByteSize(2, input_json_size), size.size().unwrap()); assert_eq!(Bytes::copy_from_slice(&writer), expected_bytes); } + + #[cfg(feature = "codecs-arrow")] + #[test] + fn test_encode_batch_arrow_emits_record_batch_error_on_type_mismatch() { + clear_recorded_events(); + + // Schema declares `message` as Int64, but the event below carries a string, + // so `build_record_batch` returns `ArrowEncodingError::ArrowJsonDecode`. + let schema = ArrowSchema::new(vec![Field::new("message", DataType::Int64, false)]); + let serializer = ArrowStreamSerializer::new(ArrowStreamSerializerConfig::new(schema)) + .expect("failed to build ArrowStreamSerializer"); + let encoder = BatchEncoder::new(BatchSerializer::Arrow(serializer)); + let encoding = (Transformer::default(), encoder); + + let event = Event::Log(LogEvent::from(BTreeMap::from([( + KeyString::from("message"), + Value::from("not_an_integer"), + )]))); + + let mut writer = Vec::new(); + let result = encoding.encode_input(vec![event], &mut writer); + assert!( + result.is_err(), + "type mismatch should fail batch encoding, got {result:?}" + ); + + contains_name_once("EncoderRecordBatchError") + .expect("EncoderRecordBatchError should be emitted on ArrowJsonDecode failure"); + contains_name_once("ComponentEventsDropped") + .expect("ComponentEventsDropped should be emitted by the wrapper"); + } } diff --git a/src/sinks/util/http.rs b/src/sinks/util/http.rs index e21ef48e4ed66..89225f4077741 100644 --- a/src/sinks/util/http.rs +++ b/src/sinks/util/http.rs @@ -7,6 +7,7 @@ use futures::{Sink, future::BoxFuture}; use headers::HeaderName; use http::{HeaderValue, Request, Response, StatusCode, header}; use http_body::Body as _; +use tracing::debug; #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct OrderedHeaderName(HeaderName); @@ -550,14 +551,137 @@ impl sink::Response for http::Response { } } +/// Serializes and deserializes a [`Vec`] +mod status_code_vec { + use http::StatusCode; + use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error}; + + /// Deserializes a [`Vec`] + pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + Vec::::deserialize(deserializer)? + .into_iter() + .map(StatusCode::from_u16) + .collect::, _>>() + .map_err(Error::custom) + } + + /// Serializes a [`Vec`] + pub fn serialize(status_codes: &[StatusCode], serializer: S) -> Result + where + S: Serializer, + { + status_codes + .iter() + .map(StatusCode::as_u16) + .collect::>() + .serialize(serializer) + } +} + +/// Configurable retry strategy for `http` based sinks. +/// +/// For more information about error responses, see [Client Error Responses][error_responses]. +/// +/// [error_responses]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Status#client_error_responses +#[configurable_component] +#[derive(Debug, Clone, Default, PartialEq)] +#[serde(tag = "type", rename_all = "snake_case")] +#[configurable(metadata(docs::enum_tag_description = "The retry strategy enum."))] +pub enum RetryStrategy { + /// Don't retry any errors, including request timeouts. + None, + + /// Default strategy. See [`RetryStrategy::retry_action`] for more details. + #[default] + Default, + + /// Retry on *all* HTTP status codes except for success codes (2xx) + All, + + /// Custom retry strategy + Custom { + /// Retry on these specific HTTP status codes + #[serde(with = "status_code_vec")] + status_codes: Vec, + }, +} + +impl RetryStrategy { + /// Returns the name of the retry strategy. + #[must_use] + const fn name(&self) -> &str { + match self { + Self::None => "Never retry strategy", + Self::Default => "Default retry strategy", + Self::All => "Retry all strategy", + Self::Custom { .. } => "Custom retry strategy", + } + } + + /// Determines if the given status code should be retried. + /// + /// For the `Default` strategy, the following status codes will be retried: + /// - 429 (Too Many Requests) + /// - 408 (Request Timeout) + /// - 5xx (Server Error) + /// + /// For the `Custom` strategy, the status codes specified in the `status_codes` field will be retried. + /// + /// For the `All` strategy, all non-success status codes will be retried. + #[must_use] + pub fn retry_action(&self, status: http::StatusCode) -> RetryAction { + if status.is_success() { + return RetryAction::Successful; + } + + let reason = format!( + "{}: {}", + self.name(), + status.canonical_reason().unwrap_or_else(|| status.as_str()) + ) + .into(); + + match self { + Self::None => RetryAction::DontRetry(reason), + Self::Default => match status { + StatusCode::TOO_MANY_REQUESTS | StatusCode::REQUEST_TIMEOUT => { + RetryAction::Retry(reason) + } + StatusCode::NOT_IMPLEMENTED => RetryAction::DontRetry(reason), + _ => { + if status.is_server_error() { + RetryAction::Retry(reason) + } else { + RetryAction::DontRetry(reason) + } + } + }, + Self::All => RetryAction::Retry(reason), + Self::Custom { status_codes } => { + if status_codes.contains(&status) { + RetryAction::Retry(reason) + } else { + RetryAction::DontRetry(reason) + } + } + } + } +} + #[derive(Debug, Clone)] pub struct HttpRetryLogic { request: PhantomData, + retry_strategy: RetryStrategy, } + impl Default for HttpRetryLogic { fn default() -> Self { Self { request: PhantomData, + retry_strategy: RetryStrategy::Default, } } } @@ -567,25 +691,28 @@ impl RetryLogic for HttpRetryLogic { type Request = Req; type Response = hyper::Response; - fn is_retriable_error(&self, _error: &Self::Error) -> bool { - true + fn is_retriable_error(&self, error: &Self::Error) -> bool { + if self.retry_strategy == RetryStrategy::None { + false + } else { + error.is_retriable() + } + } + + fn is_retriable_timeout(&self) -> bool { + self.retry_strategy != RetryStrategy::None } fn should_retry_response(&self, response: &Self::Response) -> RetryAction { let status = response.status(); - - match status { - StatusCode::TOO_MANY_REQUESTS => RetryAction::Retry("too many requests".into()), - StatusCode::REQUEST_TIMEOUT => RetryAction::Retry("request timeout".into()), - StatusCode::NOT_IMPLEMENTED => { - RetryAction::DontRetry("endpoint not implemented".into()) - } - _ if status.is_server_error() => RetryAction::Retry( - format!("{}: {}", status, String::from_utf8_lossy(response.body())).into(), - ), - _ if status.is_success() => RetryAction::Successful, - _ => RetryAction::DontRetry(format!("response status: {status}").into()), + if !status.is_success() { + debug!( + message = "HTTP response.", + %status, + body = %String::from_utf8_lossy(response.body()), + ); } + self.retry_strategy.retry_action(status) } } @@ -596,6 +723,7 @@ pub struct HttpStatusRetryLogic { func: F, request: PhantomData, response: PhantomData, + retry_strategy: RetryStrategy, } impl HttpStatusRetryLogic @@ -604,11 +732,12 @@ where Req: Send + Sync + 'static, Res: Send + Sync + 'static, { - pub const fn new(func: F) -> HttpStatusRetryLogic { + pub const fn new(func: F, retry_strategy: RetryStrategy) -> HttpStatusRetryLogic { HttpStatusRetryLogic { func, request: PhantomData, response: PhantomData, + retry_strategy, } } } @@ -623,25 +752,21 @@ where type Request = Req; type Response = Res; - fn is_retriable_error(&self, _error: &Self::Error) -> bool { - true + fn is_retriable_error(&self, error: &Self::Error) -> bool { + if self.retry_strategy == RetryStrategy::None { + false + } else { + error.is_retriable() + } + } + + fn is_retriable_timeout(&self) -> bool { + self.retry_strategy != RetryStrategy::None } fn should_retry_response(&self, response: &Res) -> RetryAction { let status = (self.func)(response); - - match status { - StatusCode::TOO_MANY_REQUESTS => RetryAction::Retry("too many requests".into()), - StatusCode::REQUEST_TIMEOUT => RetryAction::Retry("request timeout".into()), - StatusCode::NOT_IMPLEMENTED => { - RetryAction::DontRetry("endpoint not implemented".into()) - } - _ if status.is_server_error() => { - RetryAction::Retry(format!("Http Status: {status}").into()) - } - _ if status.is_success() => RetryAction::Successful, - _ => RetryAction::DontRetry(format!("Http status: {status}").into()), - } + self.retry_strategy.retry_action(status) } } @@ -654,6 +779,7 @@ where func: self.func.clone(), request: PhantomData, response: PhantomData, + retry_strategy: self.retry_strategy.clone(), } } } @@ -684,13 +810,6 @@ fn headers_examples() -> BTreeMap { } impl RequestConfig { - pub fn add_old_option(&mut self, headers: Option>) { - if let Some(headers) = headers { - warn!("Option `headers` has been deprecated. Use `request.headers` instead."); - self.headers.extend(headers); - } - } - pub fn split_headers(&self) -> (BTreeMap, BTreeMap) { let mut static_headers = BTreeMap::new(); let mut template_headers = BTreeMap::new(); @@ -827,12 +946,17 @@ impl DriverResponse for HttpResponse { } /// Creates a `RetryLogic` for use with `HttpResponse`. -pub fn http_response_retry_logic() -> HttpStatusRetryLogic< +pub fn http_response_retry_logic( + retry_strategy: RetryStrategy, +) -> HttpStatusRetryLogic< impl Fn(&HttpResponse) -> StatusCode + Clone + Send + Sync + 'static, Request, HttpResponse, > { - HttpStatusRetryLogic::new(|req: &HttpResponse| req.http_response.status()) + HttpStatusRetryLogic::new( + |req: &HttpResponse| req.http_response.status(), + retry_strategy, + ) } /// Uses the estimated json encoded size to determine batch sizing. @@ -976,6 +1100,93 @@ mod test { ); } + #[test] + fn retry_strategy_none_preserves_success_and_rejects_failures() { + let strategy = RetryStrategy::None; + + assert!(strategy.retry_action::<()>(StatusCode::OK).is_successful()); + assert!( + strategy + .retry_action::<()>(StatusCode::INTERNAL_SERVER_ERROR) + .is_not_retryable() + ); + } + + #[test] + fn retry_strategy_none_disables_timeout_retries() { + let logic = HttpRetryLogic::<()> { + request: PhantomData, + retry_strategy: RetryStrategy::None, + }; + let status_logic = + HttpStatusRetryLogic::<_, (), ()>::new(|_: &()| StatusCode::OK, RetryStrategy::None); + + assert!(!logic.is_retriable_timeout()); + assert!(!status_logic.is_retriable_timeout()); + } + + #[test] + fn retry_strategy_all_preserves_success_and_retries_failures() { + let strategy = RetryStrategy::All; + + assert!(strategy.retry_action::<()>(StatusCode::OK).is_successful()); + assert!( + strategy + .retry_action::<()>(StatusCode::BAD_REQUEST) + .is_retryable() + ); + assert!( + strategy + .retry_action::<()>(StatusCode::INTERNAL_SERVER_ERROR) + .is_retryable() + ); + } + + #[test] + fn retry_strategy_custom_only_retries_configured_statuses() { + let strategy = RetryStrategy::Custom { + status_codes: vec![StatusCode::BAD_REQUEST], + }; + + assert!(strategy.retry_action::<()>(StatusCode::OK).is_successful()); + assert!( + strategy + .retry_action::<()>(StatusCode::BAD_REQUEST) + .is_retryable() + ); + assert!( + strategy + .retry_action::<()>(StatusCode::INTERNAL_SERVER_ERROR) + .is_not_retryable() + ); + } + + #[test] + fn retry_strategy_custom_serde_roundtrips_status_codes() { + let json = r#"{"type":"custom","status_codes":[400,503]}"#; + let strategy: RetryStrategy = serde_json::from_str(json).unwrap(); + assert_eq!( + strategy, + RetryStrategy::Custom { + status_codes: vec![StatusCode::BAD_REQUEST, StatusCode::SERVICE_UNAVAILABLE], + } + ); + let encoded = serde_json::to_string(&strategy).unwrap(); + let roundtrip: RetryStrategy = serde_json::from_str(&encoded).unwrap(); + assert_eq!(roundtrip, strategy); + } + + #[test] + fn retry_strategy_custom_serde_rejects_invalid_status_codes() { + // `http::StatusCode::from_u16` only accepts 100–999; 1000 is out of range. + let json = r#"{"type":"custom","status_codes":[1000]}"#; + let result = serde_json::from_str::(json); + assert!( + result.is_err(), + "expected invalid status code to fail deserialization" + ); + } + #[tokio::test] async fn util_http_it_makes_http_requests() { let (_guard, addr) = next_addr(); diff --git a/src/sinks/util/retries.rs b/src/sinks/util/retries.rs index be05d559bb18c..2a38a2b4ee900 100644 --- a/src/sinks/util/retries.rs +++ b/src/sinks/util/retries.rs @@ -34,6 +34,12 @@ pub trait RetryLogic: Clone + Send + Sync + 'static { /// implementors to specify what kinds of errors can be retried. fn is_retriable_error(&self, error: &Self::Error) -> bool; + /// When the Service call times out, this function allows implementors to + /// specify if the timeout should be retried. + fn is_retriable_timeout(&self) -> bool { + true + } + /// When the Service call returns an `Ok` response, this function allows /// implementors to specify additional logic to determine if the success response /// is actually an error. This is particularly useful when the downstream service @@ -197,10 +203,18 @@ where None } } else if error.downcast_ref::().is_some() { - warn!( - "Request timed out. If this happens often while the events are actually reaching their destination, try decreasing `batch.max_bytes` and/or using `compression` if applicable. Alternatively `request.timeout_secs` can be increased." - ); - Some(self.build_retry()) + if self.logic.is_retriable_timeout() { + warn!( + "Request timed out. If this happens often while the events are actually reaching their destination, try decreasing `batch.max_bytes` and/or using `compression` if applicable. Alternatively `request.timeout_secs` can be increased." + ); + Some(self.build_retry()) + } else { + error!( + message = + "Request timed out and is not retriable; dropping the request." + ); + None + } } else { error!( message = "Unexpected error type; dropping the request.", @@ -338,6 +352,27 @@ mod tests { assert_eq!(fut.await.unwrap(), "world"); } + #[tokio::test] + async fn timeout_error_no_retry() { + trace_init(); + + let policy = FibonacciRetryPolicy::new( + 5, + Duration::from_secs(1), + Duration::from_secs(10), + NoTimeoutRetryLogic, + JitterMode::None, + ); + + let (mut svc, mut handle) = mock::spawn_layer(RetryLayer::new(policy)); + + assert_ready_ok!(svc.poll_ready()); + + let mut fut = task::spawn(svc.call("hello")); + assert_request_eq!(handle, "hello").send_error(Elapsed::new()); + assert_ready_err!(fut.poll()); + } + #[test] fn backoff_grows_to_max() { let mut policy = FibonacciRetryPolicy::new( @@ -425,6 +460,23 @@ mod tests { } } + #[derive(Debug, Clone)] + struct NoTimeoutRetryLogic; + + impl RetryLogic for NoTimeoutRetryLogic { + type Error = Error; + type Request = &'static str; + type Response = &'static str; + + fn is_retriable_error(&self, error: &Self::Error) -> bool { + error.0 + } + + fn is_retriable_timeout(&self) -> bool { + false + } + } + #[derive(Debug)] struct Error(bool); diff --git a/src/sinks/util/service/concurrency.rs b/src/sinks/util/service/concurrency.rs index 44c982caff0f9..43c2643807a99 100644 --- a/src/sinks/util/service/concurrency.rs +++ b/src/sinks/util/service/concurrency.rs @@ -18,7 +18,7 @@ use vector_lib::configurable::{ /// /// This can be set either to one of the below enum values or to a positive integer, which denotes /// a fixed concurrency limit. -#[derive(Clone, Copy, Debug, Derivative, Eq, PartialEq, Default)] +#[derive(Clone, Copy, Debug, Eq, PartialEq, Default)] pub enum Concurrency { /// A fixed concurrency of 1. /// diff --git a/src/sinks/util/sink.rs b/src/sinks/util/sink.rs index f8dd31c67f129..e9c35c94f3e70 100644 --- a/src/sinks/util/sink.rs +++ b/src/sinks/util/sink.rs @@ -323,7 +323,7 @@ where this.lingers.remove(partition); let batch = batch.finish(); - let future = tokio::spawn(this.service.call(batch)); + let future = crate::spawn_in_current_span(this.service.call(batch)); if let Some(map) = this.in_flight.as_mut() { map.insert(partition.clone(), future.map(|_| ()).fuse().boxed()); diff --git a/src/sinks/vector/compression.rs b/src/sinks/vector/compression.rs new file mode 100644 index 0000000000000..9165d93934985 --- /dev/null +++ b/src/sinks/vector/compression.rs @@ -0,0 +1,143 @@ +use serde::{Deserialize, Deserializer, de}; +use vector_lib::configurable::configurable_component; + +/// Compression configuration for the Vector sink. +/// +/// Only `gzip` and `zstd` are supported as compression algorithms for the +/// Vector sink's gRPC transport. Compression levels are not configurable +/// as the underlying tonic library does not support them. +#[configurable_component] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +#[serde(rename_all = "snake_case")] +#[configurable(metadata( + docs::enum_tag_description = "The compression algorithm to use for sending." +))] +pub enum VectorCompression { + /// No compression. + #[default] + None, + + /// [Gzip][gzip] compression. + /// + /// [gzip]: https://www.gzip.org/ + Gzip, + + /// [Zstandard][zstd] compression. + /// + /// [zstd]: https://facebook.github.io/zstd/ + Zstd, +} + +impl VectorCompression { + /// Returns the corresponding `tonic::codec::CompressionEncoding`, if any. + pub const fn as_tonic_encoding(self) -> Option { + match self { + VectorCompression::None => Option::None, + VectorCompression::Gzip => Some(tonic::codec::CompressionEncoding::Gzip), + VectorCompression::Zstd => Some(tonic::codec::CompressionEncoding::Zstd), + } + } +} + +/// Enables deserializing compression from a bool (legacy) or string (new). +/// +/// For backward compatibility: +/// - `true` maps to `VectorCompression::Gzip` +/// - `false` maps to `VectorCompression::None` +/// +/// New syntax: +/// - `"none"`, `"gzip"`, `"zstd"` as strings +pub fn bool_or_vector_compression<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + struct BoolOrVectorCompression; + + impl<'de> de::Visitor<'de> for BoolOrVectorCompression { + type Value = VectorCompression; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("boolean (deprecated) or string (\"none\", \"gzip\", or \"zstd\")") + } + + fn visit_bool(self, value: bool) -> Result + where + E: de::Error, + { + if value { + Ok(VectorCompression::Gzip) + } else { + Ok(VectorCompression::None) + } + } + + fn visit_str(self, value: &str) -> Result + where + E: de::Error, + { + VectorCompression::deserialize(de::value::StrDeserializer::new(value)) + } + } + + deserializer.deserialize_any(BoolOrVectorCompression) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[derive(Deserialize)] + struct TestConfig { + #[serde(deserialize_with = "bool_or_vector_compression")] + compression: VectorCompression, + } + + #[test] + fn test_legacy_true() { + let json = r#"{"compression": true}"#; + let result: TestConfig = serde_json::from_str(json).unwrap(); + assert_eq!(result.compression, VectorCompression::Gzip); + } + + #[test] + fn test_legacy_false() { + let json = r#"{"compression": false}"#; + let result: TestConfig = serde_json::from_str(json).unwrap(); + assert_eq!(result.compression, VectorCompression::None); + } + + #[test] + fn test_string_gzip() { + let json = r#"{"compression": "gzip"}"#; + let result: TestConfig = serde_json::from_str(json).unwrap(); + assert_eq!(result.compression, VectorCompression::Gzip); + } + + #[test] + fn test_string_zstd() { + let json = r#"{"compression": "zstd"}"#; + let result: TestConfig = serde_json::from_str(json).unwrap(); + assert_eq!(result.compression, VectorCompression::Zstd); + } + + #[test] + fn test_string_none() { + let json = r#"{"compression": "none"}"#; + let result: TestConfig = serde_json::from_str(json).unwrap(); + assert_eq!(result.compression, VectorCompression::None); + } + + #[test] + fn test_unsupported_algorithm_rejected() { + let json = r#"{"compression": "snappy"}"#; + let result = serde_json::from_str::(json); + assert!(result.is_err()); + } + + #[test] + fn test_object_syntax_rejected() { + let json = r#"{"compression": {"algorithm": "zstd", "level": 3}}"#; + let result = serde_json::from_str::(json); + assert!(result.is_err()); + } +} diff --git a/src/sinks/vector/config.rs b/src/sinks/vector/config.rs index 52601115cb67a..ae2847073a649 100644 --- a/src/sinks/vector/config.rs +++ b/src/sinks/vector/config.rs @@ -8,6 +8,7 @@ use vector_lib::configurable::configurable_component; use super::{ VectorSinkError, + compression::VectorCompression, service::{VectorRequest, VectorResponse, VectorService}, sink::VectorSink, }; @@ -49,14 +50,19 @@ pub struct VectorConfig { #[configurable(metadata(docs::examples = "https://somehost:6000"))] address: String, - /// Whether or not to compress requests. + /// Compression algorithm for requests. /// - /// If set to `true`, requests are compressed with [`gzip`][gzip_docs]. + /// Supports `"none"`, `"gzip"`, or `"zstd"`. /// - /// [gzip_docs]: https://www.gzip.org/ - #[configurable(metadata(docs::advanced))] - #[serde(default)] - compression: bool, + /// For backward compatibility, boolean values are still accepted: + /// - `true` defaults to gzip compression + /// - `false` disables compression (deprecated syntax) + #[configurable(derived)] + #[serde( + default, + deserialize_with = "super::compression::bool_or_vector_compression" + )] + compression: VectorCompression, #[configurable(derived)] #[serde(default)] @@ -97,7 +103,7 @@ fn default_config(address: &str) -> VectorConfig { VectorConfig { version: None, address: address.to_owned(), - compression: false, + compression: VectorCompression::None, batch: BatchConfig::default(), request: TowerRequestConfig::default(), tls: None, @@ -120,7 +126,8 @@ impl SinkConfig for VectorConfig { .clone() .map(|uri| uri.uri) .unwrap_or_else(|| uri.clone()); - let healthcheck_client = VectorService::new(client.clone(), healthcheck_uri, false); + let healthcheck_client = + VectorService::new(client.clone(), healthcheck_uri, VectorCompression::None); let healthcheck = healthcheck(healthcheck_client, cx.healthcheck); let service = VectorService::new(client, uri, self.compression); let request_settings = self.request.into_settings(); diff --git a/src/sinks/vector/mod.rs b/src/sinks/vector/mod.rs index 4cf882b1d2ee6..c3abd90d6b629 100644 --- a/src/sinks/vector/mod.rs +++ b/src/sinks/vector/mod.rs @@ -1,6 +1,7 @@ use snafu::Snafu; use vector_lib::configurable::configurable_component; +mod compression; mod config; mod service; mod sink; @@ -79,11 +80,23 @@ mod tests { } async fn run_sink_test(test_type: TestType) { + run_sink_test_with_compression(test_type, None).await; + } + + async fn run_sink_test_with_compression(test_type: TestType, compression: Option<&str>) { let num_lines = 10; let (_guard, in_addr) = next_addr(); - let config = format!(r#"address = "http://{in_addr}/""#); + let config = match compression { + Some(c) => format!( + r#" + address = "http://{in_addr}/" + compression = "{c}" + "# + ), + None => format!(r#"address = "http://{in_addr}/""#), + }; let config: VectorConfig = toml::from_str(&config).unwrap(); let cx = SinkContext::default(); @@ -115,13 +128,29 @@ mod tests { assert_eq!(receiver.try_recv(), Ok(BatchStatus::Delivered)); - let output_lines = get_received(rx, |parts| { + let expected_encoding = compression; + let output_lines = get_received(rx, move |parts| { assert_eq!(Method::POST, parts.method); assert_eq!("/vector.Vector/PushEvents", parts.uri.path()); assert_eq!( "application/grpc", parts.headers.get("content-type").unwrap().to_str().unwrap() ); + match expected_encoding { + Some(enc) => assert_eq!( + enc, + parts + .headers + .get("grpc-encoding") + .unwrap_or_else(|| panic!("missing grpc-encoding header (expected {enc})")) + .to_str() + .unwrap() + ), + None => assert!( + parts.headers.get("grpc-encoding").is_none(), + "unexpected grpc-encoding header present" + ), + } }) .await; @@ -134,6 +163,21 @@ mod tests { run_sink_test(TestType::Normal).await; } + #[tokio::test] + async fn deliver_message_gzip() { + run_sink_test_with_compression(TestType::Normal, Some("gzip")).await; + } + + #[tokio::test] + async fn deliver_message_zstd() { + run_sink_test_with_compression(TestType::Normal, Some("zstd")).await; + } + + #[tokio::test] + async fn deliver_message_none() { + run_sink_test_with_compression(TestType::Normal, None).await; + } + #[tokio::test] async fn data_volume_tags() { init_telemetry( @@ -197,9 +241,35 @@ mod tests { assert_parts: impl Fn(Parts), ) -> Vec { rx.map(|(parts, body)| { + let encoding = parts + .headers + .get("grpc-encoding") + .map(|v| v.to_str().unwrap().to_owned()); assert_parts(parts); + let compressed = body[0] == 1; let proto_body = body.slice(GRPC_HEADER_SIZE..); + let proto_body = if compressed { + use std::io::Read; + let mut out = Vec::new(); + match encoding.as_deref() { + Some("gzip") => { + flate2::read::GzDecoder::new(&proto_body[..]) + .read_to_end(&mut out) + .unwrap(); + } + Some("zstd") => { + zstd::stream::read::Decoder::new(&proto_body[..]) + .unwrap() + .read_to_end(&mut out) + .unwrap(); + } + other => panic!("unexpected grpc-encoding for compressed frame: {other:?}"), + } + Bytes::from(out) + } else { + proto_body + }; let req = proto::PushEventsRequest::decode(proto_body).unwrap(); diff --git a/src/sinks/vector/service.rs b/src/sinks/vector/service.rs index 471b3a98ed6b1..9c3d09fb0e541 100644 --- a/src/sinks/vector/service.rs +++ b/src/sinks/vector/service.rs @@ -13,7 +13,7 @@ use vector_lib::{ stream::DriverResponse, }; -use super::VectorSinkError; +use super::{VectorSinkError, compression::VectorCompression}; use crate::{ Error, event::{EventFinalizers, EventStatus, Finalizable}, @@ -70,7 +70,7 @@ impl VectorService { pub fn new( hyper_client: hyper::Client>, BoxBody>, uri: Uri, - compression: bool, + compression: VectorCompression, ) -> Self { let (protocol, endpoint) = uri::protocol_endpoint(uri.clone()); let mut proto_client = proto_vector::Client::new(HyperSvc { @@ -78,9 +78,10 @@ impl VectorService { client: hyper_client, }); - if compression { - proto_client = proto_client.send_compressed(tonic::codec::CompressionEncoding::Gzip); + if let Some(encoding) = compression.as_tonic_encoding() { + proto_client = proto_client.send_compressed(encoding); } + Self { client: proto_client, protocol, diff --git a/src/sinks/websocket_server/sink.rs b/src/sinks/websocket_server/sink.rs index 77da730fbbdda..ea38d88fdac7f 100644 --- a/src/sinks/websocket_server/sink.rs +++ b/src/sinks/websocket_server/sink.rs @@ -37,7 +37,6 @@ use tokio_tungstenite::tungstenite::{ handshake::server::{ErrorResponse, Request, Response}, }; use tokio_util::codec::Encoder as _; -use tracing::Instrument; use url::Url; use uuid::Uuid; use vector_lib::{ @@ -129,20 +128,17 @@ impl WebSocketListenerSink { let open_gauge = OpenGauge::new(); while let Ok(stream) = listener.accept().await { - tokio::spawn( - Self::handle_connection( - auth.clone(), - message_buffering.clone(), - subprotocol.clone(), - Arc::clone(&peers), - Arc::clone(&client_checkpoints), - Arc::clone(&buffer), - stream, - extra_tags_config.clone(), - open_gauge.clone(), - ) - .in_current_span(), - ); + crate::spawn_in_current_span(Self::handle_connection( + auth.clone(), + message_buffering.clone(), + subprotocol.clone(), + Arc::clone(&peers), + Arc::clone(&client_checkpoints), + Arc::clone(&buffer), + stream, + extra_tags_config.clone(), + open_gauge.clone(), + )); } } @@ -360,19 +356,16 @@ impl StreamSink for WebSocketListenerSink { ))); let client_checkpoints = Arc::new(Mutex::new(HashMap::default())); - tokio::spawn( - Self::handle_connections( - self.auth, - self.message_buffering.clone(), - self.subprotocol.clone(), - Arc::clone(&peers), - self.extra_tags_config, - Arc::clone(&client_checkpoints), - Arc::clone(&message_buffer), - listener, - ) - .in_current_span(), - ); + crate::spawn_in_current_span(Self::handle_connections( + self.auth, + self.message_buffering.clone(), + self.subprotocol.clone(), + Arc::clone(&peers), + self.extra_tags_config, + Arc::clone(&client_checkpoints), + Arc::clone(&message_buffer), + listener, + )); while input.as_mut().peek().await.is_some() { let mut event = input.next().await.unwrap(); diff --git a/src/sources/aws_kinesis_firehose/mod.rs b/src/sources/aws_kinesis_firehose/mod.rs index 4c86b74165e9b..24809473cdfb9 100644 --- a/src/sources/aws_kinesis_firehose/mod.rs +++ b/src/sources/aws_kinesis_firehose/mod.rs @@ -113,9 +113,8 @@ const fn access_keys_example() -> [&'static str; 2] { /// Compression scheme for records in a Firehose message. #[configurable_component] #[configurable(metadata(docs::advanced))] -#[derive(Clone, Copy, Debug, Derivative, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] #[serde(rename_all = "lowercase")] -#[derivative(Default)] pub enum Compression { /// Automatically attempt to determine the compression scheme. /// @@ -127,7 +126,7 @@ pub enum Compression { /// set `gzip` in this field so that any records that are not-gzipped are rejected. /// /// [magic_bytes]: https://en.wikipedia.org/wiki/List_of_file_signatures - #[derivative(Default)] + #[default] Auto, /// Uncompressed. diff --git a/src/sources/aws_s3/mod.rs b/src/sources/aws_s3/mod.rs index 1f2532bd51a24..38ca6e4cb6901 100644 --- a/src/sources/aws_s3/mod.rs +++ b/src/sources/aws_s3/mod.rs @@ -34,9 +34,8 @@ pub mod sqs; /// Compression scheme for objects retrieved from S3. #[configurable_component] #[configurable(metadata(docs::advanced))] -#[derive(Clone, Copy, Debug, Derivative, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] #[serde(rename_all = "lowercase")] -#[derivative(Default)] pub enum Compression { /// Automatically attempt to determine the compression scheme. /// @@ -44,7 +43,7 @@ pub enum Compression { /// `Content-Type` metadata, as well as the key suffix (for example, `.gz`). /// /// It is set to `none` if the compression scheme cannot be determined. - #[derivative(Default)] + #[default] Auto, /// Uncompressed. @@ -59,14 +58,13 @@ pub enum Compression { /// Strategies for consuming objects from AWS S3. #[configurable_component] -#[derive(Clone, Copy, Debug, Derivative)] +#[derive(Clone, Copy, Debug, Default)] #[serde(rename_all = "lowercase")] -#[derivative(Default)] enum Strategy { /// Consumes objects by processing bucket notification events sent to an [AWS SQS queue][aws_sqs]. /// /// [aws_sqs]: https://aws.amazon.com/sqs/ - #[derivative(Default)] + #[default] Sqs, } diff --git a/src/sources/aws_s3/sqs.rs b/src/sources/aws_s3/sqs.rs index 9ad5b47be61cc..802bdf5d38f0e 100644 --- a/src/sources/aws_s3/sqs.rs +++ b/src/sources/aws_s3/sqs.rs @@ -28,7 +28,6 @@ use smallvec::SmallVec; use snafu::{ResultExt, Snafu}; use tokio::{pin, select}; use tokio_util::codec::FramedRead; -use tracing::Instrument; use vector_lib::{ codecs::decoding::FramingError, config::{LegacyKey, LogNamespace, log_schema}, @@ -358,7 +357,7 @@ impl Ingestor { acknowledgements, ); let fut = process.run(); - let handle = tokio::spawn(fut.in_current_span()); + let handle = crate::spawn_in_current_span(fut); handles.push(handle); } diff --git a/src/sources/aws_sqs/source.rs b/src/sources/aws_sqs/source.rs index f8afdb9365ca9..7dc7c01e735e0 100644 --- a/src/sources/aws_sqs/source.rs +++ b/src/sources/aws_sqs/source.rs @@ -7,7 +7,6 @@ use aws_sdk_sqs::{ use chrono::{DateTime, TimeZone, Utc}; use futures::{FutureExt, StreamExt}; use tokio::{pin, select}; -use tracing_futures::Instrument; use vector_lib::{ config::LogNamespace, finalizer::UnorderedFinalizer, @@ -50,16 +49,13 @@ impl SqsSource { let (finalizer, mut ack_stream) = Finalizer::new(Some(shutdown.clone())); let client = self.client.clone(); let queue_url = self.queue_url.clone(); - tokio::spawn( - async move { - while let Some((status, receipts)) = ack_stream.next().await { - if status == BatchStatus::Delivered { - delete_messages(client.clone(), receipts, queue_url.clone()).await; - } + crate::spawn_in_current_span(async move { + while let Some((status, receipts)) = ack_stream.next().await { + if status == BatchStatus::Delivered { + delete_messages(client.clone(), receipts, queue_url.clone()).await; } } - .in_current_span(), - ); + }); Arc::new(finalizer) }); let events_received = register!(EventsReceived); @@ -70,19 +66,16 @@ impl SqsSource { let mut out = out.clone(); let finalizer = finalizer.clone(); let events_received = events_received.clone(); - task_handles.push(tokio::spawn( - async move { - let finalizer = finalizer.as_ref(); - pin!(shutdown); - loop { - select! { - _ = &mut shutdown => break, - _ = source.run_once(&mut out, finalizer, events_received.clone()) => {}, - } + task_handles.push(crate::spawn_in_current_span(async move { + let finalizer = finalizer.as_ref(); + pin!(shutdown); + loop { + select! { + _ = &mut shutdown => break, + _ = source.run_once(&mut out, finalizer, events_received.clone()) => {}, } } - .in_current_span(), - )); + })); } // Wait for all of the processes to finish. If any one of them panics, we resume diff --git a/src/sources/azure_blob/integration_tests.rs b/src/sources/azure_blob/integration_tests.rs new file mode 100644 index 0000000000000..d53af33940fa8 --- /dev/null +++ b/src/sources/azure_blob/integration_tests.rs @@ -0,0 +1,1058 @@ +use azure_core::http::{RequestContent, StatusCode}; +use azure_storage_blob::BlobContainerClient; +use azure_storage_queue::{QueueClient, models::QueueMessage}; +use base64::{Engine, prelude::BASE64_STANDARD}; +use flate2::{Compression as GzCompression, write::GzEncoder}; +use std::sync::Arc; +use std::{io::Write, time::Duration}; + +use super::{ + AzureBlobConfig, Compression, + queue::{Config, make_container_client, make_queue_client}, +}; +use crate::{ + config::ProxyConfig, + event::Event, + serde::default_decoding, + test_util::components::{ + COMPONENT_ERROR_TAGS, SOURCE_TAGS, run_and_assert_source_compliance, + run_and_assert_source_error, + }, +}; + +/// Test helper: build a container client with default proxy settings. +async fn test_container_client(config: &AzureBlobConfig) -> Arc { + make_container_client(config, &ProxyConfig::default()) + .await + .expect("Failed to create container client") +} + +/// Test helper: build a queue client with default proxy settings. +async fn test_queue_client(config: &AzureBlobConfig) -> Arc { + make_queue_client(config, &ProxyConfig::default()) + .await + .expect("Failed to create queue client") +} + +/// Test helper: base64-encode a message and put it on the queue, as Event Grid does. +async fn put_base64_message(queue_client: &QueueClient, message: &str) { + let queue_message = QueueMessage { + message_text: Some(BASE64_STANDARD.encode(message)), + }; + queue_client + .send_message( + queue_message.try_into().expect("infallible conversion"), + None, + ) + .await + .expect("Failed putting message"); +} + +impl AzureBlobConfig { + pub async fn new_emulator() -> AzureBlobConfig { + let address = std::env::var("AZURITE_ADDRESS").unwrap_or_else(|_| "localhost".to_string()); + let config = AzureBlobConfig { + connection_string: format!("UseDevelopmentStorage=true;DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://{}:10000/devstoreaccount1;QueueEndpoint=http://{}:10001/devstoreaccount1;TableEndpoint=http://{}:10002/devstoreaccount1;", address, address, address).into(), + container_name: "logs".to_string(), + queue: Some(Config { + queue_name: format!("test-{}", rand::random::()), + poll_secs: 1, + ..Default::default() + }), + decoding: default_decoding(), + ..Default::default() + }; + + config.ensure_container().await; + config.ensure_queue().await; + + config + } + + async fn run_assert(&self) -> Vec { + run_and_assert_source_compliance(self.clone(), Duration::from_secs(1), &SOURCE_TAGS).await + } + + async fn run_error(&self) -> Vec { + run_and_assert_source_error(self.clone(), Duration::from_secs(1), &COMPONENT_ERROR_TAGS) + .await + } + + async fn ensure_container(&self) { + let client = test_container_client(self).await; + + // HTTP 409 (Conflict) means the container already exists - this is OK + match client.create(None).await { + Ok(_) => {} + Err(e) if e.http_status() == Some(StatusCode::Conflict) => {} + Err(e) => panic!("Failed to create container: {e}"), + } + } + + async fn ensure_queue(&self) { + let client = test_queue_client(self).await; + + // HTTP 409 (Conflict) means the queue already exists - this is OK + match client.create(None).await { + Ok(_) => {} + Err(e) if e.http_status() == Some(StatusCode::Conflict) => {} + Err(e) => panic!("Failed to create queue: {e}"), + } + } + + async fn upload_blob(&self, name: String, content: String) { + let container_client = test_container_client(self).await; + let blob_client = container_client.blob_client(&name); + blob_client + .upload(RequestContent::from(content.into_bytes()), None) + .await + .expect("Failed putting blob"); + + self.queue_notify_blob_created(&name).await; + } + + async fn upload_compressed_blob(&self, name: String, content: String, compression: &str) { + let container_client = test_container_client(self).await; + let blob_client = container_client.blob_client(&name); + + let compressed_data = match compression { + "gzip" => { + let mut encoder = GzEncoder::new(Vec::new(), GzCompression::default()); + encoder + .write_all(content.as_bytes()) + .expect("Failed to write to gzip encoder"); + encoder.finish().expect("Failed to finish gzip compression") + } + "zstd" => { + zstd::encode_all(content.as_bytes(), 3).expect("Failed to compress with zstd") + } + _ => panic!("Unsupported compression type: {}", compression), + }; + + blob_client + .upload(RequestContent::from(compressed_data), None) + .await + .expect("Failed putting compressed blob"); + + self.queue_notify_blob_created(&name).await; + } + + async fn queue_notify_blob_created(&self, name: &str) { + let queue_client = test_queue_client(self).await; + let message = format!( + r#"{{ + "topic": "/subscriptions/fa5f2180-1451-4461-9b1f-aae7d4b33cf8/resourceGroups/events_poc/providers/Microsoft.Storage/storageAccounts/eventspocaccount", + "subject": "/blobServices/default/containers/logs/blobs/{}", + "eventType": "Microsoft.Storage.BlobCreated", + "id": "be3f21f7-201e-000b-7605-a29195062628", + "data": {{ + "api": "PutBlob", + "clientRequestId": "1fa42c94-6dd3-4172-95c4-fd9cf56b5009", + "requestId": "be3f21f7-201e-000b-7605-a29195000000", + "eTag": "0x8DC701C5D3FFDF6", + "contentType": "application/octet-stream", + "contentLength": 0, + "blobType": "BlockBlob", + "url": "https://eventspocaccount.blob.core.windows.net/logs/{}", + "sequencer": "0000000000000000000000000005C5360000000000276a63", + "storageDiagnostics": {{ + "batchId": "fec5b12c-2006-0034-0005-a25936000000" + }} + }}, + "dataVersion": "", + "metadataVersion": "1", + "eventTime": "2024-05-09T11:37:10.5637878Z" + }}"#, + name, name + ); + put_base64_message(&queue_client, &message).await; + } + + async fn queue_notify_blob_renamed(&self, name: &str) { + let queue_client = test_queue_client(self).await; + let message = format!( + r#"{{ + "topic": "/subscriptions/fa5f2180-1451-4461-9b1f-aae7d4b33cf8/resourceGroups/events_poc/providers/Microsoft.Storage/storageAccounts/eventspocaccount", + "subject": "/blobServices/default/containers/logs/blobs/{}", + "eventType": "Microsoft.Storage.BlobRenamed", + "id": "be3f21f7-201e-000b-7605-a29195062629", + "data": {{ + "api": "RenameFile", + "clientRequestId": "6d79dbfb-0e37-4fc4-981f-442c9ca65760", + "requestId": "831e1650-001e-001b-66ab-eeb76e000000", + "destinationUrl": "https://eventspocaccount.dfs.core.windows.net/logs/{}", + "sourceUrl": "https://eventspocaccount.dfs.core.windows.net/logs/old-name.txt", + "sequencer": "00000000000004420000000000028963", + "storageDiagnostics": {{ + "batchId": "b68529f3-68cd-4744-baa4-3c0498ec19f0" + }} + }}, + "dataVersion": "1", + "metadataVersion": "1", + "eventTime": "2024-05-09T11:37:10.5637878Z" + }}"#, + name, name + ); + put_base64_message(&queue_client, &message).await; + } + async fn queue_notify_custom_event(&self, name: &str, event_type: &str) { + let queue_client = test_queue_client(self).await; + let message = format!( + r#"{{ + "topic": "/subscriptions/fa5f2180-1451-4461-9b1f-aae7d4b33cf8/resourceGroups/events_poc/providers/Microsoft.Storage/storageAccounts/eventspocaccount", + "subject": "/blobServices/default/containers/logs/blobs/{}", + "eventType": "{}", + "id": "be3f21f7-201e-000b-7605-a29195062630", + "data": {{ + "api": "PutBlob", + "clientRequestId": "1fa42c94-6dd3-4172-95c4-fd9cf56b5009", + "requestId": "be3f21f7-201e-000b-7605-a29195000000", + "eTag": "0x8DC701C5D3FFDF6", + "contentType": "application/octet-stream", + "contentLength": 0, + "blobType": "BlockBlob", + "url": "https://eventspocaccount.blob.core.windows.net/logs/{}", + "sequencer": "0000000000000000000000000005C5360000000000276a63", + "storageDiagnostics": {{ + "batchId": "fec5b12c-2006-0034-0005-a25936000000" + }} + }}, + "dataVersion": "", + "metadataVersion": "1", + "eventTime": "2024-05-09T11:37:10.5637878Z" + }}"#, + name, event_type, name + ); + put_base64_message(&queue_client, &message).await; + } +} + +/// Test basic functionality: reading a single line from a blob. +/// +/// **Setup:** +/// - Upload a blob containing a single line of text: "a" +/// +/// **Verification:** +/// - Verify exactly 1 event is received +/// - Verify the event message matches the uploaded content +/// +/// **Purpose:** Validate basic blob reading and event generation for single-line content. +#[tokio::test] +async fn azure_blob_read_single_line_from_blob() { + let config = AzureBlobConfig::new_emulator().await; + let content = "a"; + config + .upload_blob("file.txt".to_string(), content.to_string()) + .await; + + let events = config.run_assert().await; + assert_eq!(events.len(), 1); + assert_eq!(events[0].as_log()["message"], "a".into()); +} + +/// Test handling of BlobRenamed events. +/// +/// **Setup:** +/// - Upload a blob with content to a destination path +/// - Send a BlobRenamed event notification pointing to that blob +/// +/// **Verification:** +/// - Verify exactly 1 event is received +/// - Verify the event message matches the blob content at the renamed location +/// +/// **Purpose:** Validate that BlobRenamed events trigger blob reading at the new location. +#[tokio::test] +async fn azure_blob_read_blob_renamed_event() { + let config = AzureBlobConfig::new_emulator().await; + let content = "renamed_blob_content"; + + // Upload blob to the destination path + let blob_client = test_container_client(&config) + .await + .blob_client("renamed-file.txt"); + blob_client + .upload(RequestContent::from(content.as_bytes().to_vec()), None) + .await + .expect("Failed putting blob"); + + // Send BlobRenamed event notification + config.queue_notify_blob_renamed("renamed-file.txt").await; + + let events = config.run_assert().await; + assert_eq!(events.len(), 1); + assert_eq!(events[0].as_log()["message"], "renamed_blob_content".into()); +} + +/// Test reading multiple lines from a single blob with newline-delimited framing. +/// +/// **Setup:** +/// - Upload a blob with newline-separated content: "a\nb\nc" +/// - Use default newline-delimited framing +/// +/// **Verification:** +/// - Verify exactly 3 events are received +/// - Verify each line is parsed as a separate event in order +/// +/// **Purpose:** Validate default newline-delimited framing splits blob content correctly. +#[tokio::test] +async fn azure_blob_read_multiple_lines_from_blob() { + let config = AzureBlobConfig::new_emulator().await; + let content = "a\nb\nc"; + config + .upload_blob("file.txt".to_string(), content.to_string()) + .await; + + let events = config.run_assert().await; + assert_eq!(events.len(), 3); + assert_eq!(events[0].as_log()["message"], "a".into()); + assert_eq!(events[1].as_log()["message"], "b".into()); + assert_eq!(events[2].as_log()["message"], "c".into()); +} + +/// Test processing multiple blobs from the same container. +/// +/// **Setup:** +/// - Upload 3 separate blobs, each containing a single line: "a", "b", "c" +/// - Each blob generates an Event Grid notification +/// +/// **Verification:** +/// - Verify exactly 3 events are received (one from each blob) +/// - Verify each event contains the content from its respective blob +/// +/// **Purpose:** Validate the source can handle multiple blob notifications and process them correctly. +#[tokio::test] +async fn azure_blob_read_single_line_from_multiple_blobs() { + let config = AzureBlobConfig::new_emulator().await; + let contents = vec!["a", "b", "c"]; + for (i, content) in contents.clone().iter().enumerate() { + config + .upload_blob(format!("file{}.txt", i), content.to_string()) + .await; + } + + let events = + run_and_assert_source_compliance(config.clone(), Duration::from_secs(4), &SOURCE_TAGS) + .await; + assert_eq!(events.len(), contents.len()); + for (i, event) in events.iter().enumerate() { + assert_eq!(event.as_log()["message"], contents[i].into()); + } +} + +/// Test error handling when queue messages cannot be read. +/// +/// **Setup:** +/// - Upload a blob with valid content +/// - Configure source to read from a non-existent queue +/// +/// **Verification:** +/// - Verify no events are received (source should emit error metric) +/// - Verify error tags are emitted for the component +/// +/// **Purpose:** Validate proper error handling when queue is inaccessible. +#[tokio::test] +async fn azure_blob_emit_error_on_message_read() { + let mut config = AzureBlobConfig::new_emulator().await; + let content = "a\nb\nc"; + config + .upload_blob("file.txt".to_string(), content.to_string()) + .await; + config.queue = Some(Config { + queue_name: "nonexistent".to_string(), + poll_secs: 1, + ..Default::default() + }); + + let events = config.run_error().await; + assert!(events.is_empty()); +} + +/// Test graceful handling of notifications for non-existent blobs. +/// +/// **Setup:** +/// - Send Event Grid notification for a blob that doesn't exist +/// - Then upload a valid blob with actual content +/// +/// **Verification:** +/// - Verify only the valid blob's event is received (1 event total) +/// - Verify the missing blob notification is ignored without crashing +/// +/// **Purpose:** Validate resilience when Event Grid sends notifications for deleted or non-existent blobs. +#[tokio::test] +async fn azure_blob_ignore_missing_blob() { + let config = AzureBlobConfig::new_emulator().await; + + config.queue_notify_blob_created("non-existent").await; + config + .upload_blob("file.txt".to_string(), "some_content".to_string()) + .await; + + let events = config.run_assert().await; + assert_eq!(events.len(), 1); +} + +/// Test JSON deserialization with JSON codec. +/// +/// **Setup:** +/// - Configure source with JSON deserializer +/// - Upload a blob containing a single JSON object with fields: timestamp, level, message +/// +/// **Verification:** +/// - Verify exactly 1 event is received +/// - Verify JSON fields are correctly parsed and accessible in the log event +/// +/// **Purpose:** Validate decoding configuration works correctly for JSON content. +#[tokio::test] +async fn azure_blob_read_json_content() { + use vector_lib::codecs::decoding::{DeserializerConfig, JsonDeserializerConfig}; + + let mut config = AzureBlobConfig::new_emulator().await; + config.decoding = DeserializerConfig::Json(JsonDeserializerConfig::default()); + + let json_content = + r#"{"timestamp": "2024-01-01T00:00:00Z", "level": "INFO", "message": "Test log"}"#; + config + .upload_blob("log.json".to_string(), json_content.to_string()) + .await; + + let events = config.run_assert().await; + assert_eq!(events.len(), 1); + let log = events[0].as_log(); + assert_eq!(log["level"], "INFO".into()); + assert_eq!(log["message"], "Test log".into()); +} + +/// Test error handling with malformed Event Grid messages. +/// +/// **Setup:** +/// - Manually enqueue an invalid (non-JSON) message to the queue +/// - Upload a valid blob with correct content +/// +/// **Verification:** +/// - Verify the valid blob's event is still received (1 event total) +/// - Verify the malformed message is skipped without blocking valid messages +/// +/// **Purpose:** Validate source resilience when queue contains corrupted or invalid messages. +#[tokio::test] +async fn azure_blob_handle_malformed_message() { + let config = AzureBlobConfig::new_emulator().await; + let queue_client = test_queue_client(&config).await; + put_base64_message(&queue_client, "not a valid json").await; + + config + .upload_blob("file.txt".to_string(), "correct content".to_string()) + .await; + + let events = config.run_assert().await; + assert_eq!(events.len(), 1); + assert_eq!(events[0].as_log()["message"], "correct content".into()); +} + +/// Test handling of empty blobs. +/// +/// **Setup:** +/// - Upload a blob with zero-length content (empty string) +/// +/// **Verification:** +/// - Verify the source handles empty blobs gracefully (either 0 events or 1 empty event) +/// - Verify no errors or crashes occur +/// +/// **Purpose:** Validate edge case handling for empty blob content. +#[tokio::test] +async fn azure_blob_read_empty_blob() { + let config = AzureBlobConfig::new_emulator().await; + config + .upload_blob("empty.txt".to_string(), "".to_string()) + .await; + + let events = config.run_assert().await; + if !events.is_empty() { + assert_eq!(events[0].as_log()["message"], "".into()); + } +} + +// ===== Compression Integration Tests ===== + +/// Test gzip decompression with automatic compression detection via file extension. +/// +/// **Setup:** +/// - Configure compression = "auto" +/// - Upload a gzip-compressed blob with ".gz" extension containing "line1\nline2\nline3" +/// +/// **Verification:** +/// - Verify blob is automatically detected as gzip and decompressed +/// - Verify exactly 3 events are received with correct content +/// +/// **Purpose:** Validate auto-detection of gzip compression from file extension. +#[tokio::test] +async fn azure_blob_gzip_compression_auto() { + let mut config = AzureBlobConfig::new_emulator().await; + config.compression = Compression::Auto; + + let content = "line1\nline2\nline3"; + config + .upload_compressed_blob("file.gz".to_string(), content.to_string(), "gzip") + .await; + + let events = config.run_assert().await; + assert_eq!(events.len(), 3); + assert_eq!(events[0].as_log()["message"], "line1".into()); + assert_eq!(events[1].as_log()["message"], "line2".into()); + assert_eq!(events[2].as_log()["message"], "line3".into()); +} + +/// Test gzip decompression with explicit compression configuration. +/// +/// **Setup:** +/// - Configure compression = "gzip" explicitly +/// - Upload a gzip-compressed blob WITHOUT ".gz" extension +/// +/// **Verification:** +/// - Verify blob is decompressed despite lacking compression-indicating file extension +/// - Verify exactly 2 events are received with correct content +/// +/// **Purpose:** Validate explicit gzip compression override when file extension is absent. +#[tokio::test] +async fn azure_blob_gzip_compression_explicit() { + let mut config = AzureBlobConfig::new_emulator().await; + config.compression = Compression::Gzip; + + let content = "explicit_gzip_line1\nexplicit_gzip_line2"; + config + .upload_compressed_blob("file_no_extension".to_string(), content.to_string(), "gzip") + .await; + + let events = config.run_assert().await; + assert_eq!(events.len(), 2); + assert_eq!(events[0].as_log()["message"], "explicit_gzip_line1".into()); + assert_eq!(events[1].as_log()["message"], "explicit_gzip_line2".into()); +} + +/// Test zstd decompression with automatic compression detection. +/// +/// **Setup:** +/// - Configure compression = "auto" +/// - Upload a zstd-compressed blob with ".zst" extension +/// +/// **Verification:** +/// - Verify blob is automatically detected as zstd and decompressed +/// - Verify exactly 3 events are received with correct content +/// +/// **Purpose:** Validate auto-detection of zstd compression from file extension. +#[tokio::test] +async fn azure_blob_zstd_compression() { + let mut config = AzureBlobConfig::new_emulator().await; + config.compression = Compression::Auto; + + let content = "zstd_line1\nzstd_line2\nzstd_line3"; + config + .upload_compressed_blob("file.zst".to_string(), content.to_string(), "zstd") + .await; + + let events = config.run_assert().await; + assert_eq!(events.len(), 3); + assert_eq!(events[0].as_log()["message"], "zstd_line1".into()); + assert_eq!(events[1].as_log()["message"], "zstd_line2".into()); + assert_eq!(events[2].as_log()["message"], "zstd_line3".into()); +} + +/// Test that compression can be disabled even when file extension suggests compression. +/// +/// **Setup:** +/// - Configure compression = "none" explicitly +/// - Upload an UNCOMPRESSED blob named "file.gz" (misleading extension) +/// +/// **Verification:** +/// - Verify blob is NOT decompressed despite having ".gz" extension +/// - Verify raw uncompressed content is read correctly (2 events) +/// +/// **Purpose:** Validate explicit compression=none overrides auto-detection. +#[tokio::test] +async fn azure_blob_no_compression_with_gz_extension() { + let mut config = AzureBlobConfig::new_emulator().await; + config.compression = Compression::None; + + let content = "uncompressed_content\nsecond_line"; + config + .upload_blob("file.gz".to_string(), content.to_string()) + .await; + + let events = config.run_assert().await; + assert_eq!(events.len(), 2); + assert_eq!(events[0].as_log()["message"], "uncompressed_content".into()); + assert_eq!(events[1].as_log()["message"], "second_line".into()); +} + +/// Test decompression of multipart gzip files (multiple gzip members concatenated). +/// +/// **Setup:** +/// - Create a blob with two separate gzip-compressed parts concatenated together +/// - Each part contains 2 lines with trailing newlines +/// - Configure compression = "auto" +/// +/// **Verification:** +/// - Verify all 4 lines from both gzip members are decompressed and parsed +/// - Verify events maintain correct order across both gzip members +/// +/// **Purpose:** Validate multi-member gzip decompression (common format for log rotation). +#[tokio::test] +async fn azure_blob_multipart_gzip() { + let mut config = AzureBlobConfig::new_emulator().await; + config.compression = Compression::Auto; + + // Create two separate gzip-compressed parts + // Add trailing newlines so concatenated result splits correctly + let content1 = "part1_line1\npart1_line2\n"; + let content2 = "part2_line1\npart2_line2\n"; + + let mut encoder1 = GzEncoder::new(Vec::new(), GzCompression::default()); + encoder1 + .write_all(content1.as_bytes()) + .expect("Failed to write part 1"); + let compressed1 = encoder1.finish().expect("Failed to finish part 1"); + + let mut encoder2 = GzEncoder::new(Vec::new(), GzCompression::default()); + encoder2 + .write_all(content2.as_bytes()) + .expect("Failed to write part 2"); + let compressed2 = encoder2.finish().expect("Failed to finish part 2"); + + // Concatenate the two gzip parts + let mut multipart = compressed1; + multipart.extend(compressed2); + + let container_client = test_container_client(&config).await; + let blob_client = container_client.blob_client("multipart.gz"); + blob_client + .upload(RequestContent::from(multipart), None) + .await + .expect("Failed putting multipart blob"); + + config.queue_notify_blob_created("multipart.gz").await; + + let events = config.run_assert().await; + assert_eq!(events.len(), 4); + assert_eq!(events[0].as_log()["message"], "part1_line1".into()); + assert_eq!(events[1].as_log()["message"], "part1_line2".into()); + assert_eq!(events[2].as_log()["message"], "part2_line1".into()); + assert_eq!(events[3].as_log()["message"], "part2_line2".into()); +} + +// ============================================================================ +// Framing Integration Tests +// ============================================================================ + +use vector_lib::codecs::decoding::{ + CharacterDelimitedDecoderConfig, CharacterDelimitedDecoderOptions, FramingConfig, + NewlineDelimitedDecoderConfig, NewlineDelimitedDecoderOptions, +}; + +/// Test character-delimited framing with null byte (\\0) delimiter. +/// +/// **Setup:** +/// - Configure character-delimited framing with \\0 (null) delimiter +/// - Upload blob with null-delimited content: "a\\0b\\0c" +/// +/// **Verification:** +/// - Verify blob is split on null bytes into exactly 3 events +/// - Verify each segment is parsed as a separate event: "a", "b", "c" +/// +/// **Purpose:** Validate custom character-delimited framing with non-printable delimiters. +#[tokio::test] +async fn azure_blob_character_delimited_framing() { + let mut config = AzureBlobConfig::new_emulator().await; + config.framing = FramingConfig::CharacterDelimited(CharacterDelimitedDecoderConfig { + character_delimited: CharacterDelimitedDecoderOptions { + delimiter: b'\0', + max_length: None, + }, + }); + + let content = "a\0b\0c"; + config + .upload_blob("null-delimited.txt".to_string(), content.to_string()) + .await; + + let events = config.run_assert().await; + assert_eq!(events.len(), 3); + assert_eq!(events[0].as_log()["message"], "a".into()); + assert_eq!(events[1].as_log()["message"], "b".into()); + assert_eq!(events[2].as_log()["message"], "c".into()); +} + +/// Test bytes framing (treat entire blob as single event without splitting). +/// +/// **Setup:** +/// - Configure framing = "bytes" (no splitting) +/// - Upload blob with content containing newlines: "single\\nevent\\nwith\\nnewlines" +/// +/// **Verification:** +/// - Verify exactly 1 event is received +/// - Verify event contains entire blob content including all newlines +/// +/// **Purpose:** Validate bytes framing treats entire blob as atomic event regardless of delimiters. +#[tokio::test] +async fn azure_blob_bytes_framing() { + let mut config = AzureBlobConfig::new_emulator().await; + config.framing = FramingConfig::Bytes; + + let content = "single\nevent\nwith\nnewlines"; + config + .upload_blob("bytes.txt".to_string(), content.to_string()) + .await; + + let events = config.run_assert().await; + assert_eq!(events.len(), 1); + assert_eq!( + events[0].as_log()["message"], + "single\nevent\nwith\nnewlines".into() + ); +} + +/// Test newline-delimited framing with mixed line ending styles. +/// +/// **Setup:** +/// - Configure newline-delimited framing +/// - Upload blob with mixed line endings: Unix (\\n), Windows (\\r\\n), old Mac (\\r) +/// +/// **Verification:** +/// - Verify blob is split on \\n characters (newline-delimited splits only on \\n) +/// - Verify \\r characters are preserved in content where they don't precede \\n +/// - Verify exactly 3 events are received +/// +/// **Purpose:** Validate newline-delimited framing behavior with non-uniform line endings. +#[tokio::test] +async fn azure_blob_mixed_line_endings() { + let mut config = AzureBlobConfig::new_emulator().await; + config.framing = FramingConfig::NewlineDelimited(NewlineDelimitedDecoderConfig { + newline_delimited: NewlineDelimitedDecoderOptions { max_length: None }, + }); + + // Mix of \n, \r\n, and \r line endings + // Newline-delimited decoder only splits on \n, so \r is preserved in the content + let content = "unix\nwindows\r\nold_mac\rend"; + config + .upload_blob("mixed-endings.txt".to_string(), content.to_string()) + .await; + + let events = config.run_assert().await; + // Splits on \n only: "unix\n", "windows\r\n", "old_mac\rend" (no trailing \n) + // Results in: "unix", "windows\r", "old_mac\rend" + assert_eq!(events.len(), 3); + assert_eq!(events[0].as_log()["message"], "unix".into()); + assert_eq!(events[1].as_log()["message"], "windows\r".into()); + assert_eq!(events[2].as_log()["message"], "old_mac\rend".into()); +} + +// ============================================================================ +// Combined Tests (Framing + Compression) +// ============================================================================ + +/// Test combination of gzip compression and character-delimited framing. +/// +/// **Setup:** +/// - Configure compression = "auto" and character-delimited framing with \\0 delimiter +/// - Upload gzip-compressed blob with null-delimited content: "item1\\0item2\\0item3" +/// +/// **Verification:** +/// - Verify compression is auto-detected and blob is decompressed +/// - Verify decompressed content is split on null bytes into 3 events +/// - Verify complete pipeline: decompress β†’ frame β†’ parse +/// +/// **Purpose:** Validate compression and framing work correctly together. +#[tokio::test] +async fn azure_blob_gzip_character_delimited() { + let mut config = AzureBlobConfig::new_emulator().await; + config.compression = Compression::Auto; + config.framing = FramingConfig::CharacterDelimited(CharacterDelimitedDecoderConfig { + character_delimited: CharacterDelimitedDecoderOptions { + delimiter: b'\0', + max_length: None, + }, + }); + + let content = "item1\0item2\0item3"; + config + .upload_compressed_blob( + "null-compressed.gz".to_string(), + content.to_string(), + "gzip", + ) + .await; + + let events = config.run_assert().await; + assert_eq!(events.len(), 3); + assert_eq!(events[0].as_log()["message"], "item1".into()); + assert_eq!(events[1].as_log()["message"], "item2".into()); + assert_eq!(events[2].as_log()["message"], "item3".into()); +} + +// ===== Multiline Integration Tests ===== + +/// Test multiline aggregation with continue_through mode (Java-style stack traces). +/// +/// **Setup:** +/// - Configure multiline mode = "continue_through" +/// - start_pattern: lines starting with date (^\\d{4}-\\d{2}-\\d{2}) +/// - condition_pattern: continuation lines start with whitespace (^\\s) +/// - Upload blob with Java stack trace (ERROR line + indented stack trace lines + INFO line) +/// +/// **Verification:** +/// - Verify exactly 2 events are received +/// - Verify first event contains complete stack trace (ERROR line + all indented lines) +/// - Verify second event is the single INFO line +/// +/// **Purpose:** Validate continue_through mode aggregates multi-line stack traces correctly. +#[tokio::test] +async fn azure_blob_multiline_continue_through() { + use crate::{line_agg, sources::util::MultilineConfig}; + use std::time::Duration; + + let mut config = AzureBlobConfig::new_emulator().await; + config.multiline = Some(MultilineConfig { + start_pattern: r"^\d{4}-\d{2}-\d{2}".to_string(), + condition_pattern: r"^\s".to_string(), + mode: line_agg::Mode::ContinueThrough, + timeout_ms: Duration::from_millis(1000), + }); + + let content = "2024-01-01 ERROR Something failed\n at com.example.Class.method(Class.java:10)\n at com.example.Main.main(Main.java:20)\n2024-01-01 INFO Next log"; + config + .upload_blob("stacktrace.log".to_string(), content.to_string()) + .await; + + let events = + run_and_assert_source_compliance(config.clone(), Duration::from_secs(3), &SOURCE_TAGS) + .await; + assert_eq!(events.len(), 2); + + // First event should contain the full stack trace + let first_log = events[0].as_log()["message"].to_string_lossy(); + assert!(first_log.contains("ERROR Something failed")); + assert!(first_log.contains("at com.example.Class.method")); + assert!(first_log.contains("at com.example.Main.main")); + + // Second event should be the single line + assert_eq!( + events[1].as_log()["message"], + "2024-01-01 INFO Next log".into() + ); +} + +/// Test multiline aggregation with halt_before mode. +/// +/// **Setup:** +/// - Configure multiline mode = "halt_before" +/// - start_pattern and condition_pattern: lines starting with "START" +/// - Upload blob with content: "START block1\\nline2\\nline3\\nSTART block2\\nline5" +/// +/// **Verification:** +/// - Verify exactly 2 events are received +/// - Verify first event contains "START block1" through "line3" (stops before next START) +/// - Verify second event contains "START block2\\nline5" +/// +/// **Purpose:** Validate halt_before mode stops aggregation before matching line. +#[tokio::test] +async fn azure_blob_multiline_halt_before() { + use crate::{line_agg, sources::util::MultilineConfig}; + use std::time::Duration; + + let mut config = AzureBlobConfig::new_emulator().await; + config.multiline = Some(MultilineConfig { + start_pattern: r"^START".to_string(), + condition_pattern: r"^START".to_string(), + mode: line_agg::Mode::HaltBefore, + timeout_ms: Duration::from_millis(1000), + }); + + let content = "START block1\nline2\nline3\nSTART block2\nline5"; + config + .upload_blob("halt_before.log".to_string(), content.to_string()) + .await; + + let events = + run_and_assert_source_compliance(config.clone(), Duration::from_secs(3), &SOURCE_TAGS) + .await; + assert_eq!(events.len(), 2); + + let first_log = events[0].as_log()["message"].to_string_lossy(); + assert!(first_log.contains("START block1")); + assert!(first_log.contains("line2")); + assert!(first_log.contains("line3")); + + let second_log = events[1].as_log()["message"].to_string_lossy(); + assert!(second_log.contains("START block2")); + assert!(second_log.contains("line5")); +} + +/// Test multiline aggregation with halt_with mode (include terminator). +/// +/// **Setup:** +/// - Configure multiline mode = "halt_with" +/// - start_pattern: lines starting with "BEGIN" +/// - condition_pattern: lines starting with "END" +/// - Upload blob with transactions: "BEGIN...\\nprocessing\\nEND\\nBEGIN...\\nEND" +/// +/// **Verification:** +/// - Verify exactly 2 events are received +/// - Verify each event contains complete transaction from BEGIN through END (inclusive) +/// - Verify terminator line (END) is included in the aggregated event +/// +/// **Purpose:** Validate halt_with mode includes the terminator line in aggregated event. +#[tokio::test] +async fn azure_blob_multiline_halt_with() { + use crate::{line_agg, sources::util::MultilineConfig}; + use std::time::Duration; + + let mut config = AzureBlobConfig::new_emulator().await; + config.multiline = Some(MultilineConfig { + start_pattern: r"^BEGIN".to_string(), + condition_pattern: r"^END".to_string(), + mode: line_agg::Mode::HaltWith, + timeout_ms: Duration::from_millis(1000), + }); + + let content = "BEGIN transaction1\nprocessing\nEND\nBEGIN transaction2\nmore processing\nEND"; + config + .upload_blob("halt_with.log".to_string(), content.to_string()) + .await; + + let events = + run_and_assert_source_compliance(config.clone(), Duration::from_secs(3), &SOURCE_TAGS) + .await; + assert_eq!(events.len(), 2); + + let first_log = events[0].as_log()["message"].to_string_lossy(); + assert!(first_log.contains("BEGIN transaction1")); + assert!(first_log.contains("processing")); + assert!(first_log.contains("END")); + + let second_log = events[1].as_log()["message"].to_string_lossy(); + assert!(second_log.contains("BEGIN transaction2")); + assert!(second_log.contains("more processing")); + assert!(second_log.contains("END")); +} + +/// Test that unsupported event types are ignored and their queue messages deleted. +/// +/// **Setup:** +/// - Send an Event Grid notification with unsupported event type "Microsoft.Storage.BlobDeleted" +/// - Upload a valid blob and send a BlobCreated notification +/// +/// **Verification:** +/// - Verify only the valid blob's event is received (1 event) +/// - Verify the unsupported event message is deleted from the queue (not stuck for reprocessing) +/// +/// **Purpose:** Validate that unsupported event types don't cause infinite reprocessing loops. +#[tokio::test] +async fn azure_blob_ignore_unsupported_event_type() { + let config = AzureBlobConfig::new_emulator().await; + + config + .queue_notify_custom_event("some-blob.txt", "Microsoft.Storage.BlobDeleted") + .await; + config + .upload_blob("valid.txt".to_string(), "valid content".to_string()) + .await; + + let events = config.run_assert().await; + assert_eq!(events.len(), 1); + assert_eq!(events[0].as_log()["message"], "valid content".into()); +} + +/// Test that messages for mismatching containers are ignored and deleted from the queue. +/// +/// **Setup:** +/// - Send an Event Grid notification referencing a different container than configured +/// - Upload a valid blob and send a BlobCreated notification for the correct container +/// +/// **Verification:** +/// - Verify only the valid blob's event is received (1 event) +/// - Verify the mismatching container message is deleted (not stuck for reprocessing) +/// +/// **Purpose:** Validate that messages for other containers don't cause infinite reprocessing loops. +#[tokio::test] +async fn azure_blob_ignore_mismatching_container() { + let config = AzureBlobConfig::new_emulator().await; + + // Send a BlobCreated event referencing a different container + let queue_client = test_queue_client(&config).await; + let message = r#"{ + "topic": "/subscriptions/fa5f2180-1451-4461-9b1f-aae7d4b33cf8/resourceGroups/events_poc/providers/Microsoft.Storage/storageAccounts/eventspocaccount", + "subject": "/blobServices/default/containers/other-container/blobs/some-blob.txt", + "eventType": "Microsoft.Storage.BlobCreated", + "id": "be3f21f7-201e-000b-7605-a29195062631", + "data": { + "api": "PutBlob", + "clientRequestId": "1fa42c94-6dd3-4172-95c4-fd9cf56b5009", + "requestId": "be3f21f7-201e-000b-7605-a29195000000", + "eTag": "0x8DC701C5D3FFDF6", + "contentType": "application/octet-stream", + "contentLength": 0, + "blobType": "BlockBlob", + "url": "https://eventspocaccount.blob.core.windows.net/other-container/some-blob.txt", + "sequencer": "0000000000000000000000000005C5360000000000276a63", + "storageDiagnostics": { + "batchId": "fec5b12c-2006-0034-0005-a25936000000" + } + }, + "dataVersion": "", + "metadataVersion": "1", + "eventTime": "2024-05-09T11:37:10.5637878Z" + }"#; + put_base64_message(&queue_client, message).await; + + config + .upload_blob("valid.txt".to_string(), "valid content".to_string()) + .await; + + let events = config.run_assert().await; + assert_eq!(events.len(), 1); + assert_eq!(events[0].as_log()["message"], "valid content".into()); +} + +/// Test complete pipeline: gzip compression + multiline aggregation + JSON decoding. +/// +/// **Setup:** +/// - Configure compression = "auto", JSON decoding, and multiline continue_through mode +/// - start_pattern: lines starting with "{", condition_pattern: indented lines +/// - Upload gzip-compressed blob with JSON lines and indented continuation +/// +/// **Verification:** +/// - Verify compression is detected and blob is decompressed +/// - Verify multiline aggregation combines JSON with indented details +/// - Verify JSON parsing happens after multiline aggregation +/// - Verify at least one event is received +/// +/// **Purpose:** Validate complete processing pipeline with all features enabled. +#[tokio::test] +async fn azure_blob_gzip_multiline_json() { + use crate::{line_agg, sources::util::MultilineConfig}; + use std::time::Duration; + use vector_lib::codecs::decoding::{DeserializerConfig, JsonDeserializerConfig}; + + let mut config = AzureBlobConfig::new_emulator().await; + config.compression = Compression::Auto; + config.decoding = DeserializerConfig::Json(JsonDeserializerConfig::default()); + config.multiline = Some(MultilineConfig { + start_pattern: r"^\{".to_string(), + condition_pattern: r"^\s".to_string(), + mode: line_agg::Mode::ContinueThrough, + timeout_ms: Duration::from_millis(1000), + }); + + let content = r#"{"level":"ERROR","message":"Failed"} + {"details":"Stack trace"} +{"level":"INFO","message":"Success"}"#; + + config + .upload_compressed_blob("multiline.json.gz".to_string(), content.to_string(), "gzip") + .await; + + let events = + run_and_assert_source_compliance(config.clone(), Duration::from_secs(3), &SOURCE_TAGS) + .await; + // Note: JSON parsing happens after multiline aggregation, so we expect the aggregated lines + // The exact number of events depends on whether the aggregated lines form valid JSON + assert!(!events.is_empty()); +} diff --git a/src/sources/azure_blob/mod.rs b/src/sources/azure_blob/mod.rs new file mode 100644 index 0000000000000..ad5beed844b4d --- /dev/null +++ b/src/sources/azure_blob/mod.rs @@ -0,0 +1,513 @@ +use std::{future::Future, pin::Pin, sync::Arc}; + +use async_stream::stream; +use bytes::Bytes; +use futures::{Stream, stream::StreamExt}; +use tokio::select; +use vrl::{path, value::Kind}; + +use derivative::Derivative; +use vector_lib::internal_event::Registered; +use vector_lib::{ + codecs::{ + NewlineDelimitedDecoderConfig, + decoding::{DeserializerConfig, FramingConfig, NewlineDelimitedDecoderOptions}, + }, + config::{LegacyKey, log_schema}, + configurable::configurable_component, + internal_event::{ByteSize, BytesReceived, CountByteSize, InternalEventHandle as _, Protocol}, + lookup::{PathPrefix, metadata_path, owned_value_path}, + sensitive_string::SensitiveString, +}; + +/// Compression scheme for blobs retrieved from Azure Blob Storage. +#[configurable_component] +#[configurable(metadata(docs::advanced))] +#[derive(Clone, Copy, Debug, Derivative, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +#[derivative(Default)] +pub enum Compression { + /// Automatically attempt to determine the compression scheme. + /// + /// The compression scheme is determined from the blob's `Content-Type` metadata + /// and the blob name suffix (e.g., `.gz`, `.zst`). + /// + /// Priority order: 1) Content-Type, 2) File extension. + /// + /// Supported Content-Type values: `application/gzip`, `application/x-gzip`, `application/zstd`. + /// + /// **Note:** Unlike the AWS S3 source, the `Content-Encoding` header is NOT supported + /// due to an Azure SDK limitation. When blobs have `Content-Encoding` set, use the + /// file extension or `Content-Type` header instead, or configure explicit compression. + /// + /// It is set to `none` if the compression scheme cannot be determined. + #[derivative(Default)] + Auto, + + /// Uncompressed. + None, + + /// GZIP. + Gzip, + + /// ZSTD. + Zstd, +} + +/// Returns the default framing configuration for backwards compatibility. +/// Uses newline-delimited framing to match the original hardcoded behavior. +const fn default_framing() -> FramingConfig { + FramingConfig::NewlineDelimited(NewlineDelimitedDecoderConfig { + newline_delimited: NewlineDelimitedDecoderOptions { max_length: None }, + }) +} + +use crate::{ + SourceSender, + codecs::{Decoder, DecodingConfig}, + config::{ + LogNamespace, SourceAcknowledgementsConfig, SourceConfig, SourceContext, SourceOutput, + }, + event::{BatchNotifier, BatchStatus, EstimatedJsonEncodedSizeOf, Event}, + internal_events::{ + EventsReceived, InvalidRowEventTypeError, QueueMessageProcessingErrored, + QueueMessageProcessingRejected, QueueMessageProcessingSucceeded, StreamClosedError, + }, + line_agg, + serde::{bool_or_struct, default_decoding}, + shutdown::ShutdownSignal, + sources::{azure_blob::queue::make_blob_with_ack_stream, util::MultilineConfig}, +}; + +#[cfg(all(test, feature = "azure-blob-integration-tests"))] +mod integration_tests; +pub mod queue; +#[cfg(test)] +mod test; + +/// Collects logs from Azure Blob Storage. +/// +/// This source reads objects from Azure Blob Storage by processing events from an Azure Storage Queue. +/// When a blob is created or modified in the configured container, an event is sent to the queue, +/// and this source processes those events to read and decode the blob contents. +#[configurable_component(source("azure_blob", "Collect logs from Azure Blob Storage."))] +#[derive(Clone, Derivative)] +#[derivative(Default, Debug)] +#[serde(default, deny_unknown_fields)] +pub struct AzureBlobConfig { + /// The namespace to use for logs. This overrides the global setting. + #[configurable(metadata(docs::hidden))] + #[serde(default)] + log_namespace: Option, + + /// Factory function for creating blob streams with acknowledgements. Used only for tests. + #[configurable(metadata(docs::hidden))] + #[serde(skip)] + #[derivative(Default(value = "None"), Debug = "ignore")] + pub blob_stream_factory: + Option crate::Result + Send + Sync>>, + + /// Configuration options for Storage Queue. + queue: Option, + + /// The Azure Blob Storage Account connection string. + /// + /// Authentication with access key is the only supported authentication method. + #[configurable(metadata( + docs::examples = "DefaultEndpointsProtocol=https;AccountName=mylogstorage;AccountKey=storageaccountkeybase64encoded;EndpointSuffix=core.windows.net" + ))] + pub connection_string: SensitiveString, + + /// The Azure Blob Storage Account container name. + #[configurable(metadata(docs::examples = "my-logs"))] + pub(super) container_name: String, + + #[configurable(derived)] + #[serde(default, deserialize_with = "bool_or_struct")] + pub acknowledgements: SourceAcknowledgementsConfig, + + /// Compression scheme used for decompressing blobs retrieved from Azure Blob Storage. + #[configurable(derived)] + #[serde(default)] + pub compression: Compression, + + /// Configurable framing for splitting blob contents into events. + #[configurable(derived)] + #[serde(default = "default_framing")] + #[derivative(Default(value = "default_framing()"))] + pub framing: FramingConfig, + + /// Multiline aggregation configuration. + /// + /// If not specified, multiline aggregation is disabled. + #[configurable(derived)] + pub multiline: Option, + + #[configurable(derived)] + #[serde(default = "default_decoding")] + #[derivative(Default(value = "default_decoding()"))] + pub decoding: DeserializerConfig, + + /// Whether to delete non-retryable messages from the queue. + /// + /// If a message is rejected by the sink and not retryable, setting this to `true` + /// will delete the message from the queue. When `false`, rejected messages are + /// retained in the queue and will become visible again after the visibility timeout. + #[serde(default = "default_true")] + #[derivative(Default(value = "default_true()"))] + pub delete_failed_message: bool, +} + +const fn default_true() -> bool { + true +} + +impl_generate_config_from_default!(AzureBlobConfig); + +impl AzureBlobConfig { + /// Self validation + pub fn validate(&self) -> crate::Result<()> { + let queue = match self.queue.as_ref() { + Some(queue) if !queue.queue_name.is_empty() => queue, + _ => return Err("Azure event grid queue must be set.".into()), + }; + + if self.container_name.is_empty() { + return Err("Azure Container must be set.".into()); + } + + if !(1..=32).contains(&queue.max_number_of_messages) { + return Err("Azure queue `max_number_of_messages` must be between 1 and 32.".into()); + } + + if !(1..=604800).contains(&queue.visibility_timeout_secs) { + return Err( + "Azure queue `visibility_timeout_secs` must be between 1 and 604800.".into(), + ); + } + + if queue.poll_secs == 0 { + return Err("Azure queue `poll_secs` must be greater than 0.".into()); + } + + Ok(()) + } +} + +/// Determines the compression format from content type and blob name. +/// Priority order: 1) Content-Type, 2) File extension. +/// +/// Note: Content-Encoding header is NOT supported due to Azure SDK limitation. +/// The SDK requires Content-Length header in responses, but Azure uses chunked +/// transfer encoding when Content-Encoding is set, causing parsing failures. +pub(super) fn determine_compression( + content_type: Option<&str>, + blob_name: &str, +) -> Option { + content_type + .and_then(content_type_to_compression) + .or_else(|| blob_name_to_compression(blob_name)) +} + +/// Converts Content-Type header value to Compression enum. +/// Strips MIME parameters (e.g. `application/gzip; charset=utf-8` β†’ `application/gzip`). +fn content_type_to_compression(content_type: &str) -> Option { + let base_type = content_type + .split(';') + .next() + .unwrap_or(content_type) + .trim(); + match base_type { + "application/gzip" | "application/x-gzip" => Some(Compression::Gzip), + "application/zstd" => Some(Compression::Zstd), + _ => None, + } +} + +/// Determines compression from blob file extension. +fn blob_name_to_compression(blob_name: &str) -> Option { + let extension = std::path::Path::new(blob_name) + .extension() + .and_then(std::ffi::OsStr::to_str); + + extension.and_then(|ext| match ext { + "gz" => Some(Compression::Gzip), + "zst" => Some(Compression::Zstd), + _ => None, + }) +} + +type BlobDataStream = Pin + Send>>; + +/// Outcome of blob stream consumption, passed to the completion handler. +pub(super) enum StreamResult { + /// All events delivered successfully β€” safe to delete queue message. + Delivered, + /// Upstream reported an error β€” retain queue message for retry. + Errored, + /// Permanently rejected β€” delete queue message to prevent infinite reprocessing. + Rejected, +} + +pub struct BlobWithAck { + pub(super) blob_data_stream: BlobDataStream, + /// Called after stream consumption to finalize queue message handling. + /// Encapsulates both the success action (delete queue message) and + /// read-error checking (retain queue message on framing errors). + pub(super) completion_handler: + Box Pin + Send>> + Send>, + pub(super) container: String, + pub(super) blob_name: String, +} + +type BlobWithAckStream = Pin + Send>>; + +struct AzureBlobStreamer { + shutdown: ShutdownSignal, + out: SourceSender, + log_namespace: LogNamespace, + acknowledge: bool, + delete_failed_message: bool, + decoder: Decoder, + bytes_received: Registered, + events_received: Registered, +} + +impl AzureBlobStreamer { + pub fn new( + shutdown: ShutdownSignal, + out: SourceSender, + log_namespace: LogNamespace, + acknowledge: bool, + delete_failed_message: bool, + framing: FramingConfig, + decoding: DeserializerConfig, + ) -> crate::Result { + Ok(Self { + shutdown, + out, + log_namespace, + acknowledge, + delete_failed_message, + decoder: DecodingConfig::new(framing, decoding, log_namespace).build()?, + bytes_received: register!(BytesReceived::from(Protocol::HTTP)), + events_received: register!(EventsReceived), + }) + } + + pub async fn run_streaming(mut self, mut blob_stream: BlobWithAckStream) -> Result<(), ()> { + debug!("Azure Blob source: starting blob event processing loop."); + + loop { + select! { + blob = blob_stream.next() => { + match blob{ + Some(blob) => { + self.process_blob(blob).await?; + } + None => { + break; // end of stream + } + } + }, + _ = self.shutdown.clone() => { + break; + } + } + } + + Ok(()) + } + + async fn process_blob(&mut self, blob: BlobWithAck) -> Result<(), ()> { + let (batch, receiver) = BatchNotifier::maybe_new_with_receiver(self.acknowledge); + let mut data_stream = blob.blob_data_stream; + let container = blob.container; + let blob_name = blob.blob_name; + let mut output_stream = { + let bytes_received = self.bytes_received.clone(); + let events_received = self.events_received.clone(); + let log_namespace = self.log_namespace; + let decoder = self.decoder.clone(); + let container = container.clone(); + let blob = blob_name.clone(); + stream! { + while let Some(chunk) = data_stream.next().await { + bytes_received.emit(ByteSize(chunk.len())); + let (events, _) = match decoder.deserializer_parse(chunk) { + Ok(result) => result, + Err(_error) => { + // Error is handled by codecs::Decoder, no further handling needed + continue; + } + }; + for mut event in events { + event = event.with_batch_notifier_option(&batch); + match event { + Event::Log(ref mut log_event) => { + log_namespace.insert_source_metadata( + AzureBlobConfig::NAME, + log_event, + Some(LegacyKey::Overwrite(path!("container"))), + path!("container"), + container.clone(), + ); + log_namespace.insert_source_metadata( + AzureBlobConfig::NAME, + log_event, + Some(LegacyKey::Overwrite(path!("blob"))), + path!("blob"), + blob.clone(), + ); + + // Insert timestamp metadata following AWS S3 pattern + let timestamp = chrono::Utc::now(); + match log_namespace { + LogNamespace::Vector => { + let ts_path = metadata_path!(AzureBlobConfig::NAME, "timestamp"); + log_event.insert(ts_path, timestamp); + let ingest_path = metadata_path!("vector", "ingest_timestamp"); + log_event.insert(ingest_path, timestamp); + } + LogNamespace::Legacy => { + if let Some(timestamp_key) = log_schema().timestamp_key() { + log_event.try_insert((PathPrefix::Event, timestamp_key), timestamp); + } + } + } + + events_received.emit(CountByteSize(1, event.estimated_json_encoded_size_of())); + yield event + } + _ => { + emit!(InvalidRowEventTypeError{event: &event}) + } + } + } + } + drop(batch); + }.boxed() + }; + + let send_error = match self.out.send_event_stream(&mut output_stream).await { + Ok(_) => None, + Err(error) => { + let (count, _) = output_stream.size_hint(); + emit!(StreamClosedError { count }); + Some(error) + } + }; + + drop(output_stream); + + if send_error.is_some() { + emit!(QueueMessageProcessingErrored {}); + return Ok(()); + } + + match receiver { + None => (blob.completion_handler)(StreamResult::Delivered).await, + Some(receiver) => match receiver.await { + BatchStatus::Delivered => { + (blob.completion_handler)(StreamResult::Delivered).await; + emit!(QueueMessageProcessingSucceeded {}); + } + BatchStatus::Errored => { + (blob.completion_handler)(StreamResult::Errored).await; + emit!(QueueMessageProcessingErrored {}); + } + BatchStatus::Rejected => { + if self.delete_failed_message { + warn!( + message = "Blob events rejected by sink. Deleting queue message per config.", + container = %container, + blob = %blob_name, + ); + (blob.completion_handler)(StreamResult::Rejected).await; + } + emit!(QueueMessageProcessingRejected {}); + } + }, + } + + Ok(()) + } +} + +#[async_trait::async_trait] +#[typetag::serde(name = "azure_blob")] +impl SourceConfig for AzureBlobConfig { + async fn build(&self, cx: SourceContext) -> crate::Result { + self.validate()?; + + let multiline_config: Option = self + .multiline + .as_ref() + .map(|config| config.try_into()) + .transpose()?; + + let azure_blob_streamer = AzureBlobStreamer::new( + cx.shutdown.clone(), + cx.out.clone(), + cx.log_namespace(self.log_namespace), + cx.do_acknowledgements(self.acknowledgements), + self.delete_failed_message, + self.framing.clone(), + self.decoding.clone(), + )?; + + let blob_stream: BlobWithAckStream = match self.blob_stream_factory { + Some(ref factory) => factory(cx.shutdown.clone())?, + None => { + make_blob_with_ack_stream( + self, + cx.shutdown.clone(), + self.compression, + self.framing.clone(), + multiline_config, + &cx.proxy, + ) + .await? + } + }; + Ok(Box::pin(azure_blob_streamer.run_streaming(blob_stream))) + } + + fn outputs(&self, global_log_namespace: LogNamespace) -> Vec { + let log_namespace = global_log_namespace.merge(self.log_namespace); + let schema_definition = self + .decoding + .schema_definition(log_namespace) + .with_source_metadata( + Self::NAME, + Some(LegacyKey::Overwrite(owned_value_path!("container"))), + &owned_value_path!("container"), + Kind::bytes(), + None, + ) + .with_source_metadata( + Self::NAME, + Some(LegacyKey::Overwrite(owned_value_path!("blob"))), + &owned_value_path!("blob"), + Kind::bytes(), + None, + ) + .with_source_metadata( + Self::NAME, + None, + &owned_value_path!("timestamp"), + Kind::timestamp(), + Some("timestamp"), + ) + .with_standard_vector_source_metadata(); + + vec![SourceOutput::new_maybe_logs( + self.decoding.output_type(), + schema_definition, + )] + } + + fn can_acknowledge(&self) -> bool { + true + } +} diff --git a/src/sources/azure_blob/queue.rs b/src/sources/azure_blob/queue.rs new file mode 100644 index 0000000000000..ffceba3ce0c1b --- /dev/null +++ b/src/sources/azure_blob/queue.rs @@ -0,0 +1,756 @@ +use std::{pin::Pin, sync::Arc, sync::Mutex}; + +use async_compression::tokio::bufread::{GzipDecoder, ZstdDecoder}; +use async_stream::stream; +use azure_core::http::StatusCode; +use azure_storage_blob::models::{BlobClientDownloadOptions, BlobClientGetPropertiesResultHeaders}; +use azure_storage_blob::{BlobClient, BlobContainerClient}; +use azure_storage_queue::QueueClient; +use azure_storage_queue::models::{QueueClientReceiveMessagesOptions, ReceivedMessage}; +use base64::{Engine, prelude::BASE64_STANDARD}; +use bytes::Bytes; +use futures::{FutureExt, future::ready, stream::StreamExt}; +use serde::Deserialize; +use serde_with::serde_as; +use snafu::Snafu; +use tokio::{select, time}; +use tokio_util::{codec::FramedRead, io::StreamReader}; + +use vector_lib::configurable::configurable_component; + +use crate::{ + internal_events::{ + BlobDoesntExist, QueueMessageDeleteError, QueueMessageProcessingError, + QueueMessageReceiveError, QueueStorageInvalidEventIgnored, + QueueStorageMismatchingContainerName, + }, + line_agg::{self, LineAgg}, + shutdown::ShutdownSignal, + sources::azure_blob::{ + AzureBlobConfig, BlobWithAck, BlobWithAckStream, Compression, StreamResult, + determine_compression, + }, +}; + +/// Azure Queue configuration options. +#[serde_as] +#[configurable_component] +#[derive(Clone, Debug, Derivative)] +#[derivative(Default)] +#[serde(deny_unknown_fields)] +pub(super) struct Config { + /// The name of the storage queue to poll for events. + pub(super) queue_name: String, + + /// How long to wait while polling the event grid queue for new messages, in seconds. + /// + // Stored as u32 for serde compatibility, converted to u64 Duration. + #[serde(default = "default_poll_secs")] + #[derivative(Default(value = "default_poll_secs()"))] + #[configurable(metadata(docs::type_unit = "seconds"))] + pub(super) poll_secs: u32, + + /// Maximum number of messages to receive from the queue in a single request. + /// + /// Azure Storage Queues allow between 1 and 32 messages per request. + /// Since messages are processed sequentially, a lower value reduces the risk of + /// visibility timeouts expiring before messages are processed. + #[serde(default = "default_max_number_of_messages")] + #[derivative(Default(value = "default_max_number_of_messages()"))] + pub(super) max_number_of_messages: u8, + + /// Visibility timeout for received queue messages, in seconds. + /// + /// After a message is received, it becomes invisible to other consumers for this duration. + /// If processing takes longer than this timeout, the message may be received again by another + /// consumer (or the same consumer), potentially causing duplicate processing. + /// + /// Set this value higher than the expected time to process a full batch of messages. + /// Valid range: 1 to 604800 (7 days). + #[serde(default = "default_visibility_timeout_secs")] + #[derivative(Default(value = "default_visibility_timeout_secs()"))] + #[configurable(metadata(docs::type_unit = "seconds"))] + pub(super) visibility_timeout_secs: u32, +} + +/// Creates a stream of blobs with acknowledgements from the Azure Storage Queue. +/// +/// This function polls the configured Azure Storage Queue for Event Grid notifications +/// about blob creation/modification and rename events, then streams the blob contents +/// with acknowledgement handlers that delete queue messages upon successful processing. +/// +/// # Arguments +/// * `cfg` - The Azure Blob source configuration +/// * `shutdown` - Signal to gracefully shutdown the stream +/// * `compression` - The compression scheme to use for blob decompression +/// * `framing` - The framing configuration for splitting blob contents +/// * `multiline_config` - Optional multiline aggregation configuration +/// +/// # Returns +/// A pinned boxed stream of `BlobWithAck` items +/// +/// # Errors +/// Returns an error if queue client creation fails or configuration is invalid +pub async fn make_blob_with_ack_stream( + cfg: &AzureBlobConfig, + shutdown: ShutdownSignal, + compression: Compression, + framing: vector_lib::codecs::decoding::FramingConfig, + multiline_config: Option, + proxy: &crate::config::ProxyConfig, +) -> crate::Result { + let queue_client = make_queue_client(cfg, proxy).await?; + let container_client = make_container_client(cfg, proxy).await?; + let container_name = cfg.container_name.clone(); + let poll_interval = std::time::Duration::from_secs( + cfg.queue + .as_ref() + .ok_or("Missing Event Grid queue config.")? + .poll_secs as u64, + ); + let framer = framing.build(); + let queue_config = cfg.queue.as_ref().unwrap(); + let max_messages = queue_config.max_number_of_messages; + let visibility_timeout_secs = queue_config.visibility_timeout_secs; + + Ok(Box::pin(stream! { + loop { + let messages = match queue_client + .receive_messages(Some(QueueClientReceiveMessagesOptions { + number_of_messages: Some(max_messages as i32), + visibility_timeout: Some(visibility_timeout_secs as i32), + ..Default::default() + })) + .await + .and_then(|response| response.into_model()) + { + Ok(messages) => messages.items.unwrap_or_default(), + Err(e) => { + emit!(QueueMessageReceiveError{error: &e}); + continue; + } + }; + if !messages.is_empty() { + for message in messages { + let msg_id = message.message_id.clone().unwrap_or_default(); + match process_event_grid_message( + message, + &container_name, + &container_client, + &queue_client, + compression, + framer.clone(), + multiline_config.clone(), + ).await { + Ok(Some(bp)) => yield bp, + Ok(None) => trace!("Message {msg_id} processed, no blob stream produced (event may have been ignored or blob unavailable)."), + Err(e) => { + emit!(QueueMessageProcessingError{ + error: &e, + message_id: &msg_id + }); + } + } + } + // Check shutdown between message batches to avoid processing indefinitely + // when messages keep arriving. + if shutdown.clone().now_or_never().is_some() { + info!("Shutdown signal received, stopping Azure Blob queue polling."); + break; + } + } else { + select! { + _ = shutdown.clone() => { + info!("Shutdown signal received, stopping Azure Blob queue polling."); + break; + }, + _ = time::sleep(poll_interval) => { } + } + } + } + })) +} + +/// Creates an Azure Queue client from the source configuration. +/// +/// This function initializes a queue client using the connection string from the configuration. +/// The client is used to poll for Event Grid messages about blob events. +/// +/// # Arguments +/// * `cfg` - The Azure Blob source configuration containing connection string and queue name +/// +/// # Returns +/// An Arc-wrapped Azure QueueClient configured for the specified queue +/// +/// # Errors +/// Returns an error if: +/// - The queue configuration is missing +/// - The connection string is missing or invalid +/// - The queue service client cannot be initialized +pub async fn make_queue_client( + cfg: &AzureBlobConfig, + proxy: &crate::config::ProxyConfig, +) -> crate::Result> { + let q = cfg.queue.clone().ok_or("Missing queue.")?; + crate::azure::client::build_queue_client( + None, + cfg.connection_string.inner(), + &q.queue_name, + proxy, + None, + ) + .await + .map_err(|e| format!("Failed to create Azure queue client: {}", e).into()) +} + +/// Creates an Azure Blob Storage container client from the source configuration. +/// +/// This function initializes a container client using the connection string and container name +/// from the configuration. The client is used to read blob contents when blob events are received. +/// +/// # Arguments +/// * `cfg` - The Azure Blob source configuration containing connection string and container name +/// +/// # Returns +/// An Arc-wrapped Azure ContainerClient configured for the specified container +/// +/// # Errors +/// Returns an error if: +/// - The connection string is missing +/// - The container client cannot be initialized +/// - Azure authentication fails +pub async fn make_container_client( + cfg: &AzureBlobConfig, + proxy: &crate::config::ProxyConfig, +) -> crate::Result> { + crate::azure::client::build_client( + None, + cfg.connection_string.clone().into(), + cfg.container_name.clone(), + proxy, + None, + ) + .await + .map_err(|e| format!("Failed to create Azure container client: {}", e).into()) +} + +/// Applies decompression to an async reader based on the compression type. +/// +/// # Arguments +/// * `reader` - The async reader to wrap with decompression +/// * `compression` - The compression scheme to use +/// * `blob_name` - The name of the blob (used for auto-detection) +/// * `content_type` - Optional Content-Type header value (used for auto-detection) +/// +/// # Returns +/// A boxed async reader that decompresses the input stream +fn apply_decompression( + reader: Pin>, + compression: Compression, + blob_name: &str, + content_type: Option<&str>, +) -> Box { + let r = tokio::io::BufReader::new(reader); + + let compression = match compression { + Compression::Auto => { + determine_compression(content_type, blob_name).unwrap_or(Compression::None) + } + _ => compression, + }; + + use Compression::*; + match compression { + Auto => unreachable!(), + None => Box::new(r), + Gzip => { + let mut decoder = GzipDecoder::new(r); + decoder.multiple_members(true); + Box::new(decoder) + } + Zstd => { + let mut decoder = ZstdDecoder::new(r); + decoder.multiple_members(true); + Box::new(decoder) + } + } +} + +/// Creates a streaming async reader for a blob with optional decompression. +/// +/// # Arguments +/// * `blob_client` - The Azure blob client for the blob to stream +/// * `compression` - The compression configuration +/// * `blob_name` - The name of the blob +/// * `content_type` - Optional Content-Type header value +/// +/// # Returns +/// An async reader that streams and optionally decompresses blob contents +/// +/// # Errors +/// Returns an error if blob streaming fails +async fn create_blob_stream( + blob_client: &BlobClient, + compression: Compression, + blob_name: &str, + content_type: Option<&str>, +) -> Result, ProcessingError> { + // Limit the managed download to a single in-flight range request so blob + // contents stream sequentially with bounded memory, matching the previous + // (legacy SDK) chunked streaming behavior. + let download = blob_client + .download(Some(BlobClientDownloadOptions { + parallel: std::num::NonZero::new(1), + ..Default::default() + })) + .await + .map_err(|e| ProcessingError::FailedToGetBlob { error: e })?; + + let byte_stream = download + .body + .map(|chunk| chunk.map_err(std::io::Error::other)); + + let reader = Box::pin(StreamReader::new(byte_stream)); + Ok(apply_decompression( + reader, + compression, + blob_name, + content_type, + )) +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(super) struct AzureStorageEvent { + pub subject: String, + pub event_type: String, +} + +#[derive(Debug, Snafu)] +pub enum ProcessingError { + #[snafu(display("Could not decode Queue message with id {}: {}", message_id, error))] + InvalidQueueMessage { + error: serde_json::Error, + message_id: String, + }, + + #[snafu(display("Failed to base64 decode message: {}", error))] + FailedDecodingMessageBase64 { error: base64::DecodeError }, + + #[snafu(display("Failed to utf8 decode message: {}", error))] + FailedDecodingUTF8 { error: std::string::FromUtf8Error }, + + #[snafu(display("Failed to get blob: {}", error))] + FailedToGetBlob { error: azure_core::Error }, + + #[snafu(display("Failed to parse {} as subject", subject))] + FailedToParseSubject { subject: String }, + + #[snafu(display("Queue message {} is missing required fields", message_id))] + IncompleteMessage { message_id: String }, +} + +/// The identifiers required to delete a message from the queue after processing. +#[derive(Clone, Debug)] +struct MessageHandle { + message_id: String, + pop_receipt: String, +} + +impl MessageHandle { + /// Extract the deletion handle from a received message, failing if the service + /// response is missing either identifier. + fn from_message(message: &ReceivedMessage) -> Result { + match (message.message_id.as_ref(), message.pop_receipt.as_ref()) { + (Some(message_id), Some(pop_receipt)) => Ok(Self { + message_id: message_id.clone(), + pop_receipt: pop_receipt.clone(), + }), + _ => Err(ProcessingError::IncompleteMessage { + message_id: message.message_id.clone().unwrap_or_default(), + }), + } + } +} + +/// Processes an Azure Event Grid message from the storage queue. +/// +/// Decodes the queue message, validates the blob event type (BlobCreated/BlobRenamed), +/// creates a streaming reader for the blob content with optional decompression and framing, +/// and returns a `BlobWithAck` with an acknowledgement handler that deletes the queue message. +/// +/// # Returns +/// - `Ok(Some(BlobWithAck))` - Successfully created blob stream +/// - `Ok(None)` - Event ignored (wrong type, wrong container, or blob doesn't exist) +/// - `Err(ProcessingError)` - Failed to process message or retrieve blob +async fn process_event_grid_message( + message: ReceivedMessage, + container_name: &str, + container_client: &BlobContainerClient, + queue_client: &Arc, + compression: Compression, + framer: vector_lib::codecs::decoding::Framer, + multiline_config: Option, +) -> Result, ProcessingError> { + let handle = MessageHandle::from_message(&message)?; + let message_text = + message + .message_text + .as_deref() + .ok_or_else(|| ProcessingError::IncompleteMessage { + message_id: handle.message_id.clone(), + })?; + let decoded_bytes = BASE64_STANDARD + .decode(message_text) + .map_err(|e| ProcessingError::FailedDecodingMessageBase64 { error: e })?; + let decoded_string = String::from_utf8(decoded_bytes) + .map_err(|e| ProcessingError::FailedDecodingUTF8 { error: e })?; + let body: AzureStorageEvent = serde_json::from_str(decoded_string.as_str()).map_err(|e| { + ProcessingError::InvalidQueueMessage { + error: e, + message_id: handle.message_id.clone(), + } + })?; + if body.event_type != "Microsoft.Storage.BlobCreated" + && body.event_type != "Microsoft.Storage.BlobRenamed" + { + emit!(QueueStorageInvalidEventIgnored { + container: container_name, + subject: &body.subject, + event_type: &body.event_type, + }); + remove_message_from_queue(queue_client, &handle).await; + return Ok(None); + } + let (container, blob) = + parse_subject(body.subject.clone()).ok_or(ProcessingError::FailedToParseSubject { + subject: body.subject, + })?; + + if container != container_name { + emit!(QueueStorageMismatchingContainerName { + configured_container: container_name, + container: container.as_str(), + }); + remove_message_from_queue(queue_client, &handle).await; + return Ok(None); + } + trace!( + "Detected blob event ({}) in container '{}': '{}'.", + &body.event_type, &container, &blob + ); + let blob_client = container_client.blob_client(&blob); + + // Get blob properties to determine content type for compression auto-detection. + // Note: Content-Encoding header is NOT used due to Azure SDK limitation. + // When blobs have Content-Encoding set, the SDK fails with "header not found content-length" + // because Azure uses chunked transfer encoding for such responses. + let content_type = match blob_client.get_properties(None).await { + Ok(response) => response + .content_type() + .map_err(|e| ProcessingError::FailedToGetBlob { error: e })?, + Err(e) => { + // Handle 404 (blob doesn't exist) + if e.http_status() == Some(StatusCode::NotFound) { + emit!(BlobDoesntExist { + nonexistent_blob_name: &blob, + }); + remove_message_from_queue(queue_client, &handle).await; + return Ok(None); + } + return Err(ProcessingError::FailedToGetBlob { error: e }); + } + }; + + // Create streaming decompressing reader + let object_reader = + create_blob_stream(&blob_client, compression, &blob, content_type.as_deref()).await?; + + // Use FramedRead with configurable framer + let queue_client_copy = Arc::clone(queue_client); + let read_error: Arc>> = Arc::new(Mutex::new(None)); + let read_error_for_handler = Arc::clone(&read_error); + let blob_for_warn = blob.clone(); + let container_for_warn = container.clone(); + + Ok(Some(BlobWithAck { + blob_data_stream: Box::pin({ + let blob_for_error = blob.clone(); + let read_error_flag = Arc::clone(&read_error); + let lines: Box + Send + Unpin> = Box::new( + FramedRead::new(object_reader, framer) + .map(move |res| { + res.map_err(|err| { + error!("Framing error for blob '{}': {}", blob_for_error, err); + *read_error_flag.lock().unwrap() = Some(format!("{}", err)); + }) + .ok() + }) + .take_while(|res| ready(res.is_some())) + .map(|r| r.expect("validated by take_while")), + ); + + // Apply multiline aggregation if configured + let lines: Box + Send + Unpin> = + match multiline_config { + Some(config) => Box::new( + LineAgg::new( + lines.map(|line| ((), line, ())), + line_agg::Logic::new(config), + ) + .map(|(_src, line, _ctx, _lastline_ctx)| line), + ), + None => lines, + }; + + lines + }), + completion_handler: Box::new(move |result: StreamResult| { + Box::pin(async move { + match result { + StreamResult::Delivered => { + if let Some(err) = read_error_for_handler.lock().unwrap().take() { + warn!( + "Read error for blob '{}' in container '{}': {}. Queue message retained for retry.", + blob_for_warn, container_for_warn, err + ); + return; + } + remove_message_from_queue(&queue_client_copy, &handle).await; + } + StreamResult::Rejected => { + remove_message_from_queue(&queue_client_copy, &handle).await; + } + StreamResult::Errored => { + // Retain queue message for retry on transient errors. + } + } + }) + }), + container, + blob_name: blob, + })) +} + +/// Parses the subject field from an Azure Event Grid notification. +/// +/// The subject field contains the blob path in the format: +/// `/blobServices/default/containers/{container}/blobs/{blob-path}` +/// +/// This function extracts the container name and blob path from the subject string. +/// +/// # Arguments +/// * `subject` - The subject string from an Azure Event Grid blob event +/// +/// # Returns +/// A tuple containing (container_name, blob_path) if parsing succeeds, None otherwise +/// +/// # Examples +/// ``` +/// let subject = "/blobServices/default/containers/logs/blobs/2024/01/file.txt"; +/// let result = parse_subject(subject.to_string()); +/// assert_eq!(result, Some(("logs".to_string(), "2024/01/file.txt".to_string()))); +/// ``` +fn parse_subject(subject: String) -> Option<(String, String)> { + let parts: Vec<&str> = subject.split('/').collect(); + if parts.len() < 7 + || parts[1] != "blobServices" + || parts[3] != "containers" + || parts[5] != "blobs" + { + warn!( + "Ignoring event: subject has invalid format (expected /blobServices/default/containers/{{container}}/blobs/{{blob}}), got: '{}'.", + subject + ); + return None; + } + let container = parts[4]; + let blob = parts[6..].join("/"); + Some((container.to_string(), blob)) +} + +const fn default_poll_secs() -> u32 { + 15 +} + +const fn default_max_number_of_messages() -> u8 { + 10 +} + +const fn default_visibility_timeout_secs() -> u32 { + 300 +} + +async fn remove_message_from_queue(queue_client: &QueueClient, handle: &MessageHandle) { + _ = queue_client + .delete_message(&handle.message_id, &handle.pop_receipt, None) + .await + .inspect_err(move |e| emit!(QueueMessageDeleteError { error: &e })) +} + +#[test] +fn test_azure_storage_event_deserialization() { + let event_value: AzureStorageEvent = serde_json::from_str( + r#"{ + "topic": "/subscriptions/fa5f2180-1451-4461-9b1f-aae7d4b33cf8/resourceGroups/events_poc/providers/Microsoft.Storage/storageAccounts/eventspocaccount", + "subject": "/blobServices/default/containers/content/blobs/foo", + "eventType": "Microsoft.Storage.BlobCreated", + "id": "be3f21f7-201e-000b-7605-a29195062628", + "data": { + "api": "PutBlob", + "clientRequestId": "1fa42c94-6dd3-4172-95c4-fd9cf56b5009", + "requestId": "be3f21f7-201e-000b-7605-a29195000000", + "eTag": "0x8DC701C5D3FFDF6", + "contentType": "application/octet-stream", + "contentLength": 0, + "blobType": "BlockBlob", + "url": "https://eventspocaccount.blob.core.windows.net/content/foo", + "sequencer": "0000000000000000000000000005C5360000000000276a63", + "storageDiagnostics": { + "batchId": "fec5b12c-2006-0034-0005-a25936000000" + } + }, + "dataVersion": "", + "metadataVersion": "1", + "eventTime": "2024-05-09T11:37:10.5637878Z" + }"#, + ).unwrap(); + + assert_eq!( + event_value.subject, + "/blobServices/default/containers/content/blobs/foo" + ); + assert_eq!(event_value.event_type, "Microsoft.Storage.BlobCreated"); +} + +#[test] +fn test_parse_subject() { + // (input, expected_container, expected_blob) β€” None means parse should fail + let cases: Vec<(&str, Option<(&str, &str)>)> = vec![ + // Simple blob name + ( + "/blobServices/default/containers/content/blobs/foo", + Some(("content", "foo")), + ), + // Single file in container + ( + "/blobServices/default/containers/logs/blobs/file.txt", + Some(("logs", "file.txt")), + ), + // Nested path + ( + "/blobServices/default/containers/logs/blobs/path/to/file.txt", + Some(("logs", "path/to/file.txt")), + ), + // Deep nested path with dates + ( + "/blobServices/default/containers/data/blobs/2024/01/15/logs/app.log", + Some(("data", "2024/01/15/logs/app.log")), + ), + // Real-world Azure log path + ( + "/blobServices/default/containers/insights-logs-signinlogs/blobs/tenantId=0e35ee7a-425d-45a5-9013-218c1eae8fd4/y=2024/m=06/d=20/h=05/m=00/PT1H.json", + Some(( + "insights-logs-signinlogs", + "tenantId=0e35ee7a-425d-45a5-9013-218c1eae8fd4/y=2024/m=06/d=20/h=05/m=00/PT1H.json", + )), + ), + // Special characters (URL-encoded spaces) + ( + "/blobServices/default/containers/my-container/blobs/file%20with%20spaces.txt", + Some(("my-container", "file%20with%20spaces.txt")), + ), + // Unicode + ( + "/blobServices/default/containers/donnΓ©es/blobs/Ρ„Π°ΠΉΠ».txt", + Some(("donnΓ©es", "Ρ„Π°ΠΉΠ».txt")), + ), + // Path traversal preserved as-is (Azure handles security) + ( + "/blobServices/default/containers/backup/blobs/../../../etc/passwd", + Some(("backup", "../../../etc/passwd")), + ), + // Invalid: too short + ("/blobServices/default", None), + // Invalid: empty + ("", None), + // Invalid: wrong format + ("not/a/valid/subject", None), + // Invalid: wrong fixed segment "blobServices" + ("/wrongSegment/default/containers/content/blobs/foo", None), + // Invalid: wrong fixed segment "containers" + ("/blobServices/default/wrongSegment/content/blobs/foo", None), + // Invalid: wrong fixed segment "blobs" + ( + "/blobServices/default/containers/content/wrongSegment/foo", + None, + ), + // Invalid: enough segments but all wrong + ("/a/b/c/d/e/f/g", None), + ]; + + for (subject, expected) in cases { + let result = parse_subject(subject.to_string()); + match expected { + Some((container, blob)) => { + let (c, b) = + result.unwrap_or_else(|| panic!("Expected Some for subject: {subject}")); + assert_eq!(c, container, "container mismatch for subject: {subject}"); + assert_eq!(b, blob, "blob mismatch for subject: {subject}"); + } + None => { + assert!(result.is_none(), "Expected None for subject: {subject}"); + } + } + } +} + +#[test] +fn test_config_deny_unknown_fields() { + let json = r#"{"queue_name": "test", "poll_secs": 10, "unknown_field": "value"}"#; + let result: Result = serde_json::from_str(json); + assert!(result.is_err()); +} + +#[tokio::test] +async fn test_make_queue_client_invalid_connection_string() { + use crate::sources::azure_blob::AzureBlobConfig; + + let config = AzureBlobConfig { + connection_string: "invalid-connection-string".to_string().into(), + container_name: "test".to_string(), + queue: Some(Config { + queue_name: "queue".to_string(), + poll_secs: 10, + ..Default::default() + }), + ..Default::default() + }; + + let result = make_queue_client(&config, &crate::config::ProxyConfig::default()).await; + assert!(result.is_err()); +} + +#[tokio::test] +async fn test_make_container_client_with_connection_string() { + use crate::sources::azure_blob::AzureBlobConfig; + + let config = AzureBlobConfig { + connection_string: "DefaultEndpointsProtocol=https;AccountName=test;AccountKey=dGVzdA==;EndpointSuffix=core.windows.net".to_string().into(), + container_name: "test-container".to_string(), + queue: Some(Config { + queue_name: "test-queue".to_string(), + poll_secs: default_poll_secs(), + ..Default::default() + }), + ..Default::default() + }; + + let result = make_container_client(&config, &crate::config::ProxyConfig::default()).await; + assert!(result.is_ok()); + let client = result.unwrap(); + assert_eq!( + client.url().as_str(), + "https://test.blob.core.windows.net/test-container" + ); +} diff --git a/src/sources/azure_blob/test.rs b/src/sources/azure_blob/test.rs new file mode 100644 index 0000000000000..a86e3f468329b --- /dev/null +++ b/src/sources/azure_blob/test.rs @@ -0,0 +1,345 @@ +use super::*; +use crate::{ + SourceSender, config::LogNamespace, event::EventStatus, serde::default_decoding, + shutdown::ShutdownSignal, test_util::collect_n, +}; +use std::time::Duration; +use tokio::{select, sync::oneshot, time}; + +#[tokio::test] +async fn test_messages_delivered() { + let (tx, rx) = SourceSender::new_test_finalize(EventStatus::Delivered); + let streamer = super::AzureBlobStreamer::new( + ShutdownSignal::noop(), + tx, + LogNamespace::Vector, + true, + true, + default_framing(), + default_decoding(), + ); + let mut streamer = streamer.expect("Failed to create streamer"); + let (success_sender, success_receiver) = oneshot::channel(); + let blob = BlobWithAck { + blob_data_stream: Box::pin(stream! { + let lines = vec!["foo", "bar"]; + for line in lines { + yield Bytes::from(line.as_bytes().to_vec()); + } + }), + completion_handler: Box::new(move |_result: StreamResult| { + Box::pin(async move { + success_sender.send(()).unwrap(); + }) + }), + container: "test-container".to_string(), + blob_name: "test-blob.log".to_string(), + }; + let (events_collector, events_receiver) = oneshot::channel(); + tokio::spawn(async move { + events_collector.send(collect_n(rx, 2).await).unwrap(); + }); + streamer + .process_blob(blob) + .await + .expect("Failed processing blob"); + + let events = select! { + value = events_receiver => value.expect("Failed to receive events"), + _ = time::sleep(Duration::from_secs(5)) => panic!("Timeout waiting for events"), + }; + assert_eq!(events[0].as_log().value().to_string(), "\"foo\""); + assert_eq!(events[1].as_log().value().to_string(), "\"bar\""); + select! { + _ = success_receiver => {} + _ = time::sleep(Duration::from_secs(5)) => panic!("Timeout waiting for success handler"), + } +} + +#[tokio::test] +async fn test_messages_rejected_delete_failed_message() { + let (tx, rx) = SourceSender::new_test_finalize(EventStatus::Rejected); + let streamer = super::AzureBlobStreamer::new( + ShutdownSignal::noop(), + tx, + LogNamespace::Vector, + true, + true, // delete_failed_message + default_framing(), + default_decoding(), + ); + let mut streamer = streamer.expect("Failed to create streamer"); + let (success_sender, success_receiver) = oneshot::channel(); + let blob = BlobWithAck { + blob_data_stream: Box::pin(stream! { + let lines = vec!["foo", "bar"]; + for line in lines { + yield Bytes::from(line.as_bytes().to_vec()); + } + }), + completion_handler: Box::new(move |_result: StreamResult| { + Box::pin(async move { + success_sender.send(()).unwrap(); + }) + }), + container: "test-container".to_string(), + blob_name: "test-blob.log".to_string(), + }; + let (events_collector, events_receiver) = oneshot::channel(); + tokio::spawn(async move { + events_collector.send(collect_n(rx, 2).await).unwrap(); + }); + streamer + .process_blob(blob) + .await + .expect("Failed processing blob"); + + let events = select! { + value = events_receiver => value.expect("Failed to receive events"), + _ = time::sleep(Duration::from_secs(5)) => panic!("Timeout waiting for events"), + }; + assert_eq!(events[0].as_log().value().to_string(), "\"foo\""); + assert_eq!(events[1].as_log().value().to_string(), "\"bar\""); + // With delete_failed_message=true, completion handler IS called on rejection + select! { + _ = success_receiver => {} + _ = time::sleep(Duration::from_secs(5)) => panic!("Timeout waiting for completion handler"), + } +} + +#[tokio::test] +async fn test_messages_rejected_retain_failed_message() { + let (tx, rx) = SourceSender::new_test_finalize(EventStatus::Rejected); + let streamer = super::AzureBlobStreamer::new( + ShutdownSignal::noop(), + tx, + LogNamespace::Vector, + true, + false, // delete_failed_message = false: retain on rejection + default_framing(), + default_decoding(), + ); + let mut streamer = streamer.expect("Failed to create streamer"); + let (success_sender, mut success_receiver) = oneshot::channel(); + let blob = BlobWithAck { + blob_data_stream: Box::pin(stream! { + let lines = vec!["foo", "bar"]; + for line in lines { + yield Bytes::from(line.as_bytes().to_vec()); + } + }), + completion_handler: Box::new(move |_result: StreamResult| { + Box::pin(async move { + success_sender.send(()).unwrap(); + }) + }), + container: "test-container".to_string(), + blob_name: "test-blob.log".to_string(), + }; + let (events_collector, events_receiver) = oneshot::channel(); + tokio::spawn(async move { + events_collector.send(collect_n(rx, 2).await).unwrap(); + }); + streamer + .process_blob(blob) + .await + .expect("Failed processing blob"); + + let events = select! { + value = events_receiver => value.expect("Failed to receive events"), + _ = time::sleep(Duration::from_secs(5)) => panic!("Timeout waiting for events"), + }; + assert_eq!(events[0].as_log().value().to_string(), "\"foo\""); + assert_eq!(events[1].as_log().value().to_string(), "\"bar\""); + // With delete_failed_message=false, completion handler is NOT called on rejection + assert!(success_receiver.try_recv().is_err()); +} + +// Test blob with JSON decoding +#[tokio::test] +async fn test_json_decoding_blob() { + use tokio::{select, time}; + use vector_lib::codecs::decoding::{DeserializerConfig, JsonDeserializerConfig}; + use vrl::value; + + let (tx, rx) = SourceSender::new_test_finalize(EventStatus::Delivered); + let streamer = super::AzureBlobStreamer::new( + ShutdownSignal::noop(), + tx, + LogNamespace::Vector, + true, + true, + default_framing(), + DeserializerConfig::Json(JsonDeserializerConfig::default()), + ); + let mut streamer = streamer.expect("Failed to create streamer"); + + let json_line = r#"{"level":"info","message":"test log","timestamp":"2023-01-01T00:00:00Z"}"#; + let blob = BlobWithAck { + blob_data_stream: Box::pin(stream! { + yield Bytes::from(json_line.as_bytes().to_vec()); + }), + completion_handler: Box::new(|_result: StreamResult| Box::pin(async move {})), + container: "test-container".to_string(), + blob_name: "json-blob.log".to_string(), + }; + + let (events_collector, events_receiver) = oneshot::channel(); + tokio::spawn(async move { + events_collector.send(collect_n(rx, 1).await).unwrap(); + }); + + streamer + .process_blob(blob) + .await + .expect("Failed processing blob with JSON"); + + let events = select! { + value = events_receiver => value.expect("Failed to receive events"), + _ = time::sleep(Duration::from_secs(5)) => panic!("Timeout waiting for events"), + }; + + assert_eq!(events.len(), 1); + let log = events[0].as_log(); + assert_eq!(log["level"], value!("info")); + assert_eq!(log["message"], value!("test log")); + assert_eq!(log["timestamp"], value!("2023-01-01T00:00:00Z")); +} + +// Test LogNamespace handling +#[tokio::test] +async fn test_log_namespace_legacy() { + use tokio::{select, time}; + + let (tx, rx) = SourceSender::new_test_finalize(EventStatus::Delivered); + let streamer = super::AzureBlobStreamer::new( + ShutdownSignal::noop(), + tx, + LogNamespace::Legacy, + true, + true, + default_framing(), + default_decoding(), + ); + let mut streamer = streamer.expect("Failed to create streamer"); + + let blob = BlobWithAck { + blob_data_stream: Box::pin(stream! { + yield Bytes::from("legacy_test".as_bytes().to_vec()); + }), + completion_handler: Box::new(|_result: StreamResult| Box::pin(async move {})), + container: "test-container".to_string(), + blob_name: "legacy-blob.log".to_string(), + }; + + let (events_collector, events_receiver) = oneshot::channel(); + tokio::spawn(async move { + events_collector.send(collect_n(rx, 1).await).unwrap(); + }); + + streamer + .process_blob(blob) + .await + .expect("Failed processing blob"); + + let events = select! { + value = events_receiver => value.expect("Failed to receive events"), + _ = time::sleep(Duration::from_secs(5)) => panic!("Timeout waiting for events"), + }; + + assert_eq!(events.len(), 1); + // In Legacy namespace, data should be in message field + assert_eq!( + events[0].as_log()["message"].to_string_lossy(), + "legacy_test" + ); +} + +// ===== Compression Helper Tests ===== + +#[test] +fn test_compression_detection_from_extensions() { + // Test various file extensions + assert_eq!( + super::determine_compression(None, "file.gz"), + Some(super::Compression::Gzip) + ); + assert_eq!( + super::determine_compression(None, "file.tar.gz"), + Some(super::Compression::Gzip) + ); + assert_eq!( + super::determine_compression(None, "/path/to/file.gz"), + Some(super::Compression::Gzip) + ); + assert_eq!( + super::determine_compression(None, "file.zst"), + Some(super::Compression::Zstd) + ); + assert_eq!(super::determine_compression(None, "file.txt"), None); + assert_eq!(super::determine_compression(None, "file"), None); +} + +#[test] +fn test_content_type_to_compression() { + // Test gzip variants + assert_eq!( + super::content_type_to_compression("application/gzip"), + Some(super::Compression::Gzip) + ); + assert_eq!( + super::content_type_to_compression("application/x-gzip"), + Some(super::Compression::Gzip) + ); + + // Test zstd + assert_eq!( + super::content_type_to_compression("application/zstd"), + Some(super::Compression::Zstd) + ); + + // Test unknown types + assert_eq!(super::content_type_to_compression("text/plain"), None); + assert_eq!(super::content_type_to_compression("application/json"), None); + + // Test MIME parameters are stripped + assert_eq!( + super::content_type_to_compression("application/gzip; charset=utf-8"), + Some(super::Compression::Gzip) + ); + assert_eq!( + super::content_type_to_compression("application/x-gzip; boundary=something"), + Some(super::Compression::Gzip) + ); + assert_eq!( + super::content_type_to_compression("application/zstd; q=0.9"), + Some(super::Compression::Zstd) + ); +} + +#[test] +fn test_compression_detection_with_content_type() { + // Content-Type alone + assert_eq!( + super::determine_compression(Some("application/gzip"), "file.txt"), + Some(super::Compression::Gzip) + ); + + // Content-Type takes priority over file extension + assert_eq!( + super::determine_compression(Some("application/zstd"), "file.gz"), + Some(super::Compression::Zstd) + ); + + // Content-Type with x-gzip variant + assert_eq!( + super::determine_compression(Some("application/x-gzip"), "file.txt"), + Some(super::Compression::Gzip) + ); + + // Unknown content type falls back to file extension + assert_eq!( + super::determine_compression(Some("text/plain"), "file.gz"), + Some(super::Compression::Gzip) + ); +} diff --git a/src/sources/datadog_agent/metrics.rs b/src/sources/datadog_agent/metrics.rs index d931be05fcab7..6f377e826a8b8 100644 --- a/src/sources/datadog_agent/metrics.rs +++ b/src/sources/datadog_agent/metrics.rs @@ -289,6 +289,10 @@ pub(crate) fn decode_ddseries_v2( log_schema() .host_key() .and_then(|key| tags.replace(key.to_string(), r.name)); + } else if r.r#type.eq("device") { + // The `device` resource type is used by Agent checks (disk, SNMP/NDM, etc.) + // and must be preserved as a plain `device` tag to match the v1 series behavior. + tags.replace("device".into(), r.name); } else { // But to avoid losing information if this situation changes, any other resource type/name will be saved in the tags map tags.replace(format!("resource.{}", r.r#type), r.name); diff --git a/src/sources/datadog_agent/tests.rs b/src/sources/datadog_agent/tests.rs index 065b6ec5f78f5..b2dbf9691db00 100644 --- a/src/sources/datadog_agent/tests.rs +++ b/src/sources/datadog_agent/tests.rs @@ -2648,6 +2648,66 @@ async fn series_v2_split_metric_namespace_false() { .await; } +#[tokio::test] +async fn series_v2_device_resource_preserved_as_tag() { + assert_source_compliance(&HTTP_PUSH_SOURCE_TAGS, async { + let (rx, _, _, addr, _guard) = + source(EventStatus::Delivered, true, true, false, false).await; + + let series = vec![ddmetric_proto::metric_payload::MetricSeries { + resources: vec![ + ddmetric_proto::metric_payload::Resource { + r#type: "host".to_string(), + name: "test_host".to_string(), + }, + ddmetric_proto::metric_payload::Resource { + r#type: "device".to_string(), + name: "sda".to_string(), + }, + ], + metric: "system.disk.free".to_string(), + tags: vec!["env:prod".to_string()], + points: vec![ddmetric_proto::metric_payload::MetricPoint { + value: 100.0, + timestamp: 1542182950, + }], + r#type: ddmetric_proto::metric_payload::MetricType::Gauge as i32, + unit: "".to_string(), + source_type_name: "".to_string(), + interval: 0, + metadata: None, + }]; + + let series_payload = ddmetric_proto::MetricPayload { series }; + let mut buf = Vec::new(); + series_payload.encode(&mut buf).unwrap(); + let body = unsafe { String::from_utf8_unchecked(buf) }; + + let events = send_and_collect( + addr, + body, + dd_api_key_headers(), + DD_API_SERIES_V2_PATH, + rx, + 1, + ) + .await; + + let metric = events[0].as_metric(); + let tags = metric.tags().unwrap(); + + // The `device` resource type must be preserved as a plain `device` tag, + // NOT as `resource.device`. This matches v1 series behavior. + assert_eq!(tags.get("device"), Some("sda")); + assert!( + tags.get("resource.device").is_none(), + "device should not be prefixed with 'resource.'" + ); + assert_eq!(tags.get("env"), Some("prod")); + }) + .await; +} + async fn test_sketches_split_metric_namespace_impl( split: bool, expected_name: &str, diff --git a/src/sources/demo_logs.rs b/src/sources/demo_logs.rs index 861a51a957d44..fe02da4574c80 100644 --- a/src/sources/demo_logs.rs +++ b/src/sources/demo_logs.rs @@ -94,8 +94,7 @@ pub enum DemoLogsConfigError { /// Output format configuration. #[configurable_component] -#[derive(Clone, Debug, Derivative)] -#[derivative(Default)] +#[derive(Clone, Debug, Default)] #[serde(tag = "format", rename_all = "snake_case")] #[configurable(metadata( docs::enum_tag_description = "The format of the randomly generated output." @@ -136,7 +135,7 @@ pub enum OutputFormat { /// Randomly generated HTTP server logs in [JSON][json] format. /// /// [json]: https://en.wikipedia.org/wiki/JSON - #[derivative(Default)] + #[default] Json, } diff --git a/src/sources/docker_logs/mod.rs b/src/sources/docker_logs/mod.rs index 45268788d45cc..8c3ef9bac102f 100644 --- a/src/sources/docker_logs/mod.rs +++ b/src/sources/docker_logs/mod.rs @@ -22,7 +22,6 @@ use chrono::{DateTime, FixedOffset, Local, ParseError, Utc}; use futures::{Stream, StreamExt}; use serde_with::serde_as; use tokio::sync::mpsc; -use tracing_futures::Instrument; use vector_lib::{ codecs::{BytesDeserializer, BytesDeserializerConfig}, config::{LegacyKey, LogNamespace}, @@ -740,39 +739,36 @@ impl EventStreamBuilder { /// Spawn a task to runs event stream until shutdown. fn start(&self, id: ContainerId, backoff: Option) -> ContainerState { let this = self.clone(); - tokio::spawn( - async move { - if let Some(duration) = backoff { - tokio::time::sleep(duration).await; - } + crate::spawn_in_current_span(async move { + if let Some(duration) = backoff { + tokio::time::sleep(duration).await; + } - match this - .core - .docker - .inspect_container(id.as_str(), None::) - .await - { - Ok(details) => match ContainerMetadata::from_details(details) { - Ok(metadata) => { - let info = ContainerLogInfo::new(id, metadata, this.core.now_timestamp); - this.run_event_stream(info).await; - return; - } - Err(error) => emit!(DockerLogsTimestampParseError { - error, - container_id: id.as_str() - }), - }, - Err(error) => emit!(DockerLogsContainerMetadataFetchError { + match this + .core + .docker + .inspect_container(id.as_str(), None::) + .await + { + Ok(details) => match ContainerMetadata::from_details(details) { + Ok(metadata) => { + let info = ContainerLogInfo::new(id, metadata, this.core.now_timestamp); + this.run_event_stream(info).await; + return; + } + Err(error) => emit!(DockerLogsTimestampParseError { error, container_id: id.as_str() }), - } - - this.finish(Err((id, ErrorPersistence::Transient))); + }, + Err(error) => emit!(DockerLogsContainerMetadataFetchError { + error, + container_id: id.as_str() + }), } - .in_current_span(), - ); + + this.finish(Err((id, ErrorPersistence::Transient))); + }); ContainerState::new_running() } @@ -781,7 +777,7 @@ impl EventStreamBuilder { fn restart(&self, container: &mut ContainerState) { if let Some(info) = container.take_info() { let this = self.clone(); - tokio::spawn(this.run_event_stream(info).in_current_span()); + crate::spawn_in_current_span(this.run_event_stream(info)); } } diff --git a/src/sources/docker_logs/tests.rs b/src/sources/docker_logs/tests.rs index f5af697c1b9f1..4270f45843122 100644 --- a/src/sources/docker_logs/tests.rs +++ b/src/sources/docker_logs/tests.rs @@ -30,12 +30,12 @@ mod integration_tests { } use bollard::{ + models::ContainerCreateBody, query_parameters::{ CreateContainerOptionsBuilder, CreateImageOptionsBuilder, KillContainerOptions, ListImagesOptionsBuilder, RemoveContainerOptions, StartContainerOptions, WaitContainerOptions, }, - secret::ContainerCreateBody, }; use futures::{FutureExt, stream::TryStreamExt}; use itertools::Itertools as _; @@ -182,7 +182,7 @@ mod integration_tests { .create_image(options, None, None) .for_each(|item| async move { let info = item.unwrap(); - if let Some(error) = info.error { + if let Some(error) = info.error_detail { panic!("{error:?}"); } }) diff --git a/src/sources/exec/mod.rs b/src/sources/exec/mod.rs index b87452ef107d1..48e3c14454d11 100644 --- a/src/sources/exec/mod.rs +++ b/src/sources/exec/mod.rs @@ -724,7 +724,7 @@ fn spawn_reader_thread( sender: Sender<((SmallVec<[Event; 1]>, usize), &'static str)>, ) { // Start the green background thread for collecting - drop(tokio::spawn(async move { + drop(crate::spawn_in_current_span(async move { debug!("Start capturing {} command output.", origin); let mut stream = DecoderFramedRead::new(reader, decoder); diff --git a/src/sources/file.rs b/src/sources/file.rs index 62a95db362f10..d485c6ad6b4b6 100644 --- a/src/sources/file.rs +++ b/src/sources/file.rs @@ -580,7 +580,7 @@ pub fn file_source( // checkpoints until all the acks have come in. let (send_shutdown, shutdown2) = oneshot::channel::<()>(); let checkpoints = checkpointer.view(); - tokio::spawn(async move { + crate::spawn_in_current_span(async move { while let Some((status, entry)) = ack_stream.next().await { if status == BatchStatus::Delivered { checkpoints.update(entry.file_id, entry.offset); @@ -832,6 +832,10 @@ mod tests { fs::{self, File}, future::Future, io::{Seek, Write}, + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }, }; use encoding_rs::UTF_16LE; @@ -847,7 +851,10 @@ mod tests { event::{Event, EventStatus, Value}, shutdown::ShutdownSignal, sources::file, - test_util::components::{FILE_SOURCE_TAGS, assert_source_compliance}, + test_util::{ + components::{FILE_SOURCE_TAGS, assert_source_compliance}, + wait_for_atomic_usize_timeout_ms, + }, }; #[test] @@ -856,12 +863,16 @@ mod tests { } fn test_default_file_config(dir: &tempfile::TempDir) -> file::FileConfig { + // Store checkpoints in a subdirectory so they don't appear in the + // glob-watched directory (which covers dir.path()/*). + let data_dir = dir.path().join(".data"); + fs::create_dir_all(&data_dir).unwrap(); file::FileConfig { fingerprint: FingerprintConfig::Checksum { ignored_header_bytes: 0, lines: 1, }, - data_dir: Some(dir.path().to_path_buf()), + data_dir: Some(data_dir), glob_minimum_cooldown_ms: Duration::from_millis(100), internal_metrics: FileInternalMetricsConfig { include_file_tag: true, @@ -962,9 +973,10 @@ mod tests { config.global.data_dir = global_dir.keep().into(); // local path given -- local should win + let local_data_dir = Some(local_dir.path().to_path_buf()); let res = config .global - .resolve_and_validate_data_dir(test_default_file_config(&local_dir).data_dir.as_ref()) + .resolve_and_validate_data_dir(local_data_dir.as_ref()) .unwrap(); assert_eq!(res, local_dir.path()); @@ -1153,7 +1165,7 @@ mod tests { let path1 = dir.path().join("file1"); let path2 = dir.path().join("file2"); - let received = run_file_source(&config, false, NoAcks, LogNamespace::Legacy, async { + let received = run_file_source(&config, false, NoAcks, LogNamespace::Legacy, None, async { let mut file1 = File::create(&path1).unwrap(); let mut file2 = File::create(&path2).unwrap(); @@ -1208,7 +1220,7 @@ mod tests { let path = dir.path().join("file"); - let received = run_file_source(&config, false, NoAcks, LogNamespace::Legacy, async { + let received = run_file_source(&config, false, NoAcks, LogNamespace::Legacy, None, async { let mut file = File::create(&path).unwrap(); writeln!(&mut file, "line for checkpointing").unwrap(); @@ -1234,7 +1246,7 @@ mod tests { ..test_default_file_config(&dir) }; let path = dir.path().join("file"); - let received = run_file_source(&config, false, NoAcks, LogNamespace::Legacy, async { + let received = run_file_source(&config, false, NoAcks, LogNamespace::Legacy, None, async { let mut file = File::create(&path).unwrap(); for i in 0..n { @@ -1297,7 +1309,7 @@ mod tests { let path = dir.path().join("file"); let archive_path = dir.path().join("file"); - let received = run_file_source(&config, false, NoAcks, LogNamespace::Legacy, async { + let received = run_file_source(&config, false, NoAcks, LogNamespace::Legacy, None, async { let mut file = File::create(&path).unwrap(); for i in 0..n { @@ -1365,7 +1377,7 @@ mod tests { let path2 = dir.path().join("b.txt"); let path3 = dir.path().join("a.log"); let path4 = dir.path().join("a.ignore.txt"); - let received = run_file_source(&config, false, NoAcks, LogNamespace::Legacy, async { + let received = run_file_source(&config, false, NoAcks, LogNamespace::Legacy, None, async { let mut file1 = File::create(&path1).unwrap(); let mut file2 = File::create(&path2).unwrap(); let mut file3 = File::create(&path3).unwrap(); @@ -1416,7 +1428,7 @@ mod tests { let path1 = dir.path().join("a//b/a.log.1"); let path2 = dir.path().join("a//b/test.log.1"); - let received = run_file_source(&config, false, NoAcks, LogNamespace::Legacy, async { + let received = run_file_source(&config, false, NoAcks, LogNamespace::Legacy, None, async { std::fs::create_dir_all(dir.path().join("a/b")).unwrap(); let mut file1 = File::create(&path1).unwrap(); let mut file2 = File::create(&path2).unwrap(); @@ -1469,15 +1481,16 @@ mod tests { }; let path = dir.path().join("file"); - let received = run_file_source(&config, true, acks, LogNamespace::Legacy, async { - let mut file = File::create(&path).unwrap(); + let received = + run_file_source(&config, true, acks, LogNamespace::Legacy, None, async { + let mut file = File::create(&path).unwrap(); - writeln!(&mut file, "hello there").unwrap(); - file.flush().unwrap(); + writeln!(&mut file, "hello there").unwrap(); + file.flush().unwrap(); - sleep_500_millis().await; - }) - .await; + sleep_500_millis().await; + }) + .await; assert_eq!(received.len(), 1); assert_eq!( @@ -1496,15 +1509,16 @@ mod tests { }; let path = dir.path().join("file"); - let received = run_file_source(&config, true, acks, LogNamespace::Legacy, async { - let mut file = File::create(&path).unwrap(); + let received = + run_file_source(&config, true, acks, LogNamespace::Legacy, None, async { + let mut file = File::create(&path).unwrap(); - writeln!(&mut file, "hello there").unwrap(); - file.flush().unwrap(); + writeln!(&mut file, "hello there").unwrap(); + file.flush().unwrap(); - sleep_500_millis().await; - }) - .await; + sleep_500_millis().await; + }) + .await; assert_eq!(received.len(), 1); assert_eq!( @@ -1522,15 +1536,16 @@ mod tests { }; let path = dir.path().join("file"); - let received = run_file_source(&config, true, acks, LogNamespace::Legacy, async { - let mut file = File::create(&path).unwrap(); + let received = + run_file_source(&config, true, acks, LogNamespace::Legacy, None, async { + let mut file = File::create(&path).unwrap(); - writeln!(&mut file, "hello there").unwrap(); + writeln!(&mut file, "hello there").unwrap(); - file.flush().unwrap(); - sleep_500_millis().await; - }) - .await; + file.flush().unwrap(); + sleep_500_millis().await; + }) + .await; assert_eq!(received.len(), 1); assert_eq!( @@ -1576,26 +1591,28 @@ mod tests { // First time server runs it picks up existing lines. { - let received = run_file_source(&config, true, acking, LogNamespace::Legacy, async { - sleep_500_millis().await; - writeln!(&mut file, "first line").unwrap(); - file.flush().unwrap(); - sleep_500_millis().await; - }) - .await; + let received = + run_file_source(&config, true, acking, LogNamespace::Legacy, None, async { + sleep_500_millis().await; + writeln!(&mut file, "first line").unwrap(); + file.flush().unwrap(); + sleep_500_millis().await; + }) + .await; let lines = extract_messages_string(received); assert_eq!(lines, vec!["zeroth line", "first line"]); } // Restart server, read file from checkpoint. { - let received = run_file_source(&config, true, acking, LogNamespace::Legacy, async { - sleep_500_millis().await; - writeln!(&mut file, "second line").unwrap(); - file.flush().unwrap(); - sleep_500_millis().await; - }) - .await; + let received = + run_file_source(&config, true, acking, LogNamespace::Legacy, None, async { + sleep_500_millis().await; + writeln!(&mut file, "second line").unwrap(); + file.flush().unwrap(); + sleep_500_millis().await; + }) + .await; let lines = extract_messages_string(received); assert_eq!(lines, vec!["second line"]); @@ -1608,13 +1625,14 @@ mod tests { read_from: ReadFromConfig::Beginning, ..test_default_file_config(&dir) }; - let received = run_file_source(&config, false, acking, LogNamespace::Legacy, async { - sleep_500_millis().await; - writeln!(&mut file, "third line").unwrap(); - file.flush().unwrap(); - sleep_500_millis().await; - }) - .await; + let received = + run_file_source(&config, false, acking, LogNamespace::Legacy, None, async { + sleep_500_millis().await; + writeln!(&mut file, "third line").unwrap(); + file.flush().unwrap(); + sleep_500_millis().await; + }) + .await; let lines = extract_messages_string(received); assert_eq!( @@ -1643,6 +1661,7 @@ mod tests { false, Unfinalized, LogNamespace::Legacy, + None, sleep(Duration::from_secs(5)), ) .await; @@ -1655,6 +1674,7 @@ mod tests { false, Unfinalized, LogNamespace::Legacy, + None, sleep(Duration::from_secs(5)), ) .await; @@ -1685,6 +1705,7 @@ mod tests { true, Acks, LogNamespace::Legacy, + None, // shutdown signal is sent after this duration sleep_500_millis(), ) @@ -1695,13 +1716,26 @@ mod tests { // bug we're testing for, which happens if the finalizer stream exits on shutdown with pending acks assert!(lines.len() < line_count); - // Restart the server, and it should read the rest without duplicating any + // Restart the server, and it should read the rest without duplicating any. + // Use the event counter to drain rx continuously (removing backpressure so + // the file server can read all remaining lines without being stalled), then + // trigger shutdown once all expected events have been received. + let remaining = line_count - lines.len(); + let event_count = Arc::new(AtomicUsize::new(0)); let received = run_file_source( &config, true, Acks, LogNamespace::Legacy, - sleep(Duration::from_secs(5)), + Some(Arc::clone(&event_count)), + async { + wait_for_atomic_usize_timeout_ms( + Arc::clone(&event_count), + |n| n >= remaining, + 5_000, + ) + .await; + }, ) .await; let lines2 = extract_messages_string(received); @@ -1731,13 +1765,14 @@ mod tests { let path_for_old_file = dir.path().join("file.old"); // Run server first time, collect some lines. { - let received = run_file_source(&config, true, acking, LogNamespace::Legacy, async { - let mut file = File::create(&path).unwrap(); - writeln!(&mut file, "first line").unwrap(); - file.flush().unwrap(); - sleep_500_millis().await; - }) - .await; + let received = + run_file_source(&config, true, acking, LogNamespace::Legacy, None, async { + let mut file = File::create(&path).unwrap(); + writeln!(&mut file, "first line").unwrap(); + file.flush().unwrap(); + sleep_500_millis().await; + }) + .await; let lines = extract_messages_string(received); assert_eq!(lines, vec!["first line"]); @@ -1747,13 +1782,14 @@ mod tests { // Restart the server and make sure it does not re-read the old file // even though it has a new name. { - let received = run_file_source(&config, false, acking, LogNamespace::Legacy, async { - let mut file = File::create(&path).unwrap(); - writeln!(&mut file, "second line").unwrap(); - file.flush().unwrap(); - sleep_500_millis().await; - }) - .await; + let received = + run_file_source(&config, false, acking, LogNamespace::Legacy, None, async { + let mut file = File::create(&path).unwrap(); + writeln!(&mut file, "second line").unwrap(); + file.flush().unwrap(); + sleep_500_millis().await; + }) + .await; let lines = extract_messages_string(received); assert_eq!(lines, vec!["second line"]); @@ -1815,7 +1851,7 @@ mod tests { before_file.sync_all().unwrap(); after_file.sync_all().unwrap(); - let received = run_file_source(&config, false, NoAcks, LogNamespace::Legacy, async { + let received = run_file_source(&config, false, NoAcks, LogNamespace::Legacy, None, async { sleep_500_millis().await; writeln!(&mut before_file, "second line").unwrap(); writeln!(&mut after_file, "_second line").unwrap(); @@ -1854,7 +1890,7 @@ mod tests { }; let path = dir.path().join("file"); - let received = run_file_source(&config, false, NoAcks, LogNamespace::Legacy, async { + let received = run_file_source(&config, false, NoAcks, LogNamespace::Legacy, None, async { let mut file = File::create(&path).unwrap(); writeln!(&mut file, "short").unwrap(); @@ -1891,37 +1927,48 @@ mod tests { let config = file::FileConfig { include: vec![dir.path().join("*")], message_start_indicator: Some("INFO".into()), - multi_line_timeout: 25, // less than 50 in sleep() + multi_line_timeout: 25, ..test_default_file_config(&dir) }; let path = dir.path().join("file"); - let received = run_file_source(&config, false, NoAcks, LogNamespace::Legacy, async { - let mut file = File::create(&path).unwrap(); - - writeln!(&mut file, "leftover foo").unwrap(); - writeln!(&mut file, "INFO hello").unwrap(); - writeln!(&mut file, "INFO goodbye").unwrap(); - writeln!(&mut file, "part of goodbye").unwrap(); - - file.flush().unwrap(); - sleep_500_millis().await; + let event_count = Arc::new(AtomicUsize::new(0)); + let received = run_file_source( + &config, + false, + NoAcks, + LogNamespace::Legacy, + Some(Arc::clone(&event_count)), + async { + let mut file = File::create(&path).unwrap(); - writeln!(&mut file, "INFO hi again").unwrap(); - writeln!(&mut file, "and some more").unwrap(); - writeln!(&mut file, "INFO hello").unwrap(); + // Write all lines through the second "INFO hello". Events 1-4 + // are emitted immediately by EndExclude; event 5 ("INFO hello" + // standalone) requires the 25ms timeout to fire. + writeln!(&mut file, "leftover foo").unwrap(); + writeln!(&mut file, "INFO hello").unwrap(); + writeln!(&mut file, "INFO goodbye").unwrap(); + writeln!(&mut file, "part of goodbye").unwrap(); + writeln!(&mut file, "INFO hi again").unwrap(); + writeln!(&mut file, "and some more").unwrap(); + writeln!(&mut file, "INFO hello").unwrap(); + file.flush().unwrap(); - file.flush().unwrap(); - sleep_500_millis().await; + // Block until event 5 is observed: the timeout fired and + // "INFO hello" was emitted before we write "too slow". + wait_for_atomic_usize_timeout_ms(Arc::clone(&event_count), |n| n >= 5, 500).await; - writeln!(&mut file, "too slow").unwrap(); - writeln!(&mut file, "INFO doesn't have").unwrap(); - writeln!(&mut file, "to be INFO in").unwrap(); - writeln!(&mut file, "the middle").unwrap(); + writeln!(&mut file, "too slow").unwrap(); + writeln!(&mut file, "INFO doesn't have").unwrap(); + writeln!(&mut file, "to be INFO in").unwrap(); + writeln!(&mut file, "the middle").unwrap(); + file.flush().unwrap(); - file.flush().unwrap(); - sleep_500_millis().await; - }) + // Wait for events 6 ("too slow") and 7 ("INFO doesn't have") + // before triggering shutdown. + wait_for_atomic_usize_timeout_ms(Arc::clone(&event_count), |n| n >= 7, 500).await; + }, + ) .await; let received = extract_messages_value(received); @@ -1950,38 +1997,49 @@ mod tests { start_pattern: "INFO".to_owned(), condition_pattern: "INFO".to_owned(), mode: line_agg::Mode::HaltBefore, - timeout_ms: Duration::from_millis(25), // less than 50 in sleep() + timeout_ms: Duration::from_millis(25), }), ..test_default_file_config(&dir) }; let path = dir.path().join("file"); - let received = run_file_source(&config, false, NoAcks, LogNamespace::Legacy, async { - let mut file = File::create(&path).unwrap(); - - writeln!(&mut file, "leftover foo").unwrap(); - writeln!(&mut file, "INFO hello").unwrap(); - writeln!(&mut file, "INFO goodbye").unwrap(); - writeln!(&mut file, "part of goodbye").unwrap(); - - file.flush().unwrap(); - sleep_500_millis().await; + let event_count = Arc::new(AtomicUsize::new(0)); + let received = run_file_source( + &config, + false, + NoAcks, + LogNamespace::Legacy, + Some(Arc::clone(&event_count)), + async { + let mut file = File::create(&path).unwrap(); - writeln!(&mut file, "INFO hi again").unwrap(); - writeln!(&mut file, "and some more").unwrap(); - writeln!(&mut file, "INFO hello").unwrap(); + // Write all lines through the second "INFO hello". Events 1-4 + // are emitted immediately by EndExclude; event 5 ("INFO hello" + // standalone) requires the 25ms timeout to fire. + writeln!(&mut file, "leftover foo").unwrap(); + writeln!(&mut file, "INFO hello").unwrap(); + writeln!(&mut file, "INFO goodbye").unwrap(); + writeln!(&mut file, "part of goodbye").unwrap(); + writeln!(&mut file, "INFO hi again").unwrap(); + writeln!(&mut file, "and some more").unwrap(); + writeln!(&mut file, "INFO hello").unwrap(); + file.flush().unwrap(); - file.flush().unwrap(); - sleep_500_millis().await; + // Block until event 5 is observed: the timeout fired and + // "INFO hello" was emitted before we write "too slow". + wait_for_atomic_usize_timeout_ms(Arc::clone(&event_count), |n| n >= 5, 500).await; - writeln!(&mut file, "too slow").unwrap(); - writeln!(&mut file, "INFO doesn't have").unwrap(); - writeln!(&mut file, "to be INFO in").unwrap(); - writeln!(&mut file, "the middle").unwrap(); + writeln!(&mut file, "too slow").unwrap(); + writeln!(&mut file, "INFO doesn't have").unwrap(); + writeln!(&mut file, "to be INFO in").unwrap(); + writeln!(&mut file, "the middle").unwrap(); + file.flush().unwrap(); - file.flush().unwrap(); - sleep_500_millis().await; - }) + // Wait for events 6 ("too slow") and 7 ("INFO doesn't have") + // before triggering shutdown. + wait_for_atomic_usize_timeout_ms(Arc::clone(&event_count), |n| n >= 7, 500).await; + }, + ) .await; let received = extract_messages_value(received); @@ -2024,12 +2082,14 @@ mod tests { file.sync_all().unwrap(); - // Read and aggregate existing lines + // Read and aggregate existing lines. wait_shutdown=true ensures the + // checkpoint is fully written to disk before the second run reads it. let received = run_file_source( &config, - false, + true, Acks, LogNamespace::Legacy, + None, sleep_500_millis(), ) .await; @@ -2041,7 +2101,7 @@ mod tests { // After restart, we should not see any part of the previously aggregated lines let received_after_restart = - run_file_source(&config, false, Acks, LogNamespace::Legacy, async { + run_file_source(&config, false, Acks, LogNamespace::Legacy, None, async { writeln!(&mut file, "INFO goodbye").unwrap(); file.flush().unwrap(); sleep_500_millis().await; @@ -2086,6 +2146,7 @@ mod tests { false, NoAcks, LogNamespace::Legacy, + None, sleep_500_millis(), ) .await; @@ -2148,6 +2209,7 @@ mod tests { false, NoAcks, LogNamespace::Legacy, + None, sleep_500_millis(), ) .await; @@ -2182,7 +2244,7 @@ mod tests { writeln!(&mut file, "hello i am a normal line").unwrap(); file.sync_all().unwrap(); - let received = run_file_source(&config, false, NoAcks, LogNamespace::Legacy, async { + let received = run_file_source(&config, false, NoAcks, LogNamespace::Legacy, None, async { sleep_500_millis().await; write!(&mut file, "i am not a full line").unwrap(); @@ -2229,6 +2291,7 @@ mod tests { false, NoAcks, LogNamespace::Legacy, + None, sleep_500_millis(), ) .await; @@ -2261,6 +2324,7 @@ mod tests { false, NoAcks, LogNamespace::Legacy, + None, sleep_500_millis(), ) .await; @@ -2289,7 +2353,7 @@ mod tests { }; let path = dir.path().join("file"); - let received = run_file_source(&config, false, NoAcks, LogNamespace::Legacy, async { + let received = run_file_source(&config, false, NoAcks, LogNamespace::Legacy, None, async { let mut file = File::create(&path).unwrap(); write!(&mut file, "hello i am a line\r\n").unwrap(); @@ -2328,7 +2392,7 @@ mod tests { }; let path = dir.path().join("file"); - let received = run_file_source(&config, false, NoAcks, LogNamespace::Legacy, async { + let received = run_file_source(&config, false, NoAcks, LogNamespace::Legacy, None, async { let mut file = File::create(&path).unwrap(); sleep_500_millis().await; @@ -2417,7 +2481,7 @@ mod tests { }; let path = dir.path().join("file"); - let received = run_file_source(&config, false, Acks, LogNamespace::Legacy, async { + let received = run_file_source(&config, false, Acks, LogNamespace::Legacy, None, async { let mut file = File::create(&path).unwrap(); for i in 0..n { @@ -2459,15 +2523,31 @@ mod tests { wait_shutdown: bool, acking_mode: AckingMode, log_namespace: LogNamespace, + // When `Some`, events are relayed through an unbounded channel and the + // counter is incremented for each event received. The inner future can + // call `wait_for_atomic_usize` on this counter to gate writes on + // observed events instead of relying on wall-clock sleeps. + event_counter: Option>, inner: impl Future, ) -> Vec { assert_source_compliance(&FILE_SOURCE_TAGS, async move { - let (tx, rx) = if acking_mode == Acks { - let (tx, rx) = SourceSender::new_test_finalize(EventStatus::Delivered); - (tx, rx.boxed()) - } else { - let (tx, rx) = SourceSender::new_test(); - (tx, rx.boxed()) + let (tx, rx) = match acking_mode { + Acks => { + let (tx, rx) = SourceSender::new_test_finalize(EventStatus::Delivered); + (tx, rx.boxed()) + } + Unfinalized => { + // Use Rejected so that events are finalized but checkpoints + // are NOT updated (only Delivered triggers checkpoint updates). + // This avoids a race where the default Delivered status on drop + // could leak checkpoint writes into the next run. + let (tx, rx) = SourceSender::new_test_finalize(EventStatus::Rejected); + (tx, rx.boxed()) + } + NoAcks => { + let (tx, rx) = SourceSender::new_test(); + (tx, rx.boxed()) + } }; let (trigger_shutdown, shutdown, shutdown_done) = ShutdownSignal::new_wired(); @@ -2483,21 +2563,47 @@ mod tests { log_namespace, )); - inner.await; + let result = if let Some(counter) = event_counter { + // Relay mode: a background task forwards events and increments + // the counter so `inner` can observe them without arbitrary sleeps. + let (relay_tx, mut relay_rx) = tokio::sync::mpsc::unbounded_channel::(); + tokio::spawn(async move { + let mut rx = rx; + while let Some(event) = rx.next().await { + counter.fetch_add(1, Ordering::SeqCst); + let _ = relay_tx.send(event); + } + }); - drop(trigger_shutdown); + inner.await; + drop(trigger_shutdown); - let result = if acking_mode == Unfinalized { - rx.take_until(tokio::time::sleep(Duration::from_secs(5))) - .collect::>() - .await + timeout(Duration::from_secs(5), async move { + let mut events = Vec::new(); + while let Some(event) = relay_rx.recv().await { + events.push(event); + } + events + }) + .await + .expect("Unclosed channel: may indicate file-server could not shutdown gracefully.") } else { - timeout(Duration::from_secs(5), rx.collect::>()) - .await - .expect( - "Unclosed channel: may indicate file-server could not shutdown gracefully.", - ) + inner.await; + drop(trigger_shutdown); + + if acking_mode == Unfinalized { + rx.take_until(tokio::time::sleep(Duration::from_secs(5))) + .collect::>() + .await + } else { + timeout(Duration::from_secs(5), rx.collect::>()) + .await + .expect( + "Unclosed channel: may indicate file-server could not shutdown gracefully.", + ) + } }; + if wait_shutdown { shutdown_done.await; } diff --git a/src/sources/gcp_pubsub.rs b/src/sources/gcp_pubsub.rs index b01664b7ade63..ddf96d19acf70 100644 --- a/src/sources/gcp_pubsub.rs +++ b/src/sources/gcp_pubsub.rs @@ -453,7 +453,7 @@ impl PubsubSource { // when it has an idle interval it will mark itself as not // busy. let busy_flag = Arc::new(AtomicBool::new(false)); - let task = tokio::spawn(self.clone().run(Arc::clone(&busy_flag))); + let task = crate::spawn_in_current_span(self.clone().run(Arc::clone(&busy_flag))); tasks.push(Task { task, busy_flag }); } diff --git a/src/sources/http_server.rs b/src/sources/http_server.rs index 7c444284adc6c..bc588fb713c0c 100644 --- a/src/sources/http_server.rs +++ b/src/sources/http_server.rs @@ -13,10 +13,13 @@ use vector_lib::{ }, config::{DataType, LegacyKey, LogNamespace}, configurable::configurable_component, - lookup::{lookup_v2::OptionalValuePath, owned_value_path, path}, + lookup::{PathPrefix, lookup_v2::OptionalValuePath, owned_value_path, path}, schema::Definition, }; -use vrl::value::{Kind, kind::Collection}; +use vrl::{ + path::ValuePath as _, + value::{Kind, ObjectMap, kind::Collection}, +}; use warp::http::HeaderMap; use crate::{ @@ -114,6 +117,14 @@ pub struct SimpleHttpConfig { #[configurable(metadata(docs::examples = "*"))] query_parameters: Vec, + /// HTTP authentication configuration. + /// + /// Use HTTP authentication with HTTPS only. The authentication credentials are passed as an + /// HTTP header without any additional encryption beyond what is provided by the transport itself. + /// + /// When using the `custom` strategy, the VRL program may write `%field = value` to enrich + /// authenticated events. These metadata fields are injected into the event body (legacy + /// namespace) or under `http_server.` in event metadata (Vector namespace). #[configurable(derived)] auth: Option, @@ -217,9 +228,16 @@ impl SimpleHttpConfig { ) .with_standard_vector_source_metadata(); - // for metadata that is added to the events dynamically from config options + // For metadata that is added to the events dynamically from config options. if log_namespace == LogNamespace::Legacy { - schema_definition = schema_definition.unknown_fields(Kind::bytes()); + // Custom auth programs can inject any VRL value, not just bytes; widen the unknown + // field kind accordingly so schema-aware downstream components don't reject events. + let unknown_kind = if matches!(self.auth, Some(HttpServerAuthConfig::Custom { .. })) { + Kind::any() + } else { + Kind::bytes() + }; + schema_definition = schema_definition.unknown_fields(unknown_kind); } schema_definition @@ -521,6 +539,42 @@ impl HttpSource for SimpleHttpSource { fn enable_source_ip(&self) -> bool { self.host_key.path.is_some() } + + /// Injects `%field` enrichment from a `custom` auth VRL program into events. + /// Both namespaces use insert-if-empty semantics so auth enrichment never + /// overwrites built-in source metadata (`path`, `host`, `headers`, …) that + /// `enrich_events` already populated. + /// Vector namespace: inserted into event metadata under `http_server.` for + /// all event types (Log, Metric, Trace). + /// Legacy namespace: inserted into the Log event body only (Metric/Trace are skipped). + fn inject_auth_enrichment(&self, events: &mut [Event], enrichment: ObjectMap) { + for event in events.iter_mut() { + match self.log_namespace { + LogNamespace::Vector => { + // metadata_mut() dispatches to Log, Metric, and Trace so every + // decoded event type receives the auth enrichment fields. + let meta = event.metadata_mut().value_mut(); + for (key, value) in &enrichment { + let key_str = key.as_str(); + let name_part = path!(SimpleHttpConfig::NAME); + let key_part = path!(key_str); + let full_path = name_part.concat(key_part); + if meta.get(full_path.clone()).is_none() { + meta.insert(full_path, value.clone()); + } + } + } + LogNamespace::Legacy => { + // Legacy enrichment targets the event body; only Log events have one. + if let Event::Log(log) = event { + for (key, value) in &enrichment { + log.try_insert((PathPrefix::Event, path!(key.as_str())), value.clone()); + } + } + } + } + } + } } #[cfg(test)] @@ -543,11 +597,15 @@ mod tests { config::LogNamespace, event::LogEvent, lookup::{ - OwnedTargetPath, PathPrefix, event_path, lookup_v2::OptionalValuePath, owned_value_path, + OwnedTargetPath, PathPrefix, event_path, lookup_v2::OptionalValuePath, + owned_value_path, path, }, schema::Definition, }; - use vrl::value::{Kind, ObjectMap, kind::Collection}; + use vrl::{ + path::ValuePath as _, + value::{Kind, ObjectMap, kind::Collection}, + }; use super::{SimpleHttpConfig, remove_duplicates}; use crate::{ @@ -1725,6 +1783,174 @@ mod tests { } } + #[test] + fn inject_auth_enrichment_does_not_clobber_vector_namespace_builtin_fields() { + use crate::{codecs::DecodingConfig, sources::util::HttpSource as _}; + use vector_lib::codecs::BytesDeserializerConfig; + use vrl::value::KeyString; + + let decoder = DecodingConfig::new( + BytesDecoderConfig::new().into(), + BytesDeserializerConfig::new().into(), + LogNamespace::Vector, + ) + .build() + .unwrap() + .with_log_namespace(LogNamespace::Vector); + + let source = super::SimpleHttpSource { + headers: vec![], + query_parameters: vec![], + path_key: OptionalValuePath::none(), + host_key: OptionalValuePath::none(), + decoder, + log_namespace: LogNamespace::Vector, + }; + + let mut log = LogEvent::default(); + // Pre-populate %http_server.path as enrich_events would. + log.insert( + ( + PathPrefix::Metadata, + path!(SimpleHttpConfig::NAME).concat(path!("path")), + ), + "/real/path", + ); + + let mut events = vec![Event::Log(log)]; + let mut enrichment = ObjectMap::new(); + // Attempt to clobber the built-in `path` field and inject a new field. + enrichment.insert(KeyString::from("path"), Value::from("/clobbered")); + enrichment.insert(KeyString::from("tenant_id"), Value::from("t-123")); + + source.inject_auth_enrichment(&mut events, enrichment); + + let Event::Log(log) = &events[0] else { + panic!("expected log event"); + }; + assert_eq!( + log.get(( + PathPrefix::Metadata, + path!(SimpleHttpConfig::NAME).concat(path!("path")), + )), + Some(&Value::from("/real/path")), + "auth enrichment must not overwrite built-in source metadata" + ); + assert_eq!( + log.get(( + PathPrefix::Metadata, + path!(SimpleHttpConfig::NAME).concat(path!("tenant_id")), + )), + Some(&Value::from("t-123")), + "new auth enrichment field must be injected" + ); + } + + #[test] + fn inject_auth_enrichment_does_not_overwrite_existing_metadata_in_vector_namespace() { + use crate::{codecs::DecodingConfig, sources::util::HttpSource as _}; + use vector_lib::codecs::BytesDeserializerConfig; + use vrl::value::KeyString; + + let decoder = DecodingConfig::new( + BytesDecoderConfig::new().into(), + BytesDeserializerConfig::new().into(), + LogNamespace::Vector, + ) + .build() + .unwrap() + .with_log_namespace(LogNamespace::Vector); + + let source = super::SimpleHttpSource { + headers: vec![], + query_parameters: vec![], + path_key: OptionalValuePath::none(), + host_key: OptionalValuePath::none(), + decoder, + log_namespace: LogNamespace::Vector, + }; + + let mut log = LogEvent::default(); + // Pre-populate a key (e.g. already written by enrich_events or the decoded event). + log.insert( + ( + PathPrefix::Metadata, + path!(SimpleHttpConfig::NAME).concat(path!("tenant_id")), + ), + "existing", + ); + + let mut events = vec![Event::Log(log)]; + let mut enrichment = ObjectMap::new(); + enrichment.insert(KeyString::from("tenant_id"), Value::from("auth-value")); + + source.inject_auth_enrichment(&mut events, enrichment); + + let Event::Log(log) = &events[0] else { + panic!("expected log event"); + }; + assert_eq!( + log.get(( + PathPrefix::Metadata, + path!(SimpleHttpConfig::NAME).concat(path!("tenant_id")), + )), + Some(&Value::from("existing")), + "auth enrichment must not overwrite already-present metadata keys" + ); + } + + #[test] + fn inject_auth_enrichment_applies_to_non_log_events_in_vector_namespace() { + use crate::{codecs::DecodingConfig, sources::util::HttpSource as _}; + use vector_lib::{ + codecs::BytesDeserializerConfig, + event::{Metric, MetricKind, MetricValue}, + }; + use vrl::value::KeyString; + + let decoder = DecodingConfig::new( + BytesDecoderConfig::new().into(), + BytesDeserializerConfig::new().into(), + LogNamespace::Vector, + ) + .build() + .unwrap() + .with_log_namespace(LogNamespace::Vector); + + let source = super::SimpleHttpSource { + headers: vec![], + query_parameters: vec![], + path_key: OptionalValuePath::none(), + host_key: OptionalValuePath::none(), + decoder, + log_namespace: LogNamespace::Vector, + }; + + let metric = Metric::new( + "requests", + MetricKind::Incremental, + MetricValue::Counter { value: 1.0 }, + ); + let mut events = vec![Event::Metric(metric)]; + + let mut enrichment = ObjectMap::new(); + enrichment.insert(KeyString::from("tenant_id"), Value::from("t-456")); + + source.inject_auth_enrichment(&mut events, enrichment); + + let Event::Metric(metric) = &events[0] else { + panic!("expected metric event"); + }; + assert_eq!( + metric + .metadata() + .value() + .get(path!(SimpleHttpConfig::NAME).concat(path!("tenant_id")),), + Some(&Value::from("t-456")), + "auth enrichment must be written to non-log event metadata" + ); + } + impl ValidatableComponent for SimpleHttpConfig { fn validation_configuration() -> ValidationConfiguration { let config = Self { diff --git a/src/sources/internal_logs.rs b/src/sources/internal_logs.rs index e7873a7e8a560..d29ce5e1bb121 100644 --- a/src/sources/internal_logs.rs +++ b/src/sources/internal_logs.rs @@ -210,7 +210,7 @@ async fn run( mod tests { use futures::Stream; use tokio::time::{Duration, sleep}; - use vector_lib::{event::Value, lookup::OwnedTargetPath}; + use vector_lib::{SpanField, event::Value, lookup::OwnedTargetPath}; use vrl::value::kind::Collection; use serial_test::serial; @@ -244,6 +244,10 @@ mod tests { assert_source_compliance(&SOURCE_TAGS, run_test()).await; } + // Register test-specific span fields so they appear in the SPAN_FIELDS allowlist. + inventory::submit!(SpanField("component_new_field")); + inventory::submit!(SpanField("component_numerical_field")); + async fn run_test() { let test_id: u8 = rand::random(); let start = chrono::Utc::now(); @@ -256,12 +260,16 @@ mod tests { component_id = "foo", component_type = "internal_logs", ); - let _enter = span.enter(); + let enter = span.enter(); error!(message = "Before source started.", %test_id); + drop(enter); // don't hold the span guard across an await point + let rx = start_source().await; + let enter = span.enter(); + error!(message = "After source started.", %test_id); { @@ -276,6 +284,8 @@ mod tests { error!(message = "In a nested span.", %test_id); } + drop(enter); + sleep(Duration::from_millis(1)).await; let mut events = collect_ready(rx).await; let test_id = Value::from(test_id.to_string()); @@ -344,6 +354,47 @@ mod tests { rx } + // Register a span field through the same macro downstream crates would use, then verify + // that emitting a log inside a span carrying that field captures it onto the log event. + // This is the regression check for `register_extra_span_field!` extending the + // `SpanFields::record` allowlist beyond the built-in `component_*` prefix. + vector_lib::register_extra_span_field!("internal_logs_test_extra_field"); + + #[tokio::test] + #[serial] + async fn registered_extra_span_field_is_captured() { + trace::init(false, false, "info", 10); + trace::reset_early_buffer(); + + let test_id: u8 = rand::random(); + let rx = start_source().await; + + { + let span = error_span!( + "extras", + component_id = "foo", + internal_logs_test_extra_field = "captured", + some_other_field = "dropped", + ); + let _enter = span.enter(); + error!(message = "With extra field.", %test_id); + } + + sleep(Duration::from_millis(1)).await; + let mut events = collect_ready(rx).await; + let test_id_value = Value::from(test_id.to_string()); + events.retain(|event| event.as_log().get("test_id") == Some(&test_id_value)); + + assert_eq!(events.len(), 1); + let log = events[0].as_log(); + assert_eq!( + log["vector.internal_logs_test_extra_field"], + "captured".into() + ); + // The unregistered span field is still filtered out. + assert!(log.get("vector.some_other_field").is_none()); + } + // NOTE: This test requires #[serial] because it directly interacts with global tracing state. // This is a pre-existing limitation around tracing initialization in tests. #[tokio::test] diff --git a/src/sources/internal_metrics.rs b/src/sources/internal_metrics.rs index f576efa2a71a1..76dfab8849997 100644 --- a/src/sources/internal_metrics.rs +++ b/src/sources/internal_metrics.rs @@ -196,8 +196,13 @@ impl InternalMetrics<'_> { mod tests { use std::collections::BTreeMap; - use metrics::{counter, gauge, histogram}; - use vector_lib::{metric_tags, metrics::Controller}; + use strum::IntoEnumIterator; + use vector_lib::{ + counter, gauge, histogram, + internal_event::{CounterName, GaugeName, HistogramName}, + metric_tags, + metrics::Controller, + }; use super::*; use crate::{ @@ -223,14 +228,20 @@ mod tests { // There *seems* to be a race condition here (CI was flaky), so add a slight delay. std::thread::sleep(std::time::Duration::from_millis(300)); - gauge!("foo").set(1.0); - gauge!("foo").set(2.0); - counter!("bar").increment(3); - counter!("bar").increment(4); - histogram!("baz").record(5.0); - histogram!("baz").record(6.0); - histogram!("quux", "host" => "foo").record(8.0); - histogram!("quux", "host" => "foo").record(8.1); + let gauge_name = GaugeName::iter().next().unwrap(); + let counter_name = CounterName::iter().next().unwrap(); + let mut histogram_iter = HistogramName::iter(); + let histogram_name = histogram_iter.next().unwrap(); + let histogram_tagged_name = histogram_iter.next().unwrap(); + + gauge!(gauge_name).set(1.0); + gauge!(gauge_name).set(2.0); + counter!(counter_name).increment(3); + counter!(counter_name).increment(4); + histogram!(histogram_name).record(5.0); + histogram!(histogram_name).record(6.0); + histogram!(histogram_tagged_name, "host" => "foo").record(8.0); + histogram!(histogram_tagged_name, "host" => "foo").record(8.1); let controller = Controller::get().expect("no controller"); @@ -243,10 +254,16 @@ mod tests { .map(|metric| (metric.name().to_string(), metric)) .collect::>(); - assert_eq!(&MetricValue::Gauge { value: 2.0 }, output["foo"].value()); - assert_eq!(&MetricValue::Counter { value: 7.0 }, output["bar"].value()); + assert_eq!( + &MetricValue::Gauge { value: 2.0 }, + output[gauge_name.as_str()].value() + ); + assert_eq!( + &MetricValue::Counter { value: 7.0 }, + output[counter_name.as_str()].value() + ); - match &output["baz"].value() { + match &output[histogram_name.as_str()].value() { MetricValue::AggregatedHistogram { buckets, count, @@ -263,7 +280,7 @@ mod tests { _ => panic!("wrong type"), } - match &output["quux"].value() { + match &output[histogram_tagged_name.as_str()].value() { MetricValue::AggregatedHistogram { buckets, count, @@ -282,7 +299,7 @@ mod tests { } let labels = metric_tags!("host" => "foo"); - assert_eq!(Some(&labels), output["quux"].tags()); + assert_eq!(Some(&labels), output[histogram_tagged_name.as_str()].tags()); } async fn event_from_config(config: InternalMetricsConfig) -> Event { diff --git a/src/sources/journald.rs b/src/sources/journald.rs index c6177fba1eb6c..177ee5e91663c 100644 --- a/src/sources/journald.rs +++ b/src/sources/journald.rs @@ -519,7 +519,7 @@ impl JournaldSource { let events_received = register!(EventsReceived); // Spawn stderr handler task - let stderr_handler = tokio::spawn(Self::handle_stderr(stderr_stream)); + let stderr_handler = crate::spawn_in_current_span(Self::handle_stderr(stderr_stream)); let batch_size = self.batch_size; let result = loop { @@ -1056,7 +1056,7 @@ impl Finalizer { ) -> Self { if acknowledgements { let (finalizer, mut ack_stream) = OrderedFinalizer::new(Some(shutdown)); - tokio::spawn(async move { + crate::spawn_in_current_span(async move { while let Some((status, cursor)) = ack_stream.next().await { if status == BatchStatus::Delivered { checkpointer.lock().await.set(cursor).await; diff --git a/src/sources/kafka.rs b/src/sources/kafka.rs index cae76d8e02059..f5927b365157b 100644 --- a/src/sources/kafka.rs +++ b/src/sources/kafka.rs @@ -455,7 +455,7 @@ async fn kafka_source( .map_or(config.session_timeout_ms / 2, Duration::from_millis); let consumer_state = ConsumerStateInner::::new(config, decoder, out, log_namespace, span); - tokio::spawn(async move { + crate::spawn_in_current_span(async move { coordinate_kafka_callbacks( consumer, callback_rx, diff --git a/src/sources/kubernetes_logs/mod.rs b/src/sources/kubernetes_logs/mod.rs index fa001972a59d7..04b26b4aa7c09 100644 --- a/src/sources/kubernetes_logs/mod.rs +++ b/src/sources/kubernetes_logs/mod.rs @@ -738,7 +738,7 @@ impl Source { let pod_state = pod_store_w.as_reader(); let pod_cacher = MetaCache::new(); - reflectors.push(tokio::spawn(custom_reflector( + reflectors.push(crate::spawn_in_current_span(custom_reflector( pod_store_w, pod_cacher, pod_watcher, @@ -762,7 +762,7 @@ impl Source { ) .backoff(watcher::DefaultBackoff::default()); - reflectors.push(tokio::spawn(custom_reflector( + reflectors.push(crate::spawn_in_current_span(custom_reflector( ns_store_w, MetaCache::new(), ns_watcher, @@ -787,7 +787,7 @@ impl Source { let node_state = node_store_w.as_reader(); let node_cacher = MetaCache::new(); - reflectors.push(tokio::spawn(custom_reflector( + reflectors.push(crate::spawn_in_current_span(custom_reflector( node_store_w, node_cacher, node_watcher, diff --git a/src/sources/logstash.rs b/src/sources/logstash.rs index a643ce0af73ba..a8b6c6b389e28 100644 --- a/src/sources/logstash.rs +++ b/src/sources/logstash.rs @@ -3,6 +3,7 @@ use std::{ convert::TryFrom, io::{self, Read}, net::SocketAddr, + num::NonZeroUsize, time::Duration, }; @@ -264,39 +265,47 @@ impl TcpSource for LogstashSource { } struct LogstashAcker { - sequence_number: u32, - protocol_version: Option, + // Batched reads can contain multiple writer windows. Preserve a separate + // ACK point for each completed window so Filebeat never sees an ACK that + // advances past the current window it is waiting on. If the batch ends in + // the middle of a window, ACK the last received event in that final ACK + // domain so clients are not forced to wait for the advertised window size. + // Lumberjack defines WindowSize as a maximum unacked count, so a sender can + // legitimately advertise a fresh window after a previously ACKed partial + // tail. Within a single ReadyFrames batch, the only incomplete ACK domain + // we can represent independently is the final tail we have actually seen. + // We expect most batches to need only one ACK point, either for a single + // completed window or for one partial tail. Multiple ACKs are only needed + // when ReadyFrames coalesces multiple logical windows into one batch. + acknowledgements: SmallVec<[(LogstashProtocolVersion, u32); 1]>, } impl LogstashAcker { fn new(frames: &[LogstashEventFrame]) -> Self { - let mut sequence_number = 0; - let mut protocol_version = None; - - for frame in frames { - sequence_number = std::cmp::max(sequence_number, frame.sequence_number); - // We assume that it's valid to ack via any of the protocol versions that we've seen in - // a set of frames from a single stream, so here we just take the last. In reality, we - // do not expect stream with multiple protocol versions to occur. - protocol_version = Some(frame.protocol); - } - - Self { - sequence_number, - protocol_version, - } + let acknowledgements = frames + .iter() + .enumerate() + // ACK each completed writer window and the last frame in a partial batch if ReadyFrames + // flushes before the current window is complete. + .filter(|(index, frame)| frame.window_end || index + 1 == frames.len()) + .map(|(_, frame)| (frame.protocol, frame.sequence_number)) + .collect(); + + Self { acknowledgements } } } impl TcpSourceAcker for LogstashAcker { // https://github.com/logstash-plugins/logstash-input-beats/blob/master/PROTOCOL.md#ack-frame-type fn build_ack(self, ack: TcpSourceAck) -> Option { - match (ack, self.protocol_version) { - (TcpSourceAck::Ack, Some(protocol_version)) => { - let mut bytes: Vec = Vec::with_capacity(6); - bytes.push(protocol_version.into()); - bytes.push(LogstashFrameType::Ack.into()); - bytes.extend(self.sequence_number.to_be_bytes().iter()); + match ack { + TcpSourceAck::Ack if !self.acknowledgements.is_empty() => { + let mut bytes: Vec = Vec::with_capacity(self.acknowledgements.len() * 6); + for (protocol_version, sequence_number) in self.acknowledgements { + bytes.push(protocol_version.into()); + bytes.push(LogstashFrameType::Ack.into()); + bytes.extend(sequence_number.to_be_bytes().iter()); + } Some(Bytes::from(bytes)) } _ => None, @@ -315,12 +324,51 @@ enum LogstashDecoderReadState { #[derive(Debug)] struct LogstashDecoder { state: LogstashDecoderReadState, + // Tracks how many events remain in the current writer window. This lets us + // preserve sender window boundaries even if ReadyFrames later batches + // multiple decoded windows together before ACKing. + window_events_remaining: Option, } impl LogstashDecoder { const fn new() -> Self { + Self::new_with_window_events_remaining(None) + } + + const fn new_with_window_events_remaining( + window_events_remaining: Option, + ) -> Self { Self { state: LogstashDecoderReadState::ReadProtocol, + window_events_remaining, + } + } + + /// Marks whether a decoded frame closes the current writer window. + /// + /// Filebeat expects ACKs to stay within the current window announced by the + /// most recent `WindowSize` frame. The generic TCP batching layer can merge + /// frames from multiple windows before we build an ACK, so we record the + /// per-frame window boundary here and let the acker emit one ACK frame per + /// completed window later. + /// + /// If a sender omits `WindowSize`, we keep the previous behavior and treat + /// each standalone frame as ACKable on its own. + const fn annotate_frame(&mut self, frame: &mut LogstashEventFrame) { + match self.window_events_remaining { + Some(remaining) if remaining.get() == 1 => { + frame.window_end = true; + self.window_events_remaining = None; + } + Some(remaining) => { + frame.window_end = false; + self.window_events_remaining = NonZeroUsize::new(remaining.get() - 1); // safe because we know remaining is greater than 1 + } + None => { + // Preserve existing behavior for inputs that send standalone data frames + // without an explicit WindowSize frame. + frame.window_end = true; + } } } } @@ -440,6 +488,12 @@ struct LogstashEventFrame { protocol: LogstashProtocolVersion, sequence_number: u32, fields: BTreeMap, + window_end: bool, +} + +struct DecodedCompressedFrames { + frames: VecDeque<(LogstashEventFrame, usize)>, + window_events_remaining: Option, } // Based on spec at: https://github.com/logstash-plugins/logstash-input-beats/blob/master/PROTOCOL.md @@ -492,8 +546,15 @@ impl Decoder for LogstashDecoder { } } // The window size indicates how many events the writer will send before waiting - // for acks. As we forward events as we get them, and ack as they are received, we - // do not need to keep track of this. + // for acks. We preserve this boundary so the acker can emit one ACK per + // completed window, even if multiple windows are batched together later. + // Filebeat accepts cumulative ACKs, but not ACKs that advance past the + // current writer window it is waiting on. WindowSize is a maximum unacked + // count, not necessarily an exact count of immediately following frames, so a + // sender can legitimately advertise a new window after a previously ACKed + // partial tail. If a malformed sender does this before that earlier tail has + // actually been ACKed, we tolerate the reset here even though it can collapse + // the older incomplete domain into the new one. // // https://github.com/logstash-plugins/logstash-input-beats/blob/master/PROTOCOL.md#window-size-frame-type LogstashDecoderReadState::ReadFrame(_protocol, LogstashFrameType::WindowSize) => { @@ -501,7 +562,8 @@ impl Decoder for LogstashDecoder { return Ok(None); } - let _window_size = src.get_u32(); + let window_size = src.get_u32() as usize; + self.window_events_remaining = NonZeroUsize::new(window_size); LogstashDecoderReadState::ReadProtocol } @@ -519,27 +581,37 @@ impl Decoder for LogstashDecoder { } // https://github.com/logstash-plugins/logstash-input-beats/blob/master/PROTOCOL.md#data-frame-type LogstashDecoderReadState::ReadFrame(protocol, LogstashFrameType::Data) => { - let Some(frame) = decode_data_frame(protocol, src) else { + let Some((mut frame, byte_size)) = decode_data_frame(protocol, src) else { return Ok(None); }; + self.annotate_frame(&mut frame); - LogstashDecoderReadState::PendingFrames([frame].into()) + LogstashDecoderReadState::PendingFrames([(frame, byte_size)].into()) } // https://github.com/logstash-plugins/logstash-input-beats/blob/master/PROTOCOL.md#json-frame-type LogstashDecoderReadState::ReadFrame(protocol, LogstashFrameType::Json) => { - let Some(frame) = decode_json_frame(protocol, src)? else { + let Some((mut frame, byte_size)) = decode_json_frame(protocol, src)? else { return Ok(None); }; + self.annotate_frame(&mut frame); - LogstashDecoderReadState::PendingFrames([frame].into()) + LogstashDecoderReadState::PendingFrames([(frame, byte_size)].into()) } // https://github.com/logstash-plugins/logstash-input-beats/blob/master/PROTOCOL.md#compressed-frame-type + // + // The compressed payload is still part of the same logical Lumberjack stream, so + // the nested decoder must inherit the current window state and return the updated + // state after expanding the payload. Re-annotating the emitted frames here would + // overwrite any WindowSize boundaries that were established inside the compressed + // payload and can also lose progress from a partially consumed outer window. LogstashDecoderReadState::ReadFrame(_protocol, LogstashFrameType::Compressed) => { - let Some(frames) = decode_compressed_frame(src)? else { + let Some(decoded) = decode_compressed_frame(src, self.window_events_remaining)? + else { return Ok(None); }; + self.window_events_remaining = decoded.window_events_remaining; - LogstashDecoderReadState::PendingFrames(frames) + LogstashDecoderReadState::PendingFrames(decoded.frames) } }; } @@ -581,6 +653,7 @@ fn decode_data_frame( protocol, sequence_number, fields, + window_end: false, }, byte_size, )) @@ -639,6 +712,7 @@ fn decode_json_frame( protocol, sequence_number, fields, + window_end: false, }, byte_size, ))) @@ -646,7 +720,8 @@ fn decode_json_frame( fn decode_compressed_frame( src: &mut BytesMut, -) -> Result>, DecodeError> { + window_events_remaining: Option, +) -> Result, DecodeError> { let mut rest = src.as_ref(); if rest.remaining() < 4 { @@ -674,14 +749,17 @@ fn decode_compressed_frame( let mut buf = res?; - let mut decoder = LogstashDecoder::new(); + let mut decoder = LogstashDecoder::new_with_window_events_remaining(window_events_remaining); let mut frames = VecDeque::new(); while let Some(s) = decoder.decode(&mut buf)? { frames.push_back(s); } - Ok(Some(frames)) + Ok(Some(DecodedCompressedFrames { + frames, + window_events_remaining: decoder.window_events_remaining, + })) } fn bytes_remaining(src: &BytesMut, rest: &[u8]) -> usize { @@ -709,10 +787,14 @@ impl From for SmallVec<[Event; 1]> { #[cfg(test)] mod test { + use std::io::Write; + use bytes::BufMut; - use futures::Stream; + use flate2::{Compression, write::ZlibEncoder}; + use futures::{Stream, StreamExt, stream}; use rand::{Rng, rng}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use vector_lib::codecs::ReadyFrames; use vector_lib::lookup::OwnedTargetPath; use vrl::value::kind::Collection; @@ -791,8 +873,7 @@ mod test { assert!(log.get("timestamp").is_some()); } - fn encode_req(seq: u32, pairs: &[(&str, &str)]) -> Bytes { - let mut req = BytesMut::new(); + fn push_req(req: &mut BytesMut, seq: u32, pairs: &[(&str, &str)]) { req.put_u8(b'2'); req.put_u8(b'D'); req.put_u32(seq); @@ -803,9 +884,127 @@ mod test { req.put_u32(value.len() as u32); req.put(value.as_bytes()); } + } + + fn encode_req(seq: u32, pairs: &[(&str, &str)]) -> Bytes { + let mut req = BytesMut::new(); + push_req(&mut req, seq, pairs); req.into() } + fn push_window_size(req: &mut BytesMut, size: u32) { + req.put_u8(b'2'); + req.put_u8(b'W'); + req.put_u32(size); + } + + fn push_compressed(req: &mut BytesMut, inner: &[u8]) { + let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(inner).unwrap(); + let compressed = encoder.finish().unwrap(); + + req.put_u8(b'2'); + req.put_u8(b'C'); + req.put_u32(compressed.len() as u32); + req.put(compressed.as_slice()); + } + + fn decode_frames(mut src: BytesMut) -> Vec<(LogstashEventFrame, usize)> { + let mut decoder = LogstashDecoder::new(); + let mut frames = Vec::new(); + + while let Some(frame) = decoder.decode(&mut src).unwrap() { + frames.push(frame); + } + + assert_eq!(src.len(), 0); + frames + } + + fn decode_acknowledgements(mut ack: Bytes) -> Vec { + let mut acknowledgements = Vec::new(); + + while !ack.is_empty() { + assert!( + ack.len() >= 6, + "ack stream ended with {} trailing bytes", + ack.len() + ); + assert_eq!(ack.get_u8(), b'2'); + assert_eq!(ack.get_u8(), b'A'); + acknowledgements.push(ack.get_u32()); + } + + acknowledgements + } + + fn decoded_sequence_numbers(decoded: &[(LogstashEventFrame, usize)]) -> Vec { + decoded + .iter() + .map(|(frame, _)| frame.sequence_number) + .collect::>() + } + + fn assert_decoded_sequences( + decoded: &[(LogstashEventFrame, usize)], + expected_sequences: &[u32], + ) { + assert_eq!(decoded_sequence_numbers(decoded), expected_sequences); + } + + async fn assert_acknowledgements_for_ready_frames( + decoded: Vec<(LogstashEventFrame, usize)>, + expected_sequences: &[u32], + expected_acknowledgements: &[u32], + ) { + assert_decoded_sequences(&decoded, expected_sequences); + + let stream = stream::iter(decoded.into_iter().map(Ok::<_, DecodeError>)); + let mut ready = ReadyFrames::with_capacity(stream, 16); + let (frames, _) = ready.next().await.unwrap().unwrap(); + + let ack = LogstashAcker::new(&frames) + .build_ack(TcpSourceAck::Ack) + .unwrap(); + let acknowledgements = decode_acknowledgements(ack); + + assert!(ready.next().await.is_none()); + assert_eq!(acknowledgements, expected_acknowledgements); + } + + fn decode_frames_and_assert_sequences( + src: BytesMut, + expected_sequences: &[u32], + ) -> Vec<(LogstashEventFrame, usize)> { + let decoded = decode_frames(src); + assert_decoded_sequences(&decoded, expected_sequences); + decoded + } + + fn decode_frames_with_decoder( + decoder: &mut LogstashDecoder, + mut src: BytesMut, + ) -> Vec<(LogstashEventFrame, usize)> { + let mut frames = Vec::new(); + + while let Some(frame) = decoder.decode(&mut src).unwrap() { + frames.push(frame); + } + + assert_eq!(src.len(), 0); + frames + } + + fn decode_frames_with_decoder_and_assert_sequences( + decoder: &mut LogstashDecoder, + src: BytesMut, + expected_sequences: &[u32], + ) -> Vec<(LogstashEventFrame, usize)> { + let decoded = decode_frames_with_decoder(decoder, src); + assert_decoded_sequences(&decoded, expected_sequences); + decoded + } + #[test] fn v1_decoder_does_not_panic() { let seq = rng().random_range(1..u32::MAX); @@ -818,6 +1017,106 @@ mod test { } } + #[tokio::test] + async fn distinct_windows_do_not_share_an_ack_domain() { + let mut req = BytesMut::new(); + push_window_size(&mut req, 1); + push_req(&mut req, 1, &[("message", "first window")]); + push_window_size(&mut req, 2); + push_req(&mut req, 1, &[("message", "second window first")]); + push_req(&mut req, 2, &[("message", "second window second")]); + + let decoded = decode_frames_and_assert_sequences(req, &[1, 1, 2]); + assert_acknowledgements_for_ready_frames(decoded, &[1, 1, 2], &[1, 2]).await; + } + + #[tokio::test] + async fn distinct_windows_with_monotonic_sequences_ack_the_first_window() { + let mut req = BytesMut::new(); + push_window_size(&mut req, 2); + push_req(&mut req, 1, &[("message", "first window first")]); + push_req(&mut req, 2, &[("message", "first window second")]); + push_window_size(&mut req, 2); + push_req(&mut req, 3, &[("message", "second window first")]); + push_req(&mut req, 4, &[("message", "second window second")]); + + let decoded = decode_frames_and_assert_sequences(req, &[1, 2, 3, 4]); + assert_acknowledgements_for_ready_frames(decoded, &[1, 2, 3, 4], &[2, 4]).await; + } + + #[tokio::test] + async fn incomplete_final_window_is_acked_to_the_last_received_event() { + let mut req = BytesMut::new(); + push_window_size(&mut req, 4); + push_req(&mut req, 1, &[("message", "only event in partial window")]); + + let decoded = decode_frames_and_assert_sequences(req, &[1]); + assert_acknowledgements_for_ready_frames(decoded, &[1], &[1]).await; + } + + #[tokio::test] + async fn compressed_frames_preserve_inner_window_boundaries() { + let mut inner = BytesMut::new(); + push_window_size(&mut inner, 2); + push_req(&mut inner, 1, &[("message", "compressed first")]); + push_req(&mut inner, 2, &[("message", "compressed second")]); + + let mut req = BytesMut::new(); + push_compressed(&mut req, &inner); + + let decoded = decode_frames_and_assert_sequences(req, &[1, 2]); + assert_acknowledgements_for_ready_frames(decoded, &[1, 2], &[2]).await; + } + + #[tokio::test] + async fn single_window_split_across_ready_frames_keeps_progressive_acks() { + let mut req = BytesMut::new(); + push_window_size(&mut req, 4); + push_req(&mut req, 1, &[("message", "first")]); + push_req(&mut req, 2, &[("message", "second")]); + push_req(&mut req, 3, &[("message", "third")]); + push_req(&mut req, 4, &[("message", "fourth")]); + + let decoded = decode_frames_and_assert_sequences(req, &[1, 2, 3, 4]); + + let stream = stream::iter(decoded.into_iter().map(Ok::<_, DecodeError>)); + let mut ready = ReadyFrames::with_capacity(stream, 2); + let mut acknowledgements = Vec::new(); + + while let Some(result) = ready.next().await { + let (frames, _byte_size) = result.unwrap(); + let ack = LogstashAcker::new(&frames) + .build_ack(TcpSourceAck::Ack) + .unwrap(); + acknowledgements.push(decode_acknowledgements(ack)); + } + + assert_eq!(acknowledgements, vec![vec![2], vec![4]]); + } + + #[tokio::test] + async fn fresh_window_after_acked_partial_tail_is_accepted() { + let mut decoder = LogstashDecoder::new(); + + let mut first_batch = BytesMut::new(); + push_window_size(&mut first_batch, 2); + push_req(&mut first_batch, 1, &[("message", "first partial tail")]); + let decoded = + decode_frames_with_decoder_and_assert_sequences(&mut decoder, first_batch, &[1]); + assert_acknowledgements_for_ready_frames(decoded, &[1], &[1]).await; + + let mut second_batch = BytesMut::new(); + push_window_size(&mut second_batch, 1); + push_req( + &mut second_batch, + 1, + &[("message", "fresh window after ack")], + ); + let decoded = + decode_frames_with_decoder_and_assert_sequences(&mut decoder, second_batch, &[1]); + assert_acknowledgements_for_ready_frames(decoded, &[1], &[1]).await; + } + async fn send_req(address: SocketAddr, pairs: &[(&str, &str)], sends_ack: bool) { let seq = rng().random_range(1..u32::MAX); let mut socket = tokio::net::TcpStream::connect(address).await.unwrap(); diff --git a/src/sources/mod.rs b/src/sources/mod.rs index cdc03f46fdcbb..f3ca03c1af6c1 100644 --- a/src/sources/mod.rs +++ b/src/sources/mod.rs @@ -13,11 +13,13 @@ pub mod aws_kinesis_firehose; pub mod aws_s3; #[cfg(feature = "sources-aws_sqs")] pub mod aws_sqs; +#[cfg(feature = "sources-azure_blob")] +pub mod azure_blob; #[cfg(feature = "sources-datadog_agent")] pub mod datadog_agent; #[cfg(feature = "sources-demo_logs")] pub mod demo_logs; -#[cfg(feature = "sources-dnstap")] +#[cfg(all(unix, feature = "sources-dnstap"))] pub mod dnstap; #[cfg(feature = "sources-docker_logs")] pub mod docker_logs; diff --git a/src/sources/mqtt/config.rs b/src/sources/mqtt/config.rs index 635751a3a1b0a..92b8cd78f6a5b 100644 --- a/src/sources/mqtt/config.rs +++ b/src/sources/mqtt/config.rs @@ -148,7 +148,7 @@ impl MqttSourceConfig { if let Some(tls) = tls.tls() { let ca = tls.authorities_pem().flatten().collect(); - let client_auth = None; + let client_auth = tls.identity_pem(); let alpn = Some(vec!["mqtt".into()]); options.set_transport(Transport::Tls(TlsConfiguration::Simple { ca, diff --git a/src/sources/opentelemetry/config.rs b/src/sources/opentelemetry/config.rs index c7eed8f301b50..5ae97fa491479 100644 --- a/src/sources/opentelemetry/config.rs +++ b/src/sources/opentelemetry/config.rs @@ -19,7 +19,7 @@ use crate::{ }; use futures::FutureExt; use futures_util::{TryFutureExt, future::join}; -use tonic::{codec::CompressionEncoding, transport::server::RoutesBuilder}; +use tonic::transport::server::RoutesBuilder; use vector_config::indexmap::IndexSet; use vector_lib::{ codecs::decoding::{OtlpDeserializer, OtlpSignalType}, @@ -202,13 +202,14 @@ pub struct HttpConfig { #[serde(default)] pub keepalive: KeepaliveConfig, - /// A list of HTTP headers to include in the log event. + /// A list of HTTP headers to include in the event. /// /// Accepts the wildcard (`*`) character for headers matching a specified pattern. /// - /// Specifying "*" results in all headers included in the log event. + /// Specifying "*" results in all headers included in the event. /// - /// These headers are not included in the JSON payload if a field with a conflicting name exists. + /// For log events in legacy namespace mode, headers are not included if a field with a conflicting name exists. + /// For metrics and traces, headers are always added to event metadata. #[serde(default)] #[configurable(metadata(docs::examples = "User-Agent"))] #[configurable(metadata(docs::examples = "X-My-Custom-Header"))] @@ -284,6 +285,9 @@ impl SourceConfig for OpentelemetryConfig { let metrics_deserializer = self.get_signal_deserializer(OtlpSignalType::Metrics)?; let traces_deserializer = self.get_signal_deserializer(OtlpSignalType::Traces)?; + // Compression negotiation (gzip, zstd) is handled centrally by + // `DecompressionAndMetricsLayer` in `sources::util::grpc`, so these + // services deliberately do not call `.accept_compressed(..)`. let log_service = LogsServiceServer::new(Service { pipeline: cx.out.clone(), acknowledgements, @@ -291,7 +295,6 @@ impl SourceConfig for OpentelemetryConfig { events_received: events_received.clone(), deserializer: logs_deserializer.clone(), }) - .accept_compressed(CompressionEncoding::Gzip) .max_decoding_message_size(usize::MAX); let metrics_service = MetricsServiceServer::new(Service { @@ -301,7 +304,6 @@ impl SourceConfig for OpentelemetryConfig { events_received: events_received.clone(), deserializer: metrics_deserializer.clone(), }) - .accept_compressed(CompressionEncoding::Gzip) .max_decoding_message_size(usize::MAX); let trace_service = TraceServiceServer::new(Service { @@ -311,7 +313,6 @@ impl SourceConfig for OpentelemetryConfig { events_received: events_received.clone(), deserializer: traces_deserializer.clone(), }) - .accept_compressed(CompressionEncoding::Gzip) .max_decoding_message_size(usize::MAX); let mut builder = RoutesBuilder::default(); @@ -327,7 +328,7 @@ impl SourceConfig for OpentelemetryConfig { cx.shutdown.clone(), ) .map_err(|error| { - error!(message = "Source future failed.", %error); + error!(message = "OpenTelemetry source gRPC server failed.", %error); }); let http_tls_settings = MaybeTlsSettings::from_config(self.http.tls.as_ref(), true)?; @@ -354,7 +355,10 @@ impl SourceConfig for OpentelemetryConfig { filters, cx.shutdown, self.http.keepalive.clone(), - ); + ) + .map_err(|error| { + error!(message = "OpenTelemetry source HTTP server failed.", %error); + }); Ok(join(grpc_source, http_source).map(|_| Ok(())).boxed()) } diff --git a/src/sources/opentelemetry/http.rs b/src/sources/opentelemetry/http.rs index f15e44bbd0926..42be681f1cb90 100644 --- a/src/sources/opentelemetry/http.rs +++ b/src/sources/opentelemetry/http.rs @@ -109,9 +109,11 @@ pub(crate) fn build_warp_filter( ); let metrics_filters = build_warp_metrics_filter( acknowledgements, + log_namespace, out.clone(), bytes_received.clone(), events_received.clone(), + headers.clone(), metrics_deserializer, ); let trace_filters = build_warp_trace_filter( @@ -119,6 +121,7 @@ pub(crate) fn build_warp_filter( out.clone(), bytes_received, events_received, + headers.clone(), traces_deserializer, ); log_filters @@ -257,12 +260,14 @@ fn build_warp_log_filter( } fn build_warp_metrics_filter( acknowledgements: bool, + log_namespace: LogNamespace, source_sender: SourceSender, bytes_received: Registered, events_received: Registered, + headers_cfg: Vec, deserializer: Option, ) -> BoxedFilter<(Response,)> { - let make_events = move |encoding_header: Option, _headers: HeaderMap, body: Bytes| { + let make_events = move |encoding_header: Option, headers: HeaderMap, body: Bytes| { decompress_body(encoding_header.as_deref(), body) .inspect_err(|err| { // Other status codes are already handled by `sources::util::decompress_body` (tech debt). @@ -276,15 +281,14 @@ fn build_warp_metrics_filter( .and_then(|decoded_body| { bytes_received.emit(ByteSize(decoded_body.len())); if let Some(d) = deserializer.as_ref() { - parse_with_deserializer( - d, - decoded_body, - LogNamespace::default(), - &events_received, - ) + parse_with_deserializer(d, decoded_body, log_namespace, &events_received) } else { decode_metrics_body(decoded_body, &events_received) } + .map(|mut events| { + enrich_events(&mut events, &headers_cfg, &headers, log_namespace); + events + }) }) }; @@ -301,9 +305,10 @@ fn build_warp_trace_filter( source_sender: SourceSender, bytes_received: Registered, events_received: Registered, + headers_cfg: Vec, deserializer: Option, ) -> BoxedFilter<(Response,)> { - let make_events = move |encoding_header: Option, _headers: HeaderMap, body: Bytes| { + let make_events = move |encoding_header: Option, headers: HeaderMap, body: Bytes| { decompress_body(encoding_header.as_deref(), body) .inspect_err(|err| { // Other status codes are already handled by `sources::util::decompress_body` (tech debt). @@ -326,6 +331,10 @@ fn build_warp_trace_filter( } else { decode_trace_body(decoded_body, &events_received) } + .map(|mut events| { + enrich_events(&mut events, &headers_cfg, &headers, LogNamespace::default()); + events + }) }) }; diff --git a/src/sources/opentelemetry/tests.rs b/src/sources/opentelemetry/tests.rs index de5781756dba1..9fb0bac71a1cd 100644 --- a/src/sources/opentelemetry/tests.rs +++ b/src/sources/opentelemetry/tests.rs @@ -4,12 +4,31 @@ use std::{ time::{SystemTime, UNIX_EPOCH}, }; +use crate::{ + SourceSender, + config::{OutputId, SourceConfig, SourceContext}, + event::{ + Event, EventStatus, LogEvent, Metric as MetricEvent, MetricKind, MetricTags, MetricValue, + ObjectMap, Value, into_event_stream, + metric::{Bucket, Quantile}, + }, + sources::opentelemetry::config::{ + GrpcConfig, HttpConfig, LOGS, METRICS, OpentelemetryConfig, TRACES, + }, + test_util::{ + self, + addr::next_addr, + components::{SOURCE_TAGS, assert_source_compliance}, + }, +}; use chrono::{DateTime, TimeZone, Utc}; use futures::Stream; use futures_util::StreamExt; use prost::Message; use similar_asserts::assert_eq; use tonic::Request; +use vector_lib::opentelemetry::proto::collector::trace::v1::ExportTraceServiceRequest; +use vector_lib::opentelemetry::proto::trace::v1::{ResourceSpans, ScopeSpans, Span}; use vector_lib::{ config::LogNamespace, lookup::path, @@ -33,22 +52,6 @@ use vector_lib::{ }; use vrl::value; -use crate::{ - SourceSender, - config::{OutputId, SourceConfig, SourceContext}, - event::{ - Event, EventStatus, LogEvent, Metric as MetricEvent, MetricKind, MetricTags, MetricValue, - ObjectMap, Value, into_event_stream, - metric::{Bucket, Quantile}, - }, - sources::opentelemetry::config::{GrpcConfig, HttpConfig, LOGS, METRICS, OpentelemetryConfig}, - test_util::{ - self, - addr::next_addr, - components::{SOURCE_TAGS, assert_source_compliance}, - }, -}; - fn create_test_logs_request() -> Request { Request::new(ExportLogsServiceRequest { resource_logs: vec![ResourceLogs { @@ -100,6 +103,104 @@ fn create_test_logs_request() -> Request { }) } +fn create_test_metrics_request() -> ExportMetricsServiceRequest { + ExportMetricsServiceRequest { + resource_metrics: vec![ResourceMetrics { + resource: Some(Resource { + attributes: vec![KeyValue { + key: "service.name".to_string(), + value: Some(AnyValue { + value: Some(StringValue("vector-collector".to_string())), + }), + }], + dropped_attributes_count: 0, + }), + schema_url: "".to_string(), + scope_metrics: vec![ScopeMetrics { + scope: Some(InstrumentationScope { + name: "vector-collector-instrumentation".to_string(), + version: "0.111.0".to_string(), + attributes: vec![], + dropped_attributes_count: 0, + }), + schema_url: "".to_string(), + metrics: vec![Metric { + name: "some.random.metric".to_string(), + description: "Some random metric we use for test".to_string(), + unit: "1".to_string(), + data: Some(Data::Summary(Summary { + data_points: vec![SummaryDataPoint { + attributes: vec![ + KeyValue { + key: "host".to_string(), + value: Some(AnyValue { + value: Some(StringValue("localhost".to_string())), + }), + }, + KeyValue { + key: "service".to_string(), + value: Some(AnyValue { + value: Some(StringValue("vector-collector".to_string())), + }), + }, + ], + start_time_unix_nano: 0, + time_unix_nano: 0, + count: 5, + sum: 122.5, + quantile_values: vec![ + ValueAtQuantile { + quantile: 0.5, + value: 24.5, + }, + ValueAtQuantile { + quantile: 0.9, + value: 45.0, + }, + ValueAtQuantile { + quantile: 1.0, + value: 60.0, + }, + ], + flags: 0, + }], + })), + }], + }], + }], + } +} + +fn create_test_traces_request() -> ExportTraceServiceRequest { + ExportTraceServiceRequest { + resource_spans: vec![ResourceSpans { + resource: None, + scope_spans: vec![ScopeSpans { + scope: None, + spans: vec![Span { + trace_id: (1..17).collect::>(), + span_id: (1..9).collect::>(), + parent_span_id: (1..9).collect::>(), + name: "span".to_string(), + kind: 1, + start_time_unix_nano: 1713525203000000000, + end_time_unix_nano: 1713525205000000000, + attributes: vec![], + dropped_attributes_count: 0, + events: vec![], + dropped_events_count: 0, + links: vec![], + dropped_links_count: 0, + status: None, + trace_state: "".to_string(), + }], + schema_url: "".to_string(), + }], + schema_url: "".to_string(), + }], + } +} + #[test] fn generate_config() { test_util::test_generate_config::(); @@ -1091,6 +1192,39 @@ fn get_source_config_with_headers( } } +async fn send_and_collect_otel_event( + use_otlp_decoding: bool, + output_port: &str, + endpoint: &str, + body: Vec, +) -> Event { + let (_guard_0, grpc_addr) = next_addr(); + let (_guard_1, http_addr) = next_addr(); + + let source = get_source_config_with_headers(grpc_addr, http_addr, use_otlp_decoding); + + let (sender, output, _) = new_source(EventStatus::Delivered, output_port.to_string()); + let server = source + .build(SourceContext::new_test(sender, None)) + .await + .unwrap(); + tokio::spawn(server); + test_util::wait_for_tcp(http_addr).await; + + let _res = reqwest::Client::new() + .post(format!("http://{http_addr}/{endpoint}")) + .header("Content-Type", "application/x-protobuf") + .header("User-Agent", "Test") + .body(body) + .send() + .await + .expect("Failed to send request to OpenTelemetry source."); + + let mut events = test_util::collect_ready(output).await; + assert_eq!(events.len(), 1); + events.pop().unwrap() +} + #[tokio::test] async fn http_headers_logs_use_otlp_decoding_false() { assert_source_compliance(&SOURCE_TAGS, async { @@ -1237,6 +1371,128 @@ async fn http_headers_logs_use_otlp_decoding_true() { .await; } +#[tokio::test] +async fn http_headers_metrics_use_otlp_decoding_false() { + assert_source_compliance(&SOURCE_TAGS, async { + let event = send_and_collect_otel_event( + false, + METRICS, + "v1/metrics", + create_test_metrics_request().encode_to_vec(), + ) + .await; + let metric = event.as_metric(); + assert_eq!( + metric + .metadata() + .value() + .get(path!("opentelemetry", "headers")) + .unwrap() + .get("AbsentHeader") + .unwrap(), + &Value::Null + ); + assert_eq!( + metric + .metadata() + .value() + .get(path!("opentelemetry", "headers")) + .unwrap() + .get("User-Agent") + .unwrap(), + &value!("Test") + ); + }) + .await; +} + +#[tokio::test] +async fn http_headers_metrics_use_otlp_decoding_true() { + assert_source_compliance(&SOURCE_TAGS, async { + let event = send_and_collect_otel_event( + true, + METRICS, + "v1/metrics", + create_test_metrics_request().encode_to_vec(), + ) + .await; + let log = event.as_log(); + assert_eq!(log["AbsentHeader"], Value::Null); + assert_eq!(log["User-Agent"], "Test".into()); + }) + .await; +} + +#[tokio::test] +async fn http_headers_traces_use_otlp_decoding_false() { + assert_source_compliance(&SOURCE_TAGS, async { + let event = send_and_collect_otel_event( + false, + TRACES, + "v1/traces", + create_test_traces_request().encode_to_vec(), + ) + .await; + let trace = event.as_trace(); + assert_eq!( + trace + .metadata() + .value() + .get(path!("opentelemetry", "headers")) + .unwrap() + .get("AbsentHeader") + .unwrap(), + &Value::Null + ); + assert_eq!( + trace + .metadata() + .value() + .get(path!("opentelemetry", "headers")) + .unwrap() + .get("User-Agent") + .unwrap(), + &value!("Test") + ); + }) + .await; +} + +#[tokio::test] +async fn http_headers_traces_use_otlp_decoding_true() { + assert_source_compliance(&SOURCE_TAGS, async { + let event = send_and_collect_otel_event( + true, + TRACES, + "v1/traces", + create_test_traces_request().encode_to_vec(), + ) + .await; + let trace = event.as_trace(); + assert_eq!( + trace + .metadata() + .value() + .get(path!("opentelemetry", "headers")) + .unwrap() + .get("AbsentHeader") + .unwrap(), + &Value::Null + ); + assert_eq!( + trace + .metadata() + .value() + .get(path!("opentelemetry", "headers")) + .unwrap() + .get("User-Agent") + .unwrap(), + &value!("Test") + ); + }) + .await; +} + pub struct OTelTestEnv { pub grpc_addr: String, pub config: OpentelemetryConfig, diff --git a/src/sources/postgresql_metrics.rs b/src/sources/postgresql_metrics.rs index 96ae1ab3a1263..2854c50496174 100644 --- a/src/sources/postgresql_metrics.rs +++ b/src/sources/postgresql_metrics.rs @@ -295,7 +295,7 @@ impl PostgresqlClient { endpoint: &self.endpoint, } })?; - tokio::spawn(connection); + crate::spawn_in_current_span(connection); client } None => { @@ -306,7 +306,7 @@ impl PostgresqlClient { .with_context(|_| ConnectionFailedSnafu { endpoint: &self.endpoint, })?; - tokio::spawn(connection); + crate::spawn_in_current_span(connection); client } }; diff --git a/src/sources/redis/mod.rs b/src/sources/redis/mod.rs index 0f71009701026..eb2545059ecf8 100644 --- a/src/sources/redis/mod.rs +++ b/src/sources/redis/mod.rs @@ -35,12 +35,11 @@ enum BuildError { /// Data type to use for reading messages from Redis. #[configurable_component] -#[derive(Copy, Clone, Debug, Derivative)] -#[derivative(Default)] +#[derive(Copy, Clone, Debug, Default)] #[serde(rename_all = "lowercase")] pub enum DataTypeConfig { /// The `list` data type. - #[derivative(Default)] + #[default] List, /// The `channel` data type. @@ -51,7 +50,7 @@ pub enum DataTypeConfig { /// Options for the Redis `list` data type. #[configurable_component] -#[derive(Copy, Clone, Debug, Default, Derivative, Eq, PartialEq)] +#[derive(Copy, Clone, Debug, Default, Eq, PartialEq)] #[serde(deny_unknown_fields, rename_all = "lowercase")] pub struct ListOption { #[configurable(derived)] @@ -60,12 +59,11 @@ pub struct ListOption { /// Method for getting events from the `list` data type. #[configurable_component] -#[derive(Clone, Copy, Debug, Derivative, Eq, PartialEq)] -#[derivative(Default)] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] #[serde(rename_all = "lowercase")] pub enum Method { /// Pop messages from the head of the list. - #[derivative(Default)] + #[default] Lpop, /// Pop messages from the tail of the list. diff --git a/src/sources/splunk_hec/acknowledgements.rs b/src/sources/splunk_hec/acknowledgements.rs index 3f65b7d115f95..c1688acb9945d 100644 --- a/src/sources/splunk_hec/acknowledgements.rs +++ b/src/sources/splunk_hec/acknowledgements.rs @@ -106,7 +106,7 @@ impl IndexerAcknowledgement { let idle_task_channels = Arc::clone(&channels); if config.ack_idle_cleanup { - tokio::spawn(async move { + crate::spawn_in_current_span(async move { let mut interval = interval(Duration::from_secs(max_idle_time)); loop { interval.tick().await; @@ -206,7 +206,7 @@ impl Channel { let ack_ids_status = Arc::new(Mutex::new(RoaringTreemap::new())); let finalizer_ack_ids_status = Arc::clone(&ack_ids_status); let (ack_event_finalizer, mut ack_stream) = UnorderedFinalizer::new(Some(shutdown)); - tokio::spawn(async move { + crate::spawn_in_current_span(async move { while let Some((status, ack_id)) = ack_stream.next().await { if status == BatchStatus::Delivered { let mut ack_ids_status = finalizer_ack_ids_status.lock().unwrap(); diff --git a/src/sources/splunk_hec/mod.rs b/src/sources/splunk_hec/mod.rs index b020383b9fbda..2a04b14d68d33 100644 --- a/src/sources/splunk_hec/mod.rs +++ b/src/sources/splunk_hec/mod.rs @@ -7,7 +7,7 @@ use std::{ time::Duration, }; -use bytes::{Buf, Bytes}; +use bytes::{Buf, Bytes, BytesMut}; use chrono::{DateTime, TimeZone, Utc}; use flate2::read::MultiGzDecoder; use futures::FutureExt; @@ -20,22 +20,32 @@ use serde_json::{ }; use snafu::Snafu; use tokio::net::TcpStream; +use tokio_util::codec::Decoder as _; use tower::ServiceBuilder; use tracing::Span; use vector_lib::{ EstimatedJsonEncodedSizeOf, + codecs::{ + Decoder, StreamDecodingError, + decoding::{DeserializerConfig, FramingConfig}, + }, config::{LegacyKey, LogNamespace}, configurable::configurable_component, - event::BatchNotifier, - internal_event::{CountByteSize, InternalEventHandle as _, Registered}, - lookup::{self, event_path, lookup_v2::OptionalValuePath, owned_value_path}, + event::{BatchNotifier, BatchStatusReceiver, EventMetadata}, + internal_event::{ + ComponentEventsDropped, CountByteSize, InternalEventHandle as _, Registered, UNINTENTIONAL, + }, + lookup::{ + self, OwnedValuePath, event_path, lookup_v2::OptionalValuePath, metadata_path, + owned_value_path, + }, schema::meaning, sensitive_string::SensitiveString, source_sender::SendError, tls::MaybeTlsIncomingStream, }; use vrl::{ - path::OwnedTargetPath, + path::{OwnedTargetPath, PathPrefix, ValuePath as _}, value::{Kind, kind::Collection}, }; use warp::{ @@ -56,6 +66,7 @@ use self::{ }; use crate::{ SourceSender, + codecs::DecodingConfig, config::{DataType, Resource, SourceConfig, SourceContext, SourceOutput, log_schema}, event::{Event, LogEvent, Value}, http::{KeepaliveConfig, MaxConnectionAgeLayer, build_http_trace_layer}, @@ -126,6 +137,75 @@ pub struct SplunkConfig { #[configurable(derived)] #[serde(default)] keepalive: KeepaliveConfig, + + /// Codec configuration applied to events received on `/services/collector/event`. + /// + /// When `decoding` is set, Vector applies a second decoding pass after parsing the + /// HEC envelope. The envelope's `event` field is passed through the codec, + /// and a single envelope can fan out to multiple events. Decode failures are + /// swallowed and do not return an error to the Splunk client. + /// + /// The VRL codec can access HEC envelope metadata, such as host, sourcetype, and, + /// channel, and the authentication token via `%splunk_hec.*` paths and + /// `get_secret!("splunk_hec_token")` before the program executes. + #[configurable(derived)] + #[configurable(metadata(docs::advanced))] + #[serde(default)] + pub event: CodecConfig, + + /// Codec configuration applied to events received on `/services/collector/raw`. + /// + /// When `decoding` is set, the (decompressed) request body is fed through the + /// codec instead of being emitted as a single event. Decode failures are + /// swallowed and do not return an error to the Splunk client. When unset, the + /// endpoint preserves its existing behavior of one event per request body. + #[configurable(derived)] + #[configurable(metadata(docs::advanced))] + #[serde(default)] + pub raw: CodecConfig, +} + +/// Codec configuration applied to one of the `splunk_hec` endpoints. +#[configurable_component] +#[derive(Clone, Debug, Default)] +#[serde(deny_unknown_fields, default)] +pub struct CodecConfig { + /// Framing configuration applied to the payload. + /// + /// Only used when `decoding` is also set. Defaults to a per-codec choice + /// (typically `bytes`) that produces one event per payload. + #[configurable(derived)] + #[configurable(metadata(docs::advanced))] + #[serde(default)] + pub framing: Option, + + /// Decoding configuration applied to the payload. + /// + /// When unset, the endpoint preserves its existing per-endpoint default + /// behavior. When set, the endpoint-selected payload is processed through + /// `framing` and `decoding`, and a single payload can fan out to multiple + /// events. + #[configurable(derived)] + #[configurable(metadata(docs::advanced))] + #[serde(default)] + pub decoding: Option, +} + +impl CodecConfig { + fn build_decoder(&self, log_namespace: LogNamespace) -> crate::Result> { + match &self.decoding { + Some(decoding) => { + let framing = self + .framing + .clone() + .unwrap_or_else(|| decoding.default_message_based_framing()); + Ok(Some( + DecodingConfig::new(framing, decoding.clone(), log_namespace).build()?, + )) + } + None => Ok(None), + } + } } impl_generate_config_from_default!(SplunkConfig); @@ -141,6 +221,8 @@ impl Default for SplunkConfig { store_hec_token: false, log_namespace: None, keepalive: Default::default(), + event: CodecConfig::default(), + raw: CodecConfig::default(), } } } @@ -156,7 +238,16 @@ impl SourceConfig for SplunkConfig { let tls = MaybeTlsSettings::from_config(self.tls.as_ref(), true)?; let shutdown = cx.shutdown.clone(); let out = cx.out.clone(); - let source = SplunkSource::new(self, tls.http_protocol_name(), cx); + let log_namespace = cx.log_namespace(self.log_namespace); + let event_decoder = self.event.build_decoder(log_namespace)?; + let raw_decoder = self.raw.build_decoder(log_namespace)?; + let source = SplunkSource::new( + self, + tls.http_protocol_name(), + event_decoder, + raw_decoder, + cx, + ); let event_service = source.event_service(out.clone()); let raw_service = source.raw_service(out); @@ -212,7 +303,11 @@ impl SourceConfig for SplunkConfig { fn outputs(&self, global_log_namespace: LogNamespace) -> Vec { let log_namespace = global_log_namespace.merge(self.log_namespace); - let schema_definition = match log_namespace { + // Build schemas per endpoint, then merge them. Each endpoint decides at + // runtime whether source metadata overwrites event fields or defers to a + // decoder-produced value, so applying one global strategy would make mixed + // decoder/no-decoder configurations advertise the wrong contract. + let legacy_base = || match log_namespace { LogNamespace::Legacy => { let definition = vector_lib::schema::Definition::empty_legacy_namespace() .with_event_field( @@ -238,52 +333,108 @@ impl SourceConfig for SplunkConfig { [log_namespace], ) .with_meaning(OwnedTargetPath::event_root(), meaning::MESSAGE), - } - .with_standard_vector_source_metadata() - .with_source_metadata( - SplunkConfig::NAME, - log_schema() - .host_key() - .cloned() - .map(LegacyKey::InsertIfEmpty), - &owned_value_path!("host"), - Kind::bytes(), - Some(meaning::HOST), - ) - .with_source_metadata( - SplunkConfig::NAME, - Some(LegacyKey::Overwrite(owned_value_path!(CHANNEL))), - &owned_value_path!("channel"), - Kind::bytes(), - None, - ) - .with_source_metadata( - SplunkConfig::NAME, - Some(LegacyKey::Overwrite(owned_value_path!(INDEX))), - &owned_value_path!("index"), - Kind::bytes(), - None, - ) - .with_source_metadata( - SplunkConfig::NAME, - Some(LegacyKey::Overwrite(owned_value_path!(SOURCE))), - &owned_value_path!("source"), - Kind::bytes(), - Some(meaning::SERVICE), - ) - // Not to be confused with `source_type`. - .with_source_metadata( - SplunkConfig::NAME, - Some(LegacyKey::Overwrite(owned_value_path!(SOURCETYPE))), - &owned_value_path!("sourcetype"), - Kind::bytes(), - None, + }; + + let endpoint_base = |decoding: &Option| match decoding { + Some(decoding) => decoding.schema_definition(log_namespace), + None => legacy_base(), + }; + + let splunk_legacy_key = |path: OwnedValuePath, has_decoder: bool| { + if has_decoder { + LegacyKey::InsertIfEmpty(path) + } else { + LegacyKey::Overwrite(path) + } + }; + + let add_common_metadata = |definition: vector_lib::schema::Definition| { + definition + .with_standard_vector_source_metadata() + .with_source_metadata( + SplunkConfig::NAME, + log_schema() + .host_key() + .cloned() + .map(LegacyKey::InsertIfEmpty), + &owned_value_path!("host"), + Kind::bytes(), + Some(meaning::HOST), + ) + }; + + let add_channel_metadata = |definition: vector_lib::schema::Definition, + has_decoder: bool| { + definition.with_source_metadata( + SplunkConfig::NAME, + Some(splunk_legacy_key(owned_value_path!(CHANNEL), has_decoder)), + &owned_value_path!("channel"), + Kind::bytes(), + None, + ) + }; + + let event_has_decoder = self.event.decoding.is_some(); + let raw_has_decoder = self.raw.decoding.is_some(); + + // Merge the per-endpoint base schemas (event root kind + standard Vector + // metadata). Splunk-specific fields are added once afterward with + // per-field decoder flags, avoiding the widening that occurs when the + // raw schema's open `metadata_kind.unknown` overrides specific fields + // from the event schema during merge. + let merged_base = add_common_metadata( + endpoint_base(&self.event.decoding).merge(endpoint_base(&self.raw.decoding)), ); - vec![SourceOutput::new_maybe_logs( - DataType::Log, - schema_definition, - )] + // `index`, `source`, `sourcetype` are only written by the /event endpoint. + // `channel` is written by both; use Overwrite if either endpoint has no + // decoder (some events still overwrite it). + let channel_has_decoder = event_has_decoder && raw_has_decoder; + let schema_definition = add_channel_metadata( + merged_base + .with_source_metadata( + SplunkConfig::NAME, + Some(splunk_legacy_key( + owned_value_path!(INDEX), + event_has_decoder, + )), + &owned_value_path!("index"), + Kind::bytes(), + None, + ) + .with_source_metadata( + SplunkConfig::NAME, + Some(splunk_legacy_key( + owned_value_path!(SOURCE), + event_has_decoder, + )), + &owned_value_path!("source"), + Kind::bytes(), + Some(meaning::SERVICE), + ) + // Not to be confused with `source_type`. + .with_source_metadata( + SplunkConfig::NAME, + Some(splunk_legacy_key( + owned_value_path!(SOURCETYPE), + event_has_decoder, + )), + &owned_value_path!("sourcetype"), + Kind::bytes(), + None, + ), + channel_has_decoder, + ); + + // Output type is the union of both endpoints' decoder output types + // (logs from a JSON codec, metrics from native, etc.). The legacy path + // always emits logs, so when an endpoint has no decoder we OR `Log` in. + let output_type = match (&self.event.decoding, &self.raw.decoding) { + (None, None) => DataType::Log, + (Some(d), None) | (None, Some(d)) => d.output_type() | DataType::Log, + (Some(de), Some(dr)) => de.output_type() | dr.output_type(), + }; + vec![SourceOutput::new_maybe_logs(output_type, schema_definition)] } fn resources(&self) -> Vec { @@ -303,10 +454,18 @@ struct SplunkSource { store_hec_token: bool, log_namespace: LogNamespace, events_received: Registered, + event_decoder: Option, + raw_decoder: Option, } impl SplunkSource { - fn new(config: &SplunkConfig, protocol: &'static str, cx: SourceContext) -> Self { + fn new( + config: &SplunkConfig, + protocol: &'static str, + event_decoder: Option, + raw_decoder: Option, + cx: SourceContext, + ) -> Self { let log_namespace = cx.log_namespace(config.log_namespace); let acknowledgements = cx.do_acknowledgements(config.acknowledgements.enabled.into()); let shutdown = cx.shutdown; @@ -332,6 +491,8 @@ impl SplunkSource { store_hec_token: config.store_hec_token, log_namespace, events_received: register!(EventsReceived), + event_decoder, + raw_decoder, } } @@ -349,6 +510,7 @@ impl SplunkSource { let store_hec_token = self.store_hec_token; let log_namespace = self.log_namespace; let events_received = self.events_received.clone(); + let decoder = self.event_decoder.clone(); warp::post() .and( @@ -375,6 +537,7 @@ impl SplunkSource { let mut out = out.clone(); let idx_ack = idx_ack.clone(); let events_received = events_received.clone(); + let decoder = decoder.clone(); async move { if idx_ack.is_some() && channel.is_none() { @@ -396,36 +559,44 @@ impl SplunkSource { protocol, }); - let (batch, receiver) = + let (batch, mut receiver) = BatchNotifier::maybe_new_with_receiver(idx_ack.is_some()); - let maybe_ack_id = match (idx_ack, receiver, channel.clone()) { - (Some(idx_ack), Some(receiver), Some(channel_id)) => { - match idx_ack.get_ack_id_from_channel(channel_id, receiver).await { - Ok(ack_id) => Some(ack_id), - Err(rej) => return Err(rej), - } - } - _ => None, - }; + let decoder_in_use = decoder.is_some(); + + // Without a decoder, register the ack id BEFORE iteration so + // capacity-exhaustion (`ServiceUnavailable`) short-circuits + // the request without parsing the body - byte-for-byte parity + // with the pre-decoder behavior. + let mut maybe_ack_id = None; + if !decoder_in_use { + maybe_ack_id = + register_ack(idx_ack.clone(), receiver.take(), channel.clone()) + .await?; + } let mut error = None; let mut events = Vec::new(); + let mut had_decode_errors = false; let iter: EventIterator<'_, StrRead<'_>> = EventIteratorGenerator { deserializer: Deserializer::from_str(&body).into_iter::(), - channel, + channel: channel.clone(), remote, remote_addr, batch, token: token.filter(|_| store_hec_token).map(Into::into), log_namespace, events_received, + decoder, } .into(); for result in iter { match result { - Ok(event) => events.push(event), + Ok((chunk, errored)) => { + events.extend(chunk); + had_decode_errors |= errored; + } Err(err) => { error = Some(err); break; @@ -433,6 +604,22 @@ impl SplunkSource { } } + // With a decoder, defer ack registration until we know whether + // the codec emitted anything *and* whether it dropped any + // frames. Also skip ack registration when a later envelope + // errored even if earlier ones produced events: the client + // gets a 400 and never sees the ack id, so registering it + // only leaks pending-ack capacity. + if decoder_in_use { + maybe_ack_id = + if events.is_empty() || had_decode_errors || error.is_some() { + drop(receiver); + None + } else { + register_ack(idx_ack, receiver, channel).await? + }; + } + if !events.is_empty() { match out.send_batch(events).await { Ok(()) => (), @@ -463,6 +650,7 @@ impl SplunkSource { let store_hec_token = self.store_hec_token; let events_received = self.events_received.clone(); let log_namespace = self.log_namespace; + let decoder = self.raw_decoder.clone(); warp::post() .and(path!("raw" / "1.0").or(path!("raw"))) @@ -485,6 +673,7 @@ impl SplunkSource { let mut out = out.clone(); let idx_ack = idx_ack.clone(); let events_received = events_received.clone(); + let decoder = decoder.clone(); emit!(HttpBytesReceived { byte_size: body.len(), http_path: path.as_str(), @@ -494,29 +683,79 @@ impl SplunkSource { async move { let (batch, receiver) = BatchNotifier::maybe_new_with_receiver(idx_ack.is_some()); - let maybe_ack_id = match (idx_ack, receiver) { - (Some(idx_ack), Some(receiver)) => Some( - idx_ack - .get_ack_id_from_channel(channel_id.clone(), receiver) - .await?, - ), - _ => None, + + // No-decoder path: byte-for-byte identical to the pre-decoder + // code - register ack first (fast-fail under capacity + // exhaustion), build a single event, send via `send_event` + // (avoids `send_batch_latency` emission). + let Some(decoder) = decoder else { + let maybe_ack_id = + register_ack(idx_ack, receiver, Some(channel_id.clone())).await?; + let (mut events, _) = raw_event( + body, + gzip, + channel_id, + remote, + xff, + batch, + log_namespace, + &events_received, + None, + None, + )?; + // raw_event with no decoder always produces exactly one + // event. + let mut event = events.pop().expect( + "raw_event always produces a single event when no decoder is set", + ); + if let Some(token) = token.filter(|_| store_hec_token) { + event.metadata_mut().set_splunk_hec_token(token.into()); + } + let res = out.send_event(event).await; + return res + .map(|_| maybe_ack_id) + .map_err(|_| Rejection::from(ApiError::ServerShutdown)); }; - let mut event = raw_event( + + // Decoder path: pass the optional HEC token into raw_event so + // it's stamped on each event the moment it leaves the codec + // (rather than after the whole payload is decoded). + let token: Option> = + token.filter(|_| store_hec_token).map(Arc::from); + let (events, had_decode_errors) = raw_event( body, gzip, - channel_id, + channel_id.clone(), remote, xff, batch, log_namespace, &events_received, + Some(decoder), + token, )?; - if let Some(token) = token.filter(|_| store_hec_token) { - event.metadata_mut().set_splunk_hec_token(token.into()); + + if events.is_empty() || had_decode_errors { + // With newline framing, `valid \n invalid \n valid` + // decodes to two events plus one dropped frame; returning + // an ack id there would let `/services/collector/ack` + // report success for data Vector silently lost. + drop(receiver); + if events.is_empty() { + return Ok(None); + } + // Forward the partial events with no ack so the source's + // existing partial-delivery semantics still apply. + let res = out.send_batch(events).await; + return res + .map(|_| None) + .map_err(|_| Rejection::from(ApiError::ServerShutdown)); } - let res = out.send_event(event).await; + let maybe_ack_id = + register_ack(idx_ack, receiver, Some(channel_id)).await?; + + let res = out.send_batch(events).await; res.map(|_| maybe_ack_id) .map_err(|_| Rejection::from(ApiError::ServerShutdown)) } @@ -672,8 +911,11 @@ impl SplunkSource { struct EventIterator<'de, R: JsonRead<'de>> { /// Remaining request with JSON events deserializer: serde_json::StreamDeserializer<'de, R, JsonValue>, - /// Count of sent events - events: usize, + /// Count of HEC envelopes (not fan-out events) processed so far. Used both as the + /// `InvalidEventNumber` index in Splunk error responses (zero-indexed: subtract 1 + /// for build-time errors, use as-is for parse errors that haven't entered build) + /// and as the "did we see any envelope?" check that gates the `NoData` error. + envelopes_processed: usize, /// Optional channel from headers channel: Option, /// Default time @@ -688,6 +930,9 @@ struct EventIterator<'de, R: JsonRead<'de>> { log_namespace: LogNamespace, /// handle to EventsReceived registry events_received: Registered, + /// Optional second-stage decoder applied to the envelope payload after HEC + /// envelope parsing. + decoder: Option, } /// Intermediate struct to generate an `EventIterator` @@ -700,13 +945,24 @@ struct EventIteratorGenerator<'de, R: JsonRead<'de>> { events_received: Registered, remote: Option, remote_addr: Option, + decoder: Option, } impl<'de, R: JsonRead<'de>> From> for EventIterator<'de, R> { fn from(f: EventIteratorGenerator<'de, R>) -> Self { + // The host field can collide with decoder-produced output in legacy namespace + // (its legacy key is `log_schema().host_key()`, typically `"host"`). When a + // decoder is configured, prefer the decoder's value over the envelope's so the + // user's parsed view wins on conflict. With no decoder configured, behavior is + // unchanged: every extractor uses `Overwrite`. + let extractor_strategy = if f.decoder.is_some() { + LegacyKeyStrategy::InsertIfEmpty + } else { + LegacyKeyStrategy::Overwrite + }; Self { deserializer: f.deserializer, - events: 0, + envelopes_processed: 0, channel: f.channel.map(Value::from), time: Time::Now(Utc::now()), extractors: [ @@ -721,25 +977,73 @@ impl<'de, R: JsonRead<'de>> From> for EventIterat .or_else(|| f.remote.map(|addr| addr.to_string())) .map(Value::from), f.log_namespace, - ), - DefaultExtractor::new("index", OptionalValuePath::new(INDEX), f.log_namespace), - DefaultExtractor::new("source", OptionalValuePath::new(SOURCE), f.log_namespace), + ) + .with_legacy_key_strategy(extractor_strategy), + DefaultExtractor::new("index", OptionalValuePath::new(INDEX), f.log_namespace) + .with_legacy_key_strategy(extractor_strategy), + DefaultExtractor::new("source", OptionalValuePath::new(SOURCE), f.log_namespace) + .with_legacy_key_strategy(extractor_strategy), DefaultExtractor::new( "sourcetype", OptionalValuePath::new(SOURCETYPE), f.log_namespace, - ), + ) + .with_legacy_key_strategy(extractor_strategy), ], batch: f.batch, token: f.token, log_namespace: f.log_namespace, events_received: f.events_received, + decoder: f.decoder, } } } impl<'de, R: JsonRead<'de>> EventIterator<'de, R> { + /// Process the envelope's `time` field, updating `self.time` (sticky across envelopes + /// when not explicitly provided). + fn process_time(&mut self, json: &mut JsonValue) -> Result<(), Rejection> { + let parsed_time = match json.get_mut("time").map(JsonValue::take) { + Some(JsonValue::Number(time)) => Some(Some(time)), + Some(JsonValue::String(time)) => Some(time.parse::().ok()), + _ => None, + }; + + match parsed_time { + None => Ok(()), + Some(Some(t)) => { + if let Some(t) = t.as_u64() { + let time = parse_timestamp(t as i64).ok_or(ApiError::InvalidDataFormat { + event: self.envelopes_processed.saturating_sub(1), + })?; + self.time = Time::Provided(time); + Ok(()) + } else if let Some(t) = t.as_f64() { + self.time = Time::Provided( + Utc.timestamp_opt( + t.floor() as i64, + (t.fract() * 1000.0 * 1000.0 * 1000.0) as u32, + ) + .single() + .expect("invalid timestamp"), + ); + Ok(()) + } else { + Err(ApiError::InvalidDataFormat { + event: self.envelopes_processed.saturating_sub(1), + } + .into()) + } + } + Some(None) => Err(ApiError::InvalidDataFormat { + event: self.envelopes_processed.saturating_sub(1), + } + .into()), + } + } + fn build_event(&mut self, mut json: JsonValue) -> Result { + self.envelopes_processed += 1; // Construct Event from parsed json event let mut log = match self.log_namespace { LogNamespace::Vector => self.build_log_vector(&mut json)?, @@ -787,36 +1091,7 @@ impl<'de, R: JsonRead<'de>> EventIterator<'de, R> { } } - // Process time field - let parsed_time = match json.get_mut("time").map(JsonValue::take) { - Some(JsonValue::Number(time)) => Some(Some(time)), - Some(JsonValue::String(time)) => Some(time.parse::().ok()), - _ => None, - }; - - match parsed_time { - None => (), - Some(Some(t)) => { - if let Some(t) = t.as_u64() { - let time = parse_timestamp(t as i64) - .ok_or(ApiError::InvalidDataFormat { event: self.events })?; - - self.time = Time::Provided(time); - } else if let Some(t) = t.as_f64() { - self.time = Time::Provided( - Utc.timestamp_opt( - t.floor() as i64, - (t.fract() * 1000.0 * 1000.0 * 1000.0) as u32, - ) - .single() - .expect("invalid timestamp"), - ); - } else { - return Err(ApiError::InvalidDataFormat { event: self.events }.into()); - } - } - Some(None) => return Err(ApiError::InvalidDataFormat { event: self.events }.into()), - } + self.process_time(&mut json)?; // Add time field let timestamp = match self.time.clone() { @@ -846,36 +1121,247 @@ impl<'de, R: JsonRead<'de>> EventIterator<'de, R> { log = log.with_batch_notifier(&batch); } - self.events += 1; - Ok(log.into()) } + /// Build an `EventMetadata` template from the current envelope context so + /// that VRL decoders can read source-supplied values via `%`-prefixed paths + /// before the decoder program executes. + /// + /// Peeks at the envelope `json` without consuming any fields (consumption + /// happens later in `build_events_decoded`). Falls back to sticky extractor + /// state for fields not present in the current envelope. + fn build_vrl_metadata(&self, json: &JsonValue) -> EventMetadata { + let mut metadata = EventMetadata::default(); + + // Splunk HEC token as a secret so VRL can read it via get_secret!() + if let Some(token) = &self.token { + metadata.set_splunk_hec_token(Arc::clone(token)); + } + + // Envelope host/source/sourcetype/index: peek current value; fall back + // to sticky extractor state. + let fields: &[(&str, &str)] = &[ + ("host", "splunk_hec.host"), + ("source", "splunk_hec.source"), + ("sourcetype", "splunk_hec.sourcetype"), + ("index", "splunk_hec.index"), + ]; + for (json_key, meta_path) in fields { + let val = json + .get(json_key) + .and_then(|v| v.as_str()) + .map(|s| Value::from(s.to_string())) + .or_else(|| { + self.extractors + .iter() + .find(|e| e.field == *json_key) + .and_then(|e| e.value.clone()) + }); + if let Some(v) = val { + metadata.value_mut().insert(*meta_path, v); + } + } + + // Channel: envelope field or header default + let channel = json + .get("channel") + .and_then(|v| v.as_str()) + .map(|s| Value::from(s.to_string())) + .or_else(|| self.channel.clone()); + if let Some(ch) = channel { + metadata.value_mut().insert("splunk_hec.channel", ch); + } + + metadata + } + + /// Decoded path: extract the envelope's `event` field as bytes (preserving shape), + /// run it through the second-stage decoder, and overlay envelope metadata so that + /// decoder-produced fields win on conflict. Returns the events along with a flag + /// indicating whether the codec hit any errors (so the caller can refuse to ack + /// a request that lost data). + fn build_events_decoded( + &mut self, + mut json: JsonValue, + decoder: Decoder, + ) -> Result<(Vec, bool), Rejection> { + self.envelopes_processed += 1; + let event = self.validate_event_field(&json)?; + // Strings are passed as raw bytes so decoders see the bare content + // (e.g. a JSON string event containing `{"foo":"bar"}` arrives at the + // decoder as `{"foo":"bar"}`, not `"{\"foo\":\"bar\"}"` ). All other + // JSON values (objects, arrays, numbers, bools) are serialized to JSON. + let payload = if let Some(s) = event.as_str() { + s.as_bytes().to_vec() + } else { + match serde_json::to_vec(event) { + Ok(bytes) => bytes, + Err(error) => { + let error: vector_lib::Error = Box::new(error); + emit!( + vector_lib::codecs::internal_events::DecoderDeserializeError { + error: &error + } + ); + emit!(ComponentEventsDropped:: { + count: 1, + reason: "Failed to serialize event field to bytes.", + }); + return Ok((vec![], true)); + } + } + }; + + self.process_time(&mut json)?; + + // Always forward a fallback timestamp so events without an explicit envelope + // `time` field still get one (matches the legacy /event behavior, which always + // wrote a timestamp). `decode_message` uses `try_insert`, so a decoder-supplied + // timestamp still wins on conflict. + let fallback_time = match self.time { + Time::Provided(t) | Time::Now(t) => t, + }; + + // Build a metadata template so VRL decoders can read envelope context + // via `%`-prefixed paths (e.g. `%splunk_hec.host`, `%vector.secrets.*`). + // For non-VRL decoders `with_metadata_template` is a no-op. + let decoder = decoder.with_metadata_template(self.build_vrl_metadata(&json)); + + let (decoded, had_decode_errors) = decode_payload( + decoder, + &payload, + Some(fallback_time), + true, // /event: write %splunk_hec.timestamp + DecodePayloadContext { + batch: &self.batch, + log_namespace: self.log_namespace, + events_received: &self.events_received, + splunk_hec_token: self.token.as_ref(), + }, + ); + + // Snapshot envelope metadata that has to apply uniformly to every decoded event. + let envelope_channel: Option = match json.get_mut("channel").map(JsonValue::take) { + Some(JsonValue::String(guid)) => Some(guid.into()), + _ => None, + }; + let envelope_fields: Option> = + match json.get_mut("fields").map(JsonValue::take) { + Some(JsonValue::Object(object)) => Some(object), + _ => None, + }; + let channel_path = owned_value_path!(CHANNEL); + + let mut out = Vec::with_capacity(decoded.len()); + for mut event in decoded { + if let Event::Log(log) = &mut event { + // channel: envelope value beats header default. Use `InsertIfEmpty` + // for legacy event fields, and `try_insert` for the Vector metadata + // path so a decoder-produced `%splunk_hec.channel` survives. + if let Some(channel_val) = envelope_channel.clone().or_else(|| self.channel.clone()) + { + match self.log_namespace { + LogNamespace::Legacy => { + self.log_namespace.insert_source_metadata( + SplunkConfig::NAME, + log, + Some(LegacyKey::InsertIfEmpty(&channel_path)), + lookup::path!(CHANNEL), + channel_val, + ); + } + LogNamespace::Vector => { + log.try_insert( + metadata_path!(SplunkConfig::NAME, CHANNEL), + channel_val, + ); + } + } + } + + // Top-level envelope fields (host/index/source/sourcetype) must be + // applied before `fields.*` so top-level beats `fields.*` when both + // are present β€” matching the non-decoder runtime precedence. Both + // still use InsertIfEmpty so the decoder's output wins over all + // envelope metadata. Order: decoder > top-level > fields. + for de in self.extractors.iter_mut() { + de.extract(log, &mut json); + } + + // fields: use `InsertIfEmpty` / `try_insert` to preserve decoder-wins + // and extractor-wins semantics (fields fill only what neither the + // decoder nor the top-level envelope keys have already set). + if let Some(ref fields) = envelope_fields { + for (key, value) in fields { + match self.log_namespace { + LogNamespace::Legacy => { + self.log_namespace.insert_source_metadata( + SplunkConfig::NAME, + log, + Some(LegacyKey::InsertIfEmpty(&owned_value_path!( + key.as_str() + ))), + lookup::path!(key.as_str()), + value.clone(), + ); + } + LogNamespace::Vector => { + log.try_insert( + metadata_path!(SplunkConfig::NAME, key.as_str()), + value.clone(), + ); + } + } + } + } + } + // `splunk_hec_token` is set inside `decode_payload` so the metadata is + // attached at the moment each event leaves the codec. Don't overwrite it + // here. + out.push(event); + } + + Ok((out, had_decode_errors)) + } + + /// Validate the `event` field of a HEC envelope, returning a reference to the + /// validated value or an error if it is missing, null, or (for string values) + /// empty. Shared between the decoder path and the legacy/vector construction + /// paths so they all enforce the same HEC protocol contract. + fn validate_event_field<'a>(&self, json: &'a JsonValue) -> Result<&'a JsonValue, Rejection> { + let event_idx = self.envelopes_processed.saturating_sub(1); + match json.get("event") { + None | Some(JsonValue::Null) => { + Err(ApiError::MissingEventField { event: event_idx }.into()) + } + Some(JsonValue::String(s)) if s.is_empty() => { + Err(ApiError::EmptyEventField { event: event_idx }.into()) + } + Some(event) => Ok(event), + } + } + /// Build the log event for the vector namespace. /// In this namespace the log event is created entirely from the event field. /// No renaming of the `line` field is done. fn build_log_vector(&mut self, json: &mut JsonValue) -> Result { - match json.get("event") { - Some(event) => { - let event: Value = event.into(); - let mut log = LogEvent::from(event); + let event: Value = self.validate_event_field(json)?.into(); + let mut log = LogEvent::from(event); - // EstimatedJsonSizeOf must be calculated before enrichment - self.events_received - .emit(CountByteSize(1, log.estimated_json_encoded_size_of())); + // EstimatedJsonSizeOf must be calculated before enrichment + self.events_received + .emit(CountByteSize(1, log.estimated_json_encoded_size_of())); - // The timestamp is extracted from the message for the Legacy namespace. - self.log_namespace.insert_vector_metadata( - &mut log, - log_schema().timestamp_key(), - lookup::path!("ingest_timestamp"), - chrono::Utc::now(), - ); + // The timestamp is extracted from the message for the Legacy namespace. + self.log_namespace.insert_vector_metadata( + &mut log, + log_schema().timestamp_key(), + lookup::path!("ingest_timestamp"), + chrono::Utc::now(), + ); - Ok(log) - } - None => Err(ApiError::MissingEventField { event: self.events }.into()), - } + Ok(log) } /// Build the log event for the legacy namespace. @@ -883,41 +1369,45 @@ impl<'de, R: JsonRead<'de>> EventIterator<'de, R> { /// (the docker splunk logger places the message in the event.line field) that string /// is placed in the message field. fn build_log_legacy(&mut self, json: &mut JsonValue) -> Result { + // validate_event_field checks for missing/null/empty-string + self.validate_event_field(json)?; let mut log = LogEvent::default(); - match json.get_mut("event") { - Some(event) => match event.take() { - JsonValue::String(string) => { - if string.is_empty() { - return Err(ApiError::EmptyEventField { event: self.events }.into()); + match json["event"].take() { + JsonValue::String(string) => { + log.maybe_insert(log_schema().message_key_target_path(), string); + } + JsonValue::Object(mut object) => { + if object.is_empty() { + return Err(ApiError::EmptyEventField { + event: self.envelopes_processed.saturating_sub(1), } - log.maybe_insert(log_schema().message_key_target_path(), string); + .into()); } - JsonValue::Object(mut object) => { - if object.is_empty() { - return Err(ApiError::EmptyEventField { event: self.events }.into()); - } - // Add 'line' value as 'event::schema().message_key' - if let Some(line) = object.remove("line") { - match line { - // This don't quite fit the meaning of a event::schema().message_key - JsonValue::Array(_) | JsonValue::Object(_) => { - log.insert(event_path!("line"), line); - } - _ => { - log.maybe_insert(log_schema().message_key_target_path(), line); - } + // Add 'line' value as 'event::schema().message_key' + if let Some(line) = object.remove("line") { + match line { + // This don't quite fit the meaning of a event::schema().message_key + JsonValue::Array(_) | JsonValue::Object(_) => { + log.insert(event_path!("line"), line); + } + _ => { + log.maybe_insert(log_schema().message_key_target_path(), line); } } + } - for (key, value) in object { - log.insert(event_path!(key.as_str()), value); - } + for (key, value) in object { + log.insert(event_path!(key.as_str()), value); } - _ => return Err(ApiError::InvalidDataFormat { event: self.events }.into()), - }, - None => return Err(ApiError::MissingEventField { event: self.events }.into()), - }; + } + _ => { + return Err(ApiError::InvalidDataFormat { + event: self.envelopes_processed.saturating_sub(1), + } + .into()); + } + } // EstimatedJsonSizeOf must be calculated before enrichment self.events_received @@ -928,13 +1418,23 @@ impl<'de, R: JsonRead<'de>> EventIterator<'de, R> { } impl<'de, R: JsonRead<'de>> Iterator for EventIterator<'de, R> { - type Item = Result; + /// Each item is `(events, had_decode_errors)` for one envelope - the boolean is + /// only ever `true` in the decoder path. Callers OR these together across the + /// whole request to decide whether ack registration is safe. + type Item = Result<(Vec, bool), Rejection>; fn next(&mut self) -> Option { match self.deserializer.next() { - Some(Ok(json)) => Some(self.build_event(json)), + Some(Ok(json)) => { + let result = if let Some(decoder) = self.decoder.clone() { + self.build_events_decoded(json, decoder) + } else { + self.build_event(json).map(|event| (vec![event], false)) + }; + Some(result) + } None => { - if self.events == 0 { + if self.envelopes_processed == 0 { Some(Err(ApiError::NoData.into())) } else { None @@ -944,12 +1444,112 @@ impl<'de, R: JsonRead<'de>> Iterator for EventIterator<'de, R> { emit!(SplunkHecRequestBodyInvalidError { error: error.into() }); - Some(Err( - ApiError::InvalidDataFormat { event: self.events }.into() - )) + // The deserializer failed to parse the next envelope, so the failing + // envelope's index is the count of envelopes already processed (not + // `envelopes_processed - 1`, which is what build-time errors use). + Some(Err(ApiError::InvalidDataFormat { + event: self.envelopes_processed, + } + .into())) + } + } + } +} + +struct DecodePayloadContext<'a> { + batch: &'a Option, + log_namespace: LogNamespace, + events_received: &'a Registered, + splunk_hec_token: Option<&'a Arc>, +} + +/// Run a payload through the configured `framing` + `decoding` codec. +/// +/// Returns the decoded events along with a flag indicating whether any decode error +/// occurred. The shared `crate::sources::util::decode_message` helper swallows +/// decode errors silently, which is fine for sources without ack semantics, but for +/// `splunk_hec` we need to know about errors so we can refuse to acknowledge a +/// request that lost data mid-stream. +/// +/// On each decoded event this helper sets `source_type`, `vector.ingest_timestamp`, +/// the optional `splunk_hec.timestamp` (only when `set_source_timestamp` is `true`, +/// i.e. for the `/event` endpoint which carries an HEC envelope `time` field), and +/// the optional Splunk HEC token. Pass `set_source_timestamp = false` for `/raw`, +/// which has no envelope timestamp and should only receive `%vector.ingest_timestamp`. +fn decode_payload( + mut decoder: Decoder, + payload: &[u8], + fallback_timestamp: Option>, + set_source_timestamp: bool, + ctx: DecodePayloadContext<'_>, +) -> (Vec, bool) { + let DecodePayloadContext { + batch, + log_namespace, + events_received, + splunk_hec_token, + } = ctx; + let mut buffer = BytesMut::with_capacity(payload.len()); + buffer.extend_from_slice(payload); + let now = Utc::now(); + let mut events: Vec = Vec::new(); + let mut had_errors = false; + + loop { + match decoder.decode_eof(&mut buffer) { + Ok(Some((decoded, _))) => { + for mut event in decoded { + if let Event::Log(log) = &mut event { + log_namespace.insert_vector_metadata( + log, + log_schema().source_type_key(), + lookup::path!("source_type"), + Bytes::from_static(SplunkConfig::NAME.as_bytes()), + ); + match log_namespace { + LogNamespace::Vector => { + // Only write %splunk_hec.timestamp for the /event + // endpoint, which has a real HEC envelope timestamp. + // /raw has no envelope time and should only get the + // standard %vector.ingest_timestamp below. + if set_source_timestamp && let Some(timestamp) = fallback_timestamp + { + log.try_insert( + metadata_path!(SplunkConfig::NAME, "timestamp"), + timestamp, + ); + } + log.insert(metadata_path!("vector", "ingest_timestamp"), now); + } + LogNamespace::Legacy => { + if let Some(timestamp) = fallback_timestamp + && let Some(timestamp_key) = log_schema().timestamp_key() + { + log.try_insert((PathPrefix::Event, timestamp_key), timestamp); + } + } + } + } + if let Some(token) = splunk_hec_token { + event.metadata_mut().set_splunk_hec_token(Arc::clone(token)); + } + events_received.emit(CountByteSize(1, event.estimated_json_encoded_size_of())); + events.push(event.with_batch_notifier_option(batch)); + } + } + Ok(None) => break, + Err(error) => { + // The decoder logs its own error; record that one occurred so the + // caller can refuse to ack a request that lost data. + had_errors = true; + if !error.can_continue() { + break; + } } } } + + (events, had_errors) } /// Parse a `i64` unix timestamp that can either be in seconds, milliseconds or @@ -986,12 +1586,20 @@ fn parse_timestamp(t: i64) -> Option> { Some(ts) } +/// How to write the legacy key when `DefaultExtractor::extract` applies a value. +#[derive(Clone, Copy)] +enum LegacyKeyStrategy { + Overwrite, + InsertIfEmpty, +} + /// Maintains last known extracted value of field and uses it in the absence of field. struct DefaultExtractor { field: &'static str, to_field: OptionalValuePath, value: Option, log_namespace: LogNamespace, + legacy_key_strategy: LegacyKeyStrategy, } impl DefaultExtractor { @@ -1005,6 +1613,7 @@ impl DefaultExtractor { to_field, value: None, log_namespace, + legacy_key_strategy: LegacyKeyStrategy::Overwrite, } } @@ -1019,9 +1628,18 @@ impl DefaultExtractor { to_field, value: value.into(), log_namespace, + legacy_key_strategy: LegacyKeyStrategy::Overwrite, } } + /// Set the strategy used when writing this extractor's legacy key. Defaults to + /// `Overwrite`; the decoder path uses `InsertIfEmpty` for fields that may collide + /// with decoder-produced output (e.g. `host`). + const fn with_legacy_key_strategy(mut self, strategy: LegacyKeyStrategy) -> Self { + self.legacy_key_strategy = strategy; + self + } + fn extract(&mut self, log: &mut LogEvent, value: &mut JsonValue) { // Process json_field if let Some(JsonValue::String(new_value)) = value.get_mut(self.field).map(JsonValue::take) { @@ -1032,13 +1650,33 @@ impl DefaultExtractor { if let Some(index) = self.value.as_ref() && let Some(metadata_key) = self.to_field.path.as_ref() { - self.log_namespace.insert_source_metadata( - SplunkConfig::NAME, - log, - Some(LegacyKey::Overwrite(metadata_key)), - &self.to_field.path.clone().unwrap_or(owned_value_path!("")), - index.clone(), - ) + // For Vector namespace + InsertIfEmpty (decoder mode): check the metadata + // value tree before inserting so VRL-produced values aren't overwritten. + // `insert_source_metadata` for Vector ns always calls `insert`, not + // `try_insert`, so we replicate its path construction here. + if matches!(self.log_namespace, LogNamespace::Vector) + && matches!(self.legacy_key_strategy, LegacyKeyStrategy::InsertIfEmpty) + { + log.try_insert( + ( + PathPrefix::Metadata, + lookup::path!(SplunkConfig::NAME).concat(metadata_key), + ), + index.clone(), + ); + } else { + let legacy_key = match self.legacy_key_strategy { + LegacyKeyStrategy::Overwrite => LegacyKey::Overwrite(metadata_key), + LegacyKeyStrategy::InsertIfEmpty => LegacyKey::InsertIfEmpty(metadata_key), + }; + self.log_namespace.insert_source_metadata( + SplunkConfig::NAME, + log, + Some(legacy_key), + &self.to_field.path.clone().unwrap_or(owned_value_path!("")), + index.clone(), + ); + } } } } @@ -1052,7 +1690,13 @@ enum Time { Provided(DateTime), } -/// Creates event from raw request +/// Creates events from a raw HEC request body. +/// +/// Without a decoder, returns a single event whose message is the (decompressed) +/// request body. With a decoder, the body is fed through the configured framing + +/// decoding pipeline and one or more events are returned. The boolean second tuple +/// element is `true` when the decoder hit any (recoverable or non-recoverable) +/// errors during the request, so the caller can refuse to acknowledge the request. #[allow(clippy::too_many_arguments)] fn raw_event( bytes: Bytes, @@ -1063,42 +1707,23 @@ fn raw_event( batch: Option, log_namespace: LogNamespace, events_received: &Registered, -) -> Result { + decoder: Option, + splunk_hec_token: Option>, +) -> Result<(Vec, bool), Rejection> { // Process gzip - let message: Value = if gzip { + let body_bytes: Bytes = if gzip { let mut data = Vec::new(); match MultiGzDecoder::new(bytes.reader()).read_to_end(&mut data) { Ok(0) => return Err(ApiError::NoData.into()), - Ok(_) => Value::from(Bytes::from(data)), + Ok(_) => Bytes::from(data), Err(error) => { emit!(SplunkHecRequestBodyInvalidError { error }); return Err(ApiError::InvalidDataFormat { event: 0 }.into()); } } } else { - bytes.into() - }; - - // Construct event - let mut log = match log_namespace { - LogNamespace::Vector => LogEvent::from(message), - LogNamespace::Legacy => { - let mut log = LogEvent::default(); - log.maybe_insert(log_schema().message_key_target_path(), message); - log - } + bytes }; - // We need to calculate the estimated json size of the event BEFORE enrichment. - events_received.emit(CountByteSize(1, log.estimated_json_encoded_size_of())); - - // Add channel - log_namespace.insert_source_metadata( - SplunkConfig::NAME, - &mut log, - Some(LegacyKey::Overwrite(&owned_value_path!(CHANNEL))), - lookup::path!(CHANNEL), - channel, - ); // host-field priority for raw endpoint: // - x-forwarded-for is set to `host` field first, if present. If not present: @@ -1109,23 +1734,105 @@ fn raw_event( remote.map(|remote| remote.to_string()) }; - if let Some(host) = host { - log_namespace.insert_source_metadata( - SplunkConfig::NAME, + let decoder_in_use = decoder.is_some(); + let (mut events, had_decode_errors): (Vec, bool) = if let Some(decoder) = decoder { + // Build a metadata template so VRL decoders can read raw-endpoint context + // via `%`-prefixed paths (e.g. `%splunk_hec.channel`, `%splunk_hec.host`, + // `%vector.secrets.splunk_hec_token`). No-op for non-VRL decoders. + let decoder = { + let mut meta = EventMetadata::default(); + if let Some(token) = splunk_hec_token.as_ref() { + meta.set_splunk_hec_token(Arc::clone(token)); + } + if let Some(ref h) = host { + meta.value_mut().insert("splunk_hec.host", h.clone()); + } + meta.value_mut() + .insert("splunk_hec.channel", channel.clone()); + decoder.with_metadata_template(meta) + }; + + // Pass ingest time as the fallback timestamp so decoded events always have + // one - matches `insert_standard_vector_source_metadata` in the legacy raw + // path. `decode_payload` uses `try_insert`, so a decoder-supplied timestamp + // still wins on conflict. + decode_payload( + decoder, + &body_bytes, + Some(Utc::now()), + false, // /raw: no HEC envelope timestamp; only %vector.ingest_timestamp + DecodePayloadContext { + batch: &batch, + log_namespace, + events_received, + splunk_hec_token: splunk_hec_token.as_ref(), + }, + ) + } else { + let message: Value = body_bytes.into(); + let mut log = match log_namespace { + LogNamespace::Vector => LogEvent::from(message), + LogNamespace::Legacy => { + let mut log = LogEvent::default(); + log.maybe_insert(log_schema().message_key_target_path(), message); + log + } + }; + // We need to calculate the estimated json size of the event BEFORE enrichment. + events_received.emit(CountByteSize(1, log.estimated_json_encoded_size_of())); + + log_namespace.insert_standard_vector_source_metadata( &mut log, - log_schema().host_key().map(LegacyKey::InsertIfEmpty), - lookup::path!("host"), - host, + SplunkConfig::NAME, + Utc::now(), ); - } - log_namespace.insert_standard_vector_source_metadata(&mut log, SplunkConfig::NAME, Utc::now()); + if let Some(batch) = batch.clone() { + log = log.with_batch_notifier(&batch); + } + (vec![Event::from(log)], false) + }; - if let Some(batch) = batch { - log = log.with_batch_notifier(&batch); + let channel_path = owned_value_path!(CHANNEL); + for event in &mut events { + if let Event::Log(log) = event { + // With a decoder configured, defer to anything it produced at the legacy + // When a decoder is in use, preserve decoder-wins semantics for Vector ns + // by using `try_insert` on the metadata path (insert_source_metadata for + // Vector ns always overwrites). Without a decoder the log is freshly + // constructed so overwriting is correct. + if decoder_in_use && matches!(log_namespace, LogNamespace::Vector) { + log.try_insert(metadata_path!(SplunkConfig::NAME, CHANNEL), channel.clone()); + if let Some(ref h) = host { + log.try_insert(metadata_path!(SplunkConfig::NAME, "host"), h.clone()); + } + } else { + let channel_legacy_key = if decoder_in_use { + LegacyKey::InsertIfEmpty(&channel_path) + } else { + LegacyKey::Overwrite(&channel_path) + }; + log_namespace.insert_source_metadata( + SplunkConfig::NAME, + log, + Some(channel_legacy_key), + lookup::path!(CHANNEL), + channel.clone(), + ); + if let Some(ref host) = host { + log_namespace.insert_source_metadata( + SplunkConfig::NAME, + log, + log_schema().host_key().map(LegacyKey::InsertIfEmpty), + lookup::path!("host"), + host.clone(), + ); + } + } + } } - Ok(Event::from(log)) + Ok((events, had_decode_errors)) } #[derive(Clone, Copy, Debug, Snafu)] @@ -1219,6 +1926,17 @@ mod splunk_response { pub const ACK_IS_DISABLED: HecResponse = HecResponse::new(HecStatusCode::AckIsDisabled); } +async fn register_ack( + idx_ack: Option>, + receiver: Option, + channel: Option, +) -> Result, Rejection> { + match (idx_ack, receiver, channel) { + (Some(ack), Some(rx), Some(ch)) => Ok(Some(ack.get_ack_id_from_channel(ch, rx).await?)), + _ => Ok(None), + } +} + fn finish_ok(maybe_ack_id: Option) -> Response { let body = if let Some(ack_id) = maybe_ack_id { HecResponse::new(HecStatusCode::Success).with_metadata(HecResponseMetadata::AckId(ack_id)) @@ -1311,7 +2029,10 @@ mod tests { use vector_lib::{ codecs::{ BytesDecoderConfig, JsonSerializerConfig, TextSerializerConfig, - decoding::DeserializerConfig, + decoding::{ + DeserializerConfig, + format::{VrlDeserializerConfig, VrlDeserializerOptions}, + }, }, event::EventStatus, schema::Definition, @@ -1383,6 +2104,8 @@ mod tests { store_hec_token, log_namespace: None, keepalive: Default::default(), + event: CodecConfig::default(), + raw: CodecConfig::default(), } .build(cx) .await @@ -2629,6 +3352,594 @@ mod tests { ); } + async fn source_with_codec( + event: CodecConfig, + raw: CodecConfig, + ) -> ( + impl Stream + Unpin + use<>, + SocketAddr, + PortGuard, + ) { + let (sender, recv) = SourceSender::new_test_finalize(EventStatus::Delivered); + let (_guard, address) = next_addr(); + let cx = SourceContext::new_test(sender, None); + tokio::spawn(async move { + SplunkConfig { + address, + token: Some(TOKEN.to_owned().into()), + valid_tokens: None, + tls: None, + acknowledgements: Default::default(), + store_hec_token: false, + log_namespace: None, + keepalive: Default::default(), + event, + raw, + } + .build(cx) + .await + .unwrap() + .await + .unwrap() + }); + wait_for_tcp(address).await; + (recv, address, _guard) + } + + /// Codec config that just sets `decoding` (default framing). + fn codec_decoding(decoding: DeserializerConfig) -> CodecConfig { + CodecConfig { + framing: None, + decoding: Some(decoding), + } + } + + /// Codec config that sets both `framing` and `decoding`. + fn codec_full( + framing: Option, + decoding: Option, + ) -> CodecConfig { + CodecConfig { framing, decoding } + } + + #[tokio::test] + async fn decoder_event_endpoint_json_string() { + assert_source_compliance(&HTTP_PUSH_SOURCE_TAGS, async { + let (source, address, _guard) = source_with_codec( + codec_decoding(vector_lib::codecs::JsonDeserializerConfig::default().into()), + CodecConfig::default(), + ) + .await; + let envelope = + r#"{"event":"{\"foo\":\"bar\",\"n\":42}","host":"client-host","sourcetype":"my-app"}"#; + assert_eq!( + 200, + post(address, "services/collector/event", envelope).await + ); + + let event = collect_n(source, 1).await.remove(0); + let log = event.as_log(); + assert_eq!(log["foo"], "bar".into()); + assert_eq!(log["n"], 42.into()); + assert_eq!( + log[log_schema().host_key().unwrap().to_string().as_str()], + "client-host".into() + ); + assert_eq!(log[&super::SOURCETYPE], "my-app".into()); + }) + .await; + } + + #[tokio::test] + async fn decoder_event_endpoint_json_object_round_trip() { + assert_source_compliance(&HTTP_PUSH_SOURCE_TAGS, async { + let (source, address, _guard) = source_with_codec( + codec_decoding(vector_lib::codecs::JsonDeserializerConfig::default().into()), + CodecConfig::default(), + ) + .await; + let envelope = r#"{"event":{"foo":"bar","nested":{"k":1}},"host":"h"}"#; + assert_eq!( + 200, + post(address, "services/collector/event", envelope).await + ); + + let event = collect_n(source, 1).await.remove(0); + let log = event.as_log(); + assert_eq!(log["foo"], "bar".into()); + assert_eq!(*log.get("nested.k").unwrap(), 1.into()); + assert_eq!( + log[log_schema().host_key().unwrap().to_string().as_str()], + "h".into() + ); + }) + .await; + } + + #[tokio::test] + async fn decoder_event_endpoint_all_envelope_fields_yield_to_decoder() { + // The decoded path must defer to the codec for `splunk_channel`, + // `splunk_index`, `splunk_source`, and `splunk_sourcetype` in legacy ns - + // not just `host`. Otherwise the changelog's "decoder wins on conflict" + // promise is broken for HEC envelope metadata. + assert_source_compliance(&HTTP_PUSH_SOURCE_TAGS, async { + let (source, address, _guard) = source_with_codec( + codec_decoding(vector_lib::codecs::JsonDeserializerConfig::default().into()), + CodecConfig::default(), + ) + .await; + // The string `event` decodes to a JSON object that pre-populates each + // legacy splunk_* field. The envelope sets conflicting values for the + // same fields and must lose. + let envelope = r#"{ + "event":"{\"splunk_channel\":\"decoder-channel\",\"splunk_index\":\"decoder-index\",\"splunk_source\":\"decoder-source\",\"splunk_sourcetype\":\"decoder-sourcetype\"}", + "index":"envelope-index", + "source":"envelope-source", + "sourcetype":"envelope-sourcetype" + }"#; + assert_eq!( + 200, + post(address, "services/collector/event", envelope).await + ); + + let event = collect_n(source, 1).await.remove(0); + let log = event.as_log(); + assert_eq!(log[&super::CHANNEL], "decoder-channel".into()); + assert_eq!(log[&super::INDEX], "decoder-index".into()); + assert_eq!(log[&super::SOURCE], "decoder-source".into()); + assert_eq!(log[&super::SOURCETYPE], "decoder-sourcetype".into()); + }) + .await; + } + + #[tokio::test] + async fn decoder_event_endpoint_decoder_field_wins_over_envelope() { + assert_source_compliance(&HTTP_PUSH_SOURCE_TAGS, async { + let (source, address, _guard) = source_with_codec( + codec_decoding(vector_lib::codecs::JsonDeserializerConfig::default().into()), + CodecConfig::default(), + ) + .await; + // The string `event` decodes to {host: "decoder-host"}; the envelope sets + // host: "envelope-host". The decoder's value must win. + let envelope = r#"{"event":"{\"host\":\"decoder-host\"}","host":"envelope-host"}"#; + assert_eq!( + 200, + post(address, "services/collector/event", envelope).await + ); + + let event = collect_n(source, 1).await.remove(0); + let log = event.as_log(); + assert_eq!( + log[log_schema().host_key().unwrap().to_string().as_str()], + "decoder-host".into() + ); + }) + .await; + } + + #[tokio::test] + async fn decoder_event_endpoint_decode_failure_returns_200() { + // A malformed inner JSON must not surface as an HTTP error to the Splunk + // client - decode failures are swallowed by the codec like other Vector + // sources do. + let (_source, address, _guard) = source_with_codec( + codec_decoding(vector_lib::codecs::JsonDeserializerConfig::default().into()), + CodecConfig::default(), + ) + .await; + let envelope = r#"{"event":"not valid json {","host":"h"}"#; + assert_eq!( + 200, + post(address, "services/collector/event", envelope).await + ); + } + + #[tokio::test] + async fn decoder_raw_endpoint_newline_delimited() { + assert_source_compliance(&HTTP_PUSH_SOURCE_TAGS, async { + let (source, address, _guard) = source_with_codec( + CodecConfig::default(), + codec_full( + Some(FramingConfig::NewlineDelimited(Default::default())), + Some(DeserializerConfig::Bytes), + ), + ) + .await; + let body = "line1\nline2\nline3"; + assert_eq!(200, post(address, "services/collector/raw", body).await); + + let events = collect_n(source, 3).await; + assert_eq!(events.len(), 3); + let messages: Vec = events + .iter() + .map(|e| { + e.as_log()[log_schema().message_key().unwrap().to_string()] + .to_string_lossy() + .into_owned() + }) + .collect(); + assert!(messages.contains(&"line1".to_string())); + assert!(messages.contains(&"line2".to_string())); + assert!(messages.contains(&"line3".to_string())); + + // All events share the channel from the request header. + for event in &events { + assert_eq!(event.as_log()[&super::CHANNEL], "channel".into()); + } + }) + .await; + } + + #[tokio::test] + async fn decoder_event_endpoint_envelope_without_time_has_fallback_timestamp() { + // Regression: with a decoder set, an envelope that omits `time` must still + // produce events with a timestamp (the legacy /event path always wrote one). + assert_source_compliance(&HTTP_PUSH_SOURCE_TAGS, async { + let (source, address, _guard) = source_with_codec( + codec_decoding(vector_lib::codecs::JsonDeserializerConfig::default().into()), + CodecConfig::default(), + ) + .await; + let envelope = r#"{"event":"{\"foo\":\"bar\"}"}"#; + assert_eq!( + 200, + post(address, "services/collector/event", envelope).await + ); + + let event = collect_n(source, 1).await.remove(0); + assert!( + event.as_log().get_timestamp().is_some(), + "decoded event from envelope without `time` field is missing a timestamp" + ); + }) + .await; + } + + #[tokio::test] + async fn decoder_independent_per_endpoint_codecs() { + // /event and /raw can be configured with completely different codecs and + // each endpoint applies only its own. Here /event uses JSON decoding (so a + // string `event` field decodes to fields) and /raw uses newline framing + // with a bytes decoder (so a multi-line body fans out to N events). + assert_source_compliance(&HTTP_PUSH_SOURCE_TAGS, async { + let (source, address, _guard) = source_with_codec( + codec_decoding(vector_lib::codecs::JsonDeserializerConfig::default().into()), + codec_full( + Some(FramingConfig::NewlineDelimited(Default::default())), + Some(DeserializerConfig::Bytes), + ), + ) + .await; + + // /event: JSON decoder turns the inner string into structured fields. + assert_eq!( + 200, + post( + address, + "services/collector/event", + r#"{"event":"{\"foo\":\"bar\"}"}"# + ) + .await + ); + // /raw: newline framing splits the body into three events. + assert_eq!( + 200, + post(address, "services/collector/raw", "a\nb\nc").await + ); + + let events = collect_n(source, 4).await; + assert_eq!(events.len(), 4); + + // The /event request produces one log with `foo=bar`. + let event_log = events + .iter() + .find(|e| e.as_log().contains("foo")) + .expect("expected /event request to produce a log with `foo` set"); + assert_eq!(event_log.as_log()["foo"], "bar".into()); + + // The /raw request produces three logs whose messages are the lines. + let raw_messages: Vec = events + .iter() + .filter(|e| !e.as_log().contains("foo")) + .map(|e| { + e.as_log()[log_schema().message_key().unwrap().to_string()] + .to_string_lossy() + .into_owned() + }) + .collect(); + assert_eq!(raw_messages.len(), 3); + assert!(raw_messages.contains(&"a".to_string())); + assert!(raw_messages.contains(&"b".to_string())); + assert!(raw_messages.contains(&"c".to_string())); + }) + .await; + } + + /// End-to-end test for the second-stage VRL decoder on `/services/collector/event`. + /// + /// Validates the core use case from PR #25312: a VRL program decodes the + /// inner `event` payload *and* reads HEC envelope metadata injected before + /// decoding via `%splunk_hec.*` paths. + #[tokio::test] + async fn decoder_vrl_reads_envelope_metadata() { + assert_source_compliance(&HTTP_PUSH_SOURCE_TAGS, async { + let vrl_source = r#" + # Read envelope metadata injected before this VRL program runs. + .envelope_host = string!(%splunk_hec.host) + .envelope_sourcetype = string!(%splunk_hec.sourcetype) + + # Decode the inner JSON payload (the bytes of the `event` string). + . = merge!(parse_json!(string!(.message)), .) + "#; + + let event_codec = codec_decoding( + DeserializerConfig::Vrl(VrlDeserializerConfig { + vrl: VrlDeserializerOptions { + source: vrl_source.into(), + timezone: None, + }, + }), + ); + + let (source, address, _guard) = + source_with_codec(event_codec, CodecConfig::default()).await; + + // Send a HEC event whose `event` field is a JSON-encoded string. + // The VRL decoder should parse it and also read the envelope host/sourcetype. + let payload = r#"{"event":"{\"level\":\"info\",\"msg\":\"hello\"}","host":"splunk-host","sourcetype":"my-app"}"#; + assert_eq!( + 200, + post(address, "services/collector/event", payload).await + ); + + let event = collect_n(source, 1).await.remove(0); + let log = event.as_log(); + + // Inner JSON decoded correctly. + assert_eq!(log["level"], "info".into()); + assert_eq!(log["msg"], "hello".into()); + + // VRL read envelope metadata via %splunk_hec.* and wrote it to event fields. + assert_eq!(log["envelope_host"], "splunk-host".into()); + assert_eq!(log["envelope_sourcetype"], "my-app".into()); + + // Post-decode splunk_hec metadata still applied (host, sourcetype). + assert_eq!( + log[log_schema().host_key().unwrap().to_string().as_str()], + "splunk-host".into() + ); + assert_eq!(log[&super::SOURCETYPE], "my-app".into()); + }) + .await; + } + + #[tokio::test] + async fn decoder_raw_endpoint_event_has_fallback_timestamp() { + // Regression: decoded /raw events must carry an ingest timestamp like the + // legacy raw_event path did via `insert_standard_vector_source_metadata`. + assert_source_compliance(&HTTP_PUSH_SOURCE_TAGS, async { + let (source, address, _guard) = source_with_codec( + CodecConfig::default(), + codec_full(None, Some(DeserializerConfig::Bytes)), + ) + .await; + let body = "hello"; + assert_eq!(200, post(address, "services/collector/raw", body).await); + + let event = collect_n(source, 1).await.remove(0); + assert!( + event.as_log().get_timestamp().is_some(), + "decoded /raw event is missing a timestamp" + ); + }) + .await; + } + + #[tokio::test] + async fn decoder_raw_endpoint_empty_decode_does_not_ack() { + // Regression: when the decoder produces zero events from a raw payload and + // acknowledgements are enabled, the response must not include an `ackId` + // because /services/collector/ack would otherwise report success for data + // Vector silently dropped. + let ack_config = HecAcknowledgementsConfig { + enabled: Some(true), + ..Default::default() + }; + let (sender, _recv) = SourceSender::new_test_finalize(EventStatus::Delivered); + let (_guard, address) = next_addr(); + let cx = SourceContext::new_test(sender, None); + tokio::spawn(async move { + SplunkConfig { + address, + token: Some(TOKEN.to_owned().into()), + valid_tokens: None, + tls: None, + acknowledgements: ack_config, + store_hec_token: false, + log_namespace: None, + keepalive: Default::default(), + event: CodecConfig::default(), + raw: codec_decoding(vector_lib::codecs::JsonDeserializerConfig::default().into()), + } + .build(cx) + .await + .unwrap() + .await + .unwrap() + }); + wait_for_tcp(address).await; + + let opts = SendWithOpts { + channel: Some(Channel::Header("guid")), + forwarded_for: None, + }; + // A body the JSON decoder cannot parse - codec drops it, no events emitted. + let body = "not json {"; + let response = send_with_response(address, "services/collector/raw", body, TOKEN, &opts) + .await + .json::() + .await + .unwrap(); + + assert_eq!(response["code"].as_u64(), Some(0), "response: {response:?}"); + assert!( + response.get("ackId").is_none(), + "expected no ackId in response when decoder produced zero events, got: {response:?}" + ); + } + + #[tokio::test] + async fn decoder_raw_endpoint_partial_decode_does_not_ack() { + // Regression: a request whose body decodes into some valid frames AND some + // dropped frames (e.g., `valid \n invalid \n valid` under newline framing + // with a JSON decoder) must not return an `ackId`. Otherwise + // /services/collector/ack reports success for data Vector silently dropped. + let ack_config = HecAcknowledgementsConfig { + enabled: Some(true), + ..Default::default() + }; + let (sender, _recv) = SourceSender::new_test_finalize(EventStatus::Delivered); + let (_guard, address) = next_addr(); + let cx = SourceContext::new_test(sender, None); + tokio::spawn(async move { + SplunkConfig { + address, + token: Some(TOKEN.to_owned().into()), + valid_tokens: None, + tls: None, + acknowledgements: ack_config, + store_hec_token: false, + log_namespace: None, + keepalive: Default::default(), + event: CodecConfig::default(), + raw: codec_full( + Some(FramingConfig::NewlineDelimited(Default::default())), + Some(vector_lib::codecs::JsonDeserializerConfig::default().into()), + ), + } + .build(cx) + .await + .unwrap() + .await + .unwrap() + }); + wait_for_tcp(address).await; + + let opts = SendWithOpts { + channel: Some(Channel::Header("guid")), + forwarded_for: None, + }; + // Two valid JSON frames bracketing one invalid frame. + let body = "{\"valid\":1}\nnot json\n{\"valid\":2}"; + let response = send_with_response(address, "services/collector/raw", body, TOKEN, &opts) + .await + .json::() + .await + .unwrap(); + + assert_eq!(response["code"].as_u64(), Some(0), "response: {response:?}"); + assert!( + response.get("ackId").is_none(), + "expected no ackId when the decoder dropped a frame mid-request, got: {response:?}" + ); + } + + #[tokio::test] + async fn decoder_event_endpoint_error_index_matches_envelope_not_fanout() { + // Regression: with the decoder fanning out one envelope into many events, + // `InvalidEventNumber` in error responses must still report the failing + // envelope's zero-indexed position, not the cumulative event count. + let (source, address, _guard) = source_with_codec( + codec_full( + Some(FramingConfig::NewlineDelimited(Default::default())), + Some(DeserializerConfig::Bytes), + ), + CodecConfig::default(), + ) + .await; + // Envelope 0 has an `event` string with three lines: with newline framing + // and a bytes decoder, that fans out to three events. Envelope 1 omits + // `event`, so the decoded path returns `MissingEventField { event: 1 }`. + let body = "{\"event\":\"a\\nb\\nc\"}{\"foo\":\"bar\"}"; + + let opts = SendWithOpts { + channel: Some(Channel::Header("guid")), + forwarded_for: None, + }; + let response = + send_with_response(address, "services/collector/event", body, TOKEN, &opts).await; + let status = response.status(); + let body: serde_json::Value = response.json().await.unwrap(); + + assert_eq!(status.as_u16(), 400, "body: {body:?}"); + assert_eq!( + body["invalid-event-number"].as_u64(), + Some(1), + "expected envelope index 1 (the failing envelope), not a fan-out event index. body: {body:?}" + ); + // Drain the partially-emitted events so the source task doesn't block. + let _ = collect_n(source, 3).await; + } + + #[test] + fn output_schema_definition_with_decoder_vector_namespace() { + let config = SplunkConfig { + log_namespace: Some(true), + event: codec_decoding(vector_lib::codecs::JsonDeserializerConfig::default().into()), + ..Default::default() + }; + let definition = config + .outputs(LogNamespace::Vector) + .remove(0) + .schema_definition(true); + + // The decoder's schema produces `Kind::json()` at the root, the source + // layers its envelope metadata fields on top, and the legacy log shape is + // unioned in (since /raw has no decoder and still emits legacy events) - + // contributing the `message` meaning at root. + let expected_definition = + Definition::new_with_default_metadata(Kind::json(), [LogNamespace::Vector]) + .with_meaning(OwnedTargetPath::event_root(), meaning::MESSAGE) + .with_metadata_field( + &owned_value_path!("vector", "source_type"), + Kind::bytes(), + None, + ) + .with_metadata_field( + &owned_value_path!("vector", "ingest_timestamp"), + Kind::timestamp(), + None, + ) + .with_metadata_field( + &owned_value_path!("splunk_hec", "host"), + Kind::bytes(), + Some("host"), + ) + .with_metadata_field( + &owned_value_path!("splunk_hec", "index"), + Kind::bytes(), + None, + ) + .with_metadata_field( + &owned_value_path!("splunk_hec", "source"), + Kind::bytes(), + Some("service"), + ) + .with_metadata_field( + &owned_value_path!("splunk_hec", "channel"), + Kind::bytes(), + None, + ) + .with_metadata_field( + &owned_value_path!("splunk_hec", "sourcetype"), + Kind::bytes(), + None, + ); + + assert_eq!(definition, Some(expected_definition)); + } + #[test] fn output_schema_definition_vector_namespace() { let config = SplunkConfig { diff --git a/src/sources/util/body_decoding.rs b/src/sources/util/body_decoding.rs index 6a99da3240792..d7947940e9ab4 100644 --- a/src/sources/util/body_decoding.rs +++ b/src/sources/util/body_decoding.rs @@ -2,12 +2,11 @@ use vector_lib::configurable::configurable_component; /// Content encoding. #[configurable_component] -#[derive(Clone, Copy, Debug, Derivative, Eq, PartialEq)] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] #[serde(rename_all = "snake_case")] -#[derivative(Default)] pub enum Encoding { /// Plaintext. - #[derivative(Default)] + #[default] Text, /// Newline-delimited JSON. diff --git a/src/sources/util/framestream.rs b/src/sources/util/framestream.rs index a42bae5815322..9c96b2f3fc4ef 100644 --- a/src/sources/util/framestream.rs +++ b/src/sources/util/framestream.rs @@ -908,7 +908,7 @@ async fn spawn_event_handling_tasks( ) -> JoinHandle<()> { wait_for_task_quota(&active_task_nums, max_frame_handling_tasks).await; - tokio::spawn(async move { + crate::spawn_in_current_span(async move { future::ready({ if let Some(evt) = event_handler.handle_event(received_from, event_data) && event_sink.send_event(evt).await.is_err() diff --git a/src/sources/util/grpc/decompression.rs b/src/sources/util/grpc/decompression.rs index 535c938f3cbbe..26427f27afe6a 100644 --- a/src/sources/util/grpc/decompression.rs +++ b/src/sources/util/grpc/decompression.rs @@ -1,16 +1,17 @@ use std::{ cmp, future::Future, - io::Write, + io::{self, Write}, mem, pin::Pin, + sync::LazyLock, task::{Context, Poll}, }; use bytes::{Buf, BufMut, BytesMut}; use flate2::write::GzDecoder; use futures_util::FutureExt; -use http::{Request, Response}; +use http::{HeaderValue, Request, Response}; use hyper::{ Body, body::{HttpBody, Sender}, @@ -31,8 +32,58 @@ const GRPC_MESSAGE_HEADER_LEN: usize = mem::size_of::() + mem::size_of:: &'static str { + match self { + Self::Gzip => "gzip", + Self::Zstd => "zstd", + Self::Identity => "identity", + } + } + + fn parse(s: &str) -> Option { + Self::ALL.iter().copied().find(|e| e.as_str() == s) + } + + // `identity` is the gRPC no-op encoding: the request body is already + // uncompressed, so there's nothing to decompress. + const fn to_scheme(self) -> Option { + match self { + Self::Gzip => Some(CompressionScheme::Gzip), + Self::Zstd => Some(CompressionScheme::Zstd), + Self::Identity => None, + } + } +} + +// Advertised to clients via `grpc-accept-encoding`. Derived from +// `AdvertisedEncoding::ALL` so this layer is the single owner of gRPC compression +// negotiation for all Vector gRPC sources and the header value cannot drift from +// the set of schemes actually handled. +static GRPC_ACCEPT_ENCODING_VALUE: LazyLock = LazyLock::new(|| { + AdvertisedEncoding::ALL + .iter() + .map(|e| e.as_str()) + .collect::>() + .join(",") +}); + enum CompressionScheme { Gzip, + Zstd, } impl CompressionScheme { @@ -49,17 +100,18 @@ impl CompressionScheme { .transpose() .and_then(|value| match value { None => Ok(None), - Some(scheme) => match scheme.as_str() { - "gzip" => Ok(Some(CompressionScheme::Gzip)), - other => Err(Status::unimplemented(format!( - "compression scheme `{other}` is not supported" + Some(scheme) => match AdvertisedEncoding::parse(&scheme) { + Some(encoding) => Ok(encoding.to_scheme()), + None => Err(Status::unimplemented(format!( + "compression scheme `{scheme}` is not supported" ))), }, }) .map_err(|mut status| { status.metadata_mut().insert( GRPC_ACCEPT_ENCODING_HEADER, - AsciiMetadataValue::from_static("gzip,identity"), + AsciiMetadataValue::try_from(GRPC_ACCEPT_ENCODING_VALUE.as_str()) + .expect("advertised encoding value must be valid ASCII"), ); status }) @@ -78,21 +130,64 @@ enum State { }, } -fn new_decompressor() -> GzDecoder> { - // Create the backing buffer for the decompressor and set the compression flag to false (0) and pre-allocate - // the space for the length prefix, which we'll fill out once we've finalized the decompressor. - let buf = vec![0; GRPC_MESSAGE_HEADER_LEN]; +enum Decompressor { + Gzip(Box>>), + Zstd { + compressed: Vec, + output_buf: Vec, + }, +} - GzDecoder::new(buf) +impl Decompressor { + fn new(scheme: &CompressionScheme) -> Result { + // Create the backing buffer for the decompressor and set the compression flag to false (0) + // and pre-allocate the space for the length prefix, which we'll fill out once we've + // finalized the decompressor. + let buf = vec![0; GRPC_MESSAGE_HEADER_LEN]; + match scheme { + CompressionScheme::Gzip => Ok(Decompressor::Gzip(Box::new(GzDecoder::new(buf)))), + CompressionScheme::Zstd => Ok(Decompressor::Zstd { + compressed: Vec::new(), + output_buf: buf, + }), + } + } + + fn write_all(&mut self, data: &[u8]) -> io::Result<()> { + match self { + Decompressor::Gzip(d) => d.write_all(data), + Decompressor::Zstd { compressed, .. } => { + compressed.extend_from_slice(data); + Ok(()) + } + } + } + + fn finish(self) -> io::Result> { + match self { + Decompressor::Gzip(d) => (*d).finish(), + // Decode directly into output_buf to avoid a temporary intermediate Vec that + // decode_all would produce; peak memory is compressed + decompressed rather than + // compressed + 2 Γ— decompressed. + Decompressor::Zstd { + compressed, + mut output_buf, + } => { + zstd::stream::copy_decode(io::Cursor::new(&compressed), &mut output_buf)?; + Ok(output_buf) + } + } + } } async fn drive_body_decompression( mut source: Body, mut destination: Sender, + scheme: Option, ) -> Result { let mut state = State::default(); let mut buf = BytesMut::new(); - let mut decompressor = None; + let mut decompressor: Option = None; let mut bytes_received = 0; // Drain all message chunks from the body first. @@ -131,6 +226,16 @@ async fn drive_body_decompression( // decompressor incrementally because there's no good reason to make both the internal buffer and // the decompressor buffer expand if we don't have to. if is_compressed { + // Per the gRPC compression spec, the compressed flag requires a + // negotiated encoding. Reject frames that set it under identity + // (or with no `grpc-encoding` header) rather than silently + // falling back to gzip and masking client/server mismatches. + if scheme.is_none() { + return Err(Status::internal( + "received compressed frame but no compression scheme was negotiated", + )); + } + // We skip the header in the buffer because it doesn't matter to the decompressor and we // recreate it anyways. buf.advance(GRPC_MESSAGE_HEADER_LEN); @@ -170,8 +275,18 @@ async fn drive_body_decompression( // the decompressor. This is _technically_ synchronous but there's really no way to do it // asynchronously since we already have the data, and that's the only asynchronous part. let to_take = cmp::min(available, *remaining); - let decompressor = decompressor.get_or_insert_with(new_decompressor); - if decompressor.write_all(&buf[..to_take]).is_err() { + let d = match &mut decompressor { + Some(d) => d, + slot @ None => { + let scheme = scheme.as_ref().expect( + "compressed frames without a negotiated scheme are rejected earlier", + ); + slot.insert(Decompressor::new(scheme).map_err(|_| { + Status::internal("failed to initialize decompressor") + })?) + } + }; + if d.write_all(&buf[..to_take]).is_err() { return Err(Status::internal("failed to write to decompressor")); } @@ -243,12 +358,13 @@ async fn drive_request( destination: Sender, inner: F, bytes_received: Registered, + scheme: Option, ) -> Result, E> where F: Future, E>>, E: std::fmt::Display, { - let body_decompression = drive_body_decompression(source, destination); + let body_decompression = drive_body_decompression(source, destination, scheme); pin!(inner); pin!(body_decompression); @@ -256,7 +372,7 @@ where let mut body_eof = false; let mut body_bytes_received = 0; - let result = loop { + let mut result = loop { select! { biased; @@ -291,6 +407,18 @@ where } }; + // Advertise the set of compression schemes this layer can accept to the client. + // Since this layer is the single owner of compression negotiation, individual + // services no longer call `.accept_compressed(..)` and therefore tonic would not + // set this header itself. + if let Ok(res) = result.as_mut() { + res.headers_mut().insert( + GRPC_ACCEPT_ENCODING_HEADER, + HeaderValue::from_str(&GRPC_ACCEPT_ENCODING_VALUE) + .expect("advertised encoding value must be valid ASCII"), + ); + } + result } @@ -325,16 +453,29 @@ where // The request either isn't using compression, or it has indicated compression may be used and we know we // can support decompression based on the indicated compression scheme... so wrap the body to decompress, if // need be, and then track the bytes that flowed through. - // - // TODO: Actually use the scheme given back to us to support other compression schemes. - Ok(_) => { + Ok(scheme) => { let (destination, decompressed_body) = Body::channel(); - let (req_parts, req_body) = req.into_parts(); + let (mut req_parts, req_body) = req.into_parts(); + // Since this layer owns compression negotiation and is about to hand the + // inner service a fully decompressed body (with the per-message compressed + // flag cleared), strip the `grpc-encoding` header so tonic's codegen treats + // the request as uncompressed and does not try to validate the encoding + // against any per-service `accept_compressed(..)` configuration. + if scheme.is_some() { + req_parts.headers.remove(GRPC_ENCODING_HEADER); + } let mapped_req = Request::from_parts(req_parts, decompressed_body); let inner = self.inner.call(mapped_req); - drive_request(req_body, destination, inner, self.bytes_received.clone()).boxed() + drive_request( + req_body, + destination, + inner, + self.bytes_received.clone(), + scheme, + ) + .boxed() } } } @@ -359,7 +500,7 @@ where /// request was valid, and was processed -- we can now report the number of bytes (after decompression) that were /// received _and_ processed correctly. /// -/// The only supported compression scheme is gzip, which is also the only supported compression scheme in `tonic` itself. +/// The supported compression schemes are gzip and zstd. #[derive(Clone, Default)] pub struct DecompressionAndMetricsLayer; diff --git a/src/sources/util/http/headers.rs b/src/sources/util/http/headers.rs index 8a4b73ff212df..ef8376f695723 100644 --- a/src/sources/util/http/headers.rs +++ b/src/sources/util/http/headers.rs @@ -24,14 +24,22 @@ pub fn add_headers( let value = headers.get(header_name).map(HeaderValue::as_bytes); for event in events.iter_mut() { - if let Event::Log(log) = event { - log_namespace.insert_source_metadata( - source_name, - log, - Some(LegacyKey::InsertIfEmpty(path!(header_name))), - path!("headers", header_name), - Value::from(value.map(Bytes::copy_from_slice)), - ); + match event { + Event::Log(log) => { + log_namespace.insert_source_metadata( + source_name, + log, + Some(LegacyKey::InsertIfEmpty(path!(header_name))), + path!("headers", header_name), + Value::from(value.map(Bytes::copy_from_slice)), + ); + } + Event::Metric(_) | Event::Trace(_) => { + event.metadata_mut().value_mut().insert( + path!(source_name, "headers", header_name), + Value::from(value.map(Bytes::copy_from_slice)), + ); + } } } } @@ -45,14 +53,22 @@ pub fn add_headers( let value = headers.get(header_name).map(HeaderValue::as_bytes); for event in events.iter_mut() { - if let Event::Log(log) = event { - log_namespace.insert_source_metadata( - source_name, - log, - Some(LegacyKey::InsertIfEmpty(path!(header_name.as_str()))), - path!("headers", header_name.as_str()), - Value::from(value.map(Bytes::copy_from_slice)), - ); + match event { + Event::Log(log) => { + log_namespace.insert_source_metadata( + source_name, + log, + Some(LegacyKey::InsertIfEmpty(path!(header_name.as_str()))), + path!("headers", header_name.as_str()), + Value::from(value.map(Bytes::copy_from_slice)), + ); + } + Event::Metric(_) | Event::Trace(_) => { + event.metadata_mut().value_mut().insert( + path!(source_name, "headers", header_name.as_str()), + Value::from(value.map(Bytes::copy_from_slice)), + ); + } } } } @@ -64,12 +80,15 @@ pub fn add_headers( #[cfg(test)] mod tests { + use chrono::{DateTime, Utc}; + use std::time::SystemTime; use vector_lib::config::LogNamespace; use vrl::{path, value}; use warp::http::HeaderMap; + use crate::event::{Event, MetricKind, MetricTags, MetricValue}; use crate::{ - event::LogEvent, + event::{LogEvent, Metric, TraceEvent}, sources::{http_server::HttpConfigParamKind, util::add_headers}, }; @@ -109,6 +128,53 @@ mod tests { .get(path!("test", "headers")) .unwrap() ); + + let mut metric = [Event::from( + Metric::new( + "some.random.metric", + MetricKind::Incremental, + MetricValue::Counter { value: 123.4 }, + ) + .with_timestamp(Some(DateTime::::from(SystemTime::now()))) + .with_tags(Some(MetricTags::default())), + )]; + + add_headers( + &mut metric, + &header_names, + &headers, + LogNamespace::default(), + "test", + ); + + let mut trace = [TraceEvent::from(btreemap! { + "span_id" => "abc123", + "trace_id" => "xyz789", + "span_name" => "test-span", + "service" => "my-service", + }) + .into()]; + + add_headers( + &mut trace, + &header_names, + &headers, + LogNamespace::default(), + "test", + ); + + assert_eq!( + metric[0] + .metadata() + .value() + .get(path!("test", "headers")) + .unwrap(), + trace[0] + .metadata() + .value() + .get(path!("test", "headers")) + .unwrap() + ); } #[test] @@ -162,5 +228,81 @@ mod tests { "gzip".into(), "Checking log contains Content-Encoding header" ); + + let mut metric = [Event::from( + Metric::new( + "some.random.metric", + MetricKind::Incremental, + MetricValue::Counter { value: 123.4 }, + ) + .with_timestamp(Some(DateTime::::from(SystemTime::now()))) + .with_tags(Some(MetricTags::default())), + )]; + + add_headers( + &mut metric, + &header_names, + &headers, + LogNamespace::default(), + "test", + ); + + let metric_headers = metric[0] + .metadata() + .value() + .get(path!("test", "headers")) + .unwrap(); + + assert_eq!( + metric_headers.get("content-type").unwrap(), + &value!("application/x-protobuf"), + "Checking metric contains Content-Type header" + ); + assert!( + !metric_headers.contains("user-agent"), + "Checking metric does not contain User-Agent header" + ); + assert_eq!( + metric_headers.get("content-encoding").unwrap(), + &value!("gzip"), + "Checking metric contains Content-Encoding header" + ); + + let mut trace = [TraceEvent::from(btreemap! { + "span_id" => "abc123", + "trace_id" => "xyz789", + "span_name" => "test-span", + "service" => "my-service", + }) + .into()]; + + add_headers( + &mut trace, + &header_names, + &headers, + LogNamespace::default(), + "test", + ); + + let trace_headers = trace[0] + .metadata() + .value() + .get(path!("test", "headers")) + .unwrap(); + + assert_eq!( + trace_headers.get("content-type").unwrap(), + &value!("application/x-protobuf"), + "Checking trace contains Content-Type header" + ); + assert!( + !trace_headers.contains("user-agent"), + "Checking trace does not contain User-Agent header" + ); + assert_eq!( + trace_headers.get("content-encoding").unwrap(), + &value!("gzip"), + "Checking trace contains Content-Encoding header" + ); } } diff --git a/src/sources/util/http/prelude.rs b/src/sources/util/http/prelude.rs index 3414ea00264b0..9645b432ad82b 100644 --- a/src/sources/util/http/prelude.rs +++ b/src/sources/util/http/prelude.rs @@ -11,6 +11,7 @@ use vector_lib::{ config::SourceAcknowledgementsConfig, event::{BatchNotifier, BatchStatus, BatchStatusReceiver, Event}, }; +use vrl::value::ObjectMap; use warp::{ Filter, filters::{ @@ -56,6 +57,18 @@ pub trait HttpSource: Clone + Send + Sync + 'static { path: &str, ) -> Result, ErrorMessage>; + /// Called after `enrich_events` when `custom` auth returned metadata enrichment fields. + /// Sources that do not override this will emit a warning and drop the enrichment. + fn inject_auth_enrichment(&self, _events: &mut [Event], enrichment: ObjectMap) { + if !enrichment.is_empty() { + warn!( + message = "Auth metadata enrichment is not supported by this source and will be dropped. \ + Remove %field writes from the custom auth VRL program or switch to a source that supports enrichment.", + fields = ?enrichment.keys().collect::>(), + ); + } + } + fn decode(&self, encoding_header: Option<&str>, body: Bytes) -> Result { decompress_body(encoding_header, body) } @@ -132,23 +145,27 @@ pub trait HttpSource: Clone + Send + Sync + 'static { let http_path = path.as_str(); let events = auth_matcher .as_ref() - .map_or(Ok(()), |a| { + .map_or(Ok(None), |a| { a.handle_auth( addr.as_ref().map(|a| a.0).as_ref(), &headers, path.as_str(), ) }) - .and_then(|()| self.decode(encoding_header.as_deref(), body)) - .and_then(|body| { + .and_then(|auth_enrichment| { + self.decode(encoding_header.as_deref(), body) + .map(|body| (body, auth_enrichment)) + }) + .and_then(|(body, auth_enrichment)| { emit!(HttpBytesReceived { byte_size: body.len(), http_path, protocol, }); self.build_events(body, &headers, &query_parameters, path.as_str()) + .map(|events| (events, auth_enrichment)) }) - .map(|mut events| { + .map(|(mut events, auth_enrichment)| { emit!(HttpEventsReceived { count: events.len(), byte_size: events.estimated_json_encoded_size_of(), @@ -166,6 +183,10 @@ pub trait HttpSource: Clone + Send + Sync + 'static { .as_ref(), ); + if let Some(enrichment) = auth_enrichment { + self.inject_auth_enrichment(&mut events, enrichment); + } + events }); diff --git a/src/sources/util/net/tcp/mod.rs b/src/sources/util/net/tcp/mod.rs index 079f22d84b34c..c1bf2d8600879 100644 --- a/src/sources/util/net/tcp/mod.rs +++ b/src/sources/util/net/tcp/mod.rs @@ -37,8 +37,9 @@ use crate::{ internal_events::{ ConnectionOpen, OpenGauge, SocketBindError, SocketEventsReceived, SocketMode, SocketReceiveError, StreamClosedError, TcpBytesReceived, TcpSendAckError, - TcpSocketTlsConnectionError, + TcpSocketTlsConnectionError, TcpSourceConnectionClosed, }, + net::is_graceful_tls_shutdown, sources::util::{AfterReadExt, LenientFramedRead}, }; @@ -220,6 +221,12 @@ where tokio::spawn( fut.map(move |()| { drop(open_token); + // Paired with the ConnectionOpen emit above: + // fires exactly once per accepted connection, + // including paths that return early from + // handle_stream (TLS handshake failure, + // shutdown during handshake). + emit!(TcpSourceConnectionClosed); drop(tcp_connection_permit); }) .instrument(span.or_current()), @@ -401,7 +408,19 @@ async fn handle_stream( if let Some(ack_bytes) = acker.build_ack(ack){ let stream = reader.get_mut().get_mut(); if let Err(error) = stream.write_all(&ack_bytes).await { - emit!(TcpSendAckError{ error }); + // Per spec, `*Error` events MUST only be + // emitted on real errors. A peer-initiated + // graceful TLS shutdown during the ack + // write is a lifecycle event, not an error + // β€” log at warn and skip the emit. + if is_graceful_tls_shutdown(&error) { + warn!( + message = "Connection closed by peer before acknowledgement could be sent.", + error = %error, + ); + } else { + emit!(TcpSendAckError { error }); + } break; } } diff --git a/src/sources/vector/mod.rs b/src/sources/vector/mod.rs index 5c29c975e4612..a1e0137258489 100644 --- a/src/sources/vector/mod.rs +++ b/src/sources/vector/mod.rs @@ -177,13 +177,16 @@ impl SourceConfig for VectorConfig { let acknowledgements = cx.do_acknowledgements(self.acknowledgements); let log_namespace = cx.log_namespace(self.log_namespace); - // Create the custom Vector service (existing) + // Create the custom Vector service (existing). + // + // Compression negotiation (gzip, zstd) is handled centrally by + // `DecompressionAndMetricsLayer` in `sources::util::grpc`, so we + // deliberately do not call `.accept_compressed(..)` here. let vector_service = proto::Server::new(Service { pipeline: cx.out, acknowledgements, log_namespace, }) - .accept_compressed(tonic::codec::CompressionEncoding::Gzip) // Tonic added a default of 4MB in 0.9. This replaces the old behavior. .max_decoding_message_size(usize::MAX); @@ -303,9 +306,11 @@ mod tests { test_util, }; - async fn run_test(vector_source_config_str: &str, addr: SocketAddr) { - let config = format!(r#"address = "{addr}""#); - let source: VectorConfig = toml::from_str(&config).unwrap(); + async fn run_test(compression: Option<&str>) { + let (_guard, addr) = test_util::addr::next_addr(); + + let source_config = format!(r#"address = "{addr}""#); + let source: VectorConfig = toml::from_str(&source_config).unwrap(); let (tx, rx) = SourceSender::new_test(); let server = source @@ -318,7 +323,16 @@ mod tests { // Ideally, this would be a fully custom agent to send the data, // but the sink side already does such a test and this is good // to ensure interoperability. - let sink: SinkConfig = toml::from_str(vector_source_config_str).unwrap(); + let sink_config = match compression { + Some(c) => format!( + r#" + address = "{addr}" + compression = "{c}" + "# + ), + None => format!(r#"address = "{addr}""#), + }; + let sink: SinkConfig = toml::from_str(&sink_config).unwrap(); let cx = SinkContext::default(); let (sink, _) = sink.build(cx).await.unwrap(); @@ -338,21 +352,17 @@ mod tests { #[tokio::test] async fn receive_message() { - let (_guard, addr) = test_util::addr::next_addr(); - - let config = format!(r#"address = "{addr}""#); - run_test(&config, addr).await; + run_test(None).await; } #[tokio::test] - async fn receive_compressed_message() { - let (_guard, addr) = test_util::addr::next_addr(); + async fn receive_gzip_compressed_message() { + run_test(Some("gzip")).await; + } - let config = format!( - r#"address = "{addr}" - compression=true"# - ); - run_test(&config, addr).await; + #[tokio::test] + async fn receive_zstd_compressed_message() { + run_test(Some("zstd")).await; } #[tokio::test] diff --git a/src/sources/windows_event_log/mod.rs b/src/sources/windows_event_log/mod.rs index 17d900cc07622..945bc99b54fd9 100644 --- a/src/sources/windows_event_log/mod.rs +++ b/src/sources/windows_event_log/mod.rs @@ -25,11 +25,12 @@ cfg_if::cfg_if! { use std::path::PathBuf; use std::sync::Arc; + use chrono::Utc; use futures::StreamExt; use vector_lib::EstimatedJsonEncodedSizeOf; use vector_lib::finalizer::OrderedFinalizer; use vector_lib::internal_event::{ - ByteSize, BytesReceived, CountByteSize, InternalEventHandle as _, Protocol, + ByteSize, BytesReceived, CountByteSize, InternalEventHandle, Protocol, }; use windows::Win32::Foundation::{DUPLICATE_SAME_ACCESS, DuplicateHandle, HANDLE}; use windows::Win32::System::Threading::GetCurrentProcess; @@ -48,6 +49,7 @@ cfg_if::cfg_if! { error::WindowsEventLogError, parser::EventLogParser, subscription::{EventLogSubscription, WaitResult}, + xml_parser::WindowsEvent, }; } } @@ -99,7 +101,7 @@ impl Finalizer { OrderedFinalizer::::new(Some(shutdown.clone())); // Spawn background task to process acknowledgments and update checkpoints - tokio::spawn(async move { + crate::spawn_in_current_span(async move { while let Some((status, entry)) = ack_stream.next().await { if status == BatchStatus::Delivered { if let Err(e) = checkpointer.set_batch(entry.bookmarks.clone()).await { @@ -157,6 +159,114 @@ impl Finalizer { } } +/// Parse, emit metrics for, send, and finalize a non-empty batch of pulled Windows events. +/// +/// Both the `EventsAvailable` path and the speculative-timeout path share this +/// logic. Returns `true` if the downstream pipeline closed and the caller +/// should break out of the main event loop. +async fn process_event_batch( + events: Vec, + parser: &EventLogParser, + log_namespace: LogNamespace, + acknowledgements: bool, + subscription: &EventLogSubscription, + out: &mut SourceSender, + finalizer: &Finalizer, + events_received: &impl InternalEventHandle, + bytes_received: &impl InternalEventHandle, +) -> bool { + // Rate limiting between batches (async-compatible). + if let Some(limiter) = subscription.rate_limiter() { + limiter.until_ready().await; + } + + let (batch, receiver) = BatchNotifier::maybe_new_with_receiver(acknowledgements); + let mut log_events = Vec::new(); + let mut total_byte_size = 0usize; + let mut channels_in_batch = std::collections::HashSet::new(); + + for event in events { + let channel = event.channel.clone(); + channels_in_batch.insert(channel.clone()); + let event_id = event.event_id; + match parser.parse_event(event) { + Ok(mut log_event) => { + log_namespace.insert_standard_vector_source_metadata( + &mut log_event, + WindowsEventLogConfig::NAME, + Utc::now(), + ); + + let byte_size = log_event.estimated_json_encoded_size_of(); + total_byte_size += byte_size.get(); + if let Some(ref batch) = batch { + log_event = log_event.with_batch_notifier(batch); + } + log_events.push(log_event); + } + Err(e) => { + emit!(WindowsEventLogParseError { + error: e.to_string(), + channel, + event_id: Some(event_id), + }); + } + } + } + + if !log_events.is_empty() { + let count = log_events.len(); + events_received.emit(CountByteSize(count, total_byte_size.into())); + bytes_received.emit(ByteSize(total_byte_size)); + + // BACK PRESSURE: block until the pipeline accepts the batch. + // We don't call EvtNext again until this completes. + if let Err(_error) = out.send_batch(log_events).await { + emit!(StreamClosedError { count }); + return true; // signal: break the main loop + } + + // Register checkpoint entry with the finalizer. + let bookmarks: Vec<(String, String)> = channels_in_batch + .into_iter() + .filter_map(|channel| { + subscription + .get_bookmark_xml(&channel) + .map(|xml| (channel, xml)) + }) + .collect(); + + if !bookmarks.is_empty() { + let entry = FinalizerEntry { bookmarks }; + finalizer.finalize(entry, receiver).await; + } + } + + false // pipeline still open +} + +/// Transfer ownership of `subscription` into a `spawn_blocking` task, run `f` +/// on it, then return both the subscription and the result. +/// +/// All blocking Windows APIs (`WaitForMultipleObjects`, `EvtNext`, `EvtRender`) +/// must run in `spawn_blocking` to avoid stalling the async runtime. The +/// ownership-transfer pattern ensures only one thread holds the subscription +/// at a time, preventing data races without requiring locks. +async fn with_subscription_blocking( + subscription: EventLogSubscription, + f: F, +) -> Result<(EventLogSubscription, R), WindowsEventLogError> +where + F: FnOnce(EventLogSubscription) -> (EventLogSubscription, R) + Send + 'static, + R: Send + 'static, +{ + tokio::task::spawn_blocking(move || f(subscription)) + .await + .map_err(|e| WindowsEventLogError::ConfigError { + message: format!("Blocking subscription task panicked: {e}"), + }) +} + /// Windows Event Log source implementation pub struct WindowsEventLogSource { config: WindowsEventLogConfig, @@ -251,7 +361,7 @@ impl WindowsEventLogSource { } }; let shutdown_watcher = shutdown.clone(); - tokio::spawn(async move { + crate::spawn_in_current_span(async move { shutdown_watcher.await; unsafe { let handle = @@ -281,42 +391,25 @@ impl WindowsEventLogSource { // Ownership transfer ensures no data races between the blocking thread // and async code. The shutdown watcher uses a raw HANDLE value (just an // integer) to signal shutdown without needing access to the subscription. - let (returned_sub, wait_result) = tokio::task::spawn_blocking({ - let sub = subscription; - move || { + let (returned_sub, wait_result) = + with_subscription_blocking(subscription, move |sub| { let result = sub.wait_for_events_blocking(timeout_ms); (sub, result) - } - }) - .await - .map_err(|e| WindowsEventLogError::ConfigError { - message: format!("Wait task panicked: {e}"), - })?; - + }) + .await?; subscription = returned_sub; match wait_result { WaitResult::EventsAvailable => { // Pull events via spawn_blocking (EvtNext/EvtRender are blocking APIs) - let (returned_sub, events_result) = tokio::task::spawn_blocking({ - let mut sub = subscription; - move || { + let (returned_sub, events_result) = + with_subscription_blocking(subscription, move |mut sub| { let result = sub.pull_events(batch_size); (sub, result) - } - }) - .await - .map_err(|e| WindowsEventLogError::ConfigError { - message: format!("Pull task panicked: {e}"), - })?; - + }) + .await?; subscription = returned_sub; - // Rate limiting between batches (async-compatible) - if let Some(limiter) = subscription.rate_limiter() { - limiter.until_ready().await; - } - match events_result { Ok(events) if events.is_empty() => { error_backoff = std::time::Duration::from_millis(100); @@ -328,65 +421,20 @@ impl WindowsEventLogSource { message = "Pulled Windows Event Log events.", event_count = events.len() ); - - let (batch, receiver) = - BatchNotifier::maybe_new_with_receiver(acknowledgements); - - let mut log_events = Vec::new(); - let mut total_byte_size = 0; - let mut channels_in_batch = std::collections::HashSet::new(); - - for event in events { - let channel = event.channel.clone(); - channels_in_batch.insert(channel.clone()); - let event_id = event.event_id; - match parser.parse_event(event) { - Ok(mut log_event) => { - let byte_size = log_event.estimated_json_encoded_size_of(); - total_byte_size += byte_size.get(); - - if let Some(ref batch) = batch { - log_event = log_event.with_batch_notifier(batch); - } - - log_events.push(log_event); - } - Err(e) => { - emit!(WindowsEventLogParseError { - error: e.to_string(), - channel, - event_id: Some(event_id), - }); - } - } - } - - if !log_events.is_empty() { - let count = log_events.len(); - events_received.emit(CountByteSize(count, total_byte_size.into())); - bytes_received.emit(ByteSize(total_byte_size)); - - // BACK PRESSURE: block here until the pipeline accepts - // the batch. We don't call EvtNext again until this completes. - if let Err(_error) = out.send_batch(log_events).await { - emit!(StreamClosedError { count }); - break; - } - - // Register checkpoint entry with finalizer - let bookmarks: Vec<(String, String)> = channels_in_batch - .into_iter() - .filter_map(|channel| { - subscription - .get_bookmark_xml(&channel) - .map(|xml| (channel, xml)) - }) - .collect(); - - if !bookmarks.is_empty() { - let entry = FinalizerEntry { bookmarks }; - finalizer.finalize(entry, receiver).await; - } + if process_event_batch( + events, + &parser, + self.log_namespace, + acknowledgements, + &subscription, + &mut out, + &finalizer, + &events_received, + &bytes_received, + ) + .await + { + break; } } Err(e) => { @@ -415,10 +463,6 @@ impl WindowsEventLogSource { } WaitResult::Timeout => { - // A full wait cycle without errors means the system is healthy; - // reset backoff so the next transient error starts fresh. - error_backoff = std::time::Duration::from_millis(100); - // Periodic checkpoint flush (sync mode only) if !acknowledgements && last_checkpoint.elapsed() >= checkpoint_interval { if let Err(e) = subscription.flush_bookmarks().await { @@ -448,6 +492,75 @@ impl WindowsEventLogSource { ); } } + + // Speculative pull: self-heal against any lost-wakeup scenario, + // regardless of root cause. If the OS signal was lost through any + // mechanism (not just the pre-drain race fixed in #25194), this + // ensures the source recovers within one timeout period. + // Use the speculative pull variant so idle timeout cycles don't + // refresh per-channel record-count gauges via EvtOpenLog / + // EvtGetLogInfo on every configured channel. + let (returned_sub, speculative_result) = + with_subscription_blocking(subscription, move |mut sub| { + let result = sub.pull_events_speculative(batch_size); + (sub, result) + }) + .await?; + subscription = returned_sub; + + match speculative_result { + Ok(events) if events.is_empty() => { + // Healthy cycle: reset backoff so the next transient + // error starts fresh. + error_backoff = std::time::Duration::from_millis(100); + } + Ok(events) => { + // Healthy cycle: reset backoff so the next transient + // error starts fresh. + error_backoff = std::time::Duration::from_millis(100); + warn!( + message = "Speculative timeout pull recovered events; possible lost wakeup detected.", + event_count = events.len(), + ); + if process_event_batch( + events, + &parser, + self.log_namespace, + acknowledgements, + &subscription, + &mut out, + &finalizer, + &events_received, + &bytes_received, + ) + .await + { + break; + } + } + Err(e) => { + emit!(WindowsEventLogQueryError { + channel: "all".to_string(), + query: None, + error: e.to_string(), + }); + if !e.is_recoverable() { + error!( + message = "Non-recoverable speculative pull error, shutting down.", + error = %e + ); + break; + } + // Exponential backoff mirrors the EventsAvailable error path. + warn!( + message = "Recoverable speculative pull error, backing off.", + backoff_ms = error_backoff.as_millis() as u64, + error = %e + ); + tokio::time::sleep(error_backoff).await; + error_backoff = (error_backoff * 2).min(MAX_ERROR_BACKOFF); + } + } } WaitResult::Shutdown => { @@ -561,8 +674,11 @@ impl SourceConfig for WindowsEventLogConfig { ), ])), [LogNamespace::Vector], - ), - LogNamespace::Legacy => vector_lib::schema::Definition::any(), + ) + .with_standard_vector_source_metadata(), + LogNamespace::Legacy => { + vector_lib::schema::Definition::any().with_standard_vector_source_metadata() + } }; vec![SourceOutput::new_maybe_logs( diff --git a/src/sources/windows_event_log/subscription.rs b/src/sources/windows_event_log/subscription.rs index dc92713f691d9..4571561a2d349 100644 --- a/src/sources/windows_event_log/subscription.rs +++ b/src/sources/windows_event_log/subscription.rs @@ -18,9 +18,9 @@ use windows::Win32::System::EventLog::{ EvtSubscribeStartAfterBookmark, EvtSubscribeStartAtOldestRecord, EvtSubscribeStrict, EvtSubscribeToFutureEvents, }; -#[cfg(test)] -use windows::Win32::System::Threading::SetEvent; -use windows::Win32::System::Threading::{CreateEventW, ResetEvent, WaitForMultipleObjects}; +use windows::Win32::System::Threading::{ + CreateEventW, ResetEvent, SetEvent, WaitForMultipleObjects, +}; use windows::core::HSTRING; use super::{ @@ -30,6 +30,19 @@ use super::{ use crate::internal_events::WindowsEventLogBookmarkError; +/// Test-only hook called inside the `pull_events` drain loop after each +/// `EvtNext` invocation. Used by the lost-wakeup regression test +/// (see `test_pull_events_preserves_setevent_during_drain`) to race a +/// `SetEvent` against the drain without relying on thread-timing. +/// No-op and zero-cost in non-test builds. +/// +/// Only one test should install a hook at a time; tests that install a hook +/// must use `#[serial_test::serial]` or equivalent serialization to prevent +/// concurrent tests from triggering each other's hook. +#[cfg(test)] +static DRAIN_STEP_HOOK: std::sync::Mutex>> = + std::sync::Mutex::new(None); + /// Maximum number of entries in the EvtFormatMessage result cache. pub const FORMAT_CACHE_CAPACITY: usize = 10_000; /// Maximum number of cached publisher metadata handles. @@ -80,6 +93,7 @@ struct ChannelSubscription { // SAFETY: Same rationale as EventLogSubscription - Windows kernel handles are thread-safe. unsafe impl Send for ChannelSubscription {} +unsafe impl Sync for ChannelSubscription {} /// Result of waiting for events across all channels. pub enum WaitResult { @@ -130,8 +144,10 @@ pub struct EventLogSubscription { // SAFETY: Windows HANDLE and EVT_HANDLE are kernel objects safe to use across // threads. In windows 0.58, HANDLE wraps *mut c_void which is !Send/!Sync, -// but the underlying kernel handles are thread-safe. +// but the underlying kernel handles are thread-safe. All mutation requires +// &mut self; &self methods are read-only or delegate to Sync types (RateLimiter). unsafe impl Send for EventLogSubscription {} +unsafe impl Sync for EventLogSubscription {} impl EventLogSubscription { /// Create a new pull-model subscription for all configured channels. @@ -415,21 +431,20 @@ impl EventLogSubscription { /// Wait for events to become available on any channel, or for shutdown. /// /// Uses `WaitForMultipleObjects` via `spawn_blocking` to avoid blocking the - /// Tokio runtime. The wait array includes all channel signal events plus the - /// shutdown event. + /// Tokio runtime. The wait array puts shutdown first so a stop request wins + /// over any channel that is already signaled. pub fn wait_for_events_blocking(&self, timeout_ms: u32) -> WaitResult { - // Build wait handle array: [channel0_signal, channel1_signal, ..., shutdown_event] - let mut handles: Vec = self.channels.iter().map(|c| c.signal_event).collect(); + // Build wait handle array: [shutdown_event, channel0_signal, channel1_signal, ...] + let mut handles = Vec::with_capacity(self.channels.len() + 1); handles.push(self.shutdown_event); + handles.extend(self.channels.iter().map(|c| c.signal_event)); let result = unsafe { WaitForMultipleObjects(&handles, false, timeout_ms) }; - let shutdown_index = (self.channels.len()) as u32; - match result { r if r == WAIT_TIMEOUT => WaitResult::Timeout, - r if r.0 < WAIT_OBJECT_0.0 + shutdown_index => WaitResult::EventsAvailable, - r if r.0 == WAIT_OBJECT_0.0 + shutdown_index => WaitResult::Shutdown, + r if r == WAIT_OBJECT_0 => WaitResult::Shutdown, + r if r.0 <= WAIT_OBJECT_0.0 + self.channels.len() as u32 => WaitResult::EventsAvailable, _ => { // WAIT_FAILED or unexpected - treat as timeout to avoid tight loop warn!( @@ -459,6 +474,28 @@ impl EventLogSubscription { pub fn pull_events( &mut self, max_events: usize, + ) -> Result, WindowsEventLogError> { + self.pull_events_inner(max_events, true) + } + + /// Pull events for timeout-based speculative recovery. + /// + /// This keeps the same event-drain behavior as `pull_events`, but avoids + /// refreshing per-channel record-count gauges for channels that were empty. + /// Timeout pulls can run repeatedly while the host is idle, so skipping + /// those metadata queries prevents steady `EvtOpenLog`/`EvtGetLogInfo` + /// churn without changing event recovery behavior. + pub fn pull_events_speculative( + &mut self, + max_events: usize, + ) -> Result, WindowsEventLogError> { + self.pull_events_inner(max_events, false) + } + + fn pull_events_inner( + &mut self, + max_events: usize, + update_records_for_empty_channels: bool, ) -> Result, WindowsEventLogError> { let mut all_events = Vec::with_capacity(max_events.min(1000)); let num_channels = self.channels.len().max(1); @@ -479,9 +516,25 @@ impl EventLogSubscription { let mut bookmark_failed = false; let mut channel_count = 0usize; - // Drain loop: keep calling EvtNext until ERROR_NO_MORE_ITEMS or channel budget. - // Only reset the signal once the channel is fully drained; if we hit the - // budget limit the signal stays set so WaitForMultipleObjects returns immediately. + // Reset the signal BEFORE draining to avoid a lost-wakeup race + // (see vectordotdev/vector#25194). The Windows Event Log service + // signals this manual-reset event via SetEvent each time a new + // matching event is recorded; SetEvent on an already-signaled + // event is a no-op, so if we reset AFTER draining, any signal + // that arrives between our last EvtNext and ResetEvent is lost + // β€” the subscription then hangs until the next event arrives. + // Resetting first means any signal raised during the drain is + // preserved, causing the next WaitForMultipleObjects to return + // immediately. + // + // If we exit the drain loop early (channel budget exhausted or + // bookmark update failed mid-batch), we re-SetEvent at the end + // of this iteration so the next pull_events call revisits this + // channel without waiting for a fresh OS signal. + unsafe { + let _ = ResetEvent(channel_sub.signal_event); + } + 'drain: loop { if channel_count >= channel_limit { break; @@ -501,6 +554,17 @@ impl EventLogSubscription { ) }; + // Test-only hook: lets the lost-wakeup regression test race + // a SetEvent against the drain without thread-timing. No-op + // and zero-cost in non-test builds. + #[cfg(test)] + { + let hook = DRAIN_STEP_HOOK.lock().unwrap().clone(); + if let Some(h) = hook { + h(channel_sub.signal_event); + } + } + if let Err(err) = result { let code = (err.code().0 as u32) & 0xFFFF; if code == ERROR_NO_MORE_ITEMS { @@ -513,6 +577,8 @@ impl EventLogSubscription { channel = %channel_sub.channel ); channel_drained = true; + // Speculative pull on timeout in mod.rs is a safety net if the + // re-subscribed channel does not immediately re-signal. break; } if code == ERROR_EVT_QUERY_RESULT_INVALID_POSITION { @@ -526,7 +592,9 @@ impl EventLogSubscription { message = "Re-subscription succeeded after stale query.", channel = %channel_sub.channel ); - // Retry from fresh subscription β€” the signal will fire again + // Retry from fresh subscription β€” the signal will fire again. + // Speculative pull on timeout in mod.rs is a safety net if + // the new subscription does not immediately re-signal. channel_drained = true; break; } @@ -538,10 +606,23 @@ impl EventLogSubscription { ); channel_sub.subscription_active_gauge.set(0.0); channel_drained = true; + // Speculative pull on timeout in mod.rs is a safety net if + // the failed channel does not re-signal after recovery. break; } } } + // Re-arm the signal before returning. We reset it pre-drain + // but are bailing out without confirming the drain completed, + // so if events were left un-drained the next pull_events must + // still revisit this channel without waiting for a fresh OS + // signal. This mirrors the `else` branch below that handles + // budget-exhaustion and bookmark-failure early breaks, and + // avoids the same lost-wakeup symptom (vectordotdev/vector#25194) + // on transient EvtNext failures. + unsafe { + let _ = SetEvent(channel_sub.signal_event); + } return Err(WindowsEventLogError::PullEventsError { channel: channel_sub.channel.clone(), source: err, @@ -697,15 +778,22 @@ impl EventLogSubscription { } if channel_drained && !bookmark_failed { + // Update channel record count gauge for lag detection. + if update_records_for_empty_channels || channel_count > 0 { + super::render::update_channel_records( + &channel_sub.channel, + &channel_sub.channel_records_gauge, + ); + } + } else { + // Drain exited early (budget exhausted or bookmark_failed + // mid-batch). Re-arm the signal so the next pull_events + // revisits this channel immediately without waiting for a + // fresh OS notification. Pairs with the pre-drain ResetEvent + // above. unsafe { - let _ = ResetEvent(channel_sub.signal_event); + let _ = SetEvent(channel_sub.signal_event); } - - // Update channel record count gauge for lag detection. - super::render::update_channel_records( - &channel_sub.channel, - &channel_sub.channel_records_gauge, - ); } } @@ -816,6 +904,15 @@ impl EventLogSubscription { self.shutdown_event.0 } + /// Test-only accessor for the first channel's signal event handle. Used + /// by the lost-wakeup regression test to scope its drain-loop hook to + /// exactly this subscription, so it does not fire on concurrent + /// `pull_events` calls from other tests in the same process. + #[cfg(test)] + pub(super) fn first_channel_signal_raw(&self) -> isize { + self.channels[0].signal_event.0 as isize + } + /// Returns a reference to the rate limiter, if configured. pub const fn rate_limiter( &self, @@ -1005,6 +1102,7 @@ impl Drop for EventLogSubscription { #[cfg(test)] mod tests { use super::*; + use serial_test::serial; async fn create_test_checkpointer() -> (Arc, tempfile::TempDir) { let temp_dir = tempfile::TempDir::new().unwrap(); @@ -1136,6 +1234,31 @@ mod tests { drop(subscription); } + /// Test that shutdown wins when both shutdown and channel handles are signaled. + #[tokio::test] + async fn test_shutdown_signal_takes_priority_over_channel_signal() { + let mut config = WindowsEventLogConfig::default(); + config.channels = vec!["Application".to_string()]; + config.event_timeout_ms = 500; + + let (checkpointer, _temp_dir) = create_test_checkpointer().await; + + let subscription = EventLogSubscription::new(&config, checkpointer, false) + .await + .expect("Subscription creation should succeed"); + + unsafe { + let handle = HANDLE(subscription.shutdown_event_raw()); + let _ = SetEvent(handle); + } + + let result = subscription.wait_for_events_blocking(0); + assert!( + matches!(result, WaitResult::Shutdown), + "shutdown should take priority over already-signaled channels" + ); + } + /// Test pull_events with read_existing_events=true #[tokio::test] async fn test_pull_events_returns_events() { @@ -1272,4 +1395,209 @@ mod tests { // that the subscription is functional. let _events = subscription.pull_events(100).unwrap_or_default(); } + + /// Proves that `pull_events` works independently of signal state β€” the + /// invariant the speculative timeout pull in mod.rs relies on. + /// + /// Steps: + /// 1. Subscribe to the Application log with `read_existing_events = true`. + /// 2. Manually clear the channel signal via `ResetEvent`, simulating a lost wakeup. + /// 3. Assert `wait_for_events_blocking` times out (signal cleared, no OS wake-up). + /// 4. Assert `pull_events` still returns events β€” `EvtNext` fetches from the queue + /// regardless of signal state, so the speculative pull in mod.rs self-heals. + #[tokio::test] + #[serial] + async fn test_pull_events_works_with_cleared_signal() { + // Seed the Application log with a record so the "events remain + // available despite cleared signal" assertion below does not depend + // on whatever backlog the runner happens to have. Freshly provisioned + // CI images can have an empty Application log, which would otherwise + // make `pull_events` legitimately return empty and produce a spurious + // failure unrelated to the invariant under test. + let seed_output = std::process::Command::new("eventcreate") + .args([ + "/T", + "INFORMATION", + "/ID", + "100", + "/L", + "APPLICATION", + "/SO", + "VectorTestSpeculativePullSeed", + "/D", + "seed event for #25194 speculative-pull regression test", + ]) + .output() + .expect("failed to spawn eventcreate β€” required for deterministic seeding"); + assert!( + seed_output.status.success(), + "eventcreate failed to seed Application log (exit={:?}): stdout={:?} stderr={:?}. \ + This test requires a seeded event to be deterministic; a locked-down runner \ + without the privilege to write to Application cannot run this test reliably.", + seed_output.status.code(), + String::from_utf8_lossy(&seed_output.stdout), + String::from_utf8_lossy(&seed_output.stderr), + ); + // Give the service a moment to persist the record before we subscribe. + tokio::time::sleep(std::time::Duration::from_millis(500)).await; + + let mut config = WindowsEventLogConfig::default(); + config.channels = vec!["Application".to_string()]; + config.read_existing_events = true; + config.event_timeout_ms = 500; + + let (checkpointer, _temp_dir) = create_test_checkpointer().await; + + let mut subscription = EventLogSubscription::new(&config, checkpointer, false) + .await + .expect("Subscription creation should succeed"); + + // Manually clear the signal to simulate a lost wakeup. The seeded + // event above guarantees at least one record is queued in EvtNext + // regardless of the runner's pre-existing log state. + let signal_raw = subscription.first_channel_signal_raw(); + unsafe { + let _ = ResetEvent(HANDLE(signal_raw as *mut std::ffi::c_void)); + } + + // Signal is cleared: an immediate (0ms) poll must report Timeout. + // A 0ms wait reads only the current signal state with no grace + // window, so unrelated Windows system events arriving between the + // `ResetEvent` above and the poll cannot re-signal the handle and + // cause a spurious failure. + let wait_result = subscription.wait_for_events_blocking(0); + + assert!( + matches!(wait_result, WaitResult::Timeout), + "expected Timeout after manual ResetEvent; signal was not cleared" + ); + + // Despite the cleared signal, pull_events must still return events. + // This is the invariant the speculative timeout pull in mod.rs depends on. + let events = subscription.pull_events(100).unwrap_or_default(); + assert!( + !events.is_empty(), + "pull_events must return events independently of signal state; \ + this is the invariant the speculative timeout pull in mod.rs depends on" + ); + } + + /// Regression test for vectordotdev/vector#25194. + /// + /// The Windows Event Log service signals the pull-mode wait handle via + /// `SetEvent` each time a new matching event is recorded. Because the + /// handle is manual-reset, `SetEvent` on an already-signaled handle is + /// a no-op. If `pull_events` resets the signal *after* draining events + /// via `EvtNext`, any signal that fires between the last `EvtNext` and + /// the `ResetEvent` call is silently lost β€” the subscription then + /// permanently hangs until a subsequent event arrives. + /// + /// The fix is to reset the signal *before* the drain loop, so signals + /// raised during the drain are preserved and the next wait returns + /// immediately. + /// + /// This test pins that invariant by driving the real `pull_events` + /// against a real `EvtSubscribe` handle. It installs a + /// `DRAIN_STEP_HOOK` that runs inside the drain loop after each + /// `EvtNext` and fires `SetEvent` on the subscription's signal + /// handle β€” simulating the OS signaling a new event arrival during + /// the drain window. After `pull_events` returns, the signal must + /// still be set β€” observed via a 0ms `wait_for_events_blocking` + /// so the check measures only the reset/preserve behavior of + /// `pull_events` and is not contaminated by unrelated Windows + /// system events arriving during a nonzero wait. Under the old + /// post-drain `ResetEvent` order, the hook's `SetEvent` would be + /// clobbered by the reset and the immediate poll would return + /// `Timeout` β€” which is exactly what #25194 reports. + #[tokio::test] + #[serial] + async fn test_pull_events_preserves_setevent_during_drain() { + use std::sync::Arc as StdArc; + + let mut config = WindowsEventLogConfig::default(); + config.channels = vec!["Application".to_string()]; + config.read_existing_events = true; + config.event_timeout_ms = 1000; + + let (checkpointer, _temp_dir) = create_test_checkpointer().await; + + let mut subscription = EventLogSubscription::new(&config, checkpointer, false) + .await + .expect("Subscription creation should succeed"); + + // Capture THIS subscription's signal handle so the hook can scope + // itself to this test. DRAIN_STEP_HOOK is a process-global, and + // cargo runs tests in parallel by default; without handle-keying, + // a concurrent test's pull_events could trigger our one-shot + // hook first, flip `fired`, and SetEvent on the wrong handle. + let target_signal_raw = subscription.first_channel_signal_raw(); + + // Install the drain-loop hook: every EvtNext call inside + // pull_events fires SetEvent on the subscription's signal + // handle. This simulates the OS signaling a fresh event + // mid-drain, which is exactly the race window #25194 exposes. + // The hook only needs to fire once to prove the invariant; we + // use an AtomicBool to keep it deterministic. The hook is keyed + // to `target_signal_raw` so concurrent pull_events calls from + // other tests no-op here. + let fired = StdArc::new(std::sync::atomic::AtomicBool::new(false)); + { + let fired = StdArc::clone(&fired); + let hook: StdArc = StdArc::new(move |signal: HANDLE| { + if signal.0 as isize != target_signal_raw { + return; + } + if !fired.swap(true, std::sync::atomic::Ordering::SeqCst) { + unsafe { + let _ = SetEvent(signal); + } + } + }); + *DRAIN_STEP_HOOK.lock().unwrap() = Some(hook); + } + + // Drop-guard: clear the hook even if the test panics, so it + // doesn't contaminate other tests in the same process. + struct HookGuard; + impl Drop for HookGuard { + fn drop(&mut self) { + *DRAIN_STEP_HOOK.lock().unwrap() = None; + } + } + let _guard = HookGuard; + + // Drive pull_events with a very large budget so the drain + // exits via ERROR_NO_MORE_ITEMS (channel_drained = true), + // which is the path that ran the post-drain ResetEvent in the + // old buggy code. Exiting via budget exhaustion would skip + // that reset and cause this test to false-pass against the + // pre-fix code. + let _ = subscription.pull_events(usize::MAX).unwrap_or_default(); + + assert!( + fired.load(std::sync::atomic::Ordering::SeqCst), + "drain-loop hook never ran β€” pull_events must call EvtNext \ + at least once even on an empty channel" + ); + + // Observe the signal state IMMEDIATELY with a 0ms wait. We want + // to know whether pull_events's reset clobbered the hook's + // SetEvent β€” NOT whether new real events arrive during some + // wait window. A nonzero timeout against the live Application + // channel lets arbitrary Windows system events re-signal us + // and false-pass against the pre-fix code. 0ms = WaitForMultiple- + // Objects returns the current state with no grace period, so + // only the reset/preserve behavior of pull_events is measured. + let result = subscription.wait_for_events_blocking(0); + + match result { + WaitResult::EventsAvailable => {} + WaitResult::Timeout => panic!( + "signal set during the drain window was lost β€” this is the \ + lost-wakeup race from vectordotdev/vector#25194. \ + pull_events must call ResetEvent BEFORE draining, not after." + ), + WaitResult::Shutdown => panic!("unexpected shutdown"), + } + } } diff --git a/src/tap/cmd.rs b/src/tap/cmd.rs index 46fc21f6b5684..fe2b69bbeb8b5 100644 --- a/src/tap/cmd.rs +++ b/src/tap/cmd.rs @@ -1,24 +1,24 @@ use std::time::Duration; use vector_lib::{ - api_client::Client, + api_client::{Client, RECONNECT_DELAY_MS}, tap::{EventFormatter, OutputChannel, TapRunner}, }; use crate::signal::{SignalRx, SignalTo}; -/// Delay (in milliseconds) before attempting to reconnect to the Vector API -const RECONNECT_DELAY: u64 = 5000; - /// CLI command func for issuing 'tap' queries, and communicating with a local/remote /// Vector API server via HTTP/WebSockets. +#[allow(clippy::print_stderr)] pub(crate) async fn cmd(opts: &super::Opts, signal_rx: SignalRx) -> exitcode::ExitCode { let url = opts.url(); - // Return early with instructions for enabling the API if the endpoint isn't reachable - // via a healthcheck. - let client = Client::new(url.clone()); - #[allow(clippy::print_stderr)] - if client.healthcheck().await.is_err() { + let Ok(uri) = url.as_str().parse() else { + eprintln!("Invalid API URL: {url}"); + return exitcode::USAGE; + }; + let mut client = Client::new(uri); + + if client.connect().await.is_err() || client.health().await.is_err() { eprintln!( indoc::indoc! {" Vector API server isn't reachable ({}). @@ -34,47 +34,66 @@ pub(crate) async fn cmd(opts: &super::Opts, signal_rx: SignalRx) -> exitcode::Ex return exitcode::UNAVAILABLE; } - tap(opts, signal_rx).await + tap_internal(opts, signal_rx, Some(client)).await } /// Observe event flow from specified components -pub async fn tap(opts: &super::Opts, mut signal_rx: SignalRx) -> exitcode::ExitCode { - let subscription_url = opts.web_socket_url(); +pub async fn tap(opts: &super::Opts, signal_rx: SignalRx) -> exitcode::ExitCode { + tap_internal(opts, signal_rx, None).await +} + +async fn tap_internal( + opts: &super::Opts, + mut signal_rx: SignalRx, + mut client_opt: Option, +) -> exitcode::ExitCode { + let url = opts.url(); let output_channel = OutputChannel::Stdout(EventFormatter::new(opts.meta, opts.format)); let tap_runner = TapRunner::new( - &subscription_url, + &url, opts.inputs_of.clone(), opts.outputs_patterns().clone(), &output_channel, - opts.format, ); loop { tokio::select! { biased; Ok(SignalTo::Shutdown(_) | SignalTo::Quit) = signal_rx.recv() => break, - exec_result = tap_runner.run_tap( - opts.interval as i64, - opts.limit as i64, - opts.duration_ms, - opts.quiet, - ) => { + exec_result = async { + if let Some(client) = client_opt.take() { + tap_runner.run_tap_with_client( + client, + opts.interval as i64, + opts.limit as i64, + opts.duration_ms, + opts.quiet, + ).await + } else { + tap_runner.run_tap( + opts.interval as i64, + opts.limit as i64, + opts.duration_ms, + opts.quiet, + ).await + } + } => { match exec_result { Ok(_) => { break; } Err(tap_executor_error) => { - if !opts.no_reconnect { - #[allow(clippy::print_stderr)] - { - eprintln!( - "[tap] Connection failed with error {:?}. Reconnecting in {:?} seconds.", - tap_executor_error, - RECONNECT_DELAY / 1000); - } - tokio::time::sleep(Duration::from_millis(RECONNECT_DELAY)).await; - } - else { + #[allow(clippy::print_stderr)] + if tap_executor_error.is_fatal() { + eprintln!("[tap] Error: {tap_executor_error:?}"); + break; + } else if !opts.no_reconnect { + eprintln!( + "[tap] Connection failed with error {:?}. Reconnecting in {:?} seconds.", + tap_executor_error, + RECONNECT_DELAY_MS / 1000); + tokio::time::sleep(Duration::from_millis(RECONNECT_DELAY_MS)).await; + } else { break; } } diff --git a/src/tap/mod.rs b/src/tap/mod.rs index 9ba77f3ce1a65..2e5fa1bbd4d3d 100644 --- a/src/tap/mod.rs +++ b/src/tap/mod.rs @@ -5,24 +5,24 @@ use clap::Parser; pub(crate) use cmd::cmd; pub use cmd::tap; use url::Url; -use vector_lib::api_client::gql::TapEncodingFormat; +use vector_lib::tap::TapEncodingFormat; -use crate::config::api::default_graphql_url; +use crate::config::api::default_grpc_url; /// Tap options #[derive(Parser, Debug, Clone)] #[command(rename_all = "kebab-case")] pub struct Opts { - /// Interval to sample events at, in milliseconds - #[arg(default_value = "500", short = 'i', long)] + /// Interval to sample events at, in milliseconds (must be >= 100) + #[arg(default_value = "500", short = 'i', long, value_parser = clap::value_parser!(u32).range(100..))] interval: u32, - /// GraphQL API server endpoint + /// gRPC API server endpoint (e.g. http://localhost:8686) #[arg(short, long)] url: Option, - /// Maximum number of events to sample each interval - #[arg(default_value = "100", short = 'l', long)] + /// Maximum number of events to sample each interval (must be >= 1) + #[arg(default_value = "100", short = 'l', long, value_parser = clap::value_parser!(u32).range(1..))] limit: u32, /// Encoding format for events printed to screen @@ -77,21 +77,9 @@ impl Opts { } } - /// Use the provided URL as the Vector GraphQL API server, or default to the local port + /// Use the provided URL as the Vector gRPC API server, or default to the local port /// provided by the API config. pub fn url(&self) -> Url { - self.url.clone().unwrap_or_else(default_graphql_url) - } - - /// URL with scheme set to WebSockets - pub fn web_socket_url(&self) -> Url { - let mut url = self.url(); - url.set_scheme(match url.scheme() { - "https" => "wss", - _ => "ws", - }) - .expect("Couldn't build WebSocket URL. Please report."); - - url + self.url.clone().unwrap_or_else(default_grpc_url) } } diff --git a/src/test_util/addr.rs b/src/test_util/addr.rs index 4654858cd99fc..d7d5be7afbfbb 100644 --- a/src/test_util/addr.rs +++ b/src/test_util/addr.rs @@ -8,6 +8,8 @@ //! //! This ensures no race window between port allocation and registration. +#[cfg(windows)] +use std::net::UdpSocket; use std::{ collections::HashSet, net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, TcpListener as StdTcpListener}, @@ -79,6 +81,14 @@ pub fn next_addr_for_ip(ip: IpAddr) -> (PortGuard, SocketAddr) { continue; } + // On Windows, certain ports are in OS-excluded ranges (e.g. set by Hyper-V/WSL). + // TCP bind(0) may return such a port, but UDP bind to the same port will fail with + // WSAEACCES (10013). Probe with a UDP socket and retry if it is excluded. + #[cfg(windows)] + if UdpSocket::bind(addr).is_err() { + continue; + } + // Port is unique, mark it as reserved BEFORE dropping the listener // This ensures no race window between dropping listener and registering the port reserved.insert(port); diff --git a/src/test_util/mod.rs b/src/test_util/mod.rs index 58b0ada3e8c6f..c2700ceeae2aa 100644 --- a/src/test_util/mod.rs +++ b/src/test_util/mod.rs @@ -554,6 +554,19 @@ where wait_for(|| ready(unblock(value.load(Ordering::SeqCst)))).await } +pub async fn wait_for_atomic_usize_timeout_ms(value: T, unblock: F, timeout_ms: u64) +where + T: AsRef, + F: Fn(usize) -> bool, +{ + let value = value.as_ref(); + wait_for_duration( + || ready(unblock(value.load(Ordering::SeqCst))), + Duration::from_millis(timeout_ms), + ) + .await +} + // Retries a func every `retry` duration until given an Ok(T); panics after `until` elapses pub async fn retry_until<'a, F, Fut, T, E>(mut f: F, retry: Duration, until: Duration) -> T where diff --git a/src/top/cmd.rs b/src/top/cmd.rs index 6410063781549..e411712261920 100644 --- a/src/top/cmd.rs +++ b/src/top/cmd.rs @@ -3,9 +3,10 @@ use std::time::Duration; use chrono::Local; use futures_util::future::join_all; +use http::Uri; use regex::Regex; use tokio::sync::{mpsc, oneshot}; -use vector_lib::api_client::{Client, connect_subscription_client}; +use vector_lib::api_client::{Client, RECONNECT_DELAY_MS}; use vector_lib::top::{ dashboard::{init_dashboard, is_tty}, @@ -13,26 +14,26 @@ use vector_lib::top::{ state::{self, ConnectionStatus, EventType, State}, }; -/// Delay (in milliseconds) before attempting to reconnect to the Vector API -const RECONNECT_DELAY: u64 = 5000; - /// CLI command func for displaying Vector components, and communicating with a local/remote -/// Vector API server via HTTP/WebSockets +/// Vector API server via gRPC +#[allow(clippy::print_stderr)] pub async fn cmd(opts: &super::Opts) -> exitcode::ExitCode { // Exit early if the terminal is not a teletype if !is_tty() { - #[allow(clippy::print_stderr)] - { - eprintln!("Terminal must be a teletype (TTY) to display a Vector dashboard."); - } + eprintln!("Terminal must be a teletype (TTY) to display a Vector dashboard."); return exitcode::IOERR; } let url = opts.url(); + // Create a new API client for connecting to the local/remote Vector instance. - let client = Client::new(url.clone()); - #[allow(clippy::print_stderr)] - if client.healthcheck().await.is_err() { + let Ok(uri) = url.as_str().parse::() else { + eprintln!("Invalid API URL: {url}"); + return exitcode::USAGE; + }; + let mut client = Client::new(uri.clone()); + + if client.connect().await.is_err() || client.health().await.is_err() { eprintln!( indoc::indoc! {" Vector API server isn't reachable ({}). @@ -48,11 +49,11 @@ pub async fn cmd(opts: &super::Opts) -> exitcode::ExitCode { return exitcode::UNAVAILABLE; } - top(opts, client, "Vector").await + top(opts, uri, "Vector").await } /// General monitoring -pub async fn top(opts: &super::Opts, client: Client, dashboard_title: &str) -> exitcode::ExitCode { +pub async fn top(opts: &super::Opts, uri: Uri, dashboard_title: &str) -> exitcode::ExitCode { // Channel for updating state via event messages let (tx, rx) = tokio::sync::mpsc::channel(20); let mut starting_state = State::new(BTreeMap::new()); @@ -68,7 +69,7 @@ pub async fn top(opts: &super::Opts, client: Client, dashboard_title: &str) -> e // Channel for shutdown signal let (shutdown_tx, shutdown_rx) = oneshot::channel::<()>(); - let connection = tokio::spawn(subscription(opts.clone(), client, tx.clone(), shutdown_tx)); + let connection = tokio::spawn(subscription(opts.clone(), uri, tx.clone(), shutdown_tx)); // Initialize the dashboard match init_dashboard( @@ -97,60 +98,67 @@ pub async fn top(opts: &super::Opts, client: Client, dashboard_title: &str) -> e } } -// This task handles reconnecting the subscription client and all -// subscriptions in the case of a web socket disconnect +// This task handles reconnecting the gRPC client and all +// subscriptions in the case of a connection failure async fn subscription( opts: super::Opts, - client: Client, + uri: Uri, tx: mpsc::Sender, shutdown_tx: oneshot::Sender<()>, ) { - let ws_url = opts.web_socket_url(); - loop { // Initialize state. On future reconnects, we re-initialize state in // order to accurately capture added, removed, and edited // components. - let state = match metrics::init_components(&client, &opts.components).await { + let state = match metrics::init_components(uri.clone(), &opts.components).await { Ok(state) => state, Err(_) => { - tokio::time::sleep(Duration::from_millis(RECONNECT_DELAY)).await; + tokio::time::sleep(Duration::from_millis(RECONNECT_DELAY_MS)).await; continue; } }; + let initial_components = state + .components + .keys() + .map(|k| k.id().to_string()) + .collect(); _ = tx.send(EventType::InitializeState(state)).await; - let subscription_client = match connect_subscription_client(ws_url.clone()).await { - Ok(c) => c, + // Subscribe to updated metrics via gRPC streaming + let handles = match metrics::subscribe( + uri.clone(), + tx.clone(), + opts.interval as i64, + opts.components.clone(), + initial_components, + ) + .await + { + Ok(handles) => handles, Err(_) => { - tokio::time::sleep(Duration::from_millis(RECONNECT_DELAY)).await; + tokio::time::sleep(Duration::from_millis(RECONNECT_DELAY_MS)).await; continue; } }; - // Subscribe to updated metrics - let finished = metrics::subscribe( - subscription_client, - tx.clone(), - opts.interval as i64, - opts.components.clone(), - ); - _ = tx .send(EventType::ConnectionUpdated(ConnectionStatus::Connected( Local::now(), ))) .await; - // Tasks spawned in metrics::subscribe finish when the subscription - // streams have completed. Currently, subscription streams only - // complete when the underlying web socket connection to the GraphQL - // server drops. - _ = join_all(finished).await; + + // Wait for metric stream tasks to finish. poll_components is intentionally + // excluded: it runs indefinitely while get_components succeeds, so joining + // it here would prevent reconnection when metric streams fail first. + _ = join_all(handles.metric_handles).await; + handles.poll_handle.abort(); + _ = tx .send(EventType::ConnectionUpdated( - ConnectionStatus::Disconnected(RECONNECT_DELAY), + ConnectionStatus::Disconnected(RECONNECT_DELAY_MS), )) .await; + if opts.no_reconnect { _ = shutdown_tx.send(()); break; diff --git a/src/top/mod.rs b/src/top/mod.rs index 5fc101f7b13c6..c185e3193a64d 100644 --- a/src/top/mod.rs +++ b/src/top/mod.rs @@ -8,17 +8,17 @@ pub use cmd::{cmd, top}; use url::Url; use vector_lib::top::state::{FilterColumn, SortColumn}; -use crate::config::api::default_graphql_url; +use crate::config::api::default_grpc_url; /// Top options #[derive(Parser, Debug, Clone)] #[command(rename_all = "kebab-case")] pub struct Opts { - /// Interval to sample metrics at, in milliseconds - #[arg(default_value = "1000", short = 'i', long)] + /// Interval to sample metrics at, in milliseconds (must be >= 100) + #[arg(default_value = "1000", short = 'i', long, value_parser = clap::value_parser!(u32).range(100..))] interval: u32, - /// GraphQL API server endpoint + /// gRPC API server endpoint (e.g. http://localhost:8686) #[arg(short, long)] url: Option, @@ -56,21 +56,9 @@ pub struct Opts { } impl Opts { - /// Use the provided URL as the Vector GraphQL API server, or default to the local port + /// Use the provided URL as the Vector gRPC API server, or default to the local port /// provided by the API config. pub fn url(&self) -> Url { - self.url.clone().unwrap_or_else(default_graphql_url) - } - - /// URL with scheme set to WebSockets - pub fn web_socket_url(&self) -> Url { - let mut url = self.url(); - url.set_scheme(match url.scheme() { - "https" => "wss", - _ => "ws", - }) - .expect("Couldn't build WebSocket URL. Please report."); - - url + self.url.clone().unwrap_or_else(default_grpc_url) } } diff --git a/src/topology/builder.rs b/src/topology/builder.rs index 4fda56f9b35b3..9caa9e2f11378 100644 --- a/src/topology/builder.rs +++ b/src/topology/builder.rs @@ -8,21 +8,23 @@ use std::{ use futures::{FutureExt, StreamExt, TryStreamExt, stream::FuturesOrdered}; use futures_util::stream::FuturesUnordered; -use metrics::gauge; use stream_cancel::{StreamExt as StreamCancelExt, Trigger, Tripwire}; use tokio::{ select, sync::{mpsc::UnboundedSender, oneshot}, time::timeout, }; -use tracing::Instrument; +use tracing::{Instrument, Span}; use vector_lib::{ EstimatedJsonEncodedSizeOf, buffers::{ BufferType, WhenFull, topology::{ builder::TopologyBuilder, - channel::{BufferReceiver, BufferSender, ChannelMetricMetadata, LimitedReceiver}, + channel::{ + BufferChannelKind, BufferReceiver, BufferSender, ChannelMetricMetadata, + LimitedReceiver, + }, }, }, internal_event::{self, CountByteSize, EventsSent, InternalEventHandle as _, Registered}, @@ -31,6 +33,7 @@ use vector_lib::{ source_sender::{CHUNK_SIZE, SourceSenderItem}, transform::update_runtime_schema_definition, }; +use vector_lib::{gauge, internal_event::GaugeName}; use vector_vrl_metrics::MetricsStorage; use super::{ @@ -43,7 +46,8 @@ use crate::{ SourceSender, config::{ ComponentKey, Config, DataType, EnrichmentTableConfig, Input, Inputs, OutputId, - ProxyConfig, SinkContext, SourceContext, TransformContext, TransformOuter, TransformOutput, + ProxyConfig, SinkContext, SinkOuter, SourceContext, SourceOuter, TransformContext, + TransformOuter, TransformOutput, }, event::{EventArray, EventContainer}, extra_context::ExtraContext, @@ -52,7 +56,10 @@ use crate::{ spawn_named, topology::task::TaskError, transforms::{SyncTransform, TaskTransform, Transform, TransformOutputs, TransformOutputsBuf}, - utilization::{UtilizationComponentSender, UtilizationEmitter, UtilizationRegistry, wrap}, + utilization::{ + OutputUtilization, Utilization, UtilizationComponentSender, UtilizationEmitter, + UtilizationRegistry, + }, }; static ENRICHMENT_TABLES: LazyLock = @@ -64,7 +71,6 @@ pub(crate) static SOURCE_SENDER_BUFFER_SIZE: LazyLock = const READY_ARRAY_CAPACITY: NonZeroUsize = NonZeroUsize::new(CHUNK_SIZE * 4).unwrap(); pub(crate) const TOPOLOGY_BUFFER_SIZE: NonZeroUsize = NonZeroUsize::new(100).unwrap(); -const TRANSFORM_CHANNEL_METRIC_PREFIX: &str = "transform_buffer"; static TRANSFORM_CONCURRENCY_LIMIT: LazyLock = LazyLock::new(|| { crate::app::worker_threads() @@ -247,173 +253,188 @@ impl<'a> Builder<'a> { { debug!(component_id = %key, "Building new source."); - let typetag = source.inner.get_component_name(); - let source_outputs = source.inner.outputs(self.config.schema.log_namespace()); - let span = error_span!( "source", component_kind = "source", component_id = %key.id(), component_type = %source.inner.get_component_name(), ); - let _entered_span = span.enter(); - let task_name = format!( - ">> {} ({}, pump) >>", - source.inner.get_component_name(), - key.id() + if let Ok(server) = self + .build_instrumented_source(key, source, enrichment_tables) + .instrument(span) + .await + { + source_tasks.insert(key.clone(), server); + } + } + + source_tasks + } + + async fn build_instrumented_source( + &mut self, + key: &ComponentKey, + source: &SourceOuter, + enrichment_tables: &vector_lib::enrichment::TableRegistry, + ) -> Result { + let typetag = source.inner.get_component_name(); + let source_outputs = source.inner.outputs(self.config.schema.log_namespace()); + + let task_name = format!( + ">> {} ({}, pump) >>", + source.inner.get_component_name(), + key.id() + ); + + let mut builder = SourceSender::builder() + .with_buffer(*SOURCE_SENDER_BUFFER_SIZE) + .with_timeout(source.inner.send_timeout()) + .with_ewma_half_life_seconds( + self.config.global.buffer_utilization_ewma_half_life_seconds, + ); + let mut pumps = Vec::new(); + let mut controls = HashMap::new(); + let mut schema_definitions = HashMap::with_capacity(source_outputs.len()); + + for output in source_outputs.into_iter() { + let rx = builder.add_source_output(output.clone(), key.clone()); + + let (fanout, control) = Fanout::new(key.clone()); + let source_type = source.inner.get_component_name(); + let source = Arc::new(key.clone()); + + let pump = run_source_output_pump(rx, fanout, source, source_type); + + pumps.push(pump.instrument(Span::current())); + controls.insert( + OutputId { + component: key.clone(), + port: output.port.clone(), + }, + control, ); - let mut builder = SourceSender::builder() - .with_buffer(*SOURCE_SENDER_BUFFER_SIZE) - .with_timeout(source.inner.send_timeout()) - .with_ewma_half_life_seconds( - self.config.global.buffer_utilization_ewma_half_life_seconds, - ); - let mut pumps = Vec::new(); - let mut controls = HashMap::new(); - let mut schema_definitions = HashMap::with_capacity(source_outputs.len()); - - for output in source_outputs.into_iter() { - let rx = builder.add_source_output(output.clone(), key.clone()); - - let (fanout, control) = Fanout::new(); - let source_type = source.inner.get_component_name(); - let source = Arc::new(key.clone()); - - let pump = run_source_output_pump(rx, fanout, source, source_type); - - pumps.push(pump.instrument(span.clone())); - controls.insert( - OutputId { - component: key.clone(), - port: output.port.clone(), - }, - control, - ); - - let port = output.port.clone(); - if let Some(definition) = output.schema_definition(self.config.schema.enabled) { - schema_definitions.insert(port, definition); - } + let port = output.port.clone(); + if let Some(definition) = output.schema_definition(self.config.schema.enabled) { + schema_definitions.insert(port, definition); } + } - let (pump_error_tx, mut pump_error_rx) = oneshot::channel(); - let pump = async move { - debug!("Source pump supervisor starting."); + let (pump_error_tx, mut pump_error_rx) = oneshot::channel(); + let pump = async move { + debug!("Source pump supervisor starting."); - // Spawn all of the per-output pumps and then await their completion. - // - // If any of the pumps complete with an error, or panic/are cancelled, we return - // immediately. - let mut handles = FuturesUnordered::new(); - for pump in pumps { - handles.push(spawn_named(pump, task_name.as_ref())); - } + // Spawn all of the per-output pumps and then await their completion. + // + // If any of the pumps complete with an error, or panic/are cancelled, we return + // immediately. + let mut handles = FuturesUnordered::new(); + for pump in pumps { + handles.push(spawn_named(pump, task_name.as_ref())); + } - let mut had_pump_error = false; - while let Some(output) = handles.try_next().await? { - if let Err(e) = output { - // Immediately send the error to the source's wrapper future, but ignore any - // errors during the send, since nested errors wouldn't make any sense here. - _ = pump_error_tx.send(e); - had_pump_error = true; - break; - } + let mut had_pump_error = false; + while let Some(output) = handles.try_next().await? { + if let Err(e) = output { + // Immediately send the error to the source's wrapper future, but ignore any + // errors during the send, since nested errors wouldn't make any sense here. + _ = pump_error_tx.send(e); + had_pump_error = true; + break; } + } - if had_pump_error { - debug!("Source pump supervisor task finished with an error."); - } else { - debug!("Source pump supervisor task finished normally."); - } - Ok(TaskOutput::Source) - }; - let pump = Task::new(key.clone(), typetag, pump); + if had_pump_error { + debug!("Source pump supervisor task finished with an error."); + } else { + debug!("Source pump supervisor task finished normally."); + } + Ok(TaskOutput::Source) + }; + let pump = Task::new(key.clone(), typetag, pump); + + let (shutdown_signal, force_shutdown_tripwire) = self + .shutdown_coordinator + .register_source(key, INTERNAL_SOURCES.contains(&typetag)); + + let context = SourceContext { + key: key.clone(), + globals: self.config.global.clone(), + enrichment_tables: enrichment_tables.clone(), + metrics_storage: METRICS_STORAGE.clone(), + shutdown: shutdown_signal, + out: builder.build(), + proxy: ProxyConfig::merge_with_env(&self.config.global.proxy, &source.proxy), + acknowledgements: source.sink_acknowledgements, + schema_definitions, + schema: self.config.schema, + extra_context: self.extra_context.clone(), + }; + let server = match source.inner.build(context).await { + Err(error) => { + self.errors.push(format!("Source \"{key}\": {error}")); + return Err(()); + } + Ok(server) => server, + }; - let (shutdown_signal, force_shutdown_tripwire) = self - .shutdown_coordinator - .register_source(key, INTERNAL_SOURCES.contains(&typetag)); + // Build a wrapper future that drives the actual source future, but returns early if we've + // been signalled to forcefully shutdown, or if the source pump encounters an error. + // + // The forceful shutdown will only resolve if the source itself doesn't shutdown gracefully + // within the allotted time window. This can occur normally for certain sources, like stdin, + // where the I/O is blocking (in a separate thread) and won't wake up to check if it's time + // to shutdown unless some input is given. + let server = async move { + debug!("Source starting."); + + let mut result = select! { + biased; - let context = SourceContext { - key: key.clone(), - globals: self.config.global.clone(), - enrichment_tables: enrichment_tables.clone(), - metrics_storage: METRICS_STORAGE.clone(), - shutdown: shutdown_signal, - out: builder.build(), - proxy: ProxyConfig::merge_with_env(&self.config.global.proxy, &source.proxy), - acknowledgements: source.sink_acknowledgements, - schema_definitions, - schema: self.config.schema, - extra_context: self.extra_context.clone(), - }; - let server = match source.inner.build(context).await { - Err(error) => { - self.errors.push(format!("Source \"{key}\": {error}")); - continue; - } - Ok(server) => server, + // We've been told that we must forcefully shut down. + _ = force_shutdown_tripwire => Ok(()), + + // The source pump encountered an error, which we're now bubbling up here to stop + // the source as well, since the source running makes no sense without the pump. + // + // We only match receiving a message, not the error of the sender being dropped, + // just to keep things simpler. + Ok(e) = &mut pump_error_rx => Err(e), + + // The source finished normally. + result = server => result.map_err(|_| TaskError::Opaque), }; - // Build a wrapper future that drives the actual source future, but returns early if we've - // been signalled to forcefully shutdown, or if the source pump encounters an error. + // Even though we already tried to receive any pump task error above, we may have exited + // on the source itself returning an error due to task scheduling, where the pump task + // encountered an error, sent it over the oneshot, but we were polling the source + // already and hit an error trying to send to the now-shutdown pump task. // - // The forceful shutdown will only resolve if the source itself doesn't shutdown gracefully - // within the allotted time window. This can occur normally for certain sources, like stdin, - // where the I/O is blocking (in a separate thread) and won't wake up to check if it's time - // to shutdown unless some input is given. - let server = async move { - debug!("Source starting."); - - let mut result = select! { - biased; - - // We've been told that we must forcefully shut down. - _ = force_shutdown_tripwire => Ok(()), - - // The source pump encountered an error, which we're now bubbling up here to stop - // the source as well, since the source running makes no sense without the pump. - // - // We only match receiving a message, not the error of the sender being dropped, - // just to keep things simpler. - Ok(e) = &mut pump_error_rx => Err(e), - - // The source finished normally. - result = server => result.map_err(|_| TaskError::Opaque), - }; + // Since the error from the source is opaque at the moment (i.e. `()`), we try a final + // time to see if the pump task encountered an error, using _that_ instead if so, to + // propagate the true error that caused the source to have to stop. + if let Ok(e) = pump_error_rx.try_recv() { + result = Err(e); + } - // Even though we already tried to receive any pump task error above, we may have exited - // on the source itself returning an error due to task scheduling, where the pump task - // encountered an error, sent it over the oneshot, but we were polling the source - // already and hit an error trying to send to the now-shutdown pump task. - // - // Since the error from the source is opaque at the moment (i.e. `()`), we try a final - // time to see if the pump task encountered an error, using _that_ instead if so, to - // propagate the true error that caused the source to have to stop. - if let Ok(e) = pump_error_rx.try_recv() { - result = Err(e); + match result { + Ok(()) => { + debug!("Source finished normally."); + Ok(TaskOutput::Source) } - - match result { - Ok(()) => { - debug!("Source finished normally."); - Ok(TaskOutput::Source) - } - Err(e) => { - debug!("Source finished with an error."); - Err(e) - } + Err(e) => { + debug!("Source finished with an error."); + Err(e) } - }; - let server = Task::new(key.clone(), typetag, server); + } + }; + let server = Task::new(key.clone(), typetag, server); - self.outputs.extend(controls); - self.tasks.insert(key.clone(), pump); - source_tasks.insert(key.clone(), server); - } + self.outputs.extend(controls); + self.tasks.insert(key.clone(), pump); - source_tasks + Ok(server) } async fn build_transforms( @@ -444,85 +465,94 @@ impl<'a> Builder<'a> { } }; - let merged_definition: Definition = input_definitions - .iter() - .map(|(_output_id, definition)| definition.clone()) - .reduce(Definition::merge) - // We may not have any definitions if all the inputs are from metrics sources. - .unwrap_or_else(Definition::any); - let span = error_span!( "transform", component_kind = "transform", component_id = %key.id(), component_type = %transform.inner.get_component_name(), ); - let _span = span.enter(); - - // Create a map of the outputs to the list of possible definitions from those outputs. - let schema_definitions = transform - .inner - .outputs( - &TransformContext { - enrichment_tables: enrichment_tables.clone(), - metrics_storage: METRICS_STORAGE.clone(), - schema: self.config.schema, - ..Default::default() - }, - &input_definitions, - ) - .into_iter() - .map(|output| { - let definitions = output.schema_definitions(self.config.schema.enabled); - (output.port, definitions) - }) - .collect::>(); - - let context = TransformContext { - key: Some(key.clone()), - globals: self.config.global.clone(), - enrichment_tables: enrichment_tables.clone(), - metrics_storage: METRICS_STORAGE.clone(), - schema_definitions, - merged_schema_definition: merged_definition.clone(), - schema: self.config.schema, - extra_context: self.extra_context.clone(), - }; - let node = - TransformNode::from_parts(key.clone(), &context, transform, &input_definitions); + self.build_instrumented_transform(key, transform, enrichment_tables, input_definitions) + .instrument(span) + .await; + } + } - let transform = match transform - .inner - .build(&context) - .instrument(span.clone()) - .await - { - Err(error) => { - self.errors.push(format!("Transform \"{key}\": {error}")); - continue; - } - Ok(transform) => transform, - }; + async fn build_instrumented_transform( + &mut self, + key: &ComponentKey, + transform: &TransformOuter, + enrichment_tables: &vector_lib::enrichment::TableRegistry, + input_definitions: Vec<(OutputId, Definition)>, + ) { + let merged_definition: Definition = input_definitions + .iter() + .map(|(_output_id, definition)| definition.clone()) + .reduce(Definition::merge) + // We may not have any definitions if all the inputs are from metrics sources. + .unwrap_or_else(Definition::any); + + // Create a map of the outputs to the list of possible definitions from those outputs. + let schema_definitions = transform + .inner + .outputs( + &TransformContext { + enrichment_tables: enrichment_tables.clone(), + metrics_storage: METRICS_STORAGE.clone(), + schema: self.config.schema, + ..Default::default() + }, + &input_definitions, + ) + .into_iter() + .map(|output| { + let definitions = output.schema_definitions(self.config.schema.enabled); + (output.port, definitions) + }) + .collect::>(); + + let context = TransformContext { + key: Some(key.clone()), + globals: self.config.global.clone(), + enrichment_tables: enrichment_tables.clone(), + metrics_storage: METRICS_STORAGE.clone(), + schema_definitions, + merged_schema_definition: merged_definition.clone(), + schema: self.config.schema, + extra_context: self.extra_context.clone(), + }; - let metrics = ChannelMetricMetadata::new(TRANSFORM_CHANNEL_METRIC_PREFIX, None); - let (input_tx, input_rx) = TopologyBuilder::standalone_memory( - TOPOLOGY_BUFFER_SIZE, - WhenFull::Block, - &span, - Some(metrics), - self.config.global.buffer_utilization_ewma_half_life_seconds, - ); + let node = TransformNode::from_parts(key.clone(), &context, transform, &input_definitions); + + let transform = match transform + .inner + .build(&context) + .instrument(Span::current()) + .await + { + Err(error) => { + self.errors.push(format!("Transform \"{key}\": {error}")); + return; + } + Ok(transform) => transform, + }; - self.inputs - .insert(key.clone(), (input_tx, node.inputs.clone())); + let metrics = ChannelMetricMetadata::new(BufferChannelKind::Transform, None); + let (input_tx, input_rx) = TopologyBuilder::standalone_memory( + TOPOLOGY_BUFFER_SIZE, + WhenFull::Block, + &Span::current(), + Some(metrics), + self.config.global.buffer_utilization_ewma_half_life_seconds, + ); - let (transform_task, transform_outputs) = - self.build_transform(transform, node, input_rx); + self.inputs + .insert(key.clone(), (input_tx, node.inputs.clone())); - self.outputs.extend(transform_outputs); - self.tasks.insert(key.clone(), transform_task); - } + let (transform_task, transform_outputs) = self.build_transform(transform, node, input_rx); + + self.outputs.extend(transform_outputs); + self.tasks.insert(key.clone(), transform_task); } async fn build_sinks(&mut self, enrichment_tables: &vector_lib::enrichment::TableRegistry) { @@ -545,172 +575,178 @@ impl<'a> Builder<'a> { { debug!(component_id = %key, "Building new sink."); - let sink_inputs = &sink.inputs; - let healthcheck = sink.healthcheck(); - let enable_healthcheck = healthcheck.enabled && self.config.healthchecks.enabled; - let healthcheck_timeout = healthcheck.timeout; - - let typetag = sink.inner.get_component_name(); - let input_type = sink.inner.input().data_type(); - let span = error_span!( "sink", component_kind = "sink", component_id = %key.id(), component_type = %sink.inner.get_component_name(), ); - let _entered_span = span.enter(); - - // At this point, we've validated that all transforms are valid, including any - // transform that mutates the schema provided by their sources. We can now validate the - // schema expectations of each individual sink. - if let Err(mut err) = schema::validate_sink_expectations( - key, - sink, - self.config, - enrichment_tables.clone(), - ) { - self.errors.append(&mut err); - }; - let (tx, rx) = match self.buffers.remove(key) { - Some(buffer) => buffer, - _ => { - let buffer_type = - match sink.buffer.stages().first().expect("cant ever be empty") { - BufferType::Memory { .. } => "memory", - BufferType::DiskV2 { .. } => "disk", - }; - let buffer_span = error_span!("sink", buffer_type); - let buffer = sink - .buffer - .build( - self.config.global.data_dir.clone(), - key.to_string(), - buffer_span, - ) - .await; - match buffer { - Err(error) => { - self.errors.push(format!("Sink \"{key}\": {error}")); - continue; - } - Ok((tx, rx)) => (tx, Arc::new(Mutex::new(Some(rx.into_stream())))), + self.build_instrumented_sink(key, sink, enrichment_tables) + .instrument(span) + .await; + } + } + + async fn build_instrumented_sink( + &mut self, + key: &ComponentKey, + sink: &SinkOuter, + enrichment_tables: &vector_lib::enrichment::TableRegistry, + ) { + let sink_inputs = &sink.inputs; + let healthcheck = sink.healthcheck(); + let enable_healthcheck = healthcheck.enabled && self.config.healthchecks.enabled; + let healthcheck_timeout = healthcheck.timeout; + + let typetag = sink.inner.get_component_name(); + let input_type = sink.inner.input().data_type(); + + // At this point, we've validated that all transforms are valid, including any + // transform that mutates the schema provided by their sources. We can now validate the + // schema expectations of each individual sink. + if let Err(mut err) = + schema::validate_sink_expectations(key, sink, self.config, enrichment_tables.clone()) + { + self.errors.append(&mut err); + }; + + let (tx, rx) = match self.buffers.remove(key) { + Some(buffer) => buffer, + _ => { + let buffer_type = match sink.buffer.stages().first().expect("cant ever be empty") { + BufferType::Memory { .. } => "memory", + BufferType::DiskV2 { .. } => "disk", + }; + let buffer_span = error_span!("sink", buffer_type); + let buffer = sink + .buffer + .build( + self.config.global.data_dir.clone(), + key.to_string(), + buffer_span, + ) + .await; + match buffer { + Err(error) => { + self.errors.push(format!("Sink \"{key}\": {error}")); + return; } + Ok((tx, rx)) => (tx, Arc::new(Mutex::new(Some(rx.into_stream())))), } - }; + } + }; - let cx = SinkContext { - healthcheck, - globals: self.config.global.clone(), - enrichment_tables: enrichment_tables.clone(), - metrics_storage: METRICS_STORAGE.clone(), - proxy: ProxyConfig::merge_with_env(&self.config.global.proxy, sink.proxy()), - schema: self.config.schema, - app_name: crate::get_app_name().to_string(), - app_name_slug: crate::get_slugified_app_name(), - extra_context: self.extra_context.clone(), - }; + let cx = SinkContext { + healthcheck, + globals: self.config.global.clone(), + enrichment_tables: enrichment_tables.clone(), + metrics_storage: METRICS_STORAGE.clone(), + proxy: ProxyConfig::merge_with_env(&self.config.global.proxy, sink.proxy()), + schema: self.config.schema, + app_name: crate::get_app_name().to_string(), + app_name_slug: crate::get_slugified_app_name(), + extra_context: self.extra_context.clone(), + }; - let (sink, healthcheck) = match sink.inner.build(cx).await { - Err(error) => { - self.errors.push(format!("Sink \"{key}\": {error}")); - continue; - } - Ok(built) => built, - }; + let (sink, healthcheck) = match sink.inner.build(cx).await { + Err(error) => { + self.errors.push(format!("Sink \"{key}\": {error}")); + return; + } + Ok(built) => built, + }; - let (trigger, tripwire) = Tripwire::new(); - - let utilization_sender = self - .utilization_registry - .add_component(key.clone(), gauge!("utilization")); - let component_key = key.clone(); - let sink = async move { - debug!("Sink starting."); - - // Why is this Arc>> needed you ask. - // In case when this function build_pieces errors - // this future won't be run so this rx won't be taken - // which will enable us to reuse rx to rebuild - // old configuration by passing this Arc>> - // yet again. - let rx = rx - .lock() - .unwrap() - .take() - .expect("Task started but input has been taken."); - - let mut rx = wrap(utilization_sender, component_key.clone(), rx); - - let events_received = register!(EventsReceived); - sink.run( - rx.by_ref() - .filter(|events: &EventArray| ready(filter_events_type(events, input_type))) - .inspect(|events| { - events_received.emit(CountByteSize( - events.len(), - events.estimated_json_encoded_size_of(), - )) - }) - .take_until_if(tripwire), - ) - .await - .map(|_| { - debug!("Sink finished normally."); - TaskOutput::Sink(rx) - }) - .map_err(|_| { - debug!("Sink finished with an error."); - TaskError::Opaque - }) - }; + let (trigger, tripwire) = Tripwire::new(); - let task = Task::new(key.clone(), typetag, sink); + let utilization_sender = self + .utilization_registry + .add_component(key.clone(), gauge!(GaugeName::Utilization)); + let component_key = key.clone(); + let sink = async move { + debug!("Sink starting."); + + // Why is this Arc>> needed you ask. + // In case when this function build_pieces errors + // this future won't be run so this rx won't be taken + // which will enable us to reuse rx to rebuild + // old configuration by passing this Arc>> + // yet again. + let rx = rx + .lock() + .unwrap() + .take() + .expect("Task started but input has been taken."); + + let mut rx = Utilization::new(utilization_sender, component_key.clone(), rx); + + let events_received = register!(EventsReceived); + sink.run( + rx.by_ref() + .filter(|events: &EventArray| ready(filter_events_type(events, input_type))) + .inspect(|events| { + events_received.emit(CountByteSize( + events.len(), + events.estimated_json_encoded_size_of(), + )) + }) + .take_until_if(tripwire), + ) + .await + .map(|_| { + debug!("Sink finished normally."); + TaskOutput::Sink(rx) + }) + .map_err(|_| { + debug!("Sink finished with an error."); + TaskError::Opaque + }) + }; - let component_key = key.clone(); - let healthcheck_task = async move { - if enable_healthcheck { - timeout(healthcheck_timeout, healthcheck) - .map(|result| match result { - Ok(Ok(_)) => { - info!("Healthcheck passed."); - Ok(TaskOutput::Healthcheck) - } - Ok(Err(error)) => { - error!( - msg = "Healthcheck failed.", - %error, - component_kind = "sink", - component_type = typetag, - component_id = %component_key.id(), - ); - Err(TaskError::wrapped(error)) - } - Err(e) => { - error!( - msg = "Healthcheck timed out.", - component_kind = "sink", - component_type = typetag, - component_id = %component_key.id(), - ); - Err(TaskError::wrapped(Box::new(e))) - } - }) - .await - } else { - info!("Healthcheck disabled."); - Ok(TaskOutput::Healthcheck) - } - }; + let task = Task::new(key.clone(), typetag, sink); + + let component_key = key.clone(); + let healthcheck_task = async move { + if enable_healthcheck { + timeout(healthcheck_timeout, healthcheck) + .map(|result| match result { + Ok(Ok(_)) => { + info!("Healthcheck passed."); + Ok(TaskOutput::Healthcheck) + } + Ok(Err(error)) => { + error!( + msg = "Healthcheck failed.", + %error, + component_kind = "sink", + component_type = typetag, + component_id = %component_key.id(), + ); + Err(TaskError::wrapped(error)) + } + Err(e) => { + error!( + msg = "Healthcheck timed out.", + component_kind = "sink", + component_type = typetag, + component_id = %component_key.id(), + ); + Err(TaskError::wrapped(Box::new(e))) + } + }) + .await + } else { + info!("Healthcheck disabled."); + Ok(TaskOutput::Healthcheck) + } + }; - let healthcheck_task = Task::new(key.clone(), typetag, healthcheck_task); + let healthcheck_task = Task::new(key.clone(), typetag, healthcheck_task); - self.inputs.insert(key.clone(), (tx, sink_inputs.clone())); - self.healthchecks.insert(key.clone(), healthcheck_task); - self.tasks.insert(key.clone(), task); - self.detach_triggers.insert(key.clone(), trigger); - } + self.inputs.insert(key.clone(), (tx, sink_inputs.clone())); + self.healthchecks.insert(key.clone(), healthcheck_task); + self.tasks.insert(key.clone(), task); + self.detach_triggers.insert(key.clone(), trigger); } fn build_transform( @@ -744,7 +780,7 @@ impl<'a> Builder<'a> { let sender = self .utilization_registry - .add_component(node.key.clone(), gauge!("utilization")); + .add_component(node.key.clone(), gauge!(GaugeName::Utilization)); let runner = Runner::new( t, input_rx, @@ -796,12 +832,13 @@ impl<'a> Builder<'a> { key: &ComponentKey, outputs: &[TransformOutput], ) -> (Task, HashMap) { - let (mut fanout, control) = Fanout::new(); + let (mut fanout, control) = Fanout::new(key.clone()); let sender = self .utilization_registry - .add_component(key.clone(), gauge!("utilization")); - let input_rx = wrap(sender, key.clone(), input_rx.into_stream()); + .add_component(key.clone(), gauge!(GaugeName::Utilization)); + let output_sender = sender.clone(); + let input_rx = Utilization::new(sender, key.clone(), input_rx.into_stream()); let events_received = register!(EventsReceived); let filtered = input_rx @@ -846,6 +883,7 @@ impl<'a> Builder<'a> { events.estimated_json_encoded_size_of(), )); }); + let stream = OutputUtilization::new(output_sender, stream); let transform = async move { debug!("Task transform starting."); @@ -879,30 +917,44 @@ async fn run_source_output_pump( ) -> TaskResult { debug!("Source pump starting."); - while let Some(SourceSenderItem { - events: mut array, - send_reference, - }) = rx.next().await - { - // Even though we have a `send_reference` timestamp above, that reference time is when - // the events were enqueued in the `SourceSender`, not when they were pulled out of the - // `rx` stream on this end. Since those times can be quite different (due to blocking - // inherent to the fanout send operation), we set the `last_transform_timestamp` to the - // current time instead to get an accurate reference for when the events started waiting - // for the first transform. - let now = Instant::now(); - array.for_each_metadata_mut(|metadata| { - metadata.set_source_id(Arc::clone(&source)); - metadata.set_source_type(source_type); - metadata.set_last_transform_timestamp(now); - }); - fanout - .send(array, Some(send_reference)) - .await - .map_err(|e| { - debug!("Source pump finished with an error."); - TaskError::wrapped(e) - })?; + let mut control_channel_open = true; + loop { + tokio::select! { + biased; + // Process control messages (e.g. Remove/Pause) even when the source + // is idle, so that config reloads can proceed without waiting for the + // next event. + alive = fanout.recv_control_message(), if control_channel_open => { + control_channel_open = alive; + } + item = rx.next() => { + match item { + Some(SourceSenderItem { events: mut array, send_reference }) => { + // Even though we have a `send_reference` timestamp above, that reference + // time is when the events were enqueued in the `SourceSender`, not when + // they were pulled out of the `rx` stream on this end. Since those times + // can be quite different (due to blocking inherent to the fanout send + // operation), we set the `last_transform_timestamp` to the current time + // instead to get an accurate reference for when the events started + // waiting for the first transform. + let now = Instant::now(); + array.for_each_metadata_mut(|metadata| { + metadata.set_source_id(Arc::clone(&source)); + metadata.set_source_type(source_type); + metadata.set_last_transform_timestamp(now); + }); + fanout + .send(array, Some(send_reference)) + .await + .map_err(|e| { + debug!("Source pump finished with an error."); + TaskError::wrapped(e) + })?; + } + None => break, + } + } + } } debug!("Source pump finished normally."); @@ -1228,12 +1280,12 @@ impl Runner { let mut t = self.transform.clone(); let mut outputs_buf = self.outputs.new_buf_with_capacity(len); - let task = tokio::spawn(async move { + let task = crate::spawn_in_current_span(async move { for events in input_arrays { t.transform_all(events, &mut outputs_buf); } outputs_buf - }.in_current_span()); + }); in_flight.push_back(task); } None => { diff --git a/src/topology/controller.rs b/src/topology/controller.rs index 53acc243a71f8..efbe82cf594ac 100644 --- a/src/topology/controller.rs +++ b/src/topology/controller.rs @@ -39,7 +39,7 @@ pub struct TopologyController { pub config_paths: Vec, pub require_healthy: Option, #[cfg(feature = "api")] - pub api_server: Option, + pub api_server: Option, pub extra_context: ExtraContext, } @@ -70,37 +70,33 @@ impl TopologyController { #[cfg(feature = "api")] if !new_config.api.enabled { if let Some(server) = self.api_server.take() { - debug!("Dropping api server."); - drop(server) + debug!("Stopping gRPC API server."); + drop(server); } } else if self.api_server.is_none() { - use std::sync::atomic::AtomicBool; + debug!("Starting gRPC API server."); - use tokio::runtime::Handle; - - use crate::internal_events::ApiStarted; - - debug!("Starting api server."); - - self.api_server = match api::Server::start( - self.topology.config(), - self.topology.watch(), - Arc::::clone(&self.topology.running), - &Handle::current(), - ) { + match api::GrpcServer::start(self.topology.config(), self.topology.watch()).await { Ok(api_server) => { - emit!(ApiStarted { - addr: new_config.api.address.unwrap(), - playground: new_config.api.playground, - graphql: new_config.api.graphql, - }); - - Some(api_server) + let addr = api_server.addr(); + info!( + message = "GRPC API server started.", + addr = %addr, + ); + self.api_server = Some(api_server); } Err(error) => { - let error = error.to_string(); - error!("An error occurred that Vector couldn't handle: {}.", error); - return ReloadOutcome::FatalError(ShutdownError::ApiFailed { error }); + let error_string = error.to_string(); + error!( + message = "An error occurred that Vector couldn't handle.", + error = %error_string, + internal_log_rate_limit = false, + ); + // Fail fast when API is explicitly enabled but fails to start + // This ensures users don't run with a broken configuration thinking top/tap will work + return ReloadOutcome::FatalError(ShutdownError::ApiFailed { + error: error_string, + }); } } } @@ -111,12 +107,6 @@ impl TopologyController { .await { Ok(()) => { - #[cfg(feature = "api")] - // Pass the new config to the API server. - if let Some(ref api_server) = self.api_server { - api_server.update_config(self.topology.config()); - } - emit!(VectorReloaded { config_paths: &self.config_paths }); @@ -160,7 +150,18 @@ impl TopologyController { } } - pub async fn stop(self) { + #[cfg_attr(not(feature = "api"), allow(unused_mut))] + pub async fn stop(mut self) { + // Phase 1: Mark the gRPC API as unavailable so that external probes + // (e.g. Kubernetes readiness) fail early and stop routing traffic + // to this instance. + #[cfg(feature = "api")] + if let Some(server) = self.api_server.as_mut() { + server.set_not_serving().await; + } + + // Phase 2: Drain the topology -- shuts down sources, waits for + // in-flight events to flush through transforms and sinks. self.topology.stop().await; } diff --git a/src/topology/running.rs b/src/topology/running.rs index db667e4891110..97e309ec0d0b8 100644 --- a/src/topology/running.rs +++ b/src/topology/running.rs @@ -1,9 +1,6 @@ use std::{ collections::{HashMap, HashSet}, - sync::{ - Arc, Mutex, - atomic::{AtomicBool, Ordering}, - }, + sync::{Arc, Mutex}, }; use futures::{Future, FutureExt, future}; @@ -58,6 +55,7 @@ pub struct RunningTopology { inputs_tap_metadata: HashMap>, outputs: HashMap, outputs_tap_metadata: HashMap, + component_type_names: HashMap, source_tasks: HashMap, tasks: HashMap, shutdown_coordinator: SourceShutdownCoordinator, @@ -65,7 +63,6 @@ pub struct RunningTopology { pub(crate) config: Config, pub(crate) abort_tx: mpsc::UnboundedSender, watch: (WatchTx, WatchRx), - pub(crate) running: Arc, graceful_shutdown_duration: Option, utilization_registry: Option, utilization_task: Option, @@ -82,13 +79,13 @@ impl RunningTopology { inputs_tap_metadata: HashMap::new(), outputs: HashMap::new(), outputs_tap_metadata: HashMap::new(), + component_type_names: HashMap::new(), shutdown_coordinator: SourceShutdownCoordinator::default(), detach_triggers: HashMap::new(), source_tasks: HashMap::new(), tasks: HashMap::new(), abort_tx, watch: watch::channel(TapResource::default()), - running: Arc::new(AtomicBool::new(true)), graceful_shutdown_duration: config.graceful_shutdown_duration, config, utilization_registry: None, @@ -146,8 +143,6 @@ impl RunningTopology { /// dropped then everything from this RunningTopology instance is fully /// dropped. pub fn stop(self) -> impl Future { - // Update the API's health endpoint to signal shutdown - self.running.store(false, Ordering::Relaxed); // Create handy handles collections of all tasks for the subsequent // operations. let mut wait_handles = Vec::new(); @@ -588,10 +583,26 @@ impl RunningTopology { .collect::>(); // For any existing sink that has a conflicting resource dependency with a changed/added - // sink, or for any sink that we want to reuse their buffer, we need to explicit wait for - // them to finish processing so we can reclaim ownership of those resources/buffers. + // sink, for any sink that we want to reuse their buffer, or for any changed sink with + // a disk buffer that is not being reused, we need to explicitly wait for them to finish + // processing so we can reclaim ownership of those resources/buffers. + let changed_disk_buffer_sinks = diff + .sinks + .to_change + .iter() + .filter(|key| { + !reuse_buffers.contains(*key) + && self + .config + .sink(key) + .is_some_and(|s| s.buffer.has_disk_stage()) + }) + .cloned() + .collect::>(); + let wait_for_sinks = conflicting_sinks .chain(reuse_buffers.iter().cloned()) + .chain(changed_disk_buffer_sinks.iter().cloned()) .collect::>(); // First, we remove any inputs to removed sinks so they can naturally shut down. @@ -633,24 +644,26 @@ impl RunningTopology { for key in &sinks_to_change { debug!(component_id = %key, "Changing sink."); - if reuse_buffers.contains(key) { + if reuse_buffers.contains(key) || changed_disk_buffer_sinks.contains(key) { self.detach_triggers .remove(key) .unwrap() .into_inner() .cancel(); - // We explicitly clone the input side of the buffer and store it so we don't lose - // it when we remove the inputs below. - // - // We clone instead of removing here because otherwise the input will be missing for - // the rest of the reload process, which violates the assumption that all previous - // inputs for components not being removed are still available. It's simpler to - // allow the "old" input to stick around and be replaced (even though that's - // basically a no-op since we're reusing the same buffer) than it is to pass around - // info about which sinks are having their buffers reused and treat them differently - // at other stages. - buffer_tx.insert((*key).clone(), self.inputs.get(key).unwrap().clone()); + if reuse_buffers.contains(key) { + // We explicitly clone the input side of the buffer and store it so we don't lose + // it when we remove the inputs below. + // + // We clone instead of removing here because otherwise the input will be missing for + // the rest of the reload process, which violates the assumption that all previous + // inputs for components not being removed are still available. It's simpler to + // allow the "old" input to stick around and be replaced (even though that's + // basically a no-op since we're reusing the same buffer) than it is to pass around + // info about which sinks are having their buffers reused and treat them differently + // at other stages. + buffer_tx.insert((*key).clone(), self.inputs.get(key).unwrap().clone()); + } } self.remove_inputs(key, diff, new_config).await; } @@ -713,17 +726,20 @@ impl RunningTopology { for key in &diff.sources.to_remove { // Sources only have outputs self.outputs_tap_metadata.remove(key); + self.component_type_names.remove(key); } for key in &diff.transforms.to_remove { // Transforms can have both inputs and outputs self.outputs_tap_metadata.remove(key); self.inputs_tap_metadata.remove(key); + self.component_type_names.remove(key); } for key in &diff.sinks.to_remove { // Sinks only have inputs self.inputs_tap_metadata.remove(key); + self.component_type_names.remove(key); } let removed_sinks = diff.enrichment_tables.to_remove.iter().filter(|key| { @@ -749,8 +765,10 @@ impl RunningTopology { for key in diff.sources.changed_and_added() { if let Some(task) = new_pieces.tasks.get(key) { + let typetag = task.typetag().to_string(); self.outputs_tap_metadata - .insert(key.clone(), ("source", task.typetag().to_string())); + .insert(key.clone(), ("source", typetag.clone())); + self.component_type_names.insert(key.clone(), typetag); } } @@ -771,8 +789,17 @@ impl RunningTopology { for key in diff.transforms.changed_and_added() { if let Some(task) = new_pieces.tasks.get(key) { + let typetag = task.typetag().to_string(); self.outputs_tap_metadata - .insert(key.clone(), ("transform", task.typetag().to_string())); + .insert(key.clone(), ("transform", typetag.clone())); + self.component_type_names.insert(key.clone(), typetag); + } + } + + for key in diff.sinks.changed_and_added() { + if let Some(task) = new_pieces.tasks.get(key) { + self.component_type_names + .insert(key.clone(), task.typetag().to_string()); } } @@ -888,6 +915,11 @@ impl RunningTopology { // Note, only sources and transforms are relevant. Sinks do // not have outputs to tap. removals, + type_names: self + .component_type_names + .iter() + .map(|(k, v)| (k.to_string(), v.clone())) + .collect(), }) .expect("Couldn't broadcast config changes."); } diff --git a/src/topology/test/reload.rs b/src/topology/test/reload.rs index 907be19f6ec0f..7b2db20d4f337 100644 --- a/src/topology/test/reload.rs +++ b/src/topology/test/reload.rs @@ -1,7 +1,7 @@ use std::{ collections::HashSet, net::{SocketAddr, TcpListener}, - num::NonZeroU64, + num::{NonZeroU64, NonZeroUsize}, time::Duration, }; @@ -9,7 +9,7 @@ use futures::StreamExt; use tokio::time::sleep; use tokio_stream::wrappers::UnboundedReceiverStream; use vector_lib::{ - buffers::{BufferConfig, BufferType, WhenFull}, + buffers::{BufferConfig, BufferType, MemoryBufferSize, WhenFull}, config::ComponentKey, }; @@ -293,7 +293,6 @@ async fn topology_readd_input() { #[tokio::test] async fn topology_reload_component() { test_util::trace_init(); - let (_guard, address_0) = next_addr(); let mut old_config = Config::builder(); @@ -320,6 +319,138 @@ async fn topology_reload_component() { } } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn topology_disk_buffer_config_change_does_not_stall() { + // Changing a disk buffer's configuration on a running sink (e.g. via in-situ + // config edit) must not stall the reload. Previously, the detach trigger was + // only cancelled for sinks whose buffers were being reused, so sinks with + // changed disk buffer configs would never have their input stream terminated, + // causing the reload to hang indefinitely. + test_util::trace_init(); + + let (_guard, address) = next_addr(); + + let data_dir = temp_dir(); + std::fs::create_dir(&data_dir).unwrap(); + + let mut old_config = Config::builder(); + old_config.global.data_dir = Some(data_dir); + old_config.add_source("in", internal_metrics_source()); + old_config.add_sink("out", &["in"], prom_exporter_sink(address, 1)); + + let sink_key = ComponentKey::from("out"); + old_config.sinks[&sink_key].buffer = BufferConfig::Single(BufferType::DiskV2 { + max_size: NonZeroU64::new(268435488).unwrap(), + when_full: WhenFull::Block, + }); + + // Change only the disk buffer's max_size. + let mut new_config = old_config.clone(); + new_config.sinks[&sink_key].buffer = BufferConfig::Single(BufferType::DiskV2 { + max_size: NonZeroU64::new(536870912).unwrap(), + when_full: WhenFull::Block, + }); + + let (mut topology, crash) = start_topology(old_config.build().unwrap(), true).await; + let mut crash_stream = UnboundedReceiverStream::new(crash); + + tokio::select! { + _ = wait_for_tcp(address) => {}, + _ = crash_stream.next() => panic!("topology crashed before reload"), + } + + // Simulate an in-situ config edit: the config watcher would put the changed + // sink into components_to_reload, which excludes it from reuse_buffers. + topology.extend_reload_set(HashSet::from_iter(vec![sink_key])); + + let reload_result = tokio::time::timeout( + Duration::from_secs(5), + topology.reload_config_and_respawn(new_config.build().unwrap(), Default::default()), + ) + .await; + + assert!( + reload_result.is_ok(), + "Reload stalled: changing a disk buffer config should not cause the reload to hang" + ); + reload_result.unwrap().unwrap(); + + // Verify the new sink is running. + tokio::select! { + _ = wait_for_tcp(address) => {}, + _ = crash_stream.next() => panic!("topology crashed after reload"), + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn topology_disk_buffer_config_change_chained_does_not_stall() { + // Same as above but with a chained memory β†’ disk overflow buffer to verify + // that the writer-drop notification is collected from overflow stages too. + test_util::trace_init(); + + let (_guard, address) = next_addr(); + + let data_dir = temp_dir(); + std::fs::create_dir(&data_dir).unwrap(); + + let memory_stage = BufferType::Memory { + size: MemoryBufferSize::MaxEvents(NonZeroUsize::new(100).unwrap()), + when_full: WhenFull::Overflow, + }; + + let mut old_config = Config::builder(); + old_config.global.data_dir = Some(data_dir); + old_config.add_source("in", internal_metrics_source()); + old_config.add_sink("out", &["in"], prom_exporter_sink(address, 1)); + + let sink_key = ComponentKey::from("out"); + old_config.sinks[&sink_key].buffer = BufferConfig::Chained(vec![ + memory_stage, + BufferType::DiskV2 { + max_size: NonZeroU64::new(268435488).unwrap(), + when_full: WhenFull::Block, + }, + ]); + + // Change only the disk overflow stage's max_size. + let mut new_config = old_config.clone(); + new_config.sinks[&sink_key].buffer = BufferConfig::Chained(vec![ + memory_stage, + BufferType::DiskV2 { + max_size: NonZeroU64::new(536870912).unwrap(), + when_full: WhenFull::Block, + }, + ]); + + let (mut topology, crash) = start_topology(old_config.build().unwrap(), true).await; + let mut crash_stream = UnboundedReceiverStream::new(crash); + + tokio::select! { + _ = wait_for_tcp(address) => {}, + _ = crash_stream.next() => panic!("topology crashed before reload"), + } + + topology.extend_reload_set(HashSet::from_iter(vec![sink_key])); + + let reload_result = tokio::time::timeout( + Duration::from_secs(5), + topology.reload_config_and_respawn(new_config.build().unwrap(), Default::default()), + ) + .await; + + assert!( + reload_result.is_ok(), + "Reload stalled: changing a chained disk buffer config should not cause the reload to hang" + ); + reload_result.unwrap().unwrap(); + + // Verify the new sink is running. + tokio::select! { + _ = wait_for_tcp(address) => {}, + _ = crash_stream.next() => panic!("topology crashed after reload"), + } +} + async fn reload_sink_test( old_config: Config, new_config: Config, diff --git a/src/trace.rs b/src/trace.rs index 6ecec804569ed..fe78b6f2a5aa0 100644 --- a/src/trace.rs +++ b/src/trace.rs @@ -1,10 +1,10 @@ #![allow(missing_docs)] use std::{ - collections::HashMap, + collections::{HashMap, HashSet}, marker::PhantomData, str::FromStr, sync::{ - Mutex, MutexGuard, OnceLock, + LazyLock, Mutex, MutexGuard, OnceLock, atomic::{AtomicBool, Ordering}, }, }; @@ -26,6 +26,7 @@ use tracing_subscriber::{ util::SubscriberInitExt, }; pub use tracing_tower::{InstrumentableService, InstrumentedService}; +use vector_lib::SpanField; use vector_lib::lookup::event_path; use vrl::value::Value; @@ -339,6 +340,17 @@ where #[derive(Default, Debug)] struct SpanFields(HashMap<&'static str, Value>); +inventory::submit!(SpanField("component_id")); +inventory::submit!(SpanField("component_type")); +inventory::submit!(SpanField("component_kind")); + +/// Snapshot of every registered [`SpanField`], +/// materialized once on first access. `inventory` populates submissions before `main`, so the +/// snapshot is guaranteed to capture every entry; the read path on every traced span event is +/// then a single set lookup against this static. +static SPAN_FIELDS: LazyLock> = + LazyLock::new(|| inventory::iter::().map(|f| f.0).collect()); + impl SpanFields { fn record(&mut self, field: &tracing_core::Field, value: impl Into) { let name = field.name(); @@ -347,8 +359,10 @@ impl SpanFields { // This captures all the basic component information provided in the // span that each component is spawned with. We don't capture all fields // to avoid adding unintentional noise and to prevent accidental - // security/privacy issues (e.g. leaking sensitive data). - if name.starts_with("component_") { + // security/privacy issues (e.g. leaking sensitive data). Downstream + // crates can extend the allowlist by name through + // `register_extra_span_field!` (see `vector_lib::SpanField`). + if SPAN_FIELDS.contains(name) { self.0.insert(name, value.into()); } } diff --git a/src/transforms/aggregate.rs b/src/transforms/aggregate.rs index 3c8622ec2dba5..acc0438c5a5f7 100644 --- a/src/transforms/aggregate.rs +++ b/src/transforms/aggregate.rs @@ -49,31 +49,90 @@ pub enum AggregationMode { #[default] Auto, - /// Sums incremental metrics, ignores absolute + /// Sums incremental metrics; absolute metrics pass through unchanged. Sum, - /// Returns the latest value for absolute metrics, ignores incremental + /// Returns the latest value for absolute metrics; incremental metrics pass through unchanged. Latest, /// Counts metrics for incremental and absolute metrics Count, - /// Returns difference between latest value for absolute, ignores incremental + /// Returns difference between latest value for absolute; incremental metrics pass through unchanged. Diff, - /// Max value of absolute metric, ignores incremental + /// Max value of absolute metric; incremental metrics pass through unchanged. Max, - /// Min value of absolute metric, ignores incremental + /// Min value of absolute metric; incremental metrics pass through unchanged. Min, - /// Mean value of absolute metric, ignores incremental + /// Mean value of absolute metric; incremental metrics pass through unchanged. Mean, - /// Stdev value of absolute metric, ignores incremental + /// Stdev value of absolute metric; incremental metrics pass through unchanged. Stdev, } +#[derive(Clone, Debug, Default, PartialEq)] +enum InnerMode { + /// Default mode. Sums incremental metrics and uses the latest value for absolute metrics. + #[default] + Auto, + + /// Sums incremental metrics; absolute metrics pass through unchanged. + Sum, + + /// Returns the latest value for absolute metrics; incremental metrics pass through unchanged. + Latest, + + /// Counts metrics for incremental and absolute metrics + Count, + + /// Returns difference between latest value for absolute; incremental metrics pass through unchanged. + Diff { + prev_map: HashMap, + }, + + /// Max value of absolute metric; incremental metrics pass through unchanged. + Max, + + /// Min value of absolute metric; incremental metrics pass through unchanged. + Min, + + /// Mean value of absolute metric; incremental metrics pass through unchanged. + Mean { + multi_map: HashMap>, + }, + + /// Stdev value of absolute metric; incremental metrics pass through unchanged. + Stdev { + multi_map: HashMap>, + }, +} + +impl From for InnerMode { + fn from(value: AggregationMode) -> Self { + match value { + AggregationMode::Auto => InnerMode::Auto, + AggregationMode::Sum => InnerMode::Sum, + AggregationMode::Latest => InnerMode::Latest, + AggregationMode::Count => InnerMode::Count, + AggregationMode::Diff => InnerMode::Diff { + prev_map: HashMap::default(), + }, + AggregationMode::Max => InnerMode::Max, + AggregationMode::Min => InnerMode::Min, + AggregationMode::Mean => InnerMode::Mean { + multi_map: HashMap::default(), + }, + AggregationMode::Stdev => InnerMode::Stdev { + multi_map: HashMap::default(), + }, + } + } +} + const fn default_mode() -> AggregationMode { AggregationMode::Auto } @@ -110,9 +169,7 @@ type MetricEntry = (MetricData, EventMetadata); pub struct Aggregate { interval: Duration, map: HashMap, - prev_map: HashMap, - multi_map: HashMap>, - mode: AggregationMode, + mode: InnerMode, } impl Aggregate { @@ -120,52 +177,49 @@ impl Aggregate { Ok(Self { interval: Duration::from_millis(config.interval_ms), map: Default::default(), - prev_map: Default::default(), - multi_map: Default::default(), - mode: config.mode, + mode: config.mode.into(), }) } - fn record(&mut self, event: Event) { + pub fn record(&mut self, event: Event) -> Option { let (series, data, metadata) = event.into_metric().into_parts(); - match self.mode { - AggregationMode::Auto => match data.kind { - MetricKind::Incremental => self.record_sum(series, data, metadata), - MetricKind::Absolute => { - self.map.insert(series, (data, metadata)); - } - }, - AggregationMode::Sum => self.record_sum(series, data, metadata), - AggregationMode::Latest | AggregationMode::Diff => match data.kind { - MetricKind::Incremental => (), - MetricKind::Absolute => { - self.map.insert(series, (data, metadata)); - } - }, - AggregationMode::Count => self.record_count(series, data, metadata), - AggregationMode::Max | AggregationMode::Min => { - self.record_comparison(series, data, metadata) + match (&mut self.mode, data.kind) { + (InnerMode::Sum, MetricKind::Absolute) + | (InnerMode::Latest | InnerMode::Diff { .. }, MetricKind::Incremental) + | (InnerMode::Max | InnerMode::Min, MetricKind::Incremental) + | (InnerMode::Mean { .. } | InnerMode::Stdev { .. }, MetricKind::Incremental) => { + return Some(Event::Metric(Metric::from_parts(series, data, metadata))); } - AggregationMode::Mean | AggregationMode::Stdev => match data.kind { - MetricKind::Incremental => (), - MetricKind::Absolute => { - if matches!(data.value, MetricValue::Gauge { value: _ }) { - match self.multi_map.entry(series) { - Entry::Occupied(mut entry) => { - let existing = entry.get_mut(); - existing.push((data, metadata)); - } - Entry::Vacant(entry) => { - entry.insert(vec![(data, metadata)]); - } + (InnerMode::Auto | InnerMode::Sum, MetricKind::Incremental) => { + self.record_sum(series, data, metadata); + } + (InnerMode::Auto, MetricKind::Absolute) + | (InnerMode::Latest | InnerMode::Diff { .. }, MetricKind::Absolute) => { + self.map.insert(series, (data, metadata)); + } + (InnerMode::Count, _) => { + self.record_count(series, data, metadata); + } + (InnerMode::Max | InnerMode::Min, MetricKind::Absolute) => { + self.record_comparison(series, data, metadata); + } + ( + InnerMode::Mean { multi_map } | InnerMode::Stdev { multi_map }, + MetricKind::Absolute, + ) => { + if matches!(data.value, MetricValue::Gauge { value: _ }) { + match multi_map.entry(series) { + Entry::Occupied(mut entry) => entry.get_mut().push((data, metadata)), + Entry::Vacant(entry) => { + entry.insert(vec![(data, metadata)]); } } } - }, + } } - emit!(AggregateEventRecorded); + None } fn record_count( @@ -188,23 +242,20 @@ impl Aggregate { } fn record_sum(&mut self, series: MetricSeries, data: MetricData, metadata: EventMetadata) { - match data.kind { - MetricKind::Incremental => match self.map.entry(series) { - Entry::Occupied(mut entry) => { - let existing = entry.get_mut(); - // In order to update (add) the new and old kind's must match - if existing.0.kind == data.kind && existing.0.update(&data) { - existing.1.merge(metadata); - } else { - emit!(AggregateUpdateFailed); - *existing = (data, metadata); - } - } - Entry::Vacant(entry) => { - entry.insert((data, metadata)); + match self.map.entry(series) { + Entry::Occupied(mut entry) => { + let existing = entry.get_mut(); + // In order to update (add) the new and old kind's must match + if existing.0.kind == data.kind && existing.0.update(&data) { + existing.1.merge(metadata); + } else { + emit!(AggregateUpdateFailed); + *existing = (data, metadata); } - }, - MetricKind::Absolute => {} + } + Entry::Vacant(entry) => { + entry.insert((data, metadata)); + } } } @@ -214,45 +265,42 @@ impl Aggregate { data: MetricData, metadata: EventMetadata, ) { - match data.kind { - MetricKind::Incremental => (), - MetricKind::Absolute => match self.map.entry(series) { - Entry::Occupied(mut entry) => { - let existing = entry.get_mut(); - // In order to update (add) the new and old kind's must match - if existing.0.kind == data.kind { - if let MetricValue::Gauge { - value: existing_value, - } = existing.0.value() - && let MetricValue::Gauge { value: new_value } = data.value() - { - let should_update = match self.mode { - AggregationMode::Max => new_value > existing_value, - AggregationMode::Min => new_value < existing_value, - _ => false, - }; - if should_update { - *existing = (data, metadata); - } + match self.map.entry(series) { + Entry::Occupied(mut entry) => { + let existing = entry.get_mut(); + // In order to update (add) the new and old kind's must match + if existing.0.kind == data.kind { + if let MetricValue::Gauge { + value: existing_value, + } = existing.0.value() + && let MetricValue::Gauge { value: new_value } = data.value() + { + let should_update = match self.mode { + InnerMode::Max => new_value > existing_value, + InnerMode::Min => new_value < existing_value, + _ => false, + }; + if should_update { + *existing = (data, metadata); } - } else { - emit!(AggregateUpdateFailed); - *existing = (data, metadata); } + } else { + emit!(AggregateUpdateFailed); + *existing = (data, metadata); } - Entry::Vacant(entry) => { - entry.insert((data, metadata)); - } - }, + } + Entry::Vacant(entry) => { + entry.insert((data, metadata)); + } } } - fn flush_into(&mut self, output: &mut Vec) { + pub fn flush_into(&mut self, output: &mut Vec) { let map = std::mem::take(&mut self.map); for (series, entry) in map.clone().into_iter() { let mut metric = Metric::from_parts(series, entry.0, entry.1); - if matches!(self.mode, AggregationMode::Diff) - && let Some(prev_entry) = self.prev_map.get(metric.series()) + if let InnerMode::Diff { prev_map } = &self.mode + && let Some(prev_entry) = prev_map.get(metric.series()) && metric.data().kind == prev_entry.0.kind && !metric.subtract(&prev_entry.0) { @@ -261,7 +309,13 @@ impl Aggregate { output.push(Event::Metric(metric)); } - let multi_map = std::mem::take(&mut self.multi_map); + let multi_map = match &mut self.mode { + InnerMode::Mean { multi_map } | InnerMode::Stdev { multi_map } => { + std::mem::take(multi_map) + } + _ => HashMap::default(), + }; + 'outer: for (series, entries) in multi_map.into_iter() { if entries.is_empty() { continue; @@ -287,11 +341,11 @@ impl Aggregate { let final_mean = final_sum.clone(); match self.mode { - AggregationMode::Mean => { + InnerMode::Mean { .. } => { let metric = Metric::from_parts(series, final_mean, final_metadata); output.push(Event::Metric(metric)); } - AggregationMode::Stdev => { + InnerMode::Stdev { .. } => { let variance = entries .iter() .filter_map(|(data, _)| { @@ -315,7 +369,9 @@ impl Aggregate { } } - self.prev_map = map; + if let InnerMode::Diff { prev_map } = &mut self.mode { + *prev_map = map; + } emit!(AggregateFlushed); } } @@ -344,7 +400,11 @@ impl TaskTransform for Aggregate { self.flush_into(&mut output); done = true; } - Some(event) => self.record(event), + Some(event) => { + if let Some(passthrough) = self.record(event) { + output.push(passthrough); + } + } } } }; @@ -420,7 +480,7 @@ mod tests { ); // Single item, just stored regardless of kind - agg.record(counter_a_1.clone()); + assert_eq!(agg.record(counter_a_1.clone()), None); let mut out = vec![]; // We should flush 1 item counter_a_1 agg.flush_into(&mut out); @@ -438,8 +498,8 @@ mod tests { assert_eq!(0, out.len()); // Two increments with the same series, should sum into 1 - agg.record(counter_a_1.clone()); - agg.record(counter_a_2); + assert_eq!(agg.record(counter_a_1.clone()), None); + assert_eq!(agg.record(counter_a_2), None); out.clear(); agg.flush_into(&mut out); assert_eq!(1, out.len()); @@ -451,8 +511,8 @@ mod tests { MetricValue::Counter { value: 44.0 }, ); // Two increments with the different series, should get each back as-is - agg.record(counter_a_1.clone()); - agg.record(counter_b_1.clone()); + assert_eq!(agg.record(counter_a_1.clone()), None); + assert_eq!(agg.record(counter_b_1.clone()), None); out.clear(); agg.flush_into(&mut out); assert_eq!(2, out.len()); @@ -486,7 +546,7 @@ mod tests { ); // Single item, just stored regardless of kind - agg.record(gauge_a_1.clone()); + assert_eq!(agg.record(gauge_a_1.clone()), None); let mut out = vec![]; // We should flush 1 item gauge_a_1 agg.flush_into(&mut out); @@ -504,8 +564,8 @@ mod tests { assert_eq!(0, out.len()); // Two absolutes with the same series, should get the 2nd (last) back. - agg.record(gauge_a_1.clone()); - agg.record(gauge_a_2.clone()); + assert_eq!(agg.record(gauge_a_1.clone()), None); + assert_eq!(agg.record(gauge_a_2.clone()), None); out.clear(); agg.flush_into(&mut out); assert_eq!(1, out.len()); @@ -517,8 +577,8 @@ mod tests { MetricValue::Gauge { value: 44.0 }, ); // Two increments with the different series, should get each back as-is - agg.record(gauge_a_1.clone()); - agg.record(gauge_b_1.clone()); + assert_eq!(agg.record(gauge_a_1.clone()), None); + assert_eq!(agg.record(gauge_b_1.clone()), None); out.clear(); agg.flush_into(&mut out); assert_eq!(2, out.len()); @@ -562,7 +622,7 @@ mod tests { ); // Single item, counter should be 1 - agg.record(gauge_a_1.clone()); + assert_eq!(agg.record(gauge_a_1.clone()), None); let mut out = vec![]; // We should flush 1 item gauge_a_1 agg.flush_into(&mut out); @@ -580,8 +640,8 @@ mod tests { assert_eq!(0, out.len()); // Two absolutes with the same series, counter should be 2 - agg.record(gauge_a_1.clone()); - agg.record(gauge_a_2.clone()); + assert_eq!(agg.record(gauge_a_1.clone()), None); + assert_eq!(agg.record(gauge_a_2.clone()), None); out.clear(); agg.flush_into(&mut out); assert_eq!(1, out.len()); @@ -608,7 +668,7 @@ mod tests { ); // Single item, it should be returned as is - agg.record(gauge_a_2.clone()); + assert_eq!(agg.record(gauge_a_2.clone()), None); let mut out = vec![]; // We should flush 1 item gauge_a_2 agg.flush_into(&mut out); @@ -626,8 +686,8 @@ mod tests { assert_eq!(0, out.len()); // Two absolutes, result should be higher of the 2 - agg.record(gauge_a_1.clone()); - agg.record(gauge_a_2.clone()); + assert_eq!(agg.record(gauge_a_1.clone()), None); + assert_eq!(agg.record(gauge_a_2.clone()), None); out.clear(); agg.flush_into(&mut out); assert_eq!(1, out.len()); @@ -654,7 +714,7 @@ mod tests { ); // Single item, it should be returned as is - agg.record(gauge_a_2.clone()); + assert_eq!(agg.record(gauge_a_2.clone()), None); let mut out = vec![]; // We should flush 1 item gauge_a_2 agg.flush_into(&mut out); @@ -672,8 +732,8 @@ mod tests { assert_eq!(0, out.len()); // Two absolutes, result should be lower of the 2 - agg.record(gauge_a_1.clone()); - agg.record(gauge_a_2.clone()); + assert_eq!(agg.record(gauge_a_1.clone()), None); + assert_eq!(agg.record(gauge_a_2.clone()), None); out.clear(); agg.flush_into(&mut out); assert_eq!(1, out.len()); @@ -705,7 +765,7 @@ mod tests { ); // Single item, it should be returned as is - agg.record(gauge_a_2.clone()); + assert_eq!(agg.record(gauge_a_2.clone()), None); let mut out = vec![]; // We should flush 1 item gauge_a_2 agg.flush_into(&mut out); @@ -723,13 +783,13 @@ mod tests { assert_eq!(0, out.len()); // Two absolutes in 2 separate flushes, result should be diff between the 2 - agg.record(gauge_a_1.clone()); + assert_eq!(agg.record(gauge_a_1.clone()), None); out.clear(); agg.flush_into(&mut out); assert_eq!(1, out.len()); assert_eq!(&gauge_a_1, &out[0]); - agg.record(gauge_a_2.clone()); + assert_eq!(agg.record(gauge_a_2.clone()), None); out.clear(); agg.flush_into(&mut out); assert_eq!(1, out.len()); @@ -757,13 +817,13 @@ mod tests { let mut out = vec![]; // Two absolutes in 2 separate flushes, result should be second one due to different types - agg.record(gauge_a_1.clone()); + assert_eq!(agg.record(gauge_a_1.clone()), None); out.clear(); agg.flush_into(&mut out); assert_eq!(1, out.len()); assert_eq!(&gauge_a_1, &out[0]); - agg.record(gauge_a_2.clone()); + assert_eq!(agg.record(gauge_a_2.clone()), None); out.clear(); agg.flush_into(&mut out); assert_eq!(1, out.len()); @@ -801,7 +861,7 @@ mod tests { ); // Single item, it should be returned as is - agg.record(gauge_a_2.clone()); + assert_eq!(agg.record(gauge_a_2.clone()), None); let mut out = vec![]; // We should flush 1 item gauge_a_2 agg.flush_into(&mut out); @@ -819,9 +879,9 @@ mod tests { assert_eq!(0, out.len()); // Three absolutes, result should be mean - agg.record(gauge_a_1.clone()); - agg.record(gauge_a_2.clone()); - agg.record(gauge_a_3.clone()); + assert_eq!(agg.record(gauge_a_1.clone()), None); + assert_eq!(agg.record(gauge_a_2.clone()), None); + assert_eq!(agg.record(gauge_a_3.clone()), None); out.clear(); agg.flush_into(&mut out); assert_eq!(1, out.len()); @@ -880,7 +940,7 @@ mod tests { ); for gauge in gauges { - agg.record(gauge); + assert_eq!(agg.record(gauge), None); } let mut out = vec![]; agg.flush_into(&mut out); @@ -888,6 +948,56 @@ mod tests { assert_eq!(&stdev_result, &out[0]); } + #[test] + fn passes_through_ignored_kind() { + // Sum mode aggregates incremental, passes through absolute without collapsing. + let mut agg = Aggregate::new(&AggregateConfig { + interval_ms: 1000_u64, + mode: AggregationMode::Sum, + }) + .unwrap(); + + let counter_1 = make_metric( + "counter_a", + MetricKind::Incremental, + MetricValue::Counter { value: 10.0 }, + ); + let counter_2 = make_metric( + "counter_a", + MetricKind::Incremental, + MetricValue::Counter { value: 5.0 }, + ); + let counter_summed = make_metric( + "counter_a", + MetricKind::Incremental, + MetricValue::Counter { value: 15.0 }, + ); + let gauge_1 = make_metric( + "gauge_a", + MetricKind::Absolute, + MetricValue::Gauge { value: 42.0 }, + ); + let gauge_2 = make_metric( + "gauge_a", + MetricKind::Absolute, + MetricValue::Gauge { value: 99.0 }, + ); + + // Absolute metrics pass through immediately (not held until flush). + assert_eq!(agg.record(gauge_1.clone()), Some(gauge_1)); + assert_eq!(agg.record(gauge_2.clone()), Some(gauge_2)); + + // Each is returned individually β€” no collapsing to latest. + assert_eq!(agg.record(counter_1), None); + assert_eq!(agg.record(counter_2), None); + + let mut out = vec![]; + agg.flush_into(&mut out); + // Only the summed incremental counter appears at flush; the gauges already passed through. + assert_eq!(1, out.len()); + assert_eq!(&counter_summed, &out[0]); + } + #[test] fn conflicting_value_type() { let mut agg = Aggregate::new(&AggregateConfig { @@ -918,13 +1028,13 @@ mod tests { // when types conflict the new values replaces whatever is there // Start with an counter - agg.record(counter.clone()); + assert_eq!(agg.record(counter.clone()), None); // Another will "add" to it - agg.record(counter.clone()); + assert_eq!(agg.record(counter.clone()), None); // Then an set will replace it due to a failed update - agg.record(set.clone()); + assert_eq!(agg.record(set.clone()), None); // Then a set union would be a noop - agg.record(set.clone()); + assert_eq!(agg.record(set.clone()), None); let mut out = vec![]; // We should flush 1 item counter agg.flush_into(&mut out); @@ -932,13 +1042,13 @@ mod tests { assert_eq!(&set, &out[0]); // Start out with an set - agg.record(set.clone()); + assert_eq!(agg.record(set.clone()), None); // Union with itself, a noop - agg.record(set); + assert_eq!(agg.record(set), None); // Send an counter with the same name, will replace due to a failed update - agg.record(counter.clone()); + assert_eq!(agg.record(counter.clone()), None); // Send another counter will "add" - agg.record(counter); + assert_eq!(agg.record(counter), None); let mut out = vec![]; // We should flush 1 item counter agg.flush_into(&mut out); @@ -973,13 +1083,13 @@ mod tests { // when types conflict the new values replaces whatever is there // Start with an incremental - agg.record(incremental.clone()); + assert_eq!(agg.record(incremental.clone()), None); // Another will "add" to it - agg.record(incremental.clone()); + assert_eq!(agg.record(incremental.clone()), None); // Then an absolute will replace it with a failed update - agg.record(absolute.clone()); + assert_eq!(agg.record(absolute.clone()), None); // Then another absolute will replace it normally - agg.record(absolute.clone()); + assert_eq!(agg.record(absolute.clone()), None); let mut out = vec![]; // We should flush 1 item incremental agg.flush_into(&mut out); @@ -987,13 +1097,13 @@ mod tests { assert_eq!(&absolute, &out[0]); // Start out with an absolute - agg.record(absolute.clone()); + assert_eq!(agg.record(absolute.clone()), None); // Replace it normally - agg.record(absolute); + assert_eq!(agg.record(absolute), None); // Send an incremental with the same name, will replace due to a failed update - agg.record(incremental.clone()); + assert_eq!(agg.record(incremental.clone()), None); // Send another incremental will "add" - agg.record(incremental); + assert_eq!(agg.record(incremental), None); let mut out = vec![]; // We should flush 1 item incremental agg.flush_into(&mut out); diff --git a/src/transforms/delay.rs b/src/transforms/delay.rs new file mode 100644 index 0000000000000..3c3e8093631d5 --- /dev/null +++ b/src/transforms/delay.rs @@ -0,0 +1,344 @@ +use std::{num::NonZeroUsize, pin::Pin, time::Duration}; + +use async_stream::stream; +use futures::{Stream, StreamExt}; +use serde_with::serde_as; +use snafu::Snafu; +use tokio_util::time::DelayQueue; +use vector_lib::configurable::configurable_component; +use vector_lib::internal_event::INTENTIONAL; +use vector_lib::{config::clone_input_definitions, internal_event::ComponentEventsDropped}; + +use crate::{ + conditions::{AnyCondition, Condition}, + config::{DataType, Input, OutputId, TransformConfig, TransformContext, TransformOutput}, + event::Event, + schema, + transforms::{TaskTransform, Transform}, +}; + +/// Configuration for the `delay` transform. +#[serde_as] +#[configurable_component(transform("delay", "Slow down events passing through a topology."))] +#[derive(Clone, Debug)] +#[serde(deny_unknown_fields)] +pub struct DelayConfig { + /// Time to delay each event, in milliseconds. + #[serde_as(as = "serde_with::DurationMilliSeconds")] + #[configurable(metadata(docs::human_name = "Delay in milliseconds", docs::example = 200))] + delay_ms: Duration, + + /// Limit for number of items in the delay queue. + #[serde(default = "default_queue_capacity")] + queue_capacity: NonZeroUsize, + + /// Strategy to handle full queue capacity. + #[serde(default)] + overflow_strategy: OverflowStrategy, + + /// Delay events in provided delay periods until the condition is met. + condition: Option, +} + +const fn default_queue_capacity() -> NonZeroUsize { + NonZeroUsize::new(500).expect("static non-zero number") +} + +impl Default for DelayConfig { + fn default() -> Self { + Self { + delay_ms: Default::default(), + queue_capacity: default_queue_capacity(), + overflow_strategy: Default::default(), + condition: Default::default(), + } + } +} + +/// Event handling behavior when delay queue is full. +#[configurable_component] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum OverflowStrategy { + /// Wait for free space in the queue. + /// + /// This applies backpressure up the topology, signalling that sources should slow down + /// the acceptance/consumption of events. This may cause the system to degenerate if this + /// component blocks for too long. + #[default] + Block, + + /// Drops the event instead of waiting for free space in the queue. + /// + /// The event will be intentionally dropped. This mode is typically used when performance is the + /// highest priority, and it is preferable to temporarily lose events rather than cause a + /// slowdown in the acceptance/consumption of events. + DropNewest, + + /// Forward the event without any delay to next component. + Forward, +} + +impl_generate_config_from_default!(DelayConfig); + +#[async_trait::async_trait] +#[typetag::serde(name = "delay")] +impl TransformConfig for DelayConfig { + async fn build(&self, context: &TransformContext) -> crate::Result { + if self.delay_ms.as_millis() == 0 { + return Err(Box::new(BuildError::ZeroDelayDuration)); + } + Ok(Transform::event_task(Delay::new(self, context)?)) + } + + fn input(&self) -> Input { + Input::all() + } + + fn outputs( + &self, + _: &TransformContext, + input_definitions: &[(OutputId, schema::Definition)], + ) -> Vec { + // The event is not modified, so the definition is passed through as-is + vec![TransformOutput::new( + DataType::all_bits(), + clone_input_definitions(input_definitions), + )] + } +} + +pub struct Delay { + delay: Duration, + queue: DelayQueue, + queue_capacity: NonZeroUsize, + overflow_strategy: OverflowStrategy, + condition: Option, +} + +impl Delay { + pub fn new(config: &DelayConfig, context: &TransformContext) -> crate::Result { + Ok(Self { + delay: config.delay_ms, + queue: DelayQueue::with_capacity(config.queue_capacity.get()), + queue_capacity: config.queue_capacity, + overflow_strategy: config.overflow_strategy, + condition: config + .condition + .as_ref() + .map(|c| c.build(&context.enrichment_tables, &context.metrics_storage)) + .transpose()?, + }) + } + + fn check_condition(&self, event: Event, first: bool) -> (bool, Event) { + if let Some(condition) = self.condition.as_ref() { + condition.check(event) + } else { + // If this is the first check, we need to ensure at least one delay is + // done if no condition is configured + (!first, event) + } + } +} + +impl TaskTransform for Delay { + fn transform( + mut self: Box, + mut input_rx: Pin + Send>>, + ) -> Pin + Send>> + where + Self: 'static, + { + Box::pin(stream! { + let mut done = false; + loop { + if done && self.queue.is_empty() { + break; + } + tokio::select! { + biased; + + Some(res) = self.queue.next() => { + let event = res.into_inner(); + let (result, event) = self.check_condition(event, false); + if result { + yield event; + } else { + self.queue.insert(event, self.delay); + } + if done && self.queue.is_empty() { + break; + } + }, + + maybe_event = input_rx.next(), if !done => { + match maybe_event { + None => { + done = true; + } + Some(event) => { + let (result, event) = self.check_condition(event, true); + if result { + yield event + } else { + if self.queue_capacity.get() <= self.queue.len() { + match self.overflow_strategy { + OverflowStrategy::Block => { + while self.queue_capacity.get() <= self.queue.len() && let Some(next) = self.queue.next().await { + let event = next.into_inner(); + let (result, event) = self.check_condition(event, false); + if result { + yield event; + } else { + self.queue.insert(event, self.delay); + } + } + }, + OverflowStrategy::DropNewest => { + emit!(ComponentEventsDropped:: { + count: 1, + reason: "Queue is full and overflow strategy is drop_newest", + }); + continue; + } + OverflowStrategy::Forward => { + yield event; + continue; + } + } + } + self.queue.insert(event, self.delay); + } + } + } + }, + } + } + }) + } +} + +#[derive(Debug, Snafu)] +pub enum BuildError { + #[snafu(display("The delay duration must not be zero"))] + ZeroDelayDuration, +} + +#[cfg(test)] +mod tests { + use indoc::indoc; + use std::task::Poll; + + use futures::SinkExt; + use vector_lib::event::TraceEvent; + + use super::*; + use crate::event::LogEvent; + + #[test] + fn generate_config() { + crate::test_util::test_generate_config::(); + } + + #[tokio::test] + async fn delay_events() { + let config = toml::from_str::(indoc! {" + delay_ms = 200 + "}) + .unwrap(); + + let delay = + Transform::event_task(Delay::new(&config, &TransformContext::default()).unwrap()); + + let delay = delay.into_task(); + + let (mut tx, rx) = futures::channel::mpsc::channel(10); + let mut out_stream = delay.transform_events(Box::pin(rx)); + + tx.send(LogEvent::default().into()).await.unwrap(); + + // We should be pending, because we are now waiting for the delay + assert_eq!(Poll::Pending, futures::poll!(out_stream.next())); + + // Wait long enough for delay to end + tokio::time::sleep(Duration::from_secs_f64(0.3)).await; + + if !matches!(futures::poll!(out_stream.next()), Poll::Ready(Some(_event))) { + panic!("Unexpectedly received None or Pending in output stream"); + } + } + + #[tokio::test] + async fn delay_events_at_capacity_drop_newest() { + let config = toml::from_str::(indoc! {r#" + delay_ms = 200 + queue_capacity = 1 + overflow_strategy = "drop_newest" + "#}) + .unwrap(); + + let delay = + Transform::event_task(Delay::new(&config, &TransformContext::default()).unwrap()); + + let delay = delay.into_task(); + + let (mut tx, rx) = futures::channel::mpsc::channel(10); + let mut out_stream = delay.transform_events(Box::pin(rx)); + + tx.send(LogEvent::default().into()).await.unwrap(); + tx.send(TraceEvent::default().into()).await.unwrap(); + + // We should be pending, because we are now waiting for the delay + assert_eq!(Poll::Pending, futures::poll!(out_stream.next())); + + // Wait long enough for delay to end + tokio::time::sleep(Duration::from_secs_f64(0.3)).await; + + let Poll::Ready(Some(event)) = futures::poll!(out_stream.next()) else { + panic!("Unexpectedly received None or Pending in output stream"); + }; + assert!(event.try_into_log().is_some()); + + // We should be pending, because trace event should have been dropped + assert_eq!(Poll::Pending, futures::poll!(out_stream.next())); + } + + #[tokio::test] + async fn delay_events_at_capacity_pass() { + let config = toml::from_str::(indoc! {r#" + delay_ms = 200 + queue_capacity = 1 + overflow_strategy = "forward" + "#}) + .unwrap(); + + let delay = + Transform::event_task(Delay::new(&config, &TransformContext::default()).unwrap()); + + let delay = delay.into_task(); + + let (mut tx, rx) = futures::channel::mpsc::channel(10); + let mut out_stream = delay.transform_events(Box::pin(rx)); + + tx.send(LogEvent::default().into()).await.unwrap(); + tx.send(TraceEvent::default().into()).await.unwrap(); + + // First event should be trace, because it is passed right away before delay + let Poll::Ready(Some(event)) = futures::poll!(out_stream.next()) else { + panic!("Unexpectedly received None or Pending in output stream"); + }; + assert!(event.try_into_trace().is_some()); + + // We should be pending, because we are now waiting for the delay + assert_eq!(Poll::Pending, futures::poll!(out_stream.next())); + + // Wait long enough for delay to end + tokio::time::sleep(Duration::from_secs_f64(0.3)).await; + + let Poll::Ready(Some(event)) = futures::poll!(out_stream.next()) else { + panic!("Unexpectedly received None or Pending in output stream"); + }; + assert!(event.try_into_log().is_some()); + } +} diff --git a/src/transforms/mod.rs b/src/transforms/mod.rs index e4f6671828807..2a9f606df3c6a 100644 --- a/src/transforms/mod.rs +++ b/src/transforms/mod.rs @@ -11,6 +11,8 @@ pub mod sample; pub mod aggregate; #[cfg(feature = "transforms-aws_ec2_metadata")] pub mod aws_ec2_metadata; +#[cfg(feature = "transforms-delay")] +pub mod delay; #[cfg(feature = "transforms-exclusive-route")] mod exclusive_route; #[cfg(feature = "transforms-filter")] diff --git a/src/transforms/sample/config.rs b/src/transforms/sample/config.rs index 209d2d655e7c9..1b5f7426d86a3 100644 --- a/src/transforms/sample/config.rs +++ b/src/transforms/sample/config.rs @@ -6,7 +6,7 @@ use vector_lib::{ }; use vrl::value::Kind; -use super::transform::{Sample, SampleMode}; +use super::transform::{DynamicSampleFields, Sample, SampleMode}; use crate::{ conditions::AnyCondition, config::{ @@ -29,10 +29,23 @@ pub enum SampleError { #[snafu(display("Only non-zero numbers are allowed values for `rate`"))] InvalidRate, + #[snafu(display("Only one value can be provided for either 'rate' or 'ratio', but not both"))] + InvalidStaticConfiguration, + + #[snafu(display( + "Only one value can be provided for either 'ratio_field' or 'rate_field', but not both" + ))] + InvalidDynamicConfiguration, + + #[snafu(display( + "Exactly one value must be provided for either 'rate' or 'ratio' to configure static sampling" + ))] + MissingStaticConfiguration, + #[snafu(display( - "Exactly one value must be provided for either 'rate' or 'ratio', but not both" + "'key_field' cannot be combined with 'ratio_field' or 'rate_field' because dynamic values can vary per event and break key-based coherence" ))] - InvalidConfiguration, + InvalidKeyFieldDynamicCombination, } /// Configuration for the `sample` transform. @@ -61,6 +74,24 @@ pub struct SampleConfig { #[configurable(validation(range(min = 0.0, max = 1.0)))] pub ratio: Option, + /// The event field whose numeric value is used as the sampling ratio on a per-event basis. + /// + /// Accepts integer, floating point, or string values that parse as a number. The value must be + /// in `(0, 1]` to be considered valid (for example, `0.25` keeps 25%). If the field is missing + /// or invalid, static sampling settings (`rate` or `ratio`) are used as a fallback. + /// This option cannot be used together with `rate_field`. + #[configurable(metadata(docs::examples = "sample_rate"))] + pub ratio_field: Option, + + /// The event field whose integer value is used as the sampling rate on a per-event basis, expressed as `1/N`. + /// + /// Accepts an integer, or a string that parses as a positive integer; floating point values + /// are rejected. The value must be a positive integer to be considered valid. If the field is + /// missing or invalid, static sampling settings (`rate` or `ratio`) are used as a fallback. + /// This option cannot be used together with `ratio_field`. + #[configurable(metadata(docs::examples = "sample_rate_n"))] + pub rate_field: Option, + /// The name of the field whose value is hashed to determine if the event should be /// sampled. /// @@ -72,6 +103,8 @@ pub struct SampleConfig { /// /// This can be useful to, for example, ensure that all logs for a given transaction are /// sampled together, but that overall `1/N` transactions are sampled. + /// + /// This option cannot be combined with `ratio_field` or `rate_field`. #[configurable(metadata(docs::examples = "message"))] pub key_field: Option, @@ -84,6 +117,9 @@ pub struct SampleConfig { /// /// If left unspecified, or if the event doesn't have `group_by`, then the event is not /// sampled separately. + /// + /// This can also be used with `ratio_field` or `rate_field` to apply dynamic sampling + /// independently per rendered group value. #[configurable(metadata( docs::examples = "{{ service }}", docs::examples = "{{ hostname }}-{{ service }}" @@ -96,6 +132,18 @@ pub struct SampleConfig { impl SampleConfig { fn sample_rate(&self) -> Result { + if self.ratio_field.is_some() && self.rate_field.is_some() { + return Err(SampleError::InvalidDynamicConfiguration); + } + + if self.key_field.is_some() && (self.ratio_field.is_some() || self.rate_field.is_some()) { + return Err(SampleError::InvalidKeyFieldDynamicCombination); + } + + if self.rate.is_some() && self.ratio.is_some() { + return Err(SampleError::InvalidStaticConfiguration); + } + match (self.rate, self.ratio) { (None, Some(ratio)) => { if ratio <= 0.0 { @@ -111,7 +159,8 @@ impl SampleConfig { Ok(SampleMode::new_rate(rate)) } } - _ => Err(SampleError::InvalidConfiguration), + (None, None) => Err(SampleError::MissingStaticConfiguration), + _ => Err(SampleError::InvalidStaticConfiguration), } } } @@ -121,6 +170,8 @@ impl GenerateConfig for SampleConfig { toml::Value::try_from(Self { rate: None, ratio: Some(0.1), + ratio_field: None, + rate_field: None, key_field: None, group_by: None, exclude: None::, @@ -134,19 +185,37 @@ impl GenerateConfig for SampleConfig { #[typetag::serde(name = "sample")] impl TransformConfig for SampleConfig { async fn build(&self, context: &TransformContext) -> crate::Result { - Ok(Transform::function(Sample::new( - Self::NAME.to_string(), - self.sample_rate()?, - self.key_field.clone(), - self.group_by.clone(), - self.exclude - .as_ref() - .map(|condition| { - condition.build(&context.enrichment_tables, &context.metrics_storage) - }) - .transpose()?, - self.sample_rate_key.clone(), - ))) + let sample_mode = self.sample_rate()?; + let exclude = self + .exclude + .as_ref() + .map(|condition| condition.build(&context.enrichment_tables, &context.metrics_storage)) + .transpose()?; + + let sample = if self.ratio_field.is_some() || self.rate_field.is_some() { + Sample::new_with_dynamic( + Self::NAME.to_string(), + sample_mode, + DynamicSampleFields { + ratio_field: self.ratio_field.clone(), + rate_field: self.rate_field.clone(), + }, + self.group_by.clone(), + exclude, + self.sample_rate_key.clone(), + ) + } else { + Sample::new( + Self::NAME.to_string(), + sample_mode, + self.key_field.clone(), + self.group_by.clone(), + exclude, + self.sample_rate_key.clone(), + ) + }; + + Ok(Transform::function(sample)) } fn input(&self) -> Input { @@ -191,10 +260,100 @@ pub fn default_sample_rate_key() -> OptionalValuePath { #[cfg(test)] mod tests { - use crate::transforms::sample::config::SampleConfig; + use crate::{ + config::TransformConfig, + transforms::sample::config::{SampleConfig, SampleError}, + }; #[test] fn generate_config() { crate::test_util::test_generate_config::(); } + + #[test] + fn rejects_dynamic_ratio_only_configuration() { + let config = SampleConfig { + rate: None, + ratio: None, + ratio_field: Some("sample_rate".to_string()), + rate_field: None, + key_field: None, + sample_rate_key: super::default_sample_rate_key(), + group_by: None, + exclude: None, + }; + + let err = config.sample_rate().unwrap_err(); + assert!(matches!(err, SampleError::MissingStaticConfiguration)); + } + + #[test] + fn rejects_dynamic_rate_only_configuration() { + let config = SampleConfig { + rate: None, + ratio: None, + ratio_field: None, + rate_field: Some("sample_rate_n".to_string()), + key_field: None, + sample_rate_key: super::default_sample_rate_key(), + group_by: None, + exclude: None, + }; + + let err = config.sample_rate().unwrap_err(); + assert!(matches!(err, SampleError::MissingStaticConfiguration)); + } + + #[test] + fn validates_static_with_dynamic_configuration() { + let config = SampleConfig { + rate: Some(10), + ratio: None, + ratio_field: None, + rate_field: Some("sample_rate_n".to_string()), + key_field: None, + sample_rate_key: super::default_sample_rate_key(), + group_by: None, + exclude: None, + }; + + assert!(config.validate(&crate::schema::Definition::any()).is_ok()); + } + + #[test] + fn rejects_both_dynamic_fields_configuration() { + let config = SampleConfig { + rate: Some(10), + ratio: None, + ratio_field: Some("sample_rate".to_string()), + rate_field: Some("sample_rate_n".to_string()), + key_field: None, + sample_rate_key: super::default_sample_rate_key(), + group_by: None, + exclude: None, + }; + + let err = config.sample_rate().unwrap_err(); + assert!(matches!(err, SampleError::InvalidDynamicConfiguration)); + } + + #[test] + fn rejects_key_field_with_dynamic_configuration() { + let config = SampleConfig { + rate: Some(10), + ratio: None, + ratio_field: Some("sample_ratio".to_string()), + rate_field: None, + key_field: Some("trace_id".to_string()), + sample_rate_key: super::default_sample_rate_key(), + group_by: None, + exclude: None, + }; + + let err = config.sample_rate().unwrap_err(); + assert!(matches!( + err, + SampleError::InvalidKeyFieldDynamicCombination + )); + } } diff --git a/src/transforms/sample/tests.rs b/src/transforms/sample/tests.rs index c705ff2cc4f1f..e6fec5c12a35d 100644 --- a/src/transforms/sample/tests.rs +++ b/src/transforms/sample/tests.rs @@ -1,4 +1,5 @@ use approx::assert_relative_eq; +use indoc::indoc; use tokio::sync::mpsc; use tokio_stream::wrappers::ReceiverStream; use vector_lib::lookup::lookup_v2::OptionalValuePath; @@ -14,7 +15,7 @@ use crate::{ FunctionTransform, OutputBuffer, sample::{ config::{SampleConfig, default_sample_rate_key}, - transform::{Sample, SampleMode}, + transform::{DynamicSampleFields, Sample, SampleMode}, }, test::{create_topology, transform_one}, }, @@ -26,6 +27,8 @@ async fn emits_internal_events() { let config = SampleConfig { rate: None, ratio: Some(1.0), + ratio_field: None, + rate_field: None, key_field: None, group_by: None, exclude: None, @@ -324,6 +327,330 @@ fn sample_at_rates_higher_then_half() { } } +#[test] +fn dynamic_ratio_field_overrides_static_ratio() { + let mut sampler = Sample::new_with_dynamic( + "sample".to_string(), + SampleMode::new_ratio(0.1), + DynamicSampleFields { + ratio_field: Some("dynamic_ratio".to_string()), + rate_field: None, + }, + None, + None, + default_sample_rate_key(), + ); + + let mut event = Event::Log(LogEvent::from("hello")); + let log = event.as_mut_log(); + log.insert("dynamic_ratio", 1.0); + + let output = transform_one(&mut sampler, event).expect("event should be sampled"); + assert_eq!(output.as_log()["sample_rate"], "1".into()); +} + +#[test] +fn dynamic_ratio_field_falls_back_to_static_ratio_when_missing() { + let mut sampler = Sample::new_with_dynamic( + "sample".to_string(), + SampleMode::new_ratio(1.0), + DynamicSampleFields { + ratio_field: Some("dynamic_ratio".to_string()), + rate_field: None, + }, + None, + None, + default_sample_rate_key(), + ); + + let event = Event::Log(LogEvent::from("hello")); + let output = transform_one(&mut sampler, event).expect("event should be sampled"); + assert_eq!(output.as_log()["sample_rate"], "1".into()); +} + +#[test] +fn dynamic_rate_field_overrides_static_ratio() { + let mut sampler = Sample::new_with_dynamic( + "sample".to_string(), + SampleMode::new_ratio(0.0), + DynamicSampleFields { + ratio_field: None, + rate_field: Some("dynamic_rate".to_string()), + }, + None, + None, + default_sample_rate_key(), + ); + + let mut event = Event::Log(LogEvent::from("hello")); + let log = event.as_mut_log(); + log.insert("dynamic_rate", 1); + + let output = transform_one(&mut sampler, event).expect("event should be sampled"); + assert_eq!(output.as_log()["sample_rate"], "1".into()); +} + +#[test] +fn dynamic_rate_field_falls_back_to_static_ratio_when_missing() { + let mut sampler = Sample::new_with_dynamic( + "sample".to_string(), + SampleMode::new_ratio(1.0), + DynamicSampleFields { + ratio_field: None, + rate_field: Some("dynamic_rate".to_string()), + }, + None, + None, + default_sample_rate_key(), + ); + + let event = Event::Log(LogEvent::from("hello")); + let output = transform_one(&mut sampler, event).expect("event should be sampled"); + assert_eq!(output.as_log()["sample_rate"], "1".into()); +} + +#[test] +fn dynamic_rate_field_rejects_float_and_falls_back_to_static_ratio() { + let mut sampler = Sample::new_with_dynamic( + "sample".to_string(), + SampleMode::new_ratio(1.0), + DynamicSampleFields { + ratio_field: None, + rate_field: Some("dynamic_rate".to_string()), + }, + None, + None, + default_sample_rate_key(), + ); + + let mut event = Event::Log(LogEvent::from("hello")); + let log = event.as_mut_log(); + log.insert("dynamic_rate", 2.0); + + let output = transform_one(&mut sampler, event).expect("event should be sampled"); + assert_eq!(output.as_log()["sample_rate"], "1".into()); +} + +#[test] +fn dynamic_ratio_honors_group_by_key() { + let ratio = 0.5_f64; + let events_per_service = 200; + let mut sampler = Sample::new_with_dynamic( + "sample".to_string(), + SampleMode::new_ratio(0.0), + DynamicSampleFields { + ratio_field: Some("dynamic_ratio".to_string()), + rate_field: None, + }, + Some(Template::try_from("{{ service }}").unwrap()), + None, + default_sample_rate_key(), + ); + + let mut sampled_service_a = 0; + let mut sampled_service_b = 0; + for _ in 0..events_per_service { + for service in ["service-a", "service-b"] { + let mut event = Event::Log(LogEvent::from("hello")); + let log = event.as_mut_log(); + log.insert("service", service); + log.insert("dynamic_ratio", ratio); + if let Some(output) = transform_one(&mut sampler, event) { + assert_eq!(output.as_log()["sample_rate"], "0.5".into()); + if service == "service-a" { + sampled_service_a += 1; + } else { + sampled_service_b += 1; + } + } + } + } + + assert!( + (60..140).contains(&sampled_service_a), + "service-a sampled {} out of {events_per_service}", + sampled_service_a + ); + assert!( + (60..140).contains(&sampled_service_b), + "service-b sampled {} out of {events_per_service}", + sampled_service_b + ); +} + +#[test] +fn dynamic_rate_honors_group_by_key() { + let rate = 2_i64; + let events_per_service = 200; + let mut sampler = Sample::new_with_dynamic( + "sample".to_string(), + SampleMode::new_ratio(0.0), + DynamicSampleFields { + ratio_field: None, + rate_field: Some("dynamic_rate".to_string()), + }, + Some(Template::try_from("{{ service }}").unwrap()), + None, + default_sample_rate_key(), + ); + + let mut sampled_service_a = 0; + let mut sampled_service_b = 0; + for _ in 0..events_per_service { + for service in ["service-a", "service-b"] { + let mut event = Event::Log(LogEvent::from("hello")); + let log = event.as_mut_log(); + log.insert("service", service); + log.insert("dynamic_rate", rate); + if let Some(output) = transform_one(&mut sampler, event) { + assert_eq!(output.as_log()["sample_rate"], "2".into()); + if service == "service-a" { + sampled_service_a += 1; + } else { + sampled_service_b += 1; + } + } + } + } + + assert!( + (60..140).contains(&sampled_service_a), + "service-a sampled {} out of {events_per_service}", + sampled_service_a + ); + assert!( + (60..140).contains(&sampled_service_b), + "service-b sampled {} out of {events_per_service}", + sampled_service_b + ); +} + +#[test] +fn dynamic_ratio_group_by_samples_mixed_ratios_at_expected_rates() { + let events_per_ratio = 500; + let mut sampler = Sample::new_with_dynamic( + "sample".to_string(), + SampleMode::new_ratio(0.0), + DynamicSampleFields { + ratio_field: Some("dynamic_ratio".to_string()), + rate_field: None, + }, + Some(Template::try_from("{{ service }}").unwrap()), + None, + default_sample_rate_key(), + ); + + let mut sampled_low_ratio = 0; + let mut sampled_high_ratio = 0; + for _ in 0..events_per_ratio { + for (ratio, is_low_ratio) in [(0.25_f64, true), (0.75_f64, false)] { + let mut event = Event::Log(LogEvent::from("hello")); + let log = event.as_mut_log(); + log.insert("service", "service-a"); + log.insert("dynamic_ratio", ratio); + if let Some(output) = transform_one(&mut sampler, event) { + assert_eq!(output.as_log()["sample_rate"], ratio.to_string().into()); + if is_low_ratio { + sampled_low_ratio += 1; + } else { + sampled_high_ratio += 1; + } + } + } + } + + assert!( + (80..220).contains(&sampled_low_ratio), + "ratio=0.25 sampled {} out of {events_per_ratio}", + sampled_low_ratio + ); + assert!( + (300..450).contains(&sampled_high_ratio), + "ratio=0.75 sampled {} out of {events_per_ratio}", + sampled_high_ratio + ); + assert!( + sampled_high_ratio > sampled_low_ratio, + "ratio=0.75 sampled {sampled_high_ratio}, ratio=0.25 sampled {sampled_low_ratio}" + ); +} + +#[test] +fn dynamic_rate_group_by_samples_mixed_rates_at_expected_rates() { + let events_per_rate = 600; + let mut sampler = Sample::new_with_dynamic( + "sample".to_string(), + SampleMode::new_ratio(0.0), + DynamicSampleFields { + ratio_field: None, + rate_field: Some("dynamic_rate".to_string()), + }, + Some(Template::try_from("{{ service }}").unwrap()), + None, + default_sample_rate_key(), + ); + + let mut sampled_rate_2 = 0; + let mut sampled_rate_3 = 0; + for _ in 0..events_per_rate { + for rate in [2_i64, 3_i64] { + let mut event = Event::Log(LogEvent::from("hello")); + let log = event.as_mut_log(); + log.insert("service", "service-a"); + log.insert("dynamic_rate", rate); + if let Some(output) = transform_one(&mut sampler, event) { + assert_eq!(output.as_log()["sample_rate"], rate.to_string().into()); + if rate == 2 { + sampled_rate_2 += 1; + } else { + sampled_rate_3 += 1; + } + } + } + } + + assert!( + (220..380).contains(&sampled_rate_2), + "rate=2 sampled {} out of {events_per_rate}", + sampled_rate_2 + ); + assert!( + (120..280).contains(&sampled_rate_3), + "rate=3 sampled {} out of {events_per_rate}", + sampled_rate_3 + ); + assert!( + sampled_rate_2 > sampled_rate_3, + "rate=2 sampled {sampled_rate_2}, rate=3 sampled {sampled_rate_3}" + ); +} + +#[tokio::test] +async fn dynamic_field_config_drives_dynamic_sampling() { + assert_transform_compliance(async move { + let config: SampleConfig = serde_yaml::from_str(indoc! {r#" + ratio_field: dynamic_ratio + ratio: 1.0 + "#}) + .expect("config should deserialize"); + + let (tx, rx) = mpsc::channel(1); + let (topology, mut out) = create_topology(ReceiverStream::new(rx), config).await; + + let mut log = LogEvent::from("hello"); + log.insert("dynamic_ratio", 1.0); + tx.send(log.into()).await.unwrap(); + + let event = out.recv().await.expect("event should be sampled"); + assert_eq!(event.as_log()["sample_rate"], "1".into()); + + drop(tx); + topology.stop().await; + assert_eq!(out.recv().await, None); + }) + .await +} + fn condition_contains(key: &str, needle: &str) -> Condition { let vrl_config = VrlConfig { source: format!(r#"contains!(."{key}", "{needle}")"#), diff --git a/src/transforms/sample/transform.rs b/src/transforms/sample/transform.rs index 74786a43fdc51..00728b076e68c 100644 --- a/src/transforms/sample/transform.rs +++ b/src/transforms/sample/transform.rs @@ -1,4 +1,9 @@ -use std::{collections::HashMap, fmt}; +use std::{ + collections::HashMap, + fmt, + hash::{Hash, Hasher}, + num::NonZeroU64, +}; use vector_lib::{ config::LegacyKey, @@ -91,6 +96,26 @@ impl SampleMode { } } +enum EventSampleMode { + Ratio(f64), + Rate(NonZeroU64), +} + +impl EventSampleMode { + fn sample_rate_label(&self) -> String { + match self { + Self::Ratio(ratio) => ratio.to_string(), + Self::Rate(rate) => rate.to_string(), + } + } +} + +#[derive(Clone, Default)] +pub struct DynamicSampleFields { + pub ratio_field: Option, + pub rate_field: Option, +} + impl fmt::Display for SampleMode { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { // Avoids the print of an additional '.0' which was not performed in the previous @@ -102,12 +127,24 @@ impl fmt::Display for SampleMode { } } +#[derive(Clone)] +pub enum SampleKeySource { + Static { + key_field: Option, + group_by: Option