From b6d820732f435569aafe1aaa5861245b7b4a9917 Mon Sep 17 00:00:00 2001 From: vkalintiris Date: Thu, 18 Jun 2026 21:49:42 +0300 Subject: [PATCH 1/6] Bump rand to 0.9.4 in Cargo.lock (#22772) --- src/crates/Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/crates/Cargo.lock b/src/crates/Cargo.lock index e69b3933ad17e9..5ed97e42acd2dd 100644 --- a/src/crates/Cargo.lock +++ b/src/crates/Cargo.lock @@ -4313,7 +4313,7 @@ dependencies = [ "notify", "num-bigint", "parking_lot", - "rand 0.9.2", + "rand 0.9.4", "regex", "rustc-hash", "ruzstd", From 7b5d4f410304063e924b03ddec4c0644bc59950e Mon Sep 17 00:00:00 2001 From: Ilya Mashchenko Date: Thu, 18 Jun 2026 22:28:26 +0300 Subject: [PATCH 2/6] chore(go.d/snmp-topology): extract topology Function package (#22774) refactor(go.d/snmp-topology): extract topology Function package --- .../go.d/collector/snmp_topology/collector.go | 18 -- .../go.d/collector/snmp_topology/func_deps.go | 70 ++++++ .../collector/snmp_topology/func_deps_test.go | 38 +++ .../collector/snmp_topology/func_topology.go | 43 ---- .../snmp_topology/func_topology_depth.go | 35 --- .../snmp_topology/func_topology_handler.go | 100 -------- .../func_topology_managed_focus.go | 101 -------- .../snmp_topology/func_topology_options.go | 46 ---- .../func_topology_presentation.go | 26 -- .../func_topology_presentation_params.go | 117 --------- .../func_topology_presentation_test.go | 20 +- .../snmp_topology/func_topology_test.go | 131 +++------- .../snmp_topology/snmptopologyfunc/deps.go | 27 ++ .../snmptopologyfunc/managed_focus.go | 100 ++++++++ .../snmp_topology/snmptopologyfunc/options.go | 97 ++++++++ .../snmptopologyfunc/presentation.go | 35 +++ .../snmptopologyfunc/presentation_params.go | 143 +++++++++++ .../snmptopologyfunc/topology.go | 67 +++++ .../snmptopologyfunc/topology_test.go | 232 ++++++++++++++++++ .../snmp_topology/topology_query_options.go | 151 ++++++++++++ 20 files changed, 999 insertions(+), 598 deletions(-) create mode 100644 src/go/plugin/go.d/collector/snmp_topology/func_deps.go create mode 100644 src/go/plugin/go.d/collector/snmp_topology/func_deps_test.go delete mode 100644 src/go/plugin/go.d/collector/snmp_topology/func_topology.go delete mode 100644 src/go/plugin/go.d/collector/snmp_topology/func_topology_depth.go delete mode 100644 src/go/plugin/go.d/collector/snmp_topology/func_topology_handler.go delete mode 100644 src/go/plugin/go.d/collector/snmp_topology/func_topology_managed_focus.go delete mode 100644 src/go/plugin/go.d/collector/snmp_topology/func_topology_options.go delete mode 100644 src/go/plugin/go.d/collector/snmp_topology/func_topology_presentation.go delete mode 100644 src/go/plugin/go.d/collector/snmp_topology/func_topology_presentation_params.go create mode 100644 src/go/plugin/go.d/collector/snmp_topology/snmptopologyfunc/deps.go create mode 100644 src/go/plugin/go.d/collector/snmp_topology/snmptopologyfunc/managed_focus.go create mode 100644 src/go/plugin/go.d/collector/snmp_topology/snmptopologyfunc/options.go create mode 100644 src/go/plugin/go.d/collector/snmp_topology/snmptopologyfunc/presentation.go create mode 100644 src/go/plugin/go.d/collector/snmp_topology/snmptopologyfunc/presentation_params.go create mode 100644 src/go/plugin/go.d/collector/snmp_topology/snmptopologyfunc/topology.go create mode 100644 src/go/plugin/go.d/collector/snmp_topology/snmptopologyfunc/topology_test.go create mode 100644 src/go/plugin/go.d/collector/snmp_topology/topology_query_options.go diff --git a/src/go/plugin/go.d/collector/snmp_topology/collector.go b/src/go/plugin/go.d/collector/snmp_topology/collector.go index f96c4ccad7760a..0c7aeb35ec16cb 100644 --- a/src/go/plugin/go.d/collector/snmp_topology/collector.go +++ b/src/go/plugin/go.d/collector/snmp_topology/collector.go @@ -14,7 +14,6 @@ import ( "github.com/gosnmp/gosnmp" "github.com/netdata/netdata/go/plugins/logger" - "github.com/netdata/netdata/go/plugins/pkg/funcapi" "github.com/netdata/netdata/go/plugins/pkg/metrix" "github.com/netdata/netdata/go/plugins/plugin/framework/collectorapi" "github.com/netdata/netdata/go/plugins/plugin/go.d/collector/snmp/ddsnmp" @@ -450,20 +449,3 @@ func newSNMPClientFromDeviceInfo(newClient func() gosnmp.Handler, dev ddsnmp.Dev return client, nil } - -func topologyMethods() []funcapi.MethodConfig { - return []funcapi.MethodConfig{ - topologyMethodConfig(), - } -} - -func topologyFunctionHandler(job collectorapi.RuntimeJob) funcapi.MethodHandler { - if job == nil { - return nil - } - coll, ok := job.Collector().(*Collector) - if !ok || coll == nil { - return nil - } - return &funcTopology{registry: coll.topologyRegistry} -} diff --git a/src/go/plugin/go.d/collector/snmp_topology/func_deps.go b/src/go/plugin/go.d/collector/snmp_topology/func_deps.go new file mode 100644 index 00000000000000..6997a9000198e2 --- /dev/null +++ b/src/go/plugin/go.d/collector/snmp_topology/func_deps.go @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package snmptopology + +import ( + "github.com/netdata/netdata/go/plugins/pkg/funcapi" + topologyv1 "github.com/netdata/netdata/go/plugins/pkg/topology/v1" + "github.com/netdata/netdata/go/plugins/plugin/framework/collectorapi" + "github.com/netdata/netdata/go/plugins/plugin/go.d/collector/snmp_topology/snmptopologyfunc" +) + +type funcDepsAdapter struct { + registry *topologyRegistry +} + +func (a funcDepsAdapter) Snapshot(options snmptopologyfunc.QueryOptions) (topologyv1.Data, bool, error) { + if a.registry == nil { + return topologyv1.Data{}, false, nil + } + + data, ok := a.registry.snapshotWithOptions(topologyQueryOptions{ + CollapseActorsByIP: options.CollapseActorsByIP, + EliminateNonIPInferred: options.EliminateNonIPInferred, + MapType: options.MapType, + InferenceStrategy: options.InferenceStrategy, + ManagedDeviceFocus: options.ManagedDeviceFocus, + Depth: options.Depth, + ResolveDNSName: resolveTopologyReverseDNSNameNoop, // never block on network I/O + }) + if !ok { + return topologyv1.Data{}, false, nil + } + + payload, err := snmpTopologyToV1(data) + if err != nil { + return topologyv1.Data{}, false, err + } + return payload, true, nil +} + +func (a funcDepsAdapter) ManagedDeviceFocusTargets() []snmptopologyfunc.ManagedFocusTarget { + if a.registry == nil { + return nil + } + + targets := a.registry.managedDeviceFocusTargets() + out := make([]snmptopologyfunc.ManagedFocusTarget, 0, len(targets)) + for _, target := range targets { + out = append(out, snmptopologyfunc.ManagedFocusTarget{ + Value: target.Value, + Name: target.Name, + }) + } + return out +} + +func topologyMethods() []funcapi.MethodConfig { + return snmptopologyfunc.Methods() +} + +func topologyFunctionHandler(job collectorapi.RuntimeJob) funcapi.MethodHandler { + if job == nil { + return nil + } + coll, ok := job.Collector().(*Collector) + if !ok || coll == nil { + return nil + } + return snmptopologyfunc.NewHandler(funcDepsAdapter{registry: coll.topologyRegistry}) +} diff --git a/src/go/plugin/go.d/collector/snmp_topology/func_deps_test.go b/src/go/plugin/go.d/collector/snmp_topology/func_deps_test.go new file mode 100644 index 00000000000000..fa2b1d0fb0e2c6 --- /dev/null +++ b/src/go/plugin/go.d/collector/snmp_topology/func_deps_test.go @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package snmptopology + +import ( + "testing" + + topologyv1 "github.com/netdata/netdata/go/plugins/pkg/topology/v1" + "github.com/netdata/netdata/go/plugins/plugin/go.d/collector/snmp_topology/snmptopologyfunc" + "github.com/stretchr/testify/require" +) + +func TestFuncDepsAdapterSnapshotUnavailable(t *testing.T) { + tests := map[string]struct { + adapter funcDepsAdapter + }{ + "nil registry": { + adapter: funcDepsAdapter{}, + }, + "empty registry": { + adapter: funcDepsAdapter{registry: newTopologyRegistry()}, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + data, ok, err := tc.adapter.Snapshot(snmptopologyfunc.QueryOptions{}) + + require.NoError(t, err) + require.False(t, ok) + require.Equal(t, topologyv1.Data{}, data) + }) + } +} + +func TestFuncDepsAdapterManagedDeviceFocusTargetsNilRegistry(t *testing.T) { + require.Nil(t, funcDepsAdapter{}.ManagedDeviceFocusTargets()) +} diff --git a/src/go/plugin/go.d/collector/snmp_topology/func_topology.go b/src/go/plugin/go.d/collector/snmp_topology/func_topology.go deleted file mode 100644 index 43e5a3f8b26b2a..00000000000000 --- a/src/go/plugin/go.d/collector/snmp_topology/func_topology.go +++ /dev/null @@ -1,43 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -package snmptopology - -import "github.com/netdata/netdata/go/plugins/pkg/funcapi" - -// Compile-time interface check. -var _ funcapi.MethodHandler = (*funcTopology)(nil) - -type funcTopology struct { - registry *topologyRegistry -} - -const ( - topologyFunctionName = "snmp:topology:snmp" - topologyMethodID = "topology:snmp" -) - -const ( - topologyParamNodesIdentity = "nodes_identity" - topologyParamMapType = "map_type" - topologyParamInferenceStrategy = "inference_strategy" - topologyParamManagedDeviceFocus = "managed_snmp_device_focus" - topologyParamDepth = "depth" - - topologyNodesIdentityIP = "ip" - topologyNodesIdentityMAC = "mac" - - topologyMapTypeLLDPCDPManaged = "lldp_cdp_managed" - topologyMapTypeHighConfidenceInferred = "high_confidence_inferred" - topologyMapTypeAllDevicesLowConfidence = "all_devices_low_confidence" - topologyInferenceStrategyFDBMinimumKnowledge = "fdb_minimum_knowledge" - topologyInferenceStrategySTPParentTree = "stp_parent_tree" - topologyInferenceStrategyFDBPairwise = "fdb_pairwise_minimum_knowledge" - topologyInferenceStrategySTPFDBCorrelated = "stp_fdb_correlated" - topologyInferenceStrategyCDPFDBHybrid = "cdp_fdb_hybrid" - topologyManagedFocusAllDevices = "all_devices" - topologyManagedFocusIPPrefix = "ip:" - topologyDepthAll = "all" - topologyDepthMin = 0 - topologyDepthMax = 10 - topologyDepthAllInternal = -1 -) diff --git a/src/go/plugin/go.d/collector/snmp_topology/func_topology_depth.go b/src/go/plugin/go.d/collector/snmp_topology/func_topology_depth.go deleted file mode 100644 index 6425c79a196df5..00000000000000 --- a/src/go/plugin/go.d/collector/snmp_topology/func_topology_depth.go +++ /dev/null @@ -1,35 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -package snmptopology - -import ( - "strconv" - "strings" -) - -func normalizeTopologyDepth(v string) int { - value := strings.ToLower(strings.TrimSpace(v)) - if value == "" || value == topologyDepthAll { - return topologyDepthAllInternal - } - depth, err := strconv.Atoi(value) - if err != nil { - return topologyDepthAllInternal - } - if depth < topologyDepthMin { - return topologyDepthMin - } - if depth > topologyDepthMax { - return topologyDepthMax - } - return depth -} - -func isTopologyMapTypeProbable(v string) bool { - switch strings.ToLower(strings.TrimSpace(v)) { - case "", topologyMapTypeAllDevicesLowConfidence: - return true - default: - return false - } -} diff --git a/src/go/plugin/go.d/collector/snmp_topology/func_topology_handler.go b/src/go/plugin/go.d/collector/snmp_topology/func_topology_handler.go deleted file mode 100644 index d9ad19bf3fc55f..00000000000000 --- a/src/go/plugin/go.d/collector/snmp_topology/func_topology_handler.go +++ /dev/null @@ -1,100 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -package snmptopology - -import ( - "context" - "strings" - - "github.com/netdata/netdata/go/plugins/pkg/funcapi" -) - -func (f *funcTopology) MethodParams(_ context.Context, method string) ([]funcapi.ParamConfig, error) { - if method != topologyMethodID { - return nil, nil - } - - return []funcapi.ParamConfig{ - topologyNodesIdentityParamConfig(), - topologyMapTypeParamConfig(), - topologyInferenceStrategyParamConfig(), - topologyManagedFocusParamConfig(topologyManagedFocusParamOptions(f.registry)), - topologyDepthParamConfig(), - }, nil -} - -func (f *funcTopology) Cleanup(_ context.Context) {} - -func (f *funcTopology) Handle(_ context.Context, method string, params funcapi.ResolvedParams) *funcapi.FunctionResponse { - if method != topologyMethodID { - return funcapi.NotFoundResponse(method) - } - - if f.registry == nil { - return funcapi.UnavailableResponse("topology data not available yet, please retry after topology refresh") - } - - options := resolveTopologyQueryOptions(params) - options.ResolveDNSName = resolveTopologyReverseDNSNameNoop // never block on network I/O - data, ok := f.registry.snapshotWithOptions(options) - if !ok { - return funcapi.UnavailableResponse("topology data not available yet, please retry after topology refresh") - } - payload, err := snmpTopologyToV1(data) - if err != nil { - return funcapi.InternalErrorResponse("failed to build topology response: %v", err) - } - - return &funcapi.FunctionResponse{ - Status: 200, - Help: "SNMP topology and neighbor discovery data", - ResponseType: "topology", - Data: payload, - } -} - -func topologyManagedFocusParamOptions(registry *topologyRegistry) []funcapi.ParamOption { - if registry == nil { - return nil - } - - options := make([]funcapi.ParamOption, 0) - for _, target := range registry.managedDeviceFocusTargets() { - if strings.TrimSpace(target.Value) == "" { - continue - } - options = append(options, funcapi.ParamOption{ - ID: target.Value, - Name: target.Name, - }) - } - return options -} - -func resolveTopologyQueryOptions(params funcapi.ResolvedParams) topologyQueryOptions { - options := topologyQueryOptions{ - CollapseActorsByIP: true, - EliminateNonIPInferred: true, - MapType: topologyMapTypeLLDPCDPManaged, - InferenceStrategy: topologyInferenceStrategyFDBMinimumKnowledge, - ManagedDeviceFocus: topologyManagedFocusAllDevices, - Depth: topologyDepthAllInternal, - } - - if identity := normalizeTopologyNodesIdentity(params.GetOne(topologyParamNodesIdentity)); identity == topologyNodesIdentityMAC { - options.CollapseActorsByIP = false - options.EliminateNonIPInferred = false - } - if mapType := normalizeTopologyMapType(params.GetOne(topologyParamMapType)); mapType != "" { - options.MapType = mapType - } - if strategy := normalizeTopologyInferenceStrategy(params.GetOne(topologyParamInferenceStrategy)); strategy != "" { - options.InferenceStrategy = strategy - } - if focuses := normalizeTopologyManagedFocuses(params.Get(topologyParamManagedDeviceFocus)); len(focuses) > 0 { - options.ManagedDeviceFocus = formatTopologyManagedFocuses(focuses) - } - options.Depth = normalizeTopologyDepth(params.GetOne(topologyParamDepth)) - - return options -} diff --git a/src/go/plugin/go.d/collector/snmp_topology/func_topology_managed_focus.go b/src/go/plugin/go.d/collector/snmp_topology/func_topology_managed_focus.go deleted file mode 100644 index fb7c34dbbc8fa4..00000000000000 --- a/src/go/plugin/go.d/collector/snmp_topology/func_topology_managed_focus.go +++ /dev/null @@ -1,101 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -package snmptopology - -import ( - "sort" - "strings" -) - -func normalizeTopologyManagedFocus(v string) string { - value := strings.TrimSpace(v) - if value == "" { - return topologyManagedFocusAllDevices - } - return normalizeTopologyManagedFocusValue(value) -} - -func normalizeTopologyManagedFocusValue(v string) string { - value := strings.TrimSpace(v) - switch strings.ToLower(value) { - case topologyManagedFocusAllDevices: - return topologyManagedFocusAllDevices - } - if len(value) > len(topologyManagedFocusIPPrefix) && - strings.EqualFold(value[:len(topologyManagedFocusIPPrefix)], topologyManagedFocusIPPrefix) { - ip := normalizeIPAddress(strings.TrimSpace(value[len(topologyManagedFocusIPPrefix):])) - if ip == "" { - return "" - } - return topologyManagedFocusIPPrefix + ip - } - return "" -} - -func normalizeTopologyManagedFocuses(values []string) []string { - expanded := splitTopologyManagedFocusValues(values) - if len(expanded) == 0 { - return []string{topologyManagedFocusAllDevices} - } - - seen := make(map[string]struct{}, len(expanded)) - out := make([]string, 0, len(expanded)) - for _, raw := range expanded { - normalized := normalizeTopologyManagedFocusValue(raw) - if normalized == "" { - continue - } - if normalized == topologyManagedFocusAllDevices { - return []string{topologyManagedFocusAllDevices} - } - if _, ok := seen[normalized]; ok { - continue - } - seen[normalized] = struct{}{} - out = append(out, normalized) - } - - if len(out) == 0 { - return []string{topologyManagedFocusAllDevices} - } - sort.Strings(out) - return out -} - -func splitTopologyManagedFocusValues(values []string) []string { - if len(values) == 0 { - return nil - } - - out := make([]string, 0, len(values)) - for _, raw := range values { - for token := range strings.SplitSeq(raw, ",") { - token = strings.TrimSpace(token) - if token == "" { - continue - } - out = append(out, token) - } - } - return out -} - -func parseTopologyManagedFocuses(value string) []string { - if strings.TrimSpace(value) == "" { - return []string{topologyManagedFocusAllDevices} - } - return normalizeTopologyManagedFocuses(strings.Split(value, ",")) -} - -func formatTopologyManagedFocuses(values []string) string { - normalized := normalizeTopologyManagedFocuses(values) - if len(normalized) == 0 { - return topologyManagedFocusAllDevices - } - return strings.Join(normalized, ",") -} - -func isTopologyManagedFocusAllDevices(value string) bool { - normalized := parseTopologyManagedFocuses(value) - return len(normalized) == 1 && normalized[0] == topologyManagedFocusAllDevices -} diff --git a/src/go/plugin/go.d/collector/snmp_topology/func_topology_options.go b/src/go/plugin/go.d/collector/snmp_topology/func_topology_options.go deleted file mode 100644 index c59a54c1377bd7..00000000000000 --- a/src/go/plugin/go.d/collector/snmp_topology/func_topology_options.go +++ /dev/null @@ -1,46 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -package snmptopology - -import "strings" - -func normalizeTopologyNodesIdentity(v string) string { - switch strings.ToLower(strings.TrimSpace(v)) { - case "", topologyNodesIdentityIP: - return topologyNodesIdentityIP - case topologyNodesIdentityMAC: - return topologyNodesIdentityMAC - default: - return "" - } -} - -func normalizeTopologyMapType(v string) string { - switch strings.ToLower(strings.TrimSpace(v)) { - case "", topologyMapTypeLLDPCDPManaged: - return topologyMapTypeLLDPCDPManaged - case topologyMapTypeHighConfidenceInferred: - return topologyMapTypeHighConfidenceInferred - case topologyMapTypeAllDevicesLowConfidence: - return topologyMapTypeAllDevicesLowConfidence - default: - return "" - } -} - -func normalizeTopologyInferenceStrategy(v string) string { - switch strings.ToLower(strings.TrimSpace(v)) { - case "", topologyInferenceStrategyFDBMinimumKnowledge: - return topologyInferenceStrategyFDBMinimumKnowledge - case topologyInferenceStrategySTPParentTree: - return topologyInferenceStrategySTPParentTree - case topologyInferenceStrategyFDBPairwise: - return topologyInferenceStrategyFDBPairwise - case topologyInferenceStrategySTPFDBCorrelated: - return topologyInferenceStrategySTPFDBCorrelated - case topologyInferenceStrategyCDPFDBHybrid: - return topologyInferenceStrategyCDPFDBHybrid - default: - return "" - } -} diff --git a/src/go/plugin/go.d/collector/snmp_topology/func_topology_presentation.go b/src/go/plugin/go.d/collector/snmp_topology/func_topology_presentation.go deleted file mode 100644 index caeb56dab284df..00000000000000 --- a/src/go/plugin/go.d/collector/snmp_topology/func_topology_presentation.go +++ /dev/null @@ -1,26 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -package snmptopology - -import "github.com/netdata/netdata/go/plugins/pkg/funcapi" - -func topologyMethodConfig() funcapi.MethodConfig { - return funcapi.MethodConfig{ - ID: topologyMethodID, - FunctionName: topologyFunctionName, - Aliases: []string{topologyMethodID}, - Name: "Topology (SNMP)", - UpdateEvery: 10, - Help: "SNMP Layer-2 topology and neighbor discovery data", - RequireCloud: true, - ResponseType: "topology", - AgentWide: true, - RequiredParams: []funcapi.ParamConfig{ - topologyNodesIdentityParamConfig(), - topologyMapTypeParamConfig(), - topologyInferenceStrategyParamConfig(), - topologyManagedFocusParamConfig(nil), - topologyDepthParamConfig(), - }, - } -} diff --git a/src/go/plugin/go.d/collector/snmp_topology/func_topology_presentation_params.go b/src/go/plugin/go.d/collector/snmp_topology/func_topology_presentation_params.go deleted file mode 100644 index f0d97efea266e1..00000000000000 --- a/src/go/plugin/go.d/collector/snmp_topology/func_topology_presentation_params.go +++ /dev/null @@ -1,117 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -package snmptopology - -import ( - "strconv" - - "github.com/netdata/netdata/go/plugins/pkg/funcapi" -) - -func topologyNodesIdentityParamConfig() funcapi.ParamConfig { - return funcapi.ParamConfig{ - ID: topologyParamNodesIdentity, - Name: "Nodes Identity", - Help: "Choose actor identity strategy: ip (collapse by IP, remove non-IP inferred) or mac", - Selection: funcapi.ParamSelect, - Options: []funcapi.ParamOption{ - {ID: topologyNodesIdentityIP, Name: "IP", Default: true}, - {ID: topologyNodesIdentityMAC, Name: "MAC"}, - }, - } -} - -func topologyMapTypeParamConfig() funcapi.ParamConfig { - return funcapi.ParamConfig{ - ID: topologyParamMapType, - Name: "Map", - Help: "Choose topology map mode", - Selection: funcapi.ParamSelect, - Options: []funcapi.ParamOption{ - { - ID: topologyMapTypeLLDPCDPManaged, - Name: "LLDP/CDP/Managed Devices Map", - Default: true, - }, - {ID: topologyMapTypeHighConfidenceInferred, Name: "High Confidence Inferred Map"}, - { - ID: topologyMapTypeAllDevicesLowConfidence, - Name: "All Devices (Low Confidence)", - }, - }, - } -} - -func topologyInferenceStrategyParamConfig() funcapi.ParamConfig { - return funcapi.ParamConfig{ - ID: topologyParamInferenceStrategy, - Name: "Infer Strategy", - Help: "Choose the topology inference strategy", - Selection: funcapi.ParamSelect, - Options: []funcapi.ParamOption{ - { - ID: topologyInferenceStrategyFDBMinimumKnowledge, - Name: "FDB Minimum-Knowledge (Baseline)", - Default: true, - }, - { - ID: topologyInferenceStrategySTPParentTree, - Name: "STP Parent Tree", - }, - { - ID: topologyInferenceStrategyFDBPairwise, - Name: "FDB Pairwise Minimum-Knowledge", - }, - { - ID: topologyInferenceStrategySTPFDBCorrelated, - Name: "STP + FDB Correlated", - }, - { - ID: topologyInferenceStrategyCDPFDBHybrid, - Name: "CDP + FDB Hybrid", - }, - }, - } -} - -func topologyManagedFocusParamConfig(extraOptions []funcapi.ParamOption) funcapi.ParamConfig { - options := make([]funcapi.ParamOption, 0, 1+len(extraOptions)) - options = append(options, funcapi.ParamOption{ - ID: topologyManagedFocusAllDevices, - Name: "All Devices", - Default: true, - }) - options = append(options, extraOptions...) - - return funcapi.ParamConfig{ - ID: topologyParamManagedDeviceFocus, - Name: "Focus On", - Help: "Choose focus root set for depth filtering", - Selection: funcapi.ParamMultiSelect, - Options: options, - } -} - -func topologyDepthParamConfig() funcapi.ParamConfig { - options := make([]funcapi.ParamOption, 0, 1+(topologyDepthMax-topologyDepthMin+1)) - options = append(options, funcapi.ParamOption{ - ID: topologyDepthAll, - Name: "All", - Default: true, - }) - for depth := topologyDepthMin; depth <= topologyDepthMax; depth++ { - value := strconv.Itoa(depth) - options = append(options, funcapi.ParamOption{ - ID: value, - Name: value, - }) - } - - return funcapi.ParamConfig{ - ID: topologyParamDepth, - Name: "Focus Depth", - Help: "Limit topology expansion hops from focus roots", - Selection: funcapi.ParamSelect, - Options: options, - } -} diff --git a/src/go/plugin/go.d/collector/snmp_topology/func_topology_presentation_test.go b/src/go/plugin/go.d/collector/snmp_topology/func_topology_presentation_test.go index c54bd80e700a0b..1fcb06fd117aa5 100644 --- a/src/go/plugin/go.d/collector/snmp_topology/func_topology_presentation_test.go +++ b/src/go/plugin/go.d/collector/snmp_topology/func_topology_presentation_test.go @@ -5,21 +5,13 @@ package snmptopology import ( "testing" + "github.com/netdata/netdata/go/plugins/pkg/funcapi" "github.com/netdata/netdata/go/plugins/plugin/framework/collectorapi" "github.com/netdata/netdata/go/plugins/plugin/go.d/collector/snmp/ddsnmp" + "github.com/netdata/netdata/go/plugins/plugin/go.d/collector/snmp_topology/snmptopologyfunc" "github.com/stretchr/testify/require" ) -func TestSNMPTopologyMethodConfigDoesNotUseLegacyPresentation(t *testing.T) { - method := topologyMethodConfig() - - require.Equal(t, topologyMethodID, method.ID) - require.Equal(t, topologyFunctionName, method.FunctionName) - require.Equal(t, []string{topologyMethodID}, method.Aliases) - require.Equal(t, "topology", method.ResponseType) - require.Nil(t, method.Presentation()) -} - func TestSNMPTopologyCreatorOwnsTopologyFunction(t *testing.T) { creator := newCreator(ddsnmp.NewDeviceStore(), NewTrapEnrichmentHandle()) require.Nil(t, creator.Create) @@ -32,15 +24,15 @@ func TestSNMPTopologyCreatorOwnsTopologyFunction(t *testing.T) { methods := creator.Methods() require.Len(t, methods, 1) - require.Equal(t, topologyMethodID, methods[0].ID) - require.Equal(t, topologyFunctionName, methods[0].FunctionName) + require.Equal(t, snmptopologyfunc.MethodID, methods[0].ID) + require.Equal(t, snmptopologyfunc.FunctionName, methods[0].FunctionName) require.True(t, methods[0].AgentWide) coll := newTestSNMPTopologyCollector() handler := creator.MethodHandler(&topologyRuntimeJobForTest{collector: coll}) - require.IsType(t, &funcTopology{}, handler) - require.Same(t, coll.topologyRegistry, handler.(*funcTopology).registry) + require.Implements(t, (*funcapi.MethodHandler)(nil), handler) require.Nil(t, creator.MethodHandler(nil)) + require.Nil(t, topologyFunctionHandler(nil)) } func TestSNMPTopologyCreatorRequiresSharedDependencies(t *testing.T) { diff --git a/src/go/plugin/go.d/collector/snmp_topology/func_topology_test.go b/src/go/plugin/go.d/collector/snmp_topology/func_topology_test.go index 61957e4af669f3..870e6bfce914ec 100644 --- a/src/go/plugin/go.d/collector/snmp_topology/func_topology_test.go +++ b/src/go/plugin/go.d/collector/snmp_topology/func_topology_test.go @@ -12,64 +12,13 @@ import ( "github.com/netdata/netdata/go/plugins/pkg/funcapi" topologyv1 "github.com/netdata/netdata/go/plugins/pkg/topology/v1" + "github.com/netdata/netdata/go/plugins/plugin/go.d/collector/snmp_topology/snmptopologyfunc" "github.com/santhosh-tekuri/jsonschema/v6" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func TestTopologyMethodConfigIncludesSelectors(t *testing.T) { - cfg := topologyMethodConfig() - assert.True(t, cfg.AgentWide) - require.Len(t, cfg.RequiredParams, 5) - - identity := cfg.RequiredParams[0] - assert.Equal(t, topologyParamNodesIdentity, identity.ID) - assert.Equal(t, funcapi.ParamSelect, identity.Selection) - require.Len(t, identity.Options, 2) - assert.Equal(t, topologyNodesIdentityIP, identity.Options[0].ID) - assert.True(t, identity.Options[0].Default) - assert.Equal(t, topologyNodesIdentityMAC, identity.Options[1].ID) - - mapType := cfg.RequiredParams[1] - assert.Equal(t, topologyParamMapType, mapType.ID) - assert.Equal(t, "Map", mapType.Name) - assert.Equal(t, funcapi.ParamSelect, mapType.Selection) - require.Len(t, mapType.Options, 3) - assert.Equal(t, topologyMapTypeLLDPCDPManaged, mapType.Options[0].ID) - assert.True(t, mapType.Options[0].Default) - assert.Equal(t, topologyMapTypeHighConfidenceInferred, mapType.Options[1].ID) - assert.Equal(t, topologyMapTypeAllDevicesLowConfidence, mapType.Options[2].ID) - - strategy := cfg.RequiredParams[2] - assert.Equal(t, topologyParamInferenceStrategy, strategy.ID) - assert.Equal(t, "Infer Strategy", strategy.Name) - assert.Equal(t, funcapi.ParamSelect, strategy.Selection) - require.Len(t, strategy.Options, 5) - assert.Equal(t, topologyInferenceStrategyFDBMinimumKnowledge, strategy.Options[0].ID) - assert.True(t, strategy.Options[0].Default) - assert.Equal(t, topologyInferenceStrategySTPParentTree, strategy.Options[1].ID) - assert.Equal(t, topologyInferenceStrategyFDBPairwise, strategy.Options[2].ID) - assert.Equal(t, topologyInferenceStrategySTPFDBCorrelated, strategy.Options[3].ID) - assert.Equal(t, topologyInferenceStrategyCDPFDBHybrid, strategy.Options[4].ID) - - managedFocus := cfg.RequiredParams[3] - assert.Equal(t, topologyParamManagedDeviceFocus, managedFocus.ID) - assert.Equal(t, "Focus On", managedFocus.Name) - assert.Equal(t, funcapi.ParamMultiSelect, managedFocus.Selection) - require.Len(t, managedFocus.Options, 1) - assert.Equal(t, topologyManagedFocusAllDevices, managedFocus.Options[0].ID) - assert.True(t, managedFocus.Options[0].Default) - - depth := cfg.RequiredParams[4] - assert.Equal(t, topologyParamDepth, depth.ID) - assert.Equal(t, "Focus Depth", depth.Name) - assert.Equal(t, funcapi.ParamSelect, depth.Selection) - require.NotEmpty(t, depth.Options) - assert.Equal(t, topologyDepthAll, depth.Options[0].ID) - assert.True(t, depth.Options[0].Default) -} - -func TestFuncTopology_MethodParams(t *testing.T) { +func TestTopologyFunctionAdapter_MethodParamsUsesRegistryFocusTargets(t *testing.T) { registry := newTopologyRegistry() registry.register(newTestTopologyCacheLLDP( "agent-test", @@ -84,26 +33,26 @@ func TestFuncTopology_MethodParams(t *testing.T) { "Gi0/2", )) - f := &funcTopology{registry: registry} + handler := snmptopologyfunc.NewHandler(funcDepsAdapter{registry: registry}) - params, err := f.MethodParams(context.Background(), topologyMethodID) + params, err := handler.MethodParams(context.Background(), snmptopologyfunc.MethodID) require.NoError(t, err) require.Len(t, params, 5) - assert.Equal(t, topologyParamNodesIdentity, params[0].ID) - assert.Equal(t, topologyParamMapType, params[1].ID) - assert.Equal(t, topologyParamInferenceStrategy, params[2].ID) - assert.Equal(t, topologyParamManagedDeviceFocus, params[3].ID) - assert.Equal(t, topologyParamDepth, params[4].ID) + assert.Equal(t, snmptopologyfunc.ParamNodesIdentity, params[0].ID) + assert.Equal(t, snmptopologyfunc.ParamMapType, params[1].ID) + assert.Equal(t, snmptopologyfunc.ParamInferenceStrategy, params[2].ID) + assert.Equal(t, snmptopologyfunc.ParamManagedDeviceFocus, params[3].ID) + assert.Equal(t, snmptopologyfunc.ParamDepth, params[4].ID) require.GreaterOrEqual(t, len(params[3].Options), 2) - assert.Equal(t, topologyManagedFocusAllDevices, params[3].Options[0].ID) + assert.Equal(t, snmptopologyfunc.ManagedFocusAllDevices, params[3].Options[0].ID) assert.Equal(t, "ip:10.0.0.1", params[3].Options[1].ID) - params, err = f.MethodParams(context.Background(), "unknown") + params, err = handler.MethodParams(context.Background(), "unknown") require.NoError(t, err) assert.Nil(t, params) } -func TestFuncTopology_Handle_DefaultStrictL2(t *testing.T) { +func TestTopologyFunctionAdapter_HandleDefaultStrictL2(t *testing.T) { registry := newTopologyRegistry() registry.register(newTestTopologyCacheLLDP( "agent-test", @@ -118,8 +67,8 @@ func TestFuncTopology_Handle_DefaultStrictL2(t *testing.T) { "Gi0/2", )) - f := &funcTopology{registry: registry} - resp := f.Handle(context.Background(), topologyMethodID, nil) + handler := snmptopologyfunc.NewHandler(funcDepsAdapter{registry: registry}) + resp := handler.Handle(context.Background(), snmptopologyfunc.MethodID, nil) require.NotNil(t, resp) assert.Equal(t, 200, resp.Status) assert.Equal(t, "topology", resp.ResponseType) @@ -137,7 +86,7 @@ func TestFuncTopology_Handle_DefaultStrictL2(t *testing.T) { assert.Greater(t, data.Links.Rows, 0) } -func TestFuncTopology_Handle_AcceptsSelectorParams(t *testing.T) { +func TestTopologyFunctionAdapter_HandleSelectorParams(t *testing.T) { registry := newTopologyRegistry() registry.register(newTestTopologyCacheLLDP( "agent-test", @@ -152,23 +101,17 @@ func TestFuncTopology_Handle_AcceptsSelectorParams(t *testing.T) { "Gi0/2", )) - f := &funcTopology{registry: registry} - cfg := []funcapi.ParamConfig{ - topologyNodesIdentityParamConfig(), - topologyMapTypeParamConfig(), - topologyInferenceStrategyParamConfig(), - topologyManagedFocusParamConfig(nil), - topologyDepthParamConfig(), - } + handler := snmptopologyfunc.NewHandler(funcDepsAdapter{registry: registry}) + cfg := snmptopologyfunc.Methods()[0].RequiredParams params := funcapi.ResolveParams(cfg, map[string][]string{ - topologyParamNodesIdentity: {topologyNodesIdentityMAC}, - topologyParamMapType: {topologyMapTypeHighConfidenceInferred}, - topologyParamInferenceStrategy: {topologyInferenceStrategySTPFDBCorrelated}, - topologyParamManagedDeviceFocus: {"ip:10.0.0.1"}, - topologyParamDepth: {"2"}, + snmptopologyfunc.ParamNodesIdentity: {snmptopologyfunc.NodesIdentityMAC}, + snmptopologyfunc.ParamMapType: {snmptopologyfunc.MapTypeHighConfidenceInferred}, + snmptopologyfunc.ParamInferenceStrategy: {snmptopologyfunc.InferenceStrategySTPFDBCorrelated}, + snmptopologyfunc.ParamManagedDeviceFocus: {"ip:10.0.0.1"}, + snmptopologyfunc.ParamDepth: {"2"}, }) - resp := f.Handle(context.Background(), topologyMethodID, params) + resp := handler.Handle(context.Background(), snmptopologyfunc.MethodID, params) require.NotNil(t, resp) assert.Equal(t, 200, resp.Status) data, ok := resp.Data.(topologyv1.Data) @@ -178,7 +121,7 @@ func TestFuncTopology_Handle_AcceptsSelectorParams(t *testing.T) { assert.Equal(t, "network", data.View.Scope) } -func TestFuncTopology_Handle_UnknownSelectorsFallbackToDefaults(t *testing.T) { +func TestTopologyFunctionAdapter_HandleUnknownSelectorsFallbackToDefaults(t *testing.T) { registry := newTopologyRegistry() registry.register(newTestTopologyCacheLLDP( "agent-test", @@ -193,29 +136,23 @@ func TestFuncTopology_Handle_UnknownSelectorsFallbackToDefaults(t *testing.T) { "Gi0/2", )) - f := &funcTopology{registry: registry} - cfg := []funcapi.ParamConfig{ - topologyNodesIdentityParamConfig(), - topologyMapTypeParamConfig(), - topologyInferenceStrategyParamConfig(), - topologyManagedFocusParamConfig(nil), - topologyDepthParamConfig(), - } + handler := snmptopologyfunc.NewHandler(funcDepsAdapter{registry: registry}) + cfg := snmptopologyfunc.Methods()[0].RequiredParams - defaultResp := f.Handle(context.Background(), topologyMethodID, nil) + defaultResp := handler.Handle(context.Background(), snmptopologyfunc.MethodID, nil) require.NotNil(t, defaultResp) require.Equal(t, 200, defaultResp.Status) defaultData, ok := defaultResp.Data.(topologyv1.Data) require.True(t, ok) invalidParams := funcapi.ResolveParams(cfg, map[string][]string{ - topologyParamNodesIdentity: {"unknown"}, - topologyParamMapType: {"invalid"}, - topologyParamInferenceStrategy: {"invalid"}, - topologyParamManagedDeviceFocus: {"invalid"}, - topologyParamDepth: {"invalid"}, + snmptopologyfunc.ParamNodesIdentity: {"unknown"}, + snmptopologyfunc.ParamMapType: {"invalid"}, + snmptopologyfunc.ParamInferenceStrategy: {"invalid"}, + snmptopologyfunc.ParamManagedDeviceFocus: {"invalid"}, + snmptopologyfunc.ParamDepth: {"invalid"}, }) - invalidResp := f.Handle(context.Background(), topologyMethodID, invalidParams) + invalidResp := handler.Handle(context.Background(), snmptopologyfunc.MethodID, invalidParams) require.NotNil(t, invalidResp) require.Equal(t, 200, invalidResp.Status) invalidData, ok := invalidResp.Data.(topologyv1.Data) @@ -622,8 +559,6 @@ func TestNormalizeTopologyInferenceStrategy(t *testing.T) { } func TestNormalizeTopologyManagedFocuses(t *testing.T) { - assert.Equal(t, topologyManagedFocusAllDevices, normalizeTopologyManagedFocus("")) - assert.Equal(t, "ip:10.0.0.1", normalizeTopologyManagedFocus(" ip:10.0.0.1 ")) assert.Equal(t, []string{topologyManagedFocusAllDevices}, normalizeTopologyManagedFocuses(nil)) assert.Equal(t, []string{topologyManagedFocusAllDevices}, normalizeTopologyManagedFocuses([]string{})) assert.Equal(t, []string{topologyManagedFocusAllDevices}, normalizeTopologyManagedFocuses([]string{""})) diff --git a/src/go/plugin/go.d/collector/snmp_topology/snmptopologyfunc/deps.go b/src/go/plugin/go.d/collector/snmp_topology/snmptopologyfunc/deps.go new file mode 100644 index 00000000000000..88176c4908023f --- /dev/null +++ b/src/go/plugin/go.d/collector/snmp_topology/snmptopologyfunc/deps.go @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package snmptopologyfunc + +import topologyv1 "github.com/netdata/netdata/go/plugins/pkg/topology/v1" + +// Deps defines the dependency surface required by SNMP topology function handlers. +type Deps interface { + Snapshot(QueryOptions) (topologyv1.Data, bool, error) + ManagedDeviceFocusTargets() []ManagedFocusTarget +} + +// ManagedFocusTarget describes a managed SNMP device that can be used as a focus root. +type ManagedFocusTarget struct { + Value string + Name string +} + +// QueryOptions is the function-level query shape consumed by the collector adapter. +type QueryOptions struct { + CollapseActorsByIP bool + EliminateNonIPInferred bool + MapType string + InferenceStrategy string + ManagedDeviceFocus string + Depth int +} diff --git a/src/go/plugin/go.d/collector/snmp_topology/snmptopologyfunc/managed_focus.go b/src/go/plugin/go.d/collector/snmp_topology/snmptopologyfunc/managed_focus.go new file mode 100644 index 00000000000000..e97e36c788a303 --- /dev/null +++ b/src/go/plugin/go.d/collector/snmp_topology/snmptopologyfunc/managed_focus.go @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package snmptopologyfunc + +import ( + "sort" + "strings" + + "github.com/netdata/netdata/go/plugins/pkg/funcapi" +) + +func managedFocusParamOptions(deps Deps) []funcapi.ParamOption { + if deps == nil { + return nil + } + + options := make([]funcapi.ParamOption, 0) + for _, target := range deps.ManagedDeviceFocusTargets() { + if strings.TrimSpace(target.Value) == "" { + continue + } + options = append(options, funcapi.ParamOption{ + ID: target.Value, + Name: target.Name, + }) + } + return options +} + +func normalizeManagedFocuses(values []string) []string { + expanded := splitManagedFocusValues(values) + if len(expanded) == 0 { + return []string{ManagedFocusAllDevices} + } + + seen := make(map[string]struct{}, len(expanded)) + out := make([]string, 0, len(expanded)) + for _, raw := range expanded { + normalized := normalizeManagedFocusValue(raw) + if normalized == "" { + continue + } + if normalized == ManagedFocusAllDevices { + return []string{ManagedFocusAllDevices} + } + if _, ok := seen[normalized]; ok { + continue + } + seen[normalized] = struct{}{} + out = append(out, normalized) + } + + if len(out) == 0 { + return []string{ManagedFocusAllDevices} + } + sort.Strings(out) + return out +} + +func splitManagedFocusValues(values []string) []string { + if len(values) == 0 { + return nil + } + + out := make([]string, 0, len(values)) + for _, raw := range values { + for token := range strings.SplitSeq(raw, ",") { + token = strings.TrimSpace(token) + if token == "" { + continue + } + out = append(out, token) + } + } + return out +} + +func normalizeManagedFocusValue(v string) string { + value := strings.TrimSpace(v) + if strings.EqualFold(value, ManagedFocusAllDevices) { + return ManagedFocusAllDevices + } + if len(value) <= len(ManagedFocusIPPrefix) || + !strings.EqualFold(value[:len(ManagedFocusIPPrefix)], ManagedFocusIPPrefix) { + return "" + } + ip := strings.TrimSpace(value[len(ManagedFocusIPPrefix):]) + if ip == "" { + return "" + } + return ManagedFocusIPPrefix + ip +} + +func formatManagedFocuses(values []string) string { + normalized := normalizeManagedFocuses(values) + if len(normalized) == 0 { + return ManagedFocusAllDevices + } + return strings.Join(normalized, ",") +} diff --git a/src/go/plugin/go.d/collector/snmp_topology/snmptopologyfunc/options.go b/src/go/plugin/go.d/collector/snmp_topology/snmptopologyfunc/options.go new file mode 100644 index 00000000000000..07374016589605 --- /dev/null +++ b/src/go/plugin/go.d/collector/snmp_topology/snmptopologyfunc/options.go @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package snmptopologyfunc + +import ( + "strconv" + "strings" + + "github.com/netdata/netdata/go/plugins/pkg/funcapi" +) + +func resolveQueryOptions(params funcapi.ResolvedParams) QueryOptions { + options := QueryOptions{ + CollapseActorsByIP: true, + EliminateNonIPInferred: true, + MapType: MapTypeLLDPCDPManaged, + InferenceStrategy: InferenceStrategyFDBMinimumKnowledge, + ManagedDeviceFocus: ManagedFocusAllDevices, + Depth: DepthAllInternal, + } + + if identity := normalizeNodesIdentity(params.GetOne(ParamNodesIdentity)); identity == NodesIdentityMAC { + options.CollapseActorsByIP = false + options.EliminateNonIPInferred = false + } + if mapType := normalizeMapType(params.GetOne(ParamMapType)); mapType != "" { + options.MapType = mapType + } + if strategy := normalizeInferenceStrategy(params.GetOne(ParamInferenceStrategy)); strategy != "" { + options.InferenceStrategy = strategy + } + if focuses := normalizeManagedFocuses(params.Get(ParamManagedDeviceFocus)); len(focuses) > 0 { + options.ManagedDeviceFocus = formatManagedFocuses(focuses) + } + options.Depth = normalizeDepth(params.GetOne(ParamDepth)) + + return options +} + +func normalizeNodesIdentity(v string) string { + switch strings.ToLower(strings.TrimSpace(v)) { + case "", NodesIdentityIP: + return NodesIdentityIP + case NodesIdentityMAC: + return NodesIdentityMAC + default: + return "" + } +} + +func normalizeMapType(v string) string { + switch strings.ToLower(strings.TrimSpace(v)) { + case "", MapTypeLLDPCDPManaged: + return MapTypeLLDPCDPManaged + case MapTypeHighConfidenceInferred: + return MapTypeHighConfidenceInferred + case MapTypeAllDevicesLowConfidence: + return MapTypeAllDevicesLowConfidence + default: + return "" + } +} + +func normalizeInferenceStrategy(v string) string { + switch strings.ToLower(strings.TrimSpace(v)) { + case "", InferenceStrategyFDBMinimumKnowledge: + return InferenceStrategyFDBMinimumKnowledge + case InferenceStrategySTPParentTree: + return InferenceStrategySTPParentTree + case InferenceStrategyFDBPairwise: + return InferenceStrategyFDBPairwise + case InferenceStrategySTPFDBCorrelated: + return InferenceStrategySTPFDBCorrelated + case InferenceStrategyCDPFDBHybrid: + return InferenceStrategyCDPFDBHybrid + default: + return "" + } +} + +func normalizeDepth(v string) int { + value := strings.ToLower(strings.TrimSpace(v)) + if value == "" || value == DepthAll { + return DepthAllInternal + } + depth, err := strconv.Atoi(value) + if err != nil { + return DepthAllInternal + } + if depth < DepthMin { + return DepthMin + } + if depth > DepthMax { + return DepthMax + } + return depth +} diff --git a/src/go/plugin/go.d/collector/snmp_topology/snmptopologyfunc/presentation.go b/src/go/plugin/go.d/collector/snmp_topology/snmptopologyfunc/presentation.go new file mode 100644 index 00000000000000..da54c179ad8f0f --- /dev/null +++ b/src/go/plugin/go.d/collector/snmp_topology/snmptopologyfunc/presentation.go @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package snmptopologyfunc + +import ( + "github.com/netdata/netdata/go/plugins/pkg/funcapi" + topologyv1 "github.com/netdata/netdata/go/plugins/pkg/topology/v1" +) + +func Methods() []funcapi.MethodConfig { + return []funcapi.MethodConfig{ + methodConfig(), + } +} + +func methodConfig() funcapi.MethodConfig { + return funcapi.MethodConfig{ + ID: MethodID, + FunctionName: FunctionName, + Aliases: []string{MethodID}, + Name: "Topology (SNMP)", + UpdateEvery: 10, + Help: "SNMP Layer-2 topology and neighbor discovery data", + RequireCloud: true, + ResponseType: topologyv1.ResponseType, + AgentWide: true, + RequiredParams: []funcapi.ParamConfig{ + nodesIdentityParamConfig(), + mapTypeParamConfig(), + inferenceStrategyParamConfig(), + managedFocusParamConfig(nil), + depthParamConfig(), + }, + } +} diff --git a/src/go/plugin/go.d/collector/snmp_topology/snmptopologyfunc/presentation_params.go b/src/go/plugin/go.d/collector/snmp_topology/snmptopologyfunc/presentation_params.go new file mode 100644 index 00000000000000..5a1c2664a598b4 --- /dev/null +++ b/src/go/plugin/go.d/collector/snmp_topology/snmptopologyfunc/presentation_params.go @@ -0,0 +1,143 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package snmptopologyfunc + +import ( + "strconv" + + "github.com/netdata/netdata/go/plugins/pkg/funcapi" +) + +const ( + ParamNodesIdentity = "nodes_identity" + ParamMapType = "map_type" + ParamInferenceStrategy = "inference_strategy" + ParamManagedDeviceFocus = "managed_snmp_device_focus" + ParamDepth = "depth" + + NodesIdentityIP = "ip" + NodesIdentityMAC = "mac" + + MapTypeLLDPCDPManaged = "lldp_cdp_managed" + MapTypeHighConfidenceInferred = "high_confidence_inferred" + MapTypeAllDevicesLowConfidence = "all_devices_low_confidence" + InferenceStrategyFDBMinimumKnowledge = "fdb_minimum_knowledge" + InferenceStrategySTPParentTree = "stp_parent_tree" + InferenceStrategyFDBPairwise = "fdb_pairwise_minimum_knowledge" + InferenceStrategySTPFDBCorrelated = "stp_fdb_correlated" + InferenceStrategyCDPFDBHybrid = "cdp_fdb_hybrid" + ManagedFocusAllDevices = "all_devices" + ManagedFocusIPPrefix = "ip:" + DepthAll = "all" + DepthMin = 0 + DepthMax = 10 + DepthAllInternal = -1 +) + +func nodesIdentityParamConfig() funcapi.ParamConfig { + return funcapi.ParamConfig{ + ID: ParamNodesIdentity, + Name: "Nodes Identity", + Help: "Choose actor identity strategy: ip (collapse by IP, remove non-IP inferred) or mac", + Selection: funcapi.ParamSelect, + Options: []funcapi.ParamOption{ + {ID: NodesIdentityIP, Name: "IP", Default: true}, + {ID: NodesIdentityMAC, Name: "MAC"}, + }, + } +} + +func mapTypeParamConfig() funcapi.ParamConfig { + return funcapi.ParamConfig{ + ID: ParamMapType, + Name: "Map", + Help: "Choose topology map mode", + Selection: funcapi.ParamSelect, + Options: []funcapi.ParamOption{ + { + ID: MapTypeLLDPCDPManaged, + Name: "LLDP/CDP/Managed Devices Map", + Default: true, + }, + {ID: MapTypeHighConfidenceInferred, Name: "High Confidence Inferred Map"}, + { + ID: MapTypeAllDevicesLowConfidence, + Name: "All Devices (Low Confidence)", + }, + }, + } +} + +func inferenceStrategyParamConfig() funcapi.ParamConfig { + return funcapi.ParamConfig{ + ID: ParamInferenceStrategy, + Name: "Infer Strategy", + Help: "Choose the topology inference strategy", + Selection: funcapi.ParamSelect, + Options: []funcapi.ParamOption{ + { + ID: InferenceStrategyFDBMinimumKnowledge, + Name: "FDB Minimum-Knowledge (Baseline)", + Default: true, + }, + { + ID: InferenceStrategySTPParentTree, + Name: "STP Parent Tree", + }, + { + ID: InferenceStrategyFDBPairwise, + Name: "FDB Pairwise Minimum-Knowledge", + }, + { + ID: InferenceStrategySTPFDBCorrelated, + Name: "STP + FDB Correlated", + }, + { + ID: InferenceStrategyCDPFDBHybrid, + Name: "CDP + FDB Hybrid", + }, + }, + } +} + +func managedFocusParamConfig(extraOptions []funcapi.ParamOption) funcapi.ParamConfig { + options := make([]funcapi.ParamOption, 0, 1+len(extraOptions)) + options = append(options, funcapi.ParamOption{ + ID: ManagedFocusAllDevices, + Name: "All Devices", + Default: true, + }) + options = append(options, extraOptions...) + + return funcapi.ParamConfig{ + ID: ParamManagedDeviceFocus, + Name: "Focus On", + Help: "Choose focus root set for depth filtering", + Selection: funcapi.ParamMultiSelect, + Options: options, + } +} + +func depthParamConfig() funcapi.ParamConfig { + options := make([]funcapi.ParamOption, 0, 1+(DepthMax-DepthMin+1)) + options = append(options, funcapi.ParamOption{ + ID: DepthAll, + Name: "All", + Default: true, + }) + for depth := DepthMin; depth <= DepthMax; depth++ { + value := strconv.Itoa(depth) + options = append(options, funcapi.ParamOption{ + ID: value, + Name: value, + }) + } + + return funcapi.ParamConfig{ + ID: ParamDepth, + Name: "Focus Depth", + Help: "Limit topology expansion hops from focus roots", + Selection: funcapi.ParamSelect, + Options: options, + } +} diff --git a/src/go/plugin/go.d/collector/snmp_topology/snmptopologyfunc/topology.go b/src/go/plugin/go.d/collector/snmp_topology/snmptopologyfunc/topology.go new file mode 100644 index 00000000000000..4f1c8f6b7eb499 --- /dev/null +++ b/src/go/plugin/go.d/collector/snmp_topology/snmptopologyfunc/topology.go @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package snmptopologyfunc + +import ( + "context" + + "github.com/netdata/netdata/go/plugins/pkg/funcapi" + topologyv1 "github.com/netdata/netdata/go/plugins/pkg/topology/v1" +) + +const ( + FunctionName = "snmp:topology:snmp" + MethodID = "topology:snmp" +) + +const topologyUnavailable = "topology data not available yet, please retry after topology refresh" + +type topologyHandler struct { + deps Deps +} + +var _ funcapi.MethodHandler = (*topologyHandler)(nil) + +func NewHandler(deps Deps) funcapi.MethodHandler { + return &topologyHandler{deps: deps} +} + +func (h *topologyHandler) MethodParams(_ context.Context, method string) ([]funcapi.ParamConfig, error) { + if method != MethodID { + return nil, nil + } + + return []funcapi.ParamConfig{ + nodesIdentityParamConfig(), + mapTypeParamConfig(), + inferenceStrategyParamConfig(), + managedFocusParamConfig(managedFocusParamOptions(h.deps)), + depthParamConfig(), + }, nil +} + +func (h *topologyHandler) Handle(_ context.Context, method string, params funcapi.ResolvedParams) *funcapi.FunctionResponse { + if method != MethodID { + return funcapi.NotFoundResponse(method) + } + if h.deps == nil { + return funcapi.UnavailableResponse(topologyUnavailable) + } + + payload, ok, err := h.deps.Snapshot(resolveQueryOptions(params)) + if err != nil { + return funcapi.InternalErrorResponse("failed to build topology response: %v", err) + } + if !ok { + return funcapi.UnavailableResponse(topologyUnavailable) + } + + return &funcapi.FunctionResponse{ + Status: 200, + Help: "SNMP topology and neighbor discovery data", + ResponseType: topologyv1.ResponseType, + Data: payload, + } +} + +func (h *topologyHandler) Cleanup(context.Context) {} diff --git a/src/go/plugin/go.d/collector/snmp_topology/snmptopologyfunc/topology_test.go b/src/go/plugin/go.d/collector/snmp_topology/snmptopologyfunc/topology_test.go new file mode 100644 index 00000000000000..8e5eeff4e9086e --- /dev/null +++ b/src/go/plugin/go.d/collector/snmp_topology/snmptopologyfunc/topology_test.go @@ -0,0 +1,232 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package snmptopologyfunc + +import ( + "context" + "errors" + "testing" + + "github.com/netdata/netdata/go/plugins/pkg/funcapi" + topologyv1 "github.com/netdata/netdata/go/plugins/pkg/topology/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMethodsIncludesSelectors(t *testing.T) { + methods := Methods() + require.Len(t, methods, 1) + + cfg := methods[0] + assert.Equal(t, MethodID, cfg.ID) + assert.Equal(t, FunctionName, cfg.FunctionName) + assert.Equal(t, []string{MethodID}, cfg.Aliases) + assert.Equal(t, "topology", cfg.ResponseType) + assert.True(t, cfg.AgentWide) + require.Len(t, cfg.RequiredParams, 5) + require.Nil(t, cfg.Presentation()) + + identity := cfg.RequiredParams[0] + assert.Equal(t, ParamNodesIdentity, identity.ID) + assert.Equal(t, funcapi.ParamSelect, identity.Selection) + require.Len(t, identity.Options, 2) + assert.Equal(t, NodesIdentityIP, identity.Options[0].ID) + assert.True(t, identity.Options[0].Default) + assert.Equal(t, NodesIdentityMAC, identity.Options[1].ID) + + mapType := cfg.RequiredParams[1] + assert.Equal(t, ParamMapType, mapType.ID) + assert.Equal(t, "Map", mapType.Name) + assert.Equal(t, funcapi.ParamSelect, mapType.Selection) + require.Len(t, mapType.Options, 3) + assert.Equal(t, MapTypeLLDPCDPManaged, mapType.Options[0].ID) + assert.True(t, mapType.Options[0].Default) + assert.Equal(t, MapTypeHighConfidenceInferred, mapType.Options[1].ID) + assert.Equal(t, MapTypeAllDevicesLowConfidence, mapType.Options[2].ID) + + strategy := cfg.RequiredParams[2] + assert.Equal(t, ParamInferenceStrategy, strategy.ID) + assert.Equal(t, "Infer Strategy", strategy.Name) + assert.Equal(t, funcapi.ParamSelect, strategy.Selection) + require.Len(t, strategy.Options, 5) + assert.Equal(t, InferenceStrategyFDBMinimumKnowledge, strategy.Options[0].ID) + assert.True(t, strategy.Options[0].Default) + assert.Equal(t, InferenceStrategySTPParentTree, strategy.Options[1].ID) + assert.Equal(t, InferenceStrategyFDBPairwise, strategy.Options[2].ID) + assert.Equal(t, InferenceStrategySTPFDBCorrelated, strategy.Options[3].ID) + assert.Equal(t, InferenceStrategyCDPFDBHybrid, strategy.Options[4].ID) + + managedFocus := cfg.RequiredParams[3] + assert.Equal(t, ParamManagedDeviceFocus, managedFocus.ID) + assert.Equal(t, "Focus On", managedFocus.Name) + assert.Equal(t, funcapi.ParamMultiSelect, managedFocus.Selection) + require.Len(t, managedFocus.Options, 1) + assert.Equal(t, ManagedFocusAllDevices, managedFocus.Options[0].ID) + assert.True(t, managedFocus.Options[0].Default) + + depth := cfg.RequiredParams[4] + assert.Equal(t, ParamDepth, depth.ID) + assert.Equal(t, "Focus Depth", depth.Name) + assert.Equal(t, funcapi.ParamSelect, depth.Selection) + require.NotEmpty(t, depth.Options) + assert.Equal(t, DepthAll, depth.Options[0].ID) + assert.True(t, depth.Options[0].Default) +} + +func TestTopologyHandlerMethodParams(t *testing.T) { + deps := &fakeDeps{ + targets: []ManagedFocusTarget{ + {Value: "", Name: "skip"}, + {Value: "ip:10.0.0.1", Name: "sw-a (10.0.0.1)"}, + }, + } + handler := NewHandler(deps) + + params, err := handler.MethodParams(context.Background(), MethodID) + require.NoError(t, err) + require.Len(t, params, 5) + assert.Equal(t, ParamNodesIdentity, params[0].ID) + assert.Equal(t, ParamMapType, params[1].ID) + assert.Equal(t, ParamInferenceStrategy, params[2].ID) + assert.Equal(t, ParamManagedDeviceFocus, params[3].ID) + assert.Equal(t, ParamDepth, params[4].ID) + require.Len(t, params[3].Options, 2) + assert.Equal(t, ManagedFocusAllDevices, params[3].Options[0].ID) + assert.Equal(t, "ip:10.0.0.1", params[3].Options[1].ID) + + params, err = handler.MethodParams(context.Background(), "unknown") + require.NoError(t, err) + require.Nil(t, params) +} + +func TestTopologyHandlerHandleDefaultOptions(t *testing.T) { + deps := &fakeDeps{data: topologyv1.Data{SchemaVersion: topologyv1.SchemaVersion}, ok: true} + handler := NewHandler(deps) + + resp := handler.Handle(context.Background(), MethodID, nil) + require.NotNil(t, resp) + assert.Equal(t, 200, resp.Status) + assert.Equal(t, topologyv1.ResponseType, resp.ResponseType) + assert.Equal(t, "SNMP topology and neighbor discovery data", resp.Help) + assert.Equal(t, deps.data, resp.Data) + + require.Equal(t, 1, deps.snapshotCalls) + assert.Equal(t, QueryOptions{ + CollapseActorsByIP: true, + EliminateNonIPInferred: true, + MapType: MapTypeLLDPCDPManaged, + InferenceStrategy: InferenceStrategyFDBMinimumKnowledge, + ManagedDeviceFocus: ManagedFocusAllDevices, + Depth: DepthAllInternal, + }, deps.lastOptions) +} + +func TestTopologyHandlerHandleSelectorParams(t *testing.T) { + deps := &fakeDeps{ + data: topologyv1.Data{SchemaVersion: topologyv1.SchemaVersion}, + ok: true, + targets: []ManagedFocusTarget{ + {Value: "ip:10.0.0.2", Name: "sw-b (10.0.0.2)"}, + {Value: "ip:10.0.0.1", Name: "sw-a (10.0.0.1)"}, + }, + } + handler := NewHandler(deps) + paramCfgs, err := handler.MethodParams(context.Background(), MethodID) + require.NoError(t, err) + + params := funcapi.ResolveParams(paramCfgs, map[string][]string{ + ParamNodesIdentity: {NodesIdentityMAC}, + ParamMapType: {MapTypeHighConfidenceInferred}, + ParamInferenceStrategy: {InferenceStrategySTPFDBCorrelated}, + ParamManagedDeviceFocus: {"ip:10.0.0.2", "ip:10.0.0.1"}, + ParamDepth: {"2"}, + }) + + resp := handler.Handle(context.Background(), MethodID, params) + require.NotNil(t, resp) + assert.Equal(t, 200, resp.Status) + require.Equal(t, 1, deps.snapshotCalls) + assert.Equal(t, QueryOptions{ + CollapseActorsByIP: false, + EliminateNonIPInferred: false, + MapType: MapTypeHighConfidenceInferred, + InferenceStrategy: InferenceStrategySTPFDBCorrelated, + ManagedDeviceFocus: "ip:10.0.0.1,ip:10.0.0.2", + Depth: 2, + }, deps.lastOptions) +} + +func TestTopologyHandlerHandleUnknownSelectorsFallbackToDefaults(t *testing.T) { + deps := &fakeDeps{data: topologyv1.Data{SchemaVersion: topologyv1.SchemaVersion}, ok: true} + handler := NewHandler(deps) + params := funcapi.ResolvedParams{ + ParamNodesIdentity: {IDs: []string{"unknown"}}, + ParamMapType: {IDs: []string{"invalid"}}, + ParamInferenceStrategy: {IDs: []string{"invalid"}}, + ParamManagedDeviceFocus: {IDs: []string{"invalid"}}, + ParamDepth: {IDs: []string{"invalid"}}, + } + + resp := handler.Handle(context.Background(), MethodID, params) + require.NotNil(t, resp) + assert.Equal(t, 200, resp.Status) + require.Equal(t, 1, deps.snapshotCalls) + assert.Equal(t, QueryOptions{ + CollapseActorsByIP: true, + EliminateNonIPInferred: true, + MapType: MapTypeLLDPCDPManaged, + InferenceStrategy: InferenceStrategyFDBMinimumKnowledge, + ManagedDeviceFocus: ManagedFocusAllDevices, + Depth: DepthAllInternal, + }, deps.lastOptions) +} + +func TestTopologyHandlerUnavailableAndErrors(t *testing.T) { + t.Run("unknown method", func(t *testing.T) { + resp := NewHandler(&fakeDeps{}).Handle(context.Background(), "unknown", nil) + require.NotNil(t, resp) + assert.Equal(t, 404, resp.Status) + }) + + t.Run("nil deps", func(t *testing.T) { + resp := NewHandler(nil).Handle(context.Background(), MethodID, nil) + require.NotNil(t, resp) + assert.Equal(t, 503, resp.Status) + assert.Contains(t, resp.Message, "topology data not available") + }) + + t.Run("snapshot unavailable", func(t *testing.T) { + resp := NewHandler(&fakeDeps{}).Handle(context.Background(), MethodID, nil) + require.NotNil(t, resp) + assert.Equal(t, 503, resp.Status) + assert.Contains(t, resp.Message, "topology data not available") + }) + + t.Run("snapshot error", func(t *testing.T) { + resp := NewHandler(&fakeDeps{err: errors.New("boom")}).Handle(context.Background(), MethodID, nil) + require.NotNil(t, resp) + assert.Equal(t, 500, resp.Status) + assert.Contains(t, resp.Message, "failed to build topology response") + }) +} + +type fakeDeps struct { + data topologyv1.Data + ok bool + err error + + targets []ManagedFocusTarget + + snapshotCalls int + lastOptions QueryOptions +} + +func (d *fakeDeps) Snapshot(options QueryOptions) (topologyv1.Data, bool, error) { + d.snapshotCalls++ + d.lastOptions = options + return d.data, d.ok, d.err +} + +func (d *fakeDeps) ManagedDeviceFocusTargets() []ManagedFocusTarget { + return d.targets +} diff --git a/src/go/plugin/go.d/collector/snmp_topology/topology_query_options.go b/src/go/plugin/go.d/collector/snmp_topology/topology_query_options.go new file mode 100644 index 00000000000000..395109ba9ea7a9 --- /dev/null +++ b/src/go/plugin/go.d/collector/snmp_topology/topology_query_options.go @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package snmptopology + +import ( + "sort" + "strings" + + "github.com/netdata/netdata/go/plugins/plugin/go.d/collector/snmp_topology/snmptopologyfunc" +) + +const ( + topologyMapTypeLLDPCDPManaged = snmptopologyfunc.MapTypeLLDPCDPManaged + topologyMapTypeHighConfidenceInferred = snmptopologyfunc.MapTypeHighConfidenceInferred + topologyMapTypeAllDevicesLowConfidence = snmptopologyfunc.MapTypeAllDevicesLowConfidence + topologyInferenceStrategyFDBMinimumKnowledge = snmptopologyfunc.InferenceStrategyFDBMinimumKnowledge + topologyInferenceStrategySTPParentTree = snmptopologyfunc.InferenceStrategySTPParentTree + topologyInferenceStrategyFDBPairwise = snmptopologyfunc.InferenceStrategyFDBPairwise + topologyInferenceStrategySTPFDBCorrelated = snmptopologyfunc.InferenceStrategySTPFDBCorrelated + topologyInferenceStrategyCDPFDBHybrid = snmptopologyfunc.InferenceStrategyCDPFDBHybrid + topologyManagedFocusAllDevices = snmptopologyfunc.ManagedFocusAllDevices + topologyManagedFocusIPPrefix = snmptopologyfunc.ManagedFocusIPPrefix + topologyDepthAll = snmptopologyfunc.DepthAll + topologyDepthMin = snmptopologyfunc.DepthMin + topologyDepthMax = snmptopologyfunc.DepthMax + topologyDepthAllInternal = snmptopologyfunc.DepthAllInternal +) + +func normalizeTopologyMapType(v string) string { + switch strings.ToLower(strings.TrimSpace(v)) { + case "", topologyMapTypeLLDPCDPManaged: + return topologyMapTypeLLDPCDPManaged + case topologyMapTypeHighConfidenceInferred: + return topologyMapTypeHighConfidenceInferred + case topologyMapTypeAllDevicesLowConfidence: + return topologyMapTypeAllDevicesLowConfidence + default: + return "" + } +} + +func normalizeTopologyInferenceStrategy(v string) string { + switch strings.ToLower(strings.TrimSpace(v)) { + case "", topologyInferenceStrategyFDBMinimumKnowledge: + return topologyInferenceStrategyFDBMinimumKnowledge + case topologyInferenceStrategySTPParentTree: + return topologyInferenceStrategySTPParentTree + case topologyInferenceStrategyFDBPairwise: + return topologyInferenceStrategyFDBPairwise + case topologyInferenceStrategySTPFDBCorrelated: + return topologyInferenceStrategySTPFDBCorrelated + case topologyInferenceStrategyCDPFDBHybrid: + return topologyInferenceStrategyCDPFDBHybrid + default: + return "" + } +} + +func isTopologyMapTypeProbable(v string) bool { + switch strings.ToLower(strings.TrimSpace(v)) { + case "", topologyMapTypeAllDevicesLowConfidence: + return true + default: + return false + } +} + +func normalizeTopologyManagedFocusValue(v string) string { + value := strings.TrimSpace(v) + switch strings.ToLower(value) { + case topologyManagedFocusAllDevices: + return topologyManagedFocusAllDevices + } + if len(value) > len(topologyManagedFocusIPPrefix) && + strings.EqualFold(value[:len(topologyManagedFocusIPPrefix)], topologyManagedFocusIPPrefix) { + ip := normalizeIPAddress(strings.TrimSpace(value[len(topologyManagedFocusIPPrefix):])) + if ip == "" { + return "" + } + return topologyManagedFocusIPPrefix + ip + } + return "" +} + +func normalizeTopologyManagedFocuses(values []string) []string { + expanded := splitTopologyManagedFocusValues(values) + if len(expanded) == 0 { + return []string{topologyManagedFocusAllDevices} + } + + seen := make(map[string]struct{}, len(expanded)) + out := make([]string, 0, len(expanded)) + for _, raw := range expanded { + normalized := normalizeTopologyManagedFocusValue(raw) + if normalized == "" { + continue + } + if normalized == topologyManagedFocusAllDevices { + return []string{topologyManagedFocusAllDevices} + } + if _, ok := seen[normalized]; ok { + continue + } + seen[normalized] = struct{}{} + out = append(out, normalized) + } + + if len(out) == 0 { + return []string{topologyManagedFocusAllDevices} + } + sort.Strings(out) + return out +} + +func splitTopologyManagedFocusValues(values []string) []string { + if len(values) == 0 { + return nil + } + + out := make([]string, 0, len(values)) + for _, raw := range values { + for token := range strings.SplitSeq(raw, ",") { + token = strings.TrimSpace(token) + if token == "" { + continue + } + out = append(out, token) + } + } + return out +} + +func parseTopologyManagedFocuses(value string) []string { + if strings.TrimSpace(value) == "" { + return []string{topologyManagedFocusAllDevices} + } + return normalizeTopologyManagedFocuses(strings.Split(value, ",")) +} + +func formatTopologyManagedFocuses(values []string) string { + normalized := normalizeTopologyManagedFocuses(values) + if len(normalized) == 0 { + return topologyManagedFocusAllDevices + } + return strings.Join(normalized, ",") +} + +func isTopologyManagedFocusAllDevices(value string) bool { + normalized := parseTopologyManagedFocuses(value) + return len(normalized) == 1 && normalized[0] == topologyManagedFocusAllDevices +} From 0dfdd25ae2e7eacd04acc1b7551a7d0251c4b6a0 Mon Sep 17 00:00:00 2001 From: Ilya Mashchenko Date: Thu, 18 Jun 2026 23:10:20 +0300 Subject: [PATCH 3/6] chore(snmp_topology): split v1 topology renderer (#22775) --- .../snmp_topology/func_topology_v1.go | 2005 +---------------- .../func_topology_v1_actor_details.go | 260 +++ .../func_topology_v1_actor_ports.go | 397 ++++ .../snmp_topology/func_topology_v1_actors.go | 244 ++ .../snmp_topology/func_topology_v1_dynamic.go | 158 ++ .../func_topology_v1_helpers_test.go | 164 ++ .../snmp_topology/func_topology_v1_links.go | 226 ++ .../func_topology_v1_presentation.go | 435 ++++ .../func_topology_v1_table_types.go | 160 ++ .../snmp_topology/func_topology_v1_values.go | 283 +++ 10 files changed, 2328 insertions(+), 2004 deletions(-) create mode 100644 src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_actor_details.go create mode 100644 src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_actor_ports.go create mode 100644 src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_actors.go create mode 100644 src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_dynamic.go create mode 100644 src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_helpers_test.go create mode 100644 src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_links.go create mode 100644 src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_presentation.go create mode 100644 src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_table_types.go create mode 100644 src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_values.go diff --git a/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1.go b/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1.go index 3030c08150c4b5..849526400e37d3 100644 --- a/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1.go +++ b/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1.go @@ -3,17 +3,8 @@ package snmptopology import ( - "fmt" - "maps" - "math" - "reflect" - "regexp" - "sort" - "strings" - "time" - - topologyengine "github.com/netdata/netdata/go/plugins/pkg/l2topology" topologyv1 "github.com/netdata/netdata/go/plugins/pkg/topology/v1" + "time" ) const ( @@ -35,438 +26,6 @@ const ( snmpTopologyV1LinkProbable = "probable" ) -var topologyV1IDInvalidChars = regexp.MustCompile(`[^A-Za-z0-9_.:-]+`) - -func snmpTopologyV1ActorTypes() map[string]topologyv1.ActorType { - types := make(map[string]topologyv1.ActorType) - addDevice := func(id, label, colorSlot, icon string) { - types[id] = topologyv1.ActorType{ - Layer: "network", - Identity: []string{"id"}, - MergeIdentity: []string{"chassis_ids", "mac_addresses", "ip_addresses", "sys_name"}, - AggregationScopes: []string{"device", "network"}, - Search: &topologyv1.ActorSearchPolicy{ - Columns: []string{"display_name", "sys_name", "management_ip", "vendor", "model"}, - }, - Presentation: &topologyv1.ActorPresentation{ - Label: label, - Role: "actor", - Icon: icon, - ColorSlot: colorSlot, - Border: &topologyv1.BorderPresentation{Enabled: new(true)}, - Size: &topologyv1.ActorSizePresentation{Mode: "link_count", Scale: "emphasized"}, - Layout: &topologyv1.ActorLayoutPresentation{Repulsion: "stronger"}, - LabelPolicy: &topologyv1.LabelPolicy{ - Columns: []string{"display_name", "sys_name"}, - Fallback: "type_label", - MaxLength: 80, - Array: "reject", - }, - Ports: &topologyv1.ActorPortsPresentation{ - ShowBullets: true, - Sources: []topologyv1.PortSourcePresentation{ - { - Source: "actor_table", - Table: "actor_ports", - ActorColumn: "actor", - NameColumn: "name", - TypeColumn: "topology_role", - DefaultType: "topology", - StatusColumn: "oper_status", - ModeColumn: "link_mode", - RoleColumn: "topology_role", - }, - }, - }, - Modal: snmpTopologyV1DeviceModal(), - }, - } - } - - addDevice("device", "Device", "primary", "server") - addDevice("router", "Router", "primary", "router") - addDevice("switch", "Switch", "primary", "switch") - addDevice("firewall", "Firewall", "warning", "firewall") - addDevice("access_point", "Access Point", "info", "access_point") - addDevice("server", "Server", "secondary", "server") - addDevice("storage", "Storage", "secondary", "storage") - addDevice("load_balancer", "Load Balancer", "info", "load_balancer") - addDevice("printer", "Printer", "neutral", "printer") - addDevice("phone", "Phone", "neutral", "phone") - addDevice("ups", "UPS", "structural", "ups") - addDevice("camera", "Camera", "neutral", "camera") - - types[snmpTopologyV1ActorEndpoint] = topologyv1.ActorType{ - Layer: "network", - Identity: []string{"id"}, - MergeIdentity: []string{"mac_addresses", "ip_addresses"}, - AggregationScopes: []string{"endpoint", "network"}, - Search: &topologyv1.ActorSearchPolicy{Columns: []string{"display_name"}}, - Presentation: &topologyv1.ActorPresentation{ - Label: "Inferred endpoint", - Role: "endpoint", - Icon: "remote-endpoint", - ColorSlot: "derived", - Border: &topologyv1.BorderPresentation{Enabled: new(true)}, - Size: &topologyv1.ActorSizePresentation{Mode: "fixed", Scale: "compact"}, - Layout: &topologyv1.ActorLayoutPresentation{Repulsion: "weaker"}, - LabelPolicy: &topologyv1.LabelPolicy{ - Columns: []string{"display_name"}, - Fallback: "type_label", - MaxLength: 80, - Array: "reject", - }, - Modal: snmpTopologyV1EndpointModal(), - }, - } - types[snmpTopologyV1ActorSegment] = topologyv1.ActorType{ - Layer: "network", - Identity: []string{"id"}, - MergeIdentity: []string{"id"}, - ParentIdentity: []string{"parent_devices"}, - AggregationScopes: []string{"segment", "network"}, - Search: &topologyv1.ActorSearchPolicy{Enabled: new(false)}, - Presentation: &topologyv1.ActorPresentation{ - Label: "Network segment", - Role: "group", - Icon: "segment", - ColorSlot: "dim", - Size: &topologyv1.ActorSizePresentation{Mode: "fixed", Scale: "compact"}, - Layout: &topologyv1.ActorLayoutPresentation{Repulsion: "weakest"}, - LabelPolicy: &topologyv1.LabelPolicy{ - Columns: []string{"display_name"}, - Fallback: "type_label", - MaxLength: 80, - Array: "reject", - }, - Modal: snmpTopologyV1EndpointModal(), - }, - } - types["custom"] = topologyv1.ActorType{ - Layer: "custom", - Identity: []string{"id"}, - MergeIdentity: []string{"id"}, - AggregationScopes: []string{"network"}, - Search: &topologyv1.ActorSearchPolicy{Columns: []string{"display_name"}}, - Presentation: &topologyv1.ActorPresentation{ - Label: "Custom", - Role: "actor", - Icon: "service", - ColorSlot: "neutral", - LabelPolicy: &topologyv1.LabelPolicy{ - Columns: []string{"display_name"}, - Fallback: "type_label", - MaxLength: 80, - Array: "reject", - }, - Modal: snmpTopologyV1EndpointModal(), - }, - } - return types -} - -func snmpTopologyV1DeviceModal() *topologyv1.ModalPresentation { - return &topologyv1.ModalPresentation{ - Labels: snmpTopologyV1DeviceModalLabels(), - MiniTopology: &topologyv1.ModalMiniTopologyPresentation{Depth: 1}, - Sections: []topologyv1.ModalSection{ - { - ID: "ports", - Label: "Ports", - Order: 1, - Source: topologyv1.ModalSource{ - Kind: "actor_table", - Table: "actor_ports", - }, - OwnerFilter: &topologyv1.ModalOwnerFilter{ - Mode: "actor_column", - ActorColumn: "actor", - }, - Columns: snmpTopologyV1PortModalColumns(), - Sort: &topologyv1.ModalSort{Column: "if_index", Direction: "asc"}, - }, - snmpTopologyV1PortLinksSection(2), - }, - } -} - -func snmpTopologyV1EndpointModal() *topologyv1.ModalPresentation { - return &topologyv1.ModalPresentation{ - Labels: snmpTopologyV1EndpointModalLabels(), - MiniTopology: &topologyv1.ModalMiniTopologyPresentation{Depth: 1}, - Sections: []topologyv1.ModalSection{snmpTopologyV1LinksSection(1)}, - } -} - -func snmpTopologyV1DeviceModalLabels() *topologyv1.ModalLabelsPresentation { - return &topologyv1.ModalLabelsPresentation{ - Table: "actor_labels", - Identification: &topologyv1.ModalLabelIdentificationPresentation{ - Fields: []topologyv1.ModalLabelIdentificationField{ - {Key: "display_name", Label: "Name", MaxValues: 1}, - {Key: "management_ip", Label: "Management IP", MaxValues: 1}, - {Key: "vendor", Label: "Vendor", MaxValues: 1}, - {Key: "model", Label: "Model", MaxValues: 1}, - {Key: "ports_total", Label: "Ports", MaxValues: 1}, - {Key: "lldp_neighbor_count", Label: "LLDP", MaxValues: 1}, - {Key: "cdp_neighbor_count", Label: "CDP", MaxValues: 1}, - }, - }, - } -} - -func snmpTopologyV1EndpointModalLabels() *topologyv1.ModalLabelsPresentation { - return &topologyv1.ModalLabelsPresentation{ - Table: "actor_labels", - Identification: &topologyv1.ModalLabelIdentificationPresentation{ - Fields: []topologyv1.ModalLabelIdentificationField{ - {Key: "display_name", Label: "Name", MaxValues: 1}, - {Key: "ip_address", Label: "IP", MaxValues: 2}, - {Key: "mac_address", Label: "MAC", MaxValues: 2}, - {Key: "hostname", Label: "Hostname", MaxValues: 2}, - }, - }, - } -} - -func snmpTopologyV1LinksSection(order int) topologyv1.ModalSection { - return topologyv1.ModalSection{ - ID: "links", - Label: "Links", - Order: order, - Source: topologyv1.ModalSource{ - Kind: "links", - }, - OwnerFilter: &topologyv1.ModalOwnerFilter{ - Mode: "incident_link", - SrcActorColumn: "src_actor", - DstActorColumn: "dst_actor", - }, - Columns: []topologyv1.ModalColumn{ - { - ID: "remote", - Label: "Remote Actor", - Projection: topologyv1.ModalProjection{ - Kind: "opposite_actor", - SrcActorColumn: "src_actor", - DstActorColumn: "dst_actor", - }, - Cell: "actor_link", - }, - modalSelectedSidePortColumn("local_port", "Local Port", "src_port_name", "dst_port_name"), - modalSelectedSidePortColumn("remote_port", "Remote Port", "dst_port_name", "src_port_name"), - modalDirectColumn("protocol", "Protocol", "protocol", "badge"), - modalDirectColumn("direction", "Direction", "direction", "text"), - modalDirectColumn("state", "State", "state", "badge"), - modalDirectColumn("evidence_count", "Evidence", "evidence_count", "number"), - }, - } -} - -func snmpTopologyV1PortLinksSection(order int) topologyv1.ModalSection { - return topologyv1.ModalSection{ - ID: "port_neighbors", - Label: "Port Neighbors", - Order: order, - Source: topologyv1.ModalSource{ - Kind: "actor_table", - Table: "actor_port_links", - }, - OwnerFilter: &topologyv1.ModalOwnerFilter{ - Mode: "actor_column", - ActorColumn: "actor", - }, - Columns: snmpTopologyV1PortLinkModalColumns(), - Sort: &topologyv1.ModalSort{Column: "if_index", Direction: "asc"}, - EmptyLabel: "No port neighbors", - } -} - -func modalSelectedSidePortColumn(id, label, selectedSrcPortColumn, selectedDstPortColumn string) topologyv1.ModalColumn { - return topologyv1.ModalColumn{ - ID: id, - Label: label, - Projection: topologyv1.ModalProjection{ - Kind: "selected_side_endpoint", - SrcActorColumn: "src_actor", - DstActorColumn: "dst_actor", - LocalPortColumn: selectedSrcPortColumn, - RemotePortColumn: selectedDstPortColumn, - }, - Cell: "text", - } -} - -func modalDirectColumn(id, label, sourceColumn, cell string) topologyv1.ModalColumn { - return topologyv1.ModalColumn{ - ID: id, - Label: label, - Projection: topologyv1.ModalProjection{Kind: "direct", Column: sourceColumn}, - Cell: cell, - } -} - -func modalDirectColumnWithVisibility(id, label, sourceColumn, cell, visibility string) topologyv1.ModalColumn { - column := modalDirectColumn(id, label, sourceColumn, cell) - column.Visibility = visibility - return column -} - -func modalActorRefColumn(id, label, actorColumn string) topologyv1.ModalColumn { - return topologyv1.ModalColumn{ - ID: id, - Label: label, - Projection: topologyv1.ModalProjection{ - Kind: "actor_ref_label", - ActorColumn: actorColumn, - }, - Cell: "actor_link", - } -} - -func modalActorRefColumnWithVisibility(id, label, actorColumn, visibility string) topologyv1.ModalColumn { - column := modalActorRefColumn(id, label, actorColumn) - column.Visibility = visibility - return column -} - -func snmpTopologyV1PortTypes() map[string]topologyv1.PortType { - return map[string]topologyv1.PortType{ - "lldp": {Presentation: &topologyv1.PortPresentation{Label: "lldp/cdp", ColorSlot: "accent", Opacity: "normal"}}, - "switch_facing": {Presentation: &topologyv1.PortPresentation{Label: "switch-facing", ColorSlot: "primary", Opacity: "normal"}}, - "host_facing": {Presentation: &topologyv1.PortPresentation{Label: "host-facing", ColorSlot: "secondary", Opacity: "normal"}}, - "host_candidate": {Presentation: &topologyv1.PortPresentation{Label: "host-candidate", ColorSlot: "info", Opacity: "normal"}}, - "trunk": {Presentation: &topologyv1.PortPresentation{Label: "trunk", ColorSlot: "warning", Opacity: "normal"}}, - "access": {Presentation: &topologyv1.PortPresentation{Label: "access", ColorSlot: "derived", Opacity: "normal"}}, - "topology": {Presentation: &topologyv1.PortPresentation{Label: "unclassified", ColorSlot: "neutral", Opacity: "normal"}}, - "idle": {Presentation: &topologyv1.PortPresentation{Label: "idle", ColorSlot: "muted", Opacity: "muted"}}, - "unknown": {Presentation: &topologyv1.PortPresentation{Label: "unknown", ColorSlot: "dim", Opacity: "muted"}}, - } -} - -type snmpTopologyV1LinkTypeSpec struct { - id string - label string - colorSlot string - lineStyle string - width string - semanticRole string -} - -func snmpTopologyV1LinkTypeSpecs() []snmpTopologyV1LinkTypeSpec { - return []snmpTopologyV1LinkTypeSpec{ - {id: snmpTopologyV1LinkLLDP, label: "LLDP", colorSlot: "accent", lineStyle: "solid", width: "thick", semanticRole: "discovery"}, - {id: snmpTopologyV1LinkCDP, label: "CDP", colorSlot: "accent", lineStyle: "solid", width: "thick", semanticRole: "discovery"}, - {id: snmpTopologyV1LinkBridge, label: "Bridge", colorSlot: "neutral", lineStyle: "solid", width: "normal", semanticRole: "normal"}, - {id: snmpTopologyV1LinkFDB, label: "FDB", colorSlot: "neutral", lineStyle: "solid", width: "normal", semanticRole: "normal"}, - {id: snmpTopologyV1LinkSTP, label: "STP", colorSlot: "muted", lineStyle: "solid", width: "normal", semanticRole: "normal"}, - {id: snmpTopologyV1LinkARP, label: "ARP", colorSlot: "muted", lineStyle: "solid", width: "normal", semanticRole: "normal"}, - {id: snmpTopologyV1LinkSNMP, label: "SNMP", colorSlot: "primary", lineStyle: "solid", width: "normal", semanticRole: "normal"}, - {id: snmpTopologyV1LinkProbable, label: "Probable", colorSlot: "dim", lineStyle: "solid", width: "normal", semanticRole: "normal"}, - {id: snmpTopologyV1LinkObservation, label: "L2 observation", colorSlot: "neutral", lineStyle: "solid", width: "normal", semanticRole: "normal"}, - } -} - -func snmpTopologyV1LinkTypes() map[string]topologyv1.LinkType { - types := make(map[string]topologyv1.LinkType) - for _, spec := range snmpTopologyV1LinkTypeSpecs() { - types[spec.id] = topologyv1.LinkType{ - Orientation: "observed_bidirectional", - DirectionRole: "observation", - SemanticRole: spec.semanticRole, - Aggregation: topologyv1.LinkAggregation{ - Direction: "canonicalize_unordered", - Evidence: "append", - Metrics: map[string]string{ - "evidence_count": "sum", - }, - }, - EvidenceTypes: []string{spec.id}, - Presentation: &topologyv1.LinkPresentation{ - Label: spec.label, - ColorSlot: spec.colorSlot, - LineStyle: spec.lineStyle, - Width: spec.width, - Curve: "straight", - Arrow: "none", - }, - } - } - return types -} - -func snmpTopologyV1EvidenceTypes() map[string]topologyv1.EvidenceType { - types := make(map[string]topologyv1.EvidenceType) - for _, spec := range snmpTopologyV1LinkTypeSpecs() { - types[spec.id] = topologyv1.EvidenceType{ - LinkType: spec.id, - Role: "observation_evidence", - Columns: snmpTopologyV1EvidenceColumns(), - MatchColumns: []string{ - "src_actor", - "dst_actor", - "protocol", - "src_endpoint", - "dst_endpoint", - }, - } - } - return types -} - -func snmpTopologyV1Presentation() *topologyv1.Presentation { - return &topologyv1.Presentation{ - ProfileVersion: "snmp-l2.v1", - Selection: &topologyv1.SelectionPresentation{ - ActorClick: &topologyv1.ActorClickPresentation{Mode: "highlight_connections"}, - }, - Legend: &topologyv1.PresentationLegend{ - Actors: []topologyv1.LegendEntry{ - {Type: "router", Label: "Router"}, - {Type: "switch", Label: "Switch"}, - {Type: "firewall", Label: "Firewall"}, - {Type: "access_point", Label: "Access Point"}, - {Type: "server", Label: "Server"}, - {Type: "storage", Label: "Storage"}, - {Type: "load_balancer", Label: "Load Balancer"}, - {Type: "printer", Label: "Printer"}, - {Type: "phone", Label: "IP Phone"}, - {Type: "ups", Label: "UPS / PDU"}, - {Type: "camera", Label: "Camera / Media"}, - {Type: "device", Label: "Other device"}, - {Type: "custom", Label: "Other"}, - {Type: "endpoint", Label: "Inferred endpoint"}, - {Type: "segment", Label: "Network segment"}, - }, - Links: []topologyv1.LegendEntry{ - {Type: snmpTopologyV1LinkLLDP, Label: "LLDP"}, - {Type: snmpTopologyV1LinkCDP, Label: "CDP"}, - {Type: snmpTopologyV1LinkSNMP, Label: "SNMP"}, - {Type: snmpTopologyV1LinkBridge, Label: "Bridge"}, - {Type: snmpTopologyV1LinkProbable, Label: "Probable"}, - }, - Ports: []topologyv1.LegendEntry{ - {Type: "lldp", Label: "lldp/cdp"}, - {Type: "switch_facing", Label: "switch-facing"}, - {Type: "host_facing", Label: "host-facing"}, - {Type: "host_candidate", Label: "host-candidate"}, - {Type: "trunk", Label: "trunk"}, - {Type: "access", Label: "access"}, - {Type: "topology", Label: "unclassified"}, - {Type: "idle", Label: "idle"}, - {Type: "unknown", Label: "unknown"}, - }, - }, - PortFields: []topologyv1.PresentationField{ - {Key: "type", Label: "Type"}, - {Key: "role", Label: "Role"}, - {Key: "status", Label: "Status"}, - {Key: "mode", Label: "Mode"}, - {Key: "sources", Label: "Sources"}, - }, - } -} - func snmpTopologyToV1(data topologyData) (topologyv1.Data, error) { stringsDict := topologyv1.NewStringDictionary("") actorRows, actorIndex := buildSNMPTopologyV1Actors(data.Actors, stringsDict) @@ -574,1565 +133,3 @@ func snmpTopologyToV1(data topologyData) (topologyv1.Data, error) { } return payload, nil } - -func snmpTopologyV1ActorPortsTableType() topologyv1.TableType { - return topologyv1.TableType{ - Role: "actor_detail", - Owner: "actor", - Aggregation: "append", - Columns: snmpTopologyV1ActorPortsColumns(), - Presentation: &topologyv1.TableTypePresentation{ - Label: "Ports", - Order: 1, - Columns: snmpTopologyV1PortModalColumns(), - }, - } -} - -func snmpTopologyV1PortModalColumns() []topologyv1.ModalColumn { - return []topologyv1.ModalColumn{ - modalDirectColumn("if_index", "Port ID", "if_index", "number"), - modalDirectColumn("name", "Port", "name", "text"), - modalDirectColumn("oper_status", "Status", "oper_status", "badge"), - modalDirectColumn("admin_status", "Admin", "admin_status", "badge"), - modalDirectColumn("port_type", "Type", "port_type", "badge"), - modalDirectColumn("link_mode", "Mode", "link_mode", "badge"), - modalDirectColumn("topology_role", "Role", "topology_role", "badge"), - modalDirectColumn("vlan_ids", "VLANs", "vlan_ids", "array_count"), - modalDirectColumn("fdb_mac_count", "FDB", "fdb_mac_count", "number"), - modalDirectColumn("link_count", "Links", "link_count", "number"), - modalDirectColumn("neighbor_count", "Neighbors", "neighbor_count", "number"), - modalActorRefColumnWithVisibility("neighbor_actor", "Neighbor", "neighbor_actor", "expanded"), - modalDirectColumnWithVisibility("neighbor_port_name", "Neighbor Port", "neighbor_port_name", "text", "expanded"), - modalDirectColumnWithVisibility("if_name", "ifName", "if_name", "text", "expanded"), - modalDirectColumnWithVisibility("if_descr", "ifDescr", "if_descr", "text", "expanded"), - modalDirectColumnWithVisibility("if_alias", "Alias", "if_alias", "text", "expanded"), - modalDirectColumnWithVisibility("port_id", "Source Port ID", "port_id", "text", "expanded"), - modalDirectColumnWithVisibility("mac", "MAC", "mac", "text", "expanded"), - modalDirectColumnWithVisibility("speed", "Speed", "speed", "number", "expanded"), - modalDirectColumnWithVisibility("stp_state", "STP", "stp_state", "badge", "expanded"), - modalDirectColumnWithVisibility("neighbors", "Neighbor Data", "neighbors", "debug_json", "debug"), - modalDirectColumnWithVisibility("vlans", "VLAN Data", "vlans", "debug_json", "debug"), - modalDirectColumnWithVisibility("extra", "Extra", "extra", "debug_json", "debug"), - } -} - -func snmpTopologyV1ActorPortLinksTableType() topologyv1.TableType { - return topologyv1.TableType{ - Role: "actor_detail", - Owner: "actor", - Aggregation: "append", - Columns: snmpTopologyV1ActorPortLinksColumns(), - Presentation: &topologyv1.TableTypePresentation{ - Label: "Port Neighbors", - Order: 2, - Columns: snmpTopologyV1PortLinkModalColumns(), - }, - } -} - -func snmpTopologyV1PortLinkModalColumns() []topologyv1.ModalColumn { - return []topologyv1.ModalColumn{ - modalDirectColumn("if_index", "Port ID", "if_index", "number"), - modalDirectColumn("port_name", "Port", "port_name", "text"), - modalActorRefColumn("remote_actor", "Remote Actor", "remote_actor"), - modalDirectColumn("remote_port_name", "Remote Port", "remote_port_name", "text"), - modalDirectColumn("type", "Type", "type", "badge"), - modalDirectColumn("state", "State", "state", "badge"), - modalDirectColumn("evidence_count", "Evidence", "evidence_count", "number"), - modalDirectColumnWithVisibility("protocol", "Protocol", "protocol", "badge", "expanded"), - modalDirectColumnWithVisibility("remote_if_index", "Remote Port ID", "remote_if_index", "number", "expanded"), - modalDirectColumnWithVisibility("port_id", "Source Port ID", "port_id", "text", "expanded"), - modalDirectColumnWithVisibility("remote_port_id", "Remote Source Port ID", "remote_port_id", "text", "expanded"), - modalDirectColumnWithVisibility("confidence", "Confidence", "confidence", "badge", "expanded"), - modalDirectColumnWithVisibility("inference", "Inference", "inference", "badge", "expanded"), - modalDirectColumnWithVisibility("attachment_mode", "Attachment", "attachment_mode", "badge", "expanded"), - modalDirectColumnWithVisibility("discovered_at", "Discovered", "discovered_at", "timestamp", "expanded"), - modalDirectColumnWithVisibility("last_seen", "Last Seen", "last_seen", "timestamp", "expanded"), - } -} - -func snmpTopologyV1ActorLabelsTableType() topologyv1.TableType { - return topologyv1.TableType{ - Role: "actor_inventory", - Owner: "actor", - Aggregation: "set", - Columns: []topologyv1.Column{ - topologyv1.NewColumn("actor", "actor_ref", topologyv1.WithRole("reference")), - topologyv1.NewColumn("key", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithRole("attribute")), - topologyv1.NewColumn("value", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithRole("attribute")), - topologyv1.NewColumn("source", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable(), topologyv1.WithRole("attribute")), - topologyv1.NewColumn("kind", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable(), topologyv1.WithRole("attribute")), - topologyv1.NewColumn("value_index", "uint", topologyv1.WithNullable(), topologyv1.WithRole("attribute")), - }, - Presentation: &topologyv1.TableTypePresentation{ - Label: "Labels", - Order: 0, - Columns: []topologyv1.ModalColumn{ - modalDirectColumn("key", "Label", "key", "text"), - modalDirectColumn("value", "Value", "value", "text"), - modalDirectColumn("source", "Source", "source", "badge"), - modalDirectColumn("kind", "Kind", "kind", "badge"), - }, - }, - } -} - -func snmpTopologyV1ActorPortsColumns() []topologyv1.Column { - return []topologyv1.Column{ - topologyv1.NewColumn("actor", "actor_ref", topologyv1.WithRole("reference")), - topologyv1.NewColumn("if_index", "uint", topologyv1.WithNullable()), - topologyv1.NewColumn("port_id", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), - topologyv1.NewColumn("name", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), - topologyv1.NewColumn("if_name", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), - topologyv1.NewColumn("if_descr", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), - topologyv1.NewColumn("if_alias", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), - topologyv1.NewColumn("mac", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), - topologyv1.NewColumn("speed", "uint", topologyv1.WithNullable()), - topologyv1.NewColumn("topology_role", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), - topologyv1.NewColumn("oper_status", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), - topologyv1.NewColumn("admin_status", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), - topologyv1.NewColumn("port_type", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), - topologyv1.NewColumn("link_mode", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), - topologyv1.NewColumn("stp_state", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), - topologyv1.NewColumn("vlan_ids", "array", topologyv1.WithNullable()), - topologyv1.NewColumn("fdb_mac_count", "uint", topologyv1.WithNullable()), - topologyv1.NewColumn("link_count", "uint", topologyv1.WithNullable()), - topologyv1.NewColumn("neighbor_count", "uint", topologyv1.WithNullable()), - topologyv1.NewColumn("neighbor_actor", "actor_ref", topologyv1.WithNullable(), topologyv1.WithRole("reference")), - topologyv1.NewColumn("neighbor_port_name", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), - topologyv1.NewColumn("neighbors", "json", topologyv1.WithNullable()), - topologyv1.NewColumn("vlans", "json", topologyv1.WithNullable()), - topologyv1.NewColumn("extra", "json", topologyv1.WithNullable()), - } -} - -func snmpTopologyV1ActorPortLinksColumns() []topologyv1.Column { - return []topologyv1.Column{ - topologyv1.NewColumn("actor", "actor_ref", topologyv1.WithRole("reference")), - topologyv1.NewColumn("link", "link_ref", topologyv1.WithRole("reference")), - topologyv1.NewColumn("remote_actor", "actor_ref", topologyv1.WithRole("reference")), - topologyv1.NewColumn("if_index", "uint", topologyv1.WithNullable()), - topologyv1.NewColumn("port_id", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), - topologyv1.NewColumn("port_name", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), - topologyv1.NewColumn("remote_if_index", "uint", topologyv1.WithNullable()), - topologyv1.NewColumn("remote_port_id", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), - topologyv1.NewColumn("remote_port_name", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), - topologyv1.NewColumn("type", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithRole("group_key")), - topologyv1.NewColumn("protocol", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithRole("group_key")), - topologyv1.NewColumn("state", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), - topologyv1.NewColumn("evidence_count", "uint", topologyv1.WithAggregation("sum")), - topologyv1.NewColumn("confidence", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), - topologyv1.NewColumn("inference", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), - topologyv1.NewColumn("attachment_mode", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), - topologyv1.NewColumn("discovered_at", "timestamp", topologyv1.WithNullable(), topologyv1.WithRole("timestamp")), - topologyv1.NewColumn("last_seen", "timestamp", topologyv1.WithNullable(), topologyv1.WithRole("timestamp")), - } -} - -func buildSNMPTopologyV1Actors(actors []topologyActor, stringsDict *topologyv1.StringDictionary) (topologyv1.Table, map[string]int) { - actorIndex := make(map[string]int, len(actors)) - ids := make([]any, len(actors)) - types := make([]any, len(actors)) - layers := make([]any, len(actors)) - sources := make([]any, len(actors)) - displayNames := make([]any, len(actors)) - chassisIDs := make([]any, len(actors)) - macAddresses := make([]any, len(actors)) - ipAddresses := make([]any, len(actors)) - hostnames := make([]any, len(actors)) - dnsNames := make([]any, len(actors)) - sysObjectIDs := make([]any, len(actors)) - sysNames := make([]any, len(actors)) - parentDevices := make([]any, len(actors)) - vendors := make([]any, len(actors)) - models := make([]any, len(actors)) - sysDescrs := make([]any, len(actors)) - sysLocations := make([]any, len(actors)) - sysContacts := make([]any, len(actors)) - managementIPs := make([]any, len(actors)) - protocols := make([]any, len(actors)) - capabilities := make([]any, len(actors)) - portsTotal := make([]any, len(actors)) - vlanCounts := make([]any, len(actors)) - fdbTotalMACs := make([]any, len(actors)) - lldpNeighborCounts := make([]any, len(actors)) - cdpNeighborCounts := make([]any, len(actors)) - endpointsTotal := make([]any, len(actors)) - chartIDPrefixes := make([]any, len(actors)) - netdataHostIDs := make([]any, len(actors)) - - for i, actor := range actors { - actorID := strings.TrimSpace(actor.ActorID) - if actorID == "" { - actorID = fmt.Sprintf("generated:%d", i) - } - actorIndex[actorID] = i - ids[i] = stringsDict.Ref(actorID) - types[i] = stringsDict.Ref(snmpTopologyV1ActorType(actor.ActorType)) - layers[i] = stringsDict.Ref(snmpTopologyV1ActorLayer(actor)) - sources[i] = stringsDict.Ref(firstNonEmptyString(actor.Source, snmpTopologyV1ProducerSource)) - displayNames[i] = nullableStringRef(stringsDict, snmpTopologyV1DisplayName(actor)) - chassisIDs[i] = stringArrayCell(actor.Match.ChassisIDs) - macAddresses[i] = stringArrayCell(actor.Match.MacAddresses) - ipAddresses[i] = stringArrayCell(actor.Match.IPAddresses) - hostnames[i] = stringArrayCell(actor.Match.Hostnames) - dnsNames[i] = stringArrayCell(actor.Match.DNSNames) - sysObjectIDs[i] = stringsDict.Ref(actor.Match.SysObjectID) - sysNames[i] = stringsDict.Ref(actor.Match.SysName) - parentDevices[i] = stringArrayCell(anyStringSlice(actor.Attributes["parent_devices"])) - vendors[i] = nullableStringRef(stringsDict, firstNonEmptyString(anyStringValue(actor.Attributes["vendor"]), anyStringValue(actor.Attributes["vendor_derived"]))) - models[i] = nullableStringRef(stringsDict, anyStringValue(actor.Attributes["model"])) - sysDescrs[i] = nullableStringRef(stringsDict, anyStringValue(actor.Attributes["sys_descr"])) - sysLocations[i] = nullableStringRef(stringsDict, anyStringValue(actor.Attributes["sys_location"])) - sysContacts[i] = nullableStringRef(stringsDict, anyStringValue(actor.Attributes["sys_contact"])) - managementIPs[i] = nullableStringRef(stringsDict, anyStringValue(actor.Attributes["management_ip"])) - protocols[i] = stringArrayCell(anyStringSlice(actor.Attributes["protocols"])) - if isEmptyArrayCell(protocols[i]) { - // Older SNMP topology payloads used learned_sources for discovered protocols. - protocols[i] = stringArrayCell(anyStringSlice(actor.Attributes["learned_sources"])) - } - if isEmptyArrayCell(protocols[i]) { - protocols[i] = nil - } - capabilities[i] = stringArrayCell(anyStringSlice(actor.Attributes["capabilities"])) - if isEmptyArrayCell(capabilities[i]) { - capabilities[i] = nil - } - portsTotal[i] = nullableUintValue(actor.Attributes["ports_total"]) - vlanCounts[i] = nullableUintValue(actor.Attributes["vlan_count"]) - fdbTotalMACs[i] = nullableUintValue(actor.Attributes["fdb_total_macs"]) - lldpNeighborCounts[i] = nullableUintValue(actor.Attributes["lldp_neighbor_count"]) - cdpNeighborCounts[i] = nullableUintValue(actor.Attributes["cdp_neighbor_count"]) - endpointsTotal[i] = nullableUintValue(actor.Attributes["endpoints_total"]) - chartIDPrefixes[i] = nullableStringRef(stringsDict, anyStringValue(actor.Attributes["chart_id_prefix"])) - netdataHostIDs[i] = nullableStringRef(stringsDict, anyStringValue(actor.Attributes["netdata_host_id"])) - } - - return topologyv1.MustTable(len(actors), - []topologyv1.Column{ - topologyv1.NewColumn("id", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithRole("identity")), - topologyv1.NewColumn("type", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithRole("group_key")), - topologyv1.NewColumn("layer", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithRole("group_key")), - topologyv1.NewColumn("source", "string_ref", topologyv1.WithDictionary("strings")), - topologyv1.NewColumn("display_name", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable(), topologyv1.WithRole("attribute")), - topologyv1.NewColumn("chassis_ids", "array", topologyv1.WithRole("merge_identity")), - topologyv1.NewColumn("mac_addresses", "array", topologyv1.WithRole("merge_identity")), - topologyv1.NewColumn("ip_addresses", "array", topologyv1.WithRole("merge_identity")), - topologyv1.NewColumn("hostnames", "array"), - topologyv1.NewColumn("dns_names", "array"), - topologyv1.NewColumn("sys_object_id", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithRole("merge_identity")), - topologyv1.NewColumn("sys_name", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithRole("merge_identity")), - topologyv1.NewColumn("parent_devices", "array", topologyv1.WithRole("parent_identity")), - topologyv1.NewColumn("vendor", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), - topologyv1.NewColumn("model", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), - topologyv1.NewColumn("sys_descr", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), - topologyv1.NewColumn("sys_location", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), - topologyv1.NewColumn("sys_contact", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), - topologyv1.NewColumn("management_ip", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), - topologyv1.NewColumn("protocols", "array", topologyv1.WithNullable()), - topologyv1.NewColumn("capabilities", "array", topologyv1.WithNullable()), - topologyv1.NewColumn("ports_total", "uint", topologyv1.WithNullable()), - topologyv1.NewColumn("vlan_count", "uint", topologyv1.WithNullable()), - topologyv1.NewColumn("fdb_total_macs", "uint", topologyv1.WithNullable()), - topologyv1.NewColumn("lldp_neighbor_count", "uint", topologyv1.WithNullable()), - topologyv1.NewColumn("cdp_neighbor_count", "uint", topologyv1.WithNullable()), - topologyv1.NewColumn("endpoints_total", "uint", topologyv1.WithNullable()), - topologyv1.NewColumn("chart_id_prefix", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), - topologyv1.NewColumn("netdata_host_id", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), - }, - []topologyv1.ColumnEncoding{ - topologyv1.Values(ids...), - topologyv1.Values(types...), - topologyv1.Values(layers...), - topologyv1.Values(sources...), - topologyv1.Values(displayNames...), - topologyv1.Values(chassisIDs...), - topologyv1.Values(macAddresses...), - topologyv1.Values(ipAddresses...), - topologyv1.Values(hostnames...), - topologyv1.Values(dnsNames...), - topologyv1.Values(sysObjectIDs...), - topologyv1.Values(sysNames...), - topologyv1.Values(parentDevices...), - topologyv1.Values(vendors...), - topologyv1.Values(models...), - topologyv1.Values(sysDescrs...), - topologyv1.Values(sysLocations...), - topologyv1.Values(sysContacts...), - topologyv1.Values(managementIPs...), - topologyv1.Values(protocols...), - topologyv1.Values(capabilities...), - topologyv1.Values(portsTotal...), - topologyv1.Values(vlanCounts...), - topologyv1.Values(fdbTotalMACs...), - topologyv1.Values(lldpNeighborCounts...), - topologyv1.Values(cdpNeighborCounts...), - topologyv1.Values(endpointsTotal...), - topologyv1.Values(chartIDPrefixes...), - topologyv1.Values(netdataHostIDs...), - }, - ), actorIndex -} - -func buildSNMPTopologyV1Links( - links []topologyLink, - actorIndex map[string]int, - stringsDict *topologyv1.StringDictionary, -) (topologyv1.Table, topologyv1.EvidenceMap, error) { - srcActors := make([]any, len(links)) - dstActors := make([]any, len(links)) - linkTypes := make([]any, len(links)) - protocols := make([]any, len(links)) - directions := make([]any, len(links)) - states := make([]any, len(links)) - srcPortNames := make([]any, len(links)) - dstPortNames := make([]any, len(links)) - evidenceCounts := make([]any, len(links)) - discoveredAt := make([]any, len(links)) - lastSeen := make([]any, len(links)) - evidenceRowsByType := make(map[string]*snmpTopologyV1EvidenceRows) - - for i, link := range links { - src, ok := actorIndex[strings.TrimSpace(link.SrcActorID)] - if !ok { - return topologyv1.Table{}, nil, fmt.Errorf("link %d references unknown source actor %q", i, link.SrcActorID) - } - dst, ok := actorIndex[strings.TrimSpace(link.DstActorID)] - if !ok { - return topologyv1.Table{}, nil, fmt.Errorf("link %d references unknown destination actor %q", i, link.DstActorID) - } - protocol := firstNonEmptyString(link.Protocol, link.LinkType, "l2") - linkType := snmpTopologyV1LinkType(link) - srcActors[i] = src - dstActors[i] = dst - linkTypes[i] = stringsDict.Ref(linkType) - protocols[i] = stringsDict.Ref(protocol) - directions[i] = stringsDict.Ref(firstNonEmptyString(link.Direction, "observed")) - states[i] = nullableStringRef(stringsDict, link.State) - srcPortNames[i] = nullableStringRef(stringsDict, topologyV1EndpointPortName(link.Src)) - dstPortNames[i] = nullableStringRef(stringsDict, topologyV1EndpointPortName(link.Dst)) - evidenceCounts[i] = 1 - discoveredAt[i] = nullableTime(link.DiscoveredAt) - lastSeen[i] = nullableTime(link.LastSeen) - srcEndpoint := nullableJSON(link.Src.Attributes) - dstEndpoint := nullableJSON(link.Dst.Attributes) - metrics := nullableJSON(link.Metrics) - - evidenceRows := evidenceRowsByType[linkType] - if evidenceRows == nil { - evidenceRows = &snmpTopologyV1EvidenceRows{} - evidenceRowsByType[linkType] = evidenceRows - } - evidenceRows.linkRefs = append(evidenceRows.linkRefs, i) - evidenceRows.srcActors = append(evidenceRows.srcActors, src) - evidenceRows.dstActors = append(evidenceRows.dstActors, dst) - evidenceRows.protocols = append(evidenceRows.protocols, stringsDict.Ref(protocol)) - evidenceRows.directions = append(evidenceRows.directions, stringsDict.Ref(firstNonEmptyString(link.Direction, "observed"))) - evidenceRows.states = append(evidenceRows.states, nullableStringRef(stringsDict, link.State)) - evidenceRows.srcPortNames = append(evidenceRows.srcPortNames, nullableStringRef(stringsDict, topologyV1EndpointPortName(link.Src))) - evidenceRows.dstPortNames = append(evidenceRows.dstPortNames, nullableStringRef(stringsDict, topologyV1EndpointPortName(link.Dst))) - evidenceRows.srcIfIndexes = append(evidenceRows.srcIfIndexes, nullableUintValue(link.Src.Attributes["if_index"])) - evidenceRows.dstIfIndexes = append(evidenceRows.dstIfIndexes, nullableUintValue(link.Dst.Attributes["if_index"])) - evidenceRows.srcManagementIPs = append(evidenceRows.srcManagementIPs, nullableStringRef(stringsDict, topologyV1EndpointString(link.Src, "management_ip"))) - evidenceRows.dstManagementIPs = append(evidenceRows.dstManagementIPs, nullableStringRef(stringsDict, topologyV1EndpointString(link.Dst, "management_ip"))) - evidenceRows.confidences = append(evidenceRows.confidences, nullableStringRef(stringsDict, topologyMetricValueString(link.Metrics, "confidence"))) - evidenceRows.inferences = append(evidenceRows.inferences, nullableStringRef(stringsDict, topologyMetricValueString(link.Metrics, "inference"))) - evidenceRows.attachmentModes = append(evidenceRows.attachmentModes, nullableStringRef(stringsDict, topologyMetricValueString(link.Metrics, "attachment_mode"))) - evidenceRows.srcEndpoints = append(evidenceRows.srcEndpoints, srcEndpoint) - evidenceRows.dstEndpoints = append(evidenceRows.dstEndpoints, dstEndpoint) - evidenceRows.metrics = append(evidenceRows.metrics, metrics) - } - - linkTable := topologyv1.MustTable(len(links), - []topologyv1.Column{ - topologyv1.NewColumn("src_actor", "actor_ref", topologyv1.WithRole("reference")), - topologyv1.NewColumn("dst_actor", "actor_ref", topologyv1.WithRole("reference")), - topologyv1.NewColumn("type", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithRole("group_key")), - topologyv1.NewColumn("protocol", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithRole("group_key")), - topologyv1.NewColumn("direction", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithRole("group_key")), - topologyv1.NewColumn("state", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), - topologyv1.NewColumn("src_port_name", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), - topologyv1.NewColumn("dst_port_name", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), - topologyv1.NewColumn("evidence_count", "uint", topologyv1.WithAggregation("sum")), - topologyv1.NewColumn("discovered_at", "timestamp", topologyv1.WithNullable(), topologyv1.WithRole("timestamp")), - topologyv1.NewColumn("last_seen", "timestamp", topologyv1.WithNullable(), topologyv1.WithRole("timestamp")), - }, - []topologyv1.ColumnEncoding{ - topologyv1.Values(srcActors...), - topologyv1.Values(dstActors...), - topologyv1.Values(linkTypes...), - topologyv1.Values(protocols...), - topologyv1.Values(directions...), - topologyv1.Values(states...), - topologyv1.Values(srcPortNames...), - topologyv1.Values(dstPortNames...), - topologyv1.Values(evidenceCounts...), - topologyv1.Values(discoveredAt...), - topologyv1.Values(lastSeen...), - }, - ) - - evidenceSections := make(topologyv1.EvidenceMap, len(evidenceRowsByType)) - for linkType, rows := range evidenceRowsByType { - evidenceSections[linkType] = topologyv1.EvidenceSection{ - Type: linkType, - Table: rows.table(), - } - } - - return linkTable, evidenceSections, nil -} - -type snmpTopologyV1EvidenceRows struct { - linkRefs []any - srcActors []any - dstActors []any - protocols []any - directions []any - states []any - srcPortNames []any - dstPortNames []any - srcIfIndexes []any - dstIfIndexes []any - srcManagementIPs []any - dstManagementIPs []any - confidences []any - inferences []any - attachmentModes []any - srcEndpoints []any - dstEndpoints []any - metrics []any -} - -func (rows *snmpTopologyV1EvidenceRows) table() topologyv1.Table { - return topologyv1.MustTable(len(rows.linkRefs), - snmpTopologyV1EvidenceColumns(), - []topologyv1.ColumnEncoding{ - topologyv1.Values(rows.linkRefs...), - topologyv1.Values(rows.srcActors...), - topologyv1.Values(rows.dstActors...), - topologyv1.Values(rows.protocols...), - topologyv1.Values(rows.directions...), - topologyv1.Values(rows.states...), - topologyv1.Values(rows.srcPortNames...), - topologyv1.Values(rows.dstPortNames...), - topologyv1.Values(rows.srcIfIndexes...), - topologyv1.Values(rows.dstIfIndexes...), - topologyv1.Values(rows.srcManagementIPs...), - topologyv1.Values(rows.dstManagementIPs...), - topologyv1.Values(rows.confidences...), - topologyv1.Values(rows.inferences...), - topologyv1.Values(rows.attachmentModes...), - topologyv1.Values(rows.srcEndpoints...), - topologyv1.Values(rows.dstEndpoints...), - topologyv1.Values(rows.metrics...), - }, - ) -} - -func snmpTopologyV1EvidenceColumns() []topologyv1.Column { - return []topologyv1.Column{ - topologyv1.NewColumn("link", "link_ref", topologyv1.WithRole("reference")), - topologyv1.NewColumn("src_actor", "actor_ref", topologyv1.WithRole("reference")), - topologyv1.NewColumn("dst_actor", "actor_ref", topologyv1.WithRole("reference")), - topologyv1.NewColumn("protocol", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithRole("group_key")), - topologyv1.NewColumn("direction", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithRole("group_key")), - topologyv1.NewColumn("state", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), - topologyv1.NewColumn("src_port_name", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), - topologyv1.NewColumn("dst_port_name", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), - topologyv1.NewColumn("src_if_index", "uint", topologyv1.WithNullable()), - topologyv1.NewColumn("dst_if_index", "uint", topologyv1.WithNullable()), - topologyv1.NewColumn("src_management_ip", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), - topologyv1.NewColumn("dst_management_ip", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), - topologyv1.NewColumn("confidence", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), - topologyv1.NewColumn("inference", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), - topologyv1.NewColumn("attachment_mode", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), - topologyv1.NewColumn("src_endpoint", "json", topologyv1.WithNullable()), - topologyv1.NewColumn("dst_endpoint", "json", topologyv1.WithNullable()), - topologyv1.NewColumn("metrics", "json", topologyv1.WithNullable()), - } -} - -func snmpTopologyV1LinkType(link topologyLink) string { - if snmpTopologyV1LinkIsProbable(link) { - return snmpTopologyV1LinkProbable - } - - switch strings.ToLower(strings.TrimSpace(firstNonEmptyString(link.Protocol, link.LinkType))) { - case snmpTopologyV1LinkLLDP: - return snmpTopologyV1LinkLLDP - case snmpTopologyV1LinkCDP: - return snmpTopologyV1LinkCDP - case snmpTopologyV1LinkBridge: - return snmpTopologyV1LinkBridge - case snmpTopologyV1LinkFDB: - return snmpTopologyV1LinkFDB - case snmpTopologyV1LinkSTP: - return snmpTopologyV1LinkSTP - case snmpTopologyV1LinkARP: - return snmpTopologyV1LinkARP - case snmpTopologyV1LinkSNMP: - return snmpTopologyV1LinkSNMP - default: - return snmpTopologyV1LinkObservation - } -} - -func snmpTopologyV1LinkIsProbable(link topologyLink) bool { - if strings.EqualFold(strings.TrimSpace(link.State), snmpTopologyV1LinkProbable) { - return true - } - if len(link.Metrics) == 0 { - return false - } - if strings.EqualFold(topologyMetricValueString(link.Metrics, "inference"), snmpTopologyV1LinkProbable) { - return true - } - return strings.HasPrefix(strings.ToLower(topologyMetricValueString(link.Metrics, "attachment_mode")), snmpTopologyV1LinkProbable+"_") -} - -func buildSNMPTopologyV1ActorDetails( - actors []topologyActor, - stringsDict *topologyv1.StringDictionary, - portNeighborSummaries map[snmpTopologyV1PortNeighborKey]snmpTopologyV1PortNeighborSummary, -) (map[string]topologyv1.DetailTable, map[string]topologyv1.TableType, error) { - details := make(map[string]topologyv1.DetailTable) - tableTypes := make(map[string]topologyv1.TableType) - - labelsTable := buildSNMPTopologyV1ActorLabelsTable(actors, stringsDict) - details["actor_labels"] = topologyv1.DetailTable{ - Type: "actor_labels", - Table: labelsTable, - } - tableTypes["actor_labels"] = snmpTopologyV1ActorLabelsTableType() - usedTableIDs := map[string]struct{}{ - "actor_labels": {}, - "actor_port_links": {}, - } - - metadataTable := buildSNMPTopologyV1ActorMetadataTable(actors) - if metadataTable.Rows > 0 { - tableID := "actor_metadata" - details[tableID] = topologyv1.DetailTable{ - Type: tableID, - Table: metadataTable, - } - tableTypes[tableID] = topologyv1.TableType{ - Role: "actor_detail", - Owner: "actor", - Aggregation: "append", - Columns: metadataTable.Columns, - Presentation: &topologyv1.TableTypePresentation{ - Label: "Debug metadata", - DefaultVisibility: "debug", - Columns: []topologyv1.ModalColumn{ - modalDirectColumn("attributes", "Attributes", "attributes", "debug_json"), - modalDirectColumn("labels", "Labels", "labels", "debug_json"), - }, - }, - } - usedTableIDs[tableID] = struct{}{} - } - - tableRowsByName := collectSNMPTopologyV1ActorTableRows(actors) - tableNames := sortedMapKeys(tableRowsByName) - reservedCustomTableIDs := snmpTopologyV1ReservedCustomTableIDs(tableNames) - for _, tableName := range tableNames { - rows := tableRowsByName[tableName] - if len(rows) == 0 { - continue - } - tableID := snmpTopologyV1ActorDetailTableID(tableName, usedTableIDs, reservedCustomTableIDs) - var table topologyv1.Table - var err error - if tableID == "actor_ports" { - table = buildSNMPTopologyV1ActorPortsTable(rows, stringsDict, portNeighborSummaries) - } else { - table, err = buildSNMPTopologyV1DynamicTable(rows, stringsDict) - } - if err != nil { - return nil, nil, fmt.Errorf("build actor detail table %q: %w", tableName, err) - } - details[tableID] = topologyv1.DetailTable{ - Type: tableID, - Table: table, - } - if tableID == "actor_ports" { - tableTypes[tableID] = snmpTopologyV1ActorPortsTableType() - } else { - tableTypes[tableID] = topologyv1.TableType{ - Role: "actor_detail", - Owner: "actor", - Aggregation: "append", - Columns: table.Columns, - } - } - usedTableIDs[tableID] = struct{}{} - } - return details, tableTypes, nil -} - -func snmpTopologyV1ReservedCustomTableIDs(tableNames []string) map[string]struct{} { - reserved := make(map[string]struct{}) - for _, tableName := range tableNames { - tableID := topologyID("actor_"+tableName, "actor_detail") - switch tableID { - case "actor_labels", "actor_metadata", "actor_port_links": - reserved[topologyID("actor_custom_"+tableName, "actor_detail")] = struct{}{} - } - } - return reserved -} - -func snmpTopologyV1ActorDetailTableID(tableName string, usedTableIDs, reservedCustomTableIDs map[string]struct{}) string { - tableID := topologyID("actor_"+tableName, "actor_detail") - switch tableID { - case "actor_labels", "actor_metadata", "actor_port_links": - return snmpTopologyV1UniqueActorDetailTableID(topologyID("actor_custom_"+tableName, "actor_detail"), usedTableIDs) - default: - if _, reserved := reservedCustomTableIDs[tableID]; reserved { - return snmpTopologyV1UniqueActorDetailTableID(topologyID("actor_detail_"+tableName, "actor_detail"), usedTableIDs) - } - return snmpTopologyV1UniqueActorDetailTableID(tableID, usedTableIDs) - } -} - -func snmpTopologyV1UniqueActorDetailTableID(tableID string, usedTableIDs map[string]struct{}) string { - if _, ok := usedTableIDs[tableID]; !ok { - return tableID - } - for suffix := 2; ; suffix++ { - candidate := fmt.Sprintf("%s_%d", tableID, suffix) - if _, ok := usedTableIDs[candidate]; !ok { - return candidate - } - } -} - -func buildSNMPTopologyV1ActorMetadataTable(actors []topologyActor) topologyv1.Table { - actorRefs := make([]any, 0, len(actors)) - attributes := make([]any, 0, len(actors)) - labels := make([]any, 0, len(actors)) - for i, actor := range actors { - if len(actor.Attributes) == 0 && len(actor.Labels) == 0 { - continue - } - actorRefs = append(actorRefs, i) - attributes = append(attributes, nullableJSON(actor.Attributes)) - labels = append(labels, nullableJSON(actor.Labels)) - } - if len(actorRefs) == 0 { - return topologyv1.EmptyTable() - } - return topologyv1.MustTable(len(actorRefs), - []topologyv1.Column{ - topologyv1.NewColumn("actor", "actor_ref", topologyv1.WithRole("reference")), - topologyv1.NewColumn("attributes", "json", topologyv1.WithNullable()), - topologyv1.NewColumn("labels", "json", topologyv1.WithNullable()), - }, - []topologyv1.ColumnEncoding{ - topologyv1.Values(actorRefs...), - topologyv1.Values(attributes...), - topologyv1.Values(labels...), - }, - ) -} - -func buildSNMPTopologyV1ActorLabelsTable( - actors []topologyActor, - stringsDict *topologyv1.StringDictionary, -) topologyv1.Table { - type labelRow struct { - actor int - key string - value string - source string - kind string - valueIndex any - } - - rows := make([]labelRow, 0, len(actors)*8) - add := func(actor int, key, value, source, kind string, valueIndex any) { - key = strings.TrimSpace(key) - value = strings.TrimSpace(value) - if key == "" || value == "" { - return - } - rows = append(rows, labelRow{ - actor: actor, - key: key, - value: value, - source: source, - kind: kind, - valueIndex: valueIndex, - }) - } - addSlice := func(actor int, key string, values []string, source, kind string) { - index := 0 - for _, value := range values { - value = strings.TrimSpace(value) - if value == "" { - continue - } - add(actor, key, value, source, kind, index) - index++ - } - } - - scalarAttributeKeys := []string{ - "vendor", "vendor_derived", "model", "sys_descr", "sys_location", "sys_contact", - "management_ip", "display_name", "display_source", "chart_id_prefix", "chart_context_prefix", - "netdata_host_id", "ports_total", "ports_up", "ports_down", "vlan_count", "fdb_total_macs", - "lldp_neighbor_count", "cdp_neighbor_count", "endpoints_total", "if_admin_status_counts", - "if_oper_status_counts", "if_link_mode_counts", "if_topology_role_counts", - } - arrayAttributeKeys := []string{ - "protocols", "protocols_collected", "learned_sources", "capabilities", - "capabilities_supported", "capabilities_enabled", "if_names", "if_indexes", - } - - for actorIndex, actor := range actors { - add(actorIndex, "actor_type", snmpTopologyV1ActorType(actor.ActorType), snmpTopologyV1ProducerSource, "identity", nil) - add(actorIndex, "layer", snmpTopologyV1ActorLayer(actor), snmpTopologyV1ProducerSource, "identity", nil) - add(actorIndex, "source", firstNonEmptyString(actor.Source, snmpTopologyV1ProducerSource), snmpTopologyV1ProducerSource, "identity", nil) - add(actorIndex, "display_name", snmpTopologyV1DisplayName(actor), snmpTopologyV1ProducerSource, "attribute", nil) - add(actorIndex, "sys_name", actor.Match.SysName, snmpTopologyV1ProducerSource, "match", nil) - add(actorIndex, "sys_object_id", actor.Match.SysObjectID, snmpTopologyV1ProducerSource, "match", nil) - addSlice(actorIndex, "chassis_id", actor.Match.ChassisIDs, snmpTopologyV1ProducerSource, "match") - addSlice(actorIndex, "mac_address", actor.Match.MacAddresses, snmpTopologyV1ProducerSource, "match") - addSlice(actorIndex, "ip_address", actor.Match.IPAddresses, snmpTopologyV1ProducerSource, "match") - addSlice(actorIndex, "hostname", actor.Match.Hostnames, snmpTopologyV1ProducerSource, "match") - addSlice(actorIndex, "dns_name", actor.Match.DNSNames, snmpTopologyV1ProducerSource, "match") - - for key, value := range actor.Labels { - add(actorIndex, key, value, "producer_label", "label", nil) - } - for _, key := range scalarAttributeKeys { - if value := topologyV1ScalarLabelValue(actor.Attributes[key]); value != "" { - add(actorIndex, key, value, snmpTopologyV1ProducerSource, "attribute", nil) - } - } - for _, key := range arrayAttributeKeys { - addSlice(actorIndex, key, anyStringSlice(actor.Attributes[key]), snmpTopologyV1ProducerSource, "attribute") - } - } - - actorRefs := make([]any, len(rows)) - keys := make([]any, len(rows)) - values := make([]any, len(rows)) - sources := make([]any, len(rows)) - kinds := make([]any, len(rows)) - valueIndexes := make([]any, len(rows)) - for i, row := range rows { - actorRefs[i] = row.actor - keys[i] = stringsDict.Ref(row.key) - values[i] = stringsDict.Ref(row.value) - sources[i] = nullableStringRef(stringsDict, row.source) - kinds[i] = nullableStringRef(stringsDict, row.kind) - valueIndexes[i] = row.valueIndex - } - - return topologyv1.MustTable(len(rows), snmpTopologyV1ActorLabelsTableType().Columns, []topologyv1.ColumnEncoding{ - topologyv1.Values(actorRefs...), - topologyv1.Values(keys...), - topologyv1.Values(values...), - topologyv1.Values(sources...), - topologyv1.Values(kinds...), - topologyv1.Values(valueIndexes...), - }) -} - -type topologyV1DynamicRow struct { - actorRef int - values map[string]any -} - -type snmpTopologyV1PortNeighborKey struct { - actorRef int - ifIndex uint64 - portName string -} - -type snmpTopologyV1PortNeighborSummary struct { - remoteActor any - remotePortName string - ambiguous bool -} - -func snmpTopologyV1PortNeighborKeyFor(actorRef int, ifIndex any, portName string) snmpTopologyV1PortNeighborKey { - if index, ok := uintValue(ifIndex); ok && index > 0 { - return snmpTopologyV1PortNeighborKey{actorRef: actorRef, ifIndex: index} - } - portName = strings.ToLower(strings.TrimSpace(portName)) - if portName == "" { - return snmpTopologyV1PortNeighborKey{actorRef: -1} - } - return snmpTopologyV1PortNeighborKey{actorRef: actorRef, portName: portName} -} - -func buildSNMPTopologyV1PortNeighborSummaries( - links []topologyLink, - actorIndex map[string]int, -) map[snmpTopologyV1PortNeighborKey]snmpTopologyV1PortNeighborSummary { - summaries := make(map[snmpTopologyV1PortNeighborKey]snmpTopologyV1PortNeighborSummary) - appendSide := func(actorID, remoteActorID string, endpoint, remoteEndpoint topologyLinkEndpoint) { - actorRef, ok := actorIndex[strings.TrimSpace(actorID)] - if !ok { - return - } - remoteActorRef, ok := actorIndex[strings.TrimSpace(remoteActorID)] - if !ok { - return - } - key := snmpTopologyV1PortNeighborKeyFor(actorRef, endpoint.Attributes["if_index"], topologyV1EndpointPortName(endpoint)) - if key.actorRef < 0 { - return - } - if existing, exists := summaries[key]; exists { - if existing.remoteActor != remoteActorRef || strings.TrimSpace(existing.remotePortName) != strings.TrimSpace(topologyV1EndpointPortName(remoteEndpoint)) { - existing.ambiguous = true - summaries[key] = existing - } - return - } - summaries[key] = snmpTopologyV1PortNeighborSummary{ - remoteActor: remoteActorRef, - remotePortName: topologyV1EndpointPortName(remoteEndpoint), - } - } - - for _, link := range links { - appendSide(link.SrcActorID, link.DstActorID, link.Src, link.Dst) - appendSide(link.DstActorID, link.SrcActorID, link.Dst, link.Src) - } - return summaries -} - -func snmpTopologyV1PortNeighborSummaryFor( - row topologyV1DynamicRow, - portName string, - summaries map[snmpTopologyV1PortNeighborKey]snmpTopologyV1PortNeighborSummary, -) (snmpTopologyV1PortNeighborSummary, bool) { - candidates := []string{ - portName, - topologyV1ScalarLabelValue(row.values["if_name"]), - topologyV1ScalarLabelValue(row.values["port_name"]), - topologyV1ScalarLabelValue(row.values["port_id"]), - } - seen := make(map[snmpTopologyV1PortNeighborKey]struct{}, len(candidates)+1) - keys := []snmpTopologyV1PortNeighborKey{ - snmpTopologyV1PortNeighborKeyFor(row.actorRef, row.values["if_index"], ""), - } - for _, candidate := range candidates { - keys = append(keys, snmpTopologyV1PortNeighborKeyFor(row.actorRef, nil, candidate)) - } - for _, key := range keys { - if key.actorRef < 0 { - continue - } - if _, ok := seen[key]; ok { - continue - } - seen[key] = struct{}{} - if summary, ok := summaries[key]; ok { - if summary.ambiguous { - return snmpTopologyV1PortNeighborSummary{}, false - } - return summary, true - } - } - return snmpTopologyV1PortNeighborSummary{}, false -} - -func collectSNMPTopologyV1ActorTableRows(actors []topologyActor) map[string][]topologyV1DynamicRow { - tables := make(map[string][]topologyV1DynamicRow) - for actorIndex, actor := range actors { - for tableName, rows := range actor.Tables { - tableName = strings.TrimSpace(tableName) - if tableName == "" || len(rows) == 0 { - continue - } - for _, row := range rows { - if len(row) == 0 { - continue - } - tables[tableName] = append(tables[tableName], topologyV1DynamicRow{ - actorRef: actorIndex, - values: row, - }) - } - } - } - return tables -} - -func buildSNMPTopologyV1ActorPortsTable( - rows []topologyV1DynamicRow, - stringsDict *topologyv1.StringDictionary, - portNeighborSummaries map[snmpTopologyV1PortNeighborKey]snmpTopologyV1PortNeighborSummary, -) topologyv1.Table { - actorRefs := make([]any, len(rows)) - ifIndexes := make([]any, len(rows)) - portIDs := make([]any, len(rows)) - names := make([]any, len(rows)) - ifNames := make([]any, len(rows)) - ifDescrs := make([]any, len(rows)) - ifAliases := make([]any, len(rows)) - macs := make([]any, len(rows)) - speeds := make([]any, len(rows)) - topologyRoles := make([]any, len(rows)) - operStatuses := make([]any, len(rows)) - adminStatuses := make([]any, len(rows)) - portTypes := make([]any, len(rows)) - linkModes := make([]any, len(rows)) - stpStates := make([]any, len(rows)) - vlanIDs := make([]any, len(rows)) - fdbMACCounts := make([]any, len(rows)) - linkCounts := make([]any, len(rows)) - neighborCounts := make([]any, len(rows)) - neighborActors := make([]any, len(rows)) - neighborPortNames := make([]any, len(rows)) - neighbors := make([]any, len(rows)) - vlans := make([]any, len(rows)) - extra := make([]any, len(rows)) - - for i, row := range rows { - actorRefs[i] = row.actorRef - ifIndexes[i] = nullableUintValue(row.values["if_index"]) - portIDs[i] = nullableStringRef(stringsDict, topologyV1ScalarLabelValue(row.values["port_id"])) - portName := firstNonEmptyString( - topologyV1ScalarLabelValue(row.values["name"]), - topologyV1ScalarLabelValue(row.values["if_name"]), - topologyV1ScalarLabelValue(row.values["port_name"]), - topologyV1ScalarLabelValue(row.values["port_id"]), - ) - names[i] = nullableStringRef(stringsDict, portName) - ifNames[i] = nullableStringRef(stringsDict, topologyV1ScalarLabelValue(row.values["if_name"])) - ifDescrs[i] = nullableStringRef(stringsDict, topologyV1ScalarLabelValue(row.values["if_descr"])) - ifAliases[i] = nullableStringRef(stringsDict, topologyV1ScalarLabelValue(row.values["if_alias"])) - macs[i] = nullableStringRef(stringsDict, topologyV1ScalarLabelValue(row.values["mac"])) - speeds[i] = nullableUintValue(row.values["speed"]) - topologyRoles[i] = nullableStringRef(stringsDict, topologyV1ScalarLabelValue(row.values["topology_role"])) - operStatuses[i] = nullableStringRef(stringsDict, firstNonEmptyString( - topologyV1ScalarLabelValue(row.values["oper_status"]), - topologyV1ScalarLabelValue(row.values["if_oper_status"]), - )) - adminStatuses[i] = nullableStringRef(stringsDict, firstNonEmptyString( - topologyV1ScalarLabelValue(row.values["admin_status"]), - topologyV1ScalarLabelValue(row.values["if_admin_status"]), - )) - portTypes[i] = nullableStringRef(stringsDict, firstNonEmptyString( - topologyV1ScalarLabelValue(row.values["port_type"]), - topologyV1ScalarLabelValue(row.values["if_type"]), - )) - linkModes[i] = nullableStringRef(stringsDict, topologyV1ScalarLabelValue(row.values["link_mode"])) - stpStates[i] = nullableStringRef(stringsDict, topologyV1ScalarLabelValue(row.values["stp_state"])) - vlanIDs[i] = stringArrayCell(anyStringSlice(row.values["vlan_ids"])) - if isEmptyArrayCell(vlanIDs[i]) { - vlanIDs[i] = nil - } - fdbMACCounts[i] = nullableUintValue(row.values["fdb_mac_count"]) - linkCounts[i] = nullableUintValue(row.values["link_count"]) - neighborCounts[i] = nullableUintValue(row.values["neighbor_count"]) - if neighborCounts[i] == nil { - if values, ok := anyMapSlice(row.values["neighbors"]); ok { - neighborCounts[i] = uint64(len(values)) - } - } - if summary, ok := snmpTopologyV1PortNeighborSummaryFor(row, portName, portNeighborSummaries); ok { - neighborActors[i] = summary.remoteActor - neighborPortNames[i] = nullableStringRef(stringsDict, summary.remotePortName) - } - neighbors[i] = nullableJSON(row.values["neighbors"]) - vlans[i] = nullableJSON(row.values["vlans"]) - extra[i] = nullableJSON(snmpTopologyV1ExtraPortValues(row.values)) - } - - return topologyv1.MustTable(len(rows), snmpTopologyV1ActorPortsColumns(), []topologyv1.ColumnEncoding{ - topologyv1.Values(actorRefs...), - topologyv1.Values(ifIndexes...), - topologyv1.Values(portIDs...), - topologyv1.Values(names...), - topologyv1.Values(ifNames...), - topologyv1.Values(ifDescrs...), - topologyv1.Values(ifAliases...), - topologyv1.Values(macs...), - topologyv1.Values(speeds...), - topologyv1.Values(topologyRoles...), - topologyv1.Values(operStatuses...), - topologyv1.Values(adminStatuses...), - topologyv1.Values(portTypes...), - topologyv1.Values(linkModes...), - topologyv1.Values(stpStates...), - topologyv1.Values(vlanIDs...), - topologyv1.Values(fdbMACCounts...), - topologyv1.Values(linkCounts...), - topologyv1.Values(neighborCounts...), - topologyv1.Values(neighborActors...), - topologyv1.Values(neighborPortNames...), - topologyv1.Values(neighbors...), - topologyv1.Values(vlans...), - topologyv1.Values(extra...), - }) -} - -func buildSNMPTopologyV1ActorPortLinksTable( - links []topologyLink, - actorIndex map[string]int, - stringsDict *topologyv1.StringDictionary, -) (topologyv1.Table, error) { - rows := &snmpTopologyV1ActorPortLinkRows{} - appendSide := func(linkIndex int, link topologyLink, actorID, remoteActorID string, endpoint, remoteEndpoint topologyLinkEndpoint) error { - actorRef, ok := actorIndex[strings.TrimSpace(actorID)] - if !ok { - return fmt.Errorf("link %d references unknown actor %q", linkIndex, actorID) - } - remoteActorRef, ok := actorIndex[strings.TrimSpace(remoteActorID)] - if !ok { - return fmt.Errorf("link %d references unknown remote actor %q", linkIndex, remoteActorID) - } - protocol := firstNonEmptyString(link.Protocol, link.LinkType, "l2") - linkType := snmpTopologyV1LinkType(link) - - rows.actors = append(rows.actors, actorRef) - rows.links = append(rows.links, linkIndex) - rows.remoteActors = append(rows.remoteActors, remoteActorRef) - rows.ifIndexes = append(rows.ifIndexes, nullableUintValue(endpoint.Attributes["if_index"])) - rows.portIDs = append(rows.portIDs, nullableStringRef(stringsDict, topologyV1EndpointString(endpoint, "port_id"))) - rows.portNames = append(rows.portNames, nullableStringRef(stringsDict, topologyV1EndpointPortName(endpoint))) - rows.remoteIfIndexes = append(rows.remoteIfIndexes, nullableUintValue(remoteEndpoint.Attributes["if_index"])) - rows.remotePortIDs = append(rows.remotePortIDs, nullableStringRef(stringsDict, topologyV1EndpointString(remoteEndpoint, "port_id"))) - rows.remotePortNames = append(rows.remotePortNames, nullableStringRef(stringsDict, topologyV1EndpointPortName(remoteEndpoint))) - rows.types = append(rows.types, stringsDict.Ref(linkType)) - rows.protocols = append(rows.protocols, stringsDict.Ref(protocol)) - rows.states = append(rows.states, nullableStringRef(stringsDict, link.State)) - rows.evidenceCounts = append(rows.evidenceCounts, uint64(1)) - rows.confidences = append(rows.confidences, nullableStringRef(stringsDict, topologyMetricValueString(link.Metrics, "confidence"))) - rows.inferences = append(rows.inferences, nullableStringRef(stringsDict, topologyMetricValueString(link.Metrics, "inference"))) - rows.attachmentModes = append(rows.attachmentModes, nullableStringRef(stringsDict, topologyMetricValueString(link.Metrics, "attachment_mode"))) - rows.discoveredAt = append(rows.discoveredAt, nullableTime(link.DiscoveredAt)) - rows.lastSeen = append(rows.lastSeen, nullableTime(link.LastSeen)) - return nil - } - - for i, link := range links { - if err := appendSide(i, link, link.SrcActorID, link.DstActorID, link.Src, link.Dst); err != nil { - return topologyv1.Table{}, err - } - if err := appendSide(i, link, link.DstActorID, link.SrcActorID, link.Dst, link.Src); err != nil { - return topologyv1.Table{}, err - } - } - - return rows.table(), nil -} - -type snmpTopologyV1ActorPortLinkRows struct { - actors []any - links []any - remoteActors []any - ifIndexes []any - portIDs []any - portNames []any - remoteIfIndexes []any - remotePortIDs []any - remotePortNames []any - types []any - protocols []any - states []any - evidenceCounts []any - confidences []any - inferences []any - attachmentModes []any - discoveredAt []any - lastSeen []any -} - -func (rows *snmpTopologyV1ActorPortLinkRows) table() topologyv1.Table { - return topologyv1.MustTable(len(rows.actors), - snmpTopologyV1ActorPortLinksColumns(), - []topologyv1.ColumnEncoding{ - topologyv1.Values(rows.actors...), - topologyv1.Values(rows.links...), - topologyv1.Values(rows.remoteActors...), - topologyv1.Values(rows.ifIndexes...), - topologyv1.Values(rows.portIDs...), - topologyv1.Values(rows.portNames...), - topologyv1.Values(rows.remoteIfIndexes...), - topologyv1.Values(rows.remotePortIDs...), - topologyv1.Values(rows.remotePortNames...), - topologyv1.Values(rows.types...), - topologyv1.Values(rows.protocols...), - topologyv1.Values(rows.states...), - topologyv1.Values(rows.evidenceCounts...), - topologyv1.Values(rows.confidences...), - topologyv1.Values(rows.inferences...), - topologyv1.Values(rows.attachmentModes...), - topologyv1.Values(rows.discoveredAt...), - topologyv1.Values(rows.lastSeen...), - }, - ) -} - -var snmpTopologyV1ActorPortCanonicalKeys = map[string]struct{}{ - "admin_status": {}, - "fdb_mac_count": {}, - "if_admin_status": {}, - "if_alias": {}, - "if_descr": {}, - "if_index": {}, - "if_name": {}, - "if_oper_status": {}, - "if_type": {}, - "link_count": {}, - "link_mode": {}, - "mac": {}, - "name": {}, - "neighbor_actor": {}, - "neighbor_count": {}, - "neighbor_port_name": {}, - "neighbors": {}, - "oper_status": {}, - "port_id": {}, - "port_name": {}, - "port_type": {}, - "speed": {}, - "stp_state": {}, - "topology_role": {}, - "vlan_ids": {}, - "vlans": {}, -} - -func snmpTopologyV1ExtraPortValues(values map[string]any) map[string]any { - extra := make(map[string]any) - for key, value := range values { - key = strings.TrimSpace(key) - if key == "" { - continue - } - if _, ok := snmpTopologyV1ActorPortCanonicalKeys[key]; ok { - continue - } - extra[key] = value - } - if len(extra) == 0 { - return nil - } - return extra -} - -func buildSNMPTopologyV1DynamicTable(rows []topologyV1DynamicRow, stringsDict *topologyv1.StringDictionary) (topologyv1.Table, error) { - keysSet := make(map[string]struct{}) - for _, row := range rows { - for key := range row.values { - key = strings.TrimSpace(key) - if key != "" { - keysSet[key] = struct{}{} - } - } - } - keys := sortedMapKeys(keysSet) - - columns := make([]topologyv1.Column, 0, len(keys)+1) - values := make([]topologyv1.ColumnEncoding, 0, len(keys)+1) - actorRefs := make([]any, len(rows)) - for i, row := range rows { - actorRefs[i] = row.actorRef - } - columns = append(columns, topologyv1.NewColumn("actor", "actor_ref", topologyv1.WithRole("reference"))) - values = append(values, topologyv1.Values(actorRefs...)) - - for _, key := range keys { - columnValues := make([]any, len(rows)) - for i, row := range rows { - if value, ok := row.values[key]; ok { - columnValues[i] = value - } - } - columnType := inferTopologyV1ColumnType(columnValues) - columnID := topologyID(key, "field") - column := topologyv1.NewColumn(columnID, columnType, topologyv1.WithNullable()) - if columnType == "string_ref" { - column = topologyv1.NewColumn(columnID, columnType, topologyv1.WithNullable(), topologyv1.WithDictionary("strings")) - for i, value := range columnValues { - if value == nil { - continue - } - columnValues[i] = stringsDict.Ref(fmt.Sprint(value)) - } - } - columns = append(columns, column) - values = append(values, topologyv1.Values(columnValues...)) - } - - return topologyv1.NewTable(len(rows), columns, values) -} - -func inferTopologyV1ColumnType(values []any) string { - typ := "" - for _, value := range values { - if value == nil { - continue - } - valueType := topologyV1ValueType(value) - if typ == "" { - typ = valueType - continue - } - if typ != valueType { - return "json" - } - } - if typ == "" { - return "json" - } - return typ -} - -func topologyV1ValueType(value any) string { - switch typed := value.(type) { - case bool: - return "bool" - case int, int8, int16, int32, int64: - return "int" - case uint, uint8, uint16, uint32, uint64: - return "uint" - case float32: - if math.Trunc(float64(typed)) == float64(typed) { - return "int" - } - return "float" - case float64: - if math.Trunc(typed) == typed { - return "int" - } - return "float" - case string: - return "string_ref" - case []string, []int, []int64, []uint, []uint64, []float64, []bool: - return "array" - case []any: - if scalarArray(typed) { - return "array" - } - return "json" - default: - return "json" - } -} - -func scalarArray(values []any) bool { - for _, value := range values { - switch value.(type) { - case nil, bool, int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, float32, float64, string: - default: - return false - } - } - return true -} - -func snmpTopologyV1ActorType(actorType string) string { - normalized := strings.ToLower(strings.TrimSpace(actorType)) - if topologyengine.IsDeviceActorType(normalized) { - return normalized - } - switch normalized { - case snmpTopologyV1ActorDevice: - return snmpTopologyV1ActorDevice - case snmpTopologyV1ActorEndpoint: - return snmpTopologyV1ActorEndpoint - case snmpTopologyV1ActorSegment: - return snmpTopologyV1ActorSegment - default: - return "custom" - } -} - -func snmpTopologyV1DisplayName(actor topologyActor) string { - return firstNonEmptyString( - anyStringValue(actor.Attributes["display_name"]), - anyStringValue(actor.Attributes["name"]), - actor.Labels["display_name"], - actor.Labels["name"], - actor.Match.SysName, - firstString(actor.Match.Hostnames), - firstString(actor.Match.DNSNames), - ) -} - -func snmpTopologyV1ActorLayer(actor topologyActor) string { - switch snmpTopologyV1ActorType(actor.ActorType) { - case snmpTopologyV1ActorEndpoint, snmpTopologyV1ActorSegment: - return "network" - default: - if topologyengine.IsDeviceActorType(actor.ActorType) { - return "network" - } - return "custom" - } -} - -func anyStringValue(value any) string { - switch typed := value.(type) { - case string: - return strings.TrimSpace(typed) - case fmt.Stringer: - return strings.TrimSpace(typed.String()) - default: - return "" - } -} - -func topologyV1ScalarLabelValue(value any) string { - switch typed := value.(type) { - case nil: - return "" - case string: - return strings.TrimSpace(typed) - case bool: - if typed { - return "true" - } - return "false" - case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, float32, float64: - return strings.TrimSpace(fmt.Sprint(typed)) - default: - return "" - } -} - -func nullableUintValue(value any) any { - out, ok := uintValue(value) - if !ok { - return nil - } - return out -} - -func uintValue(value any) (uint64, bool) { - switch typed := value.(type) { - case int: - if typed >= 0 { - return uint64(typed), true - } - case int8: - if typed >= 0 { - return uint64(typed), true - } - case int16: - if typed >= 0 { - return uint64(typed), true - } - case int32: - if typed >= 0 { - return uint64(typed), true - } - case int64: - if typed >= 0 { - return uint64(typed), true - } - case uint: - return uint64(typed), true - case uint8: - return uint64(typed), true - case uint16: - return uint64(typed), true - case uint32: - return uint64(typed), true - case uint64: - return typed, true - case float32: - if typed >= 0 && math.Trunc(float64(typed)) == float64(typed) { - return uint64(typed), true - } - case float64: - if typed >= 0 && math.Trunc(typed) == typed { - return uint64(typed), true - } - } - return 0, false -} - -func topologyV1EndpointString(endpoint topologyLinkEndpoint, key string) string { - return firstNonEmptyString( - anyStringValue(endpoint.Attributes[key]), - topologyV1MatchString(endpoint.Match, key), - ) -} - -func topologyV1EndpointPortName(endpoint topologyLinkEndpoint) string { - return firstNonEmptyString( - topologyV1EndpointString(endpoint, "port_name"), - topologyV1EndpointString(endpoint, "if_name"), - topologyV1EndpointString(endpoint, "if_descr"), - topologyV1EndpointString(endpoint, "port_id"), - ) -} - -func topologyV1MatchString(match topologyMatch, key string) string { - switch key { - case "sys_name": - return match.SysName - case "sys_object_id": - return match.SysObjectID - default: - return "" - } -} - -func firstString(values []string) string { - for _, value := range values { - if value = strings.TrimSpace(value); value != "" { - return value - } - } - return "" -} - -func nullableTime(value *time.Time) any { - if value == nil || value.IsZero() { - return nil - } - return value.UTC().Format(time.RFC3339Nano) -} - -func nullableStringRef(dict *topologyv1.StringDictionary, value string) any { - value = strings.TrimSpace(value) - if value == "" { - return nil - } - return dict.Ref(value) -} - -func nullableJSON(value any) any { - switch typed := value.(type) { - case nil: - return nil - case map[string]any: - if len(typed) == 0 { - return nil - } - case map[string]string: - if len(typed) == 0 { - return nil - } - case []any: - if len(typed) == 0 { - return nil - } - case []map[string]any: - if len(typed) == 0 { - return nil - } - } - return value -} - -func stringArrayCell(values []string) []any { - out := make([]any, 0, len(values)) - for _, value := range values { - value = strings.TrimSpace(value) - if value != "" { - out = append(out, value) - } - } - return out -} - -func isEmptyArrayCell(value any) bool { - values, ok := value.([]any) - return ok && len(values) == 0 -} - -func anyStringSlice(value any) []string { - switch typed := value.(type) { - case []string: - return typed - case []any: - out := make([]string, 0, len(typed)) - for _, item := range typed { - if s := strings.TrimSpace(fmt.Sprint(item)); s != "" { - out = append(out, s) - } - } - return out - default: - rv := reflect.ValueOf(value) - if rv.Kind() != reflect.Slice && rv.Kind() != reflect.Array { - return nil - } - out := make([]string, 0, rv.Len()) - for i := 0; i < rv.Len(); i++ { - if s := topologyV1ScalarLabelValue(rv.Index(i).Interface()); s != "" { - out = append(out, s) - } - } - return out - } -} - -func anyMapSlice(value any) ([]map[string]any, bool) { - switch typed := value.(type) { - case []map[string]any: - return typed, true - case []any: - out := make([]map[string]any, 0, len(typed)) - for _, item := range typed { - row, ok := item.(map[string]any) - if !ok { - return nil, false - } - out = append(out, row) - } - return out, true - default: - return nil, false - } -} - -func sortedMapKeys[T any](m map[string]T) []string { - keys := make([]string, 0, len(m)) - for key := range m { - keys = append(keys, key) - } - sort.Strings(keys) - return keys -} - -func topologyID(value, fallback string) string { - value = strings.TrimSpace(value) - if value == "" { - value = fallback - } - value = topologyV1IDInvalidChars.ReplaceAllString(value, "_") - value = strings.Trim(value, "_.:-") - if value == "" { - value = fallback - } - first := value[0] - if (first >= 'A' && first <= 'Z') || (first >= 'a' && first <= 'z') { - return value - } - return "x_" + value -} - -func firstNonEmptyString(values ...string) string { - for _, value := range values { - value = strings.TrimSpace(value) - if value != "" { - return value - } - } - return "" -} - -func cloneAnyMapForTopologyV1(in map[string]any) map[string]any { - if len(in) == 0 { - return nil - } - out := make(map[string]any, len(in)) - maps.Copy(out, in) - return out -} diff --git a/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_actor_details.go b/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_actor_details.go new file mode 100644 index 00000000000000..1f96e784527106 --- /dev/null +++ b/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_actor_details.go @@ -0,0 +1,260 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package snmptopology + +import ( + "fmt" + topologyv1 "github.com/netdata/netdata/go/plugins/pkg/topology/v1" + "strings" +) + +func buildSNMPTopologyV1ActorDetails( + actors []topologyActor, + stringsDict *topologyv1.StringDictionary, + portNeighborSummaries map[snmpTopologyV1PortNeighborKey]snmpTopologyV1PortNeighborSummary, +) (map[string]topologyv1.DetailTable, map[string]topologyv1.TableType, error) { + details := make(map[string]topologyv1.DetailTable) + tableTypes := make(map[string]topologyv1.TableType) + + labelsTable := buildSNMPTopologyV1ActorLabelsTable(actors, stringsDict) + details["actor_labels"] = topologyv1.DetailTable{ + Type: "actor_labels", + Table: labelsTable, + } + tableTypes["actor_labels"] = snmpTopologyV1ActorLabelsTableType() + usedTableIDs := map[string]struct{}{ + "actor_labels": {}, + "actor_port_links": {}, + } + + metadataTable := buildSNMPTopologyV1ActorMetadataTable(actors) + if metadataTable.Rows > 0 { + tableID := "actor_metadata" + details[tableID] = topologyv1.DetailTable{ + Type: tableID, + Table: metadataTable, + } + tableTypes[tableID] = topologyv1.TableType{ + Role: "actor_detail", + Owner: "actor", + Aggregation: "append", + Columns: metadataTable.Columns, + Presentation: &topologyv1.TableTypePresentation{ + Label: "Debug metadata", + DefaultVisibility: "debug", + Columns: []topologyv1.ModalColumn{ + modalDirectColumn("attributes", "Attributes", "attributes", "debug_json"), + modalDirectColumn("labels", "Labels", "labels", "debug_json"), + }, + }, + } + usedTableIDs[tableID] = struct{}{} + } + + tableRowsByName := collectSNMPTopologyV1ActorTableRows(actors) + tableNames := sortedMapKeys(tableRowsByName) + reservedCustomTableIDs := snmpTopologyV1ReservedCustomTableIDs(tableNames) + for _, tableName := range tableNames { + rows := tableRowsByName[tableName] + if len(rows) == 0 { + continue + } + tableID := snmpTopologyV1ActorDetailTableID(tableName, usedTableIDs, reservedCustomTableIDs) + var table topologyv1.Table + var err error + if tableID == "actor_ports" { + table = buildSNMPTopologyV1ActorPortsTable(rows, stringsDict, portNeighborSummaries) + } else { + table, err = buildSNMPTopologyV1DynamicTable(rows, stringsDict) + } + if err != nil { + return nil, nil, fmt.Errorf("build actor detail table %q: %w", tableName, err) + } + details[tableID] = topologyv1.DetailTable{ + Type: tableID, + Table: table, + } + if tableID == "actor_ports" { + tableTypes[tableID] = snmpTopologyV1ActorPortsTableType() + } else { + tableTypes[tableID] = topologyv1.TableType{ + Role: "actor_detail", + Owner: "actor", + Aggregation: "append", + Columns: table.Columns, + } + } + usedTableIDs[tableID] = struct{}{} + } + return details, tableTypes, nil +} + +func snmpTopologyV1ReservedCustomTableIDs(tableNames []string) map[string]struct{} { + reserved := make(map[string]struct{}) + for _, tableName := range tableNames { + tableID := topologyID("actor_"+tableName, "actor_detail") + switch tableID { + case "actor_labels", "actor_metadata", "actor_port_links": + reserved[topologyID("actor_custom_"+tableName, "actor_detail")] = struct{}{} + } + } + return reserved +} + +func snmpTopologyV1ActorDetailTableID(tableName string, usedTableIDs, reservedCustomTableIDs map[string]struct{}) string { + tableID := topologyID("actor_"+tableName, "actor_detail") + switch tableID { + case "actor_labels", "actor_metadata", "actor_port_links": + return snmpTopologyV1UniqueActorDetailTableID(topologyID("actor_custom_"+tableName, "actor_detail"), usedTableIDs) + default: + if _, reserved := reservedCustomTableIDs[tableID]; reserved { + return snmpTopologyV1UniqueActorDetailTableID(topologyID("actor_detail_"+tableName, "actor_detail"), usedTableIDs) + } + return snmpTopologyV1UniqueActorDetailTableID(tableID, usedTableIDs) + } +} + +func snmpTopologyV1UniqueActorDetailTableID(tableID string, usedTableIDs map[string]struct{}) string { + if _, ok := usedTableIDs[tableID]; !ok { + return tableID + } + for suffix := 2; ; suffix++ { + candidate := fmt.Sprintf("%s_%d", tableID, suffix) + if _, ok := usedTableIDs[candidate]; !ok { + return candidate + } + } +} + +func buildSNMPTopologyV1ActorMetadataTable(actors []topologyActor) topologyv1.Table { + actorRefs := make([]any, 0, len(actors)) + attributes := make([]any, 0, len(actors)) + labels := make([]any, 0, len(actors)) + for i, actor := range actors { + if len(actor.Attributes) == 0 && len(actor.Labels) == 0 { + continue + } + actorRefs = append(actorRefs, i) + attributes = append(attributes, nullableJSON(actor.Attributes)) + labels = append(labels, nullableJSON(actor.Labels)) + } + if len(actorRefs) == 0 { + return topologyv1.EmptyTable() + } + return topologyv1.MustTable(len(actorRefs), + []topologyv1.Column{ + topologyv1.NewColumn("actor", "actor_ref", topologyv1.WithRole("reference")), + topologyv1.NewColumn("attributes", "json", topologyv1.WithNullable()), + topologyv1.NewColumn("labels", "json", topologyv1.WithNullable()), + }, + []topologyv1.ColumnEncoding{ + topologyv1.Values(actorRefs...), + topologyv1.Values(attributes...), + topologyv1.Values(labels...), + }, + ) +} + +func buildSNMPTopologyV1ActorLabelsTable( + actors []topologyActor, + stringsDict *topologyv1.StringDictionary, +) topologyv1.Table { + type labelRow struct { + actor int + key string + value string + source string + kind string + valueIndex any + } + + rows := make([]labelRow, 0, len(actors)*8) + add := func(actor int, key, value, source, kind string, valueIndex any) { + key = strings.TrimSpace(key) + value = strings.TrimSpace(value) + if key == "" || value == "" { + return + } + rows = append(rows, labelRow{ + actor: actor, + key: key, + value: value, + source: source, + kind: kind, + valueIndex: valueIndex, + }) + } + addSlice := func(actor int, key string, values []string, source, kind string) { + index := 0 + for _, value := range values { + value = strings.TrimSpace(value) + if value == "" { + continue + } + add(actor, key, value, source, kind, index) + index++ + } + } + + scalarAttributeKeys := []string{ + "vendor", "vendor_derived", "model", "sys_descr", "sys_location", "sys_contact", + "management_ip", "display_name", "display_source", "chart_id_prefix", "chart_context_prefix", + "netdata_host_id", "ports_total", "ports_up", "ports_down", "vlan_count", "fdb_total_macs", + "lldp_neighbor_count", "cdp_neighbor_count", "endpoints_total", "if_admin_status_counts", + "if_oper_status_counts", "if_link_mode_counts", "if_topology_role_counts", + } + arrayAttributeKeys := []string{ + "protocols", "protocols_collected", "learned_sources", "capabilities", + "capabilities_supported", "capabilities_enabled", "if_names", "if_indexes", + } + + for actorIndex, actor := range actors { + add(actorIndex, "actor_type", snmpTopologyV1ActorType(actor.ActorType), snmpTopologyV1ProducerSource, "identity", nil) + add(actorIndex, "layer", snmpTopologyV1ActorLayer(actor), snmpTopologyV1ProducerSource, "identity", nil) + add(actorIndex, "source", firstNonEmptyString(actor.Source, snmpTopologyV1ProducerSource), snmpTopologyV1ProducerSource, "identity", nil) + add(actorIndex, "display_name", snmpTopologyV1DisplayName(actor), snmpTopologyV1ProducerSource, "attribute", nil) + add(actorIndex, "sys_name", actor.Match.SysName, snmpTopologyV1ProducerSource, "match", nil) + add(actorIndex, "sys_object_id", actor.Match.SysObjectID, snmpTopologyV1ProducerSource, "match", nil) + addSlice(actorIndex, "chassis_id", actor.Match.ChassisIDs, snmpTopologyV1ProducerSource, "match") + addSlice(actorIndex, "mac_address", actor.Match.MacAddresses, snmpTopologyV1ProducerSource, "match") + addSlice(actorIndex, "ip_address", actor.Match.IPAddresses, snmpTopologyV1ProducerSource, "match") + addSlice(actorIndex, "hostname", actor.Match.Hostnames, snmpTopologyV1ProducerSource, "match") + addSlice(actorIndex, "dns_name", actor.Match.DNSNames, snmpTopologyV1ProducerSource, "match") + + for key, value := range actor.Labels { + add(actorIndex, key, value, "producer_label", "label", nil) + } + for _, key := range scalarAttributeKeys { + if value := topologyV1ScalarLabelValue(actor.Attributes[key]); value != "" { + add(actorIndex, key, value, snmpTopologyV1ProducerSource, "attribute", nil) + } + } + for _, key := range arrayAttributeKeys { + addSlice(actorIndex, key, anyStringSlice(actor.Attributes[key]), snmpTopologyV1ProducerSource, "attribute") + } + } + + actorRefs := make([]any, len(rows)) + keys := make([]any, len(rows)) + values := make([]any, len(rows)) + sources := make([]any, len(rows)) + kinds := make([]any, len(rows)) + valueIndexes := make([]any, len(rows)) + for i, row := range rows { + actorRefs[i] = row.actor + keys[i] = stringsDict.Ref(row.key) + values[i] = stringsDict.Ref(row.value) + sources[i] = nullableStringRef(stringsDict, row.source) + kinds[i] = nullableStringRef(stringsDict, row.kind) + valueIndexes[i] = row.valueIndex + } + + return topologyv1.MustTable(len(rows), snmpTopologyV1ActorLabelsTableType().Columns, []topologyv1.ColumnEncoding{ + topologyv1.Values(actorRefs...), + topologyv1.Values(keys...), + topologyv1.Values(values...), + topologyv1.Values(sources...), + topologyv1.Values(kinds...), + topologyv1.Values(valueIndexes...), + }) +} diff --git a/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_actor_ports.go b/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_actor_ports.go new file mode 100644 index 00000000000000..57c6af564119d4 --- /dev/null +++ b/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_actor_ports.go @@ -0,0 +1,397 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package snmptopology + +import ( + "fmt" + "strings" + + topologyv1 "github.com/netdata/netdata/go/plugins/pkg/topology/v1" +) + +type snmpTopologyV1PortNeighborKey struct { + actorRef int + ifIndex uint64 + portName string +} + +type snmpTopologyV1PortNeighborSummary struct { + remoteActor any + remotePortName string + ambiguous bool +} + +func snmpTopologyV1PortNeighborKeyFor(actorRef int, ifIndex any, portName string) snmpTopologyV1PortNeighborKey { + if index, ok := uintValue(ifIndex); ok && index > 0 { + return snmpTopologyV1PortNeighborKey{actorRef: actorRef, ifIndex: index} + } + portName = strings.ToLower(strings.TrimSpace(portName)) + if portName == "" { + return snmpTopologyV1PortNeighborKey{actorRef: -1} + } + return snmpTopologyV1PortNeighborKey{actorRef: actorRef, portName: portName} +} + +func buildSNMPTopologyV1PortNeighborSummaries( + links []topologyLink, + actorIndex map[string]int, +) map[snmpTopologyV1PortNeighborKey]snmpTopologyV1PortNeighborSummary { + summaries := make(map[snmpTopologyV1PortNeighborKey]snmpTopologyV1PortNeighborSummary) + appendSide := func(actorID, remoteActorID string, endpoint, remoteEndpoint topologyLinkEndpoint) { + actorRef, ok := actorIndex[strings.TrimSpace(actorID)] + if !ok { + return + } + remoteActorRef, ok := actorIndex[strings.TrimSpace(remoteActorID)] + if !ok { + return + } + key := snmpTopologyV1PortNeighborKeyFor(actorRef, endpoint.Attributes["if_index"], topologyV1EndpointPortName(endpoint)) + if key.actorRef < 0 { + return + } + remotePortName := topologyV1EndpointPortName(remoteEndpoint) + if existing, exists := summaries[key]; exists { + if existing.remoteActor != remoteActorRef || !snmpTopologyV1SameRemotePortName(existing.remotePortName, remotePortName) { + existing.ambiguous = true + summaries[key] = existing + } else if strings.TrimSpace(existing.remotePortName) == "" && strings.TrimSpace(remotePortName) != "" { + existing.remotePortName = remotePortName + summaries[key] = existing + } + return + } + summaries[key] = snmpTopologyV1PortNeighborSummary{ + remoteActor: remoteActorRef, + remotePortName: remotePortName, + } + } + + for _, link := range links { + appendSide(link.SrcActorID, link.DstActorID, link.Src, link.Dst) + appendSide(link.DstActorID, link.SrcActorID, link.Dst, link.Src) + } + return summaries +} + +func snmpTopologyV1SameRemotePortName(left, right string) bool { + left = strings.TrimSpace(left) + right = strings.TrimSpace(right) + if left == "" || right == "" { + return true + } + return strings.EqualFold(left, right) +} + +func snmpTopologyV1PortNeighborSummaryFor( + row topologyV1DynamicRow, + portName string, + summaries map[snmpTopologyV1PortNeighborKey]snmpTopologyV1PortNeighborSummary, +) (snmpTopologyV1PortNeighborSummary, bool) { + candidates := []string{ + portName, + topologyV1ScalarLabelValue(row.values["if_name"]), + topologyV1ScalarLabelValue(row.values["port_name"]), + topologyV1ScalarLabelValue(row.values["port_id"]), + } + seen := make(map[snmpTopologyV1PortNeighborKey]struct{}, len(candidates)+1) + keys := []snmpTopologyV1PortNeighborKey{ + snmpTopologyV1PortNeighborKeyFor(row.actorRef, row.values["if_index"], ""), + } + for _, candidate := range candidates { + keys = append(keys, snmpTopologyV1PortNeighborKeyFor(row.actorRef, nil, candidate)) + } + for _, key := range keys { + if key.actorRef < 0 { + continue + } + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + if summary, ok := summaries[key]; ok { + if summary.ambiguous { + return snmpTopologyV1PortNeighborSummary{}, false + } + return summary, true + } + } + return snmpTopologyV1PortNeighborSummary{}, false +} + +func collectSNMPTopologyV1ActorTableRows(actors []topologyActor) map[string][]topologyV1DynamicRow { + tables := make(map[string][]topologyV1DynamicRow) + for actorIndex, actor := range actors { + for tableName, rows := range actor.Tables { + tableName = strings.TrimSpace(tableName) + if tableName == "" || len(rows) == 0 { + continue + } + for _, row := range rows { + if len(row) == 0 { + continue + } + tables[tableName] = append(tables[tableName], topologyV1DynamicRow{ + actorRef: actorIndex, + values: row, + }) + } + } + } + return tables +} + +func buildSNMPTopologyV1ActorPortsTable( + rows []topologyV1DynamicRow, + stringsDict *topologyv1.StringDictionary, + portNeighborSummaries map[snmpTopologyV1PortNeighborKey]snmpTopologyV1PortNeighborSummary, +) topologyv1.Table { + actorRefs := make([]any, len(rows)) + ifIndexes := make([]any, len(rows)) + portIDs := make([]any, len(rows)) + names := make([]any, len(rows)) + ifNames := make([]any, len(rows)) + ifDescrs := make([]any, len(rows)) + ifAliases := make([]any, len(rows)) + macs := make([]any, len(rows)) + speeds := make([]any, len(rows)) + topologyRoles := make([]any, len(rows)) + operStatuses := make([]any, len(rows)) + adminStatuses := make([]any, len(rows)) + portTypes := make([]any, len(rows)) + linkModes := make([]any, len(rows)) + stpStates := make([]any, len(rows)) + vlanIDs := make([]any, len(rows)) + fdbMACCounts := make([]any, len(rows)) + linkCounts := make([]any, len(rows)) + neighborCounts := make([]any, len(rows)) + neighborActors := make([]any, len(rows)) + neighborPortNames := make([]any, len(rows)) + neighbors := make([]any, len(rows)) + vlans := make([]any, len(rows)) + extra := make([]any, len(rows)) + + for i, row := range rows { + actorRefs[i] = row.actorRef + ifIndexes[i] = nullableUintValue(row.values["if_index"]) + portIDs[i] = nullableStringRef(stringsDict, topologyV1ScalarLabelValue(row.values["port_id"])) + portName := firstNonEmptyString( + topologyV1ScalarLabelValue(row.values["name"]), + topologyV1ScalarLabelValue(row.values["if_name"]), + topologyV1ScalarLabelValue(row.values["port_name"]), + topologyV1ScalarLabelValue(row.values["port_id"]), + ) + names[i] = nullableStringRef(stringsDict, portName) + ifNames[i] = nullableStringRef(stringsDict, topologyV1ScalarLabelValue(row.values["if_name"])) + ifDescrs[i] = nullableStringRef(stringsDict, topologyV1ScalarLabelValue(row.values["if_descr"])) + ifAliases[i] = nullableStringRef(stringsDict, topologyV1ScalarLabelValue(row.values["if_alias"])) + macs[i] = nullableStringRef(stringsDict, topologyV1ScalarLabelValue(row.values["mac"])) + speeds[i] = nullableUintValue(row.values["speed"]) + topologyRoles[i] = nullableStringRef(stringsDict, topologyV1ScalarLabelValue(row.values["topology_role"])) + operStatuses[i] = nullableStringRef(stringsDict, firstNonEmptyString( + topologyV1ScalarLabelValue(row.values["oper_status"]), + topologyV1ScalarLabelValue(row.values["if_oper_status"]), + )) + adminStatuses[i] = nullableStringRef(stringsDict, firstNonEmptyString( + topologyV1ScalarLabelValue(row.values["admin_status"]), + topologyV1ScalarLabelValue(row.values["if_admin_status"]), + )) + portTypes[i] = nullableStringRef(stringsDict, firstNonEmptyString( + topologyV1ScalarLabelValue(row.values["port_type"]), + topologyV1ScalarLabelValue(row.values["if_type"]), + )) + linkModes[i] = nullableStringRef(stringsDict, topologyV1ScalarLabelValue(row.values["link_mode"])) + stpStates[i] = nullableStringRef(stringsDict, topologyV1ScalarLabelValue(row.values["stp_state"])) + vlanIDs[i] = stringArrayCell(anyStringSlice(row.values["vlan_ids"])) + if isEmptyArrayCell(vlanIDs[i]) { + vlanIDs[i] = nil + } + fdbMACCounts[i] = nullableUintValue(row.values["fdb_mac_count"]) + linkCounts[i] = nullableUintValue(row.values["link_count"]) + neighborCounts[i] = nullableUintValue(row.values["neighbor_count"]) + if neighborCounts[i] == nil { + if values, ok := anyMapSlice(row.values["neighbors"]); ok { + neighborCounts[i] = uint64(len(values)) + } + } + if summary, ok := snmpTopologyV1PortNeighborSummaryFor(row, portName, portNeighborSummaries); ok { + neighborActors[i] = summary.remoteActor + neighborPortNames[i] = nullableStringRef(stringsDict, summary.remotePortName) + } + neighbors[i] = nullableJSON(row.values["neighbors"]) + vlans[i] = nullableJSON(row.values["vlans"]) + extra[i] = nullableJSON(snmpTopologyV1ExtraPortValues(row.values)) + } + + return topologyv1.MustTable(len(rows), snmpTopologyV1ActorPortsColumns(), []topologyv1.ColumnEncoding{ + topologyv1.Values(actorRefs...), + topologyv1.Values(ifIndexes...), + topologyv1.Values(portIDs...), + topologyv1.Values(names...), + topologyv1.Values(ifNames...), + topologyv1.Values(ifDescrs...), + topologyv1.Values(ifAliases...), + topologyv1.Values(macs...), + topologyv1.Values(speeds...), + topologyv1.Values(topologyRoles...), + topologyv1.Values(operStatuses...), + topologyv1.Values(adminStatuses...), + topologyv1.Values(portTypes...), + topologyv1.Values(linkModes...), + topologyv1.Values(stpStates...), + topologyv1.Values(vlanIDs...), + topologyv1.Values(fdbMACCounts...), + topologyv1.Values(linkCounts...), + topologyv1.Values(neighborCounts...), + topologyv1.Values(neighborActors...), + topologyv1.Values(neighborPortNames...), + topologyv1.Values(neighbors...), + topologyv1.Values(vlans...), + topologyv1.Values(extra...), + }) +} + +func buildSNMPTopologyV1ActorPortLinksTable( + links []topologyLink, + actorIndex map[string]int, + stringsDict *topologyv1.StringDictionary, +) (topologyv1.Table, error) { + rows := &snmpTopologyV1ActorPortLinkRows{} + appendSide := func(linkIndex int, link topologyLink, actorID, remoteActorID string, endpoint, remoteEndpoint topologyLinkEndpoint) error { + actorRef, ok := actorIndex[strings.TrimSpace(actorID)] + if !ok { + return fmt.Errorf("link %d references unknown actor %q", linkIndex, actorID) + } + remoteActorRef, ok := actorIndex[strings.TrimSpace(remoteActorID)] + if !ok { + return fmt.Errorf("link %d references unknown remote actor %q", linkIndex, remoteActorID) + } + protocol := firstNonEmptyString(link.Protocol, link.LinkType, "l2") + linkType := snmpTopologyV1LinkType(link) + + rows.actors = append(rows.actors, actorRef) + rows.links = append(rows.links, linkIndex) + rows.remoteActors = append(rows.remoteActors, remoteActorRef) + rows.ifIndexes = append(rows.ifIndexes, nullableUintValue(endpoint.Attributes["if_index"])) + rows.portIDs = append(rows.portIDs, nullableStringRef(stringsDict, topologyV1EndpointString(endpoint, "port_id"))) + rows.portNames = append(rows.portNames, nullableStringRef(stringsDict, topologyV1EndpointPortName(endpoint))) + rows.remoteIfIndexes = append(rows.remoteIfIndexes, nullableUintValue(remoteEndpoint.Attributes["if_index"])) + rows.remotePortIDs = append(rows.remotePortIDs, nullableStringRef(stringsDict, topologyV1EndpointString(remoteEndpoint, "port_id"))) + rows.remotePortNames = append(rows.remotePortNames, nullableStringRef(stringsDict, topologyV1EndpointPortName(remoteEndpoint))) + rows.types = append(rows.types, stringsDict.Ref(linkType)) + rows.protocols = append(rows.protocols, stringsDict.Ref(protocol)) + rows.states = append(rows.states, nullableStringRef(stringsDict, link.State)) + rows.evidenceCounts = append(rows.evidenceCounts, uint64(1)) + rows.confidences = append(rows.confidences, nullableStringRef(stringsDict, topologyMetricValueString(link.Metrics, "confidence"))) + rows.inferences = append(rows.inferences, nullableStringRef(stringsDict, topologyMetricValueString(link.Metrics, "inference"))) + rows.attachmentModes = append(rows.attachmentModes, nullableStringRef(stringsDict, topologyMetricValueString(link.Metrics, "attachment_mode"))) + rows.discoveredAt = append(rows.discoveredAt, nullableTime(link.DiscoveredAt)) + rows.lastSeen = append(rows.lastSeen, nullableTime(link.LastSeen)) + return nil + } + + for i, link := range links { + if err := appendSide(i, link, link.SrcActorID, link.DstActorID, link.Src, link.Dst); err != nil { + return topologyv1.Table{}, err + } + if err := appendSide(i, link, link.DstActorID, link.SrcActorID, link.Dst, link.Src); err != nil { + return topologyv1.Table{}, err + } + } + + return rows.table(), nil +} + +type snmpTopologyV1ActorPortLinkRows struct { + actors []any + links []any + remoteActors []any + ifIndexes []any + portIDs []any + portNames []any + remoteIfIndexes []any + remotePortIDs []any + remotePortNames []any + types []any + protocols []any + states []any + evidenceCounts []any + confidences []any + inferences []any + attachmentModes []any + discoveredAt []any + lastSeen []any +} + +func (rows *snmpTopologyV1ActorPortLinkRows) table() topologyv1.Table { + return topologyv1.MustTable(len(rows.actors), + snmpTopologyV1ActorPortLinksColumns(), + []topologyv1.ColumnEncoding{ + topologyv1.Values(rows.actors...), + topologyv1.Values(rows.links...), + topologyv1.Values(rows.remoteActors...), + topologyv1.Values(rows.ifIndexes...), + topologyv1.Values(rows.portIDs...), + topologyv1.Values(rows.portNames...), + topologyv1.Values(rows.remoteIfIndexes...), + topologyv1.Values(rows.remotePortIDs...), + topologyv1.Values(rows.remotePortNames...), + topologyv1.Values(rows.types...), + topologyv1.Values(rows.protocols...), + topologyv1.Values(rows.states...), + topologyv1.Values(rows.evidenceCounts...), + topologyv1.Values(rows.confidences...), + topologyv1.Values(rows.inferences...), + topologyv1.Values(rows.attachmentModes...), + topologyv1.Values(rows.discoveredAt...), + topologyv1.Values(rows.lastSeen...), + }, + ) +} + +var snmpTopologyV1ActorPortCanonicalKeys = map[string]struct{}{ + "admin_status": {}, + "fdb_mac_count": {}, + "if_admin_status": {}, + "if_alias": {}, + "if_descr": {}, + "if_index": {}, + "if_name": {}, + "if_oper_status": {}, + "if_type": {}, + "link_count": {}, + "link_mode": {}, + "mac": {}, + "name": {}, + "neighbor_actor": {}, + "neighbor_count": {}, + "neighbor_port_name": {}, + "neighbors": {}, + "oper_status": {}, + "port_id": {}, + "port_name": {}, + "port_type": {}, + "speed": {}, + "stp_state": {}, + "topology_role": {}, + "vlan_ids": {}, + "vlans": {}, +} + +func snmpTopologyV1ExtraPortValues(values map[string]any) map[string]any { + extra := make(map[string]any) + for key, value := range values { + key = strings.TrimSpace(key) + if key == "" { + continue + } + if _, ok := snmpTopologyV1ActorPortCanonicalKeys[key]; ok { + continue + } + extra[key] = value + } + if len(extra) == 0 { + return nil + } + return extra +} diff --git a/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_actors.go b/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_actors.go new file mode 100644 index 00000000000000..a76621c6b9d3c8 --- /dev/null +++ b/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_actors.go @@ -0,0 +1,244 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package snmptopology + +import ( + "fmt" + topologyengine "github.com/netdata/netdata/go/plugins/pkg/l2topology" + topologyv1 "github.com/netdata/netdata/go/plugins/pkg/topology/v1" + "strings" +) + +func buildSNMPTopologyV1Actors(actors []topologyActor, stringsDict *topologyv1.StringDictionary) (topologyv1.Table, map[string]int) { + actorIndex := make(map[string]int, len(actors)) + usedActorIDs := make(map[string]struct{}, len(actors)) + for _, actor := range actors { + if actorID := strings.TrimSpace(actor.ActorID); actorID != "" { + usedActorIDs[actorID] = struct{}{} + } + } + ids := make([]any, len(actors)) + types := make([]any, len(actors)) + layers := make([]any, len(actors)) + sources := make([]any, len(actors)) + displayNames := make([]any, len(actors)) + chassisIDs := make([]any, len(actors)) + macAddresses := make([]any, len(actors)) + ipAddresses := make([]any, len(actors)) + hostnames := make([]any, len(actors)) + dnsNames := make([]any, len(actors)) + sysObjectIDs := make([]any, len(actors)) + sysNames := make([]any, len(actors)) + parentDevices := make([]any, len(actors)) + vendors := make([]any, len(actors)) + models := make([]any, len(actors)) + sysDescrs := make([]any, len(actors)) + sysLocations := make([]any, len(actors)) + sysContacts := make([]any, len(actors)) + managementIPs := make([]any, len(actors)) + protocols := make([]any, len(actors)) + capabilities := make([]any, len(actors)) + portsTotal := make([]any, len(actors)) + vlanCounts := make([]any, len(actors)) + fdbTotalMACs := make([]any, len(actors)) + lldpNeighborCounts := make([]any, len(actors)) + cdpNeighborCounts := make([]any, len(actors)) + endpointsTotal := make([]any, len(actors)) + chartIDPrefixes := make([]any, len(actors)) + netdataHostIDs := make([]any, len(actors)) + + for i, actor := range actors { + actorID := strings.TrimSpace(actor.ActorID) + if actorID == "" { + actorID = snmpTopologyV1FallbackActorID(actor, i, usedActorIDs) + } + actorIndex[actorID] = i + ids[i] = stringsDict.Ref(actorID) + types[i] = stringsDict.Ref(snmpTopologyV1ActorType(actor.ActorType)) + layers[i] = stringsDict.Ref(snmpTopologyV1ActorLayer(actor)) + sources[i] = stringsDict.Ref(firstNonEmptyString(actor.Source, snmpTopologyV1ProducerSource)) + displayNames[i] = nullableStringRef(stringsDict, snmpTopologyV1DisplayName(actor)) + chassisIDs[i] = stringArrayCell(actor.Match.ChassisIDs) + macAddresses[i] = stringArrayCell(actor.Match.MacAddresses) + ipAddresses[i] = stringArrayCell(actor.Match.IPAddresses) + hostnames[i] = stringArrayCell(actor.Match.Hostnames) + dnsNames[i] = stringArrayCell(actor.Match.DNSNames) + sysObjectIDs[i] = stringsDict.Ref(actor.Match.SysObjectID) + sysNames[i] = stringsDict.Ref(actor.Match.SysName) + parentDevices[i] = stringArrayCell(anyStringSlice(actor.Attributes["parent_devices"])) + vendors[i] = nullableStringRef(stringsDict, firstNonEmptyString(anyStringValue(actor.Attributes["vendor"]), anyStringValue(actor.Attributes["vendor_derived"]))) + models[i] = nullableStringRef(stringsDict, anyStringValue(actor.Attributes["model"])) + sysDescrs[i] = nullableStringRef(stringsDict, anyStringValue(actor.Attributes["sys_descr"])) + sysLocations[i] = nullableStringRef(stringsDict, anyStringValue(actor.Attributes["sys_location"])) + sysContacts[i] = nullableStringRef(stringsDict, anyStringValue(actor.Attributes["sys_contact"])) + managementIPs[i] = nullableStringRef(stringsDict, anyStringValue(actor.Attributes["management_ip"])) + protocols[i] = stringArrayCell(anyStringSlice(actor.Attributes["protocols"])) + if isEmptyArrayCell(protocols[i]) { + // Older SNMP topology payloads used learned_sources for discovered protocols. + protocols[i] = stringArrayCell(anyStringSlice(actor.Attributes["learned_sources"])) + } + if isEmptyArrayCell(protocols[i]) { + protocols[i] = nil + } + capabilities[i] = stringArrayCell(anyStringSlice(actor.Attributes["capabilities"])) + if isEmptyArrayCell(capabilities[i]) { + capabilities[i] = nil + } + portsTotal[i] = nullableUintValue(actor.Attributes["ports_total"]) + vlanCounts[i] = nullableUintValue(actor.Attributes["vlan_count"]) + fdbTotalMACs[i] = nullableUintValue(actor.Attributes["fdb_total_macs"]) + lldpNeighborCounts[i] = nullableUintValue(actor.Attributes["lldp_neighbor_count"]) + cdpNeighborCounts[i] = nullableUintValue(actor.Attributes["cdp_neighbor_count"]) + endpointsTotal[i] = nullableUintValue(actor.Attributes["endpoints_total"]) + chartIDPrefixes[i] = nullableStringRef(stringsDict, anyStringValue(actor.Attributes["chart_id_prefix"])) + netdataHostIDs[i] = nullableStringRef(stringsDict, anyStringValue(actor.Attributes["netdata_host_id"])) + } + + return topologyv1.MustTable(len(actors), + []topologyv1.Column{ + topologyv1.NewColumn("id", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithRole("identity")), + topologyv1.NewColumn("type", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithRole("group_key")), + topologyv1.NewColumn("layer", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithRole("group_key")), + topologyv1.NewColumn("source", "string_ref", topologyv1.WithDictionary("strings")), + topologyv1.NewColumn("display_name", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable(), topologyv1.WithRole("attribute")), + topologyv1.NewColumn("chassis_ids", "array", topologyv1.WithRole("merge_identity")), + topologyv1.NewColumn("mac_addresses", "array", topologyv1.WithRole("merge_identity")), + topologyv1.NewColumn("ip_addresses", "array", topologyv1.WithRole("merge_identity")), + topologyv1.NewColumn("hostnames", "array"), + topologyv1.NewColumn("dns_names", "array"), + topologyv1.NewColumn("sys_object_id", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithRole("merge_identity")), + topologyv1.NewColumn("sys_name", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithRole("merge_identity")), + topologyv1.NewColumn("parent_devices", "array", topologyv1.WithRole("parent_identity")), + topologyv1.NewColumn("vendor", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), + topologyv1.NewColumn("model", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), + topologyv1.NewColumn("sys_descr", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), + topologyv1.NewColumn("sys_location", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), + topologyv1.NewColumn("sys_contact", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), + topologyv1.NewColumn("management_ip", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), + topologyv1.NewColumn("protocols", "array", topologyv1.WithNullable()), + topologyv1.NewColumn("capabilities", "array", topologyv1.WithNullable()), + topologyv1.NewColumn("ports_total", "uint", topologyv1.WithNullable()), + topologyv1.NewColumn("vlan_count", "uint", topologyv1.WithNullable()), + topologyv1.NewColumn("fdb_total_macs", "uint", topologyv1.WithNullable()), + topologyv1.NewColumn("lldp_neighbor_count", "uint", topologyv1.WithNullable()), + topologyv1.NewColumn("cdp_neighbor_count", "uint", topologyv1.WithNullable()), + topologyv1.NewColumn("endpoints_total", "uint", topologyv1.WithNullable()), + topologyv1.NewColumn("chart_id_prefix", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), + topologyv1.NewColumn("netdata_host_id", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), + }, + []topologyv1.ColumnEncoding{ + topologyv1.Values(ids...), + topologyv1.Values(types...), + topologyv1.Values(layers...), + topologyv1.Values(sources...), + topologyv1.Values(displayNames...), + topologyv1.Values(chassisIDs...), + topologyv1.Values(macAddresses...), + topologyv1.Values(ipAddresses...), + topologyv1.Values(hostnames...), + topologyv1.Values(dnsNames...), + topologyv1.Values(sysObjectIDs...), + topologyv1.Values(sysNames...), + topologyv1.Values(parentDevices...), + topologyv1.Values(vendors...), + topologyv1.Values(models...), + topologyv1.Values(sysDescrs...), + topologyv1.Values(sysLocations...), + topologyv1.Values(sysContacts...), + topologyv1.Values(managementIPs...), + topologyv1.Values(protocols...), + topologyv1.Values(capabilities...), + topologyv1.Values(portsTotal...), + topologyv1.Values(vlanCounts...), + topologyv1.Values(fdbTotalMACs...), + topologyv1.Values(lldpNeighborCounts...), + topologyv1.Values(cdpNeighborCounts...), + topologyv1.Values(endpointsTotal...), + topologyv1.Values(chartIDPrefixes...), + topologyv1.Values(netdataHostIDs...), + }, + ), actorIndex +} + +func snmpTopologyV1FallbackActorID(actor topologyActor, index int, used map[string]struct{}) string { + for _, candidate := range []struct { + kind string + value string + }{ + {kind: "netdata_node", value: actor.Match.NetdataNodeID}, + {kind: "machine", value: actor.Match.NetdataMachineGUID}, + {kind: "cloud_instance", value: actor.Match.CloudInstanceID}, + {kind: "chassis", value: firstString(actor.Match.ChassisIDs)}, + {kind: "mac", value: firstString(actor.Match.MacAddresses)}, + {kind: "ip", value: firstString(actor.Match.IPAddresses)}, + {kind: "sys_name", value: actor.Match.SysName}, + {kind: "hostname", value: firstString(actor.Match.Hostnames)}, + {kind: "dns", value: firstString(actor.Match.DNSNames)}, + {kind: "sys_object_id", value: actor.Match.SysObjectID}, + {kind: "display_name", value: snmpTopologyV1DisplayName(actor)}, + } { + if value := strings.TrimSpace(candidate.value); value != "" { + return snmpTopologyV1UniqueFallbackActorID( + fmt.Sprintf("generated:%s:%s:%s", snmpTopologyV1ActorType(actor.ActorType), candidate.kind, topologyID(value, "actor")), + used, + ) + } + } + + return snmpTopologyV1UniqueFallbackActorID(fmt.Sprintf("generated:%d", index), used) +} + +func snmpTopologyV1UniqueFallbackActorID(actorID string, used map[string]struct{}) string { + if _, ok := used[actorID]; !ok { + used[actorID] = struct{}{} + return actorID + } + for suffix := 2; ; suffix++ { + candidate := fmt.Sprintf("%s_%d", actorID, suffix) + if _, ok := used[candidate]; !ok { + used[candidate] = struct{}{} + return candidate + } + } +} + +func snmpTopologyV1ActorType(actorType string) string { + normalized := strings.ToLower(strings.TrimSpace(actorType)) + if topologyengine.IsDeviceActorType(normalized) { + return normalized + } + switch normalized { + case snmpTopologyV1ActorDevice: + return snmpTopologyV1ActorDevice + case snmpTopologyV1ActorEndpoint: + return snmpTopologyV1ActorEndpoint + case snmpTopologyV1ActorSegment: + return snmpTopologyV1ActorSegment + default: + return "custom" + } +} + +func snmpTopologyV1DisplayName(actor topologyActor) string { + return firstNonEmptyString( + anyStringValue(actor.Attributes["display_name"]), + anyStringValue(actor.Attributes["name"]), + actor.Labels["display_name"], + actor.Labels["name"], + actor.Match.SysName, + firstString(actor.Match.Hostnames), + firstString(actor.Match.DNSNames), + ) +} + +func snmpTopologyV1ActorLayer(actor topologyActor) string { + switch snmpTopologyV1ActorType(actor.ActorType) { + case snmpTopologyV1ActorEndpoint, snmpTopologyV1ActorSegment: + return "network" + default: + if topologyengine.IsDeviceActorType(actor.ActorType) { + return "network" + } + return "custom" + } +} diff --git a/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_dynamic.go b/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_dynamic.go new file mode 100644 index 00000000000000..07bf12af4e7211 --- /dev/null +++ b/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_dynamic.go @@ -0,0 +1,158 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package snmptopology + +import ( + "fmt" + topologyv1 "github.com/netdata/netdata/go/plugins/pkg/topology/v1" + "math" + "strings" +) + +type topologyV1DynamicRow struct { + actorRef int + values map[string]any +} + +func buildSNMPTopologyV1DynamicTable(rows []topologyV1DynamicRow, stringsDict *topologyv1.StringDictionary) (topologyv1.Table, error) { + keysSet := make(map[string]struct{}) + normalizedRows := make([]map[string]any, len(rows)) + for i, row := range rows { + normalizedRows[i] = normalizeTopologyV1DynamicRowValues(row.values) + for key := range normalizedRows[i] { + keysSet[key] = struct{}{} + } + } + keys := sortedMapKeys(keysSet) + + columns := make([]topologyv1.Column, 0, len(keys)+1) + values := make([]topologyv1.ColumnEncoding, 0, len(keys)+1) + actorRefs := make([]any, len(rows)) + for i, row := range rows { + actorRefs[i] = row.actorRef + } + columns = append(columns, topologyv1.NewColumn("actor", "actor_ref", topologyv1.WithRole("reference"))) + values = append(values, topologyv1.Values(actorRefs...)) + + for _, key := range keys { + columnValues := make([]any, len(rows)) + for i, values := range normalizedRows { + if value, ok := values[key]; ok { + columnValues[i] = value + } + } + columnType := inferTopologyV1ColumnType(columnValues) + columnID := topologyID(key, "field") + column := topologyv1.NewColumn(columnID, columnType, topologyv1.WithNullable()) + if columnType == "string_ref" { + column = topologyv1.NewColumn(columnID, columnType, topologyv1.WithNullable(), topologyv1.WithDictionary("strings")) + for i, value := range columnValues { + if value == nil { + continue + } + columnValues[i] = stringsDict.Ref(fmt.Sprint(value)) + } + } + columns = append(columns, column) + values = append(values, topologyv1.Values(columnValues...)) + } + + return topologyv1.NewTable(len(rows), columns, values) +} + +func normalizeTopologyV1DynamicRowValues(values map[string]any) map[string]any { + type candidate struct { + raw string + value any + } + + candidates := make(map[string]candidate, len(values)) + for rawKey, value := range values { + key := strings.TrimSpace(rawKey) + if key != "" { + existing, exists := candidates[key] + if !exists || topologyV1DynamicRowKeyCandidateLess(rawKey, existing.raw, key) { + candidates[key] = candidate{raw: rawKey, value: value} + } + } + } + + normalized := make(map[string]any, len(candidates)) + for key, candidate := range candidates { + normalized[key] = candidate.value + } + return normalized +} + +func topologyV1DynamicRowKeyCandidateLess(rawKey, existingRawKey, normalizedKey string) bool { + rawKeyExact := rawKey == normalizedKey + existingRawKeyExact := existingRawKey == normalizedKey + if rawKeyExact != existingRawKeyExact { + return rawKeyExact + } + return rawKey < existingRawKey +} + +func inferTopologyV1ColumnType(values []any) string { + typ := "" + for _, value := range values { + if value == nil { + continue + } + valueType := topologyV1ValueType(value) + if typ == "" { + typ = valueType + continue + } + if typ != valueType { + return "json" + } + } + if typ == "" { + return "json" + } + return typ +} + +func topologyV1ValueType(value any) string { + switch typed := value.(type) { + case bool: + return "bool" + case int, int8, int16, int32, int64: + return "int" + case uint, uint8, uint16, uint32, uint64: + return "uint" + case float32: + if math.Trunc(float64(typed)) == float64(typed) { + return "int" + } + return "float" + case float64: + if math.Trunc(typed) == typed { + return "int" + } + return "float" + case string: + return "string_ref" + case []string, []int, []int64, []uint, []uint64, []float64, []bool: + return "array" + case []any: + if scalarArray(typed) { + return "array" + } + return "json" + default: + return "json" + } +} + +func scalarArray(values []any) bool { + for _, value := range values { + switch value.(type) { + case nil, bool, int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, float32, float64, string: + default: + return false + } + } + return true +} diff --git a/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_helpers_test.go b/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_helpers_test.go new file mode 100644 index 00000000000000..525ae2d9e52f1b --- /dev/null +++ b/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_helpers_test.go @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package snmptopology + +import ( + "testing" + + topologyv1 "github.com/netdata/netdata/go/plugins/pkg/topology/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestBuildSNMPTopologyV1DynamicTable_TrimsColumnKeysBeforeLookup(t *testing.T) { + dict := topologyv1.NewStringDictionary("") + + table, err := buildSNMPTopologyV1DynamicTable([]topologyV1DynamicRow{ + { + actorRef: 0, + values: map[string]any{ + " speed ": uint64(1000), + }, + }, + }, dict) + + require.NoError(t, err) + assert.Equal(t, []any{uint64(1000)}, topologyV1TestColumnValues(t, table, "speed")) +} + +func TestBuildSNMPTopologyV1DynamicTable_PrefersExactKeyOnTrimCollision(t *testing.T) { + dict := topologyv1.NewStringDictionary("") + + table, err := buildSNMPTopologyV1DynamicTable([]topologyV1DynamicRow{ + { + actorRef: 0, + values: map[string]any{ + " speed ": uint64(1000), + "speed": uint64(2000), + }, + }, + }, dict) + + require.NoError(t, err) + assert.Equal(t, []any{uint64(2000)}, topologyV1TestColumnValues(t, table, "speed")) +} + +func TestAnyStringSlice_DropsNilAndNonScalarItems(t *testing.T) { + require.Nil(t, anyStringSlice(nil)) + assert.Equal(t, []string{"up", "42", "false"}, anyStringSlice([]any{ + nil, + " up ", + 42, + false, + map[string]any{"not": "scalar"}, + })) +} + +func TestBuildSNMPTopologyV1Actors_UsesStableFallbackActorID(t *testing.T) { + actors := []topologyActor{ + {ActorType: "device", Match: topologyMatch{IPAddresses: []string{"10.0.0.2"}}}, + {ActorType: "device", Match: topologyMatch{IPAddresses: []string{"10.0.0.1"}}}, + } + reorderedActors := []topologyActor{actors[1], actors[0]} + + _, actorIndex := buildSNMPTopologyV1Actors(actors, topologyv1.NewStringDictionary("")) + _, reorderedActorIndex := buildSNMPTopologyV1Actors(reorderedActors, topologyv1.NewStringDictionary("")) + + require.Contains(t, actorIndex, "generated:device:ip:x_10.0.0.1") + require.Contains(t, actorIndex, "generated:device:ip:x_10.0.0.2") + require.Contains(t, reorderedActorIndex, "generated:device:ip:x_10.0.0.1") + require.Contains(t, reorderedActorIndex, "generated:device:ip:x_10.0.0.2") +} + +func TestBuildSNMPTopologyV1Actors_FallbackDoesNotCollideWithExplicitActorID(t *testing.T) { + _, actorIndex := buildSNMPTopologyV1Actors([]topologyActor{ + { + ActorType: "device", + Match: topologyMatch{IPAddresses: []string{"10.0.0.1"}}, + }, + { + ActorID: "generated:device:ip:x_10.0.0.1", + ActorType: "device", + }, + }, topologyv1.NewStringDictionary("")) + + assert.Equal(t, map[string]int{ + "generated:device:ip:x_10.0.0.1_2": 0, + "generated:device:ip:x_10.0.0.1": 1, + }, actorIndex) +} + +func TestBuildSNMPTopologyV1PortNeighborSummaries_NormalizesRemotePortName(t *testing.T) { + summaries := buildSNMPTopologyV1PortNeighborSummaries([]topologyLink{ + { + SrcActorID: "device-a", + DstActorID: "device-b", + Src: topologyLinkEndpoint{Attributes: map[string]any{ + "if_index": uint64(1), + "port_name": "Gi0/1", + }}, + Dst: topologyLinkEndpoint{Attributes: map[string]any{ + "port_name": " Gi0/2 ", + }}, + }, + { + SrcActorID: "device-a", + DstActorID: "device-b", + Src: topologyLinkEndpoint{Attributes: map[string]any{ + "if_index": uint64(1), + "port_name": "Gi0/1", + }}, + Dst: topologyLinkEndpoint{Attributes: map[string]any{ + "port_name": "gi0/2", + }}, + }, + }, map[string]int{"device-a": 0, "device-b": 1}) + + summary, ok := summaries[snmpTopologyV1PortNeighborKey{actorRef: 0, ifIndex: 1}] + require.True(t, ok) + assert.False(t, summary.ambiguous) +} + +func TestBuildSNMPTopologyV1PortNeighborSummaries_FillsMissingRemotePortName(t *testing.T) { + summaries := buildSNMPTopologyV1PortNeighborSummaries([]topologyLink{ + { + SrcActorID: "device-a", + DstActorID: "device-b", + Src: topologyLinkEndpoint{Attributes: map[string]any{ + "if_index": uint64(1), + "port_name": "Gi0/1", + }}, + }, + { + SrcActorID: "device-a", + DstActorID: "device-b", + Src: topologyLinkEndpoint{Attributes: map[string]any{ + "if_index": uint64(1), + "port_name": "Gi0/1", + }}, + Dst: topologyLinkEndpoint{Attributes: map[string]any{ + "port_name": "Gi0/2", + }}, + }, + }, map[string]int{"device-a": 0, "device-b": 1}) + + summary, ok := summaries[snmpTopologyV1PortNeighborKey{actorRef: 0, ifIndex: 1}] + require.True(t, ok) + assert.False(t, summary.ambiguous) + assert.Equal(t, "Gi0/2", summary.remotePortName) +} + +func topologyV1TestColumnValues(t *testing.T, table topologyv1.Table, columnID string) []any { + t.Helper() + + for i, column := range table.Columns { + if column.ID != columnID { + continue + } + values, ok := table.Values[i].(topologyv1.ValuesEncoding) + require.True(t, ok) + return values.Values + } + require.Failf(t, "missing column", "column %q not found", columnID) + return nil +} diff --git a/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_links.go b/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_links.go new file mode 100644 index 00000000000000..cfd258849840f5 --- /dev/null +++ b/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_links.go @@ -0,0 +1,226 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package snmptopology + +import ( + "fmt" + topologyv1 "github.com/netdata/netdata/go/plugins/pkg/topology/v1" + "strings" +) + +func buildSNMPTopologyV1Links( + links []topologyLink, + actorIndex map[string]int, + stringsDict *topologyv1.StringDictionary, +) (topologyv1.Table, topologyv1.EvidenceMap, error) { + srcActors := make([]any, len(links)) + dstActors := make([]any, len(links)) + linkTypes := make([]any, len(links)) + protocols := make([]any, len(links)) + directions := make([]any, len(links)) + states := make([]any, len(links)) + srcPortNames := make([]any, len(links)) + dstPortNames := make([]any, len(links)) + evidenceCounts := make([]any, len(links)) + discoveredAt := make([]any, len(links)) + lastSeen := make([]any, len(links)) + evidenceRowsByType := make(map[string]*snmpTopologyV1EvidenceRows) + + for i, link := range links { + src, ok := actorIndex[strings.TrimSpace(link.SrcActorID)] + if !ok { + return topologyv1.Table{}, nil, fmt.Errorf("link %d references unknown source actor %q", i, link.SrcActorID) + } + dst, ok := actorIndex[strings.TrimSpace(link.DstActorID)] + if !ok { + return topologyv1.Table{}, nil, fmt.Errorf("link %d references unknown destination actor %q", i, link.DstActorID) + } + protocol := firstNonEmptyString(link.Protocol, link.LinkType, "l2") + linkType := snmpTopologyV1LinkType(link) + srcActors[i] = src + dstActors[i] = dst + linkTypes[i] = stringsDict.Ref(linkType) + protocols[i] = stringsDict.Ref(protocol) + directions[i] = stringsDict.Ref(firstNonEmptyString(link.Direction, "observed")) + states[i] = nullableStringRef(stringsDict, link.State) + srcPortNames[i] = nullableStringRef(stringsDict, topologyV1EndpointPortName(link.Src)) + dstPortNames[i] = nullableStringRef(stringsDict, topologyV1EndpointPortName(link.Dst)) + evidenceCounts[i] = 1 + discoveredAt[i] = nullableTime(link.DiscoveredAt) + lastSeen[i] = nullableTime(link.LastSeen) + srcEndpoint := nullableJSON(link.Src.Attributes) + dstEndpoint := nullableJSON(link.Dst.Attributes) + metrics := nullableJSON(link.Metrics) + + evidenceRows := evidenceRowsByType[linkType] + if evidenceRows == nil { + evidenceRows = &snmpTopologyV1EvidenceRows{} + evidenceRowsByType[linkType] = evidenceRows + } + evidenceRows.linkRefs = append(evidenceRows.linkRefs, i) + evidenceRows.srcActors = append(evidenceRows.srcActors, src) + evidenceRows.dstActors = append(evidenceRows.dstActors, dst) + evidenceRows.protocols = append(evidenceRows.protocols, stringsDict.Ref(protocol)) + evidenceRows.directions = append(evidenceRows.directions, stringsDict.Ref(firstNonEmptyString(link.Direction, "observed"))) + evidenceRows.states = append(evidenceRows.states, nullableStringRef(stringsDict, link.State)) + evidenceRows.srcPortNames = append(evidenceRows.srcPortNames, nullableStringRef(stringsDict, topologyV1EndpointPortName(link.Src))) + evidenceRows.dstPortNames = append(evidenceRows.dstPortNames, nullableStringRef(stringsDict, topologyV1EndpointPortName(link.Dst))) + evidenceRows.srcIfIndexes = append(evidenceRows.srcIfIndexes, nullableUintValue(link.Src.Attributes["if_index"])) + evidenceRows.dstIfIndexes = append(evidenceRows.dstIfIndexes, nullableUintValue(link.Dst.Attributes["if_index"])) + evidenceRows.srcManagementIPs = append(evidenceRows.srcManagementIPs, nullableStringRef(stringsDict, topologyV1EndpointString(link.Src, "management_ip"))) + evidenceRows.dstManagementIPs = append(evidenceRows.dstManagementIPs, nullableStringRef(stringsDict, topologyV1EndpointString(link.Dst, "management_ip"))) + evidenceRows.confidences = append(evidenceRows.confidences, nullableStringRef(stringsDict, topologyMetricValueString(link.Metrics, "confidence"))) + evidenceRows.inferences = append(evidenceRows.inferences, nullableStringRef(stringsDict, topologyMetricValueString(link.Metrics, "inference"))) + evidenceRows.attachmentModes = append(evidenceRows.attachmentModes, nullableStringRef(stringsDict, topologyMetricValueString(link.Metrics, "attachment_mode"))) + evidenceRows.srcEndpoints = append(evidenceRows.srcEndpoints, srcEndpoint) + evidenceRows.dstEndpoints = append(evidenceRows.dstEndpoints, dstEndpoint) + evidenceRows.metrics = append(evidenceRows.metrics, metrics) + } + + linkTable := topologyv1.MustTable(len(links), + []topologyv1.Column{ + topologyv1.NewColumn("src_actor", "actor_ref", topologyv1.WithRole("reference")), + topologyv1.NewColumn("dst_actor", "actor_ref", topologyv1.WithRole("reference")), + topologyv1.NewColumn("type", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithRole("group_key")), + topologyv1.NewColumn("protocol", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithRole("group_key")), + topologyv1.NewColumn("direction", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithRole("group_key")), + topologyv1.NewColumn("state", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), + topologyv1.NewColumn("src_port_name", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), + topologyv1.NewColumn("dst_port_name", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), + topologyv1.NewColumn("evidence_count", "uint", topologyv1.WithAggregation("sum")), + topologyv1.NewColumn("discovered_at", "timestamp", topologyv1.WithNullable(), topologyv1.WithRole("timestamp")), + topologyv1.NewColumn("last_seen", "timestamp", topologyv1.WithNullable(), topologyv1.WithRole("timestamp")), + }, + []topologyv1.ColumnEncoding{ + topologyv1.Values(srcActors...), + topologyv1.Values(dstActors...), + topologyv1.Values(linkTypes...), + topologyv1.Values(protocols...), + topologyv1.Values(directions...), + topologyv1.Values(states...), + topologyv1.Values(srcPortNames...), + topologyv1.Values(dstPortNames...), + topologyv1.Values(evidenceCounts...), + topologyv1.Values(discoveredAt...), + topologyv1.Values(lastSeen...), + }, + ) + + evidenceSections := make(topologyv1.EvidenceMap, len(evidenceRowsByType)) + for linkType, rows := range evidenceRowsByType { + evidenceSections[linkType] = topologyv1.EvidenceSection{ + Type: linkType, + Table: rows.table(), + } + } + + return linkTable, evidenceSections, nil +} + +type snmpTopologyV1EvidenceRows struct { + linkRefs []any + srcActors []any + dstActors []any + protocols []any + directions []any + states []any + srcPortNames []any + dstPortNames []any + srcIfIndexes []any + dstIfIndexes []any + srcManagementIPs []any + dstManagementIPs []any + confidences []any + inferences []any + attachmentModes []any + srcEndpoints []any + dstEndpoints []any + metrics []any +} + +func (rows *snmpTopologyV1EvidenceRows) table() topologyv1.Table { + return topologyv1.MustTable(len(rows.linkRefs), + snmpTopologyV1EvidenceColumns(), + []topologyv1.ColumnEncoding{ + topologyv1.Values(rows.linkRefs...), + topologyv1.Values(rows.srcActors...), + topologyv1.Values(rows.dstActors...), + topologyv1.Values(rows.protocols...), + topologyv1.Values(rows.directions...), + topologyv1.Values(rows.states...), + topologyv1.Values(rows.srcPortNames...), + topologyv1.Values(rows.dstPortNames...), + topologyv1.Values(rows.srcIfIndexes...), + topologyv1.Values(rows.dstIfIndexes...), + topologyv1.Values(rows.srcManagementIPs...), + topologyv1.Values(rows.dstManagementIPs...), + topologyv1.Values(rows.confidences...), + topologyv1.Values(rows.inferences...), + topologyv1.Values(rows.attachmentModes...), + topologyv1.Values(rows.srcEndpoints...), + topologyv1.Values(rows.dstEndpoints...), + topologyv1.Values(rows.metrics...), + }, + ) +} + +func snmpTopologyV1EvidenceColumns() []topologyv1.Column { + return []topologyv1.Column{ + topologyv1.NewColumn("link", "link_ref", topologyv1.WithRole("reference")), + topologyv1.NewColumn("src_actor", "actor_ref", topologyv1.WithRole("reference")), + topologyv1.NewColumn("dst_actor", "actor_ref", topologyv1.WithRole("reference")), + topologyv1.NewColumn("protocol", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithRole("group_key")), + topologyv1.NewColumn("direction", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithRole("group_key")), + topologyv1.NewColumn("state", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), + topologyv1.NewColumn("src_port_name", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), + topologyv1.NewColumn("dst_port_name", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), + topologyv1.NewColumn("src_if_index", "uint", topologyv1.WithNullable()), + topologyv1.NewColumn("dst_if_index", "uint", topologyv1.WithNullable()), + topologyv1.NewColumn("src_management_ip", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), + topologyv1.NewColumn("dst_management_ip", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), + topologyv1.NewColumn("confidence", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), + topologyv1.NewColumn("inference", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), + topologyv1.NewColumn("attachment_mode", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), + topologyv1.NewColumn("src_endpoint", "json", topologyv1.WithNullable()), + topologyv1.NewColumn("dst_endpoint", "json", topologyv1.WithNullable()), + topologyv1.NewColumn("metrics", "json", topologyv1.WithNullable()), + } +} + +func snmpTopologyV1LinkType(link topologyLink) string { + if snmpTopologyV1LinkIsProbable(link) { + return snmpTopologyV1LinkProbable + } + + switch strings.ToLower(strings.TrimSpace(firstNonEmptyString(link.Protocol, link.LinkType))) { + case snmpTopologyV1LinkLLDP: + return snmpTopologyV1LinkLLDP + case snmpTopologyV1LinkCDP: + return snmpTopologyV1LinkCDP + case snmpTopologyV1LinkBridge: + return snmpTopologyV1LinkBridge + case snmpTopologyV1LinkFDB: + return snmpTopologyV1LinkFDB + case snmpTopologyV1LinkSTP: + return snmpTopologyV1LinkSTP + case snmpTopologyV1LinkARP: + return snmpTopologyV1LinkARP + case snmpTopologyV1LinkSNMP: + return snmpTopologyV1LinkSNMP + default: + return snmpTopologyV1LinkObservation + } +} + +func snmpTopologyV1LinkIsProbable(link topologyLink) bool { + if strings.EqualFold(strings.TrimSpace(link.State), snmpTopologyV1LinkProbable) { + return true + } + if len(link.Metrics) == 0 { + return false + } + if strings.EqualFold(topologyMetricValueString(link.Metrics, "inference"), snmpTopologyV1LinkProbable) { + return true + } + return strings.HasPrefix(strings.ToLower(topologyMetricValueString(link.Metrics, "attachment_mode")), snmpTopologyV1LinkProbable+"_") +} diff --git a/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_presentation.go b/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_presentation.go new file mode 100644 index 00000000000000..707a905d78e14f --- /dev/null +++ b/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_presentation.go @@ -0,0 +1,435 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package snmptopology + +import topologyv1 "github.com/netdata/netdata/go/plugins/pkg/topology/v1" + +func snmpTopologyV1ActorTypes() map[string]topologyv1.ActorType { + types := make(map[string]topologyv1.ActorType) + addDevice := func(id, label, colorSlot, icon string) { + types[id] = topologyv1.ActorType{ + Layer: "network", + Identity: []string{"id"}, + MergeIdentity: []string{"chassis_ids", "mac_addresses", "ip_addresses", "sys_name"}, + AggregationScopes: []string{"device", "network"}, + Search: &topologyv1.ActorSearchPolicy{ + Columns: []string{"display_name", "sys_name", "management_ip", "vendor", "model"}, + }, + Presentation: &topologyv1.ActorPresentation{ + Label: label, + Role: "actor", + Icon: icon, + ColorSlot: colorSlot, + Border: &topologyv1.BorderPresentation{Enabled: new(true)}, + Size: &topologyv1.ActorSizePresentation{Mode: "link_count", Scale: "emphasized"}, + Layout: &topologyv1.ActorLayoutPresentation{Repulsion: "stronger"}, + LabelPolicy: &topologyv1.LabelPolicy{ + Columns: []string{"display_name", "sys_name"}, + Fallback: "type_label", + MaxLength: 80, + Array: "reject", + }, + Ports: &topologyv1.ActorPortsPresentation{ + ShowBullets: true, + Sources: []topologyv1.PortSourcePresentation{ + { + Source: "actor_table", + Table: "actor_ports", + ActorColumn: "actor", + NameColumn: "name", + TypeColumn: "topology_role", + DefaultType: "topology", + StatusColumn: "oper_status", + ModeColumn: "link_mode", + RoleColumn: "topology_role", + }, + }, + }, + Modal: snmpTopologyV1DeviceModal(), + }, + } + } + + addDevice("device", "Device", "primary", "server") + addDevice("router", "Router", "primary", "router") + addDevice("switch", "Switch", "primary", "switch") + addDevice("firewall", "Firewall", "warning", "firewall") + addDevice("access_point", "Access Point", "info", "access_point") + addDevice("server", "Server", "secondary", "server") + addDevice("storage", "Storage", "secondary", "storage") + addDevice("load_balancer", "Load Balancer", "info", "load_balancer") + addDevice("printer", "Printer", "neutral", "printer") + addDevice("phone", "Phone", "neutral", "phone") + addDevice("ups", "UPS", "structural", "ups") + addDevice("camera", "Camera", "neutral", "camera") + + types[snmpTopologyV1ActorEndpoint] = topologyv1.ActorType{ + Layer: "network", + Identity: []string{"id"}, + MergeIdentity: []string{"mac_addresses", "ip_addresses"}, + AggregationScopes: []string{"endpoint", "network"}, + Search: &topologyv1.ActorSearchPolicy{Columns: []string{"display_name"}}, + Presentation: &topologyv1.ActorPresentation{ + Label: "Inferred endpoint", + Role: "endpoint", + Icon: "remote-endpoint", + ColorSlot: "derived", + Border: &topologyv1.BorderPresentation{Enabled: new(true)}, + Size: &topologyv1.ActorSizePresentation{Mode: "fixed", Scale: "compact"}, + Layout: &topologyv1.ActorLayoutPresentation{Repulsion: "weaker"}, + LabelPolicy: &topologyv1.LabelPolicy{ + Columns: []string{"display_name"}, + Fallback: "type_label", + MaxLength: 80, + Array: "reject", + }, + Modal: snmpTopologyV1EndpointModal(), + }, + } + types[snmpTopologyV1ActorSegment] = topologyv1.ActorType{ + Layer: "network", + Identity: []string{"id"}, + MergeIdentity: []string{"id"}, + ParentIdentity: []string{"parent_devices"}, + AggregationScopes: []string{"segment", "network"}, + Search: &topologyv1.ActorSearchPolicy{Enabled: new(false)}, + Presentation: &topologyv1.ActorPresentation{ + Label: "Network segment", + Role: "group", + Icon: "segment", + ColorSlot: "dim", + Size: &topologyv1.ActorSizePresentation{Mode: "fixed", Scale: "compact"}, + Layout: &topologyv1.ActorLayoutPresentation{Repulsion: "weakest"}, + LabelPolicy: &topologyv1.LabelPolicy{ + Columns: []string{"display_name"}, + Fallback: "type_label", + MaxLength: 80, + Array: "reject", + }, + Modal: snmpTopologyV1EndpointModal(), + }, + } + types["custom"] = topologyv1.ActorType{ + Layer: "custom", + Identity: []string{"id"}, + MergeIdentity: []string{"id"}, + AggregationScopes: []string{"network"}, + Search: &topologyv1.ActorSearchPolicy{Columns: []string{"display_name"}}, + Presentation: &topologyv1.ActorPresentation{ + Label: "Custom", + Role: "actor", + Icon: "service", + ColorSlot: "neutral", + LabelPolicy: &topologyv1.LabelPolicy{ + Columns: []string{"display_name"}, + Fallback: "type_label", + MaxLength: 80, + Array: "reject", + }, + Modal: snmpTopologyV1EndpointModal(), + }, + } + return types +} + +func snmpTopologyV1DeviceModal() *topologyv1.ModalPresentation { + return &topologyv1.ModalPresentation{ + Labels: snmpTopologyV1DeviceModalLabels(), + MiniTopology: &topologyv1.ModalMiniTopologyPresentation{Depth: 1}, + Sections: []topologyv1.ModalSection{ + { + ID: "ports", + Label: "Ports", + Order: 1, + Source: topologyv1.ModalSource{ + Kind: "actor_table", + Table: "actor_ports", + }, + OwnerFilter: &topologyv1.ModalOwnerFilter{ + Mode: "actor_column", + ActorColumn: "actor", + }, + Columns: snmpTopologyV1PortModalColumns(), + Sort: &topologyv1.ModalSort{Column: "if_index", Direction: "asc"}, + }, + snmpTopologyV1PortLinksSection(2), + }, + } +} + +func snmpTopologyV1EndpointModal() *topologyv1.ModalPresentation { + return &topologyv1.ModalPresentation{ + Labels: snmpTopologyV1EndpointModalLabels(), + MiniTopology: &topologyv1.ModalMiniTopologyPresentation{Depth: 1}, + Sections: []topologyv1.ModalSection{snmpTopologyV1LinksSection(1)}, + } +} + +func snmpTopologyV1DeviceModalLabels() *topologyv1.ModalLabelsPresentation { + return &topologyv1.ModalLabelsPresentation{ + Table: "actor_labels", + Identification: &topologyv1.ModalLabelIdentificationPresentation{ + Fields: []topologyv1.ModalLabelIdentificationField{ + {Key: "display_name", Label: "Name", MaxValues: 1}, + {Key: "management_ip", Label: "Management IP", MaxValues: 1}, + {Key: "vendor", Label: "Vendor", MaxValues: 1}, + {Key: "model", Label: "Model", MaxValues: 1}, + {Key: "ports_total", Label: "Ports", MaxValues: 1}, + {Key: "lldp_neighbor_count", Label: "LLDP", MaxValues: 1}, + {Key: "cdp_neighbor_count", Label: "CDP", MaxValues: 1}, + }, + }, + } +} + +func snmpTopologyV1EndpointModalLabels() *topologyv1.ModalLabelsPresentation { + return &topologyv1.ModalLabelsPresentation{ + Table: "actor_labels", + Identification: &topologyv1.ModalLabelIdentificationPresentation{ + Fields: []topologyv1.ModalLabelIdentificationField{ + {Key: "display_name", Label: "Name", MaxValues: 1}, + {Key: "ip_address", Label: "IP", MaxValues: 2}, + {Key: "mac_address", Label: "MAC", MaxValues: 2}, + {Key: "hostname", Label: "Hostname", MaxValues: 2}, + }, + }, + } +} + +func snmpTopologyV1LinksSection(order int) topologyv1.ModalSection { + return topologyv1.ModalSection{ + ID: "links", + Label: "Links", + Order: order, + Source: topologyv1.ModalSource{ + Kind: "links", + }, + OwnerFilter: &topologyv1.ModalOwnerFilter{ + Mode: "incident_link", + SrcActorColumn: "src_actor", + DstActorColumn: "dst_actor", + }, + Columns: []topologyv1.ModalColumn{ + { + ID: "remote", + Label: "Remote Actor", + Projection: topologyv1.ModalProjection{ + Kind: "opposite_actor", + SrcActorColumn: "src_actor", + DstActorColumn: "dst_actor", + }, + Cell: "actor_link", + }, + modalSelectedSidePortColumn("local_port", "Local Port", "src_port_name", "dst_port_name"), + modalSelectedSidePortColumn("remote_port", "Remote Port", "dst_port_name", "src_port_name"), + modalDirectColumn("protocol", "Protocol", "protocol", "badge"), + modalDirectColumn("direction", "Direction", "direction", "text"), + modalDirectColumn("state", "State", "state", "badge"), + modalDirectColumn("evidence_count", "Evidence", "evidence_count", "number"), + }, + } +} + +func snmpTopologyV1PortLinksSection(order int) topologyv1.ModalSection { + return topologyv1.ModalSection{ + ID: "port_neighbors", + Label: "Port Neighbors", + Order: order, + Source: topologyv1.ModalSource{ + Kind: "actor_table", + Table: "actor_port_links", + }, + OwnerFilter: &topologyv1.ModalOwnerFilter{ + Mode: "actor_column", + ActorColumn: "actor", + }, + Columns: snmpTopologyV1PortLinkModalColumns(), + Sort: &topologyv1.ModalSort{Column: "if_index", Direction: "asc"}, + EmptyLabel: "No port neighbors", + } +} + +func modalSelectedSidePortColumn(id, label, selectedSrcPortColumn, selectedDstPortColumn string) topologyv1.ModalColumn { + return topologyv1.ModalColumn{ + ID: id, + Label: label, + Projection: topologyv1.ModalProjection{ + Kind: "selected_side_endpoint", + SrcActorColumn: "src_actor", + DstActorColumn: "dst_actor", + LocalPortColumn: selectedSrcPortColumn, + RemotePortColumn: selectedDstPortColumn, + }, + Cell: "text", + } +} + +func modalDirectColumn(id, label, sourceColumn, cell string) topologyv1.ModalColumn { + return topologyv1.ModalColumn{ + ID: id, + Label: label, + Projection: topologyv1.ModalProjection{Kind: "direct", Column: sourceColumn}, + Cell: cell, + } +} + +func modalDirectColumnWithVisibility(id, label, sourceColumn, cell, visibility string) topologyv1.ModalColumn { + column := modalDirectColumn(id, label, sourceColumn, cell) + column.Visibility = visibility + return column +} + +func modalActorRefColumn(id, label, actorColumn string) topologyv1.ModalColumn { + return topologyv1.ModalColumn{ + ID: id, + Label: label, + Projection: topologyv1.ModalProjection{ + Kind: "actor_ref_label", + ActorColumn: actorColumn, + }, + Cell: "actor_link", + } +} + +func modalActorRefColumnWithVisibility(id, label, actorColumn, visibility string) topologyv1.ModalColumn { + column := modalActorRefColumn(id, label, actorColumn) + column.Visibility = visibility + return column +} + +func snmpTopologyV1PortTypes() map[string]topologyv1.PortType { + return map[string]topologyv1.PortType{ + "lldp": {Presentation: &topologyv1.PortPresentation{Label: "lldp/cdp", ColorSlot: "accent", Opacity: "normal"}}, + "switch_facing": {Presentation: &topologyv1.PortPresentation{Label: "switch-facing", ColorSlot: "primary", Opacity: "normal"}}, + "host_facing": {Presentation: &topologyv1.PortPresentation{Label: "host-facing", ColorSlot: "secondary", Opacity: "normal"}}, + "host_candidate": {Presentation: &topologyv1.PortPresentation{Label: "host-candidate", ColorSlot: "info", Opacity: "normal"}}, + "trunk": {Presentation: &topologyv1.PortPresentation{Label: "trunk", ColorSlot: "warning", Opacity: "normal"}}, + "access": {Presentation: &topologyv1.PortPresentation{Label: "access", ColorSlot: "derived", Opacity: "normal"}}, + "topology": {Presentation: &topologyv1.PortPresentation{Label: "unclassified", ColorSlot: "neutral", Opacity: "normal"}}, + "idle": {Presentation: &topologyv1.PortPresentation{Label: "idle", ColorSlot: "muted", Opacity: "muted"}}, + "unknown": {Presentation: &topologyv1.PortPresentation{Label: "unknown", ColorSlot: "dim", Opacity: "muted"}}, + } +} + +type snmpTopologyV1LinkTypeSpec struct { + id string + label string + colorSlot string + lineStyle string + width string + semanticRole string +} + +func snmpTopologyV1LinkTypeSpecs() []snmpTopologyV1LinkTypeSpec { + return []snmpTopologyV1LinkTypeSpec{ + {id: snmpTopologyV1LinkLLDP, label: "LLDP", colorSlot: "accent", lineStyle: "solid", width: "thick", semanticRole: "discovery"}, + {id: snmpTopologyV1LinkCDP, label: "CDP", colorSlot: "accent", lineStyle: "solid", width: "thick", semanticRole: "discovery"}, + {id: snmpTopologyV1LinkBridge, label: "Bridge", colorSlot: "neutral", lineStyle: "solid", width: "normal", semanticRole: "normal"}, + {id: snmpTopologyV1LinkFDB, label: "FDB", colorSlot: "neutral", lineStyle: "solid", width: "normal", semanticRole: "normal"}, + {id: snmpTopologyV1LinkSTP, label: "STP", colorSlot: "muted", lineStyle: "solid", width: "normal", semanticRole: "normal"}, + {id: snmpTopologyV1LinkARP, label: "ARP", colorSlot: "muted", lineStyle: "solid", width: "normal", semanticRole: "normal"}, + {id: snmpTopologyV1LinkSNMP, label: "SNMP", colorSlot: "primary", lineStyle: "solid", width: "normal", semanticRole: "normal"}, + {id: snmpTopologyV1LinkProbable, label: "Probable", colorSlot: "dim", lineStyle: "solid", width: "normal", semanticRole: "normal"}, + {id: snmpTopologyV1LinkObservation, label: "L2 observation", colorSlot: "neutral", lineStyle: "solid", width: "normal", semanticRole: "normal"}, + } +} + +func snmpTopologyV1LinkTypes() map[string]topologyv1.LinkType { + types := make(map[string]topologyv1.LinkType) + for _, spec := range snmpTopologyV1LinkTypeSpecs() { + types[spec.id] = topologyv1.LinkType{ + Orientation: "observed_bidirectional", + DirectionRole: "observation", + SemanticRole: spec.semanticRole, + Aggregation: topologyv1.LinkAggregation{ + Direction: "canonicalize_unordered", + Evidence: "append", + Metrics: map[string]string{ + "evidence_count": "sum", + }, + }, + EvidenceTypes: []string{spec.id}, + Presentation: &topologyv1.LinkPresentation{ + Label: spec.label, + ColorSlot: spec.colorSlot, + LineStyle: spec.lineStyle, + Width: spec.width, + Curve: "straight", + Arrow: "none", + }, + } + } + return types +} + +func snmpTopologyV1EvidenceTypes() map[string]topologyv1.EvidenceType { + types := make(map[string]topologyv1.EvidenceType) + for _, spec := range snmpTopologyV1LinkTypeSpecs() { + types[spec.id] = topologyv1.EvidenceType{ + LinkType: spec.id, + Role: "observation_evidence", + Columns: snmpTopologyV1EvidenceColumns(), + MatchColumns: []string{ + "src_actor", + "dst_actor", + "protocol", + "src_endpoint", + "dst_endpoint", + }, + } + } + return types +} + +func snmpTopologyV1Presentation() *topologyv1.Presentation { + return &topologyv1.Presentation{ + ProfileVersion: "snmp-l2.v1", + Selection: &topologyv1.SelectionPresentation{ + ActorClick: &topologyv1.ActorClickPresentation{Mode: "highlight_connections"}, + }, + Legend: &topologyv1.PresentationLegend{ + Actors: []topologyv1.LegendEntry{ + {Type: "router", Label: "Router"}, + {Type: "switch", Label: "Switch"}, + {Type: "firewall", Label: "Firewall"}, + {Type: "access_point", Label: "Access Point"}, + {Type: "server", Label: "Server"}, + {Type: "storage", Label: "Storage"}, + {Type: "load_balancer", Label: "Load Balancer"}, + {Type: "printer", Label: "Printer"}, + {Type: "phone", Label: "IP Phone"}, + {Type: "ups", Label: "UPS / PDU"}, + {Type: "camera", Label: "Camera / Media"}, + {Type: "device", Label: "Other device"}, + {Type: "custom", Label: "Other"}, + {Type: "endpoint", Label: "Inferred endpoint"}, + {Type: "segment", Label: "Network segment"}, + }, + Links: []topologyv1.LegendEntry{ + {Type: snmpTopologyV1LinkLLDP, Label: "LLDP"}, + {Type: snmpTopologyV1LinkCDP, Label: "CDP"}, + {Type: snmpTopologyV1LinkSNMP, Label: "SNMP"}, + {Type: snmpTopologyV1LinkBridge, Label: "Bridge"}, + {Type: snmpTopologyV1LinkProbable, Label: "Probable"}, + }, + Ports: []topologyv1.LegendEntry{ + {Type: "lldp", Label: "lldp/cdp"}, + {Type: "switch_facing", Label: "switch-facing"}, + {Type: "host_facing", Label: "host-facing"}, + {Type: "host_candidate", Label: "host-candidate"}, + {Type: "trunk", Label: "trunk"}, + {Type: "access", Label: "access"}, + {Type: "topology", Label: "unclassified"}, + {Type: "idle", Label: "idle"}, + {Type: "unknown", Label: "unknown"}, + }, + }, + PortFields: []topologyv1.PresentationField{ + {Key: "type", Label: "Type"}, + {Key: "role", Label: "Role"}, + {Key: "status", Label: "Status"}, + {Key: "mode", Label: "Mode"}, + {Key: "sources", Label: "Sources"}, + }, + } +} diff --git a/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_table_types.go b/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_table_types.go new file mode 100644 index 00000000000000..cb11ae486c9b8d --- /dev/null +++ b/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_table_types.go @@ -0,0 +1,160 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package snmptopology + +import topologyv1 "github.com/netdata/netdata/go/plugins/pkg/topology/v1" + +func snmpTopologyV1ActorPortsTableType() topologyv1.TableType { + return topologyv1.TableType{ + Role: "actor_detail", + Owner: "actor", + Aggregation: "append", + Columns: snmpTopologyV1ActorPortsColumns(), + Presentation: &topologyv1.TableTypePresentation{ + Label: "Ports", + Order: 1, + Columns: snmpTopologyV1PortModalColumns(), + }, + } +} + +func snmpTopologyV1PortModalColumns() []topologyv1.ModalColumn { + return []topologyv1.ModalColumn{ + modalDirectColumn("if_index", "Port ID", "if_index", "number"), + modalDirectColumn("name", "Port", "name", "text"), + modalDirectColumn("oper_status", "Status", "oper_status", "badge"), + modalDirectColumn("admin_status", "Admin", "admin_status", "badge"), + modalDirectColumn("port_type", "Type", "port_type", "badge"), + modalDirectColumn("link_mode", "Mode", "link_mode", "badge"), + modalDirectColumn("topology_role", "Role", "topology_role", "badge"), + modalDirectColumn("vlan_ids", "VLANs", "vlan_ids", "array_count"), + modalDirectColumn("fdb_mac_count", "FDB", "fdb_mac_count", "number"), + modalDirectColumn("link_count", "Links", "link_count", "number"), + modalDirectColumn("neighbor_count", "Neighbors", "neighbor_count", "number"), + modalActorRefColumnWithVisibility("neighbor_actor", "Neighbor", "neighbor_actor", "expanded"), + modalDirectColumnWithVisibility("neighbor_port_name", "Neighbor Port", "neighbor_port_name", "text", "expanded"), + modalDirectColumnWithVisibility("if_name", "ifName", "if_name", "text", "expanded"), + modalDirectColumnWithVisibility("if_descr", "ifDescr", "if_descr", "text", "expanded"), + modalDirectColumnWithVisibility("if_alias", "Alias", "if_alias", "text", "expanded"), + modalDirectColumnWithVisibility("port_id", "Source Port ID", "port_id", "text", "expanded"), + modalDirectColumnWithVisibility("mac", "MAC", "mac", "text", "expanded"), + modalDirectColumnWithVisibility("speed", "Speed", "speed", "number", "expanded"), + modalDirectColumnWithVisibility("stp_state", "STP", "stp_state", "badge", "expanded"), + modalDirectColumnWithVisibility("neighbors", "Neighbor Data", "neighbors", "debug_json", "debug"), + modalDirectColumnWithVisibility("vlans", "VLAN Data", "vlans", "debug_json", "debug"), + modalDirectColumnWithVisibility("extra", "Extra", "extra", "debug_json", "debug"), + } +} + +func snmpTopologyV1ActorPortLinksTableType() topologyv1.TableType { + return topologyv1.TableType{ + Role: "actor_detail", + Owner: "actor", + Aggregation: "append", + Columns: snmpTopologyV1ActorPortLinksColumns(), + Presentation: &topologyv1.TableTypePresentation{ + Label: "Port Neighbors", + Order: 2, + Columns: snmpTopologyV1PortLinkModalColumns(), + }, + } +} + +func snmpTopologyV1PortLinkModalColumns() []topologyv1.ModalColumn { + return []topologyv1.ModalColumn{ + modalDirectColumn("if_index", "Port ID", "if_index", "number"), + modalDirectColumn("port_name", "Port", "port_name", "text"), + modalActorRefColumn("remote_actor", "Remote Actor", "remote_actor"), + modalDirectColumn("remote_port_name", "Remote Port", "remote_port_name", "text"), + modalDirectColumn("type", "Type", "type", "badge"), + modalDirectColumn("state", "State", "state", "badge"), + modalDirectColumn("evidence_count", "Evidence", "evidence_count", "number"), + modalDirectColumnWithVisibility("protocol", "Protocol", "protocol", "badge", "expanded"), + modalDirectColumnWithVisibility("remote_if_index", "Remote Port ID", "remote_if_index", "number", "expanded"), + modalDirectColumnWithVisibility("port_id", "Source Port ID", "port_id", "text", "expanded"), + modalDirectColumnWithVisibility("remote_port_id", "Remote Source Port ID", "remote_port_id", "text", "expanded"), + modalDirectColumnWithVisibility("confidence", "Confidence", "confidence", "badge", "expanded"), + modalDirectColumnWithVisibility("inference", "Inference", "inference", "badge", "expanded"), + modalDirectColumnWithVisibility("attachment_mode", "Attachment", "attachment_mode", "badge", "expanded"), + modalDirectColumnWithVisibility("discovered_at", "Discovered", "discovered_at", "timestamp", "expanded"), + modalDirectColumnWithVisibility("last_seen", "Last Seen", "last_seen", "timestamp", "expanded"), + } +} + +func snmpTopologyV1ActorLabelsTableType() topologyv1.TableType { + return topologyv1.TableType{ + Role: "actor_inventory", + Owner: "actor", + Aggregation: "set", + Columns: []topologyv1.Column{ + topologyv1.NewColumn("actor", "actor_ref", topologyv1.WithRole("reference")), + topologyv1.NewColumn("key", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithRole("attribute")), + topologyv1.NewColumn("value", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithRole("attribute")), + topologyv1.NewColumn("source", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable(), topologyv1.WithRole("attribute")), + topologyv1.NewColumn("kind", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable(), topologyv1.WithRole("attribute")), + topologyv1.NewColumn("value_index", "uint", topologyv1.WithNullable(), topologyv1.WithRole("attribute")), + }, + Presentation: &topologyv1.TableTypePresentation{ + Label: "Labels", + Order: 0, + Columns: []topologyv1.ModalColumn{ + modalDirectColumn("key", "Label", "key", "text"), + modalDirectColumn("value", "Value", "value", "text"), + modalDirectColumn("source", "Source", "source", "badge"), + modalDirectColumn("kind", "Kind", "kind", "badge"), + }, + }, + } +} + +func snmpTopologyV1ActorPortsColumns() []topologyv1.Column { + return []topologyv1.Column{ + topologyv1.NewColumn("actor", "actor_ref", topologyv1.WithRole("reference")), + topologyv1.NewColumn("if_index", "uint", topologyv1.WithNullable()), + topologyv1.NewColumn("port_id", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), + topologyv1.NewColumn("name", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), + topologyv1.NewColumn("if_name", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), + topologyv1.NewColumn("if_descr", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), + topologyv1.NewColumn("if_alias", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), + topologyv1.NewColumn("mac", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), + topologyv1.NewColumn("speed", "uint", topologyv1.WithNullable()), + topologyv1.NewColumn("topology_role", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), + topologyv1.NewColumn("oper_status", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), + topologyv1.NewColumn("admin_status", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), + topologyv1.NewColumn("port_type", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), + topologyv1.NewColumn("link_mode", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), + topologyv1.NewColumn("stp_state", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), + topologyv1.NewColumn("vlan_ids", "array", topologyv1.WithNullable()), + topologyv1.NewColumn("fdb_mac_count", "uint", topologyv1.WithNullable()), + topologyv1.NewColumn("link_count", "uint", topologyv1.WithNullable()), + topologyv1.NewColumn("neighbor_count", "uint", topologyv1.WithNullable()), + topologyv1.NewColumn("neighbor_actor", "actor_ref", topologyv1.WithNullable(), topologyv1.WithRole("reference")), + topologyv1.NewColumn("neighbor_port_name", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), + topologyv1.NewColumn("neighbors", "json", topologyv1.WithNullable()), + topologyv1.NewColumn("vlans", "json", topologyv1.WithNullable()), + topologyv1.NewColumn("extra", "json", topologyv1.WithNullable()), + } +} + +func snmpTopologyV1ActorPortLinksColumns() []topologyv1.Column { + return []topologyv1.Column{ + topologyv1.NewColumn("actor", "actor_ref", topologyv1.WithRole("reference")), + topologyv1.NewColumn("link", "link_ref", topologyv1.WithRole("reference")), + topologyv1.NewColumn("remote_actor", "actor_ref", topologyv1.WithRole("reference")), + topologyv1.NewColumn("if_index", "uint", topologyv1.WithNullable()), + topologyv1.NewColumn("port_id", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), + topologyv1.NewColumn("port_name", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), + topologyv1.NewColumn("remote_if_index", "uint", topologyv1.WithNullable()), + topologyv1.NewColumn("remote_port_id", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), + topologyv1.NewColumn("remote_port_name", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), + topologyv1.NewColumn("type", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithRole("group_key")), + topologyv1.NewColumn("protocol", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithRole("group_key")), + topologyv1.NewColumn("state", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), + topologyv1.NewColumn("evidence_count", "uint", topologyv1.WithAggregation("sum")), + topologyv1.NewColumn("confidence", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), + topologyv1.NewColumn("inference", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), + topologyv1.NewColumn("attachment_mode", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), + topologyv1.NewColumn("discovered_at", "timestamp", topologyv1.WithNullable(), topologyv1.WithRole("timestamp")), + topologyv1.NewColumn("last_seen", "timestamp", topologyv1.WithNullable(), topologyv1.WithRole("timestamp")), + } +} diff --git a/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_values.go b/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_values.go new file mode 100644 index 00000000000000..b2cb600d5257fb --- /dev/null +++ b/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_values.go @@ -0,0 +1,283 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package snmptopology + +import ( + "fmt" + topologyv1 "github.com/netdata/netdata/go/plugins/pkg/topology/v1" + "maps" + "math" + "reflect" + "regexp" + "sort" + "strings" + "time" +) + +var topologyV1IDInvalidChars = regexp.MustCompile(`[^A-Za-z0-9_.:-]+`) + +func anyStringValue(value any) string { + switch typed := value.(type) { + case string: + return strings.TrimSpace(typed) + case fmt.Stringer: + return strings.TrimSpace(typed.String()) + default: + return "" + } +} + +func topologyV1ScalarLabelValue(value any) string { + switch typed := value.(type) { + case nil: + return "" + case string: + return strings.TrimSpace(typed) + case bool: + if typed { + return "true" + } + return "false" + case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, float32, float64: + return strings.TrimSpace(fmt.Sprint(typed)) + default: + return "" + } +} + +func nullableUintValue(value any) any { + out, ok := uintValue(value) + if !ok { + return nil + } + return out +} + +func uintValue(value any) (uint64, bool) { + switch typed := value.(type) { + case int: + if typed >= 0 { + return uint64(typed), true + } + case int8: + if typed >= 0 { + return uint64(typed), true + } + case int16: + if typed >= 0 { + return uint64(typed), true + } + case int32: + if typed >= 0 { + return uint64(typed), true + } + case int64: + if typed >= 0 { + return uint64(typed), true + } + case uint: + return uint64(typed), true + case uint8: + return uint64(typed), true + case uint16: + return uint64(typed), true + case uint32: + return uint64(typed), true + case uint64: + return typed, true + case float32: + if typed >= 0 && math.Trunc(float64(typed)) == float64(typed) { + return uint64(typed), true + } + case float64: + if typed >= 0 && math.Trunc(typed) == typed { + return uint64(typed), true + } + } + return 0, false +} + +func topologyV1EndpointString(endpoint topologyLinkEndpoint, key string) string { + return firstNonEmptyString( + anyStringValue(endpoint.Attributes[key]), + topologyV1MatchString(endpoint.Match, key), + ) +} + +func topologyV1EndpointPortName(endpoint topologyLinkEndpoint) string { + return firstNonEmptyString( + topologyV1EndpointString(endpoint, "port_name"), + topologyV1EndpointString(endpoint, "if_name"), + topologyV1EndpointString(endpoint, "if_descr"), + topologyV1EndpointString(endpoint, "port_id"), + ) +} + +func topologyV1MatchString(match topologyMatch, key string) string { + switch key { + case "sys_name": + return match.SysName + case "sys_object_id": + return match.SysObjectID + default: + return "" + } +} + +func firstString(values []string) string { + for _, value := range values { + if value = strings.TrimSpace(value); value != "" { + return value + } + } + return "" +} + +func nullableTime(value *time.Time) any { + if value == nil || value.IsZero() { + return nil + } + return value.UTC().Format(time.RFC3339Nano) +} + +func nullableStringRef(dict *topologyv1.StringDictionary, value string) any { + value = strings.TrimSpace(value) + if value == "" { + return nil + } + return dict.Ref(value) +} + +func nullableJSON(value any) any { + switch typed := value.(type) { + case nil: + return nil + case map[string]any: + if len(typed) == 0 { + return nil + } + case map[string]string: + if len(typed) == 0 { + return nil + } + case []any: + if len(typed) == 0 { + return nil + } + case []map[string]any: + if len(typed) == 0 { + return nil + } + } + return value +} + +func stringArrayCell(values []string) []any { + out := make([]any, 0, len(values)) + for _, value := range values { + value = strings.TrimSpace(value) + if value != "" { + out = append(out, value) + } + } + return out +} + +func isEmptyArrayCell(value any) bool { + values, ok := value.([]any) + return ok && len(values) == 0 +} + +func anyStringSlice(value any) []string { + if value == nil { + return nil + } + switch typed := value.(type) { + case []string: + return typed + case []any: + out := make([]string, 0, len(typed)) + for _, item := range typed { + if s := topologyV1ScalarLabelValue(item); s != "" { + out = append(out, s) + } + } + return out + default: + rv := reflect.ValueOf(value) + if rv.Kind() != reflect.Slice && rv.Kind() != reflect.Array { + return nil + } + out := make([]string, 0, rv.Len()) + for i := 0; i < rv.Len(); i++ { + if s := topologyV1ScalarLabelValue(rv.Index(i).Interface()); s != "" { + out = append(out, s) + } + } + return out + } +} + +func anyMapSlice(value any) ([]map[string]any, bool) { + switch typed := value.(type) { + case []map[string]any: + return typed, true + case []any: + out := make([]map[string]any, 0, len(typed)) + for _, item := range typed { + row, ok := item.(map[string]any) + if !ok { + return nil, false + } + out = append(out, row) + } + return out, true + default: + return nil, false + } +} + +func sortedMapKeys[T any](m map[string]T) []string { + keys := make([]string, 0, len(m)) + for key := range m { + keys = append(keys, key) + } + sort.Strings(keys) + return keys +} + +func topologyID(value, fallback string) string { + value = strings.TrimSpace(value) + if value == "" { + value = fallback + } + value = topologyV1IDInvalidChars.ReplaceAllString(value, "_") + value = strings.Trim(value, "_.:-") + if value == "" { + value = fallback + } + first := value[0] + if (first >= 'A' && first <= 'Z') || (first >= 'a' && first <= 'z') { + return value + } + return "x_" + value +} + +func firstNonEmptyString(values ...string) string { + for _, value := range values { + value = strings.TrimSpace(value) + if value != "" { + return value + } + } + return "" +} + +func cloneAnyMapForTopologyV1(in map[string]any) map[string]any { + if len(in) == 0 { + return nil + } + out := make(map[string]any, len(in)) + maps.Copy(out, in) + return out +} From f0374d63c42d52601c77b361a7acf87dcd6e2c9a Mon Sep 17 00:00:00 2001 From: Ilya Mashchenko Date: Fri, 19 Jun 2026 00:22:06 +0300 Subject: [PATCH 4/6] chore(snmp-topology): rename graph shaping layer (#22776) --- .../snmp_topology/func_topology_test.go | 1 - .../topology_output_collapse_priority.go | 33 ----- .../topology_output_merge_test.go | 77 ---------- .../snmp_topology/topology_registry_build.go | 6 +- .../snmp_topology/topology_registry_test.go | 16 +-- ...t_cleanup.go => topology_shape_cleanup.go} | 0 ...test.go => topology_shape_cleanup_test.go} | 20 +++ ...collapse.go => topology_shape_collapse.go} | 26 ++++ ...utput_delta.go => topology_shape_delta.go} | 0 ...utput_focus.go => topology_shape_focus.go} | 0 ...lter.go => topology_shape_focus_filter.go} | 0 ...graph.go => topology_shape_focus_graph.go} | 0 ....go => topology_shape_focus_graph_test.go} | 0 ...paths.go => topology_shape_focus_paths.go} | 0 ...n.go => topology_shape_focus_selection.go} | 40 ------ .../topology_shape_focus_stats.go | 35 +++++ ...map_type.go => topology_shape_map_type.go} | 12 -- ...utput_merge.go => topology_shape_merge.go} | 0 .../topology_shape_merge_test.go | 133 ++++++++++++++++++ ...policies.go => topology_shape_policies.go} | 2 +- ...utput_stats.go => topology_shape_stats.go} | 0 .../topology_shape_stats_test.go | 53 +++++++ ...ts_helpers.go => topology_shape_values.go} | 12 ++ ..._test.go => topology_shape_values_test.go} | 2 +- 24 files changed, 292 insertions(+), 176 deletions(-) delete mode 100644 src/go/plugin/go.d/collector/snmp_topology/topology_output_collapse_priority.go delete mode 100644 src/go/plugin/go.d/collector/snmp_topology/topology_output_merge_test.go rename src/go/plugin/go.d/collector/snmp_topology/{topology_output_cleanup.go => topology_shape_cleanup.go} (100%) rename src/go/plugin/go.d/collector/snmp_topology/{topology_output_cleanup_test.go => topology_shape_cleanup_test.go} (50%) rename src/go/plugin/go.d/collector/snmp_topology/{topology_output_collapse.go => topology_shape_collapse.go} (80%) rename src/go/plugin/go.d/collector/snmp_topology/{topology_output_delta.go => topology_shape_delta.go} (100%) rename src/go/plugin/go.d/collector/snmp_topology/{topology_output_focus.go => topology_shape_focus.go} (100%) rename src/go/plugin/go.d/collector/snmp_topology/{topology_output_focus_filter.go => topology_shape_focus_filter.go} (100%) rename src/go/plugin/go.d/collector/snmp_topology/{topology_output_focus_graph.go => topology_shape_focus_graph.go} (100%) rename src/go/plugin/go.d/collector/snmp_topology/{topology_output_focus_graph_test.go => topology_shape_focus_graph_test.go} (100%) rename src/go/plugin/go.d/collector/snmp_topology/{topology_output_focus_paths.go => topology_shape_focus_paths.go} (100%) rename src/go/plugin/go.d/collector/snmp_topology/{topology_output_focus_selection.go => topology_shape_focus_selection.go} (61%) create mode 100644 src/go/plugin/go.d/collector/snmp_topology/topology_shape_focus_stats.go rename src/go/plugin/go.d/collector/snmp_topology/{topology_output_map_type.go => topology_shape_map_type.go} (89%) rename src/go/plugin/go.d/collector/snmp_topology/{topology_output_merge.go => topology_shape_merge.go} (100%) create mode 100644 src/go/plugin/go.d/collector/snmp_topology/topology_shape_merge_test.go rename src/go/plugin/go.d/collector/snmp_topology/{topology_output_policies.go => topology_shape_policies.go} (94%) rename src/go/plugin/go.d/collector/snmp_topology/{topology_output_stats.go => topology_shape_stats.go} (100%) create mode 100644 src/go/plugin/go.d/collector/snmp_topology/topology_shape_stats_test.go rename src/go/plugin/go.d/collector/snmp_topology/{topology_output_stats_helpers.go => topology_shape_values.go} (82%) rename src/go/plugin/go.d/collector/snmp_topology/{topology_output_stats_test.go => topology_shape_values_test.go} (91%) diff --git a/src/go/plugin/go.d/collector/snmp_topology/func_topology_test.go b/src/go/plugin/go.d/collector/snmp_topology/func_topology_test.go index 870e6bfce914ec..998b60a069bd63 100644 --- a/src/go/plugin/go.d/collector/snmp_topology/func_topology_test.go +++ b/src/go/plugin/go.d/collector/snmp_topology/func_topology_test.go @@ -594,7 +594,6 @@ func TestNormalizeTopologyManagedFocuses(t *testing.T) { formatTopologyManagedFocuses([]string{"ip:10.0.0.2", "ip:10.0.0.1"}), ) assert.Equal(t, []string{topologyManagedFocusAllDevices}, parseTopologyManagedFocuses("")) - assert.Equal(t, "10.0.0.1", topologyManagedFocusSelectedIP("ip:10.0.0.2,ip:10.0.0.1")) assert.Equal(t, []string{"10.0.0.1", "10.0.0.2"}, topologyManagedFocusSelectedIPs("ip:10.0.0.2,ip:10.0.0.1")) assert.True(t, isTopologyManagedFocusAllDevices(topologyManagedFocusAllDevices)) assert.False(t, isTopologyManagedFocusAllDevices("ip:10.0.0.1")) diff --git a/src/go/plugin/go.d/collector/snmp_topology/topology_output_collapse_priority.go b/src/go/plugin/go.d/collector/snmp_topology/topology_output_collapse_priority.go deleted file mode 100644 index 121327cbccfba8..00000000000000 --- a/src/go/plugin/go.d/collector/snmp_topology/topology_output_collapse_priority.go +++ /dev/null @@ -1,33 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -package snmptopology - -import ( - "strings" - - topologyengine "github.com/netdata/netdata/go/plugins/pkg/l2topology" -) - -func compareCollapseActorPriority(left, right topologyActor) int { - if leftDevice, rightDevice := topologyengine.IsDeviceActorType(left.ActorType), topologyengine.IsDeviceActorType(right.ActorType); leftDevice != rightDevice { - if leftDevice { - return -1 - } - return 1 - } - if leftInferred, rightInferred := topologyActorIsInferred(left), topologyActorIsInferred(right); leftInferred != rightInferred { - if !leftInferred { - return -1 - } - return 1 - } - leftID := strings.ToLower(strings.TrimSpace(left.ActorID)) - rightID := strings.ToLower(strings.TrimSpace(right.ActorID)) - if (leftID == "") != (rightID == "") { - if leftID != "" { - return -1 - } - return 1 - } - return strings.Compare(leftID, rightID) -} diff --git a/src/go/plugin/go.d/collector/snmp_topology/topology_output_merge_test.go b/src/go/plugin/go.d/collector/snmp_topology/topology_output_merge_test.go deleted file mode 100644 index 1f8a27b303fb4e..00000000000000 --- a/src/go/plugin/go.d/collector/snmp_topology/topology_output_merge_test.go +++ /dev/null @@ -1,77 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later - -package snmptopology - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestAppendUniqueTopologyStringsSortsAndDeduplicates(t *testing.T) { - values := appendUniqueTopologyStrings([]string{" b ", "a"}, "a", "", " c ") - require.Equal(t, []string{"a", "b", "c"}, values) -} - -func TestMergeTopologyStringMapKeepsExistingAndIgnoresBlankEntries(t *testing.T) { - base := map[string]string{ - "existing": "keep", - } - - merged := mergeTopologyStringMap(base, map[string]string{ - "existing": "replace", - " new ": " value ", - "blank": " ", - "": "ignored", - }) - - require.Equal(t, map[string]string{ - "existing": "keep", - "new": "value", - }, merged) -} - -func TestMergeTopologyAnyMapKeepsExistingAndAddsMissingKeys(t *testing.T) { - base := map[string]any{ - "existing": "keep", - } - - merged := mergeTopologyAnyMap(base, map[string]any{ - "existing": "replace", - " new ": 42, - "": "ignored", - }) - - require.Equal(t, "keep", merged["existing"]) - require.Equal(t, 42, merged["new"]) - require.NotContains(t, merged, "") -} - -func TestTopologyLinkActorKeyIncludesStateAndAttachmentMode(t *testing.T) { - base := topologyLink{ - Protocol: "bridge", - Direction: "bidirectional", - SrcActorID: "device:a", - DstActorID: "endpoint:b", - Src: topologyLinkEndpoint{Attributes: map[string]any{ - "if_name": "Gi0/1", - }}, - Dst: topologyLinkEndpoint{Attributes: map[string]any{ - "if_name": "Gi0/2", - }}, - Metrics: map[string]any{ - "bridge_domain": "vlan-200", - "attachment_mode": "probable_bridge_anchor", - "inference": "probable", - }, - State: "probable", - } - - strict := base - strict.State = "" - strict.Metrics = map[string]any{ - "bridge_domain": "vlan-200", - } - - require.NotEqual(t, topologyLinkActorKey(base), topologyLinkActorKey(strict)) -} diff --git a/src/go/plugin/go.d/collector/snmp_topology/topology_registry_build.go b/src/go/plugin/go.d/collector/snmp_topology/topology_registry_build.go index faf93041da4c38..19427932f0d233 100644 --- a/src/go/plugin/go.d/collector/snmp_topology/topology_registry_build.go +++ b/src/go/plugin/go.d/collector/snmp_topology/topology_registry_build.go @@ -32,7 +32,7 @@ func buildSingleMapTopologySnapshot(aggregate topologyObservationAggregate, opti return topologyData{}, false } augmentTopologySnapshotLocals(&data, aggregate.snapshots) - applySNMPTopologyOutputPolicies(&data, options) + applySNMPTopologyShapePolicies(&data, options) applyTopologyDepthFocusFilter(&data, options) return data, true } @@ -51,7 +51,7 @@ func buildProbableTopologySnapshot(aggregate topologyObservationAggregate, optio return topologyData{}, false } augmentTopologySnapshotLocals(&strictData, aggregate.snapshots) - applySNMPTopologyOutputPolicies(&strictData, strictOptions) + applySNMPTopologyShapePolicies(&strictData, strictOptions) probableOptions := options probableOptions.MapType = topologyMapTypeAllDevicesLowConfidence @@ -66,7 +66,7 @@ func buildProbableTopologySnapshot(aggregate topologyObservationAggregate, optio return topologyData{}, false } augmentTopologySnapshotLocals(&probableData, aggregate.snapshots) - applySNMPTopologyOutputPolicies(&probableData, probableOptions) + applySNMPTopologyShapePolicies(&probableData, probableOptions) markProbableDeltaLinks(&strictData, &probableData) applyTopologyDepthFocusFilter(&probableData, options) return probableData, true diff --git a/src/go/plugin/go.d/collector/snmp_topology/topology_registry_test.go b/src/go/plugin/go.d/collector/snmp_topology/topology_registry_test.go index 727d98317be21b..c3a598f836fb25 100644 --- a/src/go/plugin/go.d/collector/snmp_topology/topology_registry_test.go +++ b/src/go/plugin/go.d/collector/snmp_topology/topology_registry_test.go @@ -429,7 +429,7 @@ func TestCanonicalMatchKey_NormalizesEquivalentMACRepresentations(t *testing.T) require.Contains(t, topologyMatchIdentityKeys(colon), "hw:70:49:a2:65:72:cd") } -func TestApplySNMPTopologyOutputPolicies_CollapsesActorsByIP(t *testing.T) { +func TestApplySNMPTopologyShapePolicies_CollapsesActorsByIP(t *testing.T) { data := topologyData{ Actors: []topologyActor{ { @@ -461,7 +461,7 @@ func TestApplySNMPTopologyOutputPolicies_CollapsesActorsByIP(t *testing.T) { Stats: map[string]any{}, } - applySNMPTopologyOutputPolicies(&data, topologyQueryOptions{ + applySNMPTopologyShapePolicies(&data, topologyQueryOptions{ CollapseActorsByIP: true, MapType: topologyMapTypeHighConfidenceInferred, }) @@ -471,7 +471,7 @@ func TestApplySNMPTopologyOutputPolicies_CollapsesActorsByIP(t *testing.T) { require.Equal(t, 1, data.Stats["actors_collapsed_by_ip"]) } -func TestApplySNMPTopologyOutputPolicies_EliminatesNonIPInferredActorsAndSparseSegments(t *testing.T) { +func TestApplySNMPTopologyShapePolicies_EliminatesNonIPInferredActorsAndSparseSegments(t *testing.T) { data := topologyData{ Actors: []topologyActor{ { @@ -500,7 +500,7 @@ func TestApplySNMPTopologyOutputPolicies_EliminatesNonIPInferredActorsAndSparseS Stats: map[string]any{}, } - applySNMPTopologyOutputPolicies(&data, topologyQueryOptions{ + applySNMPTopologyShapePolicies(&data, topologyQueryOptions{ EliminateNonIPInferred: true, MapType: topologyMapTypeHighConfidenceInferred, }) @@ -511,7 +511,7 @@ func TestApplySNMPTopologyOutputPolicies_EliminatesNonIPInferredActorsAndSparseS require.Equal(t, 1, data.Stats["segments_sparse_suppressed"]) } -func TestApplySNMPTopologyOutputPolicies_HighConfidenceSuppressesUnlinkedInferredEndpoints(t *testing.T) { +func TestApplySNMPTopologyShapePolicies_HighConfidenceSuppressesUnlinkedInferredEndpoints(t *testing.T) { data := topologyData{ Actors: []topologyActor{ { @@ -544,7 +544,7 @@ func TestApplySNMPTopologyOutputPolicies_HighConfidenceSuppressesUnlinkedInferre Stats: map[string]any{}, } - applySNMPTopologyOutputPolicies(&data, topologyQueryOptions{ + applySNMPTopologyShapePolicies(&data, topologyQueryOptions{ MapType: topologyMapTypeHighConfidenceInferred, }) @@ -555,7 +555,7 @@ func TestApplySNMPTopologyOutputPolicies_HighConfidenceSuppressesUnlinkedInferre } } -func TestApplySNMPTopologyOutputPolicies_LLDPManagedMapKeepsOnlyLLDPCDPAndManagedDevices(t *testing.T) { +func TestApplySNMPTopologyShapePolicies_LLDPManagedMapKeepsOnlyLLDPCDPAndManagedDevices(t *testing.T) { data := topologyData{ Actors: []topologyActor{ { @@ -594,7 +594,7 @@ func TestApplySNMPTopologyOutputPolicies_LLDPManagedMapKeepsOnlyLLDPCDPAndManage Stats: map[string]any{}, } - applySNMPTopologyOutputPolicies(&data, topologyQueryOptions{ + applySNMPTopologyShapePolicies(&data, topologyQueryOptions{ MapType: topologyMapTypeLLDPCDPManaged, }) diff --git a/src/go/plugin/go.d/collector/snmp_topology/topology_output_cleanup.go b/src/go/plugin/go.d/collector/snmp_topology/topology_shape_cleanup.go similarity index 100% rename from src/go/plugin/go.d/collector/snmp_topology/topology_output_cleanup.go rename to src/go/plugin/go.d/collector/snmp_topology/topology_shape_cleanup.go diff --git a/src/go/plugin/go.d/collector/snmp_topology/topology_output_cleanup_test.go b/src/go/plugin/go.d/collector/snmp_topology/topology_shape_cleanup_test.go similarity index 50% rename from src/go/plugin/go.d/collector/snmp_topology/topology_output_cleanup_test.go rename to src/go/plugin/go.d/collector/snmp_topology/topology_shape_cleanup_test.go index bcabbad6169724..b13a61cff3d401 100644 --- a/src/go/plugin/go.d/collector/snmp_topology/topology_output_cleanup_test.go +++ b/src/go/plugin/go.d/collector/snmp_topology/topology_shape_cleanup_test.go @@ -26,3 +26,23 @@ func TestFilterDanglingLinks_TrimActorIDsBeforeLookup(t *testing.T) { require.Equal(t, " device-a ", data.Links[0].SrcActorID) require.Equal(t, "\tdevice-b\n", data.Links[0].DstActorID) } + +func TestPruneSparseSegmentsRemovesMultiRoundFixpoint(t *testing.T) { + data := &topologyData{ + Actors: []topologyActor{ + {ActorID: "device-a", ActorType: "device"}, + {ActorID: "segment-a", ActorType: "segment"}, + {ActorID: "segment-b", ActorType: "segment"}, + }, + Links: []topologyLink{ + {SrcActorID: "device-a", DstActorID: "segment-a"}, + {SrcActorID: "segment-a", DstActorID: "segment-b"}, + }, + } + + removed := pruneSparseSegments(data, 1) + + require.Equal(t, 2, removed) + require.Equal(t, []topologyActor{{ActorID: "device-a", ActorType: "device"}}, data.Actors) + require.Empty(t, data.Links) +} diff --git a/src/go/plugin/go.d/collector/snmp_topology/topology_output_collapse.go b/src/go/plugin/go.d/collector/snmp_topology/topology_shape_collapse.go similarity index 80% rename from src/go/plugin/go.d/collector/snmp_topology/topology_output_collapse.go rename to src/go/plugin/go.d/collector/snmp_topology/topology_shape_collapse.go index 627d862c5437f9..2adf013d792e9e 100644 --- a/src/go/plugin/go.d/collector/snmp_topology/topology_output_collapse.go +++ b/src/go/plugin/go.d/collector/snmp_topology/topology_shape_collapse.go @@ -4,6 +4,8 @@ package snmptopology import ( "strings" + + topologyengine "github.com/netdata/netdata/go/plugins/pkg/l2topology" ) func collapseActorsByIP(data *topologyData) int { @@ -143,3 +145,27 @@ func collapseActorsByIP(data *topologyData) int { data.Links = links return collapsed } + +func compareCollapseActorPriority(left, right topologyActor) int { + if leftDevice, rightDevice := topologyengine.IsDeviceActorType(left.ActorType), topologyengine.IsDeviceActorType(right.ActorType); leftDevice != rightDevice { + if leftDevice { + return -1 + } + return 1 + } + if leftInferred, rightInferred := topologyActorIsInferred(left), topologyActorIsInferred(right); leftInferred != rightInferred { + if !leftInferred { + return -1 + } + return 1 + } + leftID := strings.ToLower(strings.TrimSpace(left.ActorID)) + rightID := strings.ToLower(strings.TrimSpace(right.ActorID)) + if (leftID == "") != (rightID == "") { + if leftID != "" { + return -1 + } + return 1 + } + return strings.Compare(leftID, rightID) +} diff --git a/src/go/plugin/go.d/collector/snmp_topology/topology_output_delta.go b/src/go/plugin/go.d/collector/snmp_topology/topology_shape_delta.go similarity index 100% rename from src/go/plugin/go.d/collector/snmp_topology/topology_output_delta.go rename to src/go/plugin/go.d/collector/snmp_topology/topology_shape_delta.go diff --git a/src/go/plugin/go.d/collector/snmp_topology/topology_output_focus.go b/src/go/plugin/go.d/collector/snmp_topology/topology_shape_focus.go similarity index 100% rename from src/go/plugin/go.d/collector/snmp_topology/topology_output_focus.go rename to src/go/plugin/go.d/collector/snmp_topology/topology_shape_focus.go diff --git a/src/go/plugin/go.d/collector/snmp_topology/topology_output_focus_filter.go b/src/go/plugin/go.d/collector/snmp_topology/topology_shape_focus_filter.go similarity index 100% rename from src/go/plugin/go.d/collector/snmp_topology/topology_output_focus_filter.go rename to src/go/plugin/go.d/collector/snmp_topology/topology_shape_focus_filter.go diff --git a/src/go/plugin/go.d/collector/snmp_topology/topology_output_focus_graph.go b/src/go/plugin/go.d/collector/snmp_topology/topology_shape_focus_graph.go similarity index 100% rename from src/go/plugin/go.d/collector/snmp_topology/topology_output_focus_graph.go rename to src/go/plugin/go.d/collector/snmp_topology/topology_shape_focus_graph.go diff --git a/src/go/plugin/go.d/collector/snmp_topology/topology_output_focus_graph_test.go b/src/go/plugin/go.d/collector/snmp_topology/topology_shape_focus_graph_test.go similarity index 100% rename from src/go/plugin/go.d/collector/snmp_topology/topology_output_focus_graph_test.go rename to src/go/plugin/go.d/collector/snmp_topology/topology_shape_focus_graph_test.go diff --git a/src/go/plugin/go.d/collector/snmp_topology/topology_output_focus_paths.go b/src/go/plugin/go.d/collector/snmp_topology/topology_shape_focus_paths.go similarity index 100% rename from src/go/plugin/go.d/collector/snmp_topology/topology_output_focus_paths.go rename to src/go/plugin/go.d/collector/snmp_topology/topology_shape_focus_paths.go diff --git a/src/go/plugin/go.d/collector/snmp_topology/topology_output_focus_selection.go b/src/go/plugin/go.d/collector/snmp_topology/topology_shape_focus_selection.go similarity index 61% rename from src/go/plugin/go.d/collector/snmp_topology/topology_output_focus_selection.go rename to src/go/plugin/go.d/collector/snmp_topology/topology_shape_focus_selection.go index 3dda1b0992decc..bb821fd5c5127f 100644 --- a/src/go/plugin/go.d/collector/snmp_topology/topology_output_focus_selection.go +++ b/src/go/plugin/go.d/collector/snmp_topology/topology_shape_focus_selection.go @@ -8,46 +8,6 @@ import ( "strings" ) -func recordTopologyFocusAllDevicesStats(data *topologyData, options topologyQueryOptions) { - if data == nil { - return - } - if data.Stats == nil { - data.Stats = make(map[string]any) - } - data.Stats["managed_snmp_device_focus"] = options.ManagedDeviceFocus - data.Stats["depth"] = topologyDepthAll - data.Stats["actors_focus_depth_filtered"] = 0 - data.Stats["links_focus_depth_filtered"] = 0 - recomputeTopologyLinkStats(data) -} - -func recordTopologyFocusStats(data *topologyData, options topologyQueryOptions, beforeActors, beforeLinks int) { - if data == nil { - return - } - if data.Stats == nil { - data.Stats = make(map[string]any) - } - data.Stats["managed_snmp_device_focus"] = options.ManagedDeviceFocus - if options.Depth == topologyDepthAllInternal { - data.Stats["depth"] = topologyDepthAll - } else { - data.Stats["depth"] = options.Depth - } - data.Stats["actors_focus_depth_filtered"] = beforeActors - len(data.Actors) - data.Stats["links_focus_depth_filtered"] = beforeLinks - len(data.Links) - recomputeTopologyLinkStats(data) -} - -func topologyManagedFocusSelectedIP(value string) string { - ips := topologyManagedFocusSelectedIPs(value) - if len(ips) == 0 { - return "" - } - return ips[0] -} - func topologyManagedFocusSelectedIPs(value string) []string { normalized := parseTopologyManagedFocuses(value) if len(normalized) == 1 && normalized[0] == topologyManagedFocusAllDevices { diff --git a/src/go/plugin/go.d/collector/snmp_topology/topology_shape_focus_stats.go b/src/go/plugin/go.d/collector/snmp_topology/topology_shape_focus_stats.go new file mode 100644 index 00000000000000..70a0fa81788cf9 --- /dev/null +++ b/src/go/plugin/go.d/collector/snmp_topology/topology_shape_focus_stats.go @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package snmptopology + +func recordTopologyFocusAllDevicesStats(data *topologyData, options topologyQueryOptions) { + if data == nil { + return + } + if data.Stats == nil { + data.Stats = make(map[string]any) + } + data.Stats["managed_snmp_device_focus"] = options.ManagedDeviceFocus + data.Stats["depth"] = topologyDepthAll + data.Stats["actors_focus_depth_filtered"] = 0 + data.Stats["links_focus_depth_filtered"] = 0 + recomputeTopologyLinkStats(data) +} + +func recordTopologyFocusStats(data *topologyData, options topologyQueryOptions, beforeActors, beforeLinks int) { + if data == nil { + return + } + if data.Stats == nil { + data.Stats = make(map[string]any) + } + data.Stats["managed_snmp_device_focus"] = options.ManagedDeviceFocus + if options.Depth == topologyDepthAllInternal { + data.Stats["depth"] = topologyDepthAll + } else { + data.Stats["depth"] = options.Depth + } + data.Stats["actors_focus_depth_filtered"] = beforeActors - len(data.Actors) + data.Stats["links_focus_depth_filtered"] = beforeLinks - len(data.Links) + recomputeTopologyLinkStats(data) +} diff --git a/src/go/plugin/go.d/collector/snmp_topology/topology_output_map_type.go b/src/go/plugin/go.d/collector/snmp_topology/topology_shape_map_type.go similarity index 89% rename from src/go/plugin/go.d/collector/snmp_topology/topology_output_map_type.go rename to src/go/plugin/go.d/collector/snmp_topology/topology_shape_map_type.go index da42fec80b8321..bf4b709a6a685c 100644 --- a/src/go/plugin/go.d/collector/snmp_topology/topology_output_map_type.go +++ b/src/go/plugin/go.d/collector/snmp_topology/topology_shape_map_type.go @@ -4,8 +4,6 @@ package snmptopology import ( "strings" - - topologyengine "github.com/netdata/netdata/go/plugins/pkg/l2topology" ) func applyMapTypePolicy(data *topologyData, mapType string) int { @@ -119,13 +117,3 @@ func suppressUnlinkedInferredEndpoints(data *topologyData) int { data.Links = filtered return removed } - -func isManagedSNMPDeviceActor(actor topologyActor) bool { - if !topologyengine.IsDeviceActorType(actor.ActorType) { - return false - } - if strings.ToLower(strings.TrimSpace(actor.Source)) != "snmp" { - return false - } - return !topologyActorIsInferred(actor) -} diff --git a/src/go/plugin/go.d/collector/snmp_topology/topology_output_merge.go b/src/go/plugin/go.d/collector/snmp_topology/topology_shape_merge.go similarity index 100% rename from src/go/plugin/go.d/collector/snmp_topology/topology_output_merge.go rename to src/go/plugin/go.d/collector/snmp_topology/topology_shape_merge.go diff --git a/src/go/plugin/go.d/collector/snmp_topology/topology_shape_merge_test.go b/src/go/plugin/go.d/collector/snmp_topology/topology_shape_merge_test.go new file mode 100644 index 00000000000000..12aa13adf269cf --- /dev/null +++ b/src/go/plugin/go.d/collector/snmp_topology/topology_shape_merge_test.go @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package snmptopology + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestAppendUniqueTopologyStringsSortsAndDeduplicates(t *testing.T) { + values := appendUniqueTopologyStrings([]string{" b ", "a"}, "a", "", " c ") + require.Equal(t, []string{"a", "b", "c"}, values) +} + +func TestMergeTopologyMatchUnionsIdentityAndFillsMissingSystemNames(t *testing.T) { + merged := mergeTopologyMatch( + topologyMatch{ + ChassisIDs: []string{" chassis-b ", "chassis-a"}, + MacAddresses: []string{"aa:aa:aa:aa:aa:aa"}, + IPAddresses: []string{"10.0.0.2"}, + Hostnames: []string{"host-b"}, + DNSNames: []string{"dns-b"}, + SysObjectID: "1.3.6.1.4.1.1", + }, + topologyMatch{ + ChassisIDs: []string{"chassis-c", "chassis-a"}, + MacAddresses: []string{"bb:bb:bb:bb:bb:bb", "aa:aa:aa:aa:aa:aa"}, + IPAddresses: []string{"10.0.0.1", "10.0.0.2"}, + Hostnames: []string{"host-a"}, + DNSNames: []string{"dns-a"}, + SysName: " sw-a ", + SysObjectID: "1.3.6.1.4.1.2", + }, + ) + + require.Equal(t, []string{"chassis-a", "chassis-b", "chassis-c"}, merged.ChassisIDs) + require.Equal(t, []string{"aa:aa:aa:aa:aa:aa", "bb:bb:bb:bb:bb:bb"}, merged.MacAddresses) + require.Equal(t, []string{"10.0.0.1", "10.0.0.2"}, merged.IPAddresses) + require.Equal(t, []string{"host-a", "host-b"}, merged.Hostnames) + require.Equal(t, []string{"dns-a", "dns-b"}, merged.DNSNames) + require.Equal(t, "sw-a", merged.SysName) + require.Equal(t, "1.3.6.1.4.1.1", merged.SysObjectID) +} + +func TestMergeTopologyStringMapKeepsExistingAndIgnoresBlankEntries(t *testing.T) { + base := map[string]string{ + "existing": "keep", + } + + merged := mergeTopologyStringMap(base, map[string]string{ + "existing": "replace", + " new ": " value ", + "blank": " ", + "": "ignored", + }) + + require.Equal(t, map[string]string{ + "existing": "keep", + "new": "value", + }, merged) +} + +func TestMergeTopologyAnyMapKeepsExistingAndAddsMissingKeys(t *testing.T) { + base := map[string]any{ + "existing": "keep", + } + + merged := mergeTopologyAnyMap(base, map[string]any{ + "existing": "replace", + " new ": 42, + "": "ignored", + }) + + require.Equal(t, "keep", merged["existing"]) + require.Equal(t, 42, merged["new"]) + require.NotContains(t, merged, "") +} + +func TestTopologyLinkDeltaKeyUsesStableEndpointAndBridgeFields(t *testing.T) { + link := topologyLink{ + Protocol: " LLDP ", + Direction: " Bidirectional ", + SrcActorID: " device:a ", + DstActorID: " device:b ", + Src: topologyLinkEndpoint{Attributes: map[string]any{ + "if_index": 7, + "if_name": "Gi0/1", + "port_id": "port-a", + }}, + Dst: topologyLinkEndpoint{Attributes: map[string]any{ + "if_index": 8, + "if_name": "Gi0/2", + "port_id": "port-b", + }}, + Metrics: map[string]any{ + "bridge_domain": "vlan-10", + }, + State: "probable", + } + + require.Equal(t, "lldp|bidirectional|device:a|device:b|7|Gi0/1|port-a|8|Gi0/2|port-b|vlan-10", topologyLinkDeltaKey(link)) +} + +func TestTopologyLinkActorKeyIncludesStateAndAttachmentMode(t *testing.T) { + base := topologyLink{ + Protocol: "bridge", + Direction: "bidirectional", + SrcActorID: "device:a", + DstActorID: "endpoint:b", + Src: topologyLinkEndpoint{Attributes: map[string]any{ + "if_name": "Gi0/1", + }}, + Dst: topologyLinkEndpoint{Attributes: map[string]any{ + "if_name": "Gi0/2", + }}, + Metrics: map[string]any{ + "bridge_domain": "vlan-200", + "attachment_mode": "probable_bridge_anchor", + "inference": "probable", + }, + State: "probable", + } + + strict := base + strict.State = "" + strict.Metrics = map[string]any{ + "bridge_domain": "vlan-200", + } + + require.NotEqual(t, topologyLinkActorKey(base), topologyLinkActorKey(strict)) + require.NotEqual(t, topologyLinkDeltaKey(base), topologyLinkActorKey(base)) +} diff --git a/src/go/plugin/go.d/collector/snmp_topology/topology_output_policies.go b/src/go/plugin/go.d/collector/snmp_topology/topology_shape_policies.go similarity index 94% rename from src/go/plugin/go.d/collector/snmp_topology/topology_output_policies.go rename to src/go/plugin/go.d/collector/snmp_topology/topology_shape_policies.go index 13206ba48a4c34..9cc6f0b3898604 100644 --- a/src/go/plugin/go.d/collector/snmp_topology/topology_output_policies.go +++ b/src/go/plugin/go.d/collector/snmp_topology/topology_shape_policies.go @@ -4,7 +4,7 @@ package snmptopology import "sort" -func applySNMPTopologyOutputPolicies(data *topologyData, options topologyQueryOptions) { +func applySNMPTopologyShapePolicies(data *topologyData, options topologyQueryOptions) { if data == nil { return } diff --git a/src/go/plugin/go.d/collector/snmp_topology/topology_output_stats.go b/src/go/plugin/go.d/collector/snmp_topology/topology_shape_stats.go similarity index 100% rename from src/go/plugin/go.d/collector/snmp_topology/topology_output_stats.go rename to src/go/plugin/go.d/collector/snmp_topology/topology_shape_stats.go diff --git a/src/go/plugin/go.d/collector/snmp_topology/topology_shape_stats_test.go b/src/go/plugin/go.d/collector/snmp_topology/topology_shape_stats_test.go new file mode 100644 index 00000000000000..11956e6af53063 --- /dev/null +++ b/src/go/plugin/go.d/collector/snmp_topology/topology_shape_stats_test.go @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package snmptopology + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestApplySNMPTopologyShapePolicies_EmitsStatsContractKeys(t *testing.T) { + data := topologyData{ + Actors: []topologyActor{ + {ActorID: "device-a", ActorType: "device", Match: topologyMatch{IPAddresses: []string{"10.0.0.1"}}}, + {ActorID: "segment-a", ActorType: "segment"}, + {ActorID: "endpoint-a", ActorType: "endpoint"}, + }, + Links: []topologyLink{ + {SrcActorID: "device-a", DstActorID: "segment-a"}, + {SrcActorID: "segment-a", DstActorID: "endpoint-a"}, + }, + } + + applySNMPTopologyShapePolicies(&data, topologyQueryOptions{ + EliminateNonIPInferred: true, + InferenceStrategy: topologyInferenceStrategySTPParentTree, + MapType: topologyMapTypeHighConfidenceInferred, + }) + + keys := make([]string, 0, len(data.Stats)) + for key := range data.Stats { + keys = append(keys, key) + } + require.ElementsMatch(t, []string{ + "actors_collapsed_by_ip", + "actors_map_type_suppressed", + "actors_non_ip_inferred_suppressed", + "actors_total", + "inference_strategy", + "links_probable", + "links_total", + "map_type", + "segments_sparse_suppressed", + "segments_suppressed", + }, keys) + require.Equal(t, 1, data.Stats["actors_total"]) + require.Equal(t, 0, data.Stats["links_total"]) + require.Equal(t, 1, data.Stats["actors_non_ip_inferred_suppressed"]) + require.Equal(t, 1, data.Stats["segments_sparse_suppressed"]) + require.Equal(t, 1, data.Stats["segments_suppressed"]) + require.Equal(t, topologyMapTypeHighConfidenceInferred, data.Stats["map_type"]) + require.Equal(t, topologyInferenceStrategySTPParentTree, data.Stats["inference_strategy"]) +} diff --git a/src/go/plugin/go.d/collector/snmp_topology/topology_output_stats_helpers.go b/src/go/plugin/go.d/collector/snmp_topology/topology_shape_values.go similarity index 82% rename from src/go/plugin/go.d/collector/snmp_topology/topology_output_stats_helpers.go rename to src/go/plugin/go.d/collector/snmp_topology/topology_shape_values.go index b5a21f704f786a..70ee5cc2773b97 100644 --- a/src/go/plugin/go.d/collector/snmp_topology/topology_output_stats_helpers.go +++ b/src/go/plugin/go.d/collector/snmp_topology/topology_shape_values.go @@ -7,6 +7,8 @@ import ( "sort" "strconv" "strings" + + topologyengine "github.com/netdata/netdata/go/plugins/pkg/l2topology" ) func normalizedMatchIPs(match topologyMatch) []string { @@ -43,6 +45,16 @@ func topologyActorIsInferred(actor topologyActor) bool { return false } +func isManagedSNMPDeviceActor(actor topologyActor) bool { + if !topologyengine.IsDeviceActorType(actor.ActorType) { + return false + } + if strings.ToLower(strings.TrimSpace(actor.Source)) != "snmp" { + return false + } + return !topologyActorIsInferred(actor) +} + func boolStatValue(value any) bool { switch typed := value.(type) { case bool: diff --git a/src/go/plugin/go.d/collector/snmp_topology/topology_output_stats_test.go b/src/go/plugin/go.d/collector/snmp_topology/topology_shape_values_test.go similarity index 91% rename from src/go/plugin/go.d/collector/snmp_topology/topology_output_stats_test.go rename to src/go/plugin/go.d/collector/snmp_topology/topology_shape_values_test.go index a707a609829d5a..f36d8dd0e44ab9 100644 --- a/src/go/plugin/go.d/collector/snmp_topology/topology_output_stats_test.go +++ b/src/go/plugin/go.d/collector/snmp_topology/topology_shape_values_test.go @@ -8,7 +8,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestTopologyOutputStatHelpers_ClassifyActorsAndValues(t *testing.T) { +func TestTopologyShapeValues_ClassifyActorsAndValues(t *testing.T) { require.True(t, topologyActorIsInferred(topologyActor{ActorType: "endpoint"})) require.True(t, topologyActorIsInferred(topologyActor{Labels: map[string]string{"inferred": "yes"}})) require.True(t, topologyActorIsInferred(topologyActor{Attributes: map[string]any{"inferred": true}})) From 65c4ea06c390ac3539818861668b7d12306a0769 Mon Sep 17 00:00:00 2001 From: Ilya Mashchenko Date: Fri, 19 Jun 2026 01:19:01 +0300 Subject: [PATCH 5/6] chore(go.d/snmp_topology): make collection cache explicit (#22777) --- .../go.d/collector/snmp_topology/collector.go | 25 +-- .../snmp_topology/collector_refresh_test.go | 59 ++++++- .../snmp_topology/topology_cache_ingest.go | 50 ++++-- .../snmp_topology/topology_cache_lifecycle.go | 43 +++-- .../snmp_topology/topology_cache_test.go | 166 +++++------------- .../topology_snmprec_forwarding_test.go | 34 ++-- .../snmp_topology/topology_snmprec_test.go | 18 +- .../snmp_topology/topology_vlan_context.go | 8 +- .../topology_vlan_context_ingest.go | 6 +- 9 files changed, 197 insertions(+), 212 deletions(-) diff --git a/src/go/plugin/go.d/collector/snmp_topology/collector.go b/src/go/plugin/go.d/collector/snmp_topology/collector.go index 0c7aeb35ec16cb..a9ca1dd51edabf 100644 --- a/src/go/plugin/go.d/collector/snmp_topology/collector.go +++ b/src/go/plugin/go.d/collector/snmp_topology/collector.go @@ -80,7 +80,6 @@ type ( deviceCaches map[string]*topologyCache // one cache per SNMP device deviceLastCollected map[string]time.Time // last collection time per device - topologyCache *topologyCache // current device cache (set during refreshDeviceTopology) topologyRegistry *topologyRegistry deviceSource deviceSource trapEnrichment *TrapEnrichmentHandle @@ -321,17 +320,15 @@ func (c *Collector) refreshDeviceTopology(ctx context.Context, key string, dev d // Build the next snapshot off-registry. Function readers keep seeing the // previous complete snapshot until this collection is fully ingested. next := c.newDeviceCollectionCache(dev) - c.topologyCache = next - defer func() { c.topologyCache = nil }() - c.updateTopologySysUptime(sysUptime) - c.updateTopologyProfileTags(pms) - c.ingestTopologyProfileMetrics(pms) - c.collectTopologyVTPVLANContexts(ctx, dev) + next.updateTopologySysUptime(sysUptime) + next.updateTopologyProfileTags(pms) + next.ingestTopologyProfileMetrics(pms) + c.collectTopologyVTPVLANContexts(ctx, next, dev) if ctx.Err() != nil { return false } - c.finalizeTopologyCache() + c.finalizeTopologyCache(next) cache := c.getOrCreateDeviceCache(key) cache.mu.Lock() @@ -397,18 +394,6 @@ func (c *Collector) getTopologyProfiles(dev ddsnmp.DeviceConnectionInfo) []*ddsn return c.findTopologyProfiles(dev) } -func (c *Collector) ingestTopologyProfileMetrics(pms []*ddsnmp.ProfileMetrics) { - for _, pm := range pms { - c.ingestTopologyMetricSet(pm.TopologyMetrics) - } -} - -func (c *Collector) ingestTopologyMetricSet(metrics []ddsnmp.Metric) { - for _, metric := range metrics { - c.updateTopologyCacheEntry(metric) - } -} - func newSNMPClientFromDeviceInfo(newClient func() gosnmp.Handler, dev ddsnmp.DeviceConnectionInfo) (gosnmp.Handler, error) { client := newClient() diff --git a/src/go/plugin/go.d/collector/snmp_topology/collector_refresh_test.go b/src/go/plugin/go.d/collector/snmp_topology/collector_refresh_test.go index a50a51edae33f3..67132bb77912e5 100644 --- a/src/go/plugin/go.d/collector/snmp_topology/collector_refresh_test.go +++ b/src/go/plugin/go.d/collector/snmp_topology/collector_refresh_test.go @@ -4,6 +4,7 @@ package snmptopology import ( "context" + "errors" "sync" "testing" "time" @@ -359,6 +360,42 @@ func TestCollector_RefreshKeepsPublishedSnapshotWhileCollectionRuns(t *testing.T <-done } +func TestCollector_RefreshFailureKeepsPublishedSnapshot(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + dev := ddsnmp.DeviceConnectionInfo{ + Hostname: "10.0.0.10", + Port: 161, + SysObjectID: "1.3.6.1.4.1.9.1.1", + } + mockHandler := snmpmock.NewMockHandler(ctrl) + expectTopologyRefreshSNMPClientConnect(mockHandler, dev) + mockHandler.EXPECT().Close().Return(nil) + + key := "10.0.0.10:161" + published := newTopologyCache() + seedPublishedEndpointSnapshot(published) + + coll := newTestSNMPTopologyCollector() + coll.deviceCaches[key] = published + coll.topologyRegistry.register(published) + coll.newSnmpClient = func() gosnmp.Handler { return mockHandler } + coll.newDdSnmpColl = func(ddsnmpcollector.Config) ddCollector { + return ddCollectorFunc(func() ([]*ddsnmp.ProfileMetrics, error) { + return nil, errors.New("collection failed") + }) + } + + require.False(t, coll.refreshDeviceTopology(context.Background(), key, dev)) + + snapshot, ok := published.snapshotEngineObservations() + require.True(t, ok) + require.Len(t, snapshot.l2Observations, 1) + require.Len(t, snapshot.l2Observations[0].FDBEntries, 1) + require.Len(t, snapshot.l2Observations[0].ARPNDEntries, 1) +} + type blockingTopologyCollector struct { started chan<- struct{} release <-chan struct{} @@ -376,15 +413,7 @@ func (c *blockingTopologyCollector) Collect() ([]*ddsnmp.ProfileMetrics, error) } func expectTopologyRefreshSNMPClient(mockHandler *snmpmock.MockHandler, dev ddsnmp.DeviceConnectionInfo) { - mockHandler.EXPECT().SetTarget(dev.Hostname) - mockHandler.EXPECT().SetPort(uint16(dev.Port)) - mockHandler.EXPECT().SetRetries(dev.Retries) - mockHandler.EXPECT().SetTimeout(time.Duration(dev.Timeout) * time.Second) - mockHandler.EXPECT().SetMaxOids(dev.MaxOIDs) - mockHandler.EXPECT().SetMaxRepetitions(uint32(dev.MaxRepetitions)) - mockHandler.EXPECT().SetCommunity(dev.Community) - mockHandler.EXPECT().SetVersion(gosnmp.Version2c) - mockHandler.EXPECT().Connect().Return(nil) + expectTopologyRefreshSNMPClientConnect(mockHandler, dev) mockHandler.EXPECT().Get(gomock.InAnyOrder([]string{ snmputils.OidSnmpEngineTime, snmputils.OidHrSystemUptime, @@ -397,6 +426,18 @@ func expectTopologyRefreshSNMPClient(mockHandler *snmpmock.MockHandler, dev ddsn mockHandler.EXPECT().Close().Return(nil) } +func expectTopologyRefreshSNMPClientConnect(mockHandler *snmpmock.MockHandler, dev ddsnmp.DeviceConnectionInfo) { + mockHandler.EXPECT().SetTarget(dev.Hostname) + mockHandler.EXPECT().SetPort(uint16(dev.Port)) + mockHandler.EXPECT().SetRetries(dev.Retries) + mockHandler.EXPECT().SetTimeout(time.Duration(dev.Timeout) * time.Second) + mockHandler.EXPECT().SetMaxOids(dev.MaxOIDs) + mockHandler.EXPECT().SetMaxRepetitions(uint32(dev.MaxRepetitions)) + mockHandler.EXPECT().SetCommunity(dev.Community) + mockHandler.EXPECT().SetVersion(gosnmp.Version2c) + mockHandler.EXPECT().Connect().Return(nil) +} + func seedPublishedEndpointSnapshot(cache *topologyCache) { now := time.Now() cache.updateTime = now diff --git a/src/go/plugin/go.d/collector/snmp_topology/topology_cache_ingest.go b/src/go/plugin/go.d/collector/snmp_topology/topology_cache_ingest.go index 158575f0fe2f85..d9baa4ad70ce28 100644 --- a/src/go/plugin/go.d/collector/snmp_topology/topology_cache_ingest.go +++ b/src/go/plugin/go.d/collector/snmp_topology/topology_cache_ingest.go @@ -8,13 +8,13 @@ import ( "github.com/netdata/netdata/go/plugins/plugin/go.d/collector/snmp/ddsnmp" ) -func (c *Collector) updateTopologyProfileTags(pms []*ddsnmp.ProfileMetrics) { - if c.topologyCache == nil { +func (c *topologyCache) updateTopologyProfileTags(pms []*ddsnmp.ProfileMetrics) { + if c == nil { return } - c.topologyCache.mu.Lock() - defer c.topologyCache.mu.Unlock() + c.mu.Lock() + defer c.mu.Unlock() for _, pm := range pms { tags := topologyMetadataValues(pm.DeviceMetadata) @@ -30,41 +30,53 @@ func (c *Collector) updateTopologyProfileTags(pms []*ddsnmp.ProfileMetrics) { } if len(tags) > 0 { - c.topologyCache.applyLLDPLocalDeviceProfileTags(tags) - c.topologyCache.updateLocalBridgeIdentityFromTags(tags) - c.topologyCache.applySTPProfileTags(tags) - c.topologyCache.applyVTPProfileTags(tags) + c.applyLLDPLocalDeviceProfileTags(tags) + c.updateLocalBridgeIdentityFromTags(tags) + c.applySTPProfileTags(tags) + c.applyVTPProfileTags(tags) } } } -func (c *Collector) updateTopologyCacheEntry(m ddsnmp.Metric) { - if c.topologyCache == nil { +func (c *topologyCache) updateTopologyCacheEntry(m ddsnmp.Metric) { + if c == nil { return } - c.topologyCache.mu.Lock() - defer c.topologyCache.mu.Unlock() + c.mu.Lock() + defer c.mu.Unlock() - c.topologyCache.ingestMetric(m.TopologyKind, m.Tags) + c.ingestMetric(m.TopologyKind, m.Tags) } -func (c *Collector) updateTopologySysUptime(value int64) { - if c == nil || c.topologyCache == nil { +func (c *topologyCache) updateTopologySysUptime(value int64) { + if c == nil { return } if value <= 0 { return } - c.topologyCache.mu.Lock() - defer c.topologyCache.mu.Unlock() + c.mu.Lock() + defer c.mu.Unlock() - local := c.topologyCache.localDevice + local := c.localDevice local.SysUptime = value local.Labels = ensureLabels(local.Labels) setTopologyMetadataLabelIfMissing(local.Labels, "sys_uptime", strconv.FormatInt(value, 10)) - c.topologyCache.localDevice = local + c.localDevice = local +} + +func (c *topologyCache) ingestTopologyProfileMetrics(pms []*ddsnmp.ProfileMetrics) { + for _, pm := range pms { + c.ingestTopologyMetricSet(pm.TopologyMetrics) + } +} + +func (c *topologyCache) ingestTopologyMetricSet(metrics []ddsnmp.Metric) { + for _, metric := range metrics { + c.updateTopologyCacheEntry(metric) + } } func topologyMetadataValues(meta map[string]ddsnmp.MetaTag) map[string]string { diff --git a/src/go/plugin/go.d/collector/snmp_topology/topology_cache_lifecycle.go b/src/go/plugin/go.d/collector/snmp_topology/topology_cache_lifecycle.go index 881179cfeaf42a..9609d694593128 100644 --- a/src/go/plugin/go.d/collector/snmp_topology/topology_cache_lifecycle.go +++ b/src/go/plugin/go.d/collector/snmp_topology/topology_cache_lifecycle.go @@ -65,28 +65,45 @@ func (c *topologyCache) hasFreshSnapshotAt(now time.Time) bool { return true } -func (c *Collector) finalizeTopologyCache() { - cache := c.topologyCache +func (c *Collector) finalizeTopologyCache(cache *topologyCache) { if cache == nil { return } - cache.mu.Lock() - cache.updateFDBDiagnostics() - droppedNoMAC := cache.fdbRowsDroppedNoMAC - unmappedPort := cache.fdbRowsUnmappedPort - agentID := cache.agentID - cache.lastUpdate = cache.updateTime - cache.mu.Unlock() + stats := cache.finalizeTopologyCache() - if droppedNoMAC > 0 { - c.Warningf("device '%s': dropped %d topology FDB row(s) with empty MAC", agentID, droppedNoMAC) + if stats.droppedNoMAC > 0 { + c.Warningf("device '%s': dropped %d topology FDB row(s) with empty MAC", stats.agentID, stats.droppedNoMAC) } - if unmappedPort > 0 { - c.Warningf("device '%s': observed %d topology FDB row(s) with bridge ports missing ifIndex mapping", agentID, unmappedPort) + if stats.unmappedPort > 0 { + c.Warningf("device '%s': observed %d topology FDB row(s) with bridge ports missing ifIndex mapping", stats.agentID, stats.unmappedPort) } } +type topologyCacheFinalizeStats struct { + agentID string + droppedNoMAC int + unmappedPort int +} + +func (c *topologyCache) finalizeTopologyCache() topologyCacheFinalizeStats { + if c == nil { + return topologyCacheFinalizeStats{} + } + + c.mu.Lock() + defer c.mu.Unlock() + + c.updateFDBDiagnostics() + stats := topologyCacheFinalizeStats{ + agentID: c.agentID, + droppedNoMAC: c.fdbRowsDroppedNoMAC, + unmappedPort: c.fdbRowsUnmappedPort, + } + c.lastUpdate = c.updateTime + return stats +} + func (c *topologyCache) updateFDBDiagnostics() { c.fdbRowsUnmappedPort = 0 for _, entry := range c.fdbEntries { diff --git a/src/go/plugin/go.d/collector/snmp_topology/topology_cache_test.go b/src/go/plugin/go.d/collector/snmp_topology/topology_cache_test.go index 36555ff0cfae77..42ee051b875b52 100644 --- a/src/go/plugin/go.d/collector/snmp_topology/topology_cache_test.go +++ b/src/go/plugin/go.d/collector/snmp_topology/topology_cache_test.go @@ -15,15 +15,12 @@ import ( "github.com/stretchr/testify/require" ) -func newTestCollector(dev ddsnmp.DeviceConnectionInfo) *Collector { +func newTestTopologyCache(dev ddsnmp.DeviceConnectionInfo) *topologyCache { cache := newTopologyCache() cache.localDevice = buildLocalTopologyDevice(dev) cache.agentID = dev.Hostname cache.updateTime = time.Now() - return &Collector{ - topologyCache: cache, - deviceCaches: map[string]*topologyCache{dev.Hostname: cache}, - } + return cache } func TestTopologyMetricHandlersRegisteredForRowKinds(t *testing.T) { @@ -58,7 +55,7 @@ func TestTopologyMetricHandlersRegisteredForRowKinds(t *testing.T) { } func TestTopologyCache_LldpSnapshot(t *testing.T) { - coll := newTestCollector(ddsnmp.DeviceConnectionInfo{ + cache := newTestTopologyCache(ddsnmp.DeviceConnectionInfo{ Hostname: "10.0.0.1", SysObjectID: "1.3.6.1.4.1.9.1.1", SysName: "sw1", SysDescr: "Switch 1", SysLocation: "dc1", }) @@ -68,9 +65,9 @@ func TestTopologyCache_LldpSnapshot(t *testing.T) { tagLldpLocChassisIDSubtype: {Value: "4"}, }, }} - coll.updateTopologyProfileTags(pms) + cache.updateTopologyProfileTags(pms) - coll.updateTopologyCacheEntry(ddsnmp.Metric{ + cache.updateTopologyCacheEntry(ddsnmp.Metric{ TopologyKind: ddsnmp.KindLldpLocPort, Tags: map[string]string{ tagLldpLocPortNum: "1", @@ -79,7 +76,7 @@ func TestTopologyCache_LldpSnapshot(t *testing.T) { tagLldpLocPortDesc: "uplink", }, }) - coll.updateTopologyCacheEntry(ddsnmp.Metric{ + cache.updateTopologyCacheEntry(ddsnmp.Metric{ TopologyKind: ddsnmp.KindLldpRem, Tags: map[string]string{ tagLldpLocPortNum: "1", @@ -94,9 +91,9 @@ func TestTopologyCache_LldpSnapshot(t *testing.T) { }, }) - coll.finalizeTopologyCache() + cache.finalizeTopologyCache() - data, ok := snapshotTopologyCacheForTest(coll.topologyCache) + data, ok := snapshotTopologyCacheForTest(cache) require.True(t, ok) require.Len(t, data.Actors, 2) @@ -141,19 +138,19 @@ func TestTopologyCache_CdpSnapshot(t *testing.T) { } func TestTopologyCache_UpdateTopologyProfileTags_STPBridgeAddressSetsSNMPIdentity(t *testing.T) { - coll := newTestCollector(ddsnmp.DeviceConnectionInfo{Hostname: "10.20.4.2"}) - coll.topologyCache.localDevice.ChassisID = "10.20.4.2" - coll.topologyCache.localDevice.ChassisIDType = "management_ip" + cache := newTestTopologyCache(ddsnmp.DeviceConnectionInfo{Hostname: "10.20.4.2"}) + cache.localDevice.ChassisID = "10.20.4.2" + cache.localDevice.ChassisIDType = "management_ip" - coll.updateTopologyProfileTags([]*ddsnmp.ProfileMetrics{{ + cache.updateTopologyProfileTags([]*ddsnmp.ProfileMetrics{{ DeviceMetadata: map[string]ddsnmp.MetaTag{ tagBridgeBaseAddress: {Value: "\"18 FD 74 33 1A 9C \""}, }, }}) - require.Equal(t, "18:fd:74:33:1a:9c", coll.topologyCache.stpBaseBridgeAddress) - require.Equal(t, "18:fd:74:33:1a:9c", coll.topologyCache.localDevice.ChassisID) - require.Equal(t, "macAddress", coll.topologyCache.localDevice.ChassisIDType) + require.Equal(t, "18:fd:74:33:1a:9c", cache.stpBaseBridgeAddress) + require.Equal(t, "18:fd:74:33:1a:9c", cache.localDevice.ChassisID) + require.Equal(t, "macAddress", cache.localDevice.ChassisIDType) } func TestTopologyCache_UpdateFdbEntry_STPBridgeAddressTagSetsSNMPIdentity(t *testing.T) { @@ -249,12 +246,12 @@ func TestTopologyCache_UpdateIfIndexByIP_CollectsAllSNMPDeviceIPs(t *testing.T) } func TestTopologyCache_UpdateTopologyProfileTags_LLDPDoesNotOverrideExistingSNMPIdentity(t *testing.T) { - coll := newTestCollector(ddsnmp.DeviceConnectionInfo{Hostname: "10.20.4.2"}) - coll.topologyCache.localDevice.ChassisID = "18:fd:74:33:1a:9c" - coll.topologyCache.localDevice.ChassisIDType = "macAddress" - coll.topologyCache.localDevice.SysName = "MikroTik-Switch" + cache := newTestTopologyCache(ddsnmp.DeviceConnectionInfo{Hostname: "10.20.4.2"}) + cache.localDevice.ChassisID = "18:fd:74:33:1a:9c" + cache.localDevice.ChassisIDType = "macAddress" + cache.localDevice.SysName = "MikroTik-Switch" - coll.updateTopologyProfileTags([]*ddsnmp.ProfileMetrics{{ + cache.updateTopologyProfileTags([]*ddsnmp.ProfileMetrics{{ DeviceMetadata: map[string]ddsnmp.MetaTag{ tagLldpLocChassisID: {Value: "00:11:22:33:44:55"}, tagLldpLocChassisIDSubtype: {Value: "4"}, @@ -262,9 +259,9 @@ func TestTopologyCache_UpdateTopologyProfileTags_LLDPDoesNotOverrideExistingSNMP }, }}) - require.Equal(t, "18:fd:74:33:1a:9c", coll.topologyCache.localDevice.ChassisID) - require.Equal(t, "macAddress", coll.topologyCache.localDevice.ChassisIDType) - require.Equal(t, "MikroTik-Switch", coll.topologyCache.localDevice.SysName) + require.Equal(t, "18:fd:74:33:1a:9c", cache.localDevice.ChassisID) + require.Equal(t, "macAddress", cache.localDevice.ChassisIDType) + require.Equal(t, "MikroTik-Switch", cache.localDevice.SysName) } func TestTopologyCache_CdpSnapshotHexAddress(t *testing.T) { @@ -473,10 +470,10 @@ func TestTopologyCache_SnapshotMergesRemoteIdentityAcrossProtocols(t *testing.T) } func TestTopologyCache_LLDPManagementAddressesAndCaps(t *testing.T) { - coll := newTestCollector(ddsnmp.DeviceConnectionInfo{ + cache := newTestTopologyCache(ddsnmp.DeviceConnectionInfo{ Hostname: "10.0.0.1", SysObjectID: "1.3.6.1.4.1.9.1.1", SysName: "sw1", SysDescr: "Switch 1", }) - coll.updateTopologyProfileTags([]*ddsnmp.ProfileMetrics{{ + cache.updateTopologyProfileTags([]*ddsnmp.ProfileMetrics{{ DeviceMetadata: map[string]ddsnmp.MetaTag{ tagLldpLocChassisID: {Value: "00:11:22:33:44:55"}, tagLldpLocChassisIDSubtype: {Value: "4"}, @@ -485,7 +482,7 @@ func TestTopologyCache_LLDPManagementAddressesAndCaps(t *testing.T) { }, }}) - coll.updateTopologyCacheEntry(ddsnmp.Metric{ + cache.updateTopologyCacheEntry(ddsnmp.Metric{ TopologyKind: ddsnmp.KindLldpLocManAddr, Tags: map[string]string{ tagLldpLocMgmtAddrSubtype: "2", @@ -493,7 +490,7 @@ func TestTopologyCache_LLDPManagementAddressesAndCaps(t *testing.T) { tagLldpLocMgmtAddrIfID: "1", }, }) - coll.updateTopologyCacheEntry(ddsnmp.Metric{ + cache.updateTopologyCacheEntry(ddsnmp.Metric{ TopologyKind: ddsnmp.KindLldpRemManAddr, Tags: map[string]string{ tagLldpLocPortNum: "1", @@ -502,7 +499,7 @@ func TestTopologyCache_LLDPManagementAddressesAndCaps(t *testing.T) { tagLldpRemMgmtAddr: "0a000002", }, }) - coll.updateTopologyCacheEntry(ddsnmp.Metric{ + cache.updateTopologyCacheEntry(ddsnmp.Metric{ TopologyKind: ddsnmp.KindLldpRemManAddr, Tags: map[string]string{ tagLldpLocPortNum: "1", @@ -511,7 +508,7 @@ func TestTopologyCache_LLDPManagementAddressesAndCaps(t *testing.T) { tagLldpRemMgmtAddr: "31302e32302e342e3834", // "10.20.4.84" ASCII-hex }, }) - coll.updateTopologyCacheEntry(ddsnmp.Metric{ + cache.updateTopologyCacheEntry(ddsnmp.Metric{ TopologyKind: ddsnmp.KindLldpRemManAddr, Tags: map[string]string{ tagLldpLocPortNum: "1", @@ -520,7 +517,7 @@ func TestTopologyCache_LLDPManagementAddressesAndCaps(t *testing.T) { tagLldpRemMgmtAddr: "666330303a663835333a6363643a653739333a3a31", // "fc00:f853:ccd:e793::1" ASCII-hex }, }) - coll.updateTopologyCacheEntry(ddsnmp.Metric{ + cache.updateTopologyCacheEntry(ddsnmp.Metric{ TopologyKind: ddsnmp.KindLldpRemManAddr, Tags: map[string]string{ tagLldpLocPortNum: "1", @@ -533,7 +530,7 @@ func TestTopologyCache_LLDPManagementAddressesAndCaps(t *testing.T) { tagLldpRemMgmtAddrOctetPref + "4": "21", }, }) - coll.updateTopologyCacheEntry(ddsnmp.Metric{ + cache.updateTopologyCacheEntry(ddsnmp.Metric{ TopologyKind: ddsnmp.KindLldpRem, Tags: map[string]string{ tagLldpLocPortNum: "1", @@ -545,9 +542,9 @@ func TestTopologyCache_LLDPManagementAddressesAndCaps(t *testing.T) { }, }) - coll.finalizeTopologyCache() + cache.finalizeTopologyCache() - data, ok := snapshotTopologyCacheForTest(coll.topologyCache) + data, ok := snapshotTopologyCacheForTest(cache) require.True(t, ok) require.Greater(t, len(data.Actors), 1) @@ -1289,24 +1286,20 @@ func TestBuildLocalTopologyDevice_IncludesSysContactVendorAndModel(t *testing.T) require.Equal(t, topologyProfileChartContextPrefix, device.ChartContextPrefix) } -func TestCollector_UpdateTopologySysUptime_StoresSysUptime(t *testing.T) { - coll := &Collector{ - topologyCache: newTopologyCache(), - } - coll.topologyCache.localDevice = topologyDevice{} +func TestTopologyCache_UpdateTopologySysUptime_StoresSysUptime(t *testing.T) { + cache := newTopologyCache() + cache.localDevice = topologyDevice{} - coll.updateTopologySysUptime(4321) + cache.updateTopologySysUptime(4321) - require.EqualValues(t, 4321, coll.topologyCache.localDevice.SysUptime) - require.Equal(t, "4321", coll.topologyCache.localDevice.Labels["sys_uptime"]) + require.EqualValues(t, 4321, cache.localDevice.SysUptime) + require.Equal(t, "4321", cache.localDevice.Labels["sys_uptime"]) } -func TestCollector_IngestTopologyProfileMetrics_IncludesTopologyMetrics(t *testing.T) { - coll := &Collector{ - topologyCache: newTopologyCache(), - } +func TestTopologyCache_IngestTopologyProfileMetrics_IncludesTopologyMetrics(t *testing.T) { + cache := newTopologyCache() - coll.ingestTopologyProfileMetrics([]*ddsnmp.ProfileMetrics{ + cache.ingestTopologyProfileMetrics([]*ddsnmp.ProfileMetrics{ { TopologyMetrics: []ddsnmp.Metric{ { @@ -1337,10 +1330,10 @@ func TestCollector_IngestTopologyProfileMetrics_IncludesTopologyMetrics(t *testi }, }) - require.Contains(t, coll.topologyCache.lldpLocPorts, "7") - require.Contains(t, coll.topologyCache.lldpRemotes, "7:1") - require.Zero(t, coll.topologyCache.localDevice.SysUptime) - require.Empty(t, coll.topologyCache.localDevice.Labels["sys_uptime"]) + require.Contains(t, cache.lldpLocPorts, "7") + require.Contains(t, cache.lldpRemotes, "7:1") + require.Zero(t, cache.localDevice.SysUptime) + require.Empty(t, cache.localDevice.Labels["sys_uptime"]) } func TestBuildLocalTopologyDevice_MapsVersionToSoftwareOnly(t *testing.T) { @@ -1446,73 +1439,6 @@ func TestAugmentLocalActorFromCache_InjectsIdentityFields(t *testing.T) { require.Equal(t, []string{"ifErrors", "ifTraffic"}, statuses[0]["available_metrics"]) } -/* Chart cross-linking test removed — feature dropped during split. -func TestCollector_SyncTopologyChartReferences(t *testing.T) { - charts := &collectorapi.Charts{} - require.NoError(t, charts.Add( - &collectorapi.Chart{ - ID: "snmp_device_prof_sysUpTime", - Title: "System Uptime", - Units: "1", - Fam: "sys", - Ctx: "snmp.device_prof_sysUpTime", - Dims: collectorapi.Dims{ - {ID: "snmp_device_prof_sysUpTime", Name: "sysUpTime"}, - }, - }, - &collectorapi.Chart{ - ID: "snmp_device_prof_ifTraffic_swp07", - Title: "Traffic swp07", - Units: "bit/s", - Fam: "ifTraffic", - Ctx: "snmp.device_prof_ifTraffic", - Dims: collectorapi.Dims{ - {ID: "snmp_device_prof_ifTraffic_swp07_in", Name: "in"}, - }, - }, - &collectorapi.Chart{ - ID: "ping_rtt", - Title: "Ping round-trip time", - Units: "milliseconds", - Fam: "Ping/RTT", - Ctx: "snmp.device_ping_rtt", - Dims: collectorapi.Dims{ - {ID: "ping_rtt_avg", Name: "avg"}, - }, - }, - )) - - coll := &Collector{ - charts: charts, - seenScalarMetrics: map[string]bool{"sysUpTime": true}, - ifaceCache: newIfaceCache(), - topologyCache: newTopologyCache(), - vnode: &vnodes.VirtualNode{GUID: "11111111-1111-1111-1111-111111111111"}, - } - - coll.ifaceCache.interfaces["swp07"] = &ifaceEntry{ - name: "swp07", - availableMetrics: map[string]struct{}{ - "ifTraffic": {}, - "ifErrors": {}, - }, - updated: true, - } - - coll.syncTopologyChartReferences() - - local := coll.topologyCache.localDevice - require.Equal(t, "11111111-1111-1111-1111-111111111111", local.NetdataHostID) - require.Equal(t, topologyProfileChartIDPrefix, local.ChartIDPrefix) - require.Equal(t, topologyProfileChartContextPrefix, local.ChartContextPrefix) - require.Equal(t, "snmp_device_prof_sysUpTime", local.DeviceCharts["sysUpTime"]) - require.Equal(t, "ping_rtt", local.DeviceCharts["ping_rtt"]) - require.Contains(t, local.InterfaceCharts, "swp07") - require.Equal(t, "swp07", local.InterfaceCharts["swp07"].ChartIDSuffix) - require.Equal(t, []string{"ifTraffic"}, local.InterfaceCharts["swp07"].AvailableMetrics) -} -*/ - func actorHasAttributeList(snapshot topologyData, key string) bool { for _, actor := range snapshot.Actors { if actor.Attributes == nil { diff --git a/src/go/plugin/go.d/collector/snmp_topology/topology_snmprec_forwarding_test.go b/src/go/plugin/go.d/collector/snmp_topology/topology_snmprec_forwarding_test.go index 34953d1764f3e2..8c51eb98c05506 100644 --- a/src/go/plugin/go.d/collector/snmp_topology/topology_snmprec_forwarding_test.go +++ b/src/go/plugin/go.d/collector/snmp_topology/topology_snmprec_forwarding_test.go @@ -73,8 +73,8 @@ func TestTopologyCache_RealSnmprecForwardingFixtures(t *testing.T) { require.NotEmpty(t, data.bridgePorts, "fixture %q should expose bridge port mappings", tt.fixture) require.True(t, len(data.fdbEntries) > 0 || len(data.qBridgeFdb) > 0, "fixture %q should expose FDB data", tt.fixture) - coll := replaySnmprecForwardingFixture(t, tt.fixture, data) - obs := coll.topologyCache.buildEngineObservation(coll.topologyCache.localDevice) + cache := replaySnmprecForwardingFixture(t, tt.fixture, data) + obs := cache.buildEngineObservation(cache.localDevice) require.NotEmpty(t, obs.Interfaces, "expected observed interfaces from fixture %q", tt.fixture) require.NotEmpty(t, obs.BridgePorts, "expected observed bridge ports from fixture %q", tt.fixture) @@ -93,56 +93,56 @@ func TestTopologyCache_RealSnmprecForwardingFixtures(t *testing.T) { } if tt.wantVTPVLANMap { require.NotEmpty(t, data.vtpVLANs, "fixture %q should expose VTP VLAN data", tt.fixture) - require.True(t, cacheContainsAnyVLANName(coll.topologyCache.vlanIDToName, data.vtpVLANNames), "expected VTP VLAN names from fixture %q", tt.fixture) + require.True(t, cacheContainsAnyVLANName(cache.vlanIDToName, data.vtpVLANNames), "expected VTP VLAN names from fixture %q", tt.fixture) } }) } } -func replaySnmprecForwardingFixture(t *testing.T, fixture string, data snmprecForwardingFixture) *Collector { +func replaySnmprecForwardingFixture(t *testing.T, fixture string, data snmprecForwardingFixture) *topologyCache { t.Helper() - coll := newTestCollector(ddsnmp.DeviceConnectionInfo{ + cache := newTestTopologyCache(ddsnmp.DeviceConnectionInfo{ Hostname: "192.0.2.10", SysObjectID: "1.3.6.1.4.1.9.1.1", SysName: fixture, }) if len(data.bridgeMetadata) > 0 { - coll.updateTopologyProfileTags([]*ddsnmp.ProfileMetrics{{DeviceMetadata: data.bridgeMetadata}}) + cache.updateTopologyProfileTags([]*ddsnmp.ProfileMetrics{{DeviceMetadata: data.bridgeMetadata}}) } for _, tags := range data.ifNameEntries { - coll.updateTopologyCacheEntry(ddsnmp.Metric{TopologyKind: ddsnmp.KindIfName, Tags: tags}) + cache.updateTopologyCacheEntry(ddsnmp.Metric{TopologyKind: ddsnmp.KindIfName, Tags: tags}) } for _, tags := range data.ifStatusEntries { - coll.updateTopologyCacheEntry(ddsnmp.Metric{TopologyKind: ddsnmp.KindIfStatus, Tags: tags}) + cache.updateTopologyCacheEntry(ddsnmp.Metric{TopologyKind: ddsnmp.KindIfStatus, Tags: tags}) } for _, tags := range data.ipIfEntries { - coll.updateTopologyCacheEntry(ddsnmp.Metric{TopologyKind: ddsnmp.KindIpIfIndex, Tags: tags}) + cache.updateTopologyCacheEntry(ddsnmp.Metric{TopologyKind: ddsnmp.KindIpIfIndex, Tags: tags}) } for _, tags := range data.bridgePorts { - coll.updateTopologyCacheEntry(ddsnmp.Metric{TopologyKind: ddsnmp.KindBridgePortIfIndex, Tags: tags}) + cache.updateTopologyCacheEntry(ddsnmp.Metric{TopologyKind: ddsnmp.KindBridgePortIfIndex, Tags: tags}) } for _, tags := range data.qBridgeVLANs { - coll.updateTopologyCacheEntry(ddsnmp.Metric{TopologyKind: ddsnmp.KindQbridgeVlanEntry, Tags: tags}) + cache.updateTopologyCacheEntry(ddsnmp.Metric{TopologyKind: ddsnmp.KindQbridgeVlanEntry, Tags: tags}) } for _, tags := range data.vtpVLANs { - coll.updateTopologyCacheEntry(ddsnmp.Metric{TopologyKind: ddsnmp.KindVtpVlan, Tags: tags}) + cache.updateTopologyCacheEntry(ddsnmp.Metric{TopologyKind: ddsnmp.KindVtpVlan, Tags: tags}) } for _, tags := range data.fdbEntries { - coll.updateTopologyCacheEntry(ddsnmp.Metric{TopologyKind: ddsnmp.KindFdbEntry, Tags: tags}) + cache.updateTopologyCacheEntry(ddsnmp.Metric{TopologyKind: ddsnmp.KindFdbEntry, Tags: tags}) } for _, tags := range data.qBridgeFdb { - coll.updateTopologyCacheEntry(ddsnmp.Metric{TopologyKind: ddsnmp.KindQbridgeFdbEntry, Tags: tags}) + cache.updateTopologyCacheEntry(ddsnmp.Metric{TopologyKind: ddsnmp.KindQbridgeFdbEntry, Tags: tags}) } for _, tags := range data.stpPorts { - coll.updateTopologyCacheEntry(ddsnmp.Metric{TopologyKind: ddsnmp.KindStpPort, Tags: tags}) + cache.updateTopologyCacheEntry(ddsnmp.Metric{TopologyKind: ddsnmp.KindStpPort, Tags: tags}) } for _, tags := range data.arpEntries { - coll.updateTopologyCacheEntry(ddsnmp.Metric{TopologyKind: ddsnmp.KindArpEntry, Tags: tags}) + cache.updateTopologyCacheEntry(ddsnmp.Metric{TopologyKind: ddsnmp.KindArpEntry, Tags: tags}) } - return coll + return cache } func parseSnmprecForwardingFixture(t *testing.T, path string) snmprecForwardingFixture { diff --git a/src/go/plugin/go.d/collector/snmp_topology/topology_snmprec_test.go b/src/go/plugin/go.d/collector/snmp_topology/topology_snmprec_test.go index 89130d4717bbe9..814bceb6b38b29 100644 --- a/src/go/plugin/go.d/collector/snmp_topology/topology_snmprec_test.go +++ b/src/go/plugin/go.d/collector/snmp_topology/topology_snmprec_test.go @@ -43,34 +43,34 @@ func TestTopologyCache_RealSnmprecFixtures(t *testing.T) { t.Skip("no LLDP/CDP data detected") } - coll := newTestCollector(ddsnmp.DeviceConnectionInfo{ + cache := newTestTopologyCache(ddsnmp.DeviceConnectionInfo{ Hostname: "192.0.2.10", SysObjectID: "1.3.6.1.4.1.9.1.1", SysName: filepath.Base(path), }) if len(data.lldpLocalMeta) > 0 { - coll.updateTopologyProfileTags([]*ddsnmp.ProfileMetrics{{DeviceMetadata: data.lldpLocalMeta}}) + cache.updateTopologyProfileTags([]*ddsnmp.ProfileMetrics{{DeviceMetadata: data.lldpLocalMeta}}) } for _, tags := range data.lldpLocPorts { - coll.updateTopologyCacheEntry(ddsnmp.Metric{TopologyKind: ddsnmp.KindLldpLocPort, Tags: tags}) + cache.updateTopologyCacheEntry(ddsnmp.Metric{TopologyKind: ddsnmp.KindLldpLocPort, Tags: tags}) } for _, tags := range data.lldpLocManAddrs { - coll.updateTopologyCacheEntry(ddsnmp.Metric{TopologyKind: ddsnmp.KindLldpLocManAddr, Tags: tags}) + cache.updateTopologyCacheEntry(ddsnmp.Metric{TopologyKind: ddsnmp.KindLldpLocManAddr, Tags: tags}) } for _, tags := range data.lldpRemotes { - coll.updateTopologyCacheEntry(ddsnmp.Metric{TopologyKind: ddsnmp.KindLldpRem, Tags: tags}) + cache.updateTopologyCacheEntry(ddsnmp.Metric{TopologyKind: ddsnmp.KindLldpRem, Tags: tags}) } for _, tags := range data.lldpRemManAddrs { - coll.updateTopologyCacheEntry(ddsnmp.Metric{TopologyKind: ddsnmp.KindLldpRemManAddr, Tags: tags}) + cache.updateTopologyCacheEntry(ddsnmp.Metric{TopologyKind: ddsnmp.KindLldpRemManAddr, Tags: tags}) } for _, tags := range data.cdpRemotes { - coll.updateTopologyCacheEntry(ddsnmp.Metric{TopologyKind: ddsnmp.KindCdpCache, Tags: tags}) + cache.updateTopologyCacheEntry(ddsnmp.Metric{TopologyKind: ddsnmp.KindCdpCache, Tags: tags}) } - coll.finalizeTopologyCache() + cache.finalizeTopologyCache() options := defaultTopologyQueryOptionsForTest() options.CollapseActorsByIP = false options.EliminateNonIPInferred = false - snapshot, ok := snapshotTopologyCacheForTestWithOptions(coll.topologyCache, options) + snapshot, ok := snapshotTopologyCacheForTestWithOptions(cache, options) require.True(t, ok) require.GreaterOrEqual(t, len(snapshot.Actors), 1) diff --git a/src/go/plugin/go.d/collector/snmp_topology/topology_vlan_context.go b/src/go/plugin/go.d/collector/snmp_topology/topology_vlan_context.go index 851421c157fb74..da8547febf8607 100644 --- a/src/go/plugin/go.d/collector/snmp_topology/topology_vlan_context.go +++ b/src/go/plugin/go.d/collector/snmp_topology/topology_vlan_context.go @@ -8,15 +8,15 @@ import ( "github.com/netdata/netdata/go/plugins/plugin/go.d/collector/snmp/ddsnmp" ) -func (c *Collector) collectTopologyVTPVLANContexts(ctx context.Context, dev ddsnmp.DeviceConnectionInfo) { - if c.topologyCache == nil { +func (c *Collector) collectTopologyVTPVLANContexts(ctx context.Context, cache *topologyCache, dev ddsnmp.DeviceConnectionInfo) { + if cache == nil { return } if ctx.Err() != nil { return } - contexts := c.topologyCache.vtpVLANContexts() + contexts := cache.vtpVLANContexts() if len(contexts) == 0 { return } @@ -40,6 +40,6 @@ func (c *Collector) collectTopologyVTPVLANContexts(ctx context.Context, dev ddsn c.Warningf("device '%s': topology vlan-context polling failed for vlan %s: %v", dev.Hostname, context.vlanID, err) continue } - c.ingestTopologyVLANContextMetrics(context.vlanID, context.vlanName, pms) + cache.ingestTopologyVLANContextMetrics(context.vlanID, context.vlanName, pms) } } diff --git a/src/go/plugin/go.d/collector/snmp_topology/topology_vlan_context_ingest.go b/src/go/plugin/go.d/collector/snmp_topology/topology_vlan_context_ingest.go index 380867333a26b8..e539412d057b11 100644 --- a/src/go/plugin/go.d/collector/snmp_topology/topology_vlan_context_ingest.go +++ b/src/go/plugin/go.d/collector/snmp_topology/topology_vlan_context_ingest.go @@ -9,7 +9,11 @@ import ( "github.com/netdata/netdata/go/plugins/plugin/go.d/collector/snmp/ddsnmp" ) -func (c *Collector) ingestTopologyVLANContextMetrics(vlanID, vlanName string, pms []*ddsnmp.ProfileMetrics) { +func (c *topologyCache) ingestTopologyVLANContextMetrics(vlanID, vlanName string, pms []*ddsnmp.ProfileMetrics) { + if c == nil { + return + } + c.updateTopologyProfileTags(pms) for _, pm := range pms { From d47056a009998ccefa16959162c66424308e9ab4 Mon Sep 17 00:00:00 2001 From: vkalintiris Date: Fri, 19 Jun 2026 01:24:36 +0300 Subject: [PATCH 6/6] fix(journal-log-writer): accept only absolute journal directory paths. (#22771) fix(journal-log-writer): reject non-absolute journal directory at startup A relative journal_dir was accepted through Log::new init (the discarded canonicalize result left the path relative), then failed at first write with a cause-less "failed to create journal file in " because repository::File::from_path only accepts absolute paths. To OTLP clients this surfaced as a permanent "Failed to write log entry" gRPC Internal error and dropped log batches. Validate that the journal directory is absolute as the first step of create_chain, before any I/O, returning an InvalidPath error that names the offending value, and enrich the residual create-file error to describe the real remaining failure mode (unparseable generated file path). Relative-path resolution stays the consumer's responsibility: netflow resolves its relative default against the netdata cache dir when one is available, and the new Log::new check is the safety net for the standalone case. Add tests and a directory-contract spec; note the requirement in the stock otel config. --- .agents/sow/specs/README.md | 1 + .../journal-log-writer-directory-contract.md | 46 +++++++++++++ .../journal-log-writer/src/log/chain.rs | 3 +- src/crates/journal-log-writer/src/log/mod.rs | 12 ++++ .../journal-log-writer/tests/log_writer.rs | 64 +++++++++++++++++++ .../otel-plugin/configs/otel.yaml.in | 1 + 6 files changed, 126 insertions(+), 1 deletion(-) create mode 100644 .agents/sow/specs/journal-log-writer-directory-contract.md diff --git a/.agents/sow/specs/README.md b/.agents/sow/specs/README.md index 067571f43379cf..b5d23ddca1f84d 100644 --- a/.agents/sow/specs/README.md +++ b/.agents/sow/specs/README.md @@ -16,6 +16,7 @@ research cannot be mistaken for product specification. | Spec | Scope | Main consumers | |---|---|---| | [go-v2-host-scope.md](go-v2-host-scope.md) | Go framework V2 host-scope and vnode metric routing contract. | Go V2 collector work, `metrix`, `jobruntime`, `chartengine`, go.d collector docs and skills. | +| [journal-log-writer-directory-contract.md](journal-log-writer-directory-contract.md) | `Log::new` requires an absolute journal directory (rejected early otherwise); consumers must resolve relative config against a stable base. | journal-log-writer consumers: otel-plugin logs, netflow-plugin tiers. | | [netflow-tier-commit-workers.md](netflow-tier-commit-workers.md) | NetFlow rollup-tier commit worker contract: ownership, doorbell protocol, shutdown order, stretch semantics, lock discipline, telemetry. | netflow-plugin ingest/tiering work, tier benchmark and soak/crash tests, network-flows operator docs. | | [prometheus-profiles.md](prometheus-profiles.md) | go.d prometheus chart-profile format, selection modes, app-segment resolution, and chart-context assembly. | prometheus collector profile work, chart-context/UI app-section contract, profile authoring. | | [query-planner-tier-selection.md](query-planner-tier-selection.md) | Automatic storage-tier selection for metric query planning. | Query planner code, API data paths, MCP metric query behavior. | diff --git a/.agents/sow/specs/journal-log-writer-directory-contract.md b/.agents/sow/specs/journal-log-writer-directory-contract.md new file mode 100644 index 00000000000000..759a6dc45be0c6 --- /dev/null +++ b/.agents/sow/specs/journal-log-writer-directory-contract.md @@ -0,0 +1,46 @@ +# journal-log-writer directory contract + +## Scope + +Durable contract for the `journal-log-writer` crate (`Log::new`) regarding the +journal directory it is given. Applies to every consumer that constructs a +journal `Log`: the OTEL logs plugin (`otel-plugin`) and the NetFlow plugin +(`netflow-plugin`). + +## Contract + +- **The journal directory passed to `Log::new` MUST be an absolute path.** + A non-absolute path is rejected at construction with + `WriterError::InvalidPath`, naming the offending value. The rejection happens + **before** any filesystem mutation (no directory is created). +- Rationale: journal file paths are parsed by + `journal_registry::repository::File::from_path`, which only accepts absolute + paths. A relative directory therefore could never produce a writable journal — + previously it failed late, at first write, with a cause-less + `failed to create journal file in ` error. The early rejection makes the + misconfiguration diagnosable at startup. + +## Consumer responsibilities + +- **Resolve relative configuration before calling `Log::new`.** A consumer that + accepts a possibly-relative directory from its own config SHOULD resolve it to + an absolute path first, against a stable base directory it controls — never + rely on the process working directory. The `Log::new` check is the safety net, + not the primary resolution mechanism. + - NetFlow resolves `journal_dir` against the netdata cache directory **when one + is available**, via `resolve_relative_path` + (`netflow-plugin/src/plugin_config/runtime.rs`). Its default is relative + (`flows`), so under netdata (where `NETDATA_CACHE_DIR` is set) it resolves to + an absolute path. Standalone with no cache dir, the value stays relative and + is rejected by the `Log::new` check at startup (fail-fast, not a regression — + that configuration never produced a writable journal). + - The OTEL logs plugin does not resolve `journal_dir`; it requires an absolute + value. Its stock config renders one (`/otel/v1` via `@logdir_POST@`); + a relative override (`NETDATA_OTEL_LOGS_JOURNAL_DIR` env or a hand-templated + config) is treated as operator error and rejected at startup. + +## Notes + +- `Log::new` still calls `canonicalize()` on the (now guaranteed absolute) path + to verify accessibility; its result is intentionally not used to rewrite the + stored path (no symlink normalization), to keep behavior minimal. diff --git a/src/crates/journal-log-writer/src/log/chain.rs b/src/crates/journal-log-writer/src/log/chain.rs index fef3f9361dbe62..252659bb9fa586 100644 --- a/src/crates/journal-log-writer/src/log/chain.rs +++ b/src/crates/journal-log-writer/src/log/chain.rs @@ -151,7 +151,8 @@ impl OwnedChain { let Some(file) = create_chain_file(&self.path, seqnum_id, head_seqnum, head_realtime) else { return Err(WriterError::FileCreation(format!( - "failed to create journal file in {}", + "failed to create journal file in {}: the generated file path \ + could not be parsed as a journal repository file", self.path.display() ))); }; diff --git a/src/crates/journal-log-writer/src/log/mod.rs b/src/crates/journal-log-writer/src/log/mod.rs index 31112bd67aa5e1..d1676446fdf494 100644 --- a/src/crates/journal-log-writer/src/log/mod.rs +++ b/src/crates/journal-log-writer/src/log/mod.rs @@ -23,6 +23,18 @@ const STACK_ENTRY_REF_LIMIT: usize = 128; const SOURCE_REALTIME_PREFIX: &[u8] = b"_SOURCE_REALTIME_TIMESTAMP="; fn create_chain(path: &Path) -> Result { + // The journal directory must be absolute: downstream file-path parsing + // (`repository::File::from_path`) only accepts absolute paths, so a relative + // value would otherwise fail later at first write with a cause-less error. + // Checked first, before any I/O, so the rejection is FS-free and the error + // names the offending value regardless of machine-id availability. + if !path.is_absolute() { + return Err(WriterError::InvalidPath(format!( + "journal directory must be an absolute path, got: {}", + path.display() + ))); + } + let machine_id = load_machine_id() .map_err(|e| WriterError::MachineId(format!("failed to load machine ID: {}", e)))?; diff --git a/src/crates/journal-log-writer/tests/log_writer.rs b/src/crates/journal-log-writer/tests/log_writer.rs index 42bb8e3bd13cc3..60040c55421833 100644 --- a/src/crates/journal-log-writer/tests/log_writer.rs +++ b/src/crates/journal-log-writer/tests/log_writer.rs @@ -767,3 +767,67 @@ fn test_lifecycle_observer_reports_missing_retention_deletions() { "files removed from chain/accounting must still be reported for retention follow-up" ); } + +#[test] +fn relative_journal_dir_is_rejected_before_any_io() { + // A relative journal directory must fail fast at construction with a clear, + // actionable error that names the offending value, and must create no + // directory (validation happens before any filesystem mutation). + let root = "relative-journal-dir-rejected"; + let dir = format!("{root}/otel/v1"); + + // Clean any leftover from a prior run. A real removal error (e.g. permission + // denied) must surface rather than mask a spurious assertion below; only a + // missing directory is the expected, ignorable case. + match fs::remove_dir_all(root) { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => panic!("failed to clean up leftover {root}: {e}"), + } + + match Log::new(Path::new(&dir), test_config()) { + Ok(_) => panic!("expected Log::new to reject a relative journal directory"), + Err(e) => { + let msg = e.to_string(); + assert!( + msg.contains("absolute"), + "error should mention the absolute-path requirement, got: {msg}" + ); + assert!( + msg.contains(&dir), + "error should name the offending value, got: {msg}" + ); + } + } + + assert!( + !Path::new(root).exists(), + "rejection must happen before any I/O; no directory may be created" + ); +} + +#[test] +fn absolute_journal_dir_is_accepted() { + let tmp = TempDir::new().expect("create temp dir"); + assert!(tmp.path().is_absolute()); + match Log::new(tmp.path(), test_config()) { + Ok(_) => {} + Err(e) => panic!("absolute journal directory must be accepted, got: {e}"), + } +} + +#[test] +fn nonexistent_absolute_journal_dir_is_created_and_accepted() { + // Common production case: the configured absolute directory does not yet + // exist on first run and must be created. + let tmp = TempDir::new().expect("create temp dir"); + let fresh = tmp.path().join("not-yet-created/otel/v1"); + assert!(!fresh.exists()); + + match Log::new(&fresh, test_config()) { + Ok(_) => {} + Err(e) => panic!("non-existent absolute journal directory must be accepted, got: {e}"), + } + + assert!(fresh.is_dir(), "journal directory should have been created"); +} diff --git a/src/crates/netdata-otel/otel-plugin/configs/otel.yaml.in b/src/crates/netdata-otel/otel-plugin/configs/otel.yaml.in index ab51feb669cd65..774b2b6e69b3e9 100644 --- a/src/crates/netdata-otel/otel-plugin/configs/otel.yaml.in +++ b/src/crates/netdata-otel/otel-plugin/configs/otel.yaml.in @@ -35,6 +35,7 @@ metrics: logs: # Directory to store journal files for logs. + # Must be an absolute path; a relative value is rejected at startup. journal_dir: @logdir_POST@/otel/v1 # Maximum file size for individual journal files (e.g., "100MB", "1.5GB").