From b27c0f6c6f743433182dcb1e070c16ec4ee6c706 Mon Sep 17 00:00:00 2001 From: Costa Tsaousis Date: Sat, 20 Jun 2026 00:32:00 +0300 Subject: [PATCH 1/2] pulse: per-child streaming charts on parents + lock-free host status (#22784) --- CMakeLists.txt | 3 + src/daemon/main.c | 1 + src/daemon/pulse/pulse-parents.c | 370 ++++++++++++++------ src/database/rrdhost.h | 31 ++ src/libnetdata/atomics/atomics.h | 1 + src/libnetdata/atomics/single_writer.h | 47 +++ src/libnetdata/os/os.h | 1 + src/libnetdata/os/socket_egress_interface.c | 166 +++++++++ src/libnetdata/os/socket_egress_interface.h | 29 ++ src/streaming/stream-receiver.c | 10 +- src/streaming/stream-sender.c | 16 + 11 files changed, 561 insertions(+), 114 deletions(-) create mode 100644 src/libnetdata/atomics/single_writer.h create mode 100644 src/libnetdata/os/socket_egress_interface.c create mode 100644 src/libnetdata/os/socket_egress_interface.h diff --git a/CMakeLists.txt b/CMakeLists.txt index da6c42b18a2a86..a6e7fcc52a976e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1214,6 +1214,7 @@ set(LIBNETDATA_FILES src/libnetdata/locks/rw-spinlock-unittest.c src/libnetdata/atomics/atomic_flags.h src/libnetdata/atomics/atomics.h + src/libnetdata/atomics/single_writer.h src/libnetdata/locks/waitq.c src/libnetdata/locks/waitq.h src/libnetdata/object-state/object-state.c @@ -1235,6 +1236,8 @@ set(LIBNETDATA_FILES src/libnetdata/os/get_system_pagesize.h src/libnetdata/os/hostname.c src/libnetdata/os/hostname.h + src/libnetdata/os/socket_egress_interface.c + src/libnetdata/os/socket_egress_interface.h src/libnetdata/exit/exit_initiated.c src/libnetdata/exit/exit_initiated.h src/libnetdata/os/disk_space.c diff --git a/src/daemon/main.c b/src/daemon/main.c index f7ae1014332e10..5f51cd50f766a5 100644 --- a/src/daemon/main.c +++ b/src/daemon/main.c @@ -451,6 +451,7 @@ int netdata_main(int argc, char **argv) { if (ctx_unittest()) return 1; if (query_plan_unittest()) return 1; if (uuid_unittest()) return 1; + if (os_socket_egress_interface_unittest()) return 1; if (dyncfg_unittest()) return 1; if (eval_unittest()) return 1; if (duration_unittest()) return 1; diff --git a/src/daemon/pulse/pulse-parents.c b/src/daemon/pulse/pulse-parents.c index 6095bfbd183b99..ebdc676babaadf 100644 --- a/src/daemon/pulse/pulse-parents.c +++ b/src/daemon/pulse/pulse-parents.c @@ -3,8 +3,6 @@ #define PULSE_INTERNALS #include "pulse.h" -DEFINE_JUDYL_TYPED(PHOST, PULSE_HOST_STATUS); - // -------------------------------------------------------------------------------------------------------------------- // parents @@ -19,15 +17,12 @@ struct by_reason { #define STREAM_HANDSHAKE_OTHER (STREAM_HANDSHAKE_NEGATIVE_MAX + 2) struct { - SPINLOCK spinlock; - PHOST_JudyLSet index; - struct { - // counters + // event counters (event-driven) struct by_reason events_by_reason; struct by_reason disconnects_by_reason; - // gauges + // gauge chart pointers (the per-state counts are computed read-side by the traversal) struct { RRDSET *st_nodes; RRDDIM *rd_loading; @@ -39,34 +34,14 @@ struct { RRDDIM *rd_replication_waiting; RRDDIM *rd_replicating; RRDDIM *rd_running; - - ssize_t nodes_local; - ssize_t nodes_virtual; - ssize_t nodes_loading; - ssize_t nodes_archived; - ssize_t nodes_offline; - ssize_t nodes_waiting; - ssize_t nodes_replicating; - ssize_t nodes_replication_waiting; - ssize_t nodes_running; } type[2]; } parent; struct { - // counters + // event counters (event-driven) struct by_reason stream_info_failed_by_reason; struct by_reason events_by_reason; struct by_reason disconnects_by_reason; - - // gauges - ssize_t nodes_offline; - ssize_t nodes_connecting; - ssize_t nodes_pending; - ssize_t nodes_waiting; - ssize_t nodes_replicating; - ssize_t nodes_running; - ssize_t nodes_no_dst; - ssize_t nodes_no_dst_failed; } sender; } p = { 0 }; @@ -115,7 +90,6 @@ static void update_reason(struct by_reason *b, STREAM_HANDSHAKE reason) { } static void pulse_host_add_sub_status(PULSE_HOST_STATUS status, ssize_t val, STREAM_HANDSHAKE reason) { - size_t idx = status & PULSE_HOST_STATUS_EPHEMERAL ? 1 : 0; status &= ~(PULSE_HOST_STATUS_EPHEMERAL | PULSE_HOST_STATUS_PERMANENT); while(status) { @@ -128,78 +102,24 @@ static void pulse_host_add_sub_status(PULSE_HOST_STATUS status, ssize_t val, STR default: break; - case PULSE_HOST_STATUS_LOCAL: - __atomic_add_fetch(&p.parent.type[idx].nodes_local, val, __ATOMIC_RELAXED); - break; - - case PULSE_HOST_STATUS_VIRTUAL: - __atomic_add_fetch(&p.parent.type[idx].nodes_virtual, val, __ATOMIC_RELAXED); - break; - - case PULSE_HOST_STATUS_LOADING: - __atomic_add_fetch(&p.parent.type[idx].nodes_loading, val, __ATOMIC_RELAXED); - break; - - case PULSE_HOST_STATUS_ARCHIVED: - __atomic_add_fetch(&p.parent.type[idx].nodes_archived, val, __ATOMIC_RELAXED); - break; - + // inbound and outbound node gauges are now computed read-side by the pulse traversal; + // only the event-driven reason/event counters remain here. case PULSE_HOST_STATUS_RCV_OFFLINE: - __atomic_add_fetch(&p.parent.type[idx].nodes_offline, val, __ATOMIC_RELAXED); do_parent_reason = true; break; case PULSE_HOST_STATUS_RCV_WAITING: - __atomic_add_fetch(&p.parent.type[idx].nodes_waiting, val, __ATOMIC_RELAXED); do_parent_reason = true; reason = 0; break; - case PULSE_HOST_STATUS_RCV_REPLICATION_WAIT: - __atomic_add_fetch(&p.parent.type[idx].nodes_replication_waiting, val, __ATOMIC_RELAXED); - break; - - case PULSE_HOST_STATUS_RCV_REPLICATING: - __atomic_add_fetch(&p.parent.type[idx].nodes_replicating, val, __ATOMIC_RELAXED); - break; - - case PULSE_HOST_STATUS_RCV_RUNNING: - __atomic_add_fetch(&p.parent.type[idx].nodes_running, val, __ATOMIC_RELAXED); - break; - case PULSE_HOST_STATUS_SND_OFFLINE: - __atomic_add_fetch(&p.sender.nodes_offline, val, __ATOMIC_RELAXED); do_sender_reason = true; break; - case PULSE_HOST_STATUS_SND_PENDING: - __atomic_add_fetch(&p.sender.nodes_pending, val, __ATOMIC_RELAXED); - break; - case PULSE_HOST_STATUS_SND_CONNECTING: - __atomic_add_fetch(&p.sender.nodes_connecting, val, __ATOMIC_RELAXED); __atomic_add_fetch(&p.sender.events_by_reason.counters[STREAM_HANDSHAKE_CONNECT], 1, __ATOMIC_RELAXED); break; - - case PULSE_HOST_STATUS_SND_WAITING: - __atomic_add_fetch(&p.sender.nodes_waiting, val, __ATOMIC_RELAXED); - break; - - case PULSE_HOST_STATUS_SND_REPLICATING: - __atomic_add_fetch(&p.sender.nodes_replicating, val, __ATOMIC_RELAXED); - break; - - case PULSE_HOST_STATUS_SND_RUNNING: - __atomic_add_fetch(&p.sender.nodes_running, val, __ATOMIC_RELAXED); - break; - - case PULSE_HOST_STATUS_SND_NO_DST: - __atomic_add_fetch(&p.sender.nodes_no_dst, val, __ATOMIC_RELAXED); - break; - - case PULSE_HOST_STATUS_SND_NO_DST_FAILED: - __atomic_add_fetch(&p.sender.nodes_no_dst_failed, val, __ATOMIC_RELAXED); - break; } if(do_parent_reason && val > 0) @@ -236,6 +156,22 @@ void pulse_host_status(RRDHOST *host, PULSE_HOST_STATUS status, STREAM_HANDSHAKE status |= rrdhost_option_check(host, RRDHOST_OPTION_EPHEMERAL_HOST) ? PULSE_HOST_STATUS_EPHEMERAL : PULSE_HOST_STATUS_PERMANENT; + // running-latch: once the node first reaches RCV_RUNNING after a (re)connect, ignore the + // per-chart replication ripples that would otherwise flip the host status back to replicating. + // Makes no assumption that replication happens: RCV_RUNNING may be reached directly (replication + // disabled), and the block path is only ever taken for an incoming RCV_REPLICATING. + if(status & PULSE_HOST_STATUS_RCV_RUNNING) + __atomic_store_n(&host->stream.rcv.status.running_latched, true, __ATOMIC_RELAXED); + else if(status & PULSE_HOST_STATUS_RCV_REPLICATING) { + if(__atomic_load_n(&host->stream.rcv.status.running_latched, __ATOMIC_RELAXED) && + (__atomic_load_n(&host->stream.pulse_state, __ATOMIC_RELAXED) & PULSE_HOST_STATUS_RCV_RUNNING)) + status = (PULSE_HOST_STATUS)((status & ~PULSE_HOST_STATUS_RCV_REPLICATING) | PULSE_HOST_STATUS_RCV_RUNNING); + else + __atomic_store_n(&host->stream.rcv.status.running_latched, false, __ATOMIC_RELAXED); + } + else if(status & PULSE_HOST_STATUS_RCV_OFFLINE) + __atomic_store_n(&host->stream.rcv.status.running_latched, false, __ATOMIC_RELAXED); + if(status & basic) remove = basic | receiver | ephemerality | sender; else if(status & receiver) @@ -243,18 +179,28 @@ void pulse_host_status(RRDHOST *host, PULSE_HOST_STATUS status, STREAM_HANDSHAKE else if(status & sender) remove = sender; - spinlock_lock(&p.spinlock); - PULSE_HOST_STATUS old = PHOST_GET(&p.index, (uintptr_t)host); - if(status == PULSE_HOST_STATUS_DELETED) { - PHOST_DEL(&p.index, (uintptr_t)host); + // maintain the combined resolved state on the host (CAS, lock-free; replaces the global PHOST + // Judy + spinlock). The pulse traversal reads it to compute the streaming_inbound and + // streaming_outbound gauges read-side. + uint32_t cur = __atomic_load_n(&host->stream.pulse_state, __ATOMIC_RELAXED); + uint32_t next; + do { + next = (status == PULSE_HOST_STATUS_DELETED) ? 0u + : (uint32_t)((cur & ~(uint32_t)remove) | (uint32_t)status); + } while(!__atomic_compare_exchange_n(&host->stream.pulse_state, &cur, next, + false, __ATOMIC_RELAXED, __ATOMIC_RELAXED)); + + PULSE_HOST_STATUS old = (PULSE_HOST_STATUS)cur; // the state we transitioned from + + // reset the inbound-state age timer whenever the inbound (basic|receiver) state changes + if(((PULSE_HOST_STATUS)next & (basic | receiver)) != (old & (basic | receiver))) + __atomic_store_n(&host->stream.rcv.status.state_changed_s, now_realtime_sec(), __ATOMIC_RELAXED); + + if(status == PULSE_HOST_STATUS_DELETED) status = 0; // do not add anything, just remove the old flags - } - else - PHOST_SET(&p.index, (uintptr_t)host, (old & ~remove) | status); - spinlock_unlock(&p.spinlock); + // event-driven reason/event counters only (gauges are computed by the traversal) remove &= old; - pulse_host_add_sub_status(remove, -1, 0); pulse_host_add_sub_status(status, 1, reason); } @@ -330,8 +276,206 @@ static void chart_by_reason(struct by_reason *b, const char *id, const char *con rrdset_done(b->st); } +// -------------------------------------------------------------------------------------------------------------------- +// inbound aggregate + per-child charts (parent only) +// +// One read-only pass over the rrdhost dictionary computes BOTH the streaming_inbound aggregate +// (nodes per ephemerality x state) AND each child's per-instance charts on the parent's localhost. +// No global shared counters and no rrd_rdlock: dfe_start_reentrant() refcounts each host during +// iteration, and every per-host value read here is a single-writer relaxed atomic. + +typedef enum { + PULSE_INBOUND_LOCAL = 0, + PULSE_INBOUND_VIRTUAL, + PULSE_INBOUND_LOADING, + PULSE_INBOUND_ARCHIVED, + PULSE_INBOUND_OFFLINE, + PULSE_INBOUND_WAITING, + PULSE_INBOUND_REPLICATION_WAITING, + PULSE_INBOUND_REPLICATING, + PULSE_INBOUND_RUNNING, + + PULSE_INBOUND_MAX, +} PULSE_INBOUND_STATE; + +static PULSE_INBOUND_STATE pulse_inbound_state(PULSE_HOST_STATUS s) { + if(s & PULSE_HOST_STATUS_LOCAL) return PULSE_INBOUND_LOCAL; + if(s & PULSE_HOST_STATUS_VIRTUAL) return PULSE_INBOUND_VIRTUAL; + if(s & PULSE_HOST_STATUS_LOADING) return PULSE_INBOUND_LOADING; + if(s & PULSE_HOST_STATUS_ARCHIVED) return PULSE_INBOUND_ARCHIVED; + if(s & PULSE_HOST_STATUS_RCV_OFFLINE) return PULSE_INBOUND_OFFLINE; + if(s & PULSE_HOST_STATUS_RCV_WAITING) return PULSE_INBOUND_WAITING; + if(s & PULSE_HOST_STATUS_RCV_REPLICATION_WAIT) return PULSE_INBOUND_REPLICATION_WAITING; + if(s & PULSE_HOST_STATUS_RCV_REPLICATING) return PULSE_INBOUND_REPLICATING; + if(s & PULSE_HOST_STATUS_RCV_RUNNING) return PULSE_INBOUND_RUNNING; + return PULSE_INBOUND_MAX; +} + +typedef enum { + PULSE_OUTBOUND_OFFLINE = 0, + PULSE_OUTBOUND_CONNECTING, + PULSE_OUTBOUND_PENDING, + PULSE_OUTBOUND_WAITING, + PULSE_OUTBOUND_REPLICATING, + PULSE_OUTBOUND_RUNNING, + PULSE_OUTBOUND_NO_DST, + PULSE_OUTBOUND_NO_DST_FAILED, + + PULSE_OUTBOUND_MAX, +} PULSE_OUTBOUND_STATE; + +static PULSE_OUTBOUND_STATE pulse_outbound_state(PULSE_HOST_STATUS s) { + if(s & PULSE_HOST_STATUS_SND_OFFLINE) return PULSE_OUTBOUND_OFFLINE; + if(s & PULSE_HOST_STATUS_SND_CONNECTING) return PULSE_OUTBOUND_CONNECTING; + if(s & PULSE_HOST_STATUS_SND_PENDING) return PULSE_OUTBOUND_PENDING; + if(s & PULSE_HOST_STATUS_SND_WAITING) return PULSE_OUTBOUND_WAITING; + if(s & PULSE_HOST_STATUS_SND_REPLICATING) return PULSE_OUTBOUND_REPLICATING; + if(s & PULSE_HOST_STATUS_SND_RUNNING) return PULSE_OUTBOUND_RUNNING; + if(s & PULSE_HOST_STATUS_SND_NO_DST) return PULSE_OUTBOUND_NO_DST; + if(s & PULSE_HOST_STATUS_SND_NO_DST_FAILED) return PULSE_OUTBOUND_NO_DST_FAILED; + return PULSE_OUTBOUND_MAX; +} + +// per-child charts on the parent's localhost: one set per child, found-or-created by id on every +// pass. No registry/cache is kept: a host that leaves the dictionary stops being updated and the +// chart engine obsoletes its charts via the standard not-collected timer (rrdset_free_obsolete_time_s), +// exactly like every other collector. Identity + copied child labels are set once, at chart creation. + +static void pulse_child_chart_labels(RRDSET *st, RRDHOST *host) { + char node_id[UUID_STR_LEN] = ""; + if(!UUIDiszero(host->node_id)) + uuid_unparse_lower(host->node_id.uuid, node_id); + + char hops[16]; + snprintfz(hops, sizeof(hops), "%d", (int)rrdhost_ingestion_hops(host)); + + // copy the child's labels FIRST, then set the authoritative identity labels so a colliding + // child label cannot overwrite machine_guid/hostname/node_id/hops (rrdlabels_copy/add replace + // the value for a shared key) + rrdlabels_copy(st->rrdlabels, host->rrdlabels); + rrdlabels_add(st->rrdlabels, "machine_guid", host->machine_guid, RRDLABEL_SRC_AUTO); + rrdlabels_add(st->rrdlabels, "hostname", rrdhost_hostname(host), RRDLABEL_SRC_AUTO); + if(node_id[0]) + rrdlabels_add(st->rrdlabels, "node_id", node_id, RRDLABEL_SRC_AUTO); + rrdlabels_add(st->rrdlabels, "hops", hops, RRDLABEL_SRC_AUTO); +} + +static void pulse_child_charts_update(RRDHOST *host, PULSE_INBOUND_STATE state) { + char id[RRD_ID_LENGTH_MAX + 1]; + const char *guid = host->machine_guid; + + // re-apply labels + hops only when the host's labels changed (reconnect / mid-stream push), via a + // cheap version compare - so we don't re-copy every child's labels on every pass. hops can change + // when a child re-attaches via a different parent in a cluster; that reconnect bumps the version. + uint32_t lv = rrdlabels_version(host->rrdlabels); + bool refresh_labels = (lv != host->stream.rcv.status.labels_applied_version); + host->stream.rcv.status.labels_applied_version = lv; + + // --- traffic --- + snprintfz(id, sizeof(id), "streaming.in.traffic.%s", guid); + RRDSET *st_traffic = rrdset_create_localhost( + "netdata", id, NULL, "Streaming", "netdata.streaming.in.traffic", + "Inbound Streaming Traffic", "bytes/s", "netdata", "pulse", + 130160, localhost->rrd_update_every, RRDSET_TYPE_AREA); + if(unlikely(refresh_labels || !rrdlabels_exist(st_traffic->rrdlabels, "machine_guid"))) + pulse_child_chart_labels(st_traffic, host); + rrddim_set_by_pointer(st_traffic, rrddim_add(st_traffic, "in", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL), + (collected_number)single_writer_atomic_read(&host->stream.rcv.status.bytes_in)); + rrddim_set_by_pointer(st_traffic, rrddim_add(st_traffic, "out", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL), + (collected_number)single_writer_atomic_read(&host->stream.rcv.status.bytes_out)); + rrdset_done(st_traffic); + + // --- state (one-hot) --- + static const char *state_dim[PULSE_INBOUND_MAX] = { + [PULSE_INBOUND_ARCHIVED] = "archived", + [PULSE_INBOUND_OFFLINE] = "offline", + [PULSE_INBOUND_WAITING] = "waiting", + [PULSE_INBOUND_REPLICATION_WAITING] = "waiting replication", + [PULSE_INBOUND_REPLICATING] = "replicating", + [PULSE_INBOUND_RUNNING] = "running", + }; + snprintfz(id, sizeof(id), "streaming.in.state.%s", guid); + RRDSET *st_state = rrdset_create_localhost( + "netdata", id, NULL, "Streaming", "netdata.streaming.in.state", + "Inbound Streaming State", "state", "netdata", "pulse", + 130161, localhost->rrd_update_every, RRDSET_TYPE_LINE); + if(unlikely(refresh_labels || !rrdlabels_exist(st_state->rrdlabels, "machine_guid"))) + pulse_child_chart_labels(st_state, host); + for(size_t i = 0; i < PULSE_INBOUND_MAX ; i++) { + if(!state_dim[i]) continue; + rrddim_set_by_pointer(st_state, rrddim_add(st_state, state_dim[i], NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE), + (collected_number)(state == i ? 1 : 0)); + } + rrdset_done(st_state); + + // --- reconnects --- + snprintfz(id, sizeof(id), "streaming.in.reconnects.%s", guid); + RRDSET *st_reconnects = rrdset_create_localhost( + "netdata", id, NULL, "Streaming", "netdata.streaming.in.reconnects", + "Inbound Streaming Reconnects", "connects/s", "netdata", "pulse", + 130162, localhost->rrd_update_every, RRDSET_TYPE_LINE); + if(unlikely(refresh_labels || !rrdlabels_exist(st_reconnects->rrdlabels, "machine_guid"))) + pulse_child_chart_labels(st_reconnects, host); + rrddim_set_by_pointer(st_reconnects, rrddim_add(st_reconnects, "connections", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL), + (collected_number)__atomic_load_n(&host->stream.rcv.status.connections, __ATOMIC_RELAXED)); + rrdset_done(st_reconnects); + + // --- age (seconds in the current inbound state; reset to 0 on every state change) --- + snprintfz(id, sizeof(id), "streaming.in.age.%s", guid); + RRDSET *st_age = rrdset_create_localhost( + "netdata", id, NULL, "Streaming", "netdata.streaming.in.age", + "Inbound Streaming State Age", "seconds", "netdata", "pulse", + 130163, localhost->rrd_update_every, RRDSET_TYPE_LINE); + if(unlikely(refresh_labels || !rrdlabels_exist(st_age->rrdlabels, "machine_guid"))) + pulse_child_chart_labels(st_age, host); + time_t changed = __atomic_load_n(&host->stream.rcv.status.state_changed_s, __ATOMIC_RELAXED); + time_t now_s = now_realtime_sec(); + rrddim_set_by_pointer(st_age, rrddim_add(st_age, "age", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE), + (collected_number)((changed && now_s > changed) ? now_s - changed : 0)); + rrdset_done(st_age); +} + +// traverse all hosts once: tally BOTH the inbound and outbound aggregates from each host's combined +// pulse_state, and refresh the per-child charts. A host may carry both an inbound (receiver) and an +// outbound (sender) state simultaneously, so both are tallied independently. +static void pulse_parents_traverse(ssize_t inbound[2][PULSE_INBOUND_MAX], ssize_t outbound[PULSE_OUTBOUND_MAX]) { + // per-child charts only when streaming ingest is actually configured + bool do_children = stream_conf_is_parent(false); + + RRDHOST *host; + dfe_start_reentrant(rrdhost_root_index, host) { + PULSE_HOST_STATUS s = __atomic_load_n(&host->stream.pulse_state, __ATOMIC_RELAXED); + if(!s) + continue; // not classified yet + + PULSE_INBOUND_STATE in = pulse_inbound_state(s); + if(in < PULSE_INBOUND_MAX) { + size_t type = (s & PULSE_HOST_STATUS_EPHEMERAL) ? 1 : 0; + inbound[type][in]++; + + if(do_children && !rrdhost_is_local(host)) + pulse_child_charts_update(host, in); + } + + PULSE_OUTBOUND_STATE out = pulse_outbound_state(s); + if(out < PULSE_OUTBOUND_MAX) + outbound[out]++; + } + dfe_done(host); +} + void pulse_parents_do(bool extended) { - if(netdata_conf_is_parent()) { + bool is_parent = netdata_conf_is_parent(); + bool is_child = stream_conf_is_child(); + + // one read-only pass over the host dictionary: tally both streaming aggregates and refresh the + // per-child charts (no global counters, no rrd_rdlock) + ssize_t inbound[2][PULSE_INBOUND_MAX] = { 0 }; + ssize_t outbound[PULSE_OUTBOUND_MAX] = { 0 }; + if(is_parent || is_child) + pulse_parents_traverse(inbound, outbound); + + if(is_parent) { for(size_t idx = 0; idx < _countof(p.parent.type) ; idx++) { if (unlikely(!p.parent.type[idx].st_nodes)) { const char *type; @@ -373,15 +517,15 @@ void pulse_parents_do(bool extended) { p.parent.type[idx].rd_running = rrddim_add(p.parent.type[idx].st_nodes, "running", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); } - rrddim_set_by_pointer(p.parent.type[idx].st_nodes, p.parent.type[idx].rd_local, (collected_number)__atomic_load_n(&p.parent.type[idx].nodes_local, __ATOMIC_RELAXED)); - rrddim_set_by_pointer(p.parent.type[idx].st_nodes, p.parent.type[idx].rd_virtual, (collected_number)__atomic_load_n(&p.parent.type[idx].nodes_virtual, __ATOMIC_RELAXED)); - rrddim_set_by_pointer(p.parent.type[idx].st_nodes,p.parent.type[idx].rd_loading, (collected_number)__atomic_load_n(&p.parent.type[idx].nodes_loading, __ATOMIC_RELAXED)); - rrddim_set_by_pointer(p.parent.type[idx].st_nodes, p.parent.type[idx].rd_archived, (collected_number)__atomic_load_n(&p.parent.type[idx].nodes_archived, __ATOMIC_RELAXED)); - rrddim_set_by_pointer(p.parent.type[idx].st_nodes, p.parent.type[idx].rd_offline, (collected_number)__atomic_load_n(&p.parent.type[idx].nodes_offline, __ATOMIC_RELAXED)); - rrddim_set_by_pointer(p.parent.type[idx].st_nodes, p.parent.type[idx].rd_waiting, (collected_number)__atomic_load_n(&p.parent.type[idx].nodes_waiting, __ATOMIC_RELAXED)); - rrddim_set_by_pointer(p.parent.type[idx].st_nodes, p.parent.type[idx].rd_replication_waiting, (collected_number)__atomic_load_n(&p.parent.type[idx].nodes_replication_waiting, __ATOMIC_RELAXED)); - rrddim_set_by_pointer(p.parent.type[idx].st_nodes, p.parent.type[idx].rd_replicating, (collected_number)__atomic_load_n(&p.parent.type[idx].nodes_replicating, __ATOMIC_RELAXED)); - rrddim_set_by_pointer(p.parent.type[idx].st_nodes, p.parent.type[idx].rd_running, (collected_number)__atomic_load_n(&p.parent.type[idx].nodes_running, __ATOMIC_RELAXED)); + rrddim_set_by_pointer(p.parent.type[idx].st_nodes, p.parent.type[idx].rd_local, (collected_number)inbound[idx][PULSE_INBOUND_LOCAL]); + rrddim_set_by_pointer(p.parent.type[idx].st_nodes, p.parent.type[idx].rd_virtual, (collected_number)inbound[idx][PULSE_INBOUND_VIRTUAL]); + rrddim_set_by_pointer(p.parent.type[idx].st_nodes, p.parent.type[idx].rd_loading, (collected_number)inbound[idx][PULSE_INBOUND_LOADING]); + rrddim_set_by_pointer(p.parent.type[idx].st_nodes, p.parent.type[idx].rd_archived, (collected_number)inbound[idx][PULSE_INBOUND_ARCHIVED]); + rrddim_set_by_pointer(p.parent.type[idx].st_nodes, p.parent.type[idx].rd_offline, (collected_number)inbound[idx][PULSE_INBOUND_OFFLINE]); + rrddim_set_by_pointer(p.parent.type[idx].st_nodes, p.parent.type[idx].rd_waiting, (collected_number)inbound[idx][PULSE_INBOUND_WAITING]); + rrddim_set_by_pointer(p.parent.type[idx].st_nodes, p.parent.type[idx].rd_replication_waiting, (collected_number)inbound[idx][PULSE_INBOUND_REPLICATION_WAITING]); + rrddim_set_by_pointer(p.parent.type[idx].st_nodes, p.parent.type[idx].rd_replicating, (collected_number)inbound[idx][PULSE_INBOUND_REPLICATING]); + rrddim_set_by_pointer(p.parent.type[idx].st_nodes, p.parent.type[idx].rd_running, (collected_number)inbound[idx][PULSE_INBOUND_RUNNING]); rrdset_done(p.parent.type[idx].st_nodes); } @@ -404,7 +548,7 @@ void pulse_parents_do(bool extended) { } } - if(stream_conf_is_child()) { + if(is_child) { { static RRDSET *st_nodes = NULL; static RRDDIM *rd_pending = NULL; @@ -442,14 +586,14 @@ void pulse_parents_do(bool extended) { rd_no_dst_failed = rrddim_add(st_nodes, "failed", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); } - rrddim_set_by_pointer(st_nodes, rd_connecting, (collected_number)__atomic_load_n(&p.sender.nodes_connecting, __ATOMIC_RELAXED)); - rrddim_set_by_pointer(st_nodes, rd_pending, (collected_number)__atomic_load_n(&p.sender.nodes_pending, __ATOMIC_RELAXED)); - rrddim_set_by_pointer(st_nodes, rd_offline, (collected_number)__atomic_load_n(&p.sender.nodes_offline, __ATOMIC_RELAXED)); - rrddim_set_by_pointer(st_nodes, rd_waiting, (collected_number)__atomic_load_n(&p.sender.nodes_waiting, __ATOMIC_RELAXED)); - rrddim_set_by_pointer(st_nodes, rd_replicating, (collected_number)__atomic_load_n(&p.sender.nodes_replicating, __ATOMIC_RELAXED)); - rrddim_set_by_pointer(st_nodes, rd_running, (collected_number)__atomic_load_n(&p.sender.nodes_running, __ATOMIC_RELAXED)); - rrddim_set_by_pointer(st_nodes, rd_no_dst, (collected_number)__atomic_load_n(&p.sender.nodes_no_dst, __ATOMIC_RELAXED)); - rrddim_set_by_pointer(st_nodes, rd_no_dst_failed, (collected_number)__atomic_load_n(&p.sender.nodes_no_dst_failed, __ATOMIC_RELAXED)); + rrddim_set_by_pointer(st_nodes, rd_connecting, (collected_number)outbound[PULSE_OUTBOUND_CONNECTING]); + rrddim_set_by_pointer(st_nodes, rd_pending, (collected_number)outbound[PULSE_OUTBOUND_PENDING]); + rrddim_set_by_pointer(st_nodes, rd_offline, (collected_number)outbound[PULSE_OUTBOUND_OFFLINE]); + rrddim_set_by_pointer(st_nodes, rd_waiting, (collected_number)outbound[PULSE_OUTBOUND_WAITING]); + rrddim_set_by_pointer(st_nodes, rd_replicating, (collected_number)outbound[PULSE_OUTBOUND_REPLICATING]); + rrddim_set_by_pointer(st_nodes, rd_running, (collected_number)outbound[PULSE_OUTBOUND_RUNNING]); + rrddim_set_by_pointer(st_nodes, rd_no_dst, (collected_number)outbound[PULSE_OUTBOUND_NO_DST]); + rrddim_set_by_pointer(st_nodes, rd_no_dst_failed, (collected_number)outbound[PULSE_OUTBOUND_NO_DST_FAILED]); rrdset_done(st_nodes); } diff --git a/src/database/rrdhost.h b/src/database/rrdhost.h index e22b101f2dfc50..702285406c1a92 100644 --- a/src/database/rrdhost.h +++ b/src/database/rrdhost.h @@ -252,9 +252,40 @@ struct rrdhost { uint32_t charts; // the number of charts currently being replicated from a child NETDATA_DOUBLE percent; // the % of replication completion } replication; + + // single-writer (the receiver thread), relaxed-atomic; read lock-free by the + // pulse traversal. Cumulative over the host's lifetime, never reset. + uint64_t bytes_in; // raw socket bytes received from this child (all connections) + uint64_t bytes_out; // raw socket bytes sent to this child (all connections) + + // realtime second the current inbound (receiver) state was entered; reset by + // pulse_host_status() on every inbound state change, read by the pulse traversal + // to chart the age in the current state. + time_t state_changed_s; + + // "running latched": set true once the node first reaches RCV_RUNNING after a + // (re)connect. While set and the node is currently running, per-chart replication + // ripples (a new container/service/process group briefly replicating) do NOT flip + // the host status back to replicating. Reset on disconnect (RCV_OFFLINE) and when a + // replicating transition arrives while the node is NOT currently running (i.e. the + // initial catch-up of a fresh connection). Note: this masks ANY replication once + // running until the next disconnect, not only small ripples. Maintained by + // pulse_host_status() (relaxed atomic). + bool running_latched; + + // last host-label version applied to this host's per-child charts; the pulse + // traversal (single thread) re-applies labels + hops only when it changes, i.e. on + // reconnect / mid-stream label push. + uint32_t labels_applied_version; } status; } rcv; + // resolved combined PULSE_HOST_STATUS (basic|receiver|sender|ephemerality), maintained by + // pulse_host_status() lock-free (CAS) and read by the pulse traversal, which computes the + // streaming_inbound (basic|receiver bits) and streaming_outbound (sender bits) aggregates. + // Replaces the former global PHOST Judy + spinlock. + uint32_t pulse_state; + // --- configuration --- struct { diff --git a/src/libnetdata/atomics/atomics.h b/src/libnetdata/atomics/atomics.h index 1dbcf8343f4290..865c04597fc500 100644 --- a/src/libnetdata/atomics/atomics.h +++ b/src/libnetdata/atomics/atomics.h @@ -5,5 +5,6 @@ #include "atomic_flags.h" #include "refcount.h" +#include "single_writer.h" #endif //NETDATA_ATOMICS_H diff --git a/src/libnetdata/atomics/single_writer.h b/src/libnetdata/atomics/single_writer.h new file mode 100644 index 00000000000000..d1a6fa8340ea5c --- /dev/null +++ b/src/libnetdata/atomics/single_writer.h @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_SINGLE_WRITER_H +#define NETDATA_SINGLE_WRITER_H + +// Single-writer atomic counters. +// +// For counters that have EXACTLY ONE writer thread and are read by other +// threads (single-writer / multiple-readers). This is the natural shape of +// most per-host, per-chart and per-connection counters in Netdata. +// +// Why not __atomic_fetch_add(): an atomic read-modify-write is *indivisible*, +// so even at __ATOMIC_RELAXED it forces a lock-prefixed / cache-line-exclusive +// operation (x86 `lock xadd`; ARM ldxr/stxr or ldadd). RELAXED removes the +// ordering fence, NOT the atomicity cost. That cost only exists to defend +// against OTHER writers - which, by definition, a single-writer counter has +// none of. +// +// So the writer does a PLAIN read of its own value + add + a RELAXED store: +// - the writer reading its own counter cannot race (no other writer), so the +// read needs no atomic - and a plain read lets the compiler keep the running +// total in a register across increments; +// - only the STORE carries a cross-thread contract (readers must not see a torn +// value and the store must not be optimized away), so it is a relaxed store. +// On x86-64/aarch64 this is a plain load + add + plain store: no lock, no fence. +// +// Readers in OTHER threads MUST read with single_writer_atomic_read() (a relaxed +// atomic load): for them there IS a concurrent writer. +// +// MUST NOT be used when more than one thread writes the same counter. For +// genuinely multi-writer counters use __atomic_fetch_add(). On 32-bit platforms +// a 64-bit relaxed store is not a single instruction; this is the same trade-off +// Netdata's other 64-bit atomics already make. + +// add n to a single-writer counter (writer thread only) +#define single_writer_atomic_add(ptr, n) do { \ + __typeof__(ptr) _swptr = (ptr); \ + __atomic_store_n(_swptr, *_swptr + (n), __ATOMIC_RELAXED); \ + } while (0) + +// set a single-writer counter to v (writer thread only) +#define single_writer_atomic_set(ptr, v) __atomic_store_n((ptr), (v), __ATOMIC_RELAXED) + +// read a single-writer counter (any thread - readers MUST use this) +#define single_writer_atomic_read(ptr) __atomic_load_n((ptr), __ATOMIC_RELAXED) + +#endif //NETDATA_SINGLE_WRITER_H diff --git a/src/libnetdata/os/os.h b/src/libnetdata/os/os.h index f7a78abca3e7ce..4533c7d2623165 100644 --- a/src/libnetdata/os/os.h +++ b/src/libnetdata/os/os.h @@ -24,6 +24,7 @@ #include "uuid_generate.h" #include "setenv.h" #include "hostname.h" +#include "socket_egress_interface.h" #include "os-freebsd-wrappers.h" #include "os-macos-wrappers.h" #include "os-windows-wrappers.h" diff --git a/src/libnetdata/os/socket_egress_interface.c b/src/libnetdata/os/socket_egress_interface.c new file mode 100644 index 00000000000000..483c4cd6abb178 --- /dev/null +++ b/src/libnetdata/os/socket_egress_interface.c @@ -0,0 +1,166 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "libnetdata/libnetdata.h" + +#if defined(HAVE_NET_IF_H) && !defined(OS_WINDOWS) +#include +#include + +// pure address->interface match, separated from the syscalls so it can be unit-tested with +// synthetic inputs (see os_socket_egress_interface_unittest). Writes the first matching interface +// name into out (capacity out_len) and returns true on match. Handles plain IPv4, native IPv6, and +// the dual-stack case where getsockname reports the local IPv4 as ::ffff:a.b.c.d. +static bool match_egress_interface(const struct sockaddr_storage *local, + const struct ifaddrs *ifaddr, char *out, size_t out_len) { + for (const struct ifaddrs *ifa = ifaddr; ifa; ifa = ifa->ifa_next) { + if (!ifa->ifa_addr) + continue; + + bool match = false; + if (local->ss_family == AF_INET && ifa->ifa_addr->sa_family == AF_INET) + match = ((const struct sockaddr_in *)(const void *)local)->sin_addr.s_addr == + ((const struct sockaddr_in *)(void *)ifa->ifa_addr)->sin_addr.s_addr; + else if (local->ss_family == AF_INET6) { + const struct sockaddr_in6 *local6 = (const struct sockaddr_in6 *)(const void *)local; + if (IN6_IS_ADDR_V4MAPPED(&local6->sin6_addr) && ifa->ifa_addr->sa_family == AF_INET) { + // a dual-stack socket reports its local IPv4 as ::ffff:a.b.c.d - match it against the + // interface's plain AF_INET address + struct in_addr mapped4; + memcpy(&mapped4, &local6->sin6_addr.s6_addr[12], sizeof(mapped4)); + match = mapped4.s_addr == + ((const struct sockaddr_in *)(void *)ifa->ifa_addr)->sin_addr.s_addr; + } + else if (ifa->ifa_addr->sa_family == AF_INET6) { + const struct sockaddr_in6 *ifa6 = (const struct sockaddr_in6 *)(void *)ifa->ifa_addr; + match = memcmp(&local6->sin6_addr, &ifa6->sin6_addr, sizeof(struct in6_addr)) == 0; + // a link-local address (fe80::/10) is unique only per interface; the same address can + // exist on several NICs, so disambiguate by scope id (the interface index) or we could + // match the wrong interface + if (match && IN6_IS_ADDR_LINKLOCAL(&local6->sin6_addr)) + match = local6->sin6_scope_id == ifa6->sin6_scope_id; + } + } + + if (match) { + strncpyz(out, ifa->ifa_name, out_len - 1); + return true; + } + } + return false; +} + +bool os_socket_egress_interface(int fd, char *out, size_t out_len) { + if (fd < 0 || !out || out_len == 0) + return false; + + out[0] = '\0'; + + struct sockaddr_storage local; + socklen_t local_len = sizeof(local); + if (getsockname(fd, (struct sockaddr *)&local, &local_len) != 0) + return false; + + struct ifaddrs *ifaddr; + if (getifaddrs(&ifaddr) != 0) + return false; + + bool found = match_egress_interface(&local, ifaddr, out, out_len); + freeifaddrs(ifaddr); + return found; +} + +int os_socket_egress_interface_unittest(void) { + fprintf(stderr, "%s() running...\n", __FUNCTION__); + + struct sockaddr_in lo4 = { .sin_family = AF_INET }; + inet_pton(AF_INET, "127.0.0.1", &lo4.sin_addr); + struct sockaddr_in eth0_4 = { .sin_family = AF_INET }; + inet_pton(AF_INET, "10.20.4.5", ð0_4.sin_addr); + struct sockaddr_in6 eth1_6 = { .sin6_family = AF_INET6 }; + inet_pton(AF_INET6, "2001:db8::5", ð1_6.sin6_addr); + + // the SAME link-local address on two interfaces, distinguished only by scope id (interface index) + struct sockaddr_in6 eth0_ll = { .sin6_family = AF_INET6, .sin6_scope_id = 2 }; + inet_pton(AF_INET6, "fe80::1", ð0_ll.sin6_addr); + struct sockaddr_in6 eth1_ll = { .sin6_family = AF_INET6, .sin6_scope_id = 3 }; + inet_pton(AF_INET6, "fe80::1", ð1_ll.sin6_addr); + + // list order puts eth0's link-local (scope 2) BEFORE eth1's (scope 3), so a pure address match + // would wrongly pick eth0 for an eth1-scoped local - the scope id must decide + struct ifaddrs if_eth1ll = { .ifa_next = NULL, .ifa_name = (char *)"eth1", .ifa_addr = (struct sockaddr *)ð1_ll }; + struct ifaddrs if_eth0ll = { .ifa_next = &if_eth1ll, .ifa_name = (char *)"eth0", .ifa_addr = (struct sockaddr *)ð0_ll }; + struct ifaddrs if_eth1 = { .ifa_next = &if_eth0ll, .ifa_name = (char *)"eth1", .ifa_addr = (struct sockaddr *)ð1_6 }; + struct ifaddrs if_eth0 = { .ifa_next = &if_eth1, .ifa_name = (char *)"eth0", .ifa_addr = (struct sockaddr *)ð0_4 }; + struct ifaddrs if_lo = { .ifa_next = &if_eth0, .ifa_name = (char *)"lo", .ifa_addr = (struct sockaddr *)&lo4 }; + + static const struct { + const char *name; + int family; + const char *addr; // an IPv4 string when mapped==true + bool mapped; // build ::ffff:addr (a dual-stack local) + uint32_t scope; // sin6_scope_id for link-local IPv6 locals + const char *expect; // NULL = expect no match + } cases[] = { + { "ipv4 direct", AF_INET, "10.20.4.5", false, 0, "eth0" }, + { "ipv6 native", AF_INET6, "2001:db8::5", false, 0, "eth1" }, + { "ipv4-mapped ipv6", AF_INET6, "10.20.4.5", true, 0, "eth0" }, + { "link-local scope2", AF_INET6, "fe80::1", false, 2, "eth0" }, + { "link-local scope3", AF_INET6, "fe80::1", false, 3, "eth1" }, + { "no match ipv4", AF_INET, "192.0.2.1", false, 0, NULL }, + { "no match ipv6", AF_INET6, "2001:db8::99", false, 0, NULL }, + }; + + int errors = 0; + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + struct sockaddr_storage local; + memset(&local, 0, sizeof(local)); + + if (cases[i].family == AF_INET) { + struct sockaddr_in *l = (struct sockaddr_in *)&local; + l->sin_family = AF_INET; + inet_pton(AF_INET, cases[i].addr, &l->sin_addr); + } + else { + struct sockaddr_in6 *l = (struct sockaddr_in6 *)&local; + l->sin6_family = AF_INET6; + if (cases[i].mapped) { + struct in_addr v4; + inet_pton(AF_INET, cases[i].addr, &v4); + l->sin6_addr.s6_addr[10] = 0xff; + l->sin6_addr.s6_addr[11] = 0xff; + memcpy(&l->sin6_addr.s6_addr[12], &v4, sizeof(v4)); + } + else { + inet_pton(AF_INET6, cases[i].addr, &l->sin6_addr); + l->sin6_scope_id = cases[i].scope; + } + } + + char iface[OS_IFNAME_MAX] = ""; + bool found = match_egress_interface(&local, &if_lo, iface, sizeof(iface)); + + bool ok = cases[i].expect ? (found && strcmp(iface, cases[i].expect) == 0) : !found; + if (!ok) { + fprintf(stderr, " FAILED %s: expected %s, got %s\n", cases[i].name, + cases[i].expect ? cases[i].expect : "(none)", found ? iface : "(none)"); + errors++; + } + } + + fprintf(stderr, "%s() %s\n", __FUNCTION__, errors ? "FAILED" : "passed"); + return errors; +} + +#else // unsupported platform (no getifaddrs / Windows) + +bool os_socket_egress_interface(int fd __maybe_unused, char *out, size_t out_len) { + if (out && out_len) + out[0] = '\0'; + return false; +} + +int os_socket_egress_interface_unittest(void) { + return 0; +} + +#endif diff --git a/src/libnetdata/os/socket_egress_interface.h b/src/libnetdata/os/socket_egress_interface.h new file mode 100644 index 00000000000000..e912aceaa5fa4c --- /dev/null +++ b/src/libnetdata/os/socket_egress_interface.h @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_SOCKET_EGRESS_INTERFACE_H +#define NETDATA_SOCKET_EGRESS_INTERFACE_H + +#include "libnetdata/libnetdata.h" + +// portable upper bound for an interface name (IF_NAMESIZE is 16 on Linux/BSD/macOS); kept here so +// callers do not need , which is not available on every platform. +#define OS_IFNAME_MAX 32 + +/** + * Find the local network interface a connected socket egresses on. + * + * Resolves the socket's local address (getsockname) and matches it against the system interface + * addresses (getifaddrs), handling plain IPv4, native IPv6, and the dual-stack case where the local + * IPv4 is reported as an IPv6-mapped address (::ffff:a.b.c.d). + * + * @param fd a connected socket file descriptor + * @param out buffer receiving the null-terminated interface name (set empty on failure) + * @param out_len capacity of out (use OS_IFNAME_MAX) + * @return true if an interface was found, false on unsupported platforms, errors, or no match + */ +bool os_socket_egress_interface(int fd, char *out, size_t out_len); + +// unit test for the address->interface matching (incl. the IPv6-mapped-IPv4 dual-stack case) +int os_socket_egress_interface_unittest(void); + +#endif //NETDATA_SOCKET_EGRESS_INTERFACE_H diff --git a/src/streaming/stream-receiver.c b/src/streaming/stream-receiver.c index ca9396eb399ad3..ba0a7b579f8c07 100644 --- a/src/streaming/stream-receiver.c +++ b/src/streaming/stream-receiver.c @@ -154,6 +154,8 @@ static ssize_t receiver_read_uncompressed(struct receiver_state *r) { r->thread.uncompressed.read_len += bytes; r->thread.uncompressed.read_buffer[r->thread.uncompressed.read_len] = '\0'; r->thread.bytes_received += bytes; + if(likely(r->host)) + single_writer_atomic_add(&r->host->stream.rcv.status.bytes_in, (uint64_t)bytes); pulse_stream_received_bytes(bytes); } @@ -273,6 +275,8 @@ static ssize_t receiver_read_compressed(struct receiver_state *r) { if(bytes > 0) { r->thread.compressed.used += bytes; r->thread.bytes_received += bytes; + if(likely(r->host)) + single_writer_atomic_add(&r->host->stream.rcv.status.bytes_in, (uint64_t)bytes); worker_set_metric(WORKER_RECEIVER_JOB_BYTES_READ, (NETDATA_DOUBLE)bytes); pulse_stream_received_bytes(bytes); } @@ -725,6 +729,8 @@ bool stream_receiver_send_data(struct stream_thread *sth, struct receiver_state pulse_stream_sent_bytes(rc); rpt->thread.last_traffic_ut = now_ut; stream_circular_buffer_del_unsafe(scb, rc, now_ut); + if(likely(rpt->host)) + single_writer_atomic_add(&rpt->host->stream.rcv.status.bytes_out, (uint64_t)rc); if (!stats->bytes_outstanding) { rpt->thread.wanted = ND_POLL_READ; if (!nd_poll_upd(sth->run.ndpl, rpt->sock.fd, rpt->thread.wanted)) @@ -1180,7 +1186,9 @@ RRDHOST_SET_RECEIVER_RESULT rrdhost_set_receiver(RRDHOST *host, struct receiver_ rrdhost_flag_clear(host, RRDHOST_FLAG_ORPHAN); rrdhost_set_health_evloop_iteration(host); - host->stream.rcv.status.connections++; + // atomic: read lock-free by the pulse traversal (this write is under receiver_lock, but the + // read side has no lock; the connect path is rare so the atomic RMW cost is irrelevant) + __atomic_add_fetch(&host->stream.rcv.status.connections, 1, __ATOMIC_RELAXED); streaming_receiver_connected(); host->receiver = rpt; diff --git a/src/streaming/stream-sender.c b/src/streaming/stream-sender.c index d002e989fc3094..405bcd107bc547 100644 --- a/src/streaming/stream-sender.c +++ b/src/streaming/stream-sender.c @@ -141,6 +141,19 @@ static void stream_sender_on_connect_and_disconnect(struct sender_state *s) { stream_sender_unlock(s); } +// Record the interface the stream actually egresses on as the host's _net_default_iface label, so +// the parent sees the real uplink. The OS-specific lookup (getsockname + getifaddrs match) lives in +// libnetdata/os/socket_egress_interface; here we only stamp the label. This is correct under policy +// routing / multi-WAN, where the main routing table's default route can point at a different +// interface than the stream uses. Runs once per (re)connect and rides out with the first host-labels +// push in on_ready_to_dispatch(); on failover the connection breaks and reconnects over the new +// interface, so it re-evaluates automatically. +static void stream_sender_update_egress_iface_label(struct sender_state *s) { + char iface[OS_IFNAME_MAX]; + if (os_socket_egress_interface(s->sock.fd, iface, sizeof(iface)) && iface[0]) + rrdlabels_add(s->host->rrdlabels, "_net_default_iface", iface, RRDLABEL_SRC_AUTO); +} + void stream_sender_on_connect(struct sender_state *s) { nd_log(NDLS_DAEMON, NDLP_DEBUG, "STREAM SND [%s]: running on-connect hooks...", @@ -152,6 +165,9 @@ void stream_sender_on_connect(struct sender_state *s) { s->thread.last_traffic_ut = now_monotonic_usec(); + // record the real uplink interface before the first host-labels push + stream_sender_update_egress_iface_label(s); + freez(s->thread.rbuf.b); s->thread.rbuf.size = PLUGINSD_LINE_MAX + 1; s->thread.rbuf.b = mallocz(s->thread.rbuf.size); From c7d67a04b0e78e0b9f25dacae0d47476add90ea1 Mon Sep 17 00:00:00 2001 From: Ilya Mashchenko Date: Sat, 20 Jun 2026 01:33:40 +0300 Subject: [PATCH 2/2] feat(go.d/snmp_topology): add BGP adjacency links (#22790) --- .../skills/project-create-topology/SKILL.md | 18 +- .../topology-modes-correlation-aggregation.md | 28 +- .../collector/snmp/ddsnmp/profile_catalog.go | 102 ++++++ .../snmp/ddsnmp/profile_catalog_test.go | 328 ++++++++++++++++- .../go.d/collector/snmp_topology/collector.go | 3 +- .../snmp_topology/func_topology_test.go | 145 +++++++- .../snmp_topology/func_topology_v1.go | 5 + .../func_topology_v1_actor_details.go | 4 + .../func_topology_v1_bgp_peers.go | 69 ++++ .../snmp_topology/func_topology_v1_links.go | 109 ++++-- .../func_topology_v1_presentation.go | 29 ++ .../func_topology_v1_table_types.go | 57 +++ .../snmp_topology/profile_filter_test.go | 61 ++++ .../snmp_topology/topology_bgp_links.go | 335 ++++++++++++++++++ .../snmp_topology/topology_bgp_links_test.go | 331 +++++++++++++++++ .../snmp_topology/topology_bgp_peers.go | 196 ++++++++++ .../snmp_topology/topology_bgp_peers_test.go | 226 ++++++++++++ .../collector/snmp_topology/topology_cache.go | 19 + .../snmp_topology/topology_cache_lifecycle.go | 2 + .../snmp_topology/topology_cache_test.go | 67 ++++ .../snmp_topology/topology_registry_build.go | 2 + .../topology_registry_observations.go | 4 + .../topology_registry_snapshot.go | 3 + .../snmp_topology/topology_registry_test.go | 133 +++++++ .../snmp_topology/topology_shape_stats.go | 1 + .../topology_shape_stats_test.go | 16 +- 26 files changed, 2245 insertions(+), 48 deletions(-) create mode 100644 src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_bgp_peers.go create mode 100644 src/go/plugin/go.d/collector/snmp_topology/topology_bgp_links.go create mode 100644 src/go/plugin/go.d/collector/snmp_topology/topology_bgp_links_test.go create mode 100644 src/go/plugin/go.d/collector/snmp_topology/topology_bgp_peers.go create mode 100644 src/go/plugin/go.d/collector/snmp_topology/topology_bgp_peers_test.go diff --git a/.agents/skills/project-create-topology/SKILL.md b/.agents/skills/project-create-topology/SKILL.md index 72ec0512150268..2a2b07176d2bcc 100644 --- a/.agents/skills/project-create-topology/SKILL.md +++ b/.agents/skills/project-create-topology/SKILL.md @@ -430,9 +430,21 @@ For SNMP managed device actor modals: logical adjacency. It must not be presented as discovery, physical, L2, or port-neighbor evidence. - Preserve protocol-neighbor diagnostics that do not become graph links in - actor-owned detail tables, such as `actor_ospf_neighbors`. Non-full or - unresolved OSPF neighbors should remain visible there without creating loose - router/IP actors. + actor-owned detail tables, such as `actor_ospf_neighbors` and + `actor_bgp_peers`. Non-full, non-established, or unresolved protocol + neighbors should remain visible there without creating loose router/IP + actors. +- `bgp_adjacency` links represent established BGP control-plane adjacency + between resolved managed SNMP device actors. They must use explicit BGP + link/evidence types, carry `semantic_role: control`, and must not be + presented as discovery, physical, L2, or port-neighbor links. +- Deduplicate BGP adjacency by managed actor pair plus routing instance. Keep + BGP identifiers, ASNs, and endpoint IPs as evidence/display fields, but do not + make them the primary graph identity because vendor profiles expose those + fields asymmetrically. Parallel sessions between the same managed actor pair + in the same routing instance should remain actor-owned peer detail rows under + one compact graph relationship unless a future producer contract explicitly + adds a more detailed BGP graph mode. - Keep generic graph-link `Links` sections only for endpoint, segment, or custom actors that do not own port inventory. - Build link endpoint port labels only from real port fields: `port_name`, diff --git a/.agents/sow/specs/topology-modes-correlation-aggregation.md b/.agents/sow/specs/topology-modes-correlation-aggregation.md index f65f2616a952a5..28dbc7acdfa1fb 100644 --- a/.agents/sow/specs/topology-modes-correlation-aggregation.md +++ b/.agents/sow/specs/topology-modes-correlation-aggregation.md @@ -525,6 +525,20 @@ Current SNMP OSPF topology scope is OSPFv2 non-virtual neighbors from `ospfNbrTable`; OSPFv3 and OSPF virtual-neighbor tables are future scope unless a later topology feature explicitly adds their distinct semantics. +`bgp_adjacency` represents BGP control-plane adjacency between two resolved +managed SNMP device actors. It is a logical routing-protocol relationship, not +physical, L2, discovery, or port-neighbor evidence. The producer emits graph +links only for established BGP peers with resolved local and remote managed +actors. Unresolved or non-established BGP peer rows remain diagnostic +actor-owned detail rows and must not create loose router/IP graph actors. +BGP adjacency graph identity should use the managed actor pair plus routing +instance. BGP identifiers, ASNs, and endpoint IPs are useful evidence and +display fields, but they must not be the primary graph deduplication key because +SNMP BGP profiles expose those fields inconsistently across vendors. Parallel +sessions between the same managed actor pair in the same routing instance should +remain actor-owned peer detail rows under one compact graph relationship unless +a future producer contract explicitly adds a more detailed BGP graph mode. + When `ospf_adjacency` and `l3_subnet` describe the same resolved actor pair and endpoint/subnet relationship, OSPF is the stronger protocol-specific signal for the graph. The producer may suppress the matching `l3_subnet` graph link while @@ -586,6 +600,14 @@ evidence. Actor-owned OSPF neighbor detail rows should remain attached to the local device actor and are not loose-side graph materialization input unless a future producer contract explicitly defines such a policy. +For `bgp_adjacency`, aggregation MUST preserve the explicit `bgp_adjacency` +link and evidence type, including its `semantic_role: control`. Replacement may +rewire endpoint actors to stronger managed device actors, but it MUST NOT +reinterpret BGP adjacency as discovery, physical, L2, or port-neighbor +evidence. Actor-owned BGP peer detail rows should remain attached to the local +device actor and are not loose-side graph materialization input unless a future +producer contract explicitly defines such a policy. + ### UI SNMP The UI should not expose a detailed/aggregated toggle for SNMP unless the @@ -605,9 +627,9 @@ Device modals should remain port-centric: - L3 adjacency information: derived from typed `l3_subnet` relationship evidence, not from `actor_port_links`. - routing protocol neighbor information: derived from typed actor-owned detail - rows such as `actor_ospf_neighbors`, not from `actor_port_links`. The modal - may show unresolved or non-full protocol neighbors as diagnostics without - creating graph links. + rows such as `actor_ospf_neighbors` and `actor_bgp_peers`, not from + `actor_port_links`. The modal may show unresolved or non-established protocol + neighbors as diagnostics without creating graph links. ## Streaming diff --git a/src/go/plugin/go.d/collector/snmp/ddsnmp/profile_catalog.go b/src/go/plugin/go.d/collector/snmp/ddsnmp/profile_catalog.go index eeb37ee9bd4141..141b0ad5c20a02 100644 --- a/src/go/plugin/go.d/collector/snmp/ddsnmp/profile_catalog.go +++ b/src/go/plugin/go.d/collector/snmp/ddsnmp/profile_catalog.go @@ -136,6 +136,108 @@ func (v ProjectedView) FilterByKind(kinds map[ddprofiledefinition.TopologyKind]b })} } +func (v ProjectedView) FilterBGPByKind(kinds map[ddprofiledefinition.BGPRowKind]bool) ProjectedView { + for _, prof := range v.profiles { + if prof == nil || prof.Definition == nil { + continue + } + prof.Definition.BGP = slices.DeleteFunc(prof.Definition.BGP, func(row ddprofiledefinition.BGPConfig) bool { + return !kinds[row.Kind] + }) + } + return ProjectedView{profiles: slices.DeleteFunc(v.profiles, func(prof *Profile) bool { + return prof == nil || prof.Definition == nil || + (len(prof.Definition.Topology) == 0 && len(prof.Definition.Metrics) == 0 && len(prof.Definition.BGP) == 0) + })} +} + +// FilterBGPToTopologyPeers keeps BGP peer rows and prunes them to fields used +// by SNMP topology, preserving one fallback row-anchor category when needed. +func (v ProjectedView) FilterBGPToTopologyPeers() ProjectedView { + for _, prof := range v.profiles { + if prof == nil || prof.Definition == nil { + continue + } + prof.Definition.BGP = slices.DeleteFunc(prof.Definition.BGP, func(row ddprofiledefinition.BGPConfig) bool { + return row.Kind != ddprofiledefinition.BGPRowKindPeer + }) + for i := range prof.Definition.BGP { + pruneBGPConfigToTopologyFields(&prof.Definition.BGP[i]) + } + prof.Definition.BGP = slices.DeleteFunc(prof.Definition.BGP, func(row ddprofiledefinition.BGPConfig) bool { + return !bgpConfigHasSignal(row) + }) + } + return ProjectedView{profiles: slices.DeleteFunc(v.profiles, func(prof *Profile) bool { + return prof == nil || prof.Definition == nil || + (len(prof.Definition.Topology) == 0 && len(prof.Definition.Metrics) == 0 && len(prof.Definition.BGP) == 0) + })} +} + +func pruneBGPConfigToTopologyFields(row *ddprofiledefinition.BGPConfig) { + original := row.Clone() + row.Previous = ddprofiledefinition.BGPStateConfig{} + row.Traffic = ddprofiledefinition.BGPTrafficConfig{} + row.Transitions = ddprofiledefinition.BGPTransitionsConfig{} + row.Timers = ddprofiledefinition.BGPTimersConfig{} + row.LastError = ddprofiledefinition.BGPLastErrorConfig{} + row.LastNotify = ddprofiledefinition.BGPLastNotifyConfig{} + row.Reasons = ddprofiledefinition.BGPReasonsConfig{} + row.Restart = ddprofiledefinition.BGPGracefulRestartConfig{} + row.Routes = ddprofiledefinition.BGPRoutesConfig{} + row.RouteLimits = ddprofiledefinition.BGPRouteLimitsConfig{} + row.Device = ddprofiledefinition.BGPDeviceCountsConfig{} + row.StaticTags = nil + row.MetricTags = nil + + restoreBGPTopologyRowAnchor(row, original) +} + +func bgpConfigHasTopologySignal(row ddprofiledefinition.BGPConfig) bool { + return row.Admin.Enabled.IsSet() || + row.State.BGPValueConfig.IsSet() || + row.Connection.EstablishedUptime.IsSet() || + row.Connection.LastReceivedUpdateAge.IsSet() +} + +func restoreBGPTopologyRowAnchor(row *ddprofiledefinition.BGPConfig, original ddprofiledefinition.BGPConfig) { + if bgpConfigHasTopologySignal(original) { + return + } + switch { + case bgpConfigHasSignal(ddprofiledefinition.BGPConfig{Previous: original.Previous}): + row.Previous = original.Previous + case bgpConfigHasSignal(ddprofiledefinition.BGPConfig{Transitions: original.Transitions}): + row.Transitions = original.Transitions + case bgpConfigHasSignal(ddprofiledefinition.BGPConfig{Traffic: original.Traffic}): + row.Traffic = original.Traffic + case bgpConfigHasSignal(ddprofiledefinition.BGPConfig{Timers: original.Timers}): + row.Timers = original.Timers + case bgpConfigHasSignal(ddprofiledefinition.BGPConfig{LastError: original.LastError}): + row.LastError = original.LastError + case bgpConfigHasSignal(ddprofiledefinition.BGPConfig{LastNotify: original.LastNotify}): + row.LastNotify = original.LastNotify + case bgpConfigHasSignal(ddprofiledefinition.BGPConfig{Reasons: original.Reasons}): + row.Reasons = original.Reasons + case bgpConfigHasSignal(ddprofiledefinition.BGPConfig{Restart: original.Restart}): + row.Restart = original.Restart + case bgpConfigHasSignal(ddprofiledefinition.BGPConfig{Routes: original.Routes}): + row.Routes = original.Routes + case bgpConfigHasSignal(ddprofiledefinition.BGPConfig{RouteLimits: original.RouteLimits}): + row.RouteLimits = original.RouteLimits + case bgpConfigHasSignal(ddprofiledefinition.BGPConfig{Device: original.Device}): + row.Device = original.Device + } +} + +func bgpConfigHasSignal(row ddprofiledefinition.BGPConfig) bool { + hasSignal := false + ddprofiledefinition.ForEachBGPSignalValue(row, func(_ string, _ ddprofiledefinition.BGPValueConfig) { + hasSignal = true + }) + return hasSignal +} + func selectMatchedProfiles(available []*Profile, sysObjID, sysDescr string) []*Profile { matchedOIDs := make(map[*Profile]string) var selected []*Profile diff --git a/src/go/plugin/go.d/collector/snmp/ddsnmp/profile_catalog_test.go b/src/go/plugin/go.d/collector/snmp/ddsnmp/profile_catalog_test.go index 929743e9afd93d..5f04fd82bc63e1 100644 --- a/src/go/plugin/go.d/collector/snmp/ddsnmp/profile_catalog_test.go +++ b/src/go/plugin/go.d/collector/snmp/ddsnmp/profile_catalog_test.go @@ -3,6 +3,7 @@ package ddsnmp import ( + "slices" "testing" "github.com/stretchr/testify/assert" @@ -97,7 +98,7 @@ func TestResolvedProfileSetProject_SeparatesMetricsAndTopology(t *testing.T) { }, "bgp_projection": { consumer: ConsumerBGP, - bgp: 1, + bgp: 3, metadataField: "bgp_vendor", metricTag: "bgp_model", sysobjectID: "sysobjectid_bgp_vendor", @@ -161,7 +162,7 @@ func TestResolvedProfileSetProject_MetricsAndBGP(t *testing.T) { require.Len(t, profiles, 1) def := profiles[0].Definition require.Len(t, def.Metrics, 2) - require.Len(t, def.BGP, 1) + require.Len(t, def.BGP, 3) require.Len(t, def.VirtualMetrics, 1) require.Empty(t, def.Topology) require.Empty(t, def.Licensing) @@ -257,6 +258,28 @@ func TestResolvedProfileSetProject_DoesNotShareMutableProjectionState(t *testing assert.NotEqual(t, "mutated", fresh[0].Definition.Metadata["device"].Fields["lldp_loc_sys_name"].Value) } +func TestProjectedViewFilterBGPToTopologyPeersDoesNotShareMutableProjectionState(t *testing.T) { + resolved := &ResolvedProfileSet{profiles: []*Profile{projectionTestProfile()}} + + pruned := resolved.Project(ConsumerTopology, ConsumerBGP).FilterBGPToTopologyPeers().Profiles() + full := resolved.Project(ConsumerBGP).Profiles() + + require.Len(t, pruned, 1) + require.Len(t, full, 1) + require.Len(t, pruned[0].Definition.BGP, 1) + require.Len(t, full[0].Definition.BGP, 3) + + assert.Equal(t, ddprofiledefinition.BGPTrafficConfig{}, pruned[0].Definition.BGP[0].Traffic) + assert.NotEqual(t, ddprofiledefinition.BGPTrafficConfig{}, full[0].Definition.BGP[0].Traffic) + assert.NotEmpty(t, full[0].Definition.BGP[0].MetricTags) + + pruned[0].Definition.BGP[0].Identity.Neighbor.Value = "mutated" + fresh := resolved.Project(ConsumerTopology, ConsumerBGP).FilterBGPToTopologyPeers().Profiles() + require.Len(t, fresh, 1) + require.Len(t, fresh[0].Definition.BGP, 1) + assert.Equal(t, "192.0.2.1", fresh[0].Definition.BGP[0].Identity.Neighbor.Value) +} + func TestProjectedViewFilterByKind(t *testing.T) { resolved := &ResolvedProfileSet{profiles: []*Profile{projectionTestProfile()}} @@ -274,6 +297,219 @@ func TestProjectedViewFilterByKind(t *testing.T) { assert.Empty(t, unfiltered[0].Definition.Metrics) } +func TestProjectedViewFilterBGPByKind(t *testing.T) { + resolved := &ResolvedProfileSet{profiles: []*Profile{projectionTestProfile()}} + + view := resolved.Project(ConsumerTopology, ConsumerBGP).FilterBGPByKind(map[ddprofiledefinition.BGPRowKind]bool{ + ddprofiledefinition.BGPRowKindPeer: true, + }).Profiles() + + require.Len(t, view, 1) + require.Len(t, view[0].Definition.Topology, 2) + require.Len(t, view[0].Definition.BGP, 1) + assert.Equal(t, ddprofiledefinition.BGPRowKindPeer, view[0].Definition.BGP[0].Kind) + assert.Empty(t, view[0].Definition.Metrics) + + unfiltered := resolved.Project(ConsumerTopology, ConsumerBGP).Profiles() + require.Len(t, unfiltered[0].Definition.Topology, 2) + require.Len(t, unfiltered[0].Definition.BGP, 3) +} + +func TestProjectedViewFilterBGPToTopologyPeers(t *testing.T) { + resolved := &ResolvedProfileSet{profiles: []*Profile{projectionTestProfile()}} + + view := resolved.Project(ConsumerTopology, ConsumerBGP). + FilterBGPToTopologyPeers(). + Profiles() + + require.Len(t, view, 1) + require.Len(t, view[0].Definition.BGP, 1) + row := view[0].Definition.BGP[0] + + assert.Equal(t, ddprofiledefinition.BGPRowKindPeer, row.Kind) + assert.Equal(t, "192.0.2.1", row.Identity.Neighbor.Value) + assert.Equal(t, "192.0.2.2", row.Descriptors.LocalAddress.Value) + assert.Equal(t, "start", row.Admin.Enabled.Value) + assert.Equal(t, "established", row.State.Value) + assert.Equal(t, "1.3.6.1.2.1.15.3.1.16", row.Connection.EstablishedUptime.Symbol.OID) + assert.Equal(t, "1.3.6.1.2.1.15.3.1.24", row.Connection.LastReceivedUpdateAge.Symbol.OID) + + assert.Equal(t, ddprofiledefinition.BGPStateConfig{}, row.Previous) + assert.Equal(t, ddprofiledefinition.BGPTrafficConfig{}, row.Traffic) + assert.Equal(t, ddprofiledefinition.BGPTransitionsConfig{}, row.Transitions) + assert.Equal(t, ddprofiledefinition.BGPTimersConfig{}, row.Timers) + assert.Equal(t, ddprofiledefinition.BGPLastErrorConfig{}, row.LastError) + assert.Equal(t, ddprofiledefinition.BGPLastNotifyConfig{}, row.LastNotify) + assert.Equal(t, ddprofiledefinition.BGPReasonsConfig{}, row.Reasons) + assert.Equal(t, ddprofiledefinition.BGPGracefulRestartConfig{}, row.Restart) + assert.Equal(t, ddprofiledefinition.BGPRoutesConfig{}, row.Routes) + assert.Equal(t, ddprofiledefinition.BGPRouteLimitsConfig{}, row.RouteLimits) + assert.Equal(t, ddprofiledefinition.BGPDeviceCountsConfig{}, row.Device) + assert.Empty(t, row.StaticTags) + assert.Empty(t, row.MetricTags) + + assert.Empty(t, bgpSignalPathsWithPrefix(row, "traffic.")) + assert.Empty(t, bgpSignalPathsWithPrefix(row, "timers.")) + assert.Empty(t, bgpSignalPathsWithPrefix(row, "last_error.")) + assert.Empty(t, bgpSignalPathsWithPrefix(row, "routes.")) + + unfiltered := resolved.Project(ConsumerTopology, ConsumerBGP). + FilterBGPByKind(map[ddprofiledefinition.BGPRowKind]bool{ddprofiledefinition.BGPRowKindPeer: true}). + Profiles() + require.Len(t, unfiltered[0].Definition.BGP, 1) + assert.NotEqual(t, ddprofiledefinition.BGPTrafficConfig{}, unfiltered[0].Definition.BGP[0].Traffic) + assert.NotEmpty(t, unfiltered[0].Definition.BGP[0].MetricTags) +} + +func TestProjectedViewFilterBGPToTopologyPeersKeepsSingleAnchorWithoutTopologySignal(t *testing.T) { + resolved := &ResolvedProfileSet{profiles: []*Profile{{ + SourceFile: "projection.yaml", + Definition: &ddprofiledefinition.ProfileDefinition{ + BGP: []ddprofiledefinition.BGPConfig{ + { + ID: "peer", + Kind: ddprofiledefinition.BGPRowKindPeer, + Identity: ddprofiledefinition.BGPIdentityConfig{ + Neighbor: ddprofiledefinition.BGPValueConfig{Value: "192.0.2.1"}, + RemoteAS: ddprofiledefinition.BGPValueConfig{Value: "65001"}, + }, + Descriptors: ddprofiledefinition.BGPDescriptorsConfig{ + Description: ddprofiledefinition.BGPValueConfig{Value: "peer without state"}, + }, + Transitions: ddprofiledefinition.BGPTransitionsConfig{ + Established: ddprofiledefinition.BGPValueConfig{Symbol: ddprofiledefinition.SymbolConfig{OID: "1.3.6.1.4.1.2011.5.25.177.1.1.7.1.4", Name: "hwBgpPeerFsmEstablishedTransitions"}}, + Down: ddprofiledefinition.BGPValueConfig{Symbol: ddprofiledefinition.SymbolConfig{OID: "1.3.6.1.4.1.2011.5.25.177.1.1.7.1.5", Name: "hwBgpPeerDownCounts"}}, + }, + Traffic: ddprofiledefinition.BGPTrafficConfig{ + Updates: ddprofiledefinition.BGPDirectionalConfig{ + Received: ddprofiledefinition.BGPValueConfig{Symbol: ddprofiledefinition.SymbolConfig{OID: "1.3.6.1.4.1.2011.5.25.177.1.1.7.1.6", Name: "hwBgpPeerInUpdateMsgs"}}, + }, + }, + }, + }, + }, + }}} + + view := resolved.Project(ConsumerBGP).FilterBGPToTopologyPeers().Profiles() + + require.Len(t, view, 1) + require.Len(t, view[0].Definition.BGP, 1) + row := view[0].Definition.BGP[0] + assert.Equal(t, "192.0.2.1", row.Identity.Neighbor.Value) + assert.Equal(t, "peer without state", row.Descriptors.Description.Value) + assert.Equal(t, ddprofiledefinition.BGPTransitionsConfig{ + Established: ddprofiledefinition.BGPValueConfig{Symbol: ddprofiledefinition.SymbolConfig{OID: "1.3.6.1.4.1.2011.5.25.177.1.1.7.1.4", Name: "hwBgpPeerFsmEstablishedTransitions"}}, + Down: ddprofiledefinition.BGPValueConfig{Symbol: ddprofiledefinition.SymbolConfig{OID: "1.3.6.1.4.1.2011.5.25.177.1.1.7.1.5", Name: "hwBgpPeerDownCounts"}}, + }, row.Transitions) + assert.Equal(t, ddprofiledefinition.BGPTrafficConfig{}, row.Traffic) +} + +func TestProjectedViewFilterBGPToTopologyPeersPrunesAuxiliaryTableReferences(t *testing.T) { + resolved := &ResolvedProfileSet{profiles: []*Profile{{ + SourceFile: "projection.yaml", + Definition: &ddprofiledefinition.ProfileDefinition{ + BGP: []ddprofiledefinition.BGPConfig{ + { + ID: "peer", + Kind: ddprofiledefinition.BGPRowKindPeer, + Table: ddprofiledefinition.SymbolConfig{OID: "1.3.6.1.4.1.99999.10", Name: "peerTable"}, + Identity: ddprofiledefinition.BGPIdentityConfig{ + Neighbor: ddprofiledefinition.BGPValueConfig{ + Symbol: ddprofiledefinition.SymbolConfig{OID: "1.3.6.1.4.1.99999.10.1.1", Name: "peerRemoteAddr"}, + }, + RemoteAS: ddprofiledefinition.BGPValueConfig{ + Symbol: ddprofiledefinition.SymbolConfig{OID: "1.3.6.1.4.1.99999.10.1.2", Name: "peerRemoteAS"}, + }, + }, + State: ddprofiledefinition.BGPStateConfig{ + BGPValueConfig: ddprofiledefinition.BGPValueConfig{ + Symbol: ddprofiledefinition.SymbolConfig{OID: "1.3.6.1.4.1.99999.10.1.3", Name: "peerState"}, + }, + }, + Traffic: ddprofiledefinition.BGPTrafficConfig{ + Messages: ddprofiledefinition.BGPDirectionalConfig{ + Received: ddprofiledefinition.BGPValueConfig{ + Table: "peerAuxTable", + Symbol: ddprofiledefinition.SymbolConfig{OID: "1.3.6.1.4.1.99999.20.1.1", Name: "peerMessagesReceived"}, + }, + }, + }, + MetricTags: []ddprofiledefinition.MetricTagConfig{ + { + Tag: "aux_tag", + Table: "peerAuxTable", + Symbol: ddprofiledefinition.SymbolConfigCompat{ + OID: "1.3.6.1.4.1.99999.20.1.2", + Name: "peerAuxTag", + }, + }, + }, + }, + }, + }, + }}} + + full := resolved.Project(ConsumerBGP).Profiles() + require.Len(t, full, 1) + require.Len(t, full[0].Definition.BGP, 1) + assert.Contains(t, bgpValueSourceTables(full[0].Definition.BGP[0]), "peerAuxTable") + require.NotEmpty(t, full[0].Definition.BGP[0].MetricTags) + + pruned := resolved.Project(ConsumerBGP).FilterBGPToTopologyPeers().Profiles() + require.Len(t, pruned, 1) + require.Len(t, pruned[0].Definition.BGP, 1) + row := pruned[0].Definition.BGP[0] + assert.NotContains(t, bgpValueSourceTables(row), "peerAuxTable") + assert.Equal(t, ddprofiledefinition.BGPTrafficConfig{}, row.Traffic) + assert.Empty(t, row.MetricTags) +} + +func bgpSignalPathsWithPrefix(row ddprofiledefinition.BGPConfig, prefix string) []string { + var paths []string + ddprofiledefinition.ForEachBGPSignalValue(row, func(path string, _ ddprofiledefinition.BGPValueConfig) { + if len(path) >= len(prefix) && path[:len(prefix)] == prefix { + paths = append(paths, path) + } + }) + return paths +} + +func bgpValueSourceTables(row ddprofiledefinition.BGPConfig) []string { + tables := make(map[string]struct{}) + add := func(value ddprofiledefinition.BGPValueConfig) { + if value.Table != "" { + tables[value.Table] = struct{}{} + } + } + add(row.Identity.RoutingInstance) + add(row.Identity.Neighbor) + add(row.Identity.RemoteAS) + add(row.Identity.AddressFamily.BGPValueConfig) + add(row.Identity.SubsequentAddressFamily.BGPValueConfig) + add(row.Descriptors.LocalAddress) + add(row.Descriptors.LocalAS) + add(row.Descriptors.LocalIdentifier) + add(row.Descriptors.PeerIdentifier) + add(row.Descriptors.PeerType) + add(row.Descriptors.BGPVersion) + add(row.Descriptors.Description) + ddprofiledefinition.ForEachBGPSignalValue(row, func(_ string, value ddprofiledefinition.BGPValueConfig) { + add(value) + }) + for _, tag := range row.MetricTags { + if tag.Table != "" { + tables[tag.Table] = struct{}{} + } + } + + out := make([]string, 0, len(tables)) + for table := range tables { + out = append(out, table) + } + slices.Sort(out) + return out +} + func projectionTestProfile() *Profile { return &Profile{ SourceFile: "projection.yaml", @@ -389,9 +625,93 @@ func projectionTestProfile() *Profile { ID: "peer", Kind: ddprofiledefinition.BGPRowKindPeer, Identity: ddprofiledefinition.BGPIdentityConfig{ - Neighbor: ddprofiledefinition.BGPValueConfig{Value: "192.0.2.1"}, - RemoteAS: ddprofiledefinition.BGPValueConfig{Value: "65001"}, + RoutingInstance: ddprofiledefinition.BGPValueConfig{Value: "default"}, + Neighbor: ddprofiledefinition.BGPValueConfig{Value: "192.0.2.1"}, + RemoteAS: ddprofiledefinition.BGPValueConfig{Value: "65001"}, + }, + Descriptors: ddprofiledefinition.BGPDescriptorsConfig{ + LocalAddress: ddprofiledefinition.BGPValueConfig{Value: "192.0.2.2"}, + LocalAS: ddprofiledefinition.BGPValueConfig{Value: "65000"}, + LocalIdentifier: ddprofiledefinition.BGPValueConfig{Value: "10.0.0.1"}, + PeerIdentifier: ddprofiledefinition.BGPValueConfig{Value: "10.0.0.2"}, + PeerType: ddprofiledefinition.BGPValueConfig{Value: "ipv4"}, + BGPVersion: ddprofiledefinition.BGPValueConfig{Value: "4"}, + Description: ddprofiledefinition.BGPValueConfig{Value: "peer"}, + }, + Admin: ddprofiledefinition.BGPAdminConfig{ + Enabled: ddprofiledefinition.BGPValueConfig{Value: "start"}, + }, + State: ddprofiledefinition.BGPStateConfig{ + BGPValueConfig: ddprofiledefinition.BGPValueConfig{Value: "established"}, }, + Previous: ddprofiledefinition.BGPStateConfig{ + BGPValueConfig: ddprofiledefinition.BGPValueConfig{Value: "idle"}, + }, + Connection: ddprofiledefinition.BGPConnectionConfig{ + EstablishedUptime: ddprofiledefinition.BGPValueConfig{ + Symbol: ddprofiledefinition.SymbolConfig{OID: "1.3.6.1.2.1.15.3.1.16", Name: "bgpPeerFsmEstablishedTime"}, + }, + LastReceivedUpdateAge: ddprofiledefinition.BGPValueConfig{ + Symbol: ddprofiledefinition.SymbolConfig{OID: "1.3.6.1.2.1.15.3.1.24", Name: "bgpPeerInUpdateElapsedTime"}, + }, + }, + Traffic: ddprofiledefinition.BGPTrafficConfig{ + Messages: ddprofiledefinition.BGPDirectionalConfig{ + Received: ddprofiledefinition.BGPValueConfig{Symbol: ddprofiledefinition.SymbolConfig{OID: "1.3.6.1.2.1.15.3.1.12", Name: "bgpPeerInTotalMessages"}}, + Sent: ddprofiledefinition.BGPValueConfig{Symbol: ddprofiledefinition.SymbolConfig{OID: "1.3.6.1.2.1.15.3.1.13", Name: "bgpPeerOutTotalMessages"}}, + }, + }, + Transitions: ddprofiledefinition.BGPTransitionsConfig{ + Established: ddprofiledefinition.BGPValueConfig{Symbol: ddprofiledefinition.SymbolConfig{OID: "1.3.6.1.2.1.15.3.1.15", Name: "bgpPeerFsmEstablishedTransitions"}}, + }, + Timers: ddprofiledefinition.BGPTimersConfig{ + Negotiated: ddprofiledefinition.BGPTimerPairConfig{ + HoldTime: ddprofiledefinition.BGPValueConfig{Symbol: ddprofiledefinition.SymbolConfig{OID: "1.3.6.1.2.1.15.3.1.18", Name: "bgpPeerHoldTime"}}, + }, + }, + LastError: ddprofiledefinition.BGPLastErrorConfig{ + Code: ddprofiledefinition.BGPValueConfig{Symbol: ddprofiledefinition.SymbolConfig{OID: "1.3.6.1.2.1.15.3.1.14", Name: "bgpPeerLastErrorCode"}}, + }, + LastNotify: ddprofiledefinition.BGPLastNotifyConfig{ + Received: ddprofiledefinition.BGPLastNotificationConfig{ + Reason: ddprofiledefinition.BGPValueConfig{Value: "hold timer expired"}, + }, + }, + Reasons: ddprofiledefinition.BGPReasonsConfig{ + LastDown: ddprofiledefinition.BGPValueConfig{Value: "manual"}, + }, + Restart: ddprofiledefinition.BGPGracefulRestartConfig{ + State: ddprofiledefinition.BGPValueConfig{Value: "enabled"}, + }, + Routes: ddprofiledefinition.BGPRoutesConfig{ + Total: ddprofiledefinition.BGPRouteCountersConfig{ + Accepted: ddprofiledefinition.BGPValueConfig{Value: "10"}, + }, + }, + RouteLimits: ddprofiledefinition.BGPRouteLimitsConfig{ + Limit: ddprofiledefinition.BGPValueConfig{Value: "1000"}, + }, + Device: ddprofiledefinition.BGPDeviceCountsConfig{ + Peers: ddprofiledefinition.BGPValueConfig{Value: "1"}, + }, + StaticTags: []ddprofiledefinition.StaticMetricTagConfig{{Tag: "scope", Value: "bgp"}}, + MetricTags: []ddprofiledefinition.MetricTagConfig{ + {Tag: "bgp_extra", Symbol: ddprofiledefinition.SymbolConfigCompat{OID: "1.3.6.1.2.1.15.3.1.99", Name: "bgpPeerExtra"}}, + }, + }, + { + ID: "peer-family", + Kind: ddprofiledefinition.BGPRowKindPeerFamily, + Identity: ddprofiledefinition.BGPIdentityConfig{ + Neighbor: ddprofiledefinition.BGPValueConfig{Value: "192.0.2.1"}, + RemoteAS: ddprofiledefinition.BGPValueConfig{Value: "65001"}, + AddressFamily: ddprofiledefinition.BGPAddressFamilyValueConfig{BGPValueConfig: ddprofiledefinition.BGPValueConfig{Value: "ipv4"}}, + SubsequentAddressFamily: ddprofiledefinition.BGPSubsequentAddressFamilyValueConfig{BGPValueConfig: ddprofiledefinition.BGPValueConfig{Value: "unicast"}}, + }, + }, + { + ID: "device", + Kind: ddprofiledefinition.BGPRowKindDevice, }, }, VirtualMetrics: []ddprofiledefinition.VirtualMetricConfig{ diff --git a/src/go/plugin/go.d/collector/snmp_topology/collector.go b/src/go/plugin/go.d/collector/snmp_topology/collector.go index a9ca1dd51edabf..403788e4bc3e62 100644 --- a/src/go/plugin/go.d/collector/snmp_topology/collector.go +++ b/src/go/plugin/go.d/collector/snmp_topology/collector.go @@ -324,6 +324,7 @@ func (c *Collector) refreshDeviceTopology(ctx context.Context, key string, dev d next.updateTopologySysUptime(sysUptime) next.updateTopologyProfileTags(pms) next.ingestTopologyProfileMetrics(pms) + next.ingestTopologyBGPPeers(pms) c.collectTopologyVTPVLANContexts(ctx, next, dev) if ctx.Err() != nil { return false @@ -384,7 +385,7 @@ func (c *Collector) findTopologyProfiles(dev ddsnmp.DeviceConnectionInfo) []*dds SysDescr: dev.SysDescr, ManualProfiles: dev.ManualProfiles, ManualPolicy: ddsnmp.ManualProfileAugment, - }).Project(ddsnmp.ConsumerTopology).Profiles() + }).Project(ddsnmp.ConsumerTopology, ddsnmp.ConsumerBGP).FilterBGPToTopologyPeers().Profiles() } func (c *Collector) getTopologyProfiles(dev ddsnmp.DeviceConnectionInfo) []*ddsnmp.Profile { diff --git a/src/go/plugin/go.d/collector/snmp_topology/func_topology_test.go b/src/go/plugin/go.d/collector/snmp_topology/func_topology_test.go index 54ff74ec151009..48cd9f6c4c4d5f 100644 --- a/src/go/plugin/go.d/collector/snmp_topology/func_topology_test.go +++ b/src/go/plugin/go.d/collector/snmp_topology/func_topology_test.go @@ -341,7 +341,7 @@ func TestSNMPTopologyToV1_PreservesActorCustomTables(t *testing.T) { require.NotNil(t, deviceType.Presentation.Modal.Labels) require.NotNil(t, deviceType.Presentation.Modal.Labels.Identification) assert.Equal(t, "management_ip", deviceType.Presentation.Modal.Labels.Identification.Fields[1].Key) - require.Len(t, deviceType.Presentation.Modal.Sections, 4) + require.Len(t, deviceType.Presentation.Modal.Sections, 5) assert.Equal(t, "ports", deviceType.Presentation.Modal.Sections[0].ID) assert.Equal(t, "if_index", deviceType.Presentation.Modal.Sections[0].Columns[0].ID) assert.Equal(t, "Port ID", deviceType.Presentation.Modal.Sections[0].Columns[0].Label) @@ -357,6 +357,9 @@ func TestSNMPTopologyToV1_PreservesActorCustomTables(t *testing.T) { assert.Equal(t, "ospf_neighbors", deviceType.Presentation.Modal.Sections[3].ID) assert.Equal(t, "actor_table", deviceType.Presentation.Modal.Sections[3].Source.Kind) assert.Equal(t, "actor_ospf_neighbors", deviceType.Presentation.Modal.Sections[3].Source.Table) + assert.Equal(t, "bgp_peers", deviceType.Presentation.Modal.Sections[4].ID) + assert.Equal(t, "actor_table", deviceType.Presentation.Modal.Sections[4].Source.Kind) + assert.Equal(t, "actor_bgp_peers", deviceType.Presentation.Modal.Sections[4].Source.Table) endpointType := payload.Types.ActorTypes["endpoint"] require.NotNil(t, endpointType.Presentation) @@ -724,6 +727,146 @@ func TestSNMPTopologyToV1_PreservesOSPFAdjacencyPresentationEvidenceAndNeighborR assert.Empty(t, ospfSection.Columns[3].Visibility) } +func TestSNMPTopologyToV1_PreservesBGPAdjacencyPresentationEvidenceAndPeerRows(t *testing.T) { + data := topologyData{ + AgentID: "agent-test", + View: "summary", + Actors: []topologyActor{ + { + ActorID: "router-a", + ActorType: "router", + Source: "snmp", + Tables: map[string][]map[string]any{ + "bgp_peers": { + { + "routing_instance": "default", + "neighbor_ip": "192.0.2.2", + "remote_as": "65002", + "state": "established", + "admin_status": "enabled", + "local_ip": "192.0.2.1", + "local_as": "65001", + "local_identifier": "1.1.1.1", + "peer_identifier": "2.2.2.2", + "peer_type": "external", + "bgp_version": "4", + "description": "edge-peer", + "established_uptime": int64(300), + "last_received_update_age": int64(12), + "remote_actor_id": "router-b", + }, + }, + }, + }, + { + ActorID: "router-b", + ActorType: "router", + Source: "snmp", + }, + }, + Links: []topologyLink{ + { + Protocol: topologyBGPAdjacencyLinkType, + LinkType: topologyBGPAdjacencyLinkType, + Direction: "observed", + State: "established", + SrcActorID: "router-a", + DstActorID: "router-b", + Src: topologyLinkEndpoint{ + Attributes: map[string]any{ + "bgp_identifier": "1.1.1.1", + "ip": "192.0.2.1", + "as": "65001", + }, + }, + Dst: topologyLinkEndpoint{ + Attributes: map[string]any{ + "bgp_identifier": "2.2.2.2", + "ip": "192.0.2.2", + "as": "65002", + }, + }, + Metrics: map[string]any{ + "source": "bgp_mib", + "inference": "bgp_established_adjacency", + "attachment_mode": "logical_l3_bgp", + "routing_instance": "default", + "local_identifier": "1.1.1.1", + "peer_identifier": "2.2.2.2", + "local_ip": "192.0.2.1", + "neighbor_ip": "192.0.2.2", + "local_as": "65001", + "remote_as": "65002", + }, + }, + }, + } + + payload, err := snmpTopologyToV1(data) + require.NoError(t, err) + require.NoError(t, validateTopologyV1Data(payload)) + + assert.Contains(t, payload.Producer.Capabilities, "bgp") + require.Contains(t, payload.Types.LinkTypes, snmpTopologyV1LinkBGP) + linkType := payload.Types.LinkTypes[snmpTopologyV1LinkBGP] + assert.Equal(t, "observed_bidirectional", linkType.Orientation) + assert.Equal(t, "observation", linkType.DirectionRole) + assert.Equal(t, "control", linkType.SemanticRole) + require.NotNil(t, linkType.Presentation) + assert.Equal(t, "BGP adjacency", linkType.Presentation.Label) + assert.Equal(t, "accent", linkType.Presentation.ColorSlot) + assert.Equal(t, "dashed", linkType.Presentation.LineStyle) + assert.Contains(t, topologyV1LegendLinkTypes(payload), snmpTopologyV1LinkBGP) + + require.Contains(t, payload.Types.EvidenceTypes, snmpTopologyV1LinkBGP) + evidenceType := payload.Types.EvidenceTypes[snmpTopologyV1LinkBGP] + assert.Equal(t, snmpTopologyV1LinkBGP, evidenceType.LinkType) + assert.Equal(t, []string{"src_actor", "dst_actor", "routing_instance"}, evidenceType.MatchColumns) + + require.Contains(t, payload.Evidence, snmpTopologyV1LinkBGP) + evidenceTable := payload.Evidence[snmpTopologyV1LinkBGP].Table + assert.Equal(t, 1, evidenceTable.Rows) + assert.Equal(t, "string_ref", topologyV1ColumnType(evidenceTable, "routing_instance")) + assert.Equal(t, "string_ref", topologyV1ColumnType(evidenceTable, "local_identifier")) + assert.Equal(t, "string_ref", topologyV1ColumnType(evidenceTable, "peer_identifier")) + assert.Equal(t, "string_ref", topologyV1ColumnType(evidenceTable, "local_ip")) + assert.Equal(t, "string_ref", topologyV1ColumnType(evidenceTable, "neighbor_ip")) + assert.Equal(t, []string{"default"}, topologyV1StringColumnValues(t, payload, evidenceTable, "routing_instance")) + assert.Equal(t, []string{"1.1.1.1"}, topologyV1StringColumnValues(t, payload, evidenceTable, "local_identifier")) + assert.Equal(t, []string{"2.2.2.2"}, topologyV1StringColumnValues(t, payload, evidenceTable, "peer_identifier")) + assert.Equal(t, []string{"192.0.2.1"}, topologyV1StringColumnValues(t, payload, evidenceTable, "local_ip")) + assert.Equal(t, []string{"192.0.2.2"}, topologyV1StringColumnValues(t, payload, evidenceTable, "neighbor_ip")) + assert.Equal(t, []string{"65001"}, topologyV1StringColumnValues(t, payload, evidenceTable, "local_as")) + assert.Equal(t, []string{"65002"}, topologyV1StringColumnValues(t, payload, evidenceTable, "remote_as")) + + require.NotNil(t, payload.Tables) + require.Contains(t, payload.Tables.Actor, "actor_bgp_peers") + peersTable := payload.Tables.Actor["actor_bgp_peers"].Table + assert.Equal(t, 1, peersTable.Rows) + assert.Equal(t, "actor_ref", topologyV1ColumnType(peersTable, "remote_actor")) + assert.Equal(t, []any{0}, topologyV1ColumnValues(t, peersTable, "actor")) + assert.Equal(t, []any{1}, topologyV1ColumnValues(t, peersTable, "remote_actor")) + assert.Equal(t, []string{"default"}, topologyV1StringColumnValues(t, payload, peersTable, "routing_instance")) + assert.Equal(t, []string{"192.0.2.2"}, topologyV1StringColumnValues(t, payload, peersTable, "neighbor_ip")) + assert.Equal(t, []string{"65002"}, topologyV1StringColumnValues(t, payload, peersTable, "remote_as")) + assert.Equal(t, []string{"established"}, topologyV1StringColumnValues(t, payload, peersTable, "state")) + assert.Equal(t, []any{uint64(300)}, topologyV1ColumnValues(t, peersTable, "established_uptime")) + + deviceType := payload.Types.ActorTypes["router"] + require.NotNil(t, deviceType.Presentation) + require.NotNil(t, deviceType.Presentation.Modal) + bgpSection := requireTopologyV1ModalSection(t, deviceType.Presentation.Modal.Sections, "bgp_peers") + assert.Equal(t, "actor_table", bgpSection.Source.Kind) + assert.Equal(t, "actor_bgp_peers", bgpSection.Source.Table) + require.NotNil(t, bgpSection.OwnerFilter) + assert.Equal(t, "actor_column", bgpSection.OwnerFilter.Mode) + assert.Equal(t, "actor", bgpSection.OwnerFilter.ActorColumn) + require.Len(t, bgpSection.Columns, 16) + assert.Equal(t, "remote_actor", bgpSection.Columns[3].ID) + assert.Equal(t, "actor_link", bgpSection.Columns[3].Cell) + assert.Empty(t, bgpSection.Columns[3].Visibility) +} + func TestSNMPTopologyToV1_ReturnsErrorForL3SubnetWithoutSubnet(t *testing.T) { data := topologyData{ AgentID: "agent-test", diff --git a/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1.go b/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1.go index 64ca7953b7c963..f6e78dd9153cc3 100644 --- a/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1.go +++ b/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1.go @@ -26,6 +26,7 @@ const ( snmpTopologyV1LinkProbable = "probable" snmpTopologyV1LinkL3Subnet = topologyL3SubnetLinkType snmpTopologyV1LinkOSPF = topologyOSPFAdjacencyLinkType + snmpTopologyV1LinkBGP = topologyBGPAdjacencyLinkType ) func snmpTopologyToV1(data topologyData) (topologyv1.Data, error) { @@ -55,6 +56,9 @@ func snmpTopologyToV1(data topologyData) (topologyv1.Data, error) { if _, ok := tableTypes["actor_ospf_neighbors"]; !ok { tableTypes["actor_ospf_neighbors"] = snmpTopologyV1OSPFNeighborsTableType() } + if _, ok := tableTypes["actor_bgp_peers"]; !ok { + tableTypes["actor_bgp_peers"] = snmpTopologyV1BGPPeersTableType() + } portLinksTable, err := buildSNMPTopologyV1ActorPortLinksTable(data.Links, actorIndex, stringsDict) if err != nil { return topologyv1.Data{}, err @@ -112,6 +116,7 @@ func snmpTopologyToV1(data topologyData) (topologyv1.Data, error) { "stp", "l3_subnet", "ospf", + "bgp", }, }, CollectedAt: data.CollectedAt, diff --git a/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_actor_details.go b/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_actor_details.go index a284ebb44e5712..b6c76410b7ae4c 100644 --- a/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_actor_details.go +++ b/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_actor_details.go @@ -67,6 +67,8 @@ func buildSNMPTopologyV1ActorDetails( table = buildSNMPTopologyV1ActorPortsTable(rows, stringsDict, portNeighborSummaries) } else if tableID == "actor_ospf_neighbors" { table = buildSNMPTopologyV1OSPFNeighborsTable(rows, actorIndex, stringsDict) + } else if tableID == "actor_bgp_peers" { + table = buildSNMPTopologyV1BGPPeersTable(rows, actorIndex, stringsDict) } else { table, err = buildSNMPTopologyV1DynamicTable(rows, stringsDict) } @@ -81,6 +83,8 @@ func buildSNMPTopologyV1ActorDetails( tableTypes[tableID] = snmpTopologyV1ActorPortsTableType() } else if tableID == "actor_ospf_neighbors" { tableTypes[tableID] = snmpTopologyV1OSPFNeighborsTableType() + } else if tableID == "actor_bgp_peers" { + tableTypes[tableID] = snmpTopologyV1BGPPeersTableType() } else { tableTypes[tableID] = topologyv1.TableType{ Role: "actor_detail", diff --git a/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_bgp_peers.go b/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_bgp_peers.go new file mode 100644 index 00000000000000..ad4ae16ca2d9f6 --- /dev/null +++ b/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_bgp_peers.go @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package snmptopology + +import topologyv1 "github.com/netdata/netdata/go/plugins/pkg/topology/v1" + +func buildSNMPTopologyV1BGPPeersTable( + rows []topologyV1DynamicRow, + actorIndex map[string]int, + stringsDict *topologyv1.StringDictionary, +) topologyv1.Table { + actorRefs := make([]any, len(rows)) + remoteActors := make([]any, len(rows)) + routingInstances := make([]any, len(rows)) + neighborIPs := make([]any, len(rows)) + remoteASes := make([]any, len(rows)) + states := make([]any, len(rows)) + adminStatuses := make([]any, len(rows)) + localIPs := make([]any, len(rows)) + localASes := make([]any, len(rows)) + localIdentifiers := make([]any, len(rows)) + peerIdentifiers := make([]any, len(rows)) + peerTypes := make([]any, len(rows)) + bgpVersions := make([]any, len(rows)) + descriptions := make([]any, len(rows)) + establishedUptimes := make([]any, len(rows)) + lastReceivedUpdateAges := make([]any, len(rows)) + sources := make([]any, len(rows)) + + for i, row := range rows { + actorRefs[i] = row.actorRef + remoteActors[i] = nullableActorRef(actorIndex, row.values["remote_actor_id"]) + routingInstances[i] = nullableStringRef(stringsDict, firstNonEmptyString(topologyV1ScalarLabelValue(row.values["routing_instance"]), "default")) + neighborIPs[i] = nullableStringRef(stringsDict, topologyBGPPeerAddressValue(topologyV1ScalarLabelValue(row.values["neighbor_ip"]))) + remoteASes[i] = nullableStringRef(stringsDict, topologyV1ScalarLabelValue(row.values["remote_as"])) + states[i] = nullableStringRef(stringsDict, topologyV1ScalarLabelValue(row.values["state"])) + adminStatuses[i] = nullableStringRef(stringsDict, topologyV1ScalarLabelValue(row.values["admin_status"])) + localIPs[i] = nullableStringRef(stringsDict, topologyBGPPeerAddressValue(topologyV1ScalarLabelValue(row.values["local_ip"]))) + localASes[i] = nullableStringRef(stringsDict, topologyV1ScalarLabelValue(row.values["local_as"])) + localIdentifiers[i] = nullableStringRef(stringsDict, normalizeBGPRouterID(topologyV1ScalarLabelValue(row.values["local_identifier"]))) + peerIdentifiers[i] = nullableStringRef(stringsDict, normalizeBGPRouterID(topologyV1ScalarLabelValue(row.values["peer_identifier"]))) + peerTypes[i] = nullableStringRef(stringsDict, topologyV1ScalarLabelValue(row.values["peer_type"])) + bgpVersions[i] = nullableStringRef(stringsDict, topologyV1ScalarLabelValue(row.values["bgp_version"])) + descriptions[i] = nullableStringRef(stringsDict, topologyV1ScalarLabelValue(row.values["description"])) + establishedUptimes[i] = nullableUintValue(row.values["established_uptime"]) + lastReceivedUpdateAges[i] = nullableUintValue(row.values["last_received_update_age"]) + sources[i] = nullableStringRef(stringsDict, firstNonEmptyString(topologyV1ScalarLabelValue(row.values["source"]), "bgp_mib")) + } + + return topologyv1.MustTable(len(rows), snmpTopologyV1BGPPeersColumns(), []topologyv1.ColumnEncoding{ + topologyv1.Values(actorRefs...), + topologyv1.Values(remoteActors...), + topologyv1.Values(routingInstances...), + topologyv1.Values(neighborIPs...), + topologyv1.Values(remoteASes...), + topologyv1.Values(states...), + topologyv1.Values(adminStatuses...), + topologyv1.Values(localIPs...), + topologyv1.Values(localASes...), + topologyv1.Values(localIdentifiers...), + topologyv1.Values(peerIdentifiers...), + topologyv1.Values(peerTypes...), + topologyv1.Values(bgpVersions...), + topologyv1.Values(descriptions...), + topologyv1.Values(establishedUptimes...), + topologyv1.Values(lastReceivedUpdateAges...), + topologyv1.Values(sources...), + }) +} diff --git a/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_links.go b/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_links.go index d0ee305420c668..5ce5c84770c129 100644 --- a/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_links.go +++ b/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_links.go @@ -76,15 +76,29 @@ func buildSNMPTopologyV1Links( evidenceRows.confidences = append(evidenceRows.confidences, nullableStringRef(stringsDict, topologyMetricValueString(link.Metrics, "confidence"))) evidenceRows.inferences = append(evidenceRows.inferences, nullableStringRef(stringsDict, topologyMetricValueString(link.Metrics, "inference"))) evidenceRows.attachmentModes = append(evidenceRows.attachmentModes, nullableStringRef(stringsDict, topologyMetricValueString(link.Metrics, "attachment_mode"))) - evidenceRows.srcRouterIDs = append(evidenceRows.srcRouterIDs, nullableStringRef(stringsDict, topologyV1EndpointString(link.Src, "router_id"))) - evidenceRows.dstRouterIDs = append(evidenceRows.dstRouterIDs, nullableStringRef(stringsDict, topologyV1EndpointString(link.Dst, "router_id"))) - evidenceRows.srcIPs = append(evidenceRows.srcIPs, nullableStringRef(stringsDict, topologyV1EndpointString(link.Src, "ip"))) - evidenceRows.dstIPs = append(evidenceRows.dstIPs, nullableStringRef(stringsDict, topologyV1EndpointString(link.Dst, "ip"))) - evidenceRows.subnets = append(evidenceRows.subnets, nullableStringRef(stringsDict, subnet)) - evidenceRows.networks = append(evidenceRows.networks, nullableStringRef(stringsDict, topologyMetricValueString(link.Metrics, "network"))) - evidenceRows.netmasks = append(evidenceRows.netmasks, nullableStringRef(stringsDict, topologyMetricValueString(link.Metrics, "netmask"))) - evidenceRows.prefixes = append(evidenceRows.prefixes, nullableUintValue(link.Metrics["prefix"])) - evidenceRows.sources = append(evidenceRows.sources, nullableStringRef(stringsDict, topologyMetricValueString(link.Metrics, "source"))) + if linkType == snmpTopologyV1LinkL3Subnet || linkType == snmpTopologyV1LinkOSPF { + if linkType == snmpTopologyV1LinkOSPF { + evidenceRows.srcRouterIDs = append(evidenceRows.srcRouterIDs, nullableStringRef(stringsDict, topologyV1EndpointString(link.Src, "router_id"))) + evidenceRows.dstRouterIDs = append(evidenceRows.dstRouterIDs, nullableStringRef(stringsDict, topologyV1EndpointString(link.Dst, "router_id"))) + } + evidenceRows.srcIPs = append(evidenceRows.srcIPs, nullableStringRef(stringsDict, topologyV1EndpointString(link.Src, "ip"))) + evidenceRows.dstIPs = append(evidenceRows.dstIPs, nullableStringRef(stringsDict, topologyV1EndpointString(link.Dst, "ip"))) + evidenceRows.subnets = append(evidenceRows.subnets, nullableStringRef(stringsDict, subnet)) + evidenceRows.networks = append(evidenceRows.networks, nullableStringRef(stringsDict, topologyMetricValueString(link.Metrics, "network"))) + evidenceRows.netmasks = append(evidenceRows.netmasks, nullableStringRef(stringsDict, topologyMetricValueString(link.Metrics, "netmask"))) + evidenceRows.prefixes = append(evidenceRows.prefixes, nullableUintValue(link.Metrics["prefix"])) + evidenceRows.sources = append(evidenceRows.sources, nullableStringRef(stringsDict, topologyMetricValueString(link.Metrics, "source"))) + } + if linkType == snmpTopologyV1LinkBGP { + evidenceRows.routingInstances = append(evidenceRows.routingInstances, nullableStringRef(stringsDict, topologyMetricValueString(link.Metrics, "routing_instance"))) + evidenceRows.localIdentifiers = append(evidenceRows.localIdentifiers, nullableStringRef(stringsDict, topologyV1EndpointString(link.Src, "bgp_identifier"))) + evidenceRows.peerIdentifiers = append(evidenceRows.peerIdentifiers, nullableStringRef(stringsDict, topologyV1EndpointString(link.Dst, "bgp_identifier"))) + evidenceRows.localIPs = append(evidenceRows.localIPs, nullableStringRef(stringsDict, topologyV1EndpointString(link.Src, "ip"))) + evidenceRows.neighborIPs = append(evidenceRows.neighborIPs, nullableStringRef(stringsDict, topologyV1EndpointString(link.Dst, "ip"))) + evidenceRows.localASes = append(evidenceRows.localASes, nullableStringRef(stringsDict, topologyV1EndpointString(link.Src, "as"))) + evidenceRows.remoteASes = append(evidenceRows.remoteASes, nullableStringRef(stringsDict, topologyV1EndpointString(link.Dst, "as"))) + evidenceRows.sources = append(evidenceRows.sources, nullableStringRef(stringsDict, topologyMetricValueString(link.Metrics, "source"))) + } evidenceRows.srcEndpoints = append(evidenceRows.srcEndpoints, srcEndpoint) evidenceRows.dstEndpoints = append(evidenceRows.dstEndpoints, dstEndpoint) evidenceRows.metrics = append(evidenceRows.metrics, metrics) @@ -155,6 +169,13 @@ type snmpTopologyV1EvidenceRows struct { netmasks []any prefixes []any sources []any + routingInstances []any + localIdentifiers []any + peerIdentifiers []any + localASes []any + remoteASes []any + localIPs []any + neighborIPs []any srcEndpoints []any dstEndpoints []any metrics []any @@ -202,6 +223,18 @@ func (rows *snmpTopologyV1EvidenceRows) columnEncodingsForType(linkType string) topologyv1.Values(rows.sources...), ) } + if linkType == snmpTopologyV1LinkBGP { + encodings = append(encodings, + topologyv1.Values(rows.routingInstances...), + topologyv1.Values(rows.localIdentifiers...), + topologyv1.Values(rows.peerIdentifiers...), + topologyv1.Values(rows.localIPs...), + topologyv1.Values(rows.neighborIPs...), + topologyv1.Values(rows.localASes...), + topologyv1.Values(rows.remoteASes...), + topologyv1.Values(rows.sources...), + ) + } encodings = append(encodings, topologyv1.Values(rows.srcEndpoints...), topologyv1.Values(rows.dstEndpoints...), @@ -212,26 +245,34 @@ func (rows *snmpTopologyV1EvidenceRows) columnEncodingsForType(linkType string) func snmpTopologyV1EvidenceColumnsForType(linkType string) []topologyv1.Column { columns := snmpTopologyV1EvidenceColumns() - if linkType != snmpTopologyV1LinkL3Subnet { - if linkType == snmpTopologyV1LinkOSPF { - extras := []topologyv1.Column{ - topologyv1.NewColumn("src_router_id", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), - topologyv1.NewColumn("dst_router_id", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), - topologyv1.NewColumn("src_ip", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), - topologyv1.NewColumn("dst_ip", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), - topologyv1.NewColumn("subnet", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), - topologyv1.NewColumn("network", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), - topologyv1.NewColumn("netmask", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), - topologyv1.NewColumn("prefix", "uint", topologyv1.WithNullable()), - topologyv1.NewColumn("source", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), - } - insertAt := len(columns) - 3 - out := make([]topologyv1.Column, 0, len(columns)+len(extras)) - out = append(out, columns[:insertAt]...) - out = append(out, extras...) - out = append(out, columns[insertAt:]...) - return out + if linkType == snmpTopologyV1LinkOSPF { + extras := []topologyv1.Column{ + topologyv1.NewColumn("src_router_id", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), + topologyv1.NewColumn("dst_router_id", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), + topologyv1.NewColumn("src_ip", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), + topologyv1.NewColumn("dst_ip", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), + topologyv1.NewColumn("subnet", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), + topologyv1.NewColumn("network", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), + topologyv1.NewColumn("netmask", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), + topologyv1.NewColumn("prefix", "uint", topologyv1.WithNullable()), + topologyv1.NewColumn("source", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), + } + return snmpTopologyV1EvidenceColumnsWithExtras(columns, extras) + } + if linkType == snmpTopologyV1LinkBGP { + extras := []topologyv1.Column{ + topologyv1.NewColumn("routing_instance", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), + topologyv1.NewColumn("local_identifier", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), + topologyv1.NewColumn("peer_identifier", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), + topologyv1.NewColumn("local_ip", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), + topologyv1.NewColumn("neighbor_ip", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), + topologyv1.NewColumn("local_as", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), + topologyv1.NewColumn("remote_as", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), + topologyv1.NewColumn("source", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), } + return snmpTopologyV1EvidenceColumnsWithExtras(columns, extras) + } + if linkType != snmpTopologyV1LinkL3Subnet { return columns } extras := []topologyv1.Column{ @@ -243,6 +284,10 @@ func snmpTopologyV1EvidenceColumnsForType(linkType string) []topologyv1.Column { topologyv1.NewColumn("prefix", "uint", topologyv1.WithNullable()), topologyv1.NewColumn("source", "string_ref", topologyv1.WithDictionary("strings"), topologyv1.WithNullable()), } + return snmpTopologyV1EvidenceColumnsWithExtras(columns, extras) +} + +func snmpTopologyV1EvidenceColumnsWithExtras(columns, extras []topologyv1.Column) []topologyv1.Column { insertAt := len(columns) - 3 out := make([]topologyv1.Column, 0, len(columns)+len(extras)) out = append(out, columns[:insertAt]...) @@ -298,18 +343,16 @@ func snmpTopologyV1LinkType(link topologyLink) string { return snmpTopologyV1LinkL3Subnet case snmpTopologyV1LinkOSPF: return snmpTopologyV1LinkOSPF + case snmpTopologyV1LinkBGP: + return snmpTopologyV1LinkBGP default: return snmpTopologyV1LinkObservation } } -func snmpTopologyV1LinkIsL3Subnet(link topologyLink) bool { - return snmpTopologyV1LinkType(link) == snmpTopologyV1LinkL3Subnet -} - func snmpTopologyV1LinkIsLogicalL3(link topologyLink) bool { switch snmpTopologyV1LinkType(link) { - case snmpTopologyV1LinkL3Subnet, snmpTopologyV1LinkOSPF: + case snmpTopologyV1LinkL3Subnet, snmpTopologyV1LinkOSPF, snmpTopologyV1LinkBGP: return true default: return false diff --git a/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_presentation.go b/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_presentation.go index 254f3db856ccca..1412d27227f136 100644 --- a/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_presentation.go +++ b/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_presentation.go @@ -156,6 +156,7 @@ func snmpTopologyV1DeviceModal() *topologyv1.ModalPresentation { snmpTopologyV1PortLinksSection(2), snmpTopologyV1L3SubnetSection(3), snmpTopologyV1OSPFNeighborsSection(4), + snmpTopologyV1BGPPeersSection(5), }, } } @@ -313,6 +314,25 @@ func snmpTopologyV1OSPFNeighborsSection(order int) topologyv1.ModalSection { } } +func snmpTopologyV1BGPPeersSection(order int) topologyv1.ModalSection { + return topologyv1.ModalSection{ + ID: "bgp_peers", + Label: "BGP Peers", + Order: order, + Source: topologyv1.ModalSource{ + Kind: "actor_table", + Table: "actor_bgp_peers", + }, + OwnerFilter: &topologyv1.ModalOwnerFilter{ + Mode: "actor_column", + ActorColumn: "actor", + }, + Columns: snmpTopologyV1BGPPeerModalColumns(), + Sort: &topologyv1.ModalSort{Column: "neighbor_ip", Direction: "asc"}, + EmptyLabel: "No BGP peers", + } +} + func modalSelectedSidePortColumn(id, label, selectedSrcPortColumn, selectedDstPortColumn string) topologyv1.ModalColumn { return topologyv1.ModalColumn{ ID: id, @@ -411,6 +431,7 @@ func snmpTopologyV1LinkTypeSpecs() []snmpTopologyV1LinkTypeSpec { {id: snmpTopologyV1LinkARP, label: "ARP", colorSlot: "muted", lineStyle: "solid", width: "normal", semanticRole: "normal"}, {id: snmpTopologyV1LinkL3Subnet, label: "L3 subnet", colorSlot: "info", lineStyle: "dashed", width: "normal", semanticRole: "normal"}, {id: snmpTopologyV1LinkOSPF, label: "OSPF adjacency", colorSlot: "purple", lineStyle: "dashed", width: "normal", semanticRole: "control"}, + {id: snmpTopologyV1LinkBGP, label: "BGP adjacency", colorSlot: "accent", lineStyle: "dashed", width: "normal", semanticRole: "control"}, {id: snmpTopologyV1LinkSNMP, label: "SNMP", colorSlot: "primary", lineStyle: "solid", width: "normal", semanticRole: "normal"}, {id: snmpTopologyV1LinkProbable, label: "Probable", colorSlot: "dim", lineStyle: "solid", width: "normal", semanticRole: "normal"}, {id: snmpTopologyV1LinkObservation, label: "L2 observation", colorSlot: "neutral", lineStyle: "solid", width: "normal", semanticRole: "normal"}, @@ -480,6 +501,13 @@ func snmpTopologyV1EvidenceMatchColumnsForType(linkType string) []string { "dst_ip", } } + if linkType == snmpTopologyV1LinkBGP { + return []string{ + "src_actor", + "dst_actor", + "routing_instance", + } + } return []string{ "src_actor", "dst_actor", @@ -520,6 +548,7 @@ func snmpTopologyV1Presentation() *topologyv1.Presentation { {Type: snmpTopologyV1LinkBridge, Label: "Bridge"}, {Type: snmpTopologyV1LinkL3Subnet, Label: "L3 subnet"}, {Type: snmpTopologyV1LinkOSPF, Label: "OSPF adjacency"}, + {Type: snmpTopologyV1LinkBGP, Label: "BGP adjacency"}, {Type: snmpTopologyV1LinkProbable, Label: "Probable"}, }, Ports: []topologyv1.LegendEntry{ diff --git a/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_table_types.go b/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_table_types.go index 2ac6175d88eef8..a3017e7d6516cc 100644 --- a/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_table_types.go +++ b/src/go/plugin/go.d/collector/snmp_topology/func_topology_v1_table_types.go @@ -121,6 +121,41 @@ func snmpTopologyV1OSPFNeighborsTableType() topologyv1.TableType { } } +func snmpTopologyV1BGPPeersTableType() topologyv1.TableType { + return topologyv1.TableType{ + Role: "actor_detail", + Owner: "actor", + Aggregation: "append", + Columns: snmpTopologyV1BGPPeersColumns(), + Presentation: &topologyv1.TableTypePresentation{ + Label: "BGP Peers", + Order: 5, + Columns: snmpTopologyV1BGPPeerModalColumns(), + }, + } +} + +func snmpTopologyV1BGPPeerModalColumns() []topologyv1.ModalColumn { + return []topologyv1.ModalColumn{ + modalDirectColumn("neighbor_ip", "Neighbor IP", "neighbor_ip", "text"), + modalDirectColumn("remote_as", "Remote AS", "remote_as", "text"), + modalDirectColumn("state", "State", "state", "badge"), + modalActorRefColumn("remote_actor", "Remote Actor", "remote_actor"), + modalDirectColumnWithVisibility("routing_instance", "Routing Instance", "routing_instance", "text", "expanded"), + modalDirectColumnWithVisibility("admin_status", "Admin", "admin_status", "badge", "expanded"), + modalDirectColumnWithVisibility("local_ip", "Local IP", "local_ip", "text", "expanded"), + modalDirectColumnWithVisibility("local_as", "Local AS", "local_as", "text", "expanded"), + modalDirectColumnWithVisibility("local_identifier", "Local Identifier", "local_identifier", "text", "expanded"), + modalDirectColumnWithVisibility("peer_identifier", "Peer Identifier", "peer_identifier", "text", "expanded"), + modalDirectColumnWithVisibility("peer_type", "Peer Type", "peer_type", "badge", "expanded"), + modalDirectColumnWithVisibility("bgp_version", "BGP Version", "bgp_version", "text", "expanded"), + modalDirectColumnWithVisibility("established_uptime", "Established Uptime", "established_uptime", "duration", "expanded"), + modalDirectColumnWithVisibility("last_received_update_age", "Last Update Age", "last_received_update_age", "duration", "expanded"), + modalDirectColumnWithVisibility("description", "Description", "description", "text", "expanded"), + modalDirectColumnWithVisibility("source", "Source", "source", "badge", "expanded"), + } +} + func snmpTopologyV1OSPFNeighborModalColumns() []topologyv1.ModalColumn { return []topologyv1.ModalColumn{ modalDirectColumn("neighbor_router_id", "Neighbor Router ID", "neighbor_router_id", "text"), @@ -201,3 +236,25 @@ func snmpTopologyV1OSPFNeighborsColumns() []topologyv1.Column { topologyv1.NewColumn("source", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), } } + +func snmpTopologyV1BGPPeersColumns() []topologyv1.Column { + return []topologyv1.Column{ + topologyv1.NewColumn("actor", "actor_ref", topologyv1.WithRole("reference")), + topologyv1.NewColumn("remote_actor", "actor_ref", topologyv1.WithNullable(), topologyv1.WithRole("reference")), + topologyv1.NewColumn("routing_instance", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), + topologyv1.NewColumn("neighbor_ip", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), + topologyv1.NewColumn("remote_as", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), + topologyv1.NewColumn("state", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), + topologyv1.NewColumn("admin_status", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), + topologyv1.NewColumn("local_ip", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), + topologyv1.NewColumn("local_as", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), + topologyv1.NewColumn("local_identifier", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), + topologyv1.NewColumn("peer_identifier", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), + topologyv1.NewColumn("peer_type", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), + topologyv1.NewColumn("bgp_version", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), + topologyv1.NewColumn("description", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), + topologyv1.NewColumn("established_uptime", "uint", topologyv1.WithNullable()), + topologyv1.NewColumn("last_received_update_age", "uint", topologyv1.WithNullable()), + topologyv1.NewColumn("source", "string_ref", topologyv1.WithNullable(), topologyv1.WithDictionary("strings")), + } +} diff --git a/src/go/plugin/go.d/collector/snmp_topology/profile_filter_test.go b/src/go/plugin/go.d/collector/snmp_topology/profile_filter_test.go index e31a48aa405b41..2accfd8ffc373c 100644 --- a/src/go/plugin/go.d/collector/snmp_topology/profile_filter_test.go +++ b/src/go/plugin/go.d/collector/snmp_topology/profile_filter_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/require" "github.com/netdata/netdata/go/plugins/plugin/go.d/collector/snmp/ddsnmp" + "github.com/netdata/netdata/go/plugins/plugin/go.d/collector/snmp/ddsnmp/ddprofiledefinition" ) func TestFindTopologyProfiles_UsesDeclarativeProfileExtensions(t *testing.T) { @@ -70,3 +71,63 @@ func TestFindTopologyProfiles_UsesDeclarativeProfileExtensions(t *testing.T) { }) } } + +func TestFindTopologyProfiles_PrunesBGPToTopologyFields(t *testing.T) { + tests := map[string]struct { + manualProfile string + validate func(t *testing.T, row ddprofiledefinition.BGPConfig) + }{ + "standard_peer_keeps_topology_fields": { + manualProfile: "f5-big-ip", + validate: func(t *testing.T, row ddprofiledefinition.BGPConfig) { + t.Helper() + + assert.NotEqual(t, ddprofiledefinition.BGPAdminConfig{}, row.Admin) + assert.NotEqual(t, ddprofiledefinition.BGPStateConfig{}, row.State) + assert.NotEqual(t, ddprofiledefinition.BGPConnectionConfig{}, row.Connection) + assert.Equal(t, ddprofiledefinition.BGPTrafficConfig{}, row.Traffic) + assert.Equal(t, ddprofiledefinition.BGPTransitionsConfig{}, row.Transitions) + assert.Equal(t, ddprofiledefinition.BGPTimersConfig{}, row.Timers) + assert.Equal(t, ddprofiledefinition.BGPLastErrorConfig{}, row.LastError) + assert.Empty(t, row.MetricTags) + }, + }, + "huawei_peer_keeps_single_anchor_category": { + manualProfile: "huawei-routers", + validate: func(t *testing.T, row ddprofiledefinition.BGPConfig) { + t.Helper() + + assert.Equal(t, ddprofiledefinition.BGPAdminConfig{}, row.Admin) + assert.Equal(t, ddprofiledefinition.BGPStateConfig{}, row.State) + assert.Equal(t, ddprofiledefinition.BGPConnectionConfig{}, row.Connection) + assert.NotEqual(t, ddprofiledefinition.BGPTransitionsConfig{}, row.Transitions) + assert.Equal(t, ddprofiledefinition.BGPTrafficConfig{}, row.Traffic) + assert.Empty(t, row.MetricTags) + }, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + profiles := (&Collector{}).findTopologyProfiles(ddsnmp.DeviceConnectionInfo{ + ManualProfiles: []string{tc.manualProfile}, + }) + + var rows []ddprofiledefinition.BGPConfig + for _, profile := range profiles { + require.NotNil(t, profile.Definition) + for _, row := range profile.Definition.BGP { + rows = append(rows, row) + assert.Equal(t, ddprofiledefinition.BGPRowKindPeer, row.Kind) + assert.Equal(t, ddprofiledefinition.BGPRoutesConfig{}, row.Routes) + assert.Equal(t, ddprofiledefinition.BGPRouteLimitsConfig{}, row.RouteLimits) + assert.Equal(t, ddprofiledefinition.BGPDeviceCountsConfig{}, row.Device) + assert.Empty(t, row.StaticTags) + } + } + + require.Len(t, rows, 1) + tc.validate(t, rows[0]) + }) + } +} diff --git a/src/go/plugin/go.d/collector/snmp_topology/topology_bgp_links.go b/src/go/plugin/go.d/collector/snmp_topology/topology_bgp_links.go new file mode 100644 index 00000000000000..ab76b6dfd4c839 --- /dev/null +++ b/src/go/plugin/go.d/collector/snmp_topology/topology_bgp_links.go @@ -0,0 +1,335 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package snmptopology + +import ( + "sort" + "strings" +) + +const topologyBGPAdjacencyLinkType = "bgp_adjacency" + +type topologyBGPEnrichmentStats struct { + observedRows int + emittedLinks int + attachedPeerRows int + suppressedNonEstablished int + suppressedUnresolvedLocal int + suppressedUnresolvedNeighbor int + suppressedSelfActor int + suppressedDuplicateLink int +} + +func applyTopologyBGPAdjacencyEnrichment(data *topologyData, aggregate topologyObservationAggregate) topologyBGPEnrichmentStats { + var stats topologyBGPEnrichmentStats + if data == nil || len(aggregate.bgpPeers) == 0 { + recordTopologyBGPEnrichmentStats(data, stats) + return stats + } + + resolver := newTopologyBGPActorResolver(data, aggregate) + seen := existingTopologyBGPLinkKeys(data.Links) + peerRowsByActor := make(map[string][]map[string]any) + + for _, row := range aggregate.bgpPeers { + stats.observedRows++ + localRef, localOK := resolver.resolveDeviceID(row.DeviceID) + remoteRef, remoteOK := resolver.resolveBGPPeer(row) + if localOK { + modalRow := topologyBGPPeerActorRow(row) + if remoteOK { + modalRow["remote_actor_id"] = remoteRef.actorID + } + peerRowsByActor[localRef.actorID] = append(peerRowsByActor[localRef.actorID], modalRow) + stats.attachedPeerRows++ + } + + if !isBGPPeerEstablished(row) { + stats.suppressedNonEstablished++ + continue + } + if !localOK { + stats.suppressedUnresolvedLocal++ + continue + } + if !remoteOK { + stats.suppressedUnresolvedNeighbor++ + continue + } + if localRef.actorID == remoteRef.actorID { + stats.suppressedSelfActor++ + continue + } + + key := topologyBGPPeerLinkKeyParts(row, localRef.actorID, remoteRef.actorID) + if _, exists := seen[key]; exists { + stats.suppressedDuplicateLink++ + continue + } + seen[key] = struct{}{} + data.Links = append(data.Links, topologyBGPAdjacencyLink(row, localRef, remoteRef)) + stats.emittedLinks++ + } + + attachTopologyBGPPeerRows(data, peerRowsByActor) + sort.Slice(data.Links, func(i, j int) bool { + return topologyLinkSortKey(data.Links[i]) < topologyLinkSortKey(data.Links[j]) + }) + recordTopologyBGPEnrichmentStats(data, stats) + recomputeTopologyLinkStats(data) + return stats +} + +type topologyBGPActorResolver struct { + l3 topologyL3ActorResolver + byIdentifier map[string]topologyL3ActorRef +} + +func newTopologyBGPActorResolver(data *topologyData, aggregate topologyObservationAggregate) topologyBGPActorResolver { + resolver := topologyBGPActorResolver{ + l3: newTopologyL3ActorResolver(data, aggregate.snapshots), + byIdentifier: make(map[string]topologyL3ActorRef), + } + for _, row := range aggregate.l3Interfaces { + ref, ok := resolver.l3.resolveDeviceID(row.DeviceID) + if ok { + resolver.l3.addUniqueIP(row.IP, ref) + } + } + for _, row := range aggregate.bgpPeers { + ref, ok := resolver.l3.resolveDeviceID(row.DeviceID) + if !ok { + continue + } + resolver.addUniqueIdentifier(row.LocalIdentifier, ref) + resolver.l3.addUniqueIP(row.LocalIP, ref) + } + return resolver +} + +func (r topologyBGPActorResolver) resolveDeviceID(deviceID string) (topologyL3ActorRef, bool) { + return r.l3.resolveDeviceID(deviceID) +} + +func (r topologyBGPActorResolver) resolveBGPPeer(row topologyBGPPeer) (topologyL3ActorRef, bool) { + if ref, ok := r.byIdentifier[normalizeBGPRouterID(row.PeerIdentifier)]; ok && ref.actorID != "" { + return ref, true + } + if ref, ok := r.l3.byRouterID[normalizeOSPFRouterID(row.PeerIdentifier)]; ok && ref.actorID != "" { + return ref, true + } + if ref, ok := r.l3.byIP[normalizeNonUnspecifiedIPAddress(row.NeighborIP)]; ok && ref.actorID != "" { + return ref, true + } + return topologyL3ActorRef{}, false +} + +func (r topologyBGPActorResolver) addUniqueIdentifier(identifier string, ref topologyL3ActorRef) { + identifier = normalizeBGPRouterID(identifier) + if identifier == "" || ref.actorID == "" { + return + } + existing, ok := r.byIdentifier[identifier] + if !ok { + r.byIdentifier[identifier] = ref + return + } + if existing.actorID != "" && existing.actorID != ref.actorID { + r.byIdentifier[identifier] = topologyL3ActorRef{} + } +} + +type topologyBGPEndpoint struct { + identifier string + ip string + asn string +} + +func topologyBGPAdjacencyLink(row topologyBGPPeer, srcRef, dstRef topologyL3ActorRef) topologyLink { + src := topologyBGPEndpoint{ + identifier: row.LocalIdentifier, + ip: row.LocalIP, + asn: row.LocalAS, + } + dst := topologyBGPEndpoint{ + identifier: row.PeerIdentifier, + ip: row.NeighborIP, + asn: row.RemoteAS, + } + if srcRef.actorID > dstRef.actorID { + srcRef, dstRef = dstRef, srcRef + src, dst = dst, src + } + + return topologyLink{ + Layer: "3", + Protocol: topologyBGPAdjacencyLinkType, + LinkType: topologyBGPAdjacencyLinkType, + Direction: "observed", + State: "established", + SrcActorID: srcRef.actorID, + DstActorID: dstRef.actorID, + Src: topologyLinkEndpoint{ + Match: srcRef.match, + Attributes: topologyBGPEndpointAttributes(src), + }, + Dst: topologyLinkEndpoint{ + Match: dstRef.match, + Attributes: topologyBGPEndpointAttributes(dst), + }, + Metrics: topologyBGPLinkMetrics(row, src, dst), + } +} + +func topologyBGPEndpointAttributes(endpoint topologyBGPEndpoint) map[string]any { + attrs := map[string]any{ + "source": "bgp_mib", + } + if identifier := normalizeBGPRouterID(endpoint.identifier); identifier != "" { + attrs["bgp_identifier"] = identifier + } + if ip := normalizeNonUnspecifiedIPAddress(endpoint.ip); ip != "" { + attrs["ip"] = ip + } + if asn := strings.TrimSpace(endpoint.asn); asn != "" { + attrs["as"] = asn + } + return attrs +} + +func topologyBGPLinkMetrics(row topologyBGPPeer, src, dst topologyBGPEndpoint) map[string]any { + metrics := map[string]any{ + "source": "bgp_mib", + "inference": "bgp_established_adjacency", + "attachment_mode": "logical_l3_bgp", + "state": "established", + "routing_instance": row.RoutingInstance, + } + addStringMetric := func(key, value string) { + if value = strings.TrimSpace(value); value != "" { + metrics[key] = value + } + } + addStringMetric("local_ip", normalizeNonUnspecifiedIPAddress(src.ip)) + addStringMetric("neighbor_ip", normalizeNonUnspecifiedIPAddress(dst.ip)) + addStringMetric("local_as", src.asn) + addStringMetric("remote_as", dst.asn) + addStringMetric("local_identifier", normalizeBGPRouterID(src.identifier)) + addStringMetric("peer_identifier", normalizeBGPRouterID(dst.identifier)) + addStringMetric("peer_type", row.PeerType) + addStringMetric("bgp_version", row.BGPVersion) + return metrics +} + +func topologyBGPPeerActorRow(row topologyBGPPeer) map[string]any { + out := map[string]any{ + "routing_instance": row.RoutingInstance, + "remote_as": row.RemoteAS, + "state": row.State, + "source": "bgp_mib", + } + add := func(key, value string) { + if value = strings.TrimSpace(value); value != "" { + out[key] = value + } + } + add("neighbor_ip", row.NeighborIP) + add("local_ip", row.LocalIP) + add("local_as", row.LocalAS) + add("local_identifier", row.LocalIdentifier) + add("peer_identifier", row.PeerIdentifier) + add("peer_type", row.PeerType) + add("bgp_version", row.BGPVersion) + add("description", row.Description) + add("admin_status", row.AdminStatus) + if row.EstablishedUptime != nil { + out["established_uptime"] = *row.EstablishedUptime + } + if row.LastReceivedUpdateAge != nil { + out["last_received_update_age"] = *row.LastReceivedUpdateAge + } + return out +} + +func attachTopologyBGPPeerRows(data *topologyData, rowsByActor map[string][]map[string]any) { + if data == nil || len(rowsByActor) == 0 { + return + } + for i := range data.Actors { + actor := &data.Actors[i] + rows := rowsByActor[strings.TrimSpace(actor.ActorID)] + if len(rows) == 0 { + continue + } + sortTopologyBGPPeerRows(rows) + if actor.Tables == nil { + actor.Tables = make(map[string][]map[string]any) + } + actor.Tables["bgp_peers"] = rows + } +} + +func existingTopologyBGPLinkKeys(links []topologyLink) map[string]struct{} { + seen := make(map[string]struct{}) + for _, link := range links { + if strings.EqualFold(strings.TrimSpace(firstNonEmptyString(link.LinkType, link.Protocol)), topologyBGPAdjacencyLinkType) { + // Seed the key set so repeated enrichment over already-shaped data remains idempotent. + row := topologyBGPPeer{ + RoutingInstance: topologyMetricValueString(link.Metrics, "routing_instance"), + } + seen[topologyBGPPeerLinkKeyParts(row, link.SrcActorID, link.DstActorID)] = struct{}{} + } + } + return seen +} + +func topologyBGPPeerLinkKeyParts(row topologyBGPPeer, srcActorID, dstActorID string) string { + srcActorID = strings.TrimSpace(srcActorID) + dstActorID = strings.TrimSpace(dstActorID) + if srcActorID > dstActorID { + srcActorID, dstActorID = dstActorID, srcActorID + } + + return topologyL3SubnetLinkKeyParts( + srcActorID, + dstActorID, + topologyBGPRoutingInstanceValue(row.RoutingInstance), + ) +} + +func topologyBGPRoutingInstanceValue(routingInstance string) string { + return firstNonEmptyString(routingInstance, "default") +} + +func recordTopologyBGPEnrichmentStats(data *topologyData, stats topologyBGPEnrichmentStats) { + if data == nil { + return + } + if data.Stats == nil { + data.Stats = make(map[string]any) + } + data.Stats["bgp_peer_rows"] = stats.observedRows + data.Stats["bgp_peer_detail_rows"] = stats.attachedPeerRows + data.Stats["bgp_adjacency_emitted_links"] = stats.emittedLinks + data.Stats["bgp_adjacency_suppressed_non_established_state"] = stats.suppressedNonEstablished + data.Stats["bgp_adjacency_suppressed_unresolved_local"] = stats.suppressedUnresolvedLocal + data.Stats["bgp_adjacency_suppressed_unresolved_neighbor"] = stats.suppressedUnresolvedNeighbor + data.Stats["bgp_adjacency_suppressed_self_actor"] = stats.suppressedSelfActor + data.Stats["bgp_adjacency_suppressed_duplicate_link"] = stats.suppressedDuplicateLink +} + +func recomputeTopologyBGPVisibleLinkStats(data *topologyData) { + if data == nil || data.Stats == nil { + return + } + if _, ok := data.Stats["bgp_adjacency_emitted_links"]; !ok { + return + } + count := 0 + for _, link := range data.Links { + if strings.EqualFold(strings.TrimSpace(firstNonEmptyString(link.LinkType, link.Protocol)), topologyBGPAdjacencyLinkType) { + count++ + } + } + data.Stats["bgp_adjacency_visible_links"] = count +} diff --git a/src/go/plugin/go.d/collector/snmp_topology/topology_bgp_links_test.go b/src/go/plugin/go.d/collector/snmp_topology/topology_bgp_links_test.go new file mode 100644 index 00000000000000..c9b60ee014b031 --- /dev/null +++ b/src/go/plugin/go.d/collector/snmp_topology/topology_bgp_links_test.go @@ -0,0 +1,331 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package snmptopology + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestApplyTopologyBGPAdjacencyEnrichmentEmitsEstablishedManagedLink(t *testing.T) { + data := topologyData{ + Actors: []topologyActor{ + topologyBGPManagedActorForTest("router-a", "device-a", nil, "198.51.100.1"), + topologyBGPManagedActorForTest("router-b", "device-b", nil, "198.51.100.2"), + }, + } + aggregate := topologyObservationAggregate{ + bgpPeers: []topologyBGPPeer{ + bgpPeerForTest("device-a", "default", "198.51.100.1", "198.51.100.2", "65001", "65002", "1.1.1.1", "2.2.2.2", "established"), + }, + } + + stats := applyTopologyBGPAdjacencyEnrichment(&data, aggregate) + + require.Equal(t, 1, stats.emittedLinks) + require.Len(t, data.Links, 1) + link := data.Links[0] + require.Equal(t, topologyBGPAdjacencyLinkType, link.LinkType) + require.Equal(t, "established", link.State) + require.Equal(t, "65001", link.Src.Attributes["as"]) + require.Equal(t, "65002", link.Dst.Attributes["as"]) + require.Equal(t, 1, data.Stats["bgp_peer_rows"]) + require.Equal(t, 1, data.Stats["bgp_peer_detail_rows"]) + require.Equal(t, 1, data.Stats["bgp_adjacency_emitted_links"]) + require.Equal(t, 1, data.Stats["bgp_adjacency_visible_links"]) + require.Len(t, data.Actors[0].Tables["bgp_peers"], 1) + require.Equal(t, "router-b", data.Actors[0].Tables["bgp_peers"][0]["remote_actor_id"]) +} + +func TestApplyTopologyBGPAdjacencyEnrichmentKeepsSuppressedPeersAsDetailOnly(t *testing.T) { + tests := map[string]struct { + data topologyData + aggregate topologyObservationAggregate + wantNonEstablished int + wantUnresolvedNeighbor int + wantSelfActor int + wantDetailState string + wantRemoteActorID string + wantRemoteActorIDAbsent bool + wantSuppressedStatsCounter string + }{ + "non-established-peer": { + data: topologyData{ + Actors: []topologyActor{ + topologyBGPManagedActorForTest("router-a", "device-a", nil, "198.51.100.1"), + topologyBGPManagedActorForTest("router-b", "device-b", nil, "198.51.100.2"), + }, + }, + aggregate: topologyObservationAggregate{ + bgpPeers: []topologyBGPPeer{ + bgpPeerForTest("device-a", "default", "198.51.100.1", "198.51.100.2", "65001", "65002", "1.1.1.1", "2.2.2.2", "active"), + }, + }, + wantNonEstablished: 1, + wantDetailState: "active", + wantRemoteActorID: "router-b", + }, + "unresolved-neighbor": { + data: topologyData{ + Actors: []topologyActor{ + topologyBGPManagedActorForTest("router-a", "device-a", nil, "198.51.100.1"), + }, + }, + aggregate: topologyObservationAggregate{ + bgpPeers: []topologyBGPPeer{ + bgpPeerForTest("device-a", "default", "198.51.100.1", "203.0.113.2", "65001", "65002", "1.1.1.1", "2.2.2.2", "established"), + }, + }, + wantUnresolvedNeighbor: 1, + wantRemoteActorIDAbsent: true, + }, + "self-actor": { + data: topologyData{ + Actors: []topologyActor{ + topologyBGPManagedActorForTest("router-a", "device-a", nil, "198.51.100.1", "198.51.100.2"), + }, + }, + aggregate: topologyObservationAggregate{ + bgpPeers: []topologyBGPPeer{ + bgpPeerForTest("device-a", "default", "198.51.100.1", "198.51.100.2", "65001", "65002", "1.1.1.1", "2.2.2.2", "established"), + }, + }, + wantSelfActor: 1, + wantRemoteActorID: "router-a", + wantSuppressedStatsCounter: "bgp_adjacency_suppressed_self_actor", + }, + "ambiguous-peer-identifier": { + data: topologyData{ + Actors: []topologyActor{ + topologyBGPManagedActorForTest("router-a", "device-a", nil, "198.51.100.1"), + topologyBGPManagedActorForTest("router-b", "device-b", nil, "198.51.100.2"), + topologyBGPManagedActorForTest("router-c", "device-c", nil, "198.51.100.3"), + }, + }, + aggregate: topologyObservationAggregate{ + bgpPeers: []topologyBGPPeer{ + bgpPeerForTest("device-b", "default", "198.51.100.2", "203.0.113.1", "65002", "65001", "2.2.2.2", "", "idle"), + bgpPeerForTest("device-c", "default", "198.51.100.3", "203.0.113.1", "65002", "65001", "2.2.2.2", "", "idle"), + bgpPeerForTest("device-a", "default", "198.51.100.1", "", "65001", "65002", "1.1.1.1", "2.2.2.2", "established"), + }, + }, + wantNonEstablished: 2, + wantUnresolvedNeighbor: 1, + wantRemoteActorIDAbsent: true, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + stats := applyTopologyBGPAdjacencyEnrichment(&tc.data, tc.aggregate) + + require.Zero(t, stats.emittedLinks) + require.Equal(t, tc.wantNonEstablished, stats.suppressedNonEstablished) + require.Equal(t, tc.wantUnresolvedNeighbor, stats.suppressedUnresolvedNeighbor) + require.Equal(t, tc.wantSelfActor, stats.suppressedSelfActor) + require.Empty(t, tc.data.Links) + require.Len(t, tc.data.Actors[0].Tables["bgp_peers"], 1) + row := tc.data.Actors[0].Tables["bgp_peers"][0] + if tc.wantDetailState != "" { + require.Equal(t, tc.wantDetailState, row["state"]) + } + if tc.wantRemoteActorID != "" { + require.Equal(t, tc.wantRemoteActorID, row["remote_actor_id"]) + } + if tc.wantRemoteActorIDAbsent { + require.NotContains(t, row, "remote_actor_id") + } + if tc.wantSuppressedStatsCounter != "" { + require.Equal(t, 1, tc.data.Stats[tc.wantSuppressedStatsCounter]) + } + }) + } +} + +func TestApplyTopologyBGPAdjacencyEnrichmentBuildsManagedLinks(t *testing.T) { + tests := map[string]struct { + data topologyData + aggregate topologyObservationAggregate + validate func(t *testing.T, data topologyData, stats topologyBGPEnrichmentStats) + }{ + "deduplicates-asymmetric-identity-observations": { + data: topologyData{ + Actors: []topologyActor{ + topologyBGPManagedActorForTest("router-a", "device-a", nil, "198.51.100.1"), + topologyBGPManagedActorForTest("router-b", "device-b", nil, "198.51.100.2"), + }, + }, + aggregate: topologyObservationAggregate{ + bgpPeers: []topologyBGPPeer{ + bgpPeerForTest("device-a", "default", "198.51.100.1", "198.51.100.2", "65001", "65002", "1.1.1.1", "2.2.2.2", "established"), + bgpPeerForTest("device-b", "default", "198.51.100.2", "198.51.100.1", "65002", "65001", "", "", "established"), + }, + }, + validate: func(t *testing.T, data topologyData, stats topologyBGPEnrichmentStats) { + t.Helper() + + require.Equal(t, 1, stats.emittedLinks) + require.Equal(t, 1, stats.suppressedDuplicateLink) + require.Len(t, data.Links, 1) + require.Len(t, data.Actors[0].Tables["bgp_peers"], 1) + require.Len(t, data.Actors[1].Tables["bgp_peers"], 1) + require.Equal(t, 1, data.Stats["bgp_adjacency_visible_links"]) + }, + }, + "deduplicates-asymmetric-local-ip-observations": { + data: topologyData{ + Actors: []topologyActor{ + topologyBGPManagedActorForTest("router-a", "device-a", nil, "198.51.100.1"), + topologyBGPManagedActorForTest("router-b", "device-b", nil, "198.51.100.2"), + }, + }, + aggregate: topologyObservationAggregate{ + bgpPeers: []topologyBGPPeer{ + bgpPeerForTest("device-a", "default", "", "198.51.100.2", "65001", "65002", "1.1.1.1", "2.2.2.2", "established"), + bgpPeerForTest("device-b", "default", "198.51.100.2", "198.51.100.1", "65002", "65001", "2.2.2.2", "1.1.1.1", "established"), + }, + }, + validate: func(t *testing.T, data topologyData, stats topologyBGPEnrichmentStats) { + t.Helper() + + require.Equal(t, 1, stats.emittedLinks) + require.Equal(t, 1, stats.suppressedDuplicateLink) + require.Len(t, data.Links, 1) + require.Len(t, data.Actors[0].Tables["bgp_peers"], 1) + require.Len(t, data.Actors[1].Tables["bgp_peers"], 1) + require.Equal(t, 1, data.Stats["bgp_adjacency_visible_links"]) + }, + }, + "keeps-routing-instances-separate": { + data: topologyData{ + Actors: []topologyActor{ + topologyBGPManagedActorForTest("router-a", "device-a", nil, "198.51.100.1"), + topologyBGPManagedActorForTest("router-b", "device-b", nil, "198.51.100.2"), + }, + }, + aggregate: topologyObservationAggregate{ + bgpPeers: []topologyBGPPeer{ + bgpPeerForTest("device-a", "blue", "198.51.100.1", "198.51.100.2", "65001", "65002", "1.1.1.1", "2.2.2.2", "established"), + bgpPeerForTest("device-a", "red", "198.51.100.1", "198.51.100.2", "65001", "65002", "1.1.1.1", "2.2.2.2", "established"), + }, + }, + validate: func(t *testing.T, data topologyData, stats topologyBGPEnrichmentStats) { + t.Helper() + + require.Equal(t, 2, stats.emittedLinks) + require.Len(t, data.Links, 2) + require.Equal(t, 2, data.Stats["bgp_adjacency_visible_links"]) + require.ElementsMatch(t, []any{"blue", "red"}, []any{data.Links[0].Metrics["routing_instance"], data.Links[1].Metrics["routing_instance"]}) + }, + }, + "compacts-parallel-same-routing-instance-peers": { + data: topologyData{ + Actors: []topologyActor{ + topologyBGPManagedActorForTest("router-a", "device-a", nil, "198.51.100.1", "198.51.100.5"), + topologyBGPManagedActorForTest("router-b", "device-b", nil, "198.51.100.2", "198.51.100.6"), + }, + }, + aggregate: topologyObservationAggregate{ + bgpPeers: []topologyBGPPeer{ + bgpPeerForTest("device-a", "default", "198.51.100.1", "198.51.100.2", "65001", "65002", "1.1.1.1", "2.2.2.2", "established"), + bgpPeerForTest("device-a", "default", "198.51.100.5", "198.51.100.6", "65001", "65002", "1.1.1.1", "2.2.2.2", "established"), + }, + }, + validate: func(t *testing.T, data topologyData, stats topologyBGPEnrichmentStats) { + t.Helper() + + require.Equal(t, 1, stats.emittedLinks) + require.Equal(t, 1, stats.suppressedDuplicateLink) + require.Len(t, data.Links, 1) + require.Len(t, data.Actors[0].Tables["bgp_peers"], 2) + require.Equal(t, 1, data.Stats["bgp_adjacency_visible_links"]) + }, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + stats := applyTopologyBGPAdjacencyEnrichment(&tc.data, tc.aggregate) + + tc.validate(t, tc.data, stats) + }) + } +} + +func TestApplyTopologyBGPAdjacencyEnrichmentCanonicalizesUndirectedLinkEndpoints(t *testing.T) { + data := topologyData{ + Actors: []topologyActor{ + topologyBGPManagedActorForTest("router-a", "device-a", nil, "198.51.100.1"), + topologyBGPManagedActorForTest("router-b", "device-b", nil, "198.51.100.2"), + }, + } + aggregate := topologyObservationAggregate{ + bgpPeers: []topologyBGPPeer{ + bgpPeerForTest("device-b", "default", "198.51.100.2", "198.51.100.1", "65002", "65001", "2.2.2.2", "1.1.1.1", "established"), + }, + } + + stats := applyTopologyBGPAdjacencyEnrichment(&data, aggregate) + + require.Equal(t, 1, stats.emittedLinks) + require.Len(t, data.Links, 1) + link := data.Links[0] + require.Equal(t, "router-a", link.SrcActorID) + require.Equal(t, "router-b", link.DstActorID) + require.Equal(t, "1.1.1.1", link.Src.Attributes["bgp_identifier"]) + require.Equal(t, "2.2.2.2", link.Dst.Attributes["bgp_identifier"]) + require.Equal(t, "198.51.100.1", link.Src.Attributes["ip"]) + require.Equal(t, "198.51.100.2", link.Dst.Attributes["ip"]) + require.Equal(t, "65001", link.Src.Attributes["as"]) + require.Equal(t, "65002", link.Dst.Attributes["as"]) + require.Equal(t, "65001", link.Metrics["local_as"]) + require.Equal(t, "65002", link.Metrics["remote_as"]) + require.Equal(t, "1.1.1.1", link.Metrics["local_identifier"]) + require.Equal(t, "2.2.2.2", link.Metrics["peer_identifier"]) +} + +func TestApplyTopologyBGPAdjacencyEnrichmentResolvesPeerIdentifierByRouterID(t *testing.T) { + data := topologyData{ + Actors: []topologyActor{ + topologyBGPManagedActorForTest("router-a", "device-a", nil, "198.51.100.1"), + topologyBGPManagedActorForTest("router-b", "device-b", map[string]any{tagOSPFRouterID: "2.2.2.2"}, "198.51.100.2"), + }, + } + aggregate := topologyObservationAggregate{ + bgpPeers: []topologyBGPPeer{ + bgpPeerForTest("device-a", "default", "198.51.100.1", "", "65001", "65002", "1.1.1.1", "2.2.2.2", "established"), + }, + } + + stats := applyTopologyBGPAdjacencyEnrichment(&data, aggregate) + + require.Equal(t, 1, stats.emittedLinks) + require.Len(t, data.Links, 1) + require.Equal(t, "router-b", data.Links[0].DstActorID) + require.Equal(t, "2.2.2.2", data.Links[0].Dst.Attributes["bgp_identifier"]) + require.Len(t, data.Actors[0].Tables["bgp_peers"], 1) + require.Equal(t, "router-b", data.Actors[0].Tables["bgp_peers"][0]["remote_actor_id"]) +} + +func topologyBGPManagedActorForTest(actorID, deviceID string, attrs map[string]any, ips ...string) topologyActor { + if attrs == nil { + attrs = make(map[string]any) + } + attrs["device_id"] = deviceID + return topologyL3ManagedActorForTest(actorID, attrs, ips...) +} + +func bgpPeerForTest(deviceID, routingInstance, localIP, neighborIP, localAS, remoteAS, localIdentifier, peerIdentifier, state string) topologyBGPPeer { + return topologyBGPPeer{ + DeviceID: deviceID, + RoutingInstance: routingInstance, + LocalIP: localIP, + NeighborIP: neighborIP, + LocalAS: localAS, + RemoteAS: remoteAS, + LocalIdentifier: localIdentifier, + PeerIdentifier: peerIdentifier, + State: state, + } +} diff --git a/src/go/plugin/go.d/collector/snmp_topology/topology_bgp_peers.go b/src/go/plugin/go.d/collector/snmp_topology/topology_bgp_peers.go new file mode 100644 index 00000000000000..b2163b2be1be60 --- /dev/null +++ b/src/go/plugin/go.d/collector/snmp_topology/topology_bgp_peers.go @@ -0,0 +1,196 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package snmptopology + +import ( + "sort" + "strings" + + "github.com/netdata/netdata/go/plugins/plugin/go.d/collector/snmp/ddsnmp" + "github.com/netdata/netdata/go/plugins/plugin/go.d/collector/snmp/ddsnmp/ddprofiledefinition" +) + +func (c *topologyCache) ingestTopologyBGPPeers(pms []*ddsnmp.ProfileMetrics) { + if c == nil { + return + } + + c.mu.Lock() + defer c.mu.Unlock() + + if c.bgpPeersByKey == nil { + c.bgpPeersByKey = make(map[string]topologyBGPPeer) + } + for _, pm := range pms { + if pm == nil || pm.BGPCollectError != nil { + continue + } + for _, row := range pm.BGPRows { + peer, ok := topologyBGPPeerFromRow(row) + if !ok { + continue + } + c.bgpPeersByKey[topologyBGPPeerCacheKey(row, peer)] = peer + } + } +} + +func topologyBGPPeerFromRow(row ddsnmp.BGPRow) (topologyBGPPeer, bool) { + if row.Kind != ddprofiledefinition.BGPRowKindPeer { + return topologyBGPPeer{}, false + } + + neighbor := firstNonEmptyString(row.Identity.Neighbor, row.Tags["neighbor"]) + remoteAS := firstNonEmptyString(row.Identity.RemoteAS, row.Tags["remote_as"]) + if strings.TrimSpace(neighbor) == "" || strings.TrimSpace(remoteAS) == "" { + return topologyBGPPeer{}, false + } + + peer := topologyBGPPeer{ + RoutingInstance: topologyBGPRoutingInstance(row), + NeighborIP: topologyBGPPeerAddressValue(neighbor), + RemoteAS: strings.TrimSpace(remoteAS), + LocalIP: topologyBGPPeerAddressValue(row.Descriptors.LocalAddress), + LocalAS: strings.TrimSpace(row.Descriptors.LocalAS), + LocalIdentifier: normalizeBGPRouterID(row.Descriptors.LocalIdentifier), + PeerIdentifier: normalizeBGPRouterID(row.Descriptors.PeerIdentifier), + PeerType: strings.TrimSpace(row.Descriptors.PeerType), + BGPVersion: strings.TrimSpace(row.Descriptors.BGPVersion), + Description: strings.TrimSpace(row.Descriptors.Description), + AdminStatus: topologyBGPAdminStatus(row), + State: topologyBGPPeerState(row), + EstablishedUptime: topologyBGPInt64Ptr(row.Connection.EstablishedUptime), + LastReceivedUpdateAge: topologyBGPInt64Ptr(row.Connection.LastReceivedUpdateAge), + } + return peer, true +} + +func topologyBGPPeerAddressValue(value string) string { + value = strings.TrimSpace(value) + if value == "" { + return "" + } + if ip := normalizeNonUnspecifiedIPAddress(value); ip != "" { + return ip + } + if normalizeIPAddress(value) != "" { + return "" + } + return value +} + +func topologyBGPRoutingInstance(row ddsnmp.BGPRow) string { + return firstNonEmptyString(row.Identity.RoutingInstance, row.Tags["routing_instance"], "default") +} + +func topologyBGPPeerCacheKey(row ddsnmp.BGPRow, peer topologyBGPPeer) string { + if key := strings.TrimSpace(row.StructuralID); key != "" { + return key + } + return topologyL3SubnetLinkKeyParts( + row.OriginProfileID, + row.Table, + row.RowKey, + peer.RoutingInstance, + peer.NeighborIP, + peer.RemoteAS, + ) +} + +func (c *topologyCache) snapshotBGPPeers(localDeviceID string) []topologyBGPPeer { + if c == nil || len(c.bgpPeersByKey) == 0 { + return nil + } + + keys := sortedMapKeys(c.bgpPeersByKey) + rows := make([]topologyBGPPeer, 0, len(keys)) + for _, key := range keys { + row := c.bgpPeersByKey[key] + row.DeviceID = strings.TrimSpace(localDeviceID) + if row.DeviceID == "" || (row.NeighborIP == "" && row.PeerIdentifier == "") { + continue + } + rows = append(rows, row) + } + return rows +} + +func normalizeBGPRouterID(value string) string { + value = strings.TrimSpace(value) + if value == "" { + return "" + } + if ip := normalizeNonUnspecifiedIPAddress(value); ip != "" { + return ip + } + return value +} + +func topologyBGPAdminStatus(row ddsnmp.BGPRow) string { + if !row.Admin.Enabled.Has { + return "" + } + if row.Admin.Enabled.Value { + return "enabled" + } + return "disabled" +} + +func topologyBGPPeerState(row ddsnmp.BGPRow) string { + if row.State.Has && row.State.State != "" { + return normalizeTopologyBGPPeerState(string(row.State.State)) + } + return normalizeTopologyBGPPeerState(row.State.Raw) +} + +func normalizeTopologyBGPPeerState(state string) string { + state = strings.TrimSpace(state) + if state == "" { + return "" + } + normalized := strings.ToLower(strings.NewReplacer("_", "", "-", "", " ", "").Replace(state)) + switch normalized { + case "1", "idle": + return string(ddprofiledefinition.BGPPeerStateIdle) + case "2", "connect": + return string(ddprofiledefinition.BGPPeerStateConnect) + case "3", "active": + return string(ddprofiledefinition.BGPPeerStateActive) + case "4", "opensent": + return string(ddprofiledefinition.BGPPeerStateOpenSent) + case "5", "openconfirm": + return string(ddprofiledefinition.BGPPeerStateOpenConfirm) + case "6", "established": + return string(ddprofiledefinition.BGPPeerStateEstablished) + default: + return state + } +} + +func topologyBGPInt64Ptr(value ddsnmp.BGPInt64) *int64 { + if !value.Has { + return nil + } + v := value.Value + return &v +} + +func isBGPPeerEstablished(row topologyBGPPeer) bool { + return strings.EqualFold(strings.TrimSpace(row.State), string(ddprofiledefinition.BGPPeerStateEstablished)) +} + +func sortTopologyBGPPeerRows(rows []map[string]any) { + sort.Slice(rows, func(i, j int) bool { + return topologyBGPPeerActorRowSortKey(rows[i]) < topologyBGPPeerActorRowSortKey(rows[j]) + }) +} + +func topologyBGPPeerActorRowSortKey(row map[string]any) string { + return strings.Join([]string{ + anyStringValue(row["routing_instance"]), + anyStringValue(row["remote_as"]), + topologyBGPPeerAddressValue(anyStringValue(row["neighbor_ip"])), + anyStringValue(row["peer_identifier"]), + anyStringValue(row["state"]), + }, "\x00") +} diff --git a/src/go/plugin/go.d/collector/snmp_topology/topology_bgp_peers_test.go b/src/go/plugin/go.d/collector/snmp_topology/topology_bgp_peers_test.go new file mode 100644 index 00000000000000..e55705b0fd755a --- /dev/null +++ b/src/go/plugin/go.d/collector/snmp_topology/topology_bgp_peers_test.go @@ -0,0 +1,226 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +package snmptopology + +import ( + "errors" + "testing" + + topologyv1 "github.com/netdata/netdata/go/plugins/pkg/topology/v1" + "github.com/netdata/netdata/go/plugins/plugin/go.d/collector/snmp/ddsnmp" + "github.com/netdata/netdata/go/plugins/plugin/go.d/collector/snmp/ddsnmp/ddprofiledefinition" + "github.com/stretchr/testify/require" +) + +func TestTopologyBGPPeerFromRowNormalizesState(t *testing.T) { + tests := map[string]struct { + state ddsnmp.BGPState + want string + }{ + "typed-numeric": { + state: ddsnmp.BGPState{Has: true, State: ddprofiledefinition.BGPPeerState("6")}, + want: "established", + }, + "typed-case": { + state: ddsnmp.BGPState{Has: true, State: ddprofiledefinition.BGPPeerState("Established")}, + want: "established", + }, + "typed-open-sent-variant": { + state: ddsnmp.BGPState{Has: true, State: ddprofiledefinition.BGPPeerState("open_sent")}, + want: "opensent", + }, + "raw-numeric": { + state: ddsnmp.BGPState{Raw: "6"}, + want: "established", + }, + "unknown": { + state: ddsnmp.BGPState{Has: true, State: ddprofiledefinition.BGPPeerState("vendor-specific")}, + want: "vendor-specific", + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + row := bgpRowForTest("peer-1", "192.0.2.2", "65002") + row.State = tc.state + + peer, ok := topologyBGPPeerFromRow(row) + + require.True(t, ok) + require.Equal(t, tc.want, peer.State) + }) + } +} + +func TestTopologyBGPPeerFromRowKeepsOnlyDiagnosticRawAddresses(t *testing.T) { + tests := map[string]struct { + neighbor string + localAddr string + wantPeer topologyBGPPeer + }{ + "normalizes-non-unspecified-ip-addresses": { + neighbor: "C0000202", + localAddr: "192.0.2.1", + wantPeer: topologyBGPPeer{ + NeighborIP: "192.0.2.2", + LocalIP: "192.0.2.1", + }, + }, + "drops-unspecified-ip-addresses": { + neighbor: "0.0.0.0", + localAddr: "::", + wantPeer: topologyBGPPeer{}, + }, + "preserves-raw-non-ip-diagnostics": { + neighbor: "peer-token", + localAddr: "local-token", + wantPeer: topologyBGPPeer{ + NeighborIP: "peer-token", + LocalIP: "local-token", + }, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + row := bgpRowForTest("peer-1", tc.neighbor, "65002") + row.Descriptors.LocalAddress = tc.localAddr + + peer, ok := topologyBGPPeerFromRow(row) + + require.True(t, ok) + require.Equal(t, tc.wantPeer.NeighborIP, peer.NeighborIP) + require.Equal(t, tc.wantPeer.LocalIP, peer.LocalIP) + }) + } +} + +func TestSortTopologyBGPPeerRowsUsesRawNeighborFallback(t *testing.T) { + rows := []map[string]any{ + { + "routing_instance": "default", + "remote_as": "65002", + "neighbor_ip": "raw-b", + "state": "established", + }, + { + "routing_instance": "default", + "remote_as": "65002", + "neighbor_ip": "raw-a", + "state": "established", + }, + } + + sortTopologyBGPPeerRows(rows) + + require.Equal(t, "raw-a", rows[0]["neighbor_ip"]) + require.Equal(t, "raw-b", rows[1]["neighbor_ip"]) +} + +func TestBuildSNMPTopologyV1BGPPeersTableHandlesRawAndUnspecifiedAddresses(t *testing.T) { + tests := map[string]struct { + values map[string]any + wantNeighborIPs []string + wantLocalIPs []string + wantRawValues map[string][]any + }{ + "preserves-raw-non-ip-diagnostics": { + values: map[string]any{ + "neighbor_ip": "peer-token", + "local_ip": "local-token", + }, + wantNeighborIPs: []string{"peer-token"}, + wantLocalIPs: []string{"local-token"}, + }, + "drops-unspecified-ip-addresses": { + values: map[string]any{ + "neighbor_ip": "0.0.0.0", + "local_ip": "::", + }, + wantRawValues: map[string][]any{ + "neighbor_ip": {nil}, + "local_ip": {nil}, + }, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + stringsDict := topologyv1.NewStringDictionary() + + table := buildSNMPTopologyV1BGPPeersTable([]topologyV1DynamicRow{ + { + actorRef: 0, + values: tc.values, + }, + }, nil, stringsDict) + data := topologyv1.Data{ + Dictionaries: topologyv1.Dictionaries{ + "strings": stringsDict.Values(), + }, + } + + if tc.wantNeighborIPs != nil { + require.Equal(t, tc.wantNeighborIPs, topologyV1StringColumnValues(t, data, table, "neighbor_ip")) + } + if tc.wantLocalIPs != nil { + require.Equal(t, tc.wantLocalIPs, topologyV1StringColumnValues(t, data, table, "local_ip")) + } + for column, values := range tc.wantRawValues { + require.Equal(t, values, topologyV1ColumnValues(t, table, column)) + } + }) + } +} + +func TestTopologyCacheIngestTopologyBGPPeersSkipsErrorsAndInvalidRows(t *testing.T) { + cache := newTopologyCache() + + cache.ingestTopologyBGPPeers([]*ddsnmp.ProfileMetrics{ + nil, + { + BGPCollectError: errors.New("collect failed"), + BGPRows: []ddsnmp.BGPRow{ + bgpRowForTest("collect-error", "192.0.2.2", "65002"), + }, + }, + { + BGPRows: []ddsnmp.BGPRow{ + bgpRowForTest("valid", "192.0.2.3", "65003"), + bgpRowForTest("missing-neighbor", "", "65004"), + bgpRowForTest("missing-remote-as", "192.0.2.5", ""), + { + Kind: ddprofiledefinition.BGPRowKindPeerFamily, + Identity: ddsnmp.BGPIdentity{ + Neighbor: "192.0.2.6", + RemoteAS: "65006", + }, + }, + }, + }, + }) + + require.Len(t, cache.bgpPeersByKey, 1) + require.Contains(t, cache.bgpPeersByKey, "valid") + require.Equal(t, "192.0.2.3", cache.bgpPeersByKey["valid"].NeighborIP) +} + +func bgpRowForTest(structuralID, neighbor, remoteAS string) ddsnmp.BGPRow { + return ddsnmp.BGPRow{ + Kind: ddprofiledefinition.BGPRowKindPeer, + StructuralID: structuralID, + Identity: ddsnmp.BGPIdentity{ + RoutingInstance: "default", + Neighbor: neighbor, + RemoteAS: remoteAS, + }, + Descriptors: ddsnmp.BGPDescriptors{ + LocalAddress: "192.0.2.1", + LocalAS: "65001", + }, + State: ddsnmp.BGPState{ + Has: true, + State: ddprofiledefinition.BGPPeerStateEstablished, + }, + } +} diff --git a/src/go/plugin/go.d/collector/snmp_topology/topology_cache.go b/src/go/plugin/go.d/collector/snmp_topology/topology_cache.go index b7b5849151c5e6..0e5762a44f1beb 100644 --- a/src/go/plugin/go.d/collector/snmp_topology/topology_cache.go +++ b/src/go/plugin/go.d/collector/snmp_topology/topology_cache.go @@ -37,6 +37,7 @@ type topologyCache struct { stpPorts map[string]*stpPortEntry arpEntries map[string]*arpEntry ospfNeighborsByKey map[string]topologyOSPFNeighbor + bgpPeersByKey map[string]topologyBGPPeer } type ifStatus struct { @@ -153,3 +154,21 @@ type topologyOSPFNeighbor struct { Prefix int RemoteActorID string } + +type topologyBGPPeer struct { + DeviceID string + RoutingInstance string + NeighborIP string + RemoteAS string + LocalIP string + LocalAS string + LocalIdentifier string + PeerIdentifier string + PeerType string + BGPVersion string + Description string + AdminStatus string + State string + EstablishedUptime *int64 + LastReceivedUpdateAge *int64 +} diff --git a/src/go/plugin/go.d/collector/snmp_topology/topology_cache_lifecycle.go b/src/go/plugin/go.d/collector/snmp_topology/topology_cache_lifecycle.go index 14b8c3e2e4e555..ba85911fefa38e 100644 --- a/src/go/plugin/go.d/collector/snmp_topology/topology_cache_lifecycle.go +++ b/src/go/plugin/go.d/collector/snmp_topology/topology_cache_lifecycle.go @@ -24,6 +24,7 @@ func newTopologyCache() *topologyCache { stpPorts: make(map[string]*stpPortEntry), arpEntries: make(map[string]*arpEntry), ospfNeighborsByKey: make(map[string]topologyOSPFNeighbor), + bgpPeersByKey: make(map[string]topologyBGPPeer), } } @@ -57,6 +58,7 @@ func (c *topologyCache) replaceWith(src *topologyCache) { c.stpPorts = src.stpPorts c.arpEntries = src.arpEntries c.ospfNeighborsByKey = src.ospfNeighborsByKey + c.bgpPeersByKey = src.bgpPeersByKey } func (c *topologyCache) hasFreshSnapshotAt(now time.Time) bool { diff --git a/src/go/plugin/go.d/collector/snmp_topology/topology_cache_test.go b/src/go/plugin/go.d/collector/snmp_topology/topology_cache_test.go index dc11aceedb9c22..c6f58c96f95e76 100644 --- a/src/go/plugin/go.d/collector/snmp_topology/topology_cache_test.go +++ b/src/go/plugin/go.d/collector/snmp_topology/topology_cache_test.go @@ -11,6 +11,7 @@ import ( topologyengine "github.com/netdata/netdata/go/plugins/pkg/l2topology" "github.com/netdata/netdata/go/plugins/plugin/go.d/collector/snmp/ddsnmp" + "github.com/netdata/netdata/go/plugins/plugin/go.d/collector/snmp/ddsnmp/ddprofiledefinition" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -1349,6 +1350,72 @@ func TestTopologyCache_IngestTopologyProfileMetrics_IncludesTopologyMetrics(t *t require.Empty(t, cache.localDevice.Labels["sys_uptime"]) } +func TestTopologyCache_IngestTopologyBGPPeers_IncludesOnlyPeerRows(t *testing.T) { + established := int64(300) + cache := newTopologyCache() + + cache.ingestTopologyBGPPeers([]*ddsnmp.ProfileMetrics{ + { + BGPRows: []ddsnmp.BGPRow{ + { + Kind: ddprofiledefinition.BGPRowKindPeer, + StructuralID: "peer-1", + Identity: ddsnmp.BGPIdentity{ + RoutingInstance: "blue", + Neighbor: "192.0.2.2", + RemoteAS: "65002", + }, + Descriptors: ddsnmp.BGPDescriptors{ + LocalAddress: "192.0.2.1", + LocalAS: "65001", + LocalIdentifier: "1.1.1.1", + PeerIdentifier: "2.2.2.2", + PeerType: "external", + BGPVersion: "4", + Description: "edge-peer", + }, + Admin: ddsnmp.BGPAdmin{ + Enabled: ddsnmp.BGPBool{Has: true, Value: true}, + }, + State: ddsnmp.BGPState{ + Has: true, + State: ddprofiledefinition.BGPPeerStateEstablished, + }, + Connection: ddsnmp.BGPConnection{ + EstablishedUptime: ddsnmp.BGPInt64{Has: true, Value: established}, + }, + }, + { + Kind: ddprofiledefinition.BGPRowKindPeerFamily, + Identity: ddsnmp.BGPIdentity{ + Neighbor: "192.0.2.2", + RemoteAS: "65002", + AddressFamily: ddprofiledefinition.BGPAddressFamilyIPv4, + SubsequentAddressFamily: ddprofiledefinition.BGPSubsequentAddressFamilyUnicast, + }, + }, + }, + }, + }) + + require.Len(t, cache.bgpPeersByKey, 1) + peer := cache.bgpPeersByKey["peer-1"] + require.Equal(t, "blue", peer.RoutingInstance) + require.Equal(t, "192.0.2.2", peer.NeighborIP) + require.Equal(t, "65002", peer.RemoteAS) + require.Equal(t, "192.0.2.1", peer.LocalIP) + require.Equal(t, "65001", peer.LocalAS) + require.Equal(t, "1.1.1.1", peer.LocalIdentifier) + require.Equal(t, "2.2.2.2", peer.PeerIdentifier) + require.Equal(t, "external", peer.PeerType) + require.Equal(t, "4", peer.BGPVersion) + require.Equal(t, "edge-peer", peer.Description) + require.Equal(t, "enabled", peer.AdminStatus) + require.Equal(t, "established", peer.State) + require.NotNil(t, peer.EstablishedUptime) + require.Equal(t, established, *peer.EstablishedUptime) +} + func TestBuildLocalTopologyDevice_MapsVersionToSoftwareOnly(t *testing.T) { dev := ddsnmp.DeviceConnectionInfo{ Hostname: "10.0.0.2", diff --git a/src/go/plugin/go.d/collector/snmp_topology/topology_registry_build.go b/src/go/plugin/go.d/collector/snmp_topology/topology_registry_build.go index 16e6947317e564..17e04e58833acf 100644 --- a/src/go/plugin/go.d/collector/snmp_topology/topology_registry_build.go +++ b/src/go/plugin/go.d/collector/snmp_topology/topology_registry_build.go @@ -35,6 +35,7 @@ func buildSingleMapTopologySnapshot(aggregate topologyObservationAggregate, opti applySNMPTopologyShapePolicies(&data, options) applyTopologyL3SubnetEnrichment(&data, aggregate) applyTopologyOSPFAdjacencyEnrichment(&data, aggregate) + applyTopologyBGPAdjacencyEnrichment(&data, aggregate) applyTopologyDepthFocusFilter(&data, options) return data, true } @@ -72,6 +73,7 @@ func buildProbableTopologySnapshot(aggregate topologyObservationAggregate, optio markProbableDeltaLinks(&strictData, &probableData) applyTopologyL3SubnetEnrichment(&probableData, aggregate) applyTopologyOSPFAdjacencyEnrichment(&probableData, aggregate) + applyTopologyBGPAdjacencyEnrichment(&probableData, aggregate) applyTopologyDepthFocusFilter(&probableData, options) return probableData, true } diff --git a/src/go/plugin/go.d/collector/snmp_topology/topology_registry_observations.go b/src/go/plugin/go.d/collector/snmp_topology/topology_registry_observations.go index 37fa03adf9162f..0f260f2c6dda4c 100644 --- a/src/go/plugin/go.d/collector/snmp_topology/topology_registry_observations.go +++ b/src/go/plugin/go.d/collector/snmp_topology/topology_registry_observations.go @@ -76,10 +76,12 @@ func aggregateTopologyObservationSnapshots(snapshots []topologyObservationSnapsh totalObservations := 0 totalL3Interfaces := 0 totalOSPFNeighbors := 0 + totalBGPPeers := 0 for _, snapshot := range snapshots { totalObservations += len(snapshot.l2Observations) totalL3Interfaces += len(snapshot.l3Interfaces) totalOSPFNeighbors += len(snapshot.ospfNeighbors) + totalBGPPeers += len(snapshot.bgpPeers) } aggregate := topologyObservationAggregate{ @@ -87,11 +89,13 @@ func aggregateTopologyObservationSnapshots(snapshots []topologyObservationSnapsh l2Observations: make([]topologyengine.L2Observation, 0, totalObservations), l3Interfaces: make([]topologyL3Interface, 0, totalL3Interfaces), ospfNeighbors: make([]topologyOSPFNeighbor, 0, totalOSPFNeighbors), + bgpPeers: make([]topologyBGPPeer, 0, totalBGPPeers), } for _, snapshot := range snapshots { aggregate.l2Observations = append(aggregate.l2Observations, snapshot.l2Observations...) aggregate.l3Interfaces = append(aggregate.l3Interfaces, snapshot.l3Interfaces...) aggregate.ospfNeighbors = append(aggregate.ospfNeighbors, snapshot.ospfNeighbors...) + aggregate.bgpPeers = append(aggregate.bgpPeers, snapshot.bgpPeers...) if aggregate.localDeviceID == "" { aggregate.localDeviceID = snapshot.localDeviceID } diff --git a/src/go/plugin/go.d/collector/snmp_topology/topology_registry_snapshot.go b/src/go/plugin/go.d/collector/snmp_topology/topology_registry_snapshot.go index 60febf26113d12..42bb3e7acef398 100644 --- a/src/go/plugin/go.d/collector/snmp_topology/topology_registry_snapshot.go +++ b/src/go/plugin/go.d/collector/snmp_topology/topology_registry_snapshot.go @@ -13,6 +13,7 @@ type topologyObservationSnapshot struct { l2Observations []topologyengine.L2Observation l3Interfaces []topologyL3Interface ospfNeighbors []topologyOSPFNeighbor + bgpPeers []topologyBGPPeer localDevice topologyDevice localDeviceID string agentID string @@ -24,6 +25,7 @@ type topologyObservationAggregate struct { l2Observations []topologyengine.L2Observation l3Interfaces []topologyL3Interface ospfNeighbors []topologyOSPFNeighbor + bgpPeers []topologyBGPPeer localDeviceID string agentID string collectedAt time.Time @@ -58,6 +60,7 @@ func (c *topologyCache) snapshotEngineObservations() (topologyObservationSnapsho l2Observations: []topologyengine.L2Observation{localObservation}, l3Interfaces: c.snapshotL3Interfaces(localObservation.DeviceID), ospfNeighbors: c.snapshotOSPFNeighbors(localObservation.DeviceID), + bgpPeers: c.snapshotBGPPeers(localObservation.DeviceID), localDevice: local, localDeviceID: localObservation.DeviceID, agentID: strings.TrimSpace(c.agentID), diff --git a/src/go/plugin/go.d/collector/snmp_topology/topology_registry_test.go b/src/go/plugin/go.d/collector/snmp_topology/topology_registry_test.go index 066cf22760643c..593f926edd926c 100644 --- a/src/go/plugin/go.d/collector/snmp_topology/topology_registry_test.go +++ b/src/go/plugin/go.d/collector/snmp_topology/topology_registry_test.go @@ -262,6 +262,139 @@ func TestTopologyRegistry_OSPFSnapshotEnrichesSubnetAfterNeighborIngest(t *testi require.Equal(t, 1, data.Stats["ospf_adjacency_visible_links"]) } +func TestTopologyRegistry_BGPAdjacencyEmitsEstablishedManagedPeerLinkAndDetailRows(t *testing.T) { + registry := newTopologyRegistry() + + cacheA := newTopologyCache() + cacheA.updateTime = time.Now() + cacheA.lastUpdate = cacheA.updateTime + cacheA.agentID = "agent-test" + cacheA.localDevice = topologyDevice{ + ChassisID: "00:11:22:33:44:55", + ChassisIDType: "macAddress", + SysName: "router-a", + ManagementIP: "10.0.0.1", + } + cacheA.bgpPeersByKey["a"] = topologyBGPPeer{ + RoutingInstance: "default", + NeighborIP: "198.51.100.2", + RemoteAS: "65002", + LocalIP: "198.51.100.1", + LocalAS: "65001", + LocalIdentifier: "1.1.1.1", + PeerIdentifier: "2.2.2.2", + State: "established", + } + + cacheB := newTopologyCache() + cacheB.updateTime = time.Now().Add(time.Second) + cacheB.lastUpdate = cacheB.updateTime + cacheB.agentID = "agent-test" + cacheB.localDevice = topologyDevice{ + ChassisID: "aa:bb:cc:dd:ee:ff", + ChassisIDType: "macAddress", + SysName: "router-b", + ManagementIP: "10.0.0.2", + } + cacheB.bgpPeersByKey["b"] = topologyBGPPeer{ + RoutingInstance: "default", + NeighborIP: "198.51.100.1", + RemoteAS: "65001", + LocalIP: "198.51.100.2", + LocalAS: "65002", + LocalIdentifier: "2.2.2.2", + PeerIdentifier: "1.1.1.1", + State: "established", + } + + registry.register(cacheA) + registry.register(cacheB) + + data, ok := snapshotTopologyRegistryForTest(registry) + + require.True(t, ok) + require.Len(t, data.Links, 1) + link := data.Links[0] + require.Equal(t, "3", link.Layer) + require.Equal(t, topologyBGPAdjacencyLinkType, link.Protocol) + require.Equal(t, topologyBGPAdjacencyLinkType, link.LinkType) + require.Equal(t, "observed", link.Direction) + require.Equal(t, "established", link.State) + require.Equal(t, "bgp_established_adjacency", link.Metrics["inference"]) + require.Equal(t, "logical_l3_bgp", link.Metrics["attachment_mode"]) + require.Equal(t, "default", link.Metrics["routing_instance"]) + require.Equal(t, "65001", link.Src.Attributes["as"]) + require.Equal(t, "65002", link.Dst.Attributes["as"]) + require.Equal(t, 2, data.Stats["bgp_peer_rows"]) + require.Equal(t, 2, data.Stats["bgp_peer_detail_rows"]) + require.Equal(t, 1, data.Stats["bgp_adjacency_emitted_links"]) + require.Equal(t, 1, data.Stats["bgp_adjacency_suppressed_duplicate_link"]) + require.Equal(t, 1, data.Stats["bgp_adjacency_visible_links"]) + + routerA := findDeviceActorBySysName(data, "router-a") + require.NotNil(t, routerA) + routerB := findDeviceActorBySysName(data, "router-b") + require.NotNil(t, routerB) + require.Contains(t, routerA.Tables, "bgp_peers") + require.Len(t, routerA.Tables["bgp_peers"], 1) + require.Equal(t, routerB.ActorID, routerA.Tables["bgp_peers"][0]["remote_actor_id"]) +} + +func TestTopologyRegistry_BGPAdjacencyKeepsUnresolvedAndNonEstablishedPeersAsDetails(t *testing.T) { + registry := newTopologyRegistry() + + cache := newTopologyCache() + cache.updateTime = time.Now() + cache.lastUpdate = cache.updateTime + cache.agentID = "agent-test" + cache.localDevice = topologyDevice{ + ChassisID: "00:11:22:33:44:55", + ChassisIDType: "macAddress", + SysName: "router-a", + ManagementIP: "10.0.0.1", + } + cache.bgpPeersByKey["unresolved"] = topologyBGPPeer{ + RoutingInstance: "default", + NeighborIP: "203.0.113.2", + RemoteAS: "65002", + LocalIP: "198.51.100.1", + LocalAS: "65001", + LocalIdentifier: "1.1.1.1", + PeerIdentifier: "2.2.2.2", + State: "established", + } + cache.bgpPeersByKey["idle"] = topologyBGPPeer{ + RoutingInstance: "default", + NeighborIP: "203.0.113.3", + RemoteAS: "65003", + LocalIP: "198.51.100.1", + LocalAS: "65001", + LocalIdentifier: "1.1.1.1", + PeerIdentifier: "3.3.3.3", + State: "idle", + } + + registry.register(cache) + + data, ok := snapshotTopologyRegistryForTest(registry) + + require.True(t, ok) + require.Empty(t, data.Links) + require.Equal(t, 2, data.Stats["bgp_peer_rows"]) + require.Equal(t, 2, data.Stats["bgp_peer_detail_rows"]) + require.Equal(t, 1, data.Stats["bgp_adjacency_suppressed_unresolved_neighbor"]) + require.Equal(t, 1, data.Stats["bgp_adjacency_suppressed_non_established_state"]) + require.Equal(t, 0, data.Stats["bgp_adjacency_visible_links"]) + + routerA := findDeviceActorBySysName(data, "router-a") + require.NotNil(t, routerA) + require.Contains(t, routerA.Tables, "bgp_peers") + require.Len(t, routerA.Tables["bgp_peers"], 2) + for _, row := range routerA.Tables["bgp_peers"] { + require.NotContains(t, row, "remote_actor_id") + } +} + func TestCompareCollapseActorPriorityPrefersNonEmptyActorID(t *testing.T) { left := topologyActor{ ActorID: "", diff --git a/src/go/plugin/go.d/collector/snmp_topology/topology_shape_stats.go b/src/go/plugin/go.d/collector/snmp_topology/topology_shape_stats.go index 1cfe7b16b579be..a0549b995814a2 100644 --- a/src/go/plugin/go.d/collector/snmp_topology/topology_shape_stats.go +++ b/src/go/plugin/go.d/collector/snmp_topology/topology_shape_stats.go @@ -28,6 +28,7 @@ func recomputeTopologyLinkStats(data *topologyData) { data.Stats["links_probable"] = probable recomputeTopologyL3VisibleLinkStats(data) recomputeTopologyOSPFVisibleLinkStats(data) + recomputeTopologyBGPVisibleLinkStats(data) } func recomputeTopologyL3VisibleLinkStats(data *topologyData) { diff --git a/src/go/plugin/go.d/collector/snmp_topology/topology_shape_stats_test.go b/src/go/plugin/go.d/collector/snmp_topology/topology_shape_stats_test.go index 7ac7a8661ec907..a2e9416658e4de 100644 --- a/src/go/plugin/go.d/collector/snmp_topology/topology_shape_stats_test.go +++ b/src/go/plugin/go.d/collector/snmp_topology/topology_shape_stats_test.go @@ -63,13 +63,19 @@ func TestApplySNMPTopologyShapePolicies_EmitsStatsContractKeys(t *testing.T) { } } -func TestRecomputeTopologyLinkStatsRefreshesExistingL3VisibleCount(t *testing.T) { +func TestRecomputeTopologyLinkStatsRefreshesExistingLogicalL3VisibleCounts(t *testing.T) { data := &topologyData{ Stats: map[string]any{ - "l3_subnet_emitted_links": 2, + "l3_subnet_emitted_links": 2, + "ospf_adjacency_emitted_links": 1, + "bgp_adjacency_emitted_links": 1, + "l3_subnet_visible_links": 2, + "ospf_adjacency_visible_links": 1, + "bgp_adjacency_visible_links": 1, }, Links: []topologyLink{ {Protocol: topologyL3SubnetLinkType, LinkType: topologyL3SubnetLinkType}, + {Protocol: topologyBGPAdjacencyLinkType, LinkType: topologyBGPAdjacencyLinkType}, {Protocol: "lldp", LinkType: "lldp"}, }, } @@ -81,8 +87,12 @@ func TestRecomputeTopologyLinkStatsRefreshesExistingL3VisibleCount(t *testing.T) want any }{ "emitted-l3-subnet-links": {key: "l3_subnet_emitted_links", want: 2}, - "links-total": {key: "links_total", want: 2}, + "emitted-ospf-links": {key: "ospf_adjacency_emitted_links", want: 1}, + "emitted-bgp-links": {key: "bgp_adjacency_emitted_links", want: 1}, + "links-total": {key: "links_total", want: 3}, "visible-l3-subnet-links": {key: "l3_subnet_visible_links", want: 1}, + "visible-ospf-links": {key: "ospf_adjacency_visible_links", want: 0}, + "visible-bgp-links": {key: "bgp_adjacency_visible_links", want: 1}, } for name, tc := range expectedStats { t.Run("stat/"+name, func(t *testing.T) {