From e325de671c18d732d61371cf758468a415189619 Mon Sep 17 00:00:00 2001 From: Peter Haag Date: Mon, 22 Jan 2024 20:16:14 +0100 Subject: [PATCH] Implement #501. Add -X option for sfcapd --- man/sfcapd.1 | 12 ++- src/sflow/sfcapd.c | 5 +- src/sflow/sflow_nfdump.c | 211 +++++++++++++++++++++++++------------- src/sflow/sflow_nfdump.h | 4 +- src/sflow/sflow_process.c | 6 +- 5 files changed, 158 insertions(+), 80 deletions(-) diff --git a/man/sfcapd.1 b/man/sfcapd.1 index 810c82dd..41947862 100755 --- a/man/sfcapd.1 +++ b/man/sfcapd.1 @@ -1,4 +1,4 @@ -\" Copyright (c) 2023, Peter Haag +\" Copyright (c) 2024, Peter Haag .\" All rights reserved. .\" .\" Redistribution and use in source and binary forms, with or without @@ -59,6 +59,7 @@ .Op Fl m Ar metricpath .Op Fl e .Op Fl x Ar command +.Op Fl X Ar extensionList .Op Fl W Ar workers .Op Fl E .Op Fl v @@ -265,6 +266,15 @@ The full path of the new file is: %d/%f string supplied by .Fl I .El +.It Fl X Ar extensionList +.Ar extensionList +is a ',' separated list of extensions to be stored by +.Nm . +The numbers correspond to the extension list in nfxV3.h. By default extensions are added +dynamically to store all data sent by the exporter. If +.Ar extensionList +is given, only those elements matching the extension are processed and stored. Usually this +option is not needed, unless for specific requirements. .It Fl m Ar metricpath Enables the flow metric exporter. Flow metric information is sent to the UNIX socket .Ar metricpath diff --git a/src/sflow/sfcapd.c b/src/sflow/sfcapd.c index 9e761919..00b772c8 100644 --- a/src/sflow/sfcapd.c +++ b/src/sflow/sfcapd.c @@ -939,7 +939,10 @@ int main(int argc, char **argv) { SetPriv(userid, groupid); - Init_sflow(verbose, extensionList); + if (!Init_sflow(verbose, extensionList)) { + LogError("Init_sflow() failed"); + exit(EXIT_FAILURE); + } if (subdir_index && !InitHierPath(subdir_index)) { close(sock); diff --git a/src/sflow/sflow_nfdump.c b/src/sflow/sflow_nfdump.c index 5d2e7ed9..f63444d2 100644 --- a/src/sflow/sflow_nfdump.c +++ b/src/sflow/sflow_nfdump.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2009-2023, Peter Haag + * Copyright (c) 2009-2024, Peter Haag * Copyright (c) 2004-2008, SWITCH - Teleinformatikdienste fuer Lehre und Forschung * All rights reserved. * @@ -84,20 +84,67 @@ typedef struct exporter_sflow_s { } exporter_sflow_t; -static int printRecord = 0; -static uint32_t recordBaseSize; -static uint32_t numBaseElements; +static int PrintRecord = 0; + +static int ExtensionsEnabled[MAXEXTENSIONS]; +uint32_t BaseRecordSize = EXgenericFlowSize; static exporter_sflow_t *GetExporter(FlowSource_t *fs, uint32_t agentSubId, uint32_t meanSkipCount); #include "inline.c" #include "nffile_inline.c" -void Init_sflow(int verbose, char *extensionList) { - printRecord = verbose; - recordBaseSize = EXgenericFlowSize + EXflowMiscSize + EXasRoutingSize + EXvLanSize + EXmacAddrSize; - numBaseElements = 5; +int Init_sflow(int verbose, char *extensionList) { + PrintRecord = verbose; + + if (extensionList) { + // Disable all extensions + for (int i = 0; i < MAXEXTENSIONS; i++) { + ExtensionsEnabled[i] = 0; + } + + // get enabled extensions from string + int extID = ScanExtension(extensionList); + while (extID > 0) { + dbg_printf("Enable extension %d\n", extID); + ExtensionsEnabled[extID] = 1; + extID = ScanExtension(NULL); + } + + if (extID == -1) { + LogError("Failed to scan extension list."); + return 0; + } + + // make sure extension 1 is enabled + ExtensionsEnabled[1] = 1; + + } else { + // Enable all extensions + dbg_printf("Enable all extensions\n"); + for (int i = 0; i < MAXEXTENSIONS; i++) { + ExtensionsEnabled[i] = 1; + } + } + + // extension available in all flows + if (ExtensionsEnabled[EXflowMiscID]) { + BaseRecordSize += EXflowMiscSize; + } + if (ExtensionsEnabled[EXvLanID]) { + BaseRecordSize += EXvLanSize; + } + if (ExtensionsEnabled[EXasRoutingID]) { + BaseRecordSize += EXasRoutingSize; + } + if (ExtensionsEnabled[EXmacAddrID]) { + BaseRecordSize += EXmacAddrSize; + } + if (ExtensionsEnabled[EXmplsLabelID]) { + BaseRecordSize += EXmplsLabelSize; + } + return 1; } // End of Init_sflow // called by sfcapd for each packet @@ -117,7 +164,7 @@ void Process_sflow(void *in_buff, ssize_t in_buff_cnt, FlowSource_t *fs) { // TRY sample.datap = (uint32_t *)sample.rawSample; sample.endp = (u_char *)sample.rawSample + sample.rawSampleLen; - readSFlowDatagram(&sample, fs, printRecord); + readSFlowDatagram(&sample, fs, PrintRecord); } else { // CATCH dbg_printf("SFLOW: caught exception: %d\n", exceptionVal); @@ -220,43 +267,49 @@ void StoreSflowRecord(SFSample *sample, FlowSource_t *fs) { sample->dcd_dport = 0; } - uint32_t recordSize = recordBaseSize; - if (sample->gotIPV6) { + uint32_t recordSize = BaseRecordSize; + if (sample->gotIPV6 && ExtensionsEnabled[EXipv6FlowID]) { recordSize += EXipv6FlowSize; - } else { + } + + if (sample->gotIPV4 && ExtensionsEnabled[EXipv4FlowID]) { recordSize += EXipv4FlowSize; } - if (sample->mpls_num_labels > 0) { + if (sample->mpls_num_labels > 0 && ExtensionsEnabled[EXmplsLabelID]) { recordSize += EXmplsLabelSize; } - if (sample->nextHop.type == SFLADDRESSTYPE_IP_V4) { + if (sample->nextHop.type == SFLADDRESSTYPE_IP_V4 && ExtensionsEnabled[EXipNextHopV4ID]) { recordSize += EXipNextHopV4Size; } - if (sample->nextHop.type == SFLADDRESSTYPE_IP_V6) { + if (sample->nextHop.type == SFLADDRESSTYPE_IP_V6 && ExtensionsEnabled[EXipNextHopV6ID]) { recordSize += EXipNextHopV6Size; } - if (sample->bgp_nextHop.type == SFLADDRESSTYPE_IP_V4) { + if (sample->bgp_nextHop.type == SFLADDRESSTYPE_IP_V4 && ExtensionsEnabled[EXbgpNextHopV4ID]) { recordSize += EXbgpNextHopV4Size; } - if (sample->bgp_nextHop.type == SFLADDRESSTYPE_IP_V6) { + if (sample->bgp_nextHop.type == SFLADDRESSTYPE_IP_V6 && ExtensionsEnabled[EXbgpNextHopV6ID]) { recordSize += EXbgpNextHopV6Size; } if ((sample->extended_data_tag & SASAMPLE_EXTENDED_DATA_NAT) != 0) { - if (sample->nat_src.type == SFLADDRESSTYPE_IP_V4) { - recordSize += (EXnselXlateIPv4Size + EXnselXlatePortSize); + if (sample->nat_src.type == SFLADDRESSTYPE_IP_V4 && ExtensionsEnabled[EXnselXlateIPv4ID]) { + recordSize += EXnselXlateIPv4Size; + } + if (sample->nat_src.type == SFLADDRESSTYPE_IP_V6 && ExtensionsEnabled[EXnselXlateIPv6ID]) { + recordSize += EXnselXlateIPv6Size; } - if (sample->nat_src.type == SFLADDRESSTYPE_IP_V6) { - recordSize += (EXnselXlateIPv6Size + EXnselXlatePortSize); + if (ExtensionsEnabled[EXnselXlatePortID]) { + recordSize += EXnselXlatePortSize; } } - if (fs->sa_family == AF_INET6) { + if (fs->sa_family == AF_INET6 && ExtensionsEnabled[EXipReceivedV6ID]) { recordSize += EXipReceivedV6Size; - } else { + } + if (fs->sa_family == AF_INET && ExtensionsEnabled[EXipReceivedV4ID]) { recordSize += EXipReceivedV4Size; } @@ -287,7 +340,7 @@ void StoreSflowRecord(SFSample *sample, FlowSource_t *fs) { genericFlow->inBytes = sample->meanSkipCount * sample->sampledPacketSize; genericFlow->srcTos = sample->dcd_ipTos; - if (sample->gotIPV6) { + if (sample->gotIPV6 && ExtensionsEnabled[EXipv6FlowID]) { PushExtension(recordHeader, EXipv6Flow, ipv6Flow); SetFlag(recordHeader->flags, V3_FLAG_IPV6_ADDR); @@ -302,101 +355,113 @@ void StoreSflowRecord(SFSample *sample, FlowSource_t *fs) { ipv6Flow->dstAddr[0] = ntohll(*u); u = (uint64_t *)&(b[8]); ipv6Flow->dstAddr[1] = ntohll(*u); - - } else { + } + if (sample->gotIPV4 && ExtensionsEnabled[EXipv4FlowID]) { PushExtension(recordHeader, EXipv4Flow, ipv4Flow); ipv4Flow->srcAddr = ntohl(sample->dcd_srcIP.s_addr); ipv4Flow->dstAddr = ntohl(sample->dcd_dstIP.s_addr); } - PushExtension(recordHeader, EXflowMisc, flowMisc); - flowMisc->input = sample->inputPort; - flowMisc->output = sample->outputPort; - flowMisc->srcMask = sample->srcMask; - flowMisc->dstMask = sample->dstMask; + if (ExtensionsEnabled[EXflowMiscID]) { + PushExtension(recordHeader, EXflowMisc, flowMisc); + flowMisc->input = sample->inputPort; + flowMisc->output = sample->outputPort; + flowMisc->srcMask = sample->srcMask; + flowMisc->dstMask = sample->dstMask; + } - PushExtension(recordHeader, EXvLan, vLan); - vLan->srcVlan = sample->in_vlan; - vLan->dstVlan = sample->out_vlan; + if (ExtensionsEnabled[EXvLanID]) { + PushExtension(recordHeader, EXvLan, vLan); + vLan->srcVlan = sample->in_vlan; + vLan->dstVlan = sample->out_vlan; + } - PushExtension(recordHeader, EXasRouting, asRouting); - asRouting->srcAS = sample->src_as; - asRouting->dstAS = sample->dst_as; + if (ExtensionsEnabled[EXasRoutingID]) { + PushExtension(recordHeader, EXasRouting, asRouting); + asRouting->srcAS = sample->src_as; + asRouting->dstAS = sample->dst_as; + } - if (sample->nextHop.type == SFLADDRESSTYPE_IP_V4) { + if (sample->nextHop.type == SFLADDRESSTYPE_IP_V4 && ExtensionsEnabled[EXipNextHopV4ID]) { PushExtension(recordHeader, EXipNextHopV4, ipNextHopV4); ipNextHopV4->ip = ntohl(sample->nextHop.address.ip_v4.addr); } - if (sample->nextHop.type == SFLADDRESSTYPE_IP_V6) { + if (sample->nextHop.type == SFLADDRESSTYPE_IP_V6 && ExtensionsEnabled[EXipNextHopV6ID]) { uint64_t *addr = (uint64_t *)sample->nextHop.address.ip_v6.addr; PushExtension(recordHeader, EXipNextHopV6, ipNextHopV6); ipNextHopV6->ip[0] = ntohll(addr[0]); ipNextHopV6->ip[1] = ntohll(addr[1]); } - if (sample->bgp_nextHop.type == SFLADDRESSTYPE_IP_V4) { + if (sample->bgp_nextHop.type == SFLADDRESSTYPE_IP_V4 && ExtensionsEnabled[EXbgpNextHopV4ID]) { PushExtension(recordHeader, EXbgpNextHopV4, bgpNextHopV4); bgpNextHopV4->ip = ntohl(sample->bgp_nextHop.address.ip_v4.addr); } - if (sample->bgp_nextHop.type == SFLADDRESSTYPE_IP_V6) { + if (sample->bgp_nextHop.type == SFLADDRESSTYPE_IP_V6 && ExtensionsEnabled[EXbgpNextHopV6ID]) { uint64_t *addr = (void *)sample->bgp_nextHop.address.ip_v6.addr; PushExtension(recordHeader, EXipReceivedV6, ipNextHopV6); ipNextHopV6->ip[0] = ntohll(addr[0]); ipNextHopV6->ip[1] = ntohll(addr[1]); } - PushExtension(recordHeader, EXmacAddr, macAddr); - macAddr->inSrcMac = Get_val48((void *)&sample->eth_src); - macAddr->outDstMac = Get_val48((void *)&sample->eth_dst); - macAddr->inDstMac = 0; - macAddr->outSrcMac = 0; + if (ExtensionsEnabled[EXmacAddrID]) { + PushExtension(recordHeader, EXmacAddr, macAddr); + macAddr->inSrcMac = Get_val48((void *)&sample->eth_src); + macAddr->outDstMac = Get_val48((void *)&sample->eth_dst); + macAddr->inDstMac = 0; + macAddr->outSrcMac = 0; + } - if (sample->mpls_num_labels > 0) { - PushExtension(recordHeader, EXmplsLabel, mplsLabel); - for (int i = 0; i < sample->mpls_num_labels; i++) { - mplsLabel->mplsLabel[i] = sample->mpls_label[i]; + if (ExtensionsEnabled[EXmplsLabelID]) { + if (sample->mpls_num_labels > 0) { + PushExtension(recordHeader, EXmplsLabel, mplsLabel); + for (int i = 0; i < sample->mpls_num_labels; i++) { + mplsLabel->mplsLabel[i] = sample->mpls_label[i]; + } } } if ((sample->extended_data_tag & SASAMPLE_EXTENDED_DATA_NAT) != 0) { switch (sample->nat_src.type) { case SFLADDRESSTYPE_IP_V4: - dbg_printf("NAT v4 addr\n"); - PushExtension(recordHeader, EXnselXlateIPv4, nselXlateIPv4); - nselXlateIPv4->xlateSrcAddr = ntohl(sample->nat_src.address.ip_v4.addr); - nselXlateIPv4->xlateDstAddr = ntohl(sample->nat_dst.address.ip_v4.addr); - - PushExtension(recordHeader, EXnselXlatePort, nselXlatePort); - nselXlatePort->xlateSrcPort = sample->nat_src_port; - nselXlatePort->xlateDstPort = sample->nat_dst_port; + if (ExtensionsEnabled[EXnselXlateIPv4ID]) { + dbg_printf("NAT v4 addr\n"); + PushExtension(recordHeader, EXnselXlateIPv4, nselXlateIPv4); + nselXlateIPv4->xlateSrcAddr = ntohl(sample->nat_src.address.ip_v4.addr); + nselXlateIPv4->xlateDstAddr = ntohl(sample->nat_dst.address.ip_v4.addr); + } break; case SFLADDRESSTYPE_IP_V6: { - dbg_printf("NAT v6 addr\n"); - PushExtension(recordHeader, EXnselXlateIPv6, nselXlateIPv6); - uint64_t *addr = (void *)sample->nat_src.address.ip_v6.addr; - nselXlateIPv6->xlateSrcAddr[0] = ntohll(addr[0]); - nselXlateIPv6->xlateSrcAddr[1] = ntohll(addr[1]); - addr = (void *)sample->nat_dst.address.ip_v6.addr; - nselXlateIPv6->xlateDstAddr[0] = ntohll(addr[0]); - nselXlateIPv6->xlateDstAddr[1] = ntohll(addr[1]); - - PushExtension(recordHeader, EXnselXlatePort, nselXlatePort); - nselXlatePort->xlateSrcPort = sample->nat_src_port; - nselXlatePort->xlateDstPort = sample->nat_dst_port; + if (ExtensionsEnabled[EXnselXlateIPv6ID]) { + dbg_printf("NAT v6 addr\n"); + PushExtension(recordHeader, EXnselXlateIPv6, nselXlateIPv6); + uint64_t *addr = (void *)sample->nat_src.address.ip_v6.addr; + nselXlateIPv6->xlateSrcAddr[0] = ntohll(addr[0]); + nselXlateIPv6->xlateSrcAddr[1] = ntohll(addr[1]); + addr = (void *)sample->nat_dst.address.ip_v6.addr; + nselXlateIPv6->xlateDstAddr[0] = ntohll(addr[0]); + nselXlateIPv6->xlateDstAddr[1] = ntohll(addr[1]); + } } break; default: /* undefined address type - bail out */ LogError("SFLOW: getAddress() unknown address type = %d\n", sample->nat_src.type); } + if (ExtensionsEnabled[EXnselXlatePortID]) { + PushExtension(recordHeader, EXnselXlatePort, nselXlatePort); + nselXlatePort->xlateSrcPort = sample->nat_src_port; + nselXlatePort->xlateDstPort = sample->nat_dst_port; + } } // add router IP - if (fs->sa_family == PF_INET6) { + if (fs->sa_family == PF_INET6 && ExtensionsEnabled[EXipReceivedV6ID]) { PushExtension(recordHeader, EXipReceivedV6, ipReceivedV6); ipReceivedV6->ip[0] = fs->ip.V6[0]; ipReceivedV6->ip[1] = fs->ip.V6[1]; dbg_printf("Add IPv6 route IP extension\n"); - } else { + } + if (fs->sa_family == PF_INET && ExtensionsEnabled[EXipReceivedV4ID]) { PushExtension(recordHeader, EXipReceivedV4, ipReceivedV4); ipReceivedV4->ip = fs->ip.V4; dbg_printf("Add IPv4 route IP extension\n"); @@ -438,7 +503,7 @@ void StoreSflowRecord(SFSample *sample, FlowSource_t *fs) { uint32_t exporterIdent = MetricExpporterID(recordHeader); UpdateMetric(fs->nffile->ident, exporterIdent, genericFlow); - if (printRecord) { + if (PrintRecord) { flow_record_short(stdout, recordHeader); } #ifdef DEVEL diff --git a/src/sflow/sflow_nfdump.h b/src/sflow/sflow_nfdump.h index 87a187de..4b35b273 100644 --- a/src/sflow/sflow_nfdump.h +++ b/src/sflow/sflow_nfdump.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2023, Peter Haag + * Copyright (c) 2017-2024, Peter Haag * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -37,7 +37,7 @@ #include "collector.h" #include "sflow_process.h" -void Init_sflow(int verbose, char *extensionList); +int Init_sflow(int verbose, char *extensionList); void Process_sflow(void *in_buff, ssize_t in_buff_cnt, FlowSource_t *fs); diff --git a/src/sflow/sflow_process.c b/src/sflow/sflow_process.c index 6d1f15f9..2d4132ed 100644 --- a/src/sflow/sflow_process.c +++ b/src/sflow/sflow_process.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2023, Peter Haag + * Copyright (c) 2017-2024, Peter Haag * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -1948,7 +1948,7 @@ static void readExtendedProxySocket6(SFSample *sample) { */ static void readExtendedDecap(SFSample *sample, char *prefix) { - uint32_t offset = getData32(sample); + uint32_t offset __attribute__((unused)) = getData32(sample); dbg_printf("extendedType %sdecap\n", prefix); dbg_printf("%sdecap_inner_header_offset %u\n", prefix, offset); } @@ -1959,7 +1959,7 @@ static void readExtendedDecap(SFSample *sample, char *prefix) { */ static void readExtendedVNI(SFSample *sample, char *prefix) { - uint32_t vni = getData32(sample); + uint32_t vni __attribute__((unused)) = getData32(sample); dbg_printf("extendedType %sVNI\n", prefix); dbg_printf("%sVNI %u\n", prefix, vni); }