conntrack: use kprobe on nf_ct_deliver_cached_events for tracking conntrack updates

cgroup/cpu.go

@@ -6,6 +6,8 @@ import (
 	"path"
 	"strconv"
 	"strings"
+
+	"github.com/coroot/coroot-node-agent/common"
 )
 
 type CPUStat struct {
@@ -26,15 +28,15 @@ func (cg Cgroup) cpuStatV1() (*CPUStat, error) {
 	if err != nil {
 		return nil, err
 	}
-	usageNs, err := readIntFromFile(path.Join(cgRoot, "cpuacct", cg.subsystems["cpuacct"], "cpuacct.usage"))
+	usageNs, err := common.ReadIntFromFile(path.Join(cgRoot, "cpuacct", cg.subsystems["cpuacct"], "cpuacct.usage"))
 	if err != nil {
 		return nil, err
 	}
-	periodUs, err := readIntFromFile(path.Join(cgRoot, "cpu", cg.subsystems["cpu"], "cpu.cfs_period_us"))
+	periodUs, err := common.ReadIntFromFile(path.Join(cgRoot, "cpu", cg.subsystems["cpu"], "cpu.cfs_period_us"))
 	if err != nil {
 		return nil, err
 	}
-	quotaUs, err := readIntFromFile(path.Join(cgRoot, "cpu", cg.subsystems["cpu"], "cpu.cfs_quota_us"))
+	quotaUs, err := common.ReadIntFromFile(path.Join(cgRoot, "cpu", cg.subsystems["cpu"], "cpu.cfs_quota_us"))
 	if err != nil {
 		return nil, err
 	}

cgroup/memory.go

@@ -2,6 +2,8 @@ package cgroup
 
 import (
 	"path"
+
+	"github.com/coroot/coroot-node-agent/common"
 )
 
 const maxMemory = 1 << 62
@@ -24,7 +26,7 @@ func (cg *Cgroup) memoryStatV1() (*MemoryStat, error) {
 	if err != nil {
 		return nil, err
 	}
-	limit, err := readUintFromFile(path.Join(cgRoot, "memory", cg.subsystems["memory"], "memory.limit_in_bytes"))
+	limit, err := common.ReadUintFromFile(path.Join(cgRoot, "memory", cg.subsystems["memory"], "memory.limit_in_bytes"))
 	if err != nil {
 		return nil, err
 	}
@@ -51,7 +53,7 @@ func (cg *Cgroup) memoryStatV2() (*MemoryStat, error) {
 	if err != nil {
 		return nil, err
 	}
-	limit, _ := readUintFromFile(path.Join(cgRoot, cg.subsystems[""], "memory.max"))
+	limit, _ := common.ReadUintFromFile(path.Join(cgRoot, cg.subsystems[""], "memory.max"))
 	return &MemoryStat{
 		RSS:   vars["anon"] + vars["file_mapped"],
 		Cache: vars["file"],

cgroup/utils.go

@@ -27,19 +27,3 @@ func readVariablesFromFile(filePath string) (map[string]uint64, error) {
 	}
 	return res, nil
 }
-
-func readIntFromFile(filePath string) (int64, error) {
-	data, err := os.ReadFile(filePath)
-	if err != nil {
-		return 0, err
-	}
-	return strconv.ParseInt(strings.TrimSpace(string(data)), 10, 64)
-}
-
-func readUintFromFile(filePath string) (uint64, error) {
-	data, err := os.ReadFile(filePath)
-	if err != nil {
-		return 0, err
-	}
-	return strconv.ParseUint(strings.TrimSpace(string(data)), 10, 64)
-}

common/file.go

@@ -0,0 +1,23 @@
+package common
+
+import (
+	"os"
+	"strconv"
+	"strings"
+)
+
+func ReadIntFromFile(filePath string) (int64, error) {
+	data, err := os.ReadFile(filePath)
+	if err != nil {
+		return 0, err
+	}
+	return strconv.ParseInt(strings.TrimSpace(string(data)), 10, 64)
+}
+
+func ReadUintFromFile(filePath string) (uint64, error) {
+	data, err := os.ReadFile(filePath)
+	if err != nil {
+		return 0, err
+	}
+	return strconv.ParseUint(strings.TrimSpace(string(data)), 10, 64)
+}

containers/registry.go

@@ -105,7 +105,7 @@ func NewRegistry(reg prometheus.Registerer, processInfoCh chan<- ProcessInfo) (*
 
 		processInfoCh: processInfoCh,
 
-		tracer: ebpftracer.NewTracer(hostNetNs, *flags.DisableL7Tracing),
+		tracer: ebpftracer.NewTracer(hostNetNs, selfNetNs, *flags.DisableL7Tracing),
 
 		trafficStatsUpdateCh: make(chan *TrafficStatsUpdate),
 	}

ebpftracer/Dockerfile

@@ -7,13 +7,11 @@ WORKDIR /tmp/ebpf
 
 RUN clang -g -O2 -target bpf -D__KERNEL_FROM=416 -D__TARGET_ARCH_x86 -c ebpf.c -o ebpf416x86.o && llvm-strip --strip-debug ebpf416x86.o
 RUN clang -g -O2 -target bpf -D__KERNEL_FROM=420 -D__TARGET_ARCH_x86 -c ebpf.c -o ebpf420x86.o && llvm-strip --strip-debug ebpf420x86.o
-RUN clang -g -O2 -target bpf -D__KERNEL_FROM=503 -D__TARGET_ARCH_x86 -c ebpf.c -o ebpf503x86.o && llvm-strip --strip-debug ebpf503x86.o
 RUN clang -g -O2 -target bpf -D__KERNEL_FROM=506 -D__TARGET_ARCH_x86 -c ebpf.c -o ebpf506x86.o && llvm-strip --strip-debug ebpf506x86.o
 RUN clang -g -O2 -target bpf -D__KERNEL_FROM=512 -D__TARGET_ARCH_x86 -c ebpf.c -o ebpf512x86.o && llvm-strip --strip-debug ebpf512x86.o
 RUN clang -g -O2 -target bpf -D__KERNEL_FROM=512 -D__TARGET_ARCH_x86 -D__CTX_EXTRA_PADDING -c ebpf.c -o ebpf512x86cep.o && llvm-strip --strip-debug ebpf512x86cep.o
 RUN clang -g -O2 -target bpf -D__KERNEL_FROM=416 -D__TARGET_ARCH_arm64 -c ebpf.c -o ebpf416arm64.o && llvm-strip --strip-debug ebpf416arm64.o
 RUN clang -g -O2 -target bpf -D__KERNEL_FROM=420 -D__TARGET_ARCH_arm64 -c ebpf.c -o ebpf420arm64.o && llvm-strip --strip-debug ebpf420arm64.o
-RUN clang -g -O2 -target bpf -D__KERNEL_FROM=503 -D__TARGET_ARCH_arm64 -c ebpf.c -o ebpf503arm64.o && llvm-strip --strip-debug ebpf503arm64.o
 RUN clang -g -O2 -target bpf -D__KERNEL_FROM=506 -D__TARGET_ARCH_arm64 -c ebpf.c -o ebpf506arm64.o && llvm-strip --strip-debug ebpf506arm64.o
 RUN clang -g -O2 -target bpf -D__KERNEL_FROM=512 -D__TARGET_ARCH_arm64 -c ebpf.c -o ebpf512arm64.o && llvm-strip --strip-debug ebpf512arm64.o
 RUN clang -g -O2 -target bpf -D__KERNEL_FROM=512 -D__TARGET_ARCH_arm64 -D__CTX_EXTRA_PADDING -c ebpf.c -o ebpf512arm64cep.o && llvm-strip --strip-debug ebpf512arm64cep.o
@@ -26,15 +24,13 @@ RUN echo -en '// generated - do not edit\npackage ebpftracer\n\nvar ebpfProgs =
 	&& echo -en '\t"amd64": {\n' >> ebpf.go \
 	&& echo -en '\t\t{"5.12", "ctx-extra-padding", []byte("' >> ebpf.go && gzip -c ebpf512x86cep.o | base64 -w0 >> ebpf.go && echo '")},' >> ebpf.go \
 	&& echo -en '\t\t{"5.12", "", []byte("' >> ebpf.go && gzip -c ebpf512x86.o | base64 -w0 >> ebpf.go && echo '")},' >> ebpf.go \
-	&& echo -en '\t\t{"5.3", "", []byte("' >> ebpf.go && gzip -c ebpf503x86.o | base64 -w0 >> ebpf.go && echo '")},' >> ebpf.go \
 	&& echo -en '\t\t{"5.6", "", []byte("' >> ebpf.go && gzip -c ebpf506x86.o | base64 -w0 >> ebpf.go && echo '")},' >> ebpf.go \
 	&& echo -en '\t\t{"4.20", "", []byte("' >> ebpf.go && gzip -c ebpf420x86.o | base64 -w0 >> ebpf.go && echo '")},' >> ebpf.go \
 	&& echo -en '\t\t{"4.16", "", []byte("' >> ebpf.go && gzip -c ebpf416x86.o | base64 -w0 >> ebpf.go && echo '")},' >> ebpf.go \
 	&& echo -en '\t},\n'>> ebpf.go \
 	&& echo -en '\t"arm64": {\n' >> ebpf.go \
 	&& echo -en '\t\t{"5.12", "ctx-extra-padding", []byte("' >> ebpf.go && gzip -c ebpf512arm64cep.o | base64 -w0 >> ebpf.go && echo '")},' >> ebpf.go \
 	&& echo -en '\t\t{"5.12", "", []byte("' >> ebpf.go && gzip -c ebpf512arm64.o | base64 -w0 >> ebpf.go && echo '")},' >> ebpf.go \
-	&& echo -en '\t\t{"5.3", "", []byte("' >> ebpf.go && gzip -c ebpf503arm64.o | base64 -w0 >> ebpf.go && echo '")},' >> ebpf.go \
 	&& echo -en '\t\t{"5.6", "", []byte("' >> ebpf.go && gzip -c ebpf506arm64.o | base64 -w0 >> ebpf.go && echo '")},' >> ebpf.go \
 	&& echo -en '\t\t{"4.20", "", []byte("' >> ebpf.go && gzip -c ebpf420arm64.o | base64 -w0 >> ebpf.go && echo '")},' >> ebpf.go \
 	&& echo -en '\t\t{"4.16", "", []byte("' >> ebpf.go && gzip -c ebpf416arm64.o | base64 -w0 >> ebpf.go && echo '")},' >> ebpf.go \

ebpftracer/ebpf/tcp/conntrack.c

@@ -80,22 +80,11 @@ int handle_ct(struct pt_regs *ctx, struct nf_conn conn)
     return 0;
 }
 
-#if __KERNEL_FROM >= 503
-SEC("kprobe/nf_confirm")
-int nf_confirm(struct pt_regs *ctx) {
-    struct nf_conn conn;
-    if (bpf_probe_read(&conn, sizeof(conn), (void *)PT_REGS_PARM3(ctx)) != 0) {
-        return 0;
-    }
-    return handle_ct(ctx, conn);
-}
-#else
-SEC("kprobe/__nf_conntrack_hash_insert")
-int nf_conntrack_hash_insert(struct pt_regs *ctx) {
+SEC("kprobe/nf_ct_deliver_cached_events")
+int nf_ct_deliver_cached_events(struct pt_regs *ctx) {
     struct nf_conn conn;
     if (bpf_probe_read(&conn, sizeof(conn), (void *)PT_REGS_PARM1(ctx)) != 0) {
         return 0;
     }
     return handle_ct(ctx, conn);
 }
-#endif

ebpftracer/tracer.go

@@ -20,6 +20,7 @@ import (
 	"github.com/cilium/ebpf/perf"
 	"github.com/coroot/coroot-node-agent/common"
 	"github.com/coroot/coroot-node-agent/ebpftracer/l7"
+	"github.com/coroot/coroot-node-agent/proc"
 	"github.com/vishvananda/netns"
 	"golang.org/x/sys/unix"
 	"inet.af/netaddr"
@@ -82,27 +83,32 @@ const (
 type Tracer struct {
 	disableL7Tracing bool
 	hostNetNs        netns.NsHandle
+	selfNetNs        netns.NsHandle
 
 	collection *ebpf.Collection
 	readers    map[string]*perf.Reader
 	links      []link.Link
 	uprobes    map[string]*ebpf.Program
 }
 
-func NewTracer(hostNetNs netns.NsHandle, disableL7Tracing bool) *Tracer {
+func NewTracer(hostNetNs, selfNetNs netns.NsHandle, disableL7Tracing bool) *Tracer {
 	if disableL7Tracing {
 		klog.Infoln("L7 tracing is disabled")
 	}
 	return &Tracer{
 		disableL7Tracing: disableL7Tracing,
 		hostNetNs:        hostNetNs,
+		selfNetNs:        selfNetNs,
 
 		readers: map[string]*perf.Reader{},
 		uprobes: map[string]*ebpf.Program{},
 	}
 }
 
 func (t *Tracer) Run(events chan<- Event) error {
+	if err := proc.ExecuteInNetNs(t.hostNetNs, t.selfNetNs, ensureConntrackEventsAreEnabled); err != nil {
+		return err
+	}
 	if err := t.ebpf(events); err != nil {
 		return err
 	}
@@ -257,6 +263,10 @@ func (t *Tracer) ebpf(ch chan<- Event) error {
 				continue
 			}
 			l, err = link.Kprobe(programSpec.AttachTo, program, nil)
+			if err != nil && programSpec.SectionName == "kprobe/nf_ct_deliver_cached_events" {
+				klog.Warningln("nf_conntrack may not be in use:", err)
+				continue
+			}
 		}
 		if err != nil {
 			t.Close()
@@ -475,3 +485,19 @@ func isCtxExtraPaddingRequired(traceFsPath string) bool {
 	}
 	return false
 }
+
+const nfConntrackEventsParameterPath = "/proc/sys/net/netfilter/nf_conntrack_events"
+
+func ensureConntrackEventsAreEnabled() error {
+	v, err := common.ReadUintFromFile(nfConntrackEventsParameterPath)
+	if err != nil {
+		return err
+	}
+	if v != 1 {
+		klog.Infof("%s = %d, setting to 1", nfConntrackEventsParameterPath, v)
+		if err = os.WriteFile(nfConntrackEventsParameterPath, []byte("1"), 0644); err != nil {
+			return err
+		}
+	}
+	return nil
+}

ebpftracer/tracer_test.go

@@ -327,7 +327,7 @@ func runTracer(t *testing.T, verbose bool) (func() *Event, func()) {
 	assert.NoError(t, common.SetKernelVersion(string(bytes.Split(uname.Release[:], []byte{0})[0])))
 
 	go func() {
-		tt := NewTracer(0, false)
+		tt := NewTracer(0, 0, false)
 		err := tt.Run(events)
 		require.NoError(t, err)
 		<-done