map: avoid allocations during PerCPUMap batch lookups

Extend the optimization from b89a4cb ("map: avoid allocations during
batch lookup of common types") to BatchLookup with PerCPUMaps This
means that types without implicit padding will not incur allocations.
See 4609dc7 ("map: zero-allocation operations for common types") for a
detailed description of the behavior.

```
goos: linux
goarch: amd64
pkg: github.com/cilium/ebpf
cpu: AMD Ryzen 5 5600G with Radeon Graphics
                                          │   old.txt   │               new.txt               │
                                          │   sec/op    │   sec/op     vs base                │
Iterate/PerCPUHash/BatchLookup-8            551.3µ ± 1%   129.7µ ± 1%  -76.47% (p=0.000 n=10)
Iterate/PerCPUHash/BatchLookupAndDelete-8   568.2µ ± 1%   148.4µ ± 1%  -73.88% (p=0.000 n=10)
geomean                                     559.7µ        138.7µ       -75.21%

                                          │   old.txt    │              new.txt               │
                                          │     B/op     │    B/op     vs base                │
Iterate/PerCPUHash/BatchLookup-8            89709.0 ± 0%   136.0 ± 0%  -99.85% (p=0.000 n=10)
Iterate/PerCPUHash/BatchLookupAndDelete-8   89799.0 ± 0%   224.0 ± 1%  -99.75% (p=0.000 n=10)
geomean                                     87.65Ki        174.5       -99.81%

                                          │    old.txt    │              new.txt               │
                                          │   allocs/op   │ allocs/op   vs base                │
Iterate/PerCPUHash/BatchLookup-8            1006.000 ± 0%   5.000 ± 0%  -99.50% (p=0.000 n=10)
Iterate/PerCPUHash/BatchLookupAndDelete-8   1006.000 ± 0%   5.000 ± 0%  -99.50% (p=0.000 n=10)
geomean                                       1.006k        5.000       -99.50%
```

Signed-off-by: Tobias Böhm <code@aibor.de>

diff --git a/map.go b/map.go
index 55974b4ebcf965bb3dc40c8f837275c80c39528b..e8720c407d45bb6dc443bc34b63060f4aad2ba91 100644 (file)
--- a/map.go
+++ b/map.go
@@ -1220,17 +1220,18 @@ func (m *Map) batchLookupPerCPU(cmd sys.Cmd, cursor *MapBatchCursor, keysOut, va
                return 0, fmt.Errorf("keys: %w", err)
        }
 
-       valueBuf := make([]byte, count*int(m.fullValueSize))
-       valuePtr := sys.UnsafeSlicePointer(valueBuf)
+       valueBuf := sysenc.SyscallOutput(valuesOut, count*int(m.fullValueSize))
 
-       n, sysErr := m.batchLookupCmd(cmd, cursor, count, keysOut, valuePtr, opts)
+       n, sysErr := m.batchLookupCmd(cmd, cursor, count, keysOut, valueBuf.Pointer(), opts)
        if sysErr != nil && !errors.Is(sysErr, unix.ENOENT) {
                return 0, sysErr
        }
 
-       err = unmarshalBatchPerCPUValue(valuesOut, count, int(m.valueSize), valueBuf)
-       if err != nil {
-               return 0, err
+       if bytesBuf := valueBuf.Bytes(); bytesBuf != nil {
+               err = unmarshalBatchPerCPUValue(valuesOut, count, int(m.valueSize), bytesBuf)
+               if err != nil {
+                       return 0, err
+               }
        }
 
        return n, sysErr

diff --git a/map_test.go b/map_test.go
index 610c08bb2744e74a1cfae1e1d780efd6df5866fa..ece61230b68f4c47b80dc5d668354a6856dcf3bb 100644 (file)
--- a/map_test.go
+++ b/map_test.go
@@ -967,21 +967,41 @@ func TestMapIteratorAllocations(t *testing.T) {
 func TestMapBatchLookupAllocations(t *testing.T) {
        testutils.SkipIfNotSupported(t, haveBatchAPI())
 
-       arr := createMap(t, Array, 10)
+       for _, typ := range []MapType{Array, PerCPUArray} {
+               if typ == PerCPUArray {
+                       // https://lore.kernel.org/bpf/20210424214510.806627-2-pctammela@mojatatu.com/
+                       testutils.SkipOnOldKernel(t, "5.13", "batched ops support for percpu array")
+               }
 
-       var cursor MapBatchCursor
-       tmp := make([]uint32, 2)
-       input := any(tmp)
+               t.Run(typ.String(), func(t *testing.T) {
+                       m := mustNewMap(t, &MapSpec{
+                               Name:       "test",
+                               Type:       typ,
+                               KeySize:    4,
+                               ValueSize:  8, // PerCPU values must be 8 byte aligned.
+                               MaxEntries: 10,
+                       }, nil)
 
-       // AllocsPerRun warms up the function for us.
-       allocs := testing.AllocsPerRun(1, func() {
-               _, err := arr.BatchLookup(&cursor, input, input, nil)
-               if err != nil {
-                       t.Fatal(err)
-               }
-       })
+                       possibleCPU := 1
+                       if m.Type().hasPerCPUValue() {
+                               possibleCPU = MustPossibleCPU()
+                       }
+
+                       var cursor MapBatchCursor
+                       keys := any(make([]uint32, 2))
+                       values := any(make([]uint64, 2*possibleCPU))
 
-       qt.Assert(t, qt.Equals(allocs, 0))
+                       // AllocsPerRun warms up the function for us.
+                       allocs := testing.AllocsPerRun(1, func() {
+                               _, err := m.BatchLookup(&cursor, keys, values, nil)
+                               if err != nil {
+                                       t.Fatal(err)
+                               }
+                       })
+
+                       qt.Assert(t, qt.Equals(allocs, 0))
+               })
+       }
 }
 
 type customTestUnmarshaler []uint8