Skip to content

Commit 8c59958

Browse files
jschwinger233jschwinger233
authored
Merge pull request #6 from mozillazg/bpf-skb-load-bytes
Support access skb data via bpf_skb_load_bytes
2 parents fa73d86 + 83c6a6a commit 8c59958

File tree

5 files changed

+343
-80
lines changed

5 files changed

+343
-80
lines changed

‎bpf_probe_read_kernel.go

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
package elibpcap
2+
3+
import "github.com/cilium/ebpf/asm"
4+
5+
/*
6+
If PacketAccessMode == BpfProbeReadKernel, We have to adjust the ebpf instructions because verifier prevents us from
7+
directly loading data from memory. For example, the instruction "r0 = *(u8 *)(r4 +0)"
8+
will break verifier with error "R4 invalid mem access 'scalar", we therefore
9+
need to convert this direct memory load to bpf_probe_read_kernel function call:
10+
11+
- r1 = r10 // r10 is stack top
12+
- r1 += -8 // r1 = r10-8
13+
- r2 = 1 // r2 = sizeof(u8)
14+
- r3 = r4 // r4 is start of packet data, aka L3 header
15+
- r3 += 0 // r3 = r4+0
16+
- call bpf_probe_read_kernel // *(r10-8) = *(u8 *)(r4+0)
17+
- r0 = *(u8 *)(r10 -8) // r0 = *(r10-8)
18+
19+
To safely borrow R1, R2 and R3 for setting up the arguments for
20+
bpf_probe_read_kernel(), we need to save the original values of R1, R2 and R3
21+
on stack, and restore them after the function call.
22+
*/
23+
func adjustEbpfWithBpfProbeReadKernel(insts asm.Instructions, opts Options) (newInsts asm.Instructions, err error) {
24+
replaceIdx := []int{}
25+
replaceInsts := map[int]asm.Instructions{}
26+
for idx, inst := range insts {
27+
if inst.OpCode.Class().IsLoad() {
28+
replaceIdx = append(replaceIdx, idx)
29+
replaceInsts[idx] = append(replaceInsts[idx],
30+
31+
// Store R1, R2, R3 on stack.
32+
asm.StoreMem(asm.RFP, int16(R1Offset), asm.R1, asm.DWord),
33+
asm.StoreMem(asm.RFP, int16(R2Offset), asm.R2, asm.DWord),
34+
asm.StoreMem(asm.RFP, int16(R3Offset), asm.R3, asm.DWord),
35+
36+
// bpf_probe_read_kernel(RFP-8, size, inst.Src)
37+
asm.Mov.Reg(asm.R1, asm.RFP),
38+
asm.Add.Imm(asm.R1, int32(BpfReadKernelOffset)),
39+
asm.Mov.Imm(asm.R2, int32(inst.OpCode.Size().Sizeof())),
40+
asm.Mov.Reg(asm.R3, inst.Src),
41+
asm.Add.Imm(asm.R3, int32(inst.Offset)),
42+
asm.FnProbeReadKernel.Call(),
43+
44+
// inst.Dst = *(RFP-8)
45+
asm.LoadMem(inst.Dst, asm.RFP, int16(BpfReadKernelOffset), inst.OpCode.Size()),
46+
47+
// Restore R4, R5 from stack. This is needed because bpf_probe_read_kernel always resets R4 and R5 even if they are not used by bpf_probe_read_kernel.
48+
asm.LoadMem(asm.R4, asm.RFP, int16(R4Offset), asm.DWord),
49+
asm.LoadMem(asm.R5, asm.RFP, int16(R5Offset), asm.DWord),
50+
)
51+
52+
// Restore R1, R2, R3 from stack
53+
restoreInsts := asm.Instructions{
54+
asm.LoadMem(asm.R1, asm.RFP, int16(R1Offset), asm.DWord),
55+
asm.LoadMem(asm.R2, asm.RFP, int16(R2Offset), asm.DWord),
56+
asm.LoadMem(asm.R3, asm.RFP, int16(R3Offset), asm.DWord),
57+
}
58+
59+
switch inst.Dst {
60+
case asm.R1, asm.R2, asm.R3:
61+
restoreInsts = append(restoreInsts[:inst.Dst-1], restoreInsts[inst.Dst:]...)
62+
}
63+
64+
replaceInsts[idx] = append(replaceInsts[idx], restoreInsts...)
65+
66+
// Metadata is crucial for adjusting jump offsets. We
67+
// ditched original instructions, which could hold symbol
68+
// names targeted by other jump instructions, so here we
69+
// inherit the metadata from the ditched ones.
70+
replaceInsts[idx][0].Metadata = inst.Metadata
71+
}
72+
}
73+
74+
// Replace the memory load instructions with the new ones
75+
for i := len(replaceIdx) - 1; i >= 0; i-- {
76+
idx := replaceIdx[i]
77+
insts = append(insts[:idx], append(replaceInsts[idx], insts[idx+1:]...)...)
78+
}
79+
80+
// Store R4, R5 on stack.
81+
insts = append([]asm.Instruction{
82+
asm.StoreMem(asm.RFP, int16(R4Offset), asm.R4, asm.DWord),
83+
asm.StoreMem(asm.RFP, int16(R5Offset), asm.R5, asm.DWord),
84+
}, insts...)
85+
return insts, err
86+
}

‎bpf_skb_load_bytes.go

Lines changed: 212 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,212 @@
1+
package elibpcap
2+
3+
import (
4+
"fmt"
5+
"github.com/cilium/ebpf/asm"
6+
)
7+
8+
const (
9+
// Negative offsets from RFP (Frame Pointer R10)
10+
BpfDataReadOffset StackOffset = -8 * (iota + 1) // -8: Temporary buffer to store data read by bpf_skb_load_bytes
11+
R1LiveSavedOffset // -16: Slot to save the live value of R1 before a helper function call
12+
R2LiveSavedOffset // -24: Slot to save the live value of R2 before a helper function call
13+
R3LiveSavedOffset // -32: Slot to save the live value of R3 before a helper function call
14+
R4LiveSavedOffset // -40: Slot to save the live value of R4 before a helper function call (e.g., saving original R4/PacketStart before R4 is used as 'len' argument)
15+
16+
// These slots are used for values that are present at the entry of the cbpfc-generated code block
17+
// or are set up by it. They are saved once at the beginning of the adjusted eBPF code block
18+
// if needed by helper function calls or for restoration.
19+
PacketStartSavedOnStack // -48: Slot to save R4 (data/PacketStart), saved at the beginning of the adjusted eBPF code block.
20+
PacketEndSavedOnStack // -56: Slot to save R5 (data_end/PacketEnd), saved at the beginning of the adjusted eBPF code block.
21+
22+
// Slot to store the original _skb argument (R1) of the eBPF filter function.
23+
SkbPtrOriginalArgSlot // -64: Slot to save the original R1 (_skb pointer).
24+
25+
// AvailableOffset defines the start of the stack space that cbpfc can use (deepest known negative offset).
26+
// cbpfc.EBPFOpts.StackOffset will be calculated based on this.
27+
AvailableOffset // -72 (this value itself is negative, representing the size of the stack frame above cbpfc's own usage)
28+
)
29+
30+
/*
31+
If PacketAccessMode == BpfSkbLoadBytes, We have to adjust the ebpf instructions because verifier prevents us from
32+
directly loading data from memory using raw pointers (except for specific cases like TC BPF direct packet access).
33+
For example, the instruction "r0 = *(u8 *)(r4 +0)" (where R4 is skb->data)
34+
will be converted to use bpf_skb_load_bytes(skb, offset, buffer, size).
35+
36+
The conversion involves:
37+
1. At the beginning of the filtered block, save the original R1 (sk_buff*), R4 (data), and R5 (data_end) to stack slots.
38+
2. For each memory load instruction (`inst`):
39+
a. Save live registers (R1-R4) that will be clobbered by arguments setup for bpf_skb_load_bytes or by the call itself.
40+
b. Setup arguments for bpf_skb_load_bytes:
41+
- R1 (arg1): original sk_buff pointer (from SkbPtrOriginalArgSlot).
42+
- R2 (arg2): offset relative to skb->data.
43+
- If inst.Src is PacketStart (R4): offset = inst.Offset.
44+
- If inst.Src is a dynamic pointer P (e.g., R3 holding skb->data + L4_offset):
45+
offset = (P_value - PacketStart_original_value) + inst.Offset.
46+
- R3 (arg3): pointer to a temporary buffer on stack (e.g., RFP + BpfDataReadOffset).
47+
- R4 (arg4): size of data to read (from the load instruction's size).
48+
c. Call bpf_skb_load_bytes.
49+
d. Check R0 (return value). If error, jump to nomatch.
50+
e. Load the data from the temporary buffer on stack into the original destination register (inst.Dst).
51+
f. Restore R4 (data/PacketStart) and R5 (data_end/PacketEnd) from their saved slots on stack.
52+
g. Restore other saved live registers (R1-R3), taking care not to overwrite inst.Dst.
53+
*/
54+
func adjustEbpfWithBpfSkbLoadBytes(insts asm.Instructions, opts Options) (newInsts asm.Instructions, err error) {
55+
// If !DirectRead, prepend instructions to save critical initial registers
56+
// These are assumed to be R1 (_skb), R4 (data), R5 (data_end) at the entry of the filter function code.
57+
var prefixInsts asm.Instructions
58+
prefixInsts = asm.Instructions{
59+
// Save original R1 (which is _skb, the first argument to the eBPF program/filter function)
60+
asm.StoreMem(asm.RFP, int16(SkbPtrOriginalArgSlot), asm.R1, asm.DWord),
61+
// Save R4 (data pointer / PacketStart) and R5 (data_end pointer / PacketEnd)
62+
// These are used by cBPF translated code and need to be restored after helper calls.
63+
asm.StoreMem(asm.RFP, int16(PacketStartSavedOnStack), asm.R4, asm.DWord),
64+
asm.StoreMem(asm.RFP, int16(PacketEndSavedOnStack), asm.R5, asm.DWord),
65+
}
66+
67+
// tempReplaceIdx stores the indices of the original load instructions in `insts` that need replacement.
68+
tempReplaceIdx := []int{}
69+
// tempReplaceInstsMap maps the original index of a load instruction to its replacement instructions.
70+
tempReplaceInstsMap := map[int]asm.Instructions{}
71+
72+
// cbpfc.EBPFOpts.PacketStart is hardcoded to R4 in CompileEbpf.
73+
cbpfPacketStartReg := asm.R4
74+
75+
for originalIdx, inst := range insts {
76+
if inst.OpCode.Class().IsLoad() {
77+
tempReplaceIdx = append(tempReplaceIdx, originalIdx)
78+
79+
var currentReplacement asm.Instructions
80+
81+
// Save live R1, R2, R3, R4 before setting up args for bpf_skb_load_bytes.
82+
currentReplacement = append(currentReplacement,
83+
asm.StoreMem(asm.RFP, int16(R1LiveSavedOffset), asm.R1, asm.DWord),
84+
asm.StoreMem(asm.RFP, int16(R2LiveSavedOffset), asm.R2, asm.DWord),
85+
asm.StoreMem(asm.RFP, int16(R3LiveSavedOffset), asm.R3, asm.DWord),
86+
asm.StoreMem(asm.RFP, int16(R4LiveSavedOffset), asm.R4, asm.DWord),
87+
)
88+
89+
// --- Setup arguments for bpf_skb_load_bytes ---
90+
// R1 (arg1): original sk_buff pointer (from SkbPtrOriginalArgSlot)
91+
currentReplacement = append(currentReplacement,
92+
asm.LoadMem(asm.R1, asm.RFP, int16(SkbPtrOriginalArgSlot), asm.DWord),
93+
)
94+
95+
// R2 (arg2): offset_from_skb_data
96+
if inst.Src == cbpfPacketStartReg {
97+
// Case 1: Original eBPF load was relative to PacketStart (R4).
98+
// inst.Offset is the direct offset from skb->data.
99+
currentReplacement = append(currentReplacement,
100+
asm.Mov.Imm(asm.R2, int32(inst.Offset)),
101+
)
102+
} else {
103+
// Case 2: Original eBPF load used a source register (inst.Src, e.g., R0-R3)
104+
// that holds an absolute pointer to the data to be loaded,
105+
// and inst.Offset is relative to that absolute pointer.
106+
// We need to calculate offset_for_skb_load_bytes =
107+
// (value_in_inst.Src_reg - value_of_PacketStart_at_entry) + inst.Offset.
108+
109+
// Step A: Copy the value from inst.Src register into R2.
110+
// inst.Src contains the absolute pointer calculated by cbpfc.
111+
// Need to ensure this Mov doesn't conflict if inst.Src is R2 itself.
112+
// If inst.Src is R2, R2 already has the correct base pointer.
113+
// If inst.Src is R0, R1, or R3, copy it to R2.
114+
switch inst.Src {
115+
case asm.R0, asm.R1, asm.R3:
116+
currentReplacement = append(currentReplacement,
117+
asm.Mov.Reg(asm.R2, inst.Src), // R2 = absolute_pointer_base
118+
)
119+
case asm.R2:
120+
// R2 already holds the absolute_pointer_base, no Mov needed.
121+
default:
122+
// This case should ideally not be reached if cbpfc uses R0-R3 as working regs
123+
// and R4 as PacketStart for its load instructions.
124+
return nil, fmt.Errorf("adjustEbpf: unhandled inst.Src (%v) for dynamic pointer calculation", inst.Src)
125+
}
126+
127+
// Step B: Load original PacketStart value (skb->data pointer at entry) into a temporary register.
128+
// R0 is suitable as it's clobbered by the helper call / used for return value.
129+
currentReplacement = append(currentReplacement,
130+
asm.LoadMem(asm.R0, asm.RFP, int16(PacketStartSavedOnStack), asm.DWord), // R0 = original PacketStart value
131+
)
132+
133+
// Step C: R2 = R2 - R0 (i.e., absolute_pointer_base - original_PacketStart_value)
134+
// This results in offset_of_absolute_pointer_base_from_original_PacketStart.
135+
currentReplacement = append(currentReplacement,
136+
asm.Sub.Reg(asm.R2, asm.R0),
137+
)
138+
139+
// Step D: R2 = R2 + inst.Offset (add the offset relative to the absolute_pointer_base)
140+
// Now R2 holds the final offset relative to original PacketStart (skb->data).
141+
if inst.Offset != 0 { // Optimization: skip if inst.Offset is zero
142+
currentReplacement = append(currentReplacement,
143+
asm.Add.Imm(asm.R2, int32(inst.Offset)),
144+
)
145+
}
146+
}
147+
148+
// R3 (arg3): to_buf_on_stack (RFP + BpfDataReadOffset)
149+
currentReplacement = append(currentReplacement,
150+
asm.Mov.Reg(asm.R3, asm.RFP),
151+
asm.Add.Imm(asm.R3, int32(BpfDataReadOffset)),
152+
)
153+
// R4 (arg4): len (from inst.OpCode.Size().Sizeof())
154+
currentReplacement = append(currentReplacement,
155+
asm.Mov.Imm(asm.R4, int32(inst.OpCode.Size().Sizeof())),
156+
)
157+
// --- End of arguments setup ---
158+
159+
currentReplacement = append(currentReplacement, asm.FnSkbLoadBytes.Call())
160+
161+
// Check return value of bpf_skb_load_bytes (in R0)
162+
nomatchLabel := opts.labelPrefix() + "_nomatch"
163+
currentReplacement = append(currentReplacement,
164+
asm.JNE.Imm(asm.R0, 0, nomatchLabel).WithReference(nomatchLabel),
165+
)
166+
167+
// Load data from stack buffer to original inst.Dst
168+
currentReplacement = append(currentReplacement,
169+
asm.LoadMem(inst.Dst, asm.RFP, int16(BpfDataReadOffset), inst.OpCode.Size()),
170+
)
171+
172+
// Restore R4 (PacketStart) and R5 (PacketEnd) from their initially saved values
173+
currentReplacement = append(currentReplacement,
174+
asm.LoadMem(asm.R4, asm.RFP, int16(PacketStartSavedOnStack), asm.DWord),
175+
asm.LoadMem(asm.R5, asm.RFP, int16(PacketEndSavedOnStack), asm.DWord),
176+
)
177+
178+
// Restore original live R1, R2, R3 (unless inst.Dst was one of them)
179+
var restoreLiveRegs asm.Instructions
180+
if inst.Dst != asm.R1 {
181+
restoreLiveRegs = append(restoreLiveRegs, asm.LoadMem(asm.R1, asm.RFP, int16(R1LiveSavedOffset), asm.DWord))
182+
}
183+
if inst.Dst != asm.R2 {
184+
// If R2 was inst.Src in the dynamic case, its value might have been changed by the offset calculation.
185+
// However, the R2LiveSavedOffset holds the value R2 had *before* this entire replacement block.
186+
// So, restoring from R2LiveSavedOffset is correct.
187+
restoreLiveRegs = append(restoreLiveRegs, asm.LoadMem(asm.R2, asm.RFP, int16(R2LiveSavedOffset), asm.DWord))
188+
}
189+
if inst.Dst != asm.R3 {
190+
restoreLiveRegs = append(restoreLiveRegs, asm.LoadMem(asm.R3, asm.RFP, int16(R3LiveSavedOffset), asm.DWord))
191+
}
192+
// R4 was restored from PacketStartSavedOnStack, not R4LiveSavedOffset, which is correct.
193+
currentReplacement = append(currentReplacement, restoreLiveRegs...)
194+
195+
currentReplacement[0].Metadata = inst.Metadata
196+
tempReplaceInstsMap[originalIdx] = currentReplacement
197+
}
198+
}
199+
200+
finalProcessedInsts := make(asm.Instructions, 0, len(insts)+len(tempReplaceIdx)*15) // Rough estimate
201+
nextOriginalIdxToProcess := 0
202+
for _, originalIdxToReplace := range tempReplaceIdx {
203+
finalProcessedInsts = append(finalProcessedInsts, insts[nextOriginalIdxToProcess:originalIdxToReplace]...)
204+
finalProcessedInsts = append(finalProcessedInsts, tempReplaceInstsMap[originalIdxToReplace]...)
205+
nextOriginalIdxToProcess = originalIdxToReplace + 1
206+
}
207+
finalProcessedInsts = append(finalProcessedInsts, insts[nextOriginalIdxToProcess:]...)
208+
209+
resultWithPrefix := append(prefixInsts, finalProcessedInsts...)
210+
211+
return resultWithPrefix, nil
212+
}

0 commit comments

Comments
 (0)