From 33ee5d528a146b7eb995c5a69cf682e77186ffc2 Mon Sep 17 00:00:00 2001 From: Takeshi Yoneda Date: Fri, 5 Jan 2024 09:41:37 -0800 Subject: [PATCH 1/2] wazevo: adds ABIs for amd64 Signed-off-by: Takeshi Yoneda --- internal/engine/wazevo/backend/abi.go | 103 ++++++++- .../engine/wazevo/backend/compiler_lower.go | 4 +- .../engine/wazevo/backend/isa/amd64/abi.go | 87 ++++++++ .../engine/wazevo/backend/isa/amd64/instr.go | 21 ++ .../wazevo/backend/isa/amd64/machine.go | 79 +++---- .../engine/wazevo/backend/isa/amd64/reg.go | 149 +++++++++++++ .../engine/wazevo/backend/isa/arm64/abi.go | 195 ++++++------------ .../backend/isa/arm64/abi_entry_arm64.go | 2 +- .../backend/isa/arm64/abi_entry_preamble.go | 57 +++-- .../isa/arm64/abi_entry_preamble_test.go | 3 +- .../wazevo/backend/isa/arm64/abi_go_call.go | 22 +- .../backend/isa/arm64/abi_go_call_test.go | 10 +- .../wazevo/backend/isa/arm64/abi_test.go | 59 +++--- .../engine/wazevo/backend/isa/arm64/instr.go | 16 +- .../backend/isa/arm64/instr_encoding.go | 2 +- .../wazevo/backend/isa/arm64/machine.go | 13 +- .../isa/arm64/machine_pro_epi_logue.go | 4 +- .../isa/arm64/machine_pro_epi_logue_test.go | 12 +- .../wazevo/backend/isa/arm64/machine_test.go | 2 +- internal/engine/wazevo/backend/machine.go | 9 +- .../engine/wazevo/backend/machine_test.go | 23 +-- .../engine/wazevo/backend/regalloc/reg.go | 2 - .../wazevo/backend/regalloc/regalloc.go | 32 ++- .../engine/wazevo/backend/regalloc/regset.go | 22 +- 24 files changed, 578 insertions(+), 350 deletions(-) create mode 100644 internal/engine/wazevo/backend/isa/amd64/abi.go create mode 100644 internal/engine/wazevo/backend/isa/amd64/instr.go create mode 100644 internal/engine/wazevo/backend/isa/amd64/reg.go diff --git a/internal/engine/wazevo/backend/abi.go b/internal/engine/wazevo/backend/abi.go index 0d1eb4cb6b..54a5e8a2cc 100644 --- a/internal/engine/wazevo/backend/abi.go +++ b/internal/engine/wazevo/backend/abi.go @@ -7,15 +7,23 @@ import ( "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" ) -// FunctionABI represents an ABI for the specific target combined with the function signature. -type FunctionABI interface { - // CalleeGenFunctionArgsToVRegs generates instructions to move arguments to virtual registers. - CalleeGenFunctionArgsToVRegs(regs []ssa.Value) - // CalleeGenVRegsToFunctionReturns generates instructions to move virtual registers to a return value locations. - CalleeGenVRegsToFunctionReturns(regs []ssa.Value) +type FunctionABIRegInfo interface { + // ArgsResultsRegs returns the registers used for passing parameters. + ArgsResultsRegs() (argInts, argFloats, resultInt, resultFloats []regalloc.RealReg) } type ( + FunctionABI[R FunctionABIRegInfo] struct { + r R + Initialized bool + + Args, Rets []ABIArg + ArgStackSize, RetStackSize int64 + + ArgRealRegs []regalloc.VReg + RetRealRegs []regalloc.VReg + } + // ABIArg represents either argument or return value's location. ABIArg struct { // Index is the index of the argument. @@ -59,3 +67,86 @@ func (a ABIArgKind) String() string { panic("BUG") } } + +// Init initializes the abiImpl for the given signature. +func (a *FunctionABI[M]) Init(sig *ssa.Signature) { + argInts, argFloats, resultInts, resultFloats := a.r.ArgsResultsRegs() + + if len(a.Rets) < len(sig.Results) { + a.Rets = make([]ABIArg, len(sig.Results)) + } + a.Rets = a.Rets[:len(sig.Results)] + a.RetStackSize = a.setABIArgs(a.Rets, sig.Results, argInts, argFloats) + if argsNum := len(sig.Params); len(a.Args) < argsNum { + a.Args = make([]ABIArg, argsNum) + } + a.Args = a.Args[:len(sig.Params)] + a.ArgStackSize = a.setABIArgs(a.Args, sig.Params, resultInts, resultFloats) + + // Gather the real registers usages in arg/return. + a.RetRealRegs = a.RetRealRegs[:0] + for i := range a.Rets { + r := &a.Rets[i] + if r.Kind == ABIArgKindReg { + a.RetRealRegs = append(a.RetRealRegs, r.Reg) + } + } + a.ArgRealRegs = a.ArgRealRegs[:0] + for i := range a.Args { + arg := &a.Args[i] + if arg.Kind == ABIArgKindReg { + reg := arg.Reg + a.ArgRealRegs = append(a.ArgRealRegs, reg) + } + } + + a.Initialized = true +} + +// setABIArgs sets the ABI arguments in the given slice. This assumes that len(s) >= len(types) +// where if len(s) > len(types), the last elements of s is for the multi-return slot. +func (a *FunctionABI[M]) setABIArgs(s []ABIArg, types []ssa.Type, ints, floats []regalloc.RealReg) (stackSize int64) { + il, fl := len(ints), len(floats) + + var stackOffset int64 + intParamIndex, floatParamIndex := 0, 0 + for i, typ := range types { + arg := &s[i] + arg.Index = i + arg.Type = typ + if typ.IsInt() { + if intParamIndex >= il { + arg.Kind = ABIArgKindStack + const slotSize = 8 // Align 8 bytes. + arg.Offset = stackOffset + stackOffset += slotSize + } else { + arg.Kind = ABIArgKindReg + arg.Reg = regalloc.FromRealReg(ints[intParamIndex], regalloc.RegTypeInt) + intParamIndex++ + } + } else { + if floatParamIndex >= fl { + arg.Kind = ABIArgKindStack + slotSize := int64(8) // Align at least 8 bytes. + if typ.Bits() == 128 { // Vector. + slotSize = 16 + } + arg.Offset = stackOffset + stackOffset += slotSize + } else { + arg.Kind = ABIArgKindReg + arg.Reg = regalloc.FromRealReg(floats[floatParamIndex], regalloc.RegTypeFloat) + floatParamIndex++ + } + } + } + return stackOffset +} + +func (a *FunctionABI[M]) AlignedArgResultStackSlotSize() int64 { + stackSlotSize := a.RetStackSize + a.ArgStackSize + // Align stackSlotSize to 16 bytes. + stackSlotSize = (stackSlotSize + 15) &^ 15 + return stackSlotSize +} diff --git a/internal/engine/wazevo/backend/compiler_lower.go b/internal/engine/wazevo/backend/compiler_lower.go index 7a99ee546b..ddc7e67951 100644 --- a/internal/engine/wazevo/backend/compiler_lower.go +++ b/internal/engine/wazevo/backend/compiler_lower.go @@ -129,12 +129,12 @@ func (c *compiler) lowerFunctionArguments(entry ssa.BasicBlock) { c.tmpVals = append(c.tmpVals, ssa.ValueInvalid) } } - c.mach.ABI().CalleeGenFunctionArgsToVRegs(c.tmpVals) + c.mach.LowerParams(c.tmpVals) ectx.FlushPendingInstructions() } func (c *compiler) lowerFunctionReturns(returns []ssa.Value) { - c.mach.ABI().CalleeGenVRegsToFunctionReturns(returns) + c.mach.LowerReturns(returns) } // lowerBlockArguments lowers how to pass arguments to the given successor block. diff --git a/internal/engine/wazevo/backend/isa/amd64/abi.go b/internal/engine/wazevo/backend/isa/amd64/abi.go new file mode 100644 index 0000000000..11764086e1 --- /dev/null +++ b/internal/engine/wazevo/backend/isa/amd64/abi.go @@ -0,0 +1,87 @@ +package amd64 + +import ( + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend" + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc" + "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" +) + +// For the details of the ABI, see: +// https://github.com/golang/go/blob/49d42128fd8594c172162961ead19ac95e247d24/src/cmd/compile/abi-internal.md#amd64-architecture + +var ( + intArgResultRegs = []regalloc.RealReg{rax, rcx, rbx, rsi, rdi, r8, r9, r10, r11} + floatArgResultRegs = []regalloc.RealReg{xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7} +) + +var regInfo = ®alloc.RegisterInfo{ + AllocatableRegisters: [regalloc.NumRegType][]regalloc.RealReg{ + regalloc.RegTypeInt: { + rax, rcx, rdx, rbx, rsi, rdi, r8, r9, r10, r11, r12, r13, r14, r15, + }, + regalloc.RegTypeFloat: { + xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15, + }, + }, + CalleeSavedRegisters: regalloc.NewRegSet( + rdx, r12, r13, r14, r15, + xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15, + ), + CallerSavedRegisters: regalloc.NewRegSet( + rax, rcx, rbx, rsi, rdi, r8, r9, r10, r11, + xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, + ), + RealRegToVReg: []regalloc.VReg{ + rax: raxVReg, rcx: rcxVReg, rdx: rdxVReg, rbx: rbxVReg, rsp: rspVReg, rbp: rbpVReg, rsi: rsiVReg, rdi: rdiVReg, + r8: r8VReg, r9: r9VReg, r10: r10VReg, r11: r11VReg, r12: r12VReg, r13: r13VReg, r14: r14VReg, r15: r15VReg, + xmm0: xmm0VReg, xmm1: xmm1VReg, xmm2: xmm2VReg, xmm3: xmm3VReg, xmm4: xmm4VReg, xmm5: xmm5VReg, xmm6: xmm6VReg, + xmm7: xmm7VReg, xmm8: xmm8VReg, xmm9: xmm9VReg, xmm10: xmm10VReg, xmm11: xmm11VReg, xmm12: xmm12VReg, + xmm13: xmm13VReg, xmm14: xmm14VReg, xmm15: xmm15VReg, + }, + RealRegName: func(r regalloc.RealReg) string { return regNames[r] }, + RealRegType: func(r regalloc.RealReg) regalloc.RegType { + if r < xmm0 { + return regalloc.RegTypeInt + } + return regalloc.RegTypeFloat + }, +} + +// functionABIRegInfo implements backend.FunctionABIRegInfo. +type functionABIRegInfo struct{} + +// ArgsResultsRegs implements backend.FunctionABIRegInfo. +func (f functionABIRegInfo) ArgsResultsRegs() (argInts, argFloats, resultInt, resultFloats []regalloc.RealReg) { + return intArgResultRegs, floatArgResultRegs, intArgResultRegs, floatArgResultRegs +} + +type abiImpl = backend.FunctionABI[functionABIRegInfo] + +// ParamRegs implements backend.Machine. +func (m *machine) ParamRegs() (ints, floats []regalloc.RealReg) { + return intArgResultRegs, floatArgResultRegs +} + +func (m *machine) getOrCreateFunctionABI(sig *ssa.Signature) *abiImpl { + if int(sig.ID) >= len(m.abis) { + m.abis = append(m.abis, make([]abiImpl, int(sig.ID)+1)...) + } + + abi := &m.abis[sig.ID] + if abi.Initialized { + return abi + } + + abi.Init(sig) + return abi +} + +func (m *machine) LowerParams(params []ssa.Value) { + // TODO implement me + panic("implement me") +} + +func (m *machine) LowerReturns(returns []ssa.Value) { + // TODO implement me + panic("implement me") +} diff --git a/internal/engine/wazevo/backend/isa/amd64/instr.go b/internal/engine/wazevo/backend/isa/amd64/instr.go new file mode 100644 index 0000000000..8cf1369b27 --- /dev/null +++ b/internal/engine/wazevo/backend/isa/amd64/instr.go @@ -0,0 +1,21 @@ +package amd64 + +type instruction struct { + prev, next *instruction +} + +func resetInstruction(i *instruction) { + i.prev = nil + i.next = nil +} + +func setNext(i *instruction, next *instruction) { + i.next = next +} + +func setPrev(i *instruction, prev *instruction) { + i.prev = prev +} + +func asNop(*instruction) { +} diff --git a/internal/engine/wazevo/backend/isa/amd64/machine.go b/internal/engine/wazevo/backend/isa/amd64/machine.go index 4326bcf21c..321907fbe9 100644 --- a/internal/engine/wazevo/backend/isa/amd64/machine.go +++ b/internal/engine/wazevo/backend/isa/amd64/machine.go @@ -11,54 +11,49 @@ import ( // NewBackend returns a new backend for arm64. func NewBackend() backend.Machine { - m := &machine{} - return m + ectx := backend.NewExecutableContextT[instruction]( + resetInstruction, + setNext, + setPrev, + asNop, + ) + return &machine{ + ectx: ectx, + } } // machine implements backend.Machine for amd64. type machine struct { + c backend.Compiler + ectx *backend.ExecutableContextT[instruction] stackBoundsCheckDisabled bool + + // abis maps ssa.SignatureID to the ABI implementation. + abis []abiImpl + currentABI *abiImpl } -// ExecutableContext implements backend.Machine. -func (m *machine) ExecutableContext() backend.ExecutableContext { - // TODO implement me - panic("implement me") +// Reset implements backend.Machine. +func (m *machine) Reset() { + m.stackBoundsCheckDisabled = false + m.ectx.Reset() } +// ExecutableContext implements backend.Machine. +func (m *machine) ExecutableContext() backend.ExecutableContext { return m.ectx } + // DisableStackCheck implements backend.Machine. -func (m *machine) DisableStackCheck() { - m.stackBoundsCheckDisabled = true -} +func (m *machine) DisableStackCheck() { m.stackBoundsCheckDisabled = true } + +// SetCompiler implements backend.Machine. +func (m *machine) SetCompiler(compiler backend.Compiler) { m.c = compiler } // RegisterInfo implements backend.Machine. -func (m *machine) RegisterInfo() *regalloc.RegisterInfo { - // TODO implement me - panic("implement me") -} +func (m *machine) RegisterInfo() *regalloc.RegisterInfo { return regInfo } // InitializeABI implements backend.Machine. func (m *machine) InitializeABI(sig *ssa.Signature) { - // TODO implement me - panic("implement me") -} - -// ABI implements backend.Machine. -func (m *machine) ABI() backend.FunctionABI { - // TODO implement me - panic("implement me") -} - -// SetCompiler implements backend.Machine. -func (m *machine) SetCompiler(compiler backend.Compiler) { - // TODO implement me - panic("implement me") -} - -// StartLoweringFunction implements backend.Machine. -func (m *machine) StartLoweringFunction(maximumBlockID ssa.BasicBlockID) { - // TODO implement me - panic("implement me") + m.currentABI = m.getOrCreateFunctionABI(sig) } // StartBlock implements backend.Machine. @@ -85,24 +80,6 @@ func (m *machine) LowerInstr(instruction *ssa.Instruction) { panic("implement me") } -// EndBlock implements backend.Machine. -func (m *machine) EndBlock() { - // TODO implement me - panic("implement me") -} - -// LinkAdjacentBlocks implements backend.Machine. -func (m *machine) LinkAdjacentBlocks(prev, next ssa.BasicBlock) { - // TODO implement me - panic("implement me") -} - -// Reset implements backend.Machine. -func (m *machine) Reset() { - // TODO implement me - panic("implement me") -} - // FlushPendingInstructions implements backend.Machine. func (m *machine) FlushPendingInstructions() { // TODO implement me diff --git a/internal/engine/wazevo/backend/isa/amd64/reg.go b/internal/engine/wazevo/backend/isa/amd64/reg.go new file mode 100644 index 0000000000..4b1abe670d --- /dev/null +++ b/internal/engine/wazevo/backend/isa/amd64/reg.go @@ -0,0 +1,149 @@ +package amd64 + +import ( + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc" +) + +// Amd64-specific registers. +// +// Note: naming convention intentionally matches the Go assembler: https://go.dev/doc/asm +// See https://www.lri.fr/~filliatr/ens/compil/x86-64.pdf +// See https://cs.brown.edu/courses/cs033/docs/guides/x64_cheatsheet.pdf +const ( + // rax is a gp register. + rax = regalloc.RealRegInvalid + 1 + iota + // rcx is a gp register. + rcx + // rdx is a gp register. + rdx + // rbx is a gp register. + rbx + // rsp is a gp register. + rsp + // rbp is a gp register. + rbp + // rsi is a gp register. + rsi + // rdi is a gp register. + rdi + // r8 is a gp register. + r8 + // r9 is a gp register. + r9 + // r10 is a gp register. + r10 + // r11 is a gp register. + r11 + // r12 is a gp register. + r12 + // r13 is a gp register. + r13 + // r14 is a gp register. + r14 + // r15 is a gp register. + r15 + + // xmm0 is a vector register. + xmm0 + // xmm1 is a vector register. + xmm1 + // xmm2 is a vector register. + xmm2 + // xmm3 is a vector register. + xmm3 + // xmm4 is a vector register. + xmm4 + // xmm5 is a vector register. + xmm5 + // xmm6 is a vector register. + xmm6 + // xmm7 is a vector register. + xmm7 + // xmm8 is a vector register. + xmm8 + // xmm9 is a vector register. + xmm9 + // xmm10 is a vector register. + xmm10 + // xmm11 is a vector register. + xmm11 + // xmm12 is a vector register. + xmm12 + // xmm13 is a vector register. + xmm13 + // xmm14 is a vector register. + xmm14 + // xmm15 is a vector register. + xmm15 +) + +var ( + raxVReg = regalloc.FromRealReg(rax, regalloc.RegTypeInt) + rcxVReg = regalloc.FromRealReg(rcx, regalloc.RegTypeInt) + rdxVReg = regalloc.FromRealReg(rdx, regalloc.RegTypeInt) + rbxVReg = regalloc.FromRealReg(rbx, regalloc.RegTypeInt) + rspVReg = regalloc.FromRealReg(rsp, regalloc.RegTypeInt) + rbpVReg = regalloc.FromRealReg(rbp, regalloc.RegTypeInt) + rsiVReg = regalloc.FromRealReg(rsi, regalloc.RegTypeInt) + rdiVReg = regalloc.FromRealReg(rdi, regalloc.RegTypeInt) + r8VReg = regalloc.FromRealReg(r8, regalloc.RegTypeInt) + r9VReg = regalloc.FromRealReg(r9, regalloc.RegTypeInt) + r10VReg = regalloc.FromRealReg(r10, regalloc.RegTypeInt) + r11VReg = regalloc.FromRealReg(r11, regalloc.RegTypeInt) + r12VReg = regalloc.FromRealReg(r12, regalloc.RegTypeInt) + r13VReg = regalloc.FromRealReg(r13, regalloc.RegTypeInt) + r14VReg = regalloc.FromRealReg(r14, regalloc.RegTypeInt) + r15VReg = regalloc.FromRealReg(r15, regalloc.RegTypeInt) + + xmm0VReg = regalloc.FromRealReg(xmm0, regalloc.RegTypeFloat) + xmm1VReg = regalloc.FromRealReg(xmm1, regalloc.RegTypeFloat) + xmm2VReg = regalloc.FromRealReg(xmm2, regalloc.RegTypeFloat) + xmm3VReg = regalloc.FromRealReg(xmm3, regalloc.RegTypeFloat) + xmm4VReg = regalloc.FromRealReg(xmm4, regalloc.RegTypeFloat) + xmm5VReg = regalloc.FromRealReg(xmm5, regalloc.RegTypeFloat) + xmm6VReg = regalloc.FromRealReg(xmm6, regalloc.RegTypeFloat) + xmm7VReg = regalloc.FromRealReg(xmm7, regalloc.RegTypeFloat) + xmm8VReg = regalloc.FromRealReg(xmm8, regalloc.RegTypeFloat) + xmm9VReg = regalloc.FromRealReg(xmm9, regalloc.RegTypeFloat) + xmm10VReg = regalloc.FromRealReg(xmm10, regalloc.RegTypeFloat) + xmm11VReg = regalloc.FromRealReg(xmm11, regalloc.RegTypeFloat) + xmm12VReg = regalloc.FromRealReg(xmm12, regalloc.RegTypeFloat) + xmm13VReg = regalloc.FromRealReg(xmm13, regalloc.RegTypeFloat) + xmm14VReg = regalloc.FromRealReg(xmm14, regalloc.RegTypeFloat) + xmm15VReg = regalloc.FromRealReg(xmm15, regalloc.RegTypeFloat) +) + +var regNames = [...]string{ + rax: "rax", + rcx: "rcx", + rdx: "rdx", + rbx: "rbx", + rsp: "rsp", + rbp: "rbp", + rsi: "rsi", + rdi: "rdi", + r8: "r8", + r9: "r9", + r10: "r10", + r11: "r11", + r12: "r12", + r13: "r13", + r14: "r14", + r15: "r15", + xmm0: "xmm0", + xmm1: "xmm1", + xmm2: "xmm2", + xmm3: "xmm3", + xmm4: "xmm4", + xmm5: "xmm5", + xmm6: "xmm6", + xmm7: "xmm7", + xmm8: "xmm8", + xmm9: "xmm9", + xmm10: "xmm10", + xmm11: "xmm11", + xmm12: "xmm12", + xmm13: "xmm13", + xmm14: "xmm14", + xmm15: "xmm15", +} diff --git a/internal/engine/wazevo/backend/isa/arm64/abi.go b/internal/engine/wazevo/backend/isa/arm64/abi.go index 378b754abe..edca84d41b 100644 --- a/internal/engine/wazevo/backend/isa/arm64/abi.go +++ b/internal/engine/wazevo/backend/isa/arm64/abi.go @@ -10,7 +10,10 @@ import ( // * https://github.com/golang/go/blob/49d42128fd8594c172162961ead19ac95e247d24/src/cmd/compile/abi-internal.md#arm64-architecture // * https://developer.arm.com/documentation/102374/0101/Procedure-Call-Standard -const xArgRetRegMax, vArgRetRegMax = x7, v7 // x0-x7 & v0-v7. +var ( + intParamResultRegs = []regalloc.RealReg{x0, x1, x2, x3, x4, x5, x6, x7} + floatParamResultRegs = []regalloc.RealReg{v0, v1, v2, v3, v4, v5, v6, v7} +) var regInfo = ®alloc.RegisterInfo{ AllocatableRegisters: [regalloc.NumRegType][]regalloc.RealReg{ @@ -32,17 +35,14 @@ var regInfo = ®alloc.RegisterInfo{ v7, v6, v5, v4, v3, v2, v1, v0, }, }, - CalleeSavedRegisters: [regalloc.RealRegsNumMax]bool{ - x19: true, x20: true, x21: true, x22: true, x23: true, x24: true, x25: true, x26: true, x28: true, - v18: true, v19: true, v20: true, v21: true, v22: true, v23: true, v24: true, v25: true, v26: true, - v27: true, v28: true, v29: true, v30: true, v31: true, - }, - CallerSavedRegisters: [regalloc.RealRegsNumMax]bool{ - x0: true, x1: true, x2: true, x3: true, x4: true, x5: true, x6: true, x7: true, x8: true, x9: true, x10: true, - x11: true, x12: true, x13: true, x14: true, x15: true, x16: true, x17: true, x29: true, x30: true, - v0: true, v1: true, v2: true, v3: true, v4: true, v5: true, v6: true, v7: true, v8: true, v9: true, v10: true, - v11: true, v12: true, v13: true, v14: true, v15: true, v16: true, v17: true, - }, + CalleeSavedRegisters: regalloc.NewRegSet( + x19, x20, x21, x22, x23, x24, x25, x26, x28, + v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, + ), + CallerSavedRegisters: regalloc.NewRegSet( + x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15, x16, x17, x29, x30, + v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, + ), RealRegToVReg: []regalloc.VReg{ x0: x0VReg, x1: x1VReg, x2: x2VReg, x3: x3VReg, x4: x4VReg, x5: x5VReg, x6: x6VReg, x7: x7VReg, x8: x8VReg, x9: x9VReg, x10: x10VReg, x11: x11VReg, x12: x12VReg, x13: x13VReg, x14: x14VReg, x15: x15VReg, x16: x16VReg, x17: x17VReg, x18: x18VReg, x19: x19VReg, x20: x20VReg, x21: x21VReg, x22: x22VReg, x23: x23VReg, x24: x24VReg, x25: x25VReg, x26: x26VReg, x27: x27VReg, x28: x28VReg, x29: x29VReg, x30: x30VReg, v0: v0VReg, v1: v1VReg, v2: v2VReg, v3: v3VReg, v4: v4VReg, v5: v5VReg, v6: v6VReg, v7: v7VReg, v8: v8VReg, v9: v9VReg, v10: v10VReg, v11: v11VReg, v12: v12VReg, v13: v13VReg, v14: v14VReg, v15: v15VReg, v16: v16VReg, v17: v17VReg, v18: v18VReg, v19: v19VReg, v20: v20VReg, v21: v21VReg, v22: v22VReg, v23: v23VReg, v24: v24VReg, v25: v25VReg, v26: v26VReg, v27: v27VReg, v28: v28VReg, v29: v29VReg, v30: v30VReg, v31: v31VReg, @@ -56,111 +56,42 @@ var regInfo = ®alloc.RegisterInfo{ }, } -// abiImpl implements backend.FunctionABI. -type abiImpl struct { - m *machine - args, rets []backend.ABIArg - argStackSize, retStackSize int64 +// functionABIRegInfo implements backend.FunctionABIRegInfo. +type functionABIRegInfo struct{} - argRealRegs []regalloc.VReg - retRealRegs []regalloc.VReg +// ArgsResultsRegs implements backend.FunctionABIRegInfo. +func (f functionABIRegInfo) ArgsResultsRegs() (argInts, argFloats, resultInt, resultFloats []regalloc.RealReg) { + return intParamResultRegs, floatParamResultRegs, intParamResultRegs, floatParamResultRegs } -func (m *machine) getOrCreateABIImpl(sig *ssa.Signature) *abiImpl { +type functionABI = backend.FunctionABI[functionABIRegInfo] + +func (m *machine) getOrCreateFunctionABI(sig *ssa.Signature) *functionABI { if int(sig.ID) >= len(m.abis) { - m.abis = append(m.abis, make([]abiImpl, int(sig.ID)+1)...) + m.abis = append(m.abis, make([]functionABI, int(sig.ID)+1)...) } abi := &m.abis[sig.ID] - if abi.m != nil { + if abi.Initialized { return abi } - abi.m = m - abi.init(sig) + abi.Init(sig) return abi } -// int initializes the abiImpl for the given signature. -func (a *abiImpl) init(sig *ssa.Signature) { - if len(a.rets) < len(sig.Results) { - a.rets = make([]backend.ABIArg, len(sig.Results)) - } - a.rets = a.rets[:len(sig.Results)] - a.retStackSize = a.setABIArgs(a.rets, sig.Results) - if argsNum := len(sig.Params); len(a.args) < argsNum { - a.args = make([]backend.ABIArg, argsNum) - } - a.args = a.args[:len(sig.Params)] - a.argStackSize = a.setABIArgs(a.args, sig.Params) - - // Gather the real registers usages in arg/return. - a.retRealRegs = a.retRealRegs[:0] - for i := range a.rets { - r := &a.rets[i] - if r.Kind == backend.ABIArgKindReg { - a.retRealRegs = append(a.retRealRegs, r.Reg) - } - } - a.argRealRegs = a.argRealRegs[:0] - for i := range a.args { - arg := &a.args[i] - if arg.Kind == backend.ABIArgKindReg { - reg := arg.Reg - a.argRealRegs = append(a.argRealRegs, reg) - } - } -} - -// setABIArgs sets the ABI arguments in the given slice. This assumes that len(s) >= len(types) -// where if len(s) > len(types), the last elements of s is for the multi-return slot. -func (a *abiImpl) setABIArgs(s []backend.ABIArg, types []ssa.Type) (stackSize int64) { - var stackOffset int64 - nextX, nextV := x0, v0 - for i, typ := range types { - arg := &s[i] - arg.Index = i - arg.Type = typ - if typ.IsInt() { - if nextX > xArgRetRegMax { - arg.Kind = backend.ABIArgKindStack - const slotSize = 8 // Align 8 bytes. - arg.Offset = stackOffset - stackOffset += slotSize - } else { - arg.Kind = backend.ABIArgKindReg - arg.Reg = regalloc.FromRealReg(nextX, regalloc.RegTypeInt) - nextX++ - } - } else { - if nextV > vArgRetRegMax { - arg.Kind = backend.ABIArgKindStack - slotSize := int64(8) // Align at least 8 bytes. - if typ.Bits() == 128 { // Vector. - slotSize = 16 - } - arg.Offset = stackOffset - stackOffset += slotSize - } else { - arg.Kind = backend.ABIArgKindReg - arg.Reg = regalloc.FromRealReg(nextV, regalloc.RegTypeFloat) - nextV++ - } - } - } - return stackOffset -} +// LowerParams implements backend.FunctionABI. +func (m *machine) LowerParams(args []ssa.Value) { + a := m.currentABI -// CalleeGenFunctionArgsToVRegs implements backend.FunctionABI. -func (a *abiImpl) CalleeGenFunctionArgsToVRegs(args []ssa.Value) { for i, ssaArg := range args { if !ssaArg.Valid() { continue } - reg := a.m.compiler.VRegOf(ssaArg) - arg := &a.args[i] + reg := m.compiler.VRegOf(ssaArg) + arg := &a.Args[i] if arg.Kind == backend.ABIArgKindReg { - a.m.InsertMove(reg, arg.Reg, arg.Type) + m.InsertMove(reg, arg.Reg, arg.Type) } else { // TODO: we could use pair load if there's consecutive loads for the same type. // @@ -188,7 +119,6 @@ func (a *abiImpl) CalleeGenFunctionArgsToVRegs(args []ssa.Value) { // SP---> +-----------------+ <-+ // (low address) - m := a.m bits := arg.Type.Bits() // At this point of compilation, we don't yet know how much space exist below the return address. // So we instruct the address mode to add the `argStackOffset` to the offset at the later phase of compilation. @@ -203,27 +133,29 @@ func (a *abiImpl) CalleeGenFunctionArgsToVRegs(args []ssa.Value) { panic("BUG") } m.insert(load) - a.m.unresolvedAddressModes = append(a.m.unresolvedAddressModes, load) + m.unresolvedAddressModes = append(m.unresolvedAddressModes, load) } } } -// CalleeGenVRegsToFunctionReturns implements backend.FunctionABI. -func (a *abiImpl) CalleeGenVRegsToFunctionReturns(rets []ssa.Value) { +// LowerReturns lowers the given returns. +func (m *machine) LowerReturns(rets []ssa.Value) { + a := m.currentABI + l := len(rets) - 1 for i := range rets { // Reverse order in order to avoid overwriting the stack returns existing in the return registers. ret := rets[l-i] - r := &a.rets[l-i] - reg := a.m.compiler.VRegOf(ret) - if def := a.m.compiler.ValueDefinition(ret); def.IsFromInstr() { + r := &a.Rets[l-i] + reg := m.compiler.VRegOf(ret) + if def := m.compiler.ValueDefinition(ret); def.IsFromInstr() { // Constant instructions are inlined. if inst := def.Instr; inst.Constant() { - a.m.InsertLoadConstant(inst, reg) + m.InsertLoadConstant(inst, reg) } } if r.Kind == backend.ABIArgKindReg { - a.m.InsertMove(r.Reg, reg, ret.Type()) + m.InsertMove(r.Reg, reg, ret.Type()) } else { // TODO: we could use pair store if there's consecutive stores for the same type. // @@ -256,46 +188,46 @@ func (a *abiImpl) CalleeGenVRegsToFunctionReturns(rets []ssa.Value) { // At this point of compilation, we don't yet know how much space exist below the return address. // So we instruct the address mode to add the `retStackOffset` to the offset at the later phase of compilation. amode := addressMode{imm: r.Offset, rn: spVReg, kind: addressModeKindResultStackSpace} - store := a.m.allocateInstr() + store := m.allocateInstr() store.asStore(operandNR(reg), amode, bits) - a.m.insert(store) - a.m.unresolvedAddressModes = append(a.m.unresolvedAddressModes, store) + m.insert(store) + m.unresolvedAddressModes = append(m.unresolvedAddressModes, store) } } } // callerGenVRegToFunctionArg is the opposite of GenFunctionArgToVReg, which is used to generate the // caller side of the function call. -func (a *abiImpl) callerGenVRegToFunctionArg(argIndex int, reg regalloc.VReg, def *backend.SSAValueDefinition, slotBegin int64) { - arg := &a.args[argIndex] +func (m *machine) callerGenVRegToFunctionArg(a *functionABI, argIndex int, reg regalloc.VReg, def *backend.SSAValueDefinition, slotBegin int64) { + arg := &a.Args[argIndex] if def != nil && def.IsFromInstr() { // Constant instructions are inlined. if inst := def.Instr; inst.Constant() { - a.m.InsertLoadConstant(inst, reg) + m.InsertLoadConstant(inst, reg) } } if arg.Kind == backend.ABIArgKindReg { - a.m.InsertMove(arg.Reg, reg, arg.Type) + m.InsertMove(arg.Reg, reg, arg.Type) } else { // TODO: we could use pair store if there's consecutive stores for the same type. // // Note that at this point, stack pointer is already adjusted. bits := arg.Type.Bits() - amode := a.m.resolveAddressModeForOffset(arg.Offset-slotBegin, bits, spVReg, false) - store := a.m.allocateInstr() + amode := m.resolveAddressModeForOffset(arg.Offset-slotBegin, bits, spVReg, false) + store := m.allocateInstr() store.asStore(operandNR(reg), amode, bits) - a.m.insert(store) + m.insert(store) } } -func (a *abiImpl) callerGenFunctionReturnVReg(retIndex int, reg regalloc.VReg, slotBegin int64) { - r := &a.rets[retIndex] +func (m *machine) callerGenFunctionReturnVReg(a *functionABI, retIndex int, reg regalloc.VReg, slotBegin int64) { + r := &a.Rets[retIndex] if r.Kind == backend.ABIArgKindReg { - a.m.InsertMove(reg, r.Reg, r.Type) + m.InsertMove(reg, r.Reg, r.Type) } else { // TODO: we could use pair load if there's consecutive loads for the same type. - amode := a.m.resolveAddressModeForOffset(a.argStackSize+r.Offset-slotBegin, r.Type.Bits(), spVReg, false) - ldr := a.m.allocateInstr() + amode := m.resolveAddressModeForOffset(a.ArgStackSize+r.Offset-slotBegin, r.Type.Bits(), spVReg, false) + ldr := m.allocateInstr() switch r.Type { case ssa.TypeI32, ssa.TypeI64: ldr.asULoad(operandNR(reg), amode, r.Type.Bits()) @@ -304,7 +236,7 @@ func (a *abiImpl) callerGenFunctionReturnVReg(retIndex int, reg regalloc.VReg, s default: panic("BUG") } - a.m.insert(ldr) + m.insert(ldr) } } @@ -341,13 +273,6 @@ func (m *machine) resolveAddressModeForOffset(offset int64, dstBits byte, rn reg return amode } -func (a *abiImpl) alignedArgResultStackSlotSize() int64 { - stackSlotSize := a.retStackSize + a.argStackSize - // Align stackSlotSize to 16 bytes. - stackSlotSize = (stackSlotSize + 15) &^ 15 - return stackSlotSize -} - func (m *machine) lowerCall(si *ssa.Instruction) { isDirectCall := si.Opcode() == ssa.OpcodeCall var indirectCalleePtr ssa.Value @@ -359,9 +284,9 @@ func (m *machine) lowerCall(si *ssa.Instruction) { } else { indirectCalleePtr, sigID, args = si.CallIndirectData() } - calleeABI := m.getOrCreateABIImpl(m.compiler.SSABuilder().ResolveSignature(sigID)) + calleeABI := m.getOrCreateFunctionABI(m.compiler.SSABuilder().ResolveSignature(sigID)) - stackSlotSize := calleeABI.alignedArgResultStackSlotSize() + stackSlotSize := calleeABI.AlignedArgResultStackSlotSize() if m.maxRequiredStackSizeForCalls < stackSlotSize+16 { m.maxRequiredStackSizeForCalls = stackSlotSize + 16 // return address frame. } @@ -369,7 +294,7 @@ func (m *machine) lowerCall(si *ssa.Instruction) { for i, arg := range args { reg := m.compiler.VRegOf(arg) def := m.compiler.ValueDefinition(arg) - calleeABI.callerGenVRegToFunctionArg(i, reg, def, stackSlotSize) + m.callerGenVRegToFunctionArg(calleeABI, i, reg, def, stackSlotSize) } if isDirectCall { @@ -386,12 +311,12 @@ func (m *machine) lowerCall(si *ssa.Instruction) { var index int r1, rs := si.Returns() if r1.Valid() { - calleeABI.callerGenFunctionReturnVReg(0, m.compiler.VRegOf(r1), stackSlotSize) + m.callerGenFunctionReturnVReg(calleeABI, 0, m.compiler.VRegOf(r1), stackSlotSize) index++ } for _, r := range rs { - calleeABI.callerGenFunctionReturnVReg(index, m.compiler.VRegOf(r), stackSlotSize) + m.callerGenFunctionReturnVReg(calleeABI, index, m.compiler.VRegOf(r), stackSlotSize) index++ } } diff --git a/internal/engine/wazevo/backend/isa/arm64/abi_entry_arm64.go b/internal/engine/wazevo/backend/isa/arm64/abi_entry_arm64.go index 95190e0465..89fb433ac8 100644 --- a/internal/engine/wazevo/backend/isa/arm64/abi_entry_arm64.go +++ b/internal/engine/wazevo/backend/isa/arm64/abi_entry_arm64.go @@ -1,6 +1,6 @@ package arm64 -// entrypoint enters the machine code generated by this backend which begins with the preamble generated by abiImpl.EmitGoEntryPreamble below. +// entrypoint enters the machine code generated by this backend which begins with the preamble generated by functionABI.EmitGoEntryPreamble below. // This implements wazevo.entrypoint, and see the comments there for detail. func entrypoint(preambleExecutable, functionExecutable *byte, executionContextPtr uintptr, moduleContextPtr *byte, paramResultPtr *uint64, goAllocatedStackSlicePtr uintptr) diff --git a/internal/engine/wazevo/backend/isa/arm64/abi_entry_preamble.go b/internal/engine/wazevo/backend/isa/arm64/abi_entry_preamble.go index 0cec5cf582..9bdda1a1c3 100644 --- a/internal/engine/wazevo/backend/isa/arm64/abi_entry_preamble.go +++ b/internal/engine/wazevo/backend/isa/arm64/abi_entry_preamble.go @@ -16,10 +16,7 @@ import ( // // also SP and FP are correct Go-runtime-based values, and LR is the return address to the Go-side caller. func (m *machine) CompileEntryPreamble(signature *ssa.Signature) []byte { - abi := abiImpl{} - abi.m = m - abi.init(signature) - root := abi.constructEntryPreamble() + root := m.constructEntryPreamble(signature) m.encode(root) return m.compiler.Buf() } @@ -137,58 +134,60 @@ func (m *machine) goEntryPreamblePassResult(cur *instruction, resultSlicePtr reg return cur } -func (a *abiImpl) constructEntryPreamble() (root *instruction) { - m := a.m +func (m *machine) constructEntryPreamble(sig *ssa.Signature) (root *instruction) { + abi := functionABI{} + abi.Init(sig) + root = m.allocateNop() //// ----------------------------------- prologue ----------------------------------- //// // First, we save executionContextPtrReg into a callee-saved register so that it can be used in epilogue as well. // mov savedExecutionContextPtr, x0 - cur := a.move64(savedExecutionContextPtr, executionContextPtrReg, root) + cur := m.move64(savedExecutionContextPtr, executionContextPtrReg, root) // Next, save the current FP, SP and LR into the wazevo.executionContext: // str fp, [savedExecutionContextPtr, #OriginalFramePointer] // mov tmp, sp ;; sp cannot be str'ed directly. // str sp, [savedExecutionContextPtr, #OriginalStackPointer] // str lr, [savedExecutionContextPtr, #GoReturnAddress] - cur = a.loadOrStoreAtExecutionContext(fpVReg, wazevoapi.ExecutionContextOffsetOriginalFramePointer, true, cur) - cur = a.move64(tmpRegVReg, spVReg, cur) - cur = a.loadOrStoreAtExecutionContext(tmpRegVReg, wazevoapi.ExecutionContextOffsetOriginalStackPointer, true, cur) - cur = a.loadOrStoreAtExecutionContext(lrVReg, wazevoapi.ExecutionContextOffsetGoReturnAddress, true, cur) + cur = m.loadOrStoreAtExecutionContext(fpVReg, wazevoapi.ExecutionContextOffsetOriginalFramePointer, true, cur) + cur = m.move64(tmpRegVReg, spVReg, cur) + cur = m.loadOrStoreAtExecutionContext(tmpRegVReg, wazevoapi.ExecutionContextOffsetOriginalStackPointer, true, cur) + cur = m.loadOrStoreAtExecutionContext(lrVReg, wazevoapi.ExecutionContextOffsetGoReturnAddress, true, cur) // Then, move the Go-allocated stack pointer to SP: // mov sp, goAllocatedStackPtr - cur = a.move64(spVReg, goAllocatedStackPtr, cur) + cur = m.move64(spVReg, goAllocatedStackPtr, cur) prReg := paramResultSlicePtr - if len(a.args) > 2 && len(a.rets) > 0 { + if len(abi.Args) > 2 && len(abi.Rets) > 0 { // paramResultSlicePtr is modified during the execution of goEntryPreamblePassArg, // so copy it to another reg. - cur = a.move64(paramResultSliceCopied, paramResultSlicePtr, cur) + cur = m.move64(paramResultSliceCopied, paramResultSlicePtr, cur) prReg = paramResultSliceCopied } - stackSlotSize := a.alignedArgResultStackSlotSize() - for i := range a.args { + stackSlotSize := abi.AlignedArgResultStackSlotSize() + for i := range abi.Args { if i < 2 { // module context ptr and execution context ptr are passed in x0 and x1 by the Go assembly function. continue } - arg := &a.args[i] + arg := &abi.Args[i] cur = m.goEntryPreamblePassArg(cur, prReg, arg, -stackSlotSize) } // Call the real function. bl := m.allocateInstr() - bl.asCallIndirect(functionExecutable, a) + bl.asCallIndirect(functionExecutable, &abi) cur = linkInstr(cur, bl) ///// ----------------------------------- epilogue ----------------------------------- ///// // Store the register results into paramResultSlicePtr. - for i := range a.rets { - cur = m.goEntryPreamblePassResult(cur, paramResultSlicePtr, &a.rets[i], a.argStackSize-stackSlotSize) + for i := range abi.Rets { + cur = m.goEntryPreamblePassResult(cur, paramResultSlicePtr, &abi.Rets[i], abi.ArgStackSize-stackSlotSize) } // Finally, restore the FP, SP and LR, and return to the Go code. @@ -197,24 +196,24 @@ func (a *abiImpl) constructEntryPreamble() (root *instruction) { // mov sp, tmp ;; sp cannot be str'ed directly. // ldr lr, [savedExecutionContextPtr, #GoReturnAddress] // ret ;; --> return to the Go code - cur = a.loadOrStoreAtExecutionContext(fpVReg, wazevoapi.ExecutionContextOffsetOriginalFramePointer, false, cur) - cur = a.loadOrStoreAtExecutionContext(tmpRegVReg, wazevoapi.ExecutionContextOffsetOriginalStackPointer, false, cur) - cur = a.move64(spVReg, tmpRegVReg, cur) - cur = a.loadOrStoreAtExecutionContext(lrVReg, wazevoapi.ExecutionContextOffsetGoReturnAddress, false, cur) - retInst := a.m.allocateInstr() + cur = m.loadOrStoreAtExecutionContext(fpVReg, wazevoapi.ExecutionContextOffsetOriginalFramePointer, false, cur) + cur = m.loadOrStoreAtExecutionContext(tmpRegVReg, wazevoapi.ExecutionContextOffsetOriginalStackPointer, false, cur) + cur = m.move64(spVReg, tmpRegVReg, cur) + cur = m.loadOrStoreAtExecutionContext(lrVReg, wazevoapi.ExecutionContextOffsetGoReturnAddress, false, cur) + retInst := m.allocateInstr() retInst.asRet(nil) linkInstr(cur, retInst) return } -func (a *abiImpl) move64(dst, src regalloc.VReg, prev *instruction) *instruction { - instr := a.m.allocateInstr() +func (m *machine) move64(dst, src regalloc.VReg, prev *instruction) *instruction { + instr := m.allocateInstr() instr.asMove64(dst, src) return linkInstr(prev, instr) } -func (a *abiImpl) loadOrStoreAtExecutionContext(d regalloc.VReg, offset wazevoapi.Offset, store bool, prev *instruction) *instruction { - instr := a.m.allocateInstr() +func (m *machine) loadOrStoreAtExecutionContext(d regalloc.VReg, offset wazevoapi.Offset, store bool, prev *instruction) *instruction { + instr := m.allocateInstr() mode := addressMode{kind: addressModeKindRegUnsignedImm12, rn: savedExecutionContextPtr, imm: offset.I64()} if store { instr.asStore(operandNR(d), mode, 64) diff --git a/internal/engine/wazevo/backend/isa/arm64/abi_entry_preamble_test.go b/internal/engine/wazevo/backend/isa/arm64/abi_entry_preamble_test.go index d0f1ce8ee1..00ee0d871d 100644 --- a/internal/engine/wazevo/backend/isa/arm64/abi_entry_preamble_test.go +++ b/internal/engine/wazevo/backend/isa/arm64/abi_entry_preamble_test.go @@ -396,8 +396,7 @@ func TestAbiImpl_constructEntryPreamble(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { _, _, m := newSetupWithMockContext() - abi := m.getOrCreateABIImpl(tc.sig) - m.executableContext.RootInstr = abi.constructEntryPreamble() + m.executableContext.RootInstr = m.constructEntryPreamble(tc.sig) require.Equal(t, tc.exp, m.Format()) }) } diff --git a/internal/engine/wazevo/backend/isa/arm64/abi_go_call.go b/internal/engine/wazevo/backend/isa/arm64/abi_go_call.go index 8bb7a2b4e4..198d2f1b3e 100644 --- a/internal/engine/wazevo/backend/isa/arm64/abi_go_call.go +++ b/internal/engine/wazevo/backend/isa/arm64/abi_go_call.go @@ -20,8 +20,8 @@ func (m *machine) CompileGoFunctionTrampoline(exitCode wazevoapi.ExitCode, sig * argBegin++ } - abi := &abiImpl{m: m} - abi.init(sig) + abi := &functionABI{} + abi.Init(sig) m.currentABI = abi cur := m.allocateInstr() @@ -70,7 +70,7 @@ func (m *machine) CompileGoFunctionTrampoline(exitCode wazevoapi.ExitCode, sig * cur = m.insertStackBoundsCheck(goCallStackSize+frameInfoSize, cur) originalArg0Reg := x17VReg // Caller save, so we can use it for whatever we want. - if m.currentABI.alignedArgResultStackSlotSize() > 0 { + if m.currentABI.AlignedArgResultStackSlotSize() > 0 { // At this point, SP points to `ReturnAddress`, so add 16 to get the original arg 0 slot. cur = m.addsAddOrSubStackPointer(cur, originalArg0Reg, frameInfoSize, true) } @@ -102,8 +102,8 @@ func (m *machine) CompileGoFunctionTrampoline(exitCode wazevoapi.ExitCode, sig * copySp.asMove64(arg0ret0AddrReg, spVReg) cur = linkInstr(cur, copySp) - for i := range abi.args[argBegin:] { - arg := &abi.args[argBegin+i] + for i := range abi.Args[argBegin:] { + arg := &abi.Args[argBegin+i] store := m.allocateInstr() var v regalloc.VReg if arg.Kind == backend.ABIArgKindReg { @@ -157,7 +157,7 @@ func (m *machine) CompileGoFunctionTrampoline(exitCode wazevoapi.ExitCode, sig * cur = m.restoreRegistersInExecutionContext(cur, calleeSavedRegistersSorted) // Get the pointer to the arg[0]/ret[0]: We need to skip `frame_size + sliceSize`. - if len(abi.rets) > 0 { + if len(abi.Rets) > 0 { cur = m.addsAddOrSubStackPointer(cur, arg0ret0AddrReg, frameInfoSize, true) } @@ -170,17 +170,17 @@ func (m *machine) CompileGoFunctionTrampoline(exitCode wazevoapi.ExitCode, sig * cur = linkInstr(cur, ldr) originalRet0Reg := x17VReg // Caller save, so we can use it for whatever we want. - if m.currentABI.retStackSize > 0 { - cur = m.addsAddOrSubStackPointer(cur, originalRet0Reg, m.currentABI.argStackSize, true) + if m.currentABI.RetStackSize > 0 { + cur = m.addsAddOrSubStackPointer(cur, originalRet0Reg, m.currentABI.ArgStackSize, true) } // Make the SP point to the original address (above the result slot). - if s := m.currentABI.alignedArgResultStackSlotSize(); s > 0 { + if s := m.currentABI.AlignedArgResultStackSlotSize(); s > 0 { cur = m.addsAddOrSubStackPointer(cur, spVReg, s, true) } - for i := range abi.rets { - r := &abi.rets[i] + for i := range abi.Rets { + r := &abi.Rets[i] if r.Kind == backend.ABIArgKindReg { loadIntoReg := m.allocateInstr() mode := addressMode{kind: addressModeKindPostIndex, rn: arg0ret0AddrReg} diff --git a/internal/engine/wazevo/backend/isa/arm64/abi_go_call_test.go b/internal/engine/wazevo/backend/isa/arm64/abi_go_call_test.go index 2cb710e44e..d8b5c927f4 100644 --- a/internal/engine/wazevo/backend/isa/arm64/abi_go_call_test.go +++ b/internal/engine/wazevo/backend/isa/arm64/abi_go_call_test.go @@ -13,11 +13,9 @@ import ( func Test_calleeSavedRegistersSorted(t *testing.T) { var exp []regalloc.VReg - for i, r := range regInfo.CalleeSavedRegisters { - if r { - exp = append(exp, regInfo.RealRegToVReg[i]) - } - } + regInfo.CalleeSavedRegisters.Range(func(r regalloc.RealReg) { + exp = append(exp, regInfo.RealRegToVReg[r]) + }) sort.Slice(exp, func(i, j int) bool { return exp[i].RealReg() < exp[j].RealReg() }) @@ -625,7 +623,7 @@ func Test_goFunctionCallStoreStackResult(t *testing.T) { t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) { _, _, m := newSetupWithMockContext() - m.currentABI = &abiImpl{argStackSize: 8} + m.currentABI = &functionABI{ArgStackSize: 8} nop := m.allocateNop() m.goFunctionCallStoreStackResult(nop, spVReg, tc.result, tc.resultReg) diff --git a/internal/engine/wazevo/backend/isa/arm64/abi_test.go b/internal/engine/wazevo/backend/isa/arm64/abi_test.go index a917a67a64..9d4b867373 100644 --- a/internal/engine/wazevo/backend/isa/arm64/abi_test.go +++ b/internal/engine/wazevo/backend/isa/arm64/abi_test.go @@ -13,12 +13,12 @@ func TestAbiImpl_init(t *testing.T) { for _, tc := range []struct { name string sig *ssa.Signature - exp abiImpl + exp functionABI }{ { name: "empty sig", sig: &ssa.Signature{}, - exp: abiImpl{}, + exp: functionABI{}, }, { name: "small sig", @@ -26,19 +26,18 @@ func TestAbiImpl_init(t *testing.T) { Params: []ssa.Type{ssa.TypeI32, ssa.TypeF32, ssa.TypeI32}, Results: []ssa.Type{ssa.TypeI64, ssa.TypeF64}, }, - exp: abiImpl{ - m: nil, - args: []backend.ABIArg{ + exp: functionABI{ + Args: []backend.ABIArg{ {Index: 0, Kind: backend.ABIArgKindReg, Reg: x0VReg, Type: ssa.TypeI32}, {Index: 1, Kind: backend.ABIArgKindReg, Reg: v0VReg, Type: ssa.TypeF32}, {Index: 2, Kind: backend.ABIArgKindReg, Reg: x1VReg, Type: ssa.TypeI32}, }, - rets: []backend.ABIArg{ + Rets: []backend.ABIArg{ {Index: 0, Kind: backend.ABIArgKindReg, Reg: x0VReg, Type: ssa.TypeI64}, {Index: 1, Kind: backend.ABIArgKindReg, Reg: v0VReg, Type: ssa.TypeF64}, }, - argRealRegs: []regalloc.VReg{x0VReg, v0VReg, x1VReg}, - retRealRegs: []regalloc.VReg{x0VReg, v0VReg}, + ArgRealRegs: []regalloc.VReg{x0VReg, v0VReg, x1VReg}, + RetRealRegs: []regalloc.VReg{x0VReg, v0VReg}, }, }, { @@ -81,9 +80,9 @@ func TestAbiImpl_init(t *testing.T) { ssa.TypeI64, ssa.TypeF64, }, }, - exp: abiImpl{ - argStackSize: 128, retStackSize: 128, - args: []backend.ABIArg{ + exp: functionABI{ + ArgStackSize: 128, RetStackSize: 128, + Args: []backend.ABIArg{ {Index: 0, Kind: backend.ABIArgKindReg, Reg: x0VReg, Type: ssa.TypeI32}, {Index: 1, Kind: backend.ABIArgKindReg, Reg: v0VReg, Type: ssa.TypeF32}, {Index: 2, Kind: backend.ABIArgKindReg, Reg: x1VReg, Type: ssa.TypeI64}, @@ -117,7 +116,7 @@ func TestAbiImpl_init(t *testing.T) { {Index: 30, Kind: backend.ABIArgKindStack, Type: ssa.TypeI64, Offset: 112}, {Index: 31, Kind: backend.ABIArgKindStack, Type: ssa.TypeF64, Offset: 120}, }, - rets: []backend.ABIArg{ + Rets: []backend.ABIArg{ {Index: 0, Kind: backend.ABIArgKindReg, Reg: x0VReg, Type: ssa.TypeI32}, {Index: 1, Kind: backend.ABIArgKindReg, Reg: v0VReg, Type: ssa.TypeF32}, {Index: 2, Kind: backend.ABIArgKindReg, Reg: x1VReg, Type: ssa.TypeI64}, @@ -151,8 +150,8 @@ func TestAbiImpl_init(t *testing.T) { {Index: 30, Kind: backend.ABIArgKindStack, Type: ssa.TypeI64, Offset: 112}, {Index: 31, Kind: backend.ABIArgKindStack, Type: ssa.TypeF64, Offset: 120}, }, - retRealRegs: []regalloc.VReg{x0VReg, v0VReg, x1VReg, v1VReg, x2VReg, v2VReg, x3VReg, v3VReg, x4VReg, v4VReg, x5VReg, v5VReg, x6VReg, v6VReg, x7VReg, v7VReg}, - argRealRegs: []regalloc.VReg{x0VReg, v0VReg, x1VReg, v1VReg, x2VReg, v2VReg, x3VReg, v3VReg, x4VReg, v4VReg, x5VReg, v5VReg, x6VReg, v6VReg, x7VReg, v7VReg}, + RetRealRegs: []regalloc.VReg{x0VReg, v0VReg, x1VReg, v1VReg, x2VReg, v2VReg, x3VReg, v3VReg, x4VReg, v4VReg, x5VReg, v5VReg, x6VReg, v6VReg, x7VReg, v7VReg}, + ArgRealRegs: []regalloc.VReg{x0VReg, v0VReg, x1VReg, v1VReg, x2VReg, v2VReg, x3VReg, v3VReg, x4VReg, v4VReg, x5VReg, v5VReg, x6VReg, v6VReg, x7VReg, v7VReg}, }, }, { @@ -179,8 +178,8 @@ func TestAbiImpl_init(t *testing.T) { ssa.TypeI64, ssa.TypeF64, }, }, - exp: abiImpl{ - args: []backend.ABIArg{ + exp: functionABI{ + Args: []backend.ABIArg{ {Index: 0, Kind: backend.ABIArgKindReg, Reg: x0VReg, Type: ssa.TypeI32}, {Index: 1, Kind: backend.ABIArgKindReg, Reg: v0VReg, Type: ssa.TypeF32}, {Index: 2, Kind: backend.ABIArgKindReg, Reg: x1VReg, Type: ssa.TypeI64}, @@ -198,7 +197,7 @@ func TestAbiImpl_init(t *testing.T) { {Index: 14, Kind: backend.ABIArgKindReg, Reg: x7VReg, Type: ssa.TypeI64}, {Index: 15, Kind: backend.ABIArgKindReg, Reg: v7VReg, Type: ssa.TypeF64}, }, - rets: []backend.ABIArg{ + Rets: []backend.ABIArg{ {Index: 0, Kind: backend.ABIArgKindReg, Reg: x0VReg, Type: ssa.TypeI32}, {Index: 1, Kind: backend.ABIArgKindReg, Reg: v0VReg, Type: ssa.TypeF32}, {Index: 2, Kind: backend.ABIArgKindReg, Reg: x1VReg, Type: ssa.TypeI64}, @@ -216,21 +215,21 @@ func TestAbiImpl_init(t *testing.T) { {Index: 14, Kind: backend.ABIArgKindReg, Reg: x7VReg, Type: ssa.TypeI64}, {Index: 15, Kind: backend.ABIArgKindReg, Reg: v7VReg, Type: ssa.TypeF64}, }, - retRealRegs: []regalloc.VReg{x0VReg, v0VReg, x1VReg, v1VReg, x2VReg, v2VReg, x3VReg, v3VReg, x4VReg, v4VReg, x5VReg, v5VReg, x6VReg, v6VReg, x7VReg, v7VReg}, - argRealRegs: []regalloc.VReg{x0VReg, v0VReg, x1VReg, v1VReg, x2VReg, v2VReg, x3VReg, v3VReg, x4VReg, v4VReg, x5VReg, v5VReg, x6VReg, v6VReg, x7VReg, v7VReg}, + RetRealRegs: []regalloc.VReg{x0VReg, v0VReg, x1VReg, v1VReg, x2VReg, v2VReg, x3VReg, v3VReg, x4VReg, v4VReg, x5VReg, v5VReg, x6VReg, v6VReg, x7VReg, v7VReg}, + ArgRealRegs: []regalloc.VReg{x0VReg, v0VReg, x1VReg, v1VReg, x2VReg, v2VReg, x3VReg, v3VReg, x4VReg, v4VReg, x5VReg, v5VReg, x6VReg, v6VReg, x7VReg, v7VReg}, }, }, } { tc := tc t.Run(tc.name, func(t *testing.T) { - abi := abiImpl{} - abi.init(tc.sig) - require.Equal(t, tc.exp.args, abi.args) - require.Equal(t, tc.exp.rets, abi.rets) - require.Equal(t, tc.exp.argStackSize, abi.argStackSize) - require.Equal(t, tc.exp.retStackSize, abi.retStackSize) - require.Equal(t, tc.exp.retRealRegs, abi.retRealRegs) - require.Equal(t, tc.exp.argRealRegs, abi.argRealRegs) + abi := functionABI{} + abi.Init(tc.sig) + require.Equal(t, tc.exp.Args, abi.Args) + require.Equal(t, tc.exp.Rets, abi.Rets) + require.Equal(t, tc.exp.ArgStackSize, abi.ArgStackSize) + require.Equal(t, tc.exp.RetStackSize, abi.RetStackSize) + require.Equal(t, tc.exp.RetRealRegs, abi.RetRealRegs) + require.Equal(t, tc.exp.ArgRealRegs, abi.ArgRealRegs) }) } } @@ -265,9 +264,9 @@ func TestAbiImpl_callerGenVRegToFunctionArg_constant_inlining(t *testing.T) { i64 := builder.AllocateInstruction().AsIconst64(10).Insert(builder) f64 := builder.AllocateInstruction().AsF64const(3.14).Insert(builder) - abi := m.getOrCreateABIImpl(&ssa.Signature{Params: []ssa.Type{ssa.TypeI64, ssa.TypeF64}}) - abi.callerGenVRegToFunctionArg(0, regalloc.VReg(100).SetRegType(regalloc.RegTypeInt), &backend.SSAValueDefinition{Instr: i64, RefCount: 1}, 0) - abi.callerGenVRegToFunctionArg(1, regalloc.VReg(50).SetRegType(regalloc.RegTypeFloat), &backend.SSAValueDefinition{Instr: f64, RefCount: 1}, 0) + abi := m.getOrCreateFunctionABI(&ssa.Signature{Params: []ssa.Type{ssa.TypeI64, ssa.TypeF64}}) + m.callerGenVRegToFunctionArg(abi, 0, regalloc.VReg(100).SetRegType(regalloc.RegTypeInt), &backend.SSAValueDefinition{Instr: i64, RefCount: 1}, 0) + m.callerGenVRegToFunctionArg(abi, 1, regalloc.VReg(50).SetRegType(regalloc.RegTypeFloat), &backend.SSAValueDefinition{Instr: f64, RefCount: 1}, 0) require.Equal(t, `movz x100?, #0xa, lsl 0 mov x0, x100? ldr d50?, #8; b 16; data.f64 3.140000 diff --git a/internal/engine/wazevo/backend/isa/arm64/instr.go b/internal/engine/wazevo/backend/isa/arm64/instr.go index 7bd9cfb3d3..8019556977 100644 --- a/internal/engine/wazevo/backend/isa/arm64/instr.go +++ b/internal/engine/wazevo/backend/isa/arm64/instr.go @@ -26,7 +26,7 @@ type ( u1, u2, u3 uint64 rd, rm, rn, ra operand amode addressMode - abi *abiImpl + abi *functionABI targets []uint32 addedBeforeRegAlloc bool } @@ -157,7 +157,7 @@ func (i *instruction) Defs(regs *[]regalloc.VReg) []regalloc.VReg { case defKindRD: *regs = append(*regs, i.rd.nr()) case defKindCall: - *regs = append(*regs, i.abi.retRealRegs...) + *regs = append(*regs, i.abi.RetRealRegs...) default: panic(fmt.Sprintf("defKind for %v not defined", i)) } @@ -307,7 +307,7 @@ func (i *instruction) Uses(regs *[]regalloc.VReg) []regalloc.VReg { *regs = append(*regs, rm) } case useKindRet: - *regs = append(*regs, i.abi.retRealRegs...) + *regs = append(*regs, i.abi.RetRealRegs...) case useKindAMode: if amodeRN := i.amode.rn; amodeRN.Valid() { *regs = append(*regs, amodeRN) @@ -329,10 +329,10 @@ func (i *instruction) Uses(regs *[]regalloc.VReg) []regalloc.VReg { *regs = append(*regs, cnd.register()) } case useKindCall: - *regs = append(*regs, i.abi.argRealRegs...) + *regs = append(*regs, i.abi.ArgRealRegs...) case useKindCallInd: *regs = append(*regs, i.rn.nr()) - *regs = append(*regs, i.abi.argRealRegs...) + *regs = append(*regs, i.abi.ArgRealRegs...) case useKindVecRRRRewrite: *regs = append(*regs, i.rn.reg()) *regs = append(*regs, i.rm.reg()) @@ -446,13 +446,13 @@ func (i *instruction) AssignUse(index int, reg regalloc.VReg) { } } -func (i *instruction) asCall(ref ssa.FuncRef, abi *abiImpl) { +func (i *instruction) asCall(ref ssa.FuncRef, abi *functionABI) { i.kind = call i.u1 = uint64(ref) i.abi = abi } -func (i *instruction) asCallIndirect(ptr regalloc.VReg, abi *abiImpl) { +func (i *instruction) asCallIndirect(ptr regalloc.VReg, abi *functionABI) { i.kind = callInd i.rn = operandNR(ptr) i.abi = abi @@ -509,7 +509,7 @@ func (i *instruction) nop0Label() label { return label(i.u1) } -func (i *instruction) asRet(abi *abiImpl) { +func (i *instruction) asRet(abi *functionABI) { i.kind = ret i.abi = abi } diff --git a/internal/engine/wazevo/backend/isa/arm64/instr_encoding.go b/internal/engine/wazevo/backend/isa/arm64/instr_encoding.go index 52ee3ab569..2b8231e4ed 100644 --- a/internal/engine/wazevo/backend/isa/arm64/instr_encoding.go +++ b/internal/engine/wazevo/backend/isa/arm64/instr_encoding.go @@ -2150,7 +2150,7 @@ func encodeBrTableSequence(c backend.Compiler, index regalloc.VReg, targets []ui } } -// encodeExitSequence matches the implementation detail of abiImpl.emitGoEntryPreamble. +// encodeExitSequence matches the implementation detail of functionABI.emitGoEntryPreamble. func encodeExitSequence(c backend.Compiler, ctxReg regalloc.VReg) { // Restore the FP, SP and LR, and return to the Go code: // ldr lr, [ctxReg, #GoReturnAddress] diff --git a/internal/engine/wazevo/backend/isa/arm64/machine.go b/internal/engine/wazevo/backend/isa/arm64/machine.go index 60fdb34d2e..9581f3ecf4 100644 --- a/internal/engine/wazevo/backend/isa/arm64/machine.go +++ b/internal/engine/wazevo/backend/isa/arm64/machine.go @@ -16,9 +16,9 @@ type ( machine struct { compiler backend.Compiler executableContext *backend.ExecutableContextT[instruction] - currentABI *abiImpl + currentABI *functionABI // abis maps ssa.SignatureID to the ABI implementation. - abis []abiImpl + abis []functionABI regAllocFn regAllocFunctionImpl @@ -137,7 +137,7 @@ func (m *machine) Reset() { // InitializeABI implements backend.Machine InitializeABI. func (m *machine) InitializeABI(sig *ssa.Signature) { - m.currentABI = m.getOrCreateABIImpl(sig) + m.currentABI = m.getOrCreateFunctionABI(sig) } // DisableStackCheck implements backend.Machine DisableStackCheck. @@ -145,11 +145,6 @@ func (m *machine) DisableStackCheck() { m.stackBoundsCheckDisabled = true } -// ABI implements backend.Machine. -func (m *machine) ABI() backend.FunctionABI { - return m.currentABI -} - // SetCompiler implements backend.Machine. func (m *machine) SetCompiler(ctx backend.Compiler) { m.compiler = ctx @@ -477,7 +472,7 @@ func (m *machine) arg0OffsetFromSP() int64 { } func (m *machine) ret0OffsetFromSP() int64 { - return m.arg0OffsetFromSP() + m.currentABI.argStackSize + return m.arg0OffsetFromSP() + m.currentABI.ArgStackSize } func (m *machine) requiredStackSize() int64 { diff --git a/internal/engine/wazevo/backend/isa/arm64/machine_pro_epi_logue.go b/internal/engine/wazevo/backend/isa/arm64/machine_pro_epi_logue.go index fa4966407f..36436dac11 100644 --- a/internal/engine/wazevo/backend/isa/arm64/machine_pro_epi_logue.go +++ b/internal/engine/wazevo/backend/isa/arm64/machine_pro_epi_logue.go @@ -147,7 +147,7 @@ func (m *machine) SetupPrologue() { func (m *machine) createReturnAddrAndSizeOfArgRetSlot(cur *instruction) *instruction { // First we decrement the stack pointer to point the arg0 slot. var sizeOfArgRetReg regalloc.VReg - s := m.currentABI.alignedArgResultStackSlotSize() + s := m.currentABI.AlignedArgResultStackSlotSize() if s > 0 { cur = m.lowerConstantI64AndInsert(cur, tmpRegVReg, s) sizeOfArgRetReg = tmpRegVReg @@ -304,7 +304,7 @@ func (m *machine) setupEpilogueAfter(cur *instruction) { addressModePreOrPostIndex(spVReg, 16 /* stack pointer must be 16-byte aligned. */, false /* increment after loads */), 64) cur = linkInstr(cur, ldr) - if s := m.currentABI.alignedArgResultStackSlotSize(); s > 0 { + if s := m.currentABI.AlignedArgResultStackSlotSize(); s > 0 { cur = m.addsAddOrSubStackPointer(cur, spVReg, s, true) } diff --git a/internal/engine/wazevo/backend/isa/arm64/machine_pro_epi_logue_test.go b/internal/engine/wazevo/backend/isa/arm64/machine_pro_epi_logue_test.go index 70b9447124..26c6ab1cfa 100644 --- a/internal/engine/wazevo/backend/isa/arm64/machine_pro_epi_logue_test.go +++ b/internal/engine/wazevo/backend/isa/arm64/machine_pro_epi_logue_test.go @@ -12,7 +12,7 @@ func TestMachine_SetupPrologue(t *testing.T) { spillSlotSize int64 clobberedRegs []regalloc.VReg exp string - abi abiImpl + abi functionABI }{ { spillSlotSize: 0, @@ -24,7 +24,7 @@ func TestMachine_SetupPrologue(t *testing.T) { }, { spillSlotSize: 0, - abi: abiImpl{argStackSize: 16, retStackSize: 16}, + abi: functionABI{ArgStackSize: 16, RetStackSize: 16}, exp: ` orr x27, xzr, #0x20 sub sp, sp, x27 @@ -74,7 +74,7 @@ func TestMachine_SetupPrologue(t *testing.T) { }, { spillSlotSize: 320, - abi: abiImpl{argStackSize: 320, retStackSize: 160}, + abi: functionABI{ArgStackSize: 320, RetStackSize: 160}, clobberedRegs: []regalloc.VReg{v18VReg, v19VReg, x18VReg, x25VReg}, exp: ` orr x27, xzr, #0x1e0 @@ -117,7 +117,7 @@ func TestMachine_SetupPrologue(t *testing.T) { func TestMachine_SetupEpilogue(t *testing.T) { for _, tc := range []struct { exp string - abi abiImpl + abi functionABI clobberedRegs []regalloc.VReg spillSlotSize int64 }{ @@ -148,7 +148,7 @@ func TestMachine_SetupEpilogue(t *testing.T) { add sp, sp, #0x20 ret `, - abi: abiImpl{argStackSize: 16, retStackSize: 16}, + abi: functionABI{ArgStackSize: 16, RetStackSize: 16}, spillSlotSize: 16 * 5, clobberedRegs: nil, }, @@ -201,7 +201,7 @@ func TestMachine_SetupEpilogue(t *testing.T) { ret `, spillSlotSize: 16 * 10, - abi: abiImpl{argStackSize: 16, retStackSize: 320}, + abi: functionABI{ArgStackSize: 16, RetStackSize: 320}, clobberedRegs: []regalloc.VReg{v18VReg, v27VReg, x18VReg, x25VReg}, }, } { diff --git a/internal/engine/wazevo/backend/isa/arm64/machine_test.go b/internal/engine/wazevo/backend/isa/arm64/machine_test.go index 959cc62512..8d3699bd25 100644 --- a/internal/engine/wazevo/backend/isa/arm64/machine_test.go +++ b/internal/engine/wazevo/backend/isa/arm64/machine_test.go @@ -79,7 +79,7 @@ func TestMachine_arg0OffsetFromSP(t *testing.T) { func TestMachine_ret0OffsetFromSP(t *testing.T) { m := &machine{ clobberedRegs: make([]regalloc.VReg, 10), spillSlotSize: 16 * 8, - currentABI: &abiImpl{argStackSize: 180}, + currentABI: &functionABI{ArgStackSize: 180}, } require.Equal(t, int64(16*18)+32+180, m.ret0OffsetFromSP()) } diff --git a/internal/engine/wazevo/backend/machine.go b/internal/engine/wazevo/backend/machine.go index 76c58a6df2..4cdea55825 100644 --- a/internal/engine/wazevo/backend/machine.go +++ b/internal/engine/wazevo/backend/machine.go @@ -23,9 +23,6 @@ type ( // InitializeABI initializes the FunctionABI for the given signature. InitializeABI(sig *ssa.Signature) - // ABI returns the FunctionABI used for the currently compiled function. - ABI() FunctionABI - // SetCompiler sets the compilation context used for the lifetime of Machine. // This is only called once per Machine, i.e. before the first compilation. SetCompiler(Compiler) @@ -94,5 +91,11 @@ type ( // CompileEntryPreamble returns the sequence of instructions shared by multiple functions to // enter the function from Go. CompileEntryPreamble(signature *ssa.Signature) []byte + + // LowerParams lowers the given parameters. + LowerParams(params []ssa.Value) + + // LowerReturns lowers the given returns. + LowerReturns(returns []ssa.Value) } ) diff --git a/internal/engine/wazevo/backend/machine_test.go b/internal/engine/wazevo/backend/machine_test.go index 1b999f8987..666c1c7b62 100644 --- a/internal/engine/wazevo/backend/machine_test.go +++ b/internal/engine/wazevo/backend/machine_test.go @@ -10,7 +10,6 @@ import ( // mockMachine implements Machine for testing. type mockMachine struct { - abi mockABI startLoweringFunction func(id ssa.BasicBlockID) startBlock func(block ssa.BasicBlock) lowerSingleBranch func(b *ssa.Instruction) @@ -26,6 +25,10 @@ type mockMachine struct { rinfo *regalloc.RegisterInfo } +func (m mockMachine) LowerParams(params []ssa.Value) { panic("implement me") } + +func (m mockMachine) LowerReturns(returns []ssa.Value) { panic("implement me") } + func (m mockMachine) ExecutableContext() ExecutableContext { panic("implement me") } func (m mockMachine) CompileEntryPreamble(signature *ssa.Signature) []byte { @@ -76,9 +79,6 @@ func (m mockMachine) LinkAdjacentBlocks(prev, next ssa.BasicBlock) { m.linkAdjac // InitializeABI implements Machine.InitializeABI. func (m mockMachine) InitializeABI(*ssa.Signature) {} -// ABI implements Machine.ABI. -func (m mockMachine) ABI() FunctionABI { return m.abi } - // SetCompiler implements Machine.SetCompiler. func (m mockMachine) SetCompiler(Compiler) {} @@ -144,18 +144,3 @@ func (m mockMachine) Format() string { func (m mockMachine) DisableStackCheck() {} var _ Machine = (*mockMachine)(nil) - -// mockABI implements ABI for testing. -type mockABI struct{} - -func (m mockABI) EmitGoEntryPreamble() {} - -func (m mockABI) CalleeGenFunctionArgsToVRegs(regs []ssa.Value) { - panic("TODO") -} - -func (m mockABI) CalleeGenVRegsToFunctionReturns(regs []ssa.Value) { - panic("TODO") -} - -var _ FunctionABI = (*mockABI)(nil) diff --git a/internal/engine/wazevo/backend/regalloc/reg.go b/internal/engine/wazevo/backend/regalloc/reg.go index e365936b96..46df807e6f 100644 --- a/internal/engine/wazevo/backend/regalloc/reg.go +++ b/internal/engine/wazevo/backend/regalloc/reg.go @@ -121,5 +121,3 @@ func RegTypeOf(p ssa.Type) RegType { panic("invalid type") } } - -const RealRegsNumMax = 128 diff --git a/internal/engine/wazevo/backend/regalloc/regalloc.go b/internal/engine/wazevo/backend/regalloc/regalloc.go index 5b031351ea..d2d212fef0 100644 --- a/internal/engine/wazevo/backend/regalloc/regalloc.go +++ b/internal/engine/wazevo/backend/regalloc/regalloc.go @@ -39,8 +39,8 @@ type ( // AllocatableRegisters is a 2D array of allocatable RealReg, indexed by regTypeNum and regNum. // The order matters: the first element is the most preferred one when allocating. AllocatableRegisters [NumRegType][]RealReg - CalleeSavedRegisters [RealRegsNumMax]bool - CallerSavedRegisters [RealRegsNumMax]bool + CalleeSavedRegisters RegSet + CallerSavedRegisters RegSet RealRegToVReg []VReg // RealRegName returns the name of the given RealReg for debugging. RealRegName func(r RealReg) string @@ -52,7 +52,7 @@ type ( // regInfo is static per ABI/ISA, and is initialized by the machine during Machine.PrepareRegisterAllocator. regInfo *RegisterInfo // allocatableSet is a set of allocatable RealReg derived from regInfo. Static per ABI/ISA. - allocatableSet regSet + allocatableSet RegSet allocatedCalleeSavedRegs []VReg blockLivenessDataPool wazevoapi.Pool[blockLivenessData] blockLivenessData [] /* blockID to */ *blockLivenessData @@ -88,7 +88,7 @@ type ( maxVRegIDEncountered int // allocatedRegSet is a set of RealReg that are allocated during the allocation phase. This is reset per function. - allocatedRegSet regSet + allocatedRegSet RegSet } blockState struct { @@ -156,7 +156,7 @@ func (s *state) reset() { s.vrStates[i].reset() } s.maxVRegIDEncountered = -1 - s.allocatedRegSet = regSet(0) + s.allocatedRegSet = RegSet(0) s.regsInUse.reset() } @@ -243,7 +243,7 @@ func (vs *vrState) recordReload(f Function, blk Block) { } } -func (s *state) findOrSpillAllocatable(a *Allocator, allocatable []RealReg, forbiddenMask regSet) (r RealReg) { +func (s *state) findOrSpillAllocatable(a *Allocator, allocatable []RealReg, forbiddenMask RegSet) (r RealReg) { r = RealRegInvalid var lastUseAt programCounter = math.MinInt32 var spillVReg VReg @@ -276,7 +276,7 @@ func (s *state) findOrSpillAllocatable(a *Allocator, allocatable []RealReg, forb return r } -func (s *state) findAllocatable(allocatable []RealReg, forbiddenMask regSet) RealReg { +func (s *state) findAllocatable(allocatable []RealReg, forbiddenMask RegSet) RealReg { for _, r := range allocatable { if !s.regsInUse.has(r) && !forbiddenMask.has(r) { return r @@ -323,8 +323,8 @@ func (a *Allocator) DoAllocation(f Function) { func (a *Allocator) determineCalleeSavedRealRegs(f Function) { a.allocatedCalleeSavedRegs = a.allocatedCalleeSavedRegs[:0] - a.state.allocatedRegSet.range_(func(allocatedRealReg RealReg) { - if a.regInfo.isCalleeSaved(allocatedRealReg) { + a.state.allocatedRegSet.Range(func(allocatedRealReg RealReg) { + if a.regInfo.CalleeSavedRegisters.has(allocatedRealReg) { a.allocatedCalleeSavedRegs = append(a.allocatedCalleeSavedRegs, a.regInfo.RealRegToVReg[allocatedRealReg]) } }) @@ -548,7 +548,7 @@ func (a *Allocator) allocBlock(f Function, blk Block) { fmt.Println(instr) } - var currentUsedSet regSet + var currentUsedSet RegSet killSet := a.reals[:0] // Gather the set of registers that will be used in the current instruction. @@ -643,7 +643,7 @@ func (a *Allocator) allocBlock(f Function, blk Block) { } if r == RealRegInvalid { typ := def.RegType() - r = s.findOrSpillAllocatable(a, a.regInfo.AllocatableRegisters[typ], regSet(0)) + r = s.findOrSpillAllocatable(a, a.regInfo.AllocatableRegisters[typ], RegSet(0)) } s.useRealReg(r, def) } @@ -690,7 +690,7 @@ func (a *Allocator) releaseCallerSavedRegs(addrReg RealReg) { if v.IsRealReg() { continue // This is the argument register as it's already used by VReg backed by the corresponding RealReg. } - if !a.regInfo.isCallerSaved(allocated) { + if !a.regInfo.CallerSavedRegisters.has(allocated) { // If this is not a caller-saved register, it is safe to keep it across the call. continue } @@ -1010,11 +1010,3 @@ func (i *blockLivenessData) isKilledAt(vs *vrState, pos programCounter) bool { } return false } - -func (r *RegisterInfo) isCalleeSaved(reg RealReg) bool { - return r.CalleeSavedRegisters[reg] -} - -func (r *RegisterInfo) isCallerSaved(reg RealReg) bool { - return r.CallerSavedRegisters[reg] -} diff --git a/internal/engine/wazevo/backend/regalloc/regset.go b/internal/engine/wazevo/backend/regalloc/regset.go index 542c2a75a4..e9bf60661c 100644 --- a/internal/engine/wazevo/backend/regalloc/regset.go +++ b/internal/engine/wazevo/backend/regalloc/regset.go @@ -5,9 +5,19 @@ import ( "strings" ) -type regSet uint64 +// NewRegSet returns a new RegSet with the given registers. +func NewRegSet(regs ...RealReg) RegSet { + var ret RegSet + for _, r := range regs { + ret = ret.add(r) + } + return ret +} + +// RegSet represents a set of registers. +type RegSet uint64 -func (rs regSet) format(info *RegisterInfo) string { //nolint:unused +func (rs RegSet) format(info *RegisterInfo) string { //nolint:unused var ret []string for i := 0; i < 64; i++ { if rs&(1<= 64 { return rs } return rs | 1< Date: Fri, 5 Jan 2024 09:48:52 -0800 Subject: [PATCH 2/2] ok Signed-off-by: Takeshi Yoneda --- internal/engine/wazevo/backend/abi.go | 1 + internal/engine/wazevo/backend/isa/amd64/abi.go | 7 ++----- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/internal/engine/wazevo/backend/abi.go b/internal/engine/wazevo/backend/abi.go index 54a5e8a2cc..67f464a55f 100644 --- a/internal/engine/wazevo/backend/abi.go +++ b/internal/engine/wazevo/backend/abi.go @@ -13,6 +13,7 @@ type FunctionABIRegInfo interface { } type ( + // FunctionABI represents the ABI information for a function which corresponds to a ssa.Signature. FunctionABI[R FunctionABIRegInfo] struct { r R Initialized bool diff --git a/internal/engine/wazevo/backend/isa/amd64/abi.go b/internal/engine/wazevo/backend/isa/amd64/abi.go index 11764086e1..9771fb4fdc 100644 --- a/internal/engine/wazevo/backend/isa/amd64/abi.go +++ b/internal/engine/wazevo/backend/isa/amd64/abi.go @@ -57,11 +57,6 @@ func (f functionABIRegInfo) ArgsResultsRegs() (argInts, argFloats, resultInt, re type abiImpl = backend.FunctionABI[functionABIRegInfo] -// ParamRegs implements backend.Machine. -func (m *machine) ParamRegs() (ints, floats []regalloc.RealReg) { - return intArgResultRegs, floatArgResultRegs -} - func (m *machine) getOrCreateFunctionABI(sig *ssa.Signature) *abiImpl { if int(sig.ID) >= len(m.abis) { m.abis = append(m.abis, make([]abiImpl, int(sig.ID)+1)...) @@ -76,11 +71,13 @@ func (m *machine) getOrCreateFunctionABI(sig *ssa.Signature) *abiImpl { return abi } +// LowerParams implements backend.Machine. func (m *machine) LowerParams(params []ssa.Value) { // TODO implement me panic("implement me") } +// LowerReturns implements backend.Machine. func (m *machine) LowerReturns(returns []ssa.Value) { // TODO implement me panic("implement me")