From 638f86cb4fd2cea0aea3a60103034215e45aef01 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Mon, 31 Dec 2018 16:47:05 -0800 Subject: [PATCH] fix #9253 : enable FFI at CT --- compiler/evalffi.nim | 233 ++++++++++++++++++++++--------------------- compiler/vm.nim | 15 ++- compiler/vmdef.nim | 6 +- compiler/vmgen.nim | 2 +- 4 files changed, 135 insertions(+), 121 deletions(-) diff --git a/compiler/evalffi.nim b/compiler/evalffi.nim index e863c89954bd5..e25bbd500a05b 100644 --- a/compiler/evalffi.nim +++ b/compiler/evalffi.nim @@ -10,41 +10,45 @@ ## This file implements the FFI part of the evaluator for Nim code. import ast, astalgo, ropes, types, options, tables, dynlib, libffi, msgs, os +import compiler/lineinfos when defined(windows): const libcDll = "msvcrt.dll" -else: +elif defined(linux): const libcDll = "libc.so(.6|.5|)" +elif defined(osx): + const libcDll = "/usr/lib/libSystem.dylib" +else: + static: doAssert false type - TDllCache = tables.TTable[string, TLibHandle] + TDllCache = tables.Table[string, LibHandle] var - gDllCache = initTable[string, TLibHandle]() + gDllCache = initTable[string, LibHandle]() when defined(windows): var gExeHandle = loadLib(os.getAppFilename()) else: var gExeHandle = loadLib() -proc getDll(cache: var TDllCache; dll: string; info: TLineInfo): pointer = - result = cache[dll] +proc getDll(conf: ConfigRef, cache: var TDllCache; dll: string; info: TLineInfo): pointer = + if dll in cache: return cache[dll] + var libs: seq[string] + libCandidates(dll, libs) + for c in libs: + result = loadLib(c) + if not result.isNil: break if result.isNil: - var libs: seq[string] = @[] - libCandidates(dll, libs) - for c in libs: - result = loadLib(c) - if not result.isNil: break - if result.isNil: - globalError(info, "cannot load: " & dll) - cache[dll] = result + globalError(conf, info, "cannot load: " & dll) + cache[dll] = result const nkPtrLit = nkIntLit # hopefully we can get rid of this hack soon var myerrno {.importc: "errno", header: "".}: cint ## error variable -proc importcSymbol*(sym: PSym): PNode = - let name = ropeToStr(sym.loc.r) +proc importcSymbol*(conf: ConfigRef, sym: PSym): PNode = + let name = $sym.loc.r # the AST does not support untyped pointers directly, so we use an nkIntLit # that contains the address instead: @@ -57,28 +61,28 @@ proc importcSymbol*(sym: PSym): PNode = else: let lib = sym.annex if lib != nil and lib.path.kind notin {nkStrLit..nkTripleStrLit}: - globalError(sym.info, "dynlib needs to be a string lit for the REPL") + globalError(conf, sym.info, "dynlib needs to be a string lit for the REPL") var theAddr: pointer if lib.isNil and not gExehandle.isNil: # first try this exe itself: theAddr = gExehandle.symAddr(name) # then try libc: if theAddr.isNil: - let dllhandle = gDllCache.getDll(libcDll, sym.info) + let dllhandle = getDll(conf, gDllCache, libcDll, sym.info) theAddr = dllhandle.symAddr(name) elif not lib.isNil: - let dllhandle = gDllCache.getDll(if lib.kind == libHeader: libcDll + let dllhandle = getDll(conf, gDllCache, if lib.kind == libHeader: libcDll else: lib.path.strVal, sym.info) theAddr = dllhandle.symAddr(name) - if theAddr.isNil: globalError(sym.info, "cannot import: " & sym.name.s) + if theAddr.isNil: globalError(conf, sym.info, "cannot import: " & sym.name.s) result.intVal = cast[ByteAddress](theAddr) -proc mapType(t: ast.PType): ptr libffi.TType = +proc mapType(conf: ConfigRef, t: ast.PType): ptr libffi.TType = if t == nil: return addr libffi.type_void case t.kind of tyBool, tyEnum, tyChar, tyInt..tyInt64, tyUInt..tyUInt64, tySet: - case t.getSize + case getSize(conf, t) of 1: result = addr libffi.type_uint8 of 2: result = addr libffi.type_sint16 of 4: result = addr libffi.type_sint32 @@ -90,87 +94,87 @@ proc mapType(t: ast.PType): ptr libffi.TType = tyStmt, tyTypeDesc, tyProc, tyArray, tyStatic, tyNil: result = addr libffi.type_pointer of tyDistinct, tyAlias, tySink: - result = mapType(t.sons[0]) + result = mapType(conf, t.sons[0]) else: result = nil # too risky: #of tyFloat128: result = addr libffi.type_longdouble -proc mapCallConv(cc: TCallingConvention, info: TLineInfo): TABI = +proc mapCallConv(conf: ConfigRef, cc: TCallingConvention, info: TLineInfo): TABI = case cc of ccDefault: result = DEFAULT_ABI of ccStdCall: result = when defined(windows): STDCALL else: DEFAULT_ABI of ccCDecl: result = DEFAULT_ABI else: - globalError(info, "cannot map calling convention to FFI") + globalError(conf, info, "cannot map calling convention to FFI") template rd(T, p: untyped): untyped = (cast[ptr T](p))[] template wr(T, p, v: untyped): untyped = (cast[ptr T](p))[] = v template `+!`(x, y: untyped): untyped = cast[pointer](cast[ByteAddress](x) + y) -proc packSize(v: PNode, typ: PType): int = +proc packSize(conf: ConfigRef, v: PNode, typ: PType): int = ## computes the size of the blob case typ.kind of tyPtr, tyRef, tyVar, tyLent: if v.kind in {nkNilLit, nkPtrLit}: result = sizeof(pointer) else: - result = sizeof(pointer) + packSize(v.sons[0], typ.lastSon) + result = sizeof(pointer) + packSize(conf, v.sons[0], typ.lastSon) of tyDistinct, tyGenericInst, tyAlias, tySink: - result = packSize(v, typ.sons[0]) + result = packSize(conf, v, typ.sons[0]) of tyArray: # consider: ptr array[0..1000_000, int] which is common for interfacing; # we use the real length here instead if v.kind in {nkNilLit, nkPtrLit}: result = sizeof(pointer) elif v.len != 0: - result = v.len * packSize(v.sons[0], typ.sons[1]) + result = v.len * packSize(conf, v.sons[0], typ.sons[1]) else: - result = typ.getSize.int + result = getSize(conf, typ).int -proc pack(v: PNode, typ: PType, res: pointer) +proc pack(conf: ConfigRef, v: PNode, typ: PType, res: pointer) -proc getField(n: PNode; position: int): PSym = +proc getField(conf: ConfigRef, n: PNode; position: int): PSym = case n.kind of nkRecList: for i in countup(0, sonsLen(n) - 1): - result = getField(n.sons[i], position) + result = getField(conf, n.sons[i], position) if result != nil: return of nkRecCase: - result = getField(n.sons[0], position) + result = getField(conf, n.sons[0], position) if result != nil: return for i in countup(1, sonsLen(n) - 1): case n.sons[i].kind of nkOfBranch, nkElse: - result = getField(lastSon(n.sons[i]), position) + result = getField(conf, lastSon(n.sons[i]), position) if result != nil: return - else: internalError(n.info, "getField(record case branch)") + else: internalError(conf, n.info, "getField(record case branch)") of nkSym: if n.sym.position == position: result = n.sym else: discard -proc packObject(x: PNode, typ: PType, res: pointer) = - internalAssert x.kind in {nkObjConstr, nkPar, nkTupleConstr} +proc packObject(conf: ConfigRef, x: PNode, typ: PType, res: pointer) = + internalAssert conf, x.kind in {nkObjConstr, nkPar, nkTupleConstr} # compute the field's offsets: - discard typ.getSize + discard getSize(conf, typ) for i in countup(ord(x.kind == nkObjConstr), sonsLen(x) - 1): var it = x.sons[i] if it.kind == nkExprColonExpr: - internalAssert it.sons[0].kind == nkSym + internalAssert conf, it.sons[0].kind == nkSym let field = it.sons[0].sym - pack(it.sons[1], field.typ, res +! field.offset) + pack(conf, it.sons[1], field.typ, res +! field.offset) elif typ.n != nil: - let field = getField(typ.n, i) - pack(it, field.typ, res +! field.offset) + let field = getField(conf, typ.n, i) + pack(conf, it, field.typ, res +! field.offset) else: # XXX: todo - globalError(x.info, "cannot pack unnamed tuple") + globalError(conf, x.info, "cannot pack unnamed tuple") const maxPackDepth = 20 var packRecCheck = 0 -proc pack(v: PNode, typ: PType, res: pointer) = +proc pack(conf: ConfigRef, v: PNode, typ: PType, res: pointer) = template awr(T, v: untyped): untyped = wr(T, res, v) @@ -188,13 +192,13 @@ proc pack(v: PNode, typ: PType, res: pointer) = of tyUInt32: awr(uint32, v.intVal.uint32) of tyUInt64: awr(uint64, v.intVal.uint64) of tyEnum, tySet: - case v.typ.getSize + case getSize(conf, v.typ) of 1: awr(uint8, v.intVal.uint8) of 2: awr(uint16, v.intVal.uint16) of 4: awr(int32, v.intVal.int32) of 8: awr(int64, v.intVal.int64) else: - globalError(v.info, "cannot map value to FFI (tyEnum, tySet)") + globalError(conf, v.info, "cannot map value to FFI (tyEnum, tySet)") of tyFloat: awr(float, v.floatVal) of tyFloat32: awr(float32, v.floatVal) of tyFloat64: awr(float64, v.floatVal) @@ -208,7 +212,7 @@ proc pack(v: PNode, typ: PType, res: pointer) = elif v.kind in {nkStrLit..nkTripleStrLit}: awr(cstring, cstring(v.strVal)) else: - globalError(v.info, "cannot map pointer/proc value to FFI") + globalError(conf, v.info, "cannot map pointer/proc value to FFI") of tyPtr, tyRef, tyVar, tyLent: if v.kind == nkNilLit: # nothing to do since the memory is 0 initialized anyway @@ -218,44 +222,44 @@ proc pack(v: PNode, typ: PType, res: pointer) = else: if packRecCheck > maxPackDepth: packRecCheck = 0 - globalError(v.info, "cannot map value to FFI " & typeToString(v.typ)) + globalError(conf, v.info, "cannot map value to FFI " & typeToString(v.typ)) inc packRecCheck - pack(v.sons[0], typ.lastSon, res +! sizeof(pointer)) + pack(conf, v.sons[0], typ.lastSon, res +! sizeof(pointer)) dec packRecCheck awr(pointer, res +! sizeof(pointer)) of tyArray: - let baseSize = typ.sons[1].getSize + let baseSize = getSize(conf, typ.sons[1]) for i in 0 ..< v.len: - pack(v.sons[i], typ.sons[1], res +! i * baseSize) + pack(conf, v.sons[i], typ.sons[1], res +! i * baseSize) of tyObject, tyTuple: - packObject(v, typ, res) + packObject(conf, v, typ, res) of tyNil: discard of tyDistinct, tyGenericInst, tyAlias, tySink: - pack(v, typ.sons[0], res) + pack(conf, v, typ.sons[0], res) else: - globalError(v.info, "cannot map value to FFI " & typeToString(v.typ)) + globalError(conf, v.info, "cannot map value to FFI " & typeToString(v.typ)) -proc unpack(x: pointer, typ: PType, n: PNode): PNode +proc unpack(conf: ConfigRef, x: pointer, typ: PType, n: PNode): PNode -proc unpackObjectAdd(x: pointer, n, result: PNode) = +proc unpackObjectAdd(conf: ConfigRef, x: pointer, n, result: PNode) = case n.kind of nkRecList: for i in countup(0, sonsLen(n) - 1): - unpackObjectAdd(x, n.sons[i], result) + unpackObjectAdd(conf, x, n.sons[i], result) of nkRecCase: - globalError(result.info, "case objects cannot be unpacked") + globalError(conf, result.info, "case objects cannot be unpacked") of nkSym: var pair = newNodeI(nkExprColonExpr, result.info, 2) pair.sons[0] = n - pair.sons[1] = unpack(x +! n.sym.offset, n.sym.typ, nil) + pair.sons[1] = unpack(conf, x +! n.sym.offset, n.sym.typ, nil) #echo "offset: ", n.sym.name.s, " ", n.sym.offset result.add pair else: discard -proc unpackObject(x: pointer, typ: PType, n: PNode): PNode = +proc unpackObject(conf: ConfigRef, x: pointer, typ: PType, n: PNode): PNode = # compute the field's offsets: - discard typ.getSize + discard getSize(conf, typ) # iterate over any actual field of 'n' ... if n is nil we need to create # the nkPar node: @@ -263,36 +267,36 @@ proc unpackObject(x: pointer, typ: PType, n: PNode): PNode = result = newNode(nkTupleConstr) result.typ = typ if typ.n.isNil: - internalError("cannot unpack unnamed tuple") - unpackObjectAdd(x, typ.n, result) + internalError(conf, "cannot unpack unnamed tuple") + unpackObjectAdd(conf, x, typ.n, result) else: result = n if result.kind notin {nkObjConstr, nkPar, nkTupleConstr}: - globalError(n.info, "cannot map value from FFI") + globalError(conf, n.info, "cannot map value from FFI") if typ.n.isNil: - globalError(n.info, "cannot unpack unnamed tuple") + globalError(conf, n.info, "cannot unpack unnamed tuple") for i in countup(ord(n.kind == nkObjConstr), sonsLen(n) - 1): var it = n.sons[i] if it.kind == nkExprColonExpr: - internalAssert it.sons[0].kind == nkSym + internalAssert conf, it.sons[0].kind == nkSym let field = it.sons[0].sym - it.sons[1] = unpack(x +! field.offset, field.typ, it.sons[1]) + it.sons[1] = unpack(conf, x +! field.offset, field.typ, it.sons[1]) else: - let field = getField(typ.n, i) - n.sons[i] = unpack(x +! field.offset, field.typ, it) + let field = getField(conf, typ.n, i) + n.sons[i] = unpack(conf, x +! field.offset, field.typ, it) -proc unpackArray(x: pointer, typ: PType, n: PNode): PNode = +proc unpackArray(conf: ConfigRef, x: pointer, typ: PType, n: PNode): PNode = if n.isNil: result = newNode(nkBracket) result.typ = typ - newSeq(result.sons, lengthOrd(typ).int) + newSeq(result.sons, lengthOrd(conf, typ).int) else: result = n if result.kind != nkBracket: - globalError(n.info, "cannot map value from FFI") - let baseSize = typ.sons[1].getSize + globalError(conf, n.info, "cannot map value from FFI") + let baseSize = getSize(conf, typ.sons[1]) for i in 0 ..< result.len: - result.sons[i] = unpack(x +! i * baseSize, typ.sons[1], result.sons[i]) + result.sons[i] = unpack(conf, x +! i * baseSize, typ.sons[1], result.sons[i]) proc canonNodeKind(k: TNodeKind): TNodeKind = case k @@ -301,7 +305,7 @@ proc canonNodeKind(k: TNodeKind): TNodeKind = of nkStrLit..nkTripleStrLit: result = nkStrLit else: result = k -proc unpack(x: pointer, typ: PType, n: PNode): PNode = +proc unpack(conf: ConfigRef, x: pointer, typ: PType, n: PNode): PNode = template aw(k, v, field: untyped): untyped = if n.isNil: result = newNode(k) @@ -313,7 +317,7 @@ proc unpack(x: pointer, typ: PType, n: PNode): PNode = #echo "expected ", k, " but got ", result.kind #debug result return newNodeI(nkExceptBranch, n.info) - #globalError(n.info, "cannot map value from FFI") + #globalError(conf, n.info, "cannot map value from FFI") result.field = v template setNil() = @@ -344,13 +348,13 @@ proc unpack(x: pointer, typ: PType, n: PNode): PNode = of tyUInt32: awi(nkUInt32Lit, rd(uint32, x).BiggestInt) of tyUInt64: awi(nkUInt64Lit, rd(uint64, x).BiggestInt) of tyEnum: - case typ.getSize + case getSize(conf, typ) of 1: awi(nkIntLit, rd(uint8, x).BiggestInt) of 2: awi(nkIntLit, rd(uint16, x).BiggestInt) of 4: awi(nkIntLit, rd(int32, x).BiggestInt) of 8: awi(nkIntLit, rd(int64, x).BiggestInt) else: - globalError(n.info, "cannot map value from FFI (tyEnum, tySet)") + globalError(conf, n.info, "cannot map value from FFI (tyEnum, tySet)") of tyFloat: awf(nkFloatLit, rd(float, x)) of tyFloat32: awf(nkFloat32Lit, rd(float32, x)) of tyFloat64: awf(nkFloat64Lit, rd(float64, x)) @@ -371,15 +375,15 @@ proc unpack(x: pointer, typ: PType, n: PNode): PNode = elif n == nil or n.kind == nkPtrLit: awi(nkPtrLit, cast[ByteAddress](p)) elif n != nil and n.len == 1: - internalAssert n.kind == nkRefTy - n.sons[0] = unpack(p, typ.lastSon, n.sons[0]) + internalAssert(conf, n.kind == nkRefTy) + n.sons[0] = unpack(conf, p, typ.lastSon, n.sons[0]) result = n else: - globalError(n.info, "cannot map value from FFI " & typeToString(typ)) + globalError(conf, n.info, "cannot map value from FFI " & typeToString(typ)) of tyObject, tyTuple: - result = unpackObject(x, typ, n) + result = unpackObject(conf, x, typ, n) of tyArray: - result = unpackArray(x, typ, n) + result = unpackArray(conf, x, typ, n) of tyCString, tyString: let p = rd(cstring, x) if p.isNil: @@ -389,12 +393,12 @@ proc unpack(x: pointer, typ: PType, n: PNode): PNode = of tyNil: setNil() of tyDistinct, tyGenericInst, tyAlias, tySink: - result = unpack(x, typ.lastSon, n) + result = unpack(conf, x, typ.lastSon, n) else: # XXX what to do with 'array' here? - globalError(n.info, "cannot map value from FFI " & typeToString(typ)) + globalError(conf, n.info, "cannot map value from FFI " & typeToString(typ)) -proc fficast*(x: PNode, destTyp: PType): PNode = +proc fficast*(conf: ConfigRef, x: PNode, destTyp: PType): PNode = if x.kind == nkPtrLit and x.typ.kind in {tyPtr, tyRef, tyVar, tyLent, tyPointer, tyProc, tyCString, tyString, tySequence}: @@ -404,93 +408,94 @@ proc fficast*(x: PNode, destTyp: PType): PNode = result = newNodeIT(x.kind, x.info, destTyp) else: # we play safe here and allocate the max possible size: - let size = max(packSize(x, x.typ), packSize(x, destTyp)) + let size = max(packSize(conf, x, x.typ), packSize(conf, x, destTyp)) var a = alloc0(size) - pack(x, x.typ, a) + pack(conf, x, x.typ, a) # cast through a pointer needs a new inner object: let y = if x.kind == nkRefTy: newNodeI(nkRefTy, x.info, 1) else: x.copyTree y.typ = x.typ - result = unpack(a, destTyp, y) + result = unpack(conf, a, destTyp, y) dealloc a -proc callForeignFunction*(call: PNode): PNode = - internalAssert call.sons[0].kind == nkPtrLit +proc callForeignFunction*(conf: ConfigRef, call: PNode): PNode = + internalAssert conf, call.sons[0].kind == nkPtrLit var cif: TCif var sig: TParamList # use the arguments' types for varargs support: for i in 1..call.len-1: - sig[i-1] = mapType(call.sons[i].typ) + sig[i-1] = mapType(conf, call.sons[i].typ) if sig[i-1].isNil: - globalError(call.info, "cannot map FFI type") + globalError(conf, call.info, "cannot map FFI type") let typ = call.sons[0].typ - if prep_cif(cif, mapCallConv(typ.callConv, call.info), cuint(call.len-1), - mapType(typ.sons[0]), sig) != OK: - globalError(call.info, "error in FFI call") + if prep_cif(cif, mapCallConv(conf, typ.callConv, call.info), cuint(call.len-1), + mapType(conf, typ.sons[0]), sig) != OK: + globalError(conf, call.info, "error in FFI call") var args: TArgList let fn = cast[pointer](call.sons[0].intVal) for i in 1 .. call.len-1: var t = call.sons[i].typ - args[i-1] = alloc0(packSize(call.sons[i], t)) - pack(call.sons[i], t, args[i-1]) + args[i-1] = alloc0(packSize(conf, call.sons[i], t)) + pack(conf, call.sons[i], t, args[i-1]) let retVal = if isEmptyType(typ.sons[0]): pointer(nil) - else: alloc(typ.sons[0].getSize.int) + else: alloc(getSize(conf, typ.sons[0]).int) libffi.call(cif, fn, retVal, args) if retVal.isNil: result = newNode(nkEmpty) else: - result = unpack(retVal, typ.sons[0], nil) + result = unpack(conf, retVal, typ.sons[0], nil) result.info = call.info if retVal != nil: dealloc retVal for i in 1 .. call.len-1: - call.sons[i] = unpack(args[i-1], typ.sons[i], call[i]) + call.sons[i] = unpack(conf, args[i-1], typ.sons[i], call[i]) dealloc args[i-1] -proc callForeignFunction*(fn: PNode, fntyp: PType, +proc callForeignFunction*(conf: ConfigRef, fn: PNode, fntyp: PType, args: var TNodeSeq, start, len: int, info: TLineInfo): PNode = - internalAssert fn.kind == nkPtrLit + internalAssert conf, fn.kind == nkPtrLit var cif: TCif var sig: TParamList for i in 0..len-1: var aTyp = args[i+start].typ if aTyp.isNil: - internalAssert i+1 < fntyp.len + internalAssert conf, i+1 < fntyp.len aTyp = fntyp.sons[i+1] args[i+start].typ = aTyp - sig[i] = mapType(aTyp) - if sig[i].isNil: globalError(info, "cannot map FFI type") + sig[i] = mapType(conf, aTyp) + if sig[i].isNil: globalError(conf, info, "cannot map FFI type") - if prep_cif(cif, mapCallConv(fntyp.callConv, info), cuint(len), - mapType(fntyp.sons[0]), sig) != OK: - globalError(info, "error in FFI call") + if prep_cif(cif, mapCallConv(conf, fntyp.callConv, info), cuint(len), + mapType(conf, fntyp.sons[0]), sig) != OK: + globalError(conf, info, "error in FFI call") var cargs: TArgList let fn = cast[pointer](fn.intVal) for i in 0 .. len-1: let t = args[i+start].typ - cargs[i] = alloc0(packSize(args[i+start], t)) - pack(args[i+start], t, cargs[i]) + cargs[i] = alloc0(packSize(conf, args[i+start], t)) + pack(conf, args[i+start], t, cargs[i]) let retVal = if isEmptyType(fntyp.sons[0]): pointer(nil) - else: alloc(fntyp.sons[0].getSize.int) + else: alloc(getSize(conf, fntyp.sons[0]).int) libffi.call(cif, fn, retVal, cargs) if retVal.isNil: result = newNode(nkEmpty) else: - result = unpack(retVal, fntyp.sons[0], nil) + result = unpack(conf, retVal, fntyp.sons[0], nil) result.info = info if retVal != nil: dealloc retVal for i in 0 .. len-1: let t = args[i+start].typ - args[i+start] = unpack(cargs[i], t, args[i+start]) + args[i+start] = unpack(conf, cargs[i], t, args[i+start]) dealloc cargs[i] + diff --git a/compiler/vm.nim b/compiler/vm.nim index c8784c3e7dcd2..7e0cd524b9089 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -1020,7 +1020,11 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = let prcValue = c.globals.sons[prc.position-1] if prcValue.kind == nkEmpty: globalError(c.config, c.debug[pc], "cannot run " & prc.name.s) - let newValue = callForeignFunction(prcValue, prc.typ, tos.slots, + var slots2: TNodeSeq + slots2.setLen(tos.slots.len) + for i in 0..