Skip to content

Commit

Permalink
Add Zephyr Support (#19003)
Browse files Browse the repository at this point in the history 
* Porting Nim to run on Zephyr.

Includes changes to `std/net`.

Squashed commit of the following:
    tweaking more memory / malloc things
    revert back bitmasks
    tweaking nim to use kernel heap as C malloc doesn't work
    fixing socket polling on zephyr
    cleanup getting maximum sockets for process or for rtos'es
    reorganizing and fixing net for async / system
    merge netlite changes back into nativesockets
    merge netlite changes back into nativesockets
    reverting native sockets back
    tweaking nim / zephyr network
    adding option to run 'net-lite' from linux
    bridging zephyr's max connections
    fixing net errors
    fixing compilation with getAddrString
    fixing compilation with getAddrString
    experimenting with a nativesockets_lite ... getAddrString
    experimenting with a nativesockets_lite ... getAddrString
    experimenting with a nativesockets_lite ... getLocalAddr
    experimenting with a nativesockets_lite ... getLocalAddr
    experimenting with a nativesockets_lite ...
    add note regarding incorrect FreeRTOS Sockadd_in fields
    changing to NIM_STATIC_ASSERT
    cleaning up the static_assert error messages
    cleaning up the static_assert error messages
    setting up static assert ftw!
    testing compile time asserts
    reworking Sockaddr objects to more closely match various platforms
    reworking Sockaddr objects to more closely match various platforms
    reworking Sockaddr objects to more closely match various platforms
    finding missing items (issue  #18684)
    fixup posix constants (issue  #18684)
    adding plumbing for zephyr os (issue  #18684)
    adding plumbing for zephyr os (issue  #18684)

* fixing constant capitalizations

* remove extra debug prints and fix TSa_Family/cint issue

* remove extra debug prints and fix TSa_Family/cint issue

* Porting Nim to run on Zephyr.

Includes changes to `std/net`.

Squashed commit of the following:
    tweaking more memory / malloc things
    revert back bitmasks
    tweaking nim to use kernel heap as C malloc doesn't work
    fixing socket polling on zephyr
    cleanup getting maximum sockets for process or for rtos'es
    reorganizing and fixing net for async / system
    merge netlite changes back into nativesockets
    merge netlite changes back into nativesockets
    reverting native sockets back
    tweaking nim / zephyr network
    adding option to run 'net-lite' from linux
    bridging zephyr's max connections
    fixing net errors
    fixing compilation with getAddrString
    fixing compilation with getAddrString
    experimenting with a nativesockets_lite ... getAddrString
    experimenting with a nativesockets_lite ... getAddrString
    experimenting with a nativesockets_lite ... getLocalAddr
    experimenting with a nativesockets_lite ... getLocalAddr
    experimenting with a nativesockets_lite ...
    add note regarding incorrect FreeRTOS Sockadd_in fields
    changing to NIM_STATIC_ASSERT
    cleaning up the static_assert error messages
    cleaning up the static_assert error messages
    setting up static assert ftw!
    testing compile time asserts
    reworking Sockaddr objects to more closely match various platforms
    reworking Sockaddr objects to more closely match various platforms
    reworking Sockaddr objects to more closely match various platforms
    finding missing items (issue  #18684)
    fixup posix constants (issue  #18684)
    adding plumbing for zephyr os (issue  #18684)
    adding plumbing for zephyr os (issue  #18684)

* fixing constant capitalizations

* remove extra debug prints and fix TSa_Family/cint issue

* remove extra debug prints and fix TSa_Family/cint issue

* fixing PR issues

* Porting Nim to run on Zephyr.

Includes changes to `std/net`.

Squashed commit of the following:
    tweaking more memory / malloc things
    revert back bitmasks
    tweaking nim to use kernel heap as C malloc doesn't work
    fixing socket polling on zephyr
    cleanup getting maximum sockets for process or for rtos'es
    reorganizing and fixing net for async / system
    merge netlite changes back into nativesockets
    merge netlite changes back into nativesockets
    reverting native sockets back
    tweaking nim / zephyr network
    adding option to run 'net-lite' from linux
    bridging zephyr's max connections
    fixing net errors
    fixing compilation with getAddrString
    fixing compilation with getAddrString
    experimenting with a nativesockets_lite ... getAddrString
    experimenting with a nativesockets_lite ... getAddrString
    experimenting with a nativesockets_lite ... getLocalAddr
    experimenting with a nativesockets_lite ... getLocalAddr
    experimenting with a nativesockets_lite ...
    add note regarding incorrect FreeRTOS Sockadd_in fields
    changing to NIM_STATIC_ASSERT
    cleaning up the static_assert error messages
    cleaning up the static_assert error messages
    setting up static assert ftw!
    testing compile time asserts
    reworking Sockaddr objects to more closely match various platforms
    reworking Sockaddr objects to more closely match various platforms
    reworking Sockaddr objects to more closely match various platforms
    finding missing items (issue  #18684)
    fixup posix constants (issue  #18684)
    adding plumbing for zephyr os (issue  #18684)
    adding plumbing for zephyr os (issue  #18684)

* fixing constant capitalizations

* remove extra debug prints and fix TSa_Family/cint issue

* remove extra debug prints and fix TSa_Family/cint issue

* Remerge

* fixing constant capitalizations

* remove extra debug prints and fix TSa_Family/cint issue

* remove extra debug prints and fix TSa_Family/cint issue

* fixing PR issues

* fix maxDescriptors on zephyr/freertos

* move maxDescriptors to selector.nim -- fixes compile issue

* change realloc impl on zephyr to match ansi c behavior

* change realloc impl on zephyr to match ansi c behavior

* force compileOnly mode for tlwip

Co-authored-by: Jaremy J. Creechley <[email protected]>
Co-authored-by: Jaremy Creechley <[email protected]>
(cherry picked from commit 141b76e)
  • Loading branch information
@elcritch @narimiran
elcritch authored and narimiran committed Mar 24, 2022
1 parent b741f3c commit 1dc4769
Show file tree
Hide file tree
Showing 20 changed files with 517 additions and 341 deletions.
4 changes: 3 additions & 1 deletion compiler/options.nim
View file
Original file line number Diff line number Diff line change
Expand Up @@ -597,7 +597,7 @@ proc isDefined*(conf: ConfigRef; symbol: string): bool =
osQnx, osAtari, osAix,
osHaiku, osVxWorks, osSolaris, osNetbsd,
osFreebsd, osOpenbsd, osDragonfly, osMacosx, osIos,
osAndroid, osNintendoSwitch, osFreeRTOS, osCrossos}
osAndroid, osNintendoSwitch, osFreeRTOS, osCrossos, osZephyr}
of "linux":
result = conf.target.targetOS in {osLinux, osAndroid}
of "bsd":
Expand All @@ -617,6 +617,8 @@ proc isDefined*(conf: ConfigRef; symbol: string): bool =
result = conf.target.targetOS == osNintendoSwitch
of "freertos", "lwip":
result = conf.target.targetOS == osFreeRTOS
of "zephyr":
result = conf.target.targetOS == osZephyr
of "littleendian": result = CPU[conf.target.targetCPU].endian == littleEndian
of "bigendian": result = CPU[conf.target.targetCPU].endian == bigEndian
of "cpu8": result = CPU[conf.target.targetCPU].bit == 8
Expand Down
6 changes: 5 additions & 1 deletion compiler/platform.nim
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ type
osNone, osDos, osWindows, osOs2, osLinux, osMorphos, osSkyos, osSolaris,
osIrix, osNetbsd, osFreebsd, osOpenbsd, osDragonfly, osCrossos, osAix, osPalmos, osQnx,
osAmiga, osAtari, osNetware, osMacos, osMacosx, osIos, osHaiku, osAndroid, osVxWorks
osGenode, osJS, osNimVM, osStandalone, osNintendoSwitch, osFreeRTOS, osAny
osGenode, osJS, osNimVM, osStandalone, osNintendoSwitch, osFreeRTOS, osZephyr, osAny

type
TInfoOSProp* = enum
Expand Down Expand Up @@ -185,6 +185,10 @@ const
objExt: ".o", newLine: "\x0A", pathSep: ":", dirSep: "/",
scriptExt: ".sh", curDir: ".", exeExt: "", extSep: ".",
props: {ospPosix}),
(name: "Zephyr", parDir: "..", dllFrmt: "lib$1.so", altDirSep: "/",
objExt: ".o", newLine: "\x0A", pathSep: ":", dirSep: "/",
scriptExt: ".sh", curDir: ".", exeExt: "", extSep: ".",
props: {ospPosix}),
(name: "Any", parDir: "..", dllFrmt: "lib$1.so", altDirSep: "/",
objExt: ".o", newLine: "\x0A", pathSep: ":", dirSep: "/",
scriptExt: ".sh", curDir: ".", exeExt: "", extSep: ".",
Expand Down
6 changes: 4 additions & 2 deletions lib/posix/posix.nim
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,8 +151,10 @@ proc htons*(a1: uint16): uint16 {.importc, header: "<arpa/inet.h>".}
proc ntohl*(a1: uint32): uint32 {.importc, header: "<arpa/inet.h>".}
proc ntohs*(a1: uint16): uint16 {.importc, header: "<arpa/inet.h>".}

proc inet_addr*(a1: cstring): InAddrT {.importc, header: "<arpa/inet.h>".}
proc inet_ntoa*(a1: InAddr): cstring {.importc, header: "<arpa/inet.h>".}
when not defined(zephyr):
proc inet_addr*(a1: cstring): InAddrT {.importc, header: "<arpa/inet.h>".}
proc inet_ntoa*(a1: InAddr): cstring {.importc, header: "<arpa/inet.h>".}

proc inet_ntop*(a1: cint, a2: pointer, a3: cstring, a4: int32): cstring {.
importc:"(char *)$1", header: "<arpa/inet.h>".}
proc inet_pton*(a1: cint, a2: cstring, a3: pointer): cint {.
Expand Down
2 changes: 2 additions & 0 deletions lib/posix/posix_freertos_consts.nim
View file
Original file line number Diff line number Diff line change
Expand Up @@ -363,6 +363,8 @@ var SEM_FAILED* {.importc: "SEM_FAILED", header: "<semaphore.h>".}: pointer
# <sys/resource.h>
# var RLIMIT_NOFILE* {.importc: "RLIMIT_NOFILE", header: "<sys/resource.h>".}: cint

var FD_MAX* {.importc: "CONFIG_LWIP_MAX_SOCKETS", header: "<lwipopts.h>".}: cint

# <sys/select.h>
var FD_SETSIZE* {.importc: "FD_SETSIZE", header: "<sys/select.h>".}: cint

Expand Down
94 changes: 61 additions & 33 deletions lib/posix/posix_other.nim
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
when defined(nimHasStyleChecks):
{.push styleChecks: off.}

when defined(freertos):
when defined(freertos) or defined(zephyr):
const
hasSpawnH = false # should exist for every Posix system nowadays
hasAioH = false
Expand Down Expand Up @@ -391,9 +391,29 @@ when hasSpawnH:
header: "<spawn.h>", final, pure.} = object

when defined(linux):
const Sockaddr_max_length* = 255
# from sys/un.h
const Sockaddr_un_path_length* = 108
elif defined(zephyr):
when defined(net_ipv6):
const Sockaddr_max_length* = 24
elif defined(net_raw):
const Sockaddr_max_length* = 20
else:
const Sockaddr_max_length* = 8
const Sockaddr_un_path_length* = Sockaddr_max_length
# Zephyr is heavily customizable so it's easy to get to a state
# where Nim & Zephyr IPv6 settings are out of sync, causing painful runtime failures.
{.emit: ["NIM_STATIC_ASSERT(NET_SOCKADDR_MAX_SIZE == ",
Sockaddr_max_length,
",\"NET_SOCKADDR_MAX_SIZE and Sockaddr_max_length size mismatch!",
" Check that Nim and Zephyr IPv4/IPv6 settings match.",
" Try adding -d:net_ipv6 to enable IPv6 for Nim on Zephyr.\" );"].}
elif defined(freertos) or defined(lwip):
const Sockaddr_max_length* = 14
const Sockaddr_un_path_length* = 108
else:
const Sockaddr_max_length* = 255
# according to http://pubs.opengroup.org/onlinepubs/009604499/basedefs/sys/un.h.html
# this is >=92
const Sockaddr_un_path_length* = 92
Expand All @@ -408,48 +428,50 @@ when defined(lwip):
pure, final.} = object ## struct sockaddr
sa_len*: uint8 ## Address family.
sa_family*: TSa_Family ## Address family.
sa_data*: array[0..255, char] ## Socket address (variable-length data).
else:
sa_data*: array[0..Sockaddr_max_length-sizeof(uint8)-sizeof(TSa_Family), char] ## Socket address (variable-length data).

Sockaddr_storage* {.importc: "struct sockaddr_storage",
header: "<sys/socket.h>",
pure, final.} = object ## struct sockaddr_storage
s2_len*: uint8 ## Address family.
ss_family*: TSa_Family ## Address family.
s2_data1*: array[2, char] ## Address family.
s2_data2*: array[3, uint32] ## Address family.
when defined(lwip6) or defined(net_ipv6):
s2_data3*: array[3, uint32] ## Address family.
elif defined(zephyr):
type
SockAddr* {.importc: "struct sockaddr", header: "<sys/socket.h>",
pure, final.} = object ## struct sockaddr
sa_family*: TSa_Family ## Address family.
sa_data*: array[0..255, char] ## Socket address (variable-length data).
data*: array[0..Sockaddr_max_length-sizeof(TSa_Family), char] ## Socket address (variable-length data).

type
Sockaddr_un* {.importc: "struct sockaddr_un", header: "<sys/un.h>",
pure, final.} = object ## struct sockaddr_un
sun_family*: TSa_Family ## Address family.
sun_path*: array[0..Sockaddr_un_path_length-1, char] ## Socket path


when defined(lwip):
when not defined(lwip6):
type
Sockaddr_storage* {.importc: "struct sockaddr_storage",
header: "<sys/socket.h>",
pure, final.} = object ## struct sockaddr_storage
s2_len*: uint8 ## Address family.
ss_family*: TSa_Family ## Address family.
s2_data1*: array[2, char] ## Address family.
s2_data2*: array[3, uint32] ## Address family.
else:
type
Sockaddr_storage* {.importc: "struct sockaddr_storage",
header: "<sys/socket.h>",
pure, final.} = object ## struct sockaddr_storage
s2_len*: uint8 ## Address family.
ss_family*: TSa_Family ## Address family.
s2_data1*: array[2, char] ## Address family.
s2_data2*: array[3, uint32] ## Address family.
s2_data3*: array[3, uint32] ## Address family.
Sockaddr_storage* {.importc: "struct sockaddr_storage",
header: "<sys/socket.h>",
pure, final.} = object ## struct sockaddr_storage
ss_family*: TSa_Family ## Address family.
data*: array[0..Sockaddr_max_length-sizeof(TSa_Family), char] ## Socket address (variable-length data).
{.emit: ["NIM_STATIC_ASSERT(sizeof(struct sockaddr) == ", sizeof(Sockaddr), ",\"struct size mismatch\" );"].}
{.emit: ["NIM_STATIC_ASSERT(sizeof(struct sockaddr_storage) == ", sizeof(Sockaddr_storage), ",\"struct size mismatch\" );"].}
else:
type
SockAddr* {.importc: "struct sockaddr", header: "<sys/socket.h>",
pure, final.} = object ## struct sockaddr
sa_family*: TSa_Family ## Address family.
sa_data*: array[0..Sockaddr_max_length-sizeof(TSa_Family), char] ## Socket address (variable-length data).

Sockaddr_storage* {.importc: "struct sockaddr_storage",
header: "<sys/socket.h>",
pure, final.} = object ## struct sockaddr_storage
ss_family*: TSa_Family ## Address family.

type
Sockaddr_un* {.importc: "struct sockaddr_un", header: "<sys/un.h>",
pure, final.} = object ## struct sockaddr_un
sun_family*: TSa_Family ## Address family.
sun_path*: array[0..Sockaddr_un_path_length-sizeof(TSa_Family), char] ## Socket path


type
Tif_nameindex* {.importc: "struct if_nameindex", final,
pure, header: "<net/if.h>".} = object ## struct if_nameindex
Expand Down Expand Up @@ -494,6 +516,7 @@ type
header: "<netinet/in.h>".} = object ## struct in_addr
s_addr*: InAddrScalar

# TODO: Fixme for FreeRTOS/LwIP, these are incorrect
Sockaddr_in* {.importc: "struct sockaddr_in", pure, final,
header: "<netinet/in.h>".} = object ## struct sockaddr_in
sin_family*: TSa_Family ## AF_INET.
Expand Down Expand Up @@ -577,7 +600,12 @@ when not defined(lwip):
events*: cshort ## The input event flags (see below).
revents*: cshort ## The output event flags (see below).

Tnfds* {.importc: "nfds_t", header: "<poll.h>".} = cint
when defined(zephyr):
type
Tnfds* = distinct cint
else:
type
Tnfds* {.importc: "nfds_t", header: "<poll.h>".} = cint

var
errno* {.importc, header: "<errno.h>".}: cint ## error variable
Expand Down Expand Up @@ -625,7 +653,7 @@ elif defined(solaris):
# Solaris doesn't have MSG_NOSIGNAL
const
MSG_NOSIGNAL* = 0'i32
elif defined(freertos) or defined(lwip):
elif defined(zephyr) or defined(freertos) or defined(lwip):
# LwIP/FreeRTOS doesn't have MSG_NOSIGNAL
const
MSG_NOSIGNAL* = 0x20'i32
Expand Down
12 changes: 10 additions & 2 deletions lib/posix/posix_other_consts.nim
View file
Original file line number Diff line number Diff line change
Expand Up @@ -462,6 +462,9 @@ var RLIMIT_NOFILE* {.importc: "RLIMIT_NOFILE", header: "<sys/resource.h>".}: cin

# <sys/select.h>
var FD_SETSIZE* {.importc: "FD_SETSIZE", header: "<sys/select.h>".}: cint
when defined(zephyr):
# Zephyr specific hardcoded value
var FD_MAX* {.importc: "CONFIG_POSIX_MAX_FDS ", header: "<sys/select.h>".}: cint

# <sys/socket.h>
var MSG_CTRUNC* {.importc: "MSG_CTRUNC", header: "<sys/socket.h>".}: cint
Expand All @@ -487,10 +490,15 @@ var SO_SNDTIMEO* {.importc: "SO_SNDTIMEO", header: "<sys/socket.h>".}: cint
var SO_TYPE* {.importc: "SO_TYPE", header: "<sys/socket.h>".}: cint
var SOCK_DGRAM* {.importc: "SOCK_DGRAM", header: "<sys/socket.h>".}: cint
var SOCK_RAW* {.importc: "SOCK_RAW", header: "<sys/socket.h>".}: cint
var SOCK_SEQPACKET* {.importc: "SOCK_SEQPACKET", header: "<sys/socket.h>".}: cint
when defined(zephyr):
const SOCK_SEQPACKET* = cint(5)
var SOMAXCONN* {.importc: "CONFIG_NET_SOCKETS_POLL_MAX", header: "<sys/socket.h>".}: cint
else:
var SOCK_SEQPACKET* {.importc: "SOCK_SEQPACKET", header: "<sys/socket.h>".}: cint
var SOMAXCONN* {.importc: "SOMAXCONN", header: "<sys/socket.h>".}: cint

var SOCK_STREAM* {.importc: "SOCK_STREAM", header: "<sys/socket.h>".}: cint
var SOL_SOCKET* {.importc: "SOL_SOCKET", header: "<sys/socket.h>".}: cint
var SOMAXCONN* {.importc: "SOMAXCONN", header: "<sys/socket.h>".}: cint
var MSG_PEEK* {.importc: "MSG_PEEK", header: "<sys/socket.h>".}: cint
var MSG_TRUNC* {.importc: "MSG_TRUNC", header: "<sys/socket.h>".}: cint
var MSG_WAITALL* {.importc: "MSG_WAITALL", header: "<sys/socket.h>".}: cint
Expand Down
7 changes: 5 additions & 2 deletions lib/pure/asyncdispatch.nim
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1973,15 +1973,18 @@ proc activeDescriptors*(): int {.inline.} =
when defined(posix):
import posix

when defined(linux) or defined(windows) or defined(macosx) or defined(bsd):
when defined(linux) or defined(windows) or defined(macosx) or defined(bsd) or
defined(zephyr) or defined(freertos):
proc maxDescriptors*(): int {.raises: OSError.} =
## Returns the maximum number of active file descriptors for the current
## process. This involves a system call. For now `maxDescriptors` is
## supported on the following OSes: Windows, Linux, OSX, BSD.
when defined(windows):
result = 16_700_000
elif defined(zephyr) or defined(freertos):
result = FD_MAX
else:
var fdLim: RLimit
if getrlimit(RLIMIT_NOFILE, fdLim) < 0:
raiseOSError(osLastError())
result = int(fdLim.rlim_cur) - 1
result = int(fdLim.rlim_cur) - 1
14 changes: 8 additions & 6 deletions lib/pure/asyncnet.nim
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,7 @@ export SOBool
# TODO: Remove duplication introduced by PR #4683.

const defineSsl = defined(ssl) or defined(nimdoc)
const useNimNetLite = defined(nimNetLite) or defined(freertos) or defined(zephyr)

when defineSsl:
import openssl
Expand Down Expand Up @@ -178,11 +179,12 @@ proc getLocalAddr*(socket: AsyncSocket): (string, Port) =
## This is high-level interface for `getsockname`:idx:.
getLocalAddr(socket.fd, socket.domain)

proc getPeerAddr*(socket: AsyncSocket): (string, Port) =
## Get the socket's peer address and port number.
##
## This is high-level interface for `getpeername`:idx:.
getPeerAddr(socket.fd, socket.domain)
when not useNimNetLite:
proc getPeerAddr*(socket: AsyncSocket): (string, Port) =
## Get the socket's peer address and port number.
##
## This is high-level interface for `getpeername`:idx:.
getPeerAddr(socket.fd, socket.domain)

proc newAsyncSocket*(domain, sockType, protocol: cint,
buffered = true,
Expand Down Expand Up @@ -655,7 +657,7 @@ proc hasDataBuffered*(s: AsyncSocket): bool {.since: (1, 5).} =
# xxx dedup with std/net
s.isBuffered and s.bufLen > 0 and s.currPos != s.bufLen

when defined(posix):
when defined(posix) and not useNimNetLite:

proc connectUnix*(socket: AsyncSocket, path: string): owned(Future[void]) =
## Binds Unix socket to `path`.
Expand Down
5 changes: 1 addition & 4 deletions lib/pure/ioselects/ioselectors_epoll.nim
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,10 +73,7 @@ type

proc newSelector*[T](): Selector[T] =
# Retrieve the maximum fd count (for current OS) via getrlimit()
var a = RLimit()
if getrlimit(posix.RLIMIT_NOFILE, a) != 0:
raiseOSError(osLastError())
var maxFD = int(a.rlim_max)
var maxFD = maxDescriptors()
doAssert(maxFD > 0)
# Start with a reasonable size, checkFd() will grow this on demand
const numFD = 1024
Expand Down
5 changes: 1 addition & 4 deletions lib/pure/ioselects/ioselectors_poll.nim
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,10 +53,7 @@ else:
body

proc newSelector*[T](): Selector[T] =
var a = RLimit()
if getrlimit(posix.RLIMIT_NOFILE, a) != 0:
raiseIOSelectorsError(osLastError())
var maxFD = int(a.rlim_max)
var maxFD = maxDescriptors()

when hasThreadSupport:
result = cast[Selector[T]](allocShared0(sizeof(SelectorImpl[T])))
Expand Down
2 changes: 1 addition & 1 deletion lib/pure/ioselects/ioselectors_select.nim
View file
Original file line number Diff line number Diff line change
Expand Up @@ -313,7 +313,7 @@ proc selectInto*[T](s: Selector[T], timeout: int,
verifySelectParams(timeout)

if timeout != -1:
when defined(genode) or defined(freertos):
when defined(genode) or defined(freertos) or defined(zephyr):
tv.tv_sec = Time(timeout div 1_000)
else:
tv.tv_sec = timeout.int32 div 1_000
Expand Down
Loading

0 comments on commit 1dc4769

Please sign in to comment.