From 5d7c06fc70b4ac0094bc646543f05ad26e9fe937 Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Wed, 18 Sep 2024 01:45:45 +0100 Subject: [PATCH 01/76] icmp: Fix csum in echo reply We were calculating the csum without initializing the icmp header's csum field, which made for spurious echo reply drops. Signed-off-by: Pedro Falcato --- kernel/kernel/net/ipv4/icmp.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kernel/kernel/net/ipv4/icmp.cpp b/kernel/kernel/net/ipv4/icmp.cpp index 5ef15d6f4..7a28ef2b9 100644 --- a/kernel/kernel/net/ipv4/icmp.cpp +++ b/kernel/kernel/net/ipv4/icmp.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 - 2022 Pedro Falcato + * Copyright (c) 2016 - 2024 Pedro Falcato * This file is part of Onyx, and is released under the terms of the GPLv2 License * check LICENSE at the root directory for more information * @@ -72,6 +72,7 @@ void send_echo_reply(ip_header *iphdr, icmp_header *icmphdr, uint16_t length, ne response_icmp->code = 0; response_icmp->rest = icmphdr->rest; memcpy(buf->put(data_length), &icmphdr->echo.data, data_length); + response_icmp->checksum = 0; response_icmp->checksum = ipsum(response_icmp, length); inet_sock_address from{src, 0}; From 7514b80c31c71d9191e915b01a8f7ba5592b175e Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Wed, 18 Sep 2024 01:47:08 +0100 Subject: [PATCH 02/76] printk: Add %pI4 support Add %pI4 support, per the Linux kernel printk specifiers. %pI4 prints an IPv4 address in xx.xx.xx.xx form. Example: in_addr_t addr = INADDR_LOOPBACK; printf("%pI4\n", &addr) => 127.0.0.1 Signed-off-by: Pedro Falcato --- kernel/lib/libk/stdio/sprintf.c | 48 +++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/kernel/lib/libk/stdio/sprintf.c b/kernel/lib/libk/stdio/sprintf.c index 2aef5b004..2b5fffd9d 100644 --- a/kernel/lib/libk/stdio/sprintf.c +++ b/kernel/lib/libk/stdio/sprintf.c @@ -12,6 +12,7 @@ #include #include +#include #include #include @@ -399,6 +400,50 @@ static int dump_buffer(struct stream *stream, void *ptr, struct printf_specifier return written; } +static int print_ipaddr(struct stream *stream, void *ptr, struct printf_specifier *spec, + const char **pfmt) +{ + const char *fmt = *pfmt; + if (*fmt == '4') + { + u32 *inp = ptr; + u32 in = *inp; + char buf[sizeof("255.255.255.255")]; + int j = 0; + for (int i = 0; i < 4; i++, in >>= 8) + { + int k = 3; + u8 byte = in & 0xff; + char tmp[4]; + + if (i > 0) + buf[j++] = '.'; + + if (byte > 0) + { + while (byte) + { + tmp[k--] = '0' + (byte % 10); + byte /= 10; + } + } + else + { + tmp[k--] = '0'; + } + + while (k < 3) + buf[j++] = tmp[++k]; + } + + buf[j] = '\0'; + *pfmt = ++fmt; + return printf_do_string(stream, buf, spec->fwidth, spec->precision, spec->flags); + } + + return 0; +} + static int print_pointer(struct stream *stream, void *ptr, struct printf_specifier *spec, const char **pfmt) { @@ -415,6 +460,9 @@ static int print_pointer(struct stream *stream, void *ptr, struct printf_specifi case 'h': *pfmt = ++fmt; return dump_buffer(stream, ptr, spec); + case 'I': + *pfmt = ++fmt; + return print_ipaddr(stream, ptr, spec, pfmt); case 'x': *pfmt = ++fmt; /* fallthrough */ From 3974dac22560c82a0f40e45f410e7448731a045d Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Wed, 18 Sep 2024 01:49:01 +0100 Subject: [PATCH 03/76] x86/isr: Break when printing a bad stack Break instead of continuing when printing a bad stack. This should lead to led recursive faults (which helps GDB). Signed-off-by: Pedro Falcato --- kernel/arch/x86_64/isr.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/arch/x86_64/isr.cpp b/kernel/arch/x86_64/isr.cpp index 8301650ba..ba546ebd1 100644 --- a/kernel/arch/x86_64/isr.cpp +++ b/kernel/arch/x86_64/isr.cpp @@ -658,7 +658,7 @@ void print_int_stacks() /* User or corrupted stack, skip */ pr_emerg(" (%s stack skipped, bad stack)\n", type); tps = next; - continue; + break; } if (is_trap) From f59b601d7cb015d92b9b4255cc347134f90e769b Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Wed, 18 Sep 2024 02:09:38 +0100 Subject: [PATCH 04/76] net: Rework neighbours Add a brand new neighbour system, more correct, non-blocking, and takes full advantage of RCU. This commit does not yet support failure. While we're at it, we add small fixes on the IPv4 and v6 core code. One notable fix is the adding of longest matching prefix logic (which is required for e.g proper IPv4 routing on local networks). Signed-off-by: Pedro Falcato --- .clang-format | 3 +- kernel/drivers/net/rtl8168/rtl8168.cpp | 1 + kernel/include/onyx/atomic.h | 4 + kernel/include/onyx/list.h | 1 + kernel/include/onyx/net/arp.h | 8 +- kernel/include/onyx/net/dll.h | 3 +- kernel/include/onyx/net/inet_route.h | 55 ++++++- kernel/include/onyx/net/ip.h | 2 + kernel/include/onyx/net/ipv6.h | 2 + kernel/include/onyx/net/ndp.h | 9 +- kernel/include/onyx/net/neighbour.h | 179 ++++++++++++++--------- kernel/include/onyx/net/netkernel.h | 1 + kernel/include/onyx/rculist.h | 17 +++ kernel/kernel/net/ipv4/arp.cpp | 123 ++++++---------- kernel/kernel/net/ipv4/ipv4.cpp | 109 +++++++------- kernel/kernel/net/ipv6/ipv6.cpp | 96 +++++++++---- kernel/kernel/net/ipv6/ndp.cpp | 105 +++++--------- kernel/kernel/net/neighbour.cpp | 191 +++++++++++++++++++------ 18 files changed, 553 insertions(+), 356 deletions(-) diff --git a/.clang-format b/.clang-format index 0cbe2ddf1..3675a83a5 100644 --- a/.clang-format +++ b/.clang-format @@ -15,7 +15,8 @@ AllowShortLoopsOnASingleLine: false DerivePointerAlignment: true PointerAlignment: Right ColumnLimit: 100 -ForEachMacros: ['list_for_every','list_for_every_safe','list_for_every_rcu','mt_for_each'] +ForEachMacros: ['list_for_every','list_for_every_safe','list_for_every_rcu','mt_for_each','list_for_each_entry', + 'list_for_each_entry_safe', 'list_for_each_entry_rcu'] IncludeBlocks: Regroup IndentExternBlock: NoIndent IncludeCategories: diff --git a/kernel/drivers/net/rtl8168/rtl8168.cpp b/kernel/drivers/net/rtl8168/rtl8168.cpp index e66e8d5dc..46b7f6264 100644 --- a/kernel/drivers/net/rtl8168/rtl8168.cpp +++ b/kernel/drivers/net/rtl8168/rtl8168.cpp @@ -13,6 +13,7 @@ #include #include +#include #include #include diff --git a/kernel/include/onyx/atomic.h b/kernel/include/onyx/atomic.h index 720d2720a..80bd01aed 100644 --- a/kernel/include/onyx/atomic.h +++ b/kernel/include/onyx/atomic.h @@ -16,6 +16,10 @@ #define atomic_and_relaxed(var, mask) (__atomic_and_fetch(&(var), mask, __ATOMIC_RELAXED)) #define atomic_or_relaxed(var, mask) (__atomic_or_fetch(&(var), mask, __ATOMIC_RELAXED)) +#ifdef __cplusplus +#define __auto_type auto +#endif + #define cmpxchg(ptr, old, new) \ ({ \ __auto_type __old = (old); \ diff --git a/kernel/include/onyx/list.h b/kernel/include/onyx/list.h index c96b406b3..04ce27cc8 100644 --- a/kernel/include/onyx/list.h +++ b/kernel/include/onyx/list.h @@ -216,6 +216,7 @@ static inline int list_is_head(const struct list_head *list, const struct list_h #define list_prepare_entry(pos, head, member) ((pos) ?: list_entry(head, __typeof__(*pos), member)) #define list_entry_is_head(pos, head, member) list_is_head(&pos->member, (head)) #define list_first_entry(ptr, type, member) list_entry((ptr)->next, type, member) +#define list_last_entry(ptr, type, member) list_entry((ptr)->prev, type, member) #define list_for_each_entry_continue(pos, head, member) \ for (pos = list_next_entry(pos, member); !list_entry_is_head(pos, head, member); \ diff --git a/kernel/include/onyx/net/arp.h b/kernel/include/onyx/net/arp.h index b4a9b5cc3..0ca432b36 100644 --- a/kernel/include/onyx/net/arp.h +++ b/kernel/include/onyx/net/arp.h @@ -1,7 +1,9 @@ /* - * Copyright (c) 2016, 2017 Pedro Falcato + * Copyright (c) 2016 - 2024 Pedro Falcato * This file is part of Onyx, and is released under the terms of the GPLv2 License * check LICENSE at the root directory for more information + * + * SPDX-License-Identifier: GPL-2.0-only */ #ifndef _ONYX_NET_ARP_H @@ -38,7 +40,7 @@ typedef struct struct netif; -expected, int> arp_resolve_in(uint32_t ip, struct netif *netif); -int arp_handle_packet(netif *netif, packetbuf *buf); +struct neighbour *arp_resolve_in(uint32_t ip, struct netif *netif); +int arp_handle_packet(struct netif *netif, struct packetbuf *buf); #endif diff --git a/kernel/include/onyx/net/dll.h b/kernel/include/onyx/net/dll.h index 96bde9923..c9a210a4d 100644 --- a/kernel/include/onyx/net/dll.h +++ b/kernel/include/onyx/net/dll.h @@ -6,7 +6,8 @@ #ifndef _ONYX_NET_DLL_H #define _ONYX_NET_DLL_H -struct packetbuf; +#include + struct netif; enum class tx_type diff --git a/kernel/include/onyx/net/inet_route.h b/kernel/include/onyx/net/inet_route.h index a9ce68114..d373b7e3f 100644 --- a/kernel/include/onyx/net/inet_route.h +++ b/kernel/include/onyx/net/inet_route.h @@ -1,12 +1,15 @@ /* - * Copyright (c) 2020 Pedro Falcato + * Copyright (c) 2020 - 2024 Pedro Falcato * This file is part of Onyx, and is released under the terms of the GPLv2 License * check LICENSE at the root directory for more information + * + * SPDX-License-Identifier: GPL-2.0-only */ #ifndef _ONYX_NET_INET_ROUTE_H #define _ONYX_NET_INET_ROUTE_H +#include #include #include @@ -57,7 +60,55 @@ struct inet_route netif *nif; unsigned short flags; - shared_ptr dst_hw; + struct neighbour *dst_hw{}; + + inet_route() = default; + ~inet_route() + { + if (!IS_ERR_OR_NULL(dst_hw)) + neigh_put(dst_hw); + } + + inet_route &operator=(const inet_route &rhs) + { + if (&rhs == this) + return *this; + this->src_addr = rhs.src_addr; + this->dst_addr = rhs.dst_addr; + this->mask = rhs.mask; + this->gateway_addr = rhs.gateway_addr; + this->nif = rhs.nif; + this->flags = rhs.flags; + this->dst_hw = rhs.dst_hw; + if (rhs.dst_hw) + neigh_get(rhs.dst_hw); + return *this; + } + + inet_route(const inet_route &rhs) + { + *this = rhs; + } + + inet_route &operator=(inet_route &&rhs) + { + if (&rhs == this) + return *this; + this->src_addr = rhs.src_addr; + this->dst_addr = rhs.dst_addr; + this->mask = rhs.mask; + this->gateway_addr = rhs.gateway_addr; + this->nif = rhs.nif; + this->flags = rhs.flags; + this->dst_hw = rhs.dst_hw; + rhs.dst_hw = nullptr; + return *this; + } + + inet_route(inet_route &&rhs) + { + *this = cul::move(rhs); + } }; #endif diff --git a/kernel/include/onyx/net/ip.h b/kernel/include/onyx/net/ip.h index e96774258..83519d305 100644 --- a/kernel/include/onyx/net/ip.h +++ b/kernel/include/onyx/net/ip.h @@ -231,4 +231,6 @@ constexpr size_t inet_header_size(int domain) return size; } +int ip_finish_output(struct neighbour *n, struct packetbuf *pbf, struct netif *nif); + #endif diff --git a/kernel/include/onyx/net/ipv6.h b/kernel/include/onyx/net/ipv6.h index 2d2f26bdf..9f94b3cd6 100644 --- a/kernel/include/onyx/net/ipv6.h +++ b/kernel/include/onyx/net/ipv6.h @@ -120,4 +120,6 @@ int netif_addrcfg(netif *nif, const in6_addr &if_id); const struct inet_proto_family *get_v6_proto(); } // namespace ip::v6 +int ip6_finish_output(struct neighbour *neigh, struct packetbuf *pbf, struct netif *nif); + #endif diff --git a/kernel/include/onyx/net/ndp.h b/kernel/include/onyx/net/ndp.h index 0f24e18ec..8691df484 100644 --- a/kernel/include/onyx/net/ndp.h +++ b/kernel/include/onyx/net/ndp.h @@ -1,7 +1,9 @@ /* - * Copyright (c) 2020 Pedro Falcato + * Copyright (c) 2020 - 2024 Pedro Falcato * This file is part of Onyx, and is released under the terms of the GPLv2 License * check LICENSE at the root directory for more information + * + * SPDX-License-Identifier: GPL-2.0-only */ #ifndef _ONYX_NET_NDP_H @@ -9,10 +11,7 @@ #include -#include -#include - -expected, int> ndp_resolve(const in6_addr &ip, struct netif *netif); +struct neighbour *ndp_resolve(const in6_addr &ip, struct netif *netif); int ndp_handle_packet(netif *netif, packetbuf *buf); #endif diff --git a/kernel/include/onyx/net/neighbour.h b/kernel/include/onyx/net/neighbour.h index 5dfaa377e..a42e51a62 100644 --- a/kernel/include/onyx/net/neighbour.h +++ b/kernel/include/onyx/net/neighbour.h @@ -1,7 +1,9 @@ /* - * Copyright (c) 2020 Pedro Falcato + * Copyright (c) 2020 - 2024 Pedro Falcato * This file is part of Onyx, and is released under the terms of the GPLv2 License * check LICENSE at the root directory for more information + * + * SPDX-License-Identifier: GPL-2.0-only */ #ifndef _ONYX_NET_NEIGHBOUR_H @@ -9,54 +11,67 @@ #include +#include #include #include #include #include +#include +#include #include #include -#include -#include -#include -#include - #define NEIGHBOUR_VALIDITY_STATIC (~0UL) void neighbour_revalidate(clockevent* ev); -class neighbour; - #define NEIGHBOUR_FLAG_UNINITIALISED (1 << 0) #define NEIGHBOUR_FLAG_BADENTRY (1 << 1) #define NEIGHBOUR_FLAG_HAS_RESPONSE (1 << 2) #define NEIGHBOUR_FLAG_BROADCAST (1 << 3) +#define NUD_REACHABLE (1 << 4) +#define NUD_INCOMPLETE (1 << 5) +#define NUD_STALE (1 << 6) +#define NUD_PROBE (1 << 7) +#define NUD_FAILED (1 << 8) -class neighbour_table; +struct neighbour_table; union neigh_proto_addr { - in_addr in4addr; - in6_addr in6addr; + struct in_addr in4addr; + struct in6_addr in6addr; +}; + +struct neigh_ops +{ + int (*resolve)(struct neighbour* neigh, struct netif* nif); + int (*output)(struct neighbour* neigh, struct packetbuf* pbf, struct netif* nif); }; -class neighbour +struct neighbour { -protected: - unsigned char* hwaddr_; - unsigned int hwaddr_len_; + unsigned int refcount; + unsigned int hwaddr_len; + union { + unsigned char hwaddr[16]; + struct rcu_head rcu_head; + }; + int domain; struct clockevent expiry_timer; unsigned long validity_ms; - -public: unsigned int flags; - neighbour_table* table; - + struct neighbour_table* table; union neigh_proto_addr proto_addr; + const struct neigh_ops* neigh_ops; + struct list_head list_node; + seqlock_t neigh_seqlock; + struct list_head packet_queue; explicit neighbour(int _domain, const neigh_proto_addr& addr) - : hwaddr_{}, hwaddr_len_{}, domain{_domain}, flags{0} + : refcount{1}, + hwaddr_len{}, domain{_domain}, flags{NEIGHBOUR_FLAG_UNINITIALISED}, neigh_ops{} { if (_domain == AF_INET) proto_addr.in4addr.s_addr = addr.in4addr.s_addr; @@ -64,17 +79,11 @@ class neighbour memcpy(&proto_addr.in6addr, &addr.in6addr, sizeof(in6_addr)); else __builtin_unreachable(); + neigh_seqlock = {}; + INIT_LIST_HEAD(&packet_queue); } - ~neighbour() - { - delete[] hwaddr_; - } - - cul::slice hwaddr() - { - return {hwaddr_, hwaddr_len_}; - } + ~neighbour() = default; void set_validity(unsigned long validity) { @@ -118,14 +127,36 @@ class neighbour else __builtin_unreachable(); } - - void set_hwaddr(cul::slice& addr) - { - hwaddr_ = addr.data(); - hwaddr_len_ = addr.size_bytes(); - } }; +static inline bool neigh_needs_resolve(struct neighbour* neigh) +{ + /* Only bother trying to resolve neighbours if they're not yet resolved, or if there are no + * requests pending. */ + return !(READ_ONCE(neigh->flags) & (NUD_PROBE | NUD_REACHABLE | NUD_INCOMPLETE)); +} + +void neigh_start_resolve(struct neighbour* neigh, struct netif* nif); +void neigh_output_queued(struct neighbour* neigh); + +static inline void __neigh_complete_lookup(struct neighbour* neigh, const void* hwaddr, + unsigned int len) +{ + memcpy(neigh->hwaddr, hwaddr, len); + neigh->hwaddr_len = len; + neigh->flags &= ~(NUD_PROBE | NUD_INCOMPLETE | NUD_FAILED | NUD_STALE); + neigh->flags |= NUD_REACHABLE; + neigh_output_queued(neigh); +} + +static inline void neigh_complete_lookup(struct neighbour* neigh, const void* hwaddr, + unsigned int len) +{ + write_seqlock(&neigh->neigh_seqlock); + __neigh_complete_lookup(neigh, hwaddr, len); + write_sequnlock(&neigh->neigh_seqlock); +} + static inline fnv_hash_t hash_protoaddr(const neigh_proto_addr& addr, int domain) { if (domain == AF_INET) @@ -136,41 +167,61 @@ static inline fnv_hash_t hash_protoaddr(const neigh_proto_addr& addr, int domain __builtin_unreachable(); } -fnv_hash_t hash_neighbour(shared_ptr& neigh); +#define NEIGH_TAB_NR_CHAINS 32 -class neighbour_table +struct neighbour_table { -protected: - cul::hashtable, 32, fnv_hash_t, hash_neighbour> neighbour_cache; - - /* TODO: Is another lock type optimal here? Note that spinlocks have low overhead vs rwlocks - * and lookups are *usually* quick. - */ - spinlock lock; + struct list_head neigh_tab[NEIGH_TAB_NR_CHAINS]; + struct spinlock lock; const int domain; - -public: - neighbour_table(int domain) : neighbour_cache{}, lock{}, domain{domain} + neighbour_table(int domain) : lock{}, domain{domain} { spinlock_init(&lock); + for (int i = 0; i < NEIGH_TAB_NR_CHAINS; i++) + INIT_LIST_HEAD(&neigh_tab[i]); } - - /** - * @brief Add or get an existing neighbour entry - * - * @param addr the protocol address of the neighbour - * @return cul::pair, bool> a shared pointer to the neighbour + a bool - * signaling if we created it or not. - */ - cul::pair, bool> add(const neigh_proto_addr& addr, - bool only_lookup = false); - void remove(neighbour* neigh); - - /** - * @brief Clears cache entries - * - */ - void clear_cache(); }; +typedef unsigned int gfp_t; + +struct neighbour* neigh_find(struct neighbour_table* table, const union neigh_proto_addr* addr); +struct neighbour* neigh_add(struct neighbour_table* table, const union neigh_proto_addr* addr, + gfp_t gfp, const struct neigh_ops* ops, int* added); +void neigh_remove(struct neighbour_table* table, struct neighbour* neigh); +void neigh_clear(struct neighbour_table* table); +void neigh_free(struct neighbour* neigh); + +static inline void neigh_get(struct neighbour* neigh) +{ + __atomic_add_fetch(&neigh->refcount, 1, __ATOMIC_RELAXED); +} + +static inline void neigh_put(struct neighbour* neigh) +{ + if (__atomic_sub_fetch(&neigh->refcount, 1, __ATOMIC_RELAXED) == 0) + neigh_free(neigh); +} + +static inline bool neigh_get_careful(struct neighbour* neigh) +{ + unsigned int ref = READ_ONCE(neigh->refcount); + unsigned int old; + + do + { + if (unlikely(ref == 0)) + return false; + old = ref; + } while ((ref = cmpxchg(&neigh->refcount, ref, ref + 1)) != old); + + return true; +} + +int neigh_output(struct neighbour* neigh, struct packetbuf* pbf, struct netif* nif); + +static inline void neigh_set_ops(struct neighbour* neigh, struct neigh_ops* ops) +{ + WRITE_ONCE(neigh->neigh_ops, ops); +} + #endif diff --git a/kernel/include/onyx/net/netkernel.h b/kernel/include/onyx/net/netkernel.h index 5d37966f8..ccd9c3f83 100644 --- a/kernel/include/onyx/net/netkernel.h +++ b/kernel/include/onyx/net/netkernel.h @@ -10,6 +10,7 @@ #define _ONYX_NET_NETKERNEL_H #include +#include #include #include diff --git a/kernel/include/onyx/rculist.h b/kernel/include/onyx/rculist.h index ebf44c55f..28335a5d1 100644 --- a/kernel/include/onyx/rculist.h +++ b/kernel/include/onyx/rculist.h @@ -8,6 +8,7 @@ #ifndef _ONYX_RCULIST_H #define _ONYX_RCULIST_H +#include #include #include @@ -39,4 +40,20 @@ static inline void list_remove_rcu(struct list_head *node) list_remove_bulk(node->prev, node->next); } +#define list_entry_rcu(ptr, type, member) container_of(READ_ONCE(ptr), type, member) +/** + * list_for_each_entry_rcu - iterate over rcu list of given type + * @pos: the type * to use as a loop cursor. + * @head: the head for your list. + * @member: the name of the list_head within the struct. + * @cond: optional lockdep expression if called from non-RCU protection. + * + * This list-traversal primitive may safely run concurrently with + * the _rcu list-mutation primitives such as list_add_rcu() + * as long as the traversal is guarded by rcu_read_lock(). + */ +#define list_for_each_entry_rcu(pos, head, member, cond...) \ + for (pos = list_entry_rcu((head)->next, __typeof__(*pos), member); &pos->member != (head); \ + pos = list_entry_rcu(pos->member.next, __typeof__(*pos), member)) + #endif diff --git a/kernel/kernel/net/ipv4/arp.cpp b/kernel/kernel/net/ipv4/arp.cpp index 4db8b7dcd..d9fc9d865 100644 --- a/kernel/kernel/net/ipv4/arp.cpp +++ b/kernel/kernel/net/ipv4/arp.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 - 2022 Pedro Falcato + * Copyright (c) 2016 - 2024 Pedro Falcato * This file is part of Onyx, and is released under the terms of the GPLv2 License * check LICENSE at the root directory for more information * @@ -9,9 +9,9 @@ #include #include -/* Don't change the include order! Maybe TOFIX? */ #include #include +#include #include #include #include @@ -22,21 +22,21 @@ #include -/* TODO: Maybe the neighbour_table could replace some code below, if we add a few virtual functions - */ static neighbour_table arp_table{AF_INET}; static constexpr hrtime_t arp_response_timeout = 250 * NS_PER_MS; /* 20 minutes in milis */ static constexpr unsigned long arp_validity_time_ms = 1200000; -int arp_do_request(netif *netif, packetbuf *packet) +static int arp_do_request(netif *netif, packetbuf *packet, arp_request_t *arp_hdr) { - auto arp_hdr = (arp_request_t *) packet->data; - auto target_addr = arp_hdr->target_proto_address; uint8_t hw_address[6]; + /* TODO */ + (void) arp_validity_time_ms; + (void) arp_response_timeout; + if (netif->local_ip.sin_addr.s_addr == target_addr) { memcpy(hw_address, netif->mac_address, 6); @@ -73,60 +73,51 @@ int arp_do_request(netif *netif, packetbuf *packet) return netif_send_packet(netif, buf.get()); } +static int arp_resolve(struct neighbour *neigh, struct netif *netif); + +static const struct neigh_ops arp_ops = { + .resolve = arp_resolve, + .output = ip_finish_output, +}; + int arp_handle_packet(netif *netif, packetbuf *buf) { - auto arp = (arp_request_t *) buf->data; + struct neighbour *neigh; + int added; + arp_request_t *arp = (arp_request_t *) pbf_pull(buf, sizeof(arp_request_t)); + if (!arp) + return -1; - if (buf->length() < sizeof(arp_request_t)) - return -EINVAL; auto op = htons(arp->operation); if (op == ARP_OP_REQUEST) - return arp_do_request(netif, buf); + return arp_do_request(netif, buf, arp); if (op != ARP_OP_REPLY) return 0; in_addr_t req_ip = arp->sender_proto_address; - neigh_proto_addr addr; + union neigh_proto_addr addr; addr.in4addr.s_addr = req_ip; - auto [ptr, __created] = arp_table.add(addr, true); - if (!ptr) - return 0; - - unsigned char *mac = new unsigned char[ETH_ALEN]; - if (!mac) + neigh = neigh_add(&arp_table, &addr, GFP_ATOMIC, &arp_ops, &added); + if (!neigh) return 0; - memcpy(mac, arp->sender_hw_address, ETH_ALEN); - - cul::slice sl{mac, ETH_ALEN}; - ptr->set_hwaddr(sl); - ptr->flags |= NEIGHBOUR_FLAG_HAS_RESPONSE; - + neigh_complete_lookup(neigh, arp->sender_hw_address, ETH_ALEN); return 0; } -int arp_submit_request(shared_ptr &ptr, uint32_t target_addr, struct netif *netif) +static int arp_resolve(struct neighbour *neigh, struct netif *netif) { + in_addr_t target_addr = neigh->proto_addr.in4addr.s_addr; + if (target_addr == INADDR_BROADCAST || target_addr == INADDR_LOOPBACK || netif->flags & NETIF_LOOPBACK) { - auto _ptr = new unsigned char[ETH_ALEN]; - - if (_ptr) - { - bool is_bcast = target_addr == INADDR_BROADCAST; - memset(_ptr, is_bcast ? 0xff : 0, ETH_ALEN); - auto sl = cul::slice{_ptr, ETH_ALEN}; - ptr->set_hwaddr(sl); - ptr->flags |= NEIGHBOUR_FLAG_HAS_RESPONSE | (is_bcast ? NEIGHBOUR_FLAG_BROADCAST : 0); - } - else - { - return -ENOMEM; - } - + const unsigned char bcast_eth[ETH_ALEN] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff}; + const unsigned char loopback_eth[ETH_ALEN] = {}; + __neigh_complete_lookup(neigh, target_addr == INADDR_BROADCAST ? bcast_eth : loopback_eth, + ETH_ALEN); return 0; } @@ -155,7 +146,7 @@ int arp_submit_request(shared_ptr &ptr, uint32_t target_addr, struct arp->target_hw_address[4] = 0xFF; arp->target_hw_address[5] = 0xFF; arp->sender_proto_address = netif->local_ip.sin_addr.s_addr; - arp->target_proto_address = target_addr; + arp->target_proto_address = neigh->proto_addr.in4addr.s_addr; if (int st = netif->dll_ops->setup_header(buf.get(), tx_type::broadcast, tx_protocol::arp, netif, nullptr); st < 0) @@ -164,48 +155,18 @@ int arp_submit_request(shared_ptr &ptr, uint32_t target_addr, struct return netif_send_packet(netif, buf.get()); } -expected, int> arp_resolve_in(uint32_t ip, struct netif *netif) +struct neighbour *arp_resolve_in(uint32_t ip, struct netif *netif) { - neigh_proto_addr addr; + struct neighbour *neigh; + int added; + union neigh_proto_addr addr; addr.in4addr.s_addr = ip; - auto [ptr, created] = arp_table.add(addr); - if (!ptr) - return unexpected{-ENOMEM}; - - if (ptr->flags & NEIGHBOUR_FLAG_UNINITIALISED) - { - if (created) - { - if (arp_submit_request(ptr, ip, netif) < 0) - { - arp_table.remove(ptr.get_data()); - return unexpected{-ENOMEM}; - } - } - - auto t0 = clocksource_get_time(); - - /* TODO: Add a wait_for_bit that can let us wait for random things - * without taking up permanent space in the structure - */ - while (!(ptr->flags & NEIGHBOUR_FLAG_HAS_RESPONSE) && - clocksource_get_time() - t0 <= arp_response_timeout) - sched_sleep_ms(15); - - if (!(ptr->flags & NEIGHBOUR_FLAG_HAS_RESPONSE)) - { - if (created) - arp_table.remove(ptr.get_data()); - return unexpected{-ENETUNREACH}; - } - - if (created) - { - ptr->set_validity(arp_validity_time_ms); - ptr->set_initialised(); - } - } + neigh = neigh_add(&arp_table, &addr, GFP_ATOMIC, &arp_ops, &added); + if (!neigh) + return (struct neighbour *) ERR_PTR(-ENOMEM); - return ptr; + if (neigh_needs_resolve(neigh)) + neigh_start_resolve(neigh, netif); + return neigh; } diff --git a/kernel/kernel/net/ipv4/ipv4.cpp b/kernel/kernel/net/ipv4/ipv4.cpp index 3b0f0070c..2c8716b92 100644 --- a/kernel/kernel/net/ipv4/ipv4.cpp +++ b/kernel/kernel/net/ipv4/ipv4.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 - 2022 Pedro Falcato + * Copyright (c) 2016 - 2024 Pedro Falcato * This file is part of Onyx, and is released under the terms of the GPLv2 License * check LICENSE at the root directory for more information * @@ -13,6 +13,7 @@ #include #include +#include #include #include #include @@ -253,26 +254,27 @@ static tx_type detect_tx_type(const inet_route &route) return tx_type::unicast; } -int send_fragment(const inet_route &route, fragment *frag, netif *nif) +int send_frag_multicast(tx_type type, const inet_route &route, fragment *frag, netif *nif) { - auto buf = frag->this_buf; - auto type = detect_tx_type(route); - int st = 0; - - const void *hwaddr = nullptr; - - if (type != tx_type::broadcast) [[likely]] - { - if (route.dst_hw->flags & NEIGHBOUR_FLAG_BADENTRY) - return -EHOSTUNREACH; - hwaddr = route.dst_hw->hwaddr().data(); - } - - if ((st = nif->dll_ops->setup_header(buf, type, tx_protocol::ipv4, nif, hwaddr)) < 0) - [[unlikely]] + /* Sending an IPv4 fragment deviates from the neighbour paths because we truly don't communicate + * with a neighbour - at least not with a single one. */ + int st; + if ((st = nif->dll_ops->setup_header(frag->this_buf, type, tx_protocol::ipv4, nif, nullptr)) < + 0) [[unlikely]] return st; - return netif_send_packet(nif, buf); + return netif_send_packet(nif, frag->this_buf); +} + +int send_fragment(const inet_route &route, fragment *frag, netif *nif) +{ + tx_type type = detect_tx_type(route); + if (type != tx_type::unicast) + return send_frag_multicast(type, route, frag, nif); + struct neighbour *neigh = route.dst_hw; + CHECK(neigh != NULL); + frag->this_buf->route = route; + return neigh_output(neigh, frag->this_buf, nif); } int do_fragmentation(struct send_info *sinfo, size_t payload_size, packetbuf *buf, @@ -290,9 +292,7 @@ int do_fragmentation(struct send_info *sinfo, size_t payload_size, packetbuf *bu list_for_every (&frags) { struct fragment *frag = container_of(l, struct fragment, list_node); - st = send_fragment(sinfo->route, frag, netif); - if (st < 0) goto out; } @@ -324,7 +324,7 @@ int send_packet(const iflow &flow, packetbuf *buf, const cul::slice & sinfo.type = flow.protocol; sinfo.frags_following = false; - if (needs_fragmentation(buf->length(), netif)) + if (needs_fragmentation(payload_size, netif)) { /* TODO: Support ISO(IP segmentation offloading) */ sinfo.identification = allocate_id(); @@ -451,14 +451,14 @@ int handle_packet(netif *nif, packetbuf *buf) route.flags |= INET4_ROUTE_FLAG_BROADCAST; } - new (&buf->route) inet_route{route}; + buf->route = cul::move(route); if (header->proto == IPPROTO_UDP) - return udp_handle_packet(route, buf); + return udp_handle_packet(buf->route, buf); else if (header->proto == IPPROTO_TCP) - return tcp_handle_packet(route, buf); + return tcp_handle_packet(buf->route, buf); else if (header->proto == IPPROTO_ICMP) - return icmp::handle_packet(route, buf); + return icmp::handle_packet(buf->route, buf); else { /* Oh, no, an unhandled protocol! Send an ICMP error message */ @@ -478,7 +478,7 @@ int handle_packet(netif *nif, packetbuf *buf) dst_un.dgram = dgram; dst_un.next_hop_mtu = 0; - icmp::send_dst_unreachable(dst_un, nif); + // icmp::send_dst_unreachable(dst_un, nif); } return 0; @@ -615,6 +615,7 @@ expected route(const inet_sock_address &from, const inet_sock_a */ shared_ptr best_route; int highest_metric = 0; + int longest_prefix = -1; auto dest = to.in4.s_addr; // TODO: Multicast @@ -641,8 +642,9 @@ expected route(const inet_sock_address &from, const inet_sock_a /* Do a bitwise and between the destination address and the mask * If the result = r.dest, we can use this interface. */ + int mask_bits; #if 0 - printk("dest %x, mask %x, supposed dest %x\n", dest, r->mask, r->dest); + pr_info("dest %pI4, mask %pI4, supposed dest %pI4\n", &dest, &r->mask, &r->dest); #endif if ((dest & r->mask) != r->dest) continue; @@ -650,27 +652,27 @@ expected route(const inet_sock_address &from, const inet_sock_a if (required_netif && r->nif != required_netif) continue; #if 0 - printk("%s is good\n", r->nif->name); - printk("is loopback set %u\n", r->nif->flags & NETIF_LOOPBACK); -#endif - - int mods = 0; + pr_info("%s is good\n", r->nif->name); + pr_info("is loopback set %u\n", r->nif->flags & NETIF_LOOPBACK); if (r->flags & INET4_ROUTE_FLAG_GATEWAY) - mods--; + pr_info("gateway %pI4\n", &r->gateway); +#endif + /* TODO: This can and should be computed beforehand */ + mask_bits = count_bits(r->mask); - if (r->metric + mods > highest_metric) + if (mask_bits > longest_prefix || + (longest_prefix == mask_bits && r->metric > highest_metric)) { best_route = r; highest_metric = r->metric; + longest_prefix = mask_bits; } } routing_table_lock.unlock_read(); if (!best_route) - { return unexpected{-ENETUNREACH}; - } inet_route r; r.dst_addr.in4 = to.in4; @@ -681,30 +683,18 @@ expected route(const inet_sock_address &from, const inet_sock_a r.gateway_addr.in4.s_addr = best_route->gateway; if (addr_is_broadcast(to.in4.s_addr, r)) - { r.flags |= INET4_ROUTE_FLAG_BROADCAST; - } else if (addr_is_multicast(to.in4.s_addr)) - { r.flags |= INET4_ROUTE_FLAG_MULTICAST; - } auto to_resolve = r.dst_addr.in4.s_addr; if (r.flags & INET4_ROUTE_FLAG_GATEWAY) - { to_resolve = r.gateway_addr.in4.s_addr; - } - - auto res = arp_resolve_in(to_resolve, r.nif); - - if (res.has_error()) [[unlikely]] - { - return unexpected(-ENETUNREACH); - } - - r.dst_hw = res.value(); + r.dst_hw = arp_resolve_in(to_resolve, r.nif); + if (IS_ERR(r.dst_hw)) + return unexpected{PTR_ERR(r.dst_hw)}; return r; } @@ -901,3 +891,20 @@ bool inet_proto_family::add_socket(inet_socket *sock) auto sock_table = proto_info->get_socket_table(); return sock_table->add_socket(sock, ADD_SOCKET_UNLOCKED); } + +int ip_finish_output(struct neighbour *neigh, struct packetbuf *pbf, struct netif *nif) +{ + auto type = ip::v4::detect_tx_type(pbf->route); + int st = 0; + const void *hwaddr = nullptr; + + if (neigh->flags & NUD_FAILED) + return -EHOSTUNREACH; + hwaddr = neigh->hwaddr; + + if ((st = nif->dll_ops->setup_header(pbf, type, tx_protocol::ipv4, nif, hwaddr)) < 0) + [[unlikely]] + return st; + + return netif_send_packet(nif, pbf); +} diff --git a/kernel/kernel/net/ipv6/ipv6.cpp b/kernel/kernel/net/ipv6/ipv6.cpp index 79fc0876e..12eb1f075 100644 --- a/kernel/kernel/net/ipv6/ipv6.cpp +++ b/kernel/kernel/net/ipv6/ipv6.cpp @@ -1,11 +1,12 @@ /* - * Copyright (c) 2020 - 2022 Pedro Falcato + * Copyright (c) 2020 - 2024 Pedro Falcato * This file is part of Onyx, and is released under the terms of the GPLv2 License * check LICENSE at the root directory for more information * * SPDX-License-Identifier: GPL-2.0-only */ +#include #include #include #include @@ -38,6 +39,16 @@ static constexpr tx_type ipv6_addr_to_tx_type(const in6_addr &dst) return dst.s6_addr[0] == 0xff ? tx_type::multicast : tx_type::unicast; } +static int output_mcast(struct packetbuf *pbf, struct netif *nif, struct ip6hdr *hdr) +{ + int st; + if ((st = nif->dll_ops->setup_header(pbf, tx_type::multicast, tx_protocol::ipv6, nif, + &hdr->dst_addr)) < 0) + return st; + + return netif_send_packet(nif, pbf); +} + int send_packet(const iflow &flow, packetbuf *buf) { const auto &route = flow.route; @@ -72,25 +83,12 @@ int send_packet(const iflow &flow, packetbuf *buf) hdr->next_header = next_header; hdr->hop_limit = flow.ttl; - int st = 0; - const auto ttype = ipv6_addr_to_tx_type(route.dst_addr.in6); - const void *dst_hw = nullptr; + if (ttype == tx_type::multicast) + return output_mcast(buf, netif, hdr); - if (ttype == tx_type::unicast) - { - dst_hw = route.dst_hw->hwaddr().data(); - } - else if (ttype == tx_type::multicast) - { - /* Let the lower layer figure out the multicast address */ - dst_hw = &hdr->dst_addr; - } - - if ((st = netif->dll_ops->setup_header(buf, ttype, tx_protocol::ipv6, netif, dst_hw)) < 0) - return st; - - return netif_send_packet(netif, buf); + buf->route = route; + return neigh_output(buf->route.dst_hw, buf, netif); } int bind_internal(sockaddr_in6 *in, inet_socket *sock) @@ -223,6 +221,7 @@ expected route_from_routing_table(const inet_sock_address &to, shared_ptr best_route; int highest_metric = 0; auto dest = to.in6; + int longest_prefix = -1; { @@ -233,12 +232,14 @@ expected route_from_routing_table(const inet_sock_address &to, /* Do a bitwise and between the destination address and the mask * If the result = r.dest, we can use this interface. */ + int mask_bits; + const auto masked = dest & r->mask; #if 0 print_v6_addr(dest); print_v6_addr(r->mask); print_v6_addr(r->dest); #endif - if ((dest & r->mask) != r->dest) + if (masked != r->dest) continue; if (required_netif && r->nif != required_netif) @@ -247,15 +248,22 @@ expected route_from_routing_table(const inet_sock_address &to, printk("%s is good\n", r->nif->name); printk("is loopback set %u\n", r->nif->flags & NETIF_LOOPBACK); #endif + /* TODO: This can and should be computed beforehand */ + mask_bits = count_bits(r->mask.__in6_union.__s6_addr32[0]) + + count_bits(r->mask.__in6_union.__s6_addr32[1]) + + count_bits(r->mask.__in6_union.__s6_addr32[2]) + + count_bits(r->mask.__in6_union.__s6_addr32[3]); - int mods = 0; - if (r->flags & INET4_ROUTE_FLAG_GATEWAY) - mods--; - - if (r->metric + mods > highest_metric) +#if 0 + pr_warn("dest %pI6 mask %pI6 route dest %pI6 mask_bits %d\n", &dest, &r->mask, &r->dest, + mask_bits); +#endif + if (mask_bits > longest_prefix || + (longest_prefix == mask_bits && r->metric > highest_metric)) { best_route = r; highest_metric = r->metric; + longest_prefix = mask_bits; } } } @@ -351,14 +359,10 @@ expected route(const inet_sock_address &from, const inet_sock_a // printk("Gateway %x\n", ntohs(r.gateway_addr.in6.s6_addr16[0])); } - auto res = ndp_resolve(to_resolve, rt.nif); - - if (res.has_error()) [[unlikely]] - { - return unexpected{-ENETUNREACH}; - } + rt.dst_hw = ndp_resolve(to_resolve, rt.nif); - rt.dst_hw = res.value(); + if (IS_ERR(rt.dst_hw)) [[unlikely]] + return unexpected{PTR_ERR(rt.dst_hw)}; return rt; } @@ -516,6 +520,16 @@ int inet_socket::setsockopt_inet6(int level, int opt, const void *optval, sockle this->ttl = ttl; return 0; } + + case IPV6_V6ONLY: { + auto ex = get_socket_option(optval, len); + + if (ex.has_error()) + return ex.error(); + + ipv6_only = ex.value() ? 1 : 0; + return 0; + } } return -ENOPROTOOPT; @@ -528,7 +542,27 @@ int inet_socket::getsockopt_inet6(int level, int opt, void *optval, socklen_t *l case IPV6_MULTICAST_HOPS: case IPV6_UNICAST_HOPS: return put_option(ttl, optval, len); + case IPV6_V6ONLY: { + int val = ipv6_only; + return put_option(val, optval, len); + } } return -ENOPROTOOPT; } + +int ip6_finish_output(struct neighbour *neigh, struct packetbuf *pbf, struct netif *nif) +{ + int st = 0; + const void *hwaddr = nullptr; + + if (neigh->flags & NUD_FAILED) + return -EHOSTUNREACH; + hwaddr = neigh->hwaddr; + + if ((st = nif->dll_ops->setup_header(pbf, tx_type::unicast, tx_protocol::ipv6, nif, hwaddr)) < + 0) [[unlikely]] + return st; + + return netif_send_packet(nif, pbf); +} diff --git a/kernel/kernel/net/ipv6/ndp.cpp b/kernel/kernel/net/ipv6/ndp.cpp index 058e5827a..f19712735 100644 --- a/kernel/kernel/net/ipv6/ndp.cpp +++ b/kernel/kernel/net/ipv6/ndp.cpp @@ -1,7 +1,9 @@ /* - * Copyright (c) 2020 Pedro Falcato + * Copyright (c) 2020 - 2024 Pedro Falcato * This file is part of Onyx, and is released under the terms of the GPLv2 License * check LICENSE at the root directory for more information + * + * SPDX-License-Identifier: GPL-2.0-only */ #include #include @@ -12,6 +14,7 @@ #include #include +#include #include #include #include @@ -36,12 +39,17 @@ struct icmp6_source_link_layer_opt unsigned char hwaddr[]; }; -/* TODO: Maybe the neighbour_table could replace some code below, if we add a few virtual functions - */ - /* FIXME: The ndp table should be per interface */ static neighbour_table ndp_table{AF_INET6}; + +int ndp_submit_request(struct neighbour *neigh, struct netif *netif); + +const struct neigh_ops ndp_ops = { + .resolve = ndp_submit_request, + .output = ip6_finish_output, +}; + static constexpr hrtime_t ndp_response_timeout = 250 * NS_PER_MS; /* 20 minutes in milis */ @@ -49,17 +57,19 @@ static constexpr unsigned long ndp_validity_time_ms = 1200000; int ndp_handle_na(netif *netif, packetbuf *buf) { + (void) ndp_response_timeout; + (void) ndp_validity_time_ms; if (buf->length() < sizeof(nd_neighbor_advert)) return -EINVAL; auto ndp = (struct nd_neighbor_advert *) buf->data; - + int added; neigh_proto_addr addr; addr.in6addr = ndp->nd_na_target; - auto [ptr, __created] = ndp_table.add(addr, true); - if (!ptr) - return 0; + struct neighbour *neigh = neigh_add(&ndp_table, &addr, GFP_ATOMIC, &ndp_ops, &added); + if (!neigh) + return -ENOMEM; const char *optptr = (const char *) (ndp + 1); ssize_t options_len = buf->length() - sizeof(nd_neighbor_solicit); @@ -91,15 +101,7 @@ int ndp_handle_na(netif *netif, packetbuf *buf) if (!target) return 0; - unsigned char *mac = new unsigned char[ETH_ALEN]; - if (!mac) - return 0; - memcpy(mac, target, ETH_ALEN); - - cul::slice sl{mac, ETH_ALEN}; - ptr->set_hwaddr(sl); - ptr->flags |= NEIGHBOUR_FLAG_HAS_RESPONSE; - + neigh_complete_lookup(neigh, target, ETH_ALEN); return 0; } @@ -183,24 +185,13 @@ in6_addr solicited_node_address(const in6_addr &our_address) return ret; } -int ndp_submit_request(shared_ptr &ptr, const in6_addr &target_addr, struct netif *netif) +int ndp_submit_request(struct neighbour *neigh, struct netif *netif) { + const in6_addr &target_addr = neigh->proto_addr.in6addr; if (target_addr == in6addr_loopback || netif->flags & NETIF_LOOPBACK) { - auto _ptr = new unsigned char[ETH_ALEN]; - - if (_ptr) - { - memset(_ptr, 0, ETH_ALEN); - auto sl = cul::slice{_ptr, ETH_ALEN}; - ptr->set_hwaddr(sl); - ptr->flags |= NEIGHBOUR_FLAG_HAS_RESPONSE; - } - else - { - return -ENOMEM; - } - + const unsigned char loopback_eth[ETH_ALEN] = {}; + __neigh_complete_lookup(neigh, loopback_eth, ETH_ALEN); return 0; } @@ -229,48 +220,18 @@ int ndp_submit_request(shared_ptr &ptr, const in6_addr &target_addr, sizeof(buf_) - sizeof(icmp6_hdr)}); } -expected, int> ndp_resolve(const in6_addr &ip, struct netif *netif) +struct neighbour *ndp_resolve(const in6_addr &ip, struct netif *netif) { - neigh_proto_addr addr; + struct neighbour *neigh; + int added; + union neigh_proto_addr addr; addr.in6addr = ip; - auto [ptr, created] = ndp_table.add(addr); - if (!ptr) - return unexpected{-ENOMEM}; - - if (ptr->flags & NEIGHBOUR_FLAG_UNINITIALISED) - { - if (created) - { - if (ndp_submit_request(ptr, ip, netif) < 0) - { - ndp_table.remove(ptr.get_data()); - return unexpected{-ENOMEM}; - } - } - - auto t0 = clocksource_get_time(); - - /* TODO: Add a wait_for_bit that can let us wait for random things - * without taking up permanent space in the structure - */ - while (!(ptr->flags & NEIGHBOUR_FLAG_HAS_RESPONSE) && - clocksource_get_time() - t0 <= ndp_response_timeout) - sched_sleep_ms(15); - - if (!(ptr->flags & NEIGHBOUR_FLAG_HAS_RESPONSE)) - { - if (created) - ndp_table.remove(ptr.get_data()); - return unexpected{-ENETUNREACH}; - } - - if (created) - { - ptr->set_validity(ndp_validity_time_ms); - ptr->set_initialised(); - } - } + neigh = neigh_add(&ndp_table, &addr, GFP_ATOMIC, &ndp_ops, &added); + if (!neigh) + return (struct neighbour *) ERR_PTR(-ENOMEM); - return ptr; + if (neigh_needs_resolve(neigh)) + neigh_start_resolve(neigh, netif); + return neigh; } diff --git a/kernel/kernel/net/neighbour.cpp b/kernel/kernel/net/neighbour.cpp index 11e6f1b41..8887139cf 100644 --- a/kernel/kernel/net/neighbour.cpp +++ b/kernel/kernel/net/neighbour.cpp @@ -1,86 +1,187 @@ /* - * Copyright (c) 2020 Pedro Falcato + * Copyright (c) 2020 - 2024 Pedro Falcato * This file is part of Onyx, and is released under the terms of the GPLv2 License * check LICENSE at the root directory for more information + * + * SPDX-License-Identifier: GPL-2.0-only */ +#include #include +#include +#include #include +#include #include -#include -#include - void neighbour_revalidate(clockevent* ev) { // TODO: Implement ev->deadline = clocksource_get_time() + NS_PER_SEC * 60 * 20; } -cul::pair, bool> neighbour_table::add(const neigh_proto_addr& addr, - bool only_lookup) +struct neighbour* neigh_find(struct neighbour_table* table, const union neigh_proto_addr* addr) { - scoped_lock g{lock}; + struct neighbour* neigh; + u32 hash = hash_protoaddr(*addr, table->domain); + u32 index = hash & (NEIGH_TAB_NR_CHAINS - 1); - auto hash = hash_protoaddr(addr, domain); + rcu_read_lock(); - auto it = neighbour_cache.get_hash_list_begin(hash); - auto end = neighbour_cache.get_hash_list_end(hash); + list_for_each_entry_rcu (neigh, &table->neigh_tab[index], list_node) + { + if (neigh->addr_equals(*addr)) + { + if (neigh_get_careful(neigh)) + goto out; + } + } - while (it != end) + neigh = NULL; +out: + rcu_read_unlock(); + return neigh; +} + +void neigh_free(struct neighbour* neigh) +{ + kfree_rcu(neigh, rcu_head); +} + +struct neighbour* neigh_add(struct neighbour_table* table, const union neigh_proto_addr* addr, + gfp_t gfp, const struct neigh_ops* ops, int* added) +{ + struct neighbour *neigh, *n2; + u32 hash, index; + *added = 1; + + neigh = neigh_find(table, addr); + if (neigh) { - auto neigh = *it; - auto n_hwaddr = neigh->hwaddr(); - if (neigh->addr_equals(addr)) - return {neigh, false}; - it++; + *added = 0; + return neigh; } - if (only_lookup) - return {nullptr, false}; + neigh = (struct neighbour*) kmalloc(sizeof(*neigh), gfp); + if (!neigh) + return NULL; - auto ptr = make_shared(domain, addr); + hash = hash_protoaddr(*addr, table->domain); + index = hash & (NEIGH_TAB_NR_CHAINS - 1); - if (!ptr) - return {nullptr, false}; + new (neigh) neighbour(table->domain, *addr); + neigh->neigh_ops = ops; + spin_lock(&table->lock); - ptr->flags |= NEIGHBOUR_FLAG_UNINITIALISED; + /* No need for _rcu since we hold the lock */ + list_for_each_entry (n2, &table->neigh_tab[index], list_node) + { + if (n2->addr_equals(*addr)) + { + /* We can skip neigh_get_careful here since we hold the spinlock */ + neigh_get(n2); + spin_unlock(&table->lock); + kfree(neigh); + *added = 0; + return n2; + } + } - if (!neighbour_cache.add_element(ptr)) - return {nullptr, false}; + /* Not found, add */ + neigh_get(neigh); + list_add_tail_rcu(&neigh->list_node, &table->neigh_tab[index]); - return {ptr, true}; + spin_unlock(&table->lock); + return neigh; } -void neighbour_table::remove(neighbour* ptr) +void neigh_remove(struct neighbour_table* table, struct neighbour* neigh) { - scoped_lock g{lock}; - auto& hw = ptr->proto_addr; - auto hash = hash_protoaddr(hw, domain); - - auto it = neighbour_cache.get_hash_list_begin(hash); - auto end = neighbour_cache.get_hash_list_end(hash); + spin_lock(&table->lock); + list_remove_rcu(&neigh->list_node); + spin_unlock(&table->lock); + neigh_put(neigh); +} - while (it != end) +static void neigh_clear_chain(struct neighbour_table* table, u32 i) +{ + struct neighbour* n; + list_for_each_entry (n, &table->neigh_tab[i], list_node) { - auto neigh = *it; - if (neigh == ptr) - { - neighbour_cache.remove_element(neigh, hash, it); - return; - } + list_remove(&n->list_node); + neigh_put(n); + } +} + +void neigh_clear(struct neighbour_table* table) +{ + spin_lock(&table->lock); + + for (u32 i = 0; i < NEIGH_TAB_NR_CHAINS; i++) + neigh_clear_chain(table, i); + + spin_unlock(&table->lock); +} + +void neigh_start_resolve(struct neighbour* neigh, struct netif* nif) +{ + write_seqlock(&neigh->neigh_seqlock); + + if (neigh_needs_resolve(neigh)) + neigh->neigh_ops->resolve(neigh, nif); - it++; + if (neigh->flags & NUD_STALE) + { + neigh->flags &= ~NUD_STALE; + neigh->flags |= NUD_PROBE; + } + else + { + neigh->flags |= NUD_INCOMPLETE; } + + write_sequnlock(&neigh->neigh_seqlock); } -void neighbour_table::clear_cache() +int neigh_output(struct neighbour* neigh, struct packetbuf* pbf, struct netif* nif) { - scoped_lock g{lock}; + CHECK(pbf->route.nif); + unsigned int flags = READ_ONCE(neigh->flags); + if (likely(flags & NUD_REACHABLE)) + return neigh->neigh_ops->output(neigh, pbf, nif); + /* Slow path - check the neighbour's state, try to resolve it and queue our own packet. Per + * RFC1122: The link layer SHOULD save (rather than discard) at least one (the latest) + * packet of each set of packets destined to the same unresolved IP address, and transmit + * the saved packet when the address has been resolved. + */ + + if (flags & NUD_REACHABLE) + { + /* Just send it */ + return neigh->neigh_ops->output(neigh, pbf, nif); + } + + /* Probe pending (or will be). Append our packet and leave. This requires the lock. */ + spin_lock(&neigh->neigh_seqlock.lock); + list_add_tail(&pbf->list_node, &neigh->packet_queue); + pbf_get(pbf); + spin_unlock(&neigh->neigh_seqlock.lock); + + if (flags & (NUD_PROBE | NUD_INCOMPLETE)) + return 0; - neighbour_cache.empty(); + /* Not reachable nor probe nor incomplete - we don't have a probe. Start a resolve. */ + neigh_start_resolve(neigh, nif); + return 0; } -fnv_hash_t hash_neighbour(shared_ptr& neigh) +void neigh_output_queued(struct neighbour* neigh) { - return hash_protoaddr(neigh->proto_addr, neigh->get_domain()); + struct packetbuf *pbf, *next; + list_for_each_entry_safe (pbf, next, &neigh->packet_queue, list_node) + { + CHECK(pbf->route.nif != NULL); + list_remove(&pbf->list_node); + neigh->neigh_ops->output(neigh, pbf, pbf->route.nif); + pbf_put_ref(pbf); + } } From 437fcb34012347cf1dcb329808e508ffa13d2d12 Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Wed, 18 Sep 2024 02:11:20 +0100 Subject: [PATCH 05/76] netif: Fix RX path Rework the RX netif logic. This should be safe(r) and probably faster. Signed-off-by: Pedro Falcato --- kernel/include/onyx/net/netif.h | 4 +-- kernel/kernel/net/netif.cpp | 58 ++++++++++----------------------- 2 files changed, 19 insertions(+), 43 deletions(-) diff --git a/kernel/include/onyx/net/netif.h b/kernel/include/onyx/net/netif.h index 1f87763de..511432048 100644 --- a/kernel/include/onyx/net/netif.h +++ b/kernel/include/onyx/net/netif.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017 - 2022 Pedro Falcato + * Copyright (c) 2017 - 2024 Pedro Falcato * This file is part of Onyx, and is released under the terms of the GPLv2 License * check LICENSE at the root directory for more information * @@ -14,7 +14,6 @@ #include #include struct netif; -#include #include #include @@ -29,7 +28,6 @@ struct netif; #define NETIF_LOOPBACK (1 << 5) #define NETIF_HAS_RX_AVAILABLE (1 << 6) #define NETIF_DOING_RX_POLL (1 << 7) -#define NETIF_MISSED_RX (1 << 8) struct packetbuf; diff --git a/kernel/kernel/net/netif.cpp b/kernel/kernel/net/netif.cpp index 7d9c4b929..ff1bd6a0f 100644 --- a/kernel/kernel/net/netif.cpp +++ b/kernel/kernel/net/netif.cpp @@ -13,10 +13,9 @@ #include #include #include +#include #include #include -#include -#include #include #include #include @@ -126,10 +125,6 @@ void netif_register_if(struct netif *netif) { INIT_LIST_HEAD(&netif->inet6_addr_list); - assert(udp_init_netif(netif) == 0); - - assert(tcp_init_netif(netif) == 0); - auto ex = dev_register_chardevs(0, 1, 0, &netif_fops, netif->name); if (ex.has_error()) panic("netif_register_if failed"); @@ -290,9 +285,6 @@ void netif_signal_rx(netif *nif) flags |= NETIF_HAS_RX_AVAILABLE; - if (og_flags & NETIF_DOING_RX_POLL) - flags |= NETIF_MISSED_RX; - } while (!__atomic_compare_exchange_n(&nif->flags, &og_flags, flags, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)); @@ -302,59 +294,45 @@ void netif_signal_rx(netif *nif) auto queue = get_per_cpu_ptr(rx_queue); unsigned long cpu_flags = spin_lock_irqsave(&queue->lock); - list_add_tail(&nif->rx_queue_node, &queue->to_rx_list); - spin_unlock_irqrestore(&queue->lock, cpu_flags); - softirq_raise(softirq_vector::SOFTIRQ_VECTOR_NETRX); } void netif_do_rxpoll(netif *nif) { - __atomic_or_fetch(&nif->flags, NETIF_DOING_RX_POLL, __ATOMIC_RELAXED); + unsigned int desired; + atomic_or_relaxed(nif->flags, NETIF_DOING_RX_POLL); - while (true) + do { + retry: + atomic_and_relaxed(nif->flags, ~NETIF_HAS_RX_AVAILABLE); nif->poll_rx(nif); - - unsigned int flags, og_flags; - - do - { - og_flags = flags = nif->flags; - - if (!(og_flags & NETIF_MISSED_RX)) - { - nif->rx_end(nif); - flags &= ~(NETIF_HAS_RX_AVAILABLE | NETIF_DOING_RX_POLL); - } - - flags &= ~NETIF_MISSED_RX; - - } while (!__atomic_compare_exchange_n(&nif->flags, &og_flags, flags, false, - __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)); - - if (!(flags & NETIF_DOING_RX_POLL)) - break; - } + desired = READ_ONCE(nif->flags); + if (desired & NETIF_HAS_RX_AVAILABLE) + goto retry; + nif->rx_end(nif); + } while (cmpxchg(&nif->flags, desired, desired & ~NETIF_DOING_RX_POLL) != desired); } int netif_do_rx() { auto queue = get_per_cpu_ptr(rx_queue); + DEFINE_LIST(to_rx); - scoped_lock g{queue->lock}; + { + scoped_lock g{queue->lock}; + list_splice(&queue->to_rx_list, &to_rx); + } - list_for_every (&queue->to_rx_list) + list_for_every_safe (&to_rx) { netif *n = container_of(l, netif, rx_queue_node); - + list_remove(&n->rx_queue_node); netif_do_rxpoll(n); } - list_reset(&queue->to_rx_list); - return 0; } From c0ad04d8e04ca42e6898f9972a638b7fe8b1f439 Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Tue, 24 Sep 2024 00:58:53 +0100 Subject: [PATCH 06/76] vterm: Escape sequence fixes Make our terminal less buggy, A LOT FASTER scrolling, and add more sequences such that it's more similar to TERM=linux. Signed-off-by: Pedro Falcato --- kernel/include/onyx/tty.h | 33 +-- kernel/kernel/tty/vt/vterm.cpp | 461 +++++++++++++++++++++------------ 2 files changed, 320 insertions(+), 174 deletions(-) diff --git a/kernel/include/onyx/tty.h b/kernel/include/onyx/tty.h index d7dbdf406..434b573e1 100644 --- a/kernel/include/onyx/tty.h +++ b/kernel/include/onyx/tty.h @@ -186,6 +186,7 @@ __END_CDECLS #define ANSI_SAVE_CURSOR 's' #define ANSI_RESTORE_CURSOR 'u' #define CSI_DELETE_CHARS 'P' +#define CSI_ERASE_CHARS 'X' #define CSI_INSERT_BLANK '@' #define CSI_INSERT_LINE 'L' #define CSI_DELETE_LINE 'M' @@ -197,21 +198,23 @@ __END_CDECLS #define ESC_SAVECUR '7' #define ESC_RESTORECUR '8' -#define ANSI_SGR_RESET 0 -#define ANSI_SGR_BOLD 1 -#define ANSI_SGR_FAINT 2 -#define ANSI_SGR_ITALIC 3 -#define ANSI_SGR_UNDERLINE 4 -#define ANSI_SGR_SLOWBLINK 5 -#define ANSI_SGR_RAPIDBLINK 6 -#define ANSI_SGR_REVERSE 7 -#define ANSI_SGR_BLINKOFF 25 -#define ANSI_SGR_SETFGMIN 30 -#define ANSI_SGR_SETFGMAX 37 -#define ANSI_SGR_DEFAULTFG 39 -#define ANSI_SGR_SETBGMIN 40 -#define ANSI_SGR_SETBGMAX 47 -#define ANSI_SGR_DEFAULTBG 49 +#define ANSI_SGR_RESET 0 +#define ANSI_SGR_BOLD 1 +#define ANSI_SGR_FAINT 2 +#define ANSI_SGR_ITALIC 3 +#define ANSI_SGR_UNDERLINE 4 +#define ANSI_SGR_SLOWBLINK 5 +#define ANSI_SGR_RAPIDBLINK 6 +#define ANSI_SGR_REVERSE 7 +#define ANSI_SGR_NOUNDERLINE 24 +#define ANSI_SGR_BLINKOFF 25 +#define ANSI_SGR_NOREVERSE 27 +#define ANSI_SGR_SETFGMIN 30 +#define ANSI_SGR_SETFGMAX 37 +#define ANSI_SGR_DEFAULTFG 39 +#define ANSI_SGR_SETBGMIN 40 +#define ANSI_SGR_SETBGMAX 47 +#define ANSI_SGR_DEFAULTBG 49 #define ANSI_COLOR_RED "\x1b[31m" #define ANSI_COLOR_GREEN "\x1b[32m" diff --git a/kernel/kernel/tty/vt/vterm.cpp b/kernel/kernel/tty/vt/vterm.cpp index 7abd7359c..e271559d0 100644 --- a/kernel/kernel/tty/vt/vterm.cpp +++ b/kernel/kernel/tty/vt/vterm.cpp @@ -47,18 +47,18 @@ struct color uint8_t a; }; -static struct color default_fg = {.r = 0xaa, .g = 0xaa, .b = 0xaa}; +static struct color default_fg = {204, 204, 204}; static struct color default_bg = {}; const struct color color_table[] = { - {.r = 0, .g = 0, .b = 0}, /* Black */ - {.r = 0xff}, /* Red */ - {.g = 0xff}, /* Green */ - {.r = 0xff, .g = 0xff}, /* Yellow */ - {.b = 0xff}, /* Blue */ - {.r = 0xff, .b = 0xff}, /* Magenta */ - {.g = 0xff, .b = 0xff}, /* Cyan */ - {.r = 0xaa, .g = 0xaa, .b = 0xaa} /* White */ + {.r = 0, .g = 0, .b = 0}, /* Black */ + {.r = 205, .g = 49, .b = 49}, /* Red */ + {.r = 19, .g = 161, .b = 14}, /* Green */ + {229, 229, 16}, /* Yellow */ + {36, 114, 200}, /* Blue */ + {188, 63, 188}, /* Magenta */ + {17, 168, 205}, /* Cyan */ + {204, 204, 204} /* White */ }; #define VTERM_CONSOLE_CELL_DIRTY (1 << 0) @@ -120,6 +120,11 @@ struct vterm struct cond condvar; struct mutex condvar_mutex; struct vterm_message *msgs; + bool reversed; + bool flush_all; + bool numlck; + unsigned long *dirty_row_bitmap; + unsigned int bitmap_size; // Buffer used for any multibyte buffering for utf8 char multibyte_buffer[10]; @@ -157,6 +162,7 @@ struct vterm void do_ri(); void do_nl(); void do_cr(); + void delete_lines(unsigned long nr); }; void vterm_append_msg(struct vterm *vterm, struct vterm_message *msg) ACQUIRE(vterm->condvar_mutex) @@ -191,6 +197,15 @@ void vterm_send_message(struct vterm *vterm, unsigned long message, void *ctx) mutex_unlock(&vterm->condvar_mutex); } +#define LONG_SIZE_BITS __LONG_WIDTH__ + +static inline void vterm_dirty_cell(unsigned int x, unsigned int y, struct vterm *vt) +{ + struct console_cell *cell = &vt->cells[y * vt->columns + x]; + vterm_set_dirty(cell); + vt->dirty_row_bitmap[y / LONG_SIZE_BITS] |= (1UL << (y % LONG_SIZE_BITS)); +} + static inline uint32_t unpack_rgba(struct color color, struct framebuffer *fb) { uint32_t c = ((color.r << fb->color.red_shift) & fb->color.red_mask) | @@ -270,21 +285,77 @@ void vterm_flush_all(struct vterm *vterm) do_vterm_flush_all(vterm); } +static inline bool same_colour(const struct color *c1, const struct color *c2) +{ + return c1->a == c2->a && c1->r == c2->r && c1->g == c2->g && c1->b == c2->b; +} + +static void vterm_clear_range(struct vterm *vt, unsigned int start_x, unsigned int start_y, + unsigned int end_x, unsigned int end_y) +{ + unsigned int x = start_x, y = start_y; + unsigned int len; + + if (start_y == end_y) + len = end_x - start_x; + else + len = (vt->columns - start_x) + end_x + (end_y - start_y - 1) * vt->columns; + + struct console_cell *cell = vt->cells + (y * vt->columns) + x; + + for (unsigned int i = 0; i < len; i++) + { + CHECK(cell < vt->cells + (vt->columns * vt->rows)); + if (cell->codepoint != ' ' || !same_colour(&cell->bg, &vt->bg) || + !same_colour(&cell->fg, &vt->fg)) + { + cell->codepoint = ' '; + cell->bg = vt->bg; + cell->fg = vt->fg; + vterm_dirty_cell(x, y, vt); + } + + cell++; + if (unlikely(++x == vt->columns)) + { + x = 0; + y++; + } + } +} + static void __vterm_scroll(struct framebuffer *fb, struct vterm *vt, unsigned int nr_lines, unsigned int top, unsigned int bottom) { + if (top + nr_lines >= bottom) + nr_lines = (bottom - top) - 1; + unsigned int start = vt->columns * top; unsigned int dest = vt->columns * (top + nr_lines); unsigned int end = vt->columns * (bottom - nr_lines); - memcpy(vt->cells + start, vt->cells + dest, sizeof(struct console_cell) * (end - start)); + unsigned int x = 0, y = top; + + if (bottom > vt->rows || top >= bottom || nr_lines < 1) + return; - for (unsigned int i = 0; i < vt->columns * nr_lines; i++) + for (unsigned int i = 0; i < end - start; i++) { - struct console_cell *c = &vt->cells[(bottom - nr_lines) * vt->columns + i]; - c->codepoint = ' '; - c->bg = vt->bg; - c->fg = vt->fg; + struct console_cell *dst = vt->cells + start + i; + struct console_cell *src = vt->cells + dest + i; + if (dst->codepoint != src->codepoint || !same_colour(&dst->bg, &src->bg) || + !same_colour(&dst->fg, &src->fg)) + vterm_dirty_cell(x, y, vt); + dst->bg = src->bg; + dst->fg = src->fg; + dst->codepoint = src->codepoint; + if (++x == vt->columns) + { + x = 0; + y++; + } } + + vterm_clear_range(vt, 0, bottom - nr_lines, 0, bottom); } static void vterm_scroll(struct framebuffer *fb, struct vterm *vt) @@ -295,15 +366,15 @@ static void vterm_scroll(struct framebuffer *fb, struct vterm *vt) static void __vterm_scroll_down(struct framebuffer *fb, struct vterm *vt, unsigned int nr, unsigned int top, unsigned int bottom) { - unsigned int start = vt->columns * (top + nr); - unsigned int dest = vt->columns * top; - unsigned int end = vt->columns * (bottom - nr); - DCHECK(end > start); - memmove(vt->cells + start, vt->cells + dest, sizeof(struct console_cell) * (end - start)); + unsigned int src, clear, dst; + src = clear = top * vt->columns; + dst = (top + nr) * vt->columns; + memmove(vt->cells + dst, vt->cells + src, + (bottom - top - nr) * vt->columns * sizeof(struct console_cell)); for (unsigned int i = 0; i < vt->columns * nr; i++) { - struct console_cell *c = &vt->cells[dest + i]; + struct console_cell *c = &vt->cells[clear + i]; c->codepoint = ' '; c->bg = vt->bg; c->fg = vt->fg; @@ -315,20 +386,20 @@ static void vterm_scroll_down(struct framebuffer *fb, struct vterm *vt) __vterm_scroll_down(fb, vt, 1, vt->top, vt->bottom); } +static inline bool vterm_needs_dirty(struct console_cell *cell, utf32_t c, struct color fg, + struct color bg) +{ + return c != cell->codepoint || !same_colour(&fg, &cell->fg) || !same_colour(&bg, &cell->bg); +} + void vterm_set_char(utf32_t c, unsigned int x, unsigned int y, struct color fg, struct color bg, struct vterm *vterm) { struct console_cell *cell = &vterm->cells[y * vterm->columns + x]; + vterm_dirty_cell(x, y, vterm); cell->codepoint = c; cell->fg = fg; cell->bg = bg; - vterm_set_dirty(cell); -} - -void vterm_dirty_cell(unsigned int x, unsigned int y, struct vterm *vt) -{ - struct console_cell *cell = &vt->cells[y * vt->columns + x]; - vterm_set_dirty(cell); } bool vterm_putc(utf32_t c, struct vterm *vt) @@ -483,17 +554,25 @@ void vterm_flush(struct vterm *vterm); void do_vterm_flush(struct vterm *vterm) { struct font *f = get_font_data(); - for (unsigned int i = 0; i < vterm->columns; i++) + int base_row = 0; + for (unsigned int i = 0; i < vterm->bitmap_size; i++, base_row += LONG_SIZE_BITS) { - for (unsigned int j = 0; j < vterm->rows; j++) + while (vterm->dirty_row_bitmap[i] != 0) { - struct console_cell *cell = &vterm->cells[j * vterm->columns + i]; + int bit = __builtin_ffsl(vterm->dirty_row_bitmap[i]) - 1; + int row = bit + base_row; + vterm->dirty_row_bitmap[i] &= ~(1UL << bit); - if (vterm_is_dirty(cell)) + for (unsigned int j = 0; j < vterm->columns; j++) { - draw_char(cell->codepoint, i * f->width, j * f->height, vterm->fb, cell->fg, - cell->bg); - vterm_clear_dirty(cell); + struct console_cell *cell = &vterm->cells[row * vterm->columns + j]; + + if (vterm_is_dirty(cell)) + { + draw_char(cell->codepoint, j * f->width, row * f->height, vterm->fb, cell->fg, + cell->bg); + vterm_clear_dirty(cell); + } } } } @@ -503,6 +582,13 @@ void platform_serial_write(const char *s, size_t size); void vterm_flush(struct vterm *vterm) { + if (vterm->flush_all) + { + do_vterm_flush_all(vterm); + vterm->flush_all = false; + return; + } + if (vterm->multithread_enabled) vterm_send_message(vterm, VTERM_MESSAGE_FLUSH, NULL); else @@ -516,10 +602,10 @@ void vterm_fill_screen(struct vterm *vterm, uint32_t character, struct color fg, for (unsigned int j = 0; j < vterm->rows; j++) { struct console_cell *cell = &vterm->cells[i * vterm->rows + j]; + vterm_dirty_cell(i, j, vterm); cell->codepoint = character; cell->bg = bg; cell->fg = fg; - vterm_set_dirty(cell); } } } @@ -672,13 +758,27 @@ void vterm_ansi_do_sgr(unsigned long n, struct vterm *vt) case ANSI_SGR_RESET: { vt->bg = default_bg; vt->fg = default_fg; + vt->reversed = false; break; } case ANSI_SGR_REVERSE: { + if (vt->reversed) + return; struct color temp = vt->bg; vt->bg = vt->fg; vt->fg = temp; + vt->reversed = true; + break; + } + + case ANSI_SGR_NOREVERSE: { + if (!vt->reversed) + return; + struct color temp = vt->bg; + vt->bg = vt->fg; + vt->fg = temp; + vt->reversed = false; break; } @@ -737,42 +837,19 @@ void vterm_ansi_erase_in_line(unsigned long n, struct vterm *vt) { /* Clear from cursor to end */ case 0: { - for (unsigned int i = vt->cursor_x; i < vt->columns; i++) - { - struct console_cell *c = &vt->cells[vt->cursor_y * vt->columns + i]; - c->codepoint = ' '; - c->fg = vt->fg; - c->bg = vt->bg; - vterm_set_dirty(c); - } + vterm_clear_range(vt, vt->cursor_x, vt->cursor_y, vt->columns, vt->cursor_y); break; } /* Clear from cursor to beginning */ case 1: { - unsigned int x = vt->cursor_x; - - for (unsigned int i = 0; i <= x; i++) - { - struct console_cell *c = &vt->cells[vt->cursor_y * vt->columns + i]; - c->codepoint = ' '; - c->fg = vt->fg; - c->bg = vt->bg; - vterm_set_dirty(c); - } + vterm_clear_range(vt, 0, vt->cursor_y, vt->cursor_x + 1, vt->cursor_y); break; } /* Clear entire line */ case 2: { - for (unsigned int i = 0; i < vt->columns; i++) - { - struct console_cell *c = &vt->cells[vt->cursor_y * vt->columns + i]; - c->codepoint = ' '; - c->fg = vt->fg; - c->bg = vt->bg; - vterm_set_dirty(c); - } + vterm_clear_range(vt, 0, vt->cursor_y, 0, vt->cursor_y + 1); break; } } @@ -784,39 +861,13 @@ void vterm_ansi_erase_in_display(unsigned long n, struct vterm *vt) { /* Cursor to end of display */ case 0: { - /* Calculate the cidx, then loop through until the end of the array */ - unsigned int cidx = vt->cursor_y * vt->columns + vt->cursor_x; - unsigned int max = vt->rows * vt->columns; - if (cidx + 1 < cidx) - break; - - unsigned int iters = max - cidx; - - for (unsigned int i = 0; i < iters; i++) - { - struct console_cell *c = &vt->cells[cidx + i]; - c->codepoint = ' '; - c->fg = vt->fg; - c->bg = vt->bg; - vterm_set_dirty(c); - } - + vterm_clear_range(vt, vt->cursor_x, vt->cursor_y, 0, vt->rows); break; } /* Cursor to start of display */ case 1: { - unsigned int cidx = vt->cursor_y * vt->columns + vt->cursor_x; - - for (unsigned int i = 0; i <= cidx; i++) - { - struct console_cell *c = &vt->cells[i]; - c->codepoint = ' '; - c->fg = vt->fg; - c->bg = vt->bg; - vterm_set_dirty(c); - } - + vterm_clear_range(vt, 0, 0, vt->cursor_x + 1, vt->cursor_y); break; } @@ -853,7 +904,7 @@ void vterm_csi_delete_chars(unsigned long chars, struct vterm *vt) c->bg = vt->bg; } - vterm_set_dirty(c); + vterm_dirty_cell(i, vt->cursor_y, vt); } } @@ -874,7 +925,7 @@ void vterm::insert_blank(unsigned long nr) cell.bg = bg; } - vterm_set_dirty(&cell); + vterm_dirty_cell(i, cursor_y, this); } } @@ -1114,6 +1165,12 @@ void vterm::process_escape_char(char c) } } +template +static inline T clamp(T val, T min, T max) +{ + return cul::min(cul::max(val, min), max); +} + void vterm::do_csi_command(char escape) { if (csi_data.dec_private) @@ -1148,7 +1205,7 @@ void vterm::do_csi_command(char escape) case ANSI_CURSOR_HORIZONTAL_ABS: { if (args[0] > columns - 1) args[0] = columns - 1; - cursor_x = args[0]; + cursor_x = args[0] - 1; break; } @@ -1166,7 +1223,6 @@ void vterm::do_csi_command(char escape) case ANSI_SCROLL_UP: { for (unsigned long i = 0; i < args[0]; i++) vterm_scroll(fb, this); - vterm_flush_all(this); break; } @@ -1256,7 +1312,7 @@ void vterm::do_csi_command(char escape) if (args[0] == 0) args[0] = 1; - // delete_lines(args[0]); + delete_lines(args[0]); break; } @@ -1277,12 +1333,21 @@ void vterm::do_csi_command(char escape) { top = args[0] - 1; bottom = args[1]; + vterm_dirty_cell(cursor_x, cursor_y, this); + cursor_x = 0; + cursor_y = 0; } break; } + case CSI_ERASE_CHARS: { + unsigned long nr = clamp(args[0], 1, columns - cursor_x); + vterm_clear_range(this, cursor_x, cursor_y, cursor_x + nr, cursor_y); + break; + } + default: { - // printf("Unimplemented escape %c\n", escape); + // pr_info("vt: Unimplemented escape %c\n", escape); break; } } @@ -1309,6 +1374,12 @@ void vterm::insert_lines(unsigned long nr) vterm_flush_all(this); } +void vterm::delete_lines(unsigned long nr) +{ + nr = clamp(nr, 1, rows - cursor_y); + __vterm_scroll(fb, this, nr, cursor_y, bottom); +} + size_t vterm::do_escape(const char *buffer, size_t len) { size_t processed = 0; @@ -1327,32 +1398,23 @@ size_t vterm::do_escape(const char *buffer, size_t len) char escape = csi_data.escape_character; #if 0 - char buf[50]; if (in_csi) - snprintf(buf, 50, "Seq: %c nargs %lu args {%lu, %lu}\n", escape, csi_data.nr_args, - csi_data.args[0], csi_data.args[1]); - if (in_csi && !csi_data.dec_private) - platform_serial_write(buf, strlen(buf)); - // platform_serial_write("Seq: ", strlen("Seq: ")); - // platform_serial_write(&escape, 1); - // platform_serial_write("\n", 1); + pr_info("Seq: %c nargs %lu args {%lu, %lu}\n", escape, csi_data.nr_args, csi_data.args[0], + csi_data.args[1]); + else if (in_dec) + pr_info("doing DEC escape %c\n", escape); + else + pr_info("doing generic escape %c\n", escape); #endif if (in_dec) - { do_dec_command(escape); - } else if (in_csi) - { do_csi_command(escape); - } else - { do_generic_escape(escape); - } reset_escape_status(); - return processed; } @@ -1364,7 +1426,6 @@ ssize_t vterm_write_tty(const void *buffer, size_t size, struct tty *tty) mutex_lock(&vt->vt_lock); size_t i = 0; const char *data = (const char *) buffer; - bool did_scroll = false; for (; i < size; i++) { @@ -1393,18 +1454,14 @@ ssize_t vterm_write_tty(const void *buffer, size_t size, struct tty *tty) platform_serial_write(x, strlen(x)); #endif // platform_serial_write(data + i, 1); - if (vterm_putc(codepoint, vt)) - did_scroll = true; + vterm_putc(codepoint, vt); /* We sub a 1 because we're incrementing on the for loop */ i += codepoint_length - 1; } } - if (!did_scroll) - vterm_flush(vt); - else - vterm_flush_all(vt); + vterm_flush(vt); update_cursor(vt); mutex_unlock(&vt->vt_lock); @@ -1456,7 +1513,12 @@ void vterm_init(struct tty *tty) vt->fg = default_fg; vt->bg = default_bg; - assert(vt->cells != NULL); + int bitmap_size = + vt->rows / (sizeof(unsigned long) * 8) + ((vt->rows % (sizeof(unsigned long) * 8)) != 0); + vt->dirty_row_bitmap = + (unsigned long *) kcalloc(bitmap_size, sizeof(unsigned long), GFP_KERNEL); + CHECK(vt->dirty_row_bitmap); + vt->bitmap_size = bitmap_size; vterm_fill_screen(vt, ' ', vt->fg, vt->bg); @@ -1590,42 +1652,78 @@ struct key_action }; struct key_action key_actions[] = { - {KEYMAP_KEY_A, "a", "A", "\01"}, {KEYMAP_KEY_B, "b", "B", "\02"}, - {KEYMAP_KEY_C, "c", "C", "\03"}, {KEYMAP_KEY_D, "d", "D", "\04"}, - {KEYMAP_KEY_E, "e", "E", "\05"}, {KEYMAP_KEY_F, "f", "F", "\06"}, - {KEYMAP_KEY_G, "g", "G", "\07"}, {KEYMAP_KEY_H, "h", "H", "\010"}, - {KEYMAP_KEY_I, "i", "I", "\011"}, {KEYMAP_KEY_J, "j", "J", "\012"}, - {KEYMAP_KEY_K, "k", "K", "\013"}, {KEYMAP_KEY_L, "l", "L", "\014"}, - {KEYMAP_KEY_M, "m", "M", "\015"}, {KEYMAP_KEY_N, "n", "N", "\016"}, - {KEYMAP_KEY_O, "o", "O", "\017"}, {KEYMAP_KEY_P, "p", "P", "\020"}, - {KEYMAP_KEY_Q, "q", "Q", "\021"}, {KEYMAP_KEY_R, "r", "R", "\022"}, - {KEYMAP_KEY_S, "s", "S", "\023"}, {KEYMAP_KEY_T, "t", "T", "\024"}, - {KEYMAP_KEY_U, "u", "U", "\025"}, {KEYMAP_KEY_V, "v", "V", "\026"}, - {KEYMAP_KEY_W, "w", "W", "\027"}, {KEYMAP_KEY_X, "x", "X", "\030"}, - {KEYMAP_KEY_Y, "y", "Y", "\031"}, {KEYMAP_KEY_Z, "z", "Z", "\032"}, - {KEYMAP_KEY_0, "0", ")"}, {KEYMAP_KEY_1, "1", "!"}, - {KEYMAP_KEY_2, "2", "@"}, {KEYMAP_KEY_3, "3", "#"}, - {KEYMAP_KEY_4, "4", "$"}, {KEYMAP_KEY_5, "5", "%"}, - {KEYMAP_KEY_6, "6", "^"}, {KEYMAP_KEY_7, "7", "&"}, - {KEYMAP_KEY_8, "8", "*"}, {KEYMAP_KEY_9, "9", "("}, - {KEYMAP_KEY_COMMA, ",", "<"}, {KEYMAP_KEY_DOT, ".", ">"}, - {KEYMAP_KEY_KEYPAD_0, "0"}, {KEYMAP_KEY_KEYPAD_1, "1"}, - {KEYMAP_KEY_KEYPAD_2, "2"}, {KEYMAP_KEY_KEYPAD_3, "3"}, - {KEYMAP_KEY_KEYPAD_4, "4"}, {KEYMAP_KEY_KEYPAD_5, "5"}, - {KEYMAP_KEY_KEYPAD_6, "6"}, {KEYMAP_KEY_KEYPAD_7, "7"}, - {KEYMAP_KEY_KEYPAD_8, "8"}, {KEYMAP_KEY_KEYPAD_9, "9"}, - {KEYMAP_KEY_MINUS, "-", "_"}, {KEYMAP_KEY_EQUALS, "=", "+"}, - {KEYMAP_KEY_LEFTBRACE, "[", "{"}, {KEYMAP_KEY_RIGHTBRACE, "]", "}"}, - {KEYMAP_KEY_ENTER, "\r"}, {KEYMAP_KEY_SEMICOLON, ";", ":"}, - {KEYMAP_KEY_GRAVE, "`", "~"}, {KEYMAP_KEY_TAB, "\t"}, - {KEYMAP_KEY_APOSTROPHE, "'", "\""}, {KEYMAP_KEY_SLASH, "/", "?"}, - {KEYMAP_KEY_BACKSLASH, "|"}, {KEYMAP_KEY_BACKSPACE, "\x7f"}, - {KEYMAP_KEY_KEYPAD_DOT, "."}, {KEYMAP_KEY_KEYPAD_SLASH, "/"}, - {KEYMAP_KEY_KEYPAD_ASTERISK, "*"}, {KEYMAP_KEY_KEYPAD_MINUS, "-"}, - {KEYMAP_KEY_KEYPAD_PLUS, "+"}, {KEYMAP_KEY_KEYPAD_ENTER, "\n"}, - {KEYMAP_KEY_SPACE, " ", " "}, {KEYMAP_KEY_ARROW_LEFT, "\033[D"}, - {KEYMAP_KEY_ARROW_UP, "\033[A"}, {KEYMAP_KEY_ARROW_DOWN, "\033[B"}, - {KEYMAP_KEY_ARROW_RIGHT, "\033[C"}, {KEYMAP_KEY_ESC, "\033"}, + {KEYMAP_KEY_A, "a", "A", "\01"}, + {KEYMAP_KEY_B, "b", "B", "\02"}, + {KEYMAP_KEY_C, "c", "C", "\03"}, + {KEYMAP_KEY_D, "d", "D", "\04"}, + {KEYMAP_KEY_E, "e", "E", "\05"}, + {KEYMAP_KEY_F, "f", "F", "\06"}, + {KEYMAP_KEY_G, "g", "G", "\07"}, + {KEYMAP_KEY_H, "h", "H", "\010"}, + {KEYMAP_KEY_I, "i", "I", "\011"}, + {KEYMAP_KEY_J, "j", "J", "\012"}, + {KEYMAP_KEY_K, "k", "K", "\013"}, + {KEYMAP_KEY_L, "l", "L", "\014"}, + {KEYMAP_KEY_M, "m", "M", "\015"}, + {KEYMAP_KEY_N, "n", "N", "\016"}, + {KEYMAP_KEY_O, "o", "O", "\017"}, + {KEYMAP_KEY_P, "p", "P", "\020"}, + {KEYMAP_KEY_Q, "q", "Q", "\021"}, + {KEYMAP_KEY_R, "r", "R", "\022"}, + {KEYMAP_KEY_S, "s", "S", "\023"}, + {KEYMAP_KEY_T, "t", "T", "\024"}, + {KEYMAP_KEY_U, "u", "U", "\025"}, + {KEYMAP_KEY_V, "v", "V", "\026"}, + {KEYMAP_KEY_W, "w", "W", "\027"}, + {KEYMAP_KEY_X, "x", "X", "\030"}, + {KEYMAP_KEY_Y, "y", "Y", "\031"}, + {KEYMAP_KEY_Z, "z", "Z", "\032"}, + {KEYMAP_KEY_0, "0", ")"}, + {KEYMAP_KEY_1, "1", "!"}, + {KEYMAP_KEY_2, "2", "@"}, + {KEYMAP_KEY_3, "3", "#"}, + {KEYMAP_KEY_4, "4", "$"}, + {KEYMAP_KEY_5, "5", "%"}, + {KEYMAP_KEY_6, "6", "^"}, + {KEYMAP_KEY_7, "7", "&"}, + {KEYMAP_KEY_8, "8", "*"}, + {KEYMAP_KEY_9, "9", "("}, + {KEYMAP_KEY_COMMA, ",", "<"}, + {KEYMAP_KEY_DOT, ".", ">"}, + {KEYMAP_KEY_KEYPAD_0, "0"}, + {KEYMAP_KEY_KEYPAD_1, "1"}, + {KEYMAP_KEY_KEYPAD_2, "2"}, + {KEYMAP_KEY_KEYPAD_3, "3"}, + {KEYMAP_KEY_KEYPAD_4, "4"}, + {KEYMAP_KEY_KEYPAD_5, "5"}, + {KEYMAP_KEY_KEYPAD_6, "6"}, + {KEYMAP_KEY_KEYPAD_7, "7"}, + {KEYMAP_KEY_KEYPAD_8, "8"}, + {KEYMAP_KEY_KEYPAD_9, "9"}, + {KEYMAP_KEY_MINUS, "-", "_"}, + {KEYMAP_KEY_EQUALS, "=", "+"}, + {KEYMAP_KEY_LEFTBRACE, "[", "{"}, + {KEYMAP_KEY_RIGHTBRACE, "]", "}"}, + {KEYMAP_KEY_ENTER, "\r"}, + {KEYMAP_KEY_SEMICOLON, ";", ":"}, + {KEYMAP_KEY_GRAVE, "`", "~"}, + {KEYMAP_KEY_TAB, "\t"}, + {KEYMAP_KEY_APOSTROPHE, "'", "\""}, + {KEYMAP_KEY_SLASH, "/", "?"}, + {KEYMAP_KEY_BACKSLASH, "|"}, + {KEYMAP_KEY_BACKSPACE, "\x7f"}, + {KEYMAP_KEY_KEYPAD_DOT, "."}, + {KEYMAP_KEY_KEYPAD_SLASH, "/"}, + {KEYMAP_KEY_KEYPAD_ASTERISK, "*"}, + {KEYMAP_KEY_KEYPAD_MINUS, "-"}, + {KEYMAP_KEY_KEYPAD_PLUS, "+"}, + {KEYMAP_KEY_KEYPAD_ENTER, "\n"}, + {KEYMAP_KEY_SPACE, " ", " "}, + {KEYMAP_KEY_ARROW_LEFT, "\033[D", NULL, "\033[1;5D"}, + {KEYMAP_KEY_ARROW_UP, "\033[A", NULL, "\033[1;5A"}, + {KEYMAP_KEY_ARROW_DOWN, "\033[B", NULL, "\033[1;5B"}, + {KEYMAP_KEY_ARROW_RIGHT, "\033[C", NULL, "\033[1;5C"}, + {KEYMAP_KEY_ESC, "\033"}, }; struct key_action pt_pt_key_actions[] = { @@ -1697,10 +1795,10 @@ struct key_action pt_pt_key_actions[] = { {KEYMAP_KEY_KEYPAD_ENTER, "\n"}, {KEYMAP_KEY_SPACE, " ", " "}, {KEYMAP_102ND, "<", ">"}, - {KEYMAP_KEY_ARROW_LEFT, "\033[D"}, - {KEYMAP_KEY_ARROW_UP, "\033[A"}, - {KEYMAP_KEY_ARROW_DOWN, "\033[B"}, - {KEYMAP_KEY_ARROW_RIGHT, "\033[C"}, + {KEYMAP_KEY_ARROW_LEFT, "\033[D", NULL, "\033[1;5D"}, + {KEYMAP_KEY_ARROW_UP, "\033[A", NULL, "\033[1;5A"}, + {KEYMAP_KEY_ARROW_DOWN, "\033[B", NULL, "\033[1;5B"}, + {KEYMAP_KEY_ARROW_RIGHT, "\033[C", NULL, "\033[1;5C"}, {KEYMAP_KEY_ESC, "\033"}, }; @@ -1714,6 +1812,45 @@ void __vterm_receive_input(void *p) void sched_dump_threads(void); +static bool is_numpad_code(keycode_t code) +{ + switch (code) + { + /* Ew... Depends on the KEYPAD's enum layout */ + case KEYMAP_KEY_KEYPAD_7 ... KEYMAP_KEY_KEYPAD_PLUS: + return true; + default: + return false; + } +} + +static const char *numpad_replacement(keycode_t code, const char *str) +{ + switch (code) + { + case KEYMAP_KEY_KEYPAD_7: + return "\033[H"; + case KEYMAP_KEY_KEYPAD_8: + return "\033[A"; + case KEYMAP_KEY_KEYPAD_9: + return "\033[5~"; + case KEYMAP_KEY_KEYPAD_4: + return "\033[D"; + case KEYMAP_KEY_KEYPAD_5: + return "\033[E"; + case KEYMAP_KEY_KEYPAD_6: + return "\033[C"; + case KEYMAP_KEY_KEYPAD_1: + return "\033[F"; + case KEYMAP_KEY_KEYPAD_2: + return "\033[B"; + case KEYMAP_KEY_KEYPAD_3: + return "\033[6~"; + default: + return str; + } +} + int vterm_handle_key(struct vterm *vt, struct input_device *dev, struct input_event *ev) { /* We have no interest in release events */ @@ -1725,6 +1862,9 @@ int vterm_handle_key(struct vterm *vt, struct input_device *dev, struct input_ev sched_dump_threads(); #endif + if (ev->code == KEYMAP_KEY_KEYPAD_NUMLCK) + vt->numlck = !vt->numlck; + struct key_action *acts = pt_pt_key_actions; struct key_action *desired_action = NULL; @@ -1762,6 +1902,9 @@ int vterm_handle_key(struct vterm *vt, struct input_device *dev, struct input_ev action_string = desired_action->action; } + if (!vt->numlck && is_numpad_code(ev->code)) + action_string = numpad_replacement(ev->code, action_string); + if (likely(action_string)) { struct dpc_work w; From 2280b726b8e32dd1d82a982f86eb557ddf27f621 Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Tue, 24 Sep 2024 02:02:48 +0100 Subject: [PATCH 07/76] vterm: Add box-drawing support Add box-drawing support by including a unicode font, and implementing a (very primitive!) version of G0 and G1 + shift-in/shift-out. Tested by running KBuild menuconfig on Onyx. Things look okay. Signed-off-by: Pedro Falcato --- kernel/include/onyx/font.h | 1 + kernel/kernel/Makefile | 3 +- kernel/kernel/fonts/.gitignore | 1 + kernel/kernel/fonts/Makefile | 4 + kernel/kernel/fonts/font.c | 4661 ---------------------------- kernel/kernel/fonts/u_vga16.bdf.gz | Bin 0 -> 40987 bytes kernel/kernel/tty/vt/vterm.cpp | 93 +- kernel/scripts/bdf2c.py | 353 +++ 8 files changed, 452 insertions(+), 4664 deletions(-) create mode 100644 kernel/kernel/fonts/.gitignore create mode 100644 kernel/kernel/fonts/Makefile delete mode 100644 kernel/kernel/fonts/font.c create mode 100644 kernel/kernel/fonts/u_vga16.bdf.gz create mode 100755 kernel/scripts/bdf2c.py diff --git a/kernel/include/onyx/font.h b/kernel/include/onyx/font.h index 5e6fee1ef..72cf2d89b 100644 --- a/kernel/include/onyx/font.h +++ b/kernel/include/onyx/font.h @@ -15,6 +15,7 @@ struct font unsigned int chars; unsigned int *mask; unsigned char *cursor_bitmap; + unsigned int (*utf2char)(unsigned int codepoint); }; struct font *get_font_data(void); diff --git a/kernel/kernel/Makefile b/kernel/kernel/Makefile index 5d70cae79..4913d89b8 100644 --- a/kernel/kernel/Makefile +++ b/kernel/kernel/Makefile @@ -9,7 +9,7 @@ kern-y+= arc4random.o binfmt.o compression.o copy.o cppnew.o cpprt.o crc32.o dev kern-$(CONFIG_UBSAN)+= ubsan.o -kern-y+= fonts/font.o photon/photon.o binfmt/elf.o binfmt/module_loader.o binfmt/exec.o \ +kern-y+= photon/photon.o binfmt/elf.o binfmt/module_loader.o binfmt/exec.o \ libdict/rb_tree.o libdict/tree_common.o libdict/wb_tree.o time/tickless.o binfmt/shebang.o kern-$(CONFIG_COMPAT)+= binfmt/elf_compat.o @@ -32,6 +32,7 @@ include kernel/input/Makefile include kernel/tty/Makefile include kernel/sched/Makefile include kernel/kcsan/Makefile +include kernel/fonts/Makefile kernel/syscall_thunk.cpp: generate_syscall_bits.py $(ARCHDIR)/syscall_table.json mkdir -p include/onyx/gen diff --git a/kernel/kernel/fonts/.gitignore b/kernel/kernel/fonts/.gitignore new file mode 100644 index 000000000..064a8d8ef --- /dev/null +++ b/kernel/kernel/fonts/.gitignore @@ -0,0 +1 @@ +*.c diff --git a/kernel/kernel/fonts/Makefile b/kernel/kernel/fonts/Makefile new file mode 100644 index 000000000..fb81f82a4 --- /dev/null +++ b/kernel/kernel/fonts/Makefile @@ -0,0 +1,4 @@ +kernel/fonts/font.c: kernel/fonts/u_vga16.bdf.gz scripts/bdf2c.py + scripts/bdf2c.py kernel/fonts/u_vga16.bdf.gz $@ + +obj-y+= kernel/fonts/font.o diff --git a/kernel/kernel/fonts/font.c b/kernel/kernel/fonts/font.c deleted file mode 100644 index dde161557..000000000 --- a/kernel/kernel/fonts/font.c +++ /dev/null @@ -1,4661 +0,0 @@ -// Created from bdf2c Version 3, (c) 2009, 2010 by Lutz Sammer -// License AGPLv3: GNU Affero General Public License version 3 - -#include - -#include - -const unsigned char __cursor__bitmap[] = { - XXXXXXXX, XXXXXXXX, XXXXXXXX, XXXXXXXX, XXXXXXXX, XXXXXXXX, XXXXXXXX, XXXXXXXX, - XXXXXXXX, XXXXXXXX, XXXXXXXX, XXXXXXXX, XXXXXXXX, XXXXXXXX, ________, ________, -}; - -static const unsigned char __font_bitmap__[] = { - /* 0 $00 'C0000' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 1 $01 'C0001' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - _XXXXXX_, - X______X, - X_X__X_X, - X______X, - X______X, - X_X__X_X, - X__XX__X, - X______X, - X______X, - _XXXXXX_, - ________, - ________, - ________, - ________, - /* 2 $02 'C0002' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - _XXXXXX_, - XXXXXXXX, - XX_XX_XX, - XXXXXXXX, - XXXXXXXX, - XX_XX_XX, - XXX__XXX, - XXXXXXXX, - XXXXXXXX, - _XXXXXX_, - ________, - ________, - ________, - ________, - /* 3 $03 'C0003' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - _XX_XX__, - XXXXXXX_, - XXXXXXX_, - XXXXXXX_, - XXXXXXX_, - _XXXXX__, - __XXX___, - ___X____, - ________, - ________, - ________, - ________, - /* 4 $04 'C0004' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ___X____, - __XXX___, - _XXXXX__, - XXXXXXX_, - _XXXXX__, - __XXX___, - ___X____, - ________, - ________, - ________, - ________, - ________, - /* 5 $05 'C0005' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ___XX___, - __XXXX__, - __XXXX__, - XXX__XXX, - XXX__XXX, - XXX__XXX, - ___XX___, - ___XX___, - __XXXX__, - ________, - ________, - ________, - ________, - /* 6 $06 'C0006' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ___XX___, - __XXXX__, - _XXXXXX_, - XXXXXXXX, - XXXXXXXX, - _XXXXXX_, - ___XX___, - ___XX___, - __XXXX__, - ________, - ________, - ________, - ________, - /* 7 $07 'C0007' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - ________, - ___XX___, - __XXXX__, - __XXXX__, - ___XX___, - ________, - ________, - ________, - ________, - ________, - ________, - /* 8 $08 'C0008' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXX__XXX, - XX____XX, - XX____XX, - XXX__XXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - /* 9 $09 'C0009' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - __XXXX__, - _XX__XX_, - _X____X_, - _X____X_, - _XX__XX_, - __XXXX__, - ________, - ________, - ________, - ________, - ________, - /* 10 $0a 'C000a' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XX____XX, - X__XX__X, - X_XXXX_X, - X_XXXX_X, - X__XX__X, - XX____XX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - /* 11 $0b 'C000b' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ___XXXX_, - _____XX_, - ____XXX_, - ___XX_X_, - _XXXX___, - XX__XX__, - XX__XX__, - XX__XX__, - XX__XX__, - _XXXX___, - ________, - ________, - ________, - ________, - /* 12 $0c 'C000c' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - __XXXX__, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - __XXXX__, - ___XX___, - _XXXXXX_, - ___XX___, - ___XX___, - ________, - ________, - ________, - ________, - /* 13 $0d 'C000d' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - __XXXXXX, - __XX__XX, - __XXXXXX, - __XX____, - __XX____, - __XX____, - __XX____, - _XXX____, - XXXX____, - XXX_____, - ________, - ________, - ________, - ________, - /* 14 $0e 'C000e' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - _XXXXXXX, - _XX___XX, - _XXXXXXX, - _XX___XX, - _XX___XX, - _XX___XX, - _XX___XX, - _XX__XXX, - XXX__XXX, - XXX__XX_, - XX______, - ________, - ________, - ________, - /* 15 $0f 'C000f' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ___XX___, - ___XX___, - XX_XX_XX, - __XXXX__, - XXX__XXX, - __XXXX__, - XX_XX_XX, - ___XX___, - ___XX___, - ________, - ________, - ________, - ________, - /* 16 $10 'C0010' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - X_______, - XX______, - XXX_____, - XXXX____, - XXXXX___, - XXXXXXX_, - XXXXX___, - XXXX____, - XXX_____, - XX______, - X_______, - ________, - ________, - ________, - ________, - /* 17 $11 'C0011' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ______X_, - _____XX_, - ____XXX_, - ___XXXX_, - __XXXXX_, - XXXXXXX_, - __XXXXX_, - ___XXXX_, - ____XXX_, - _____XX_, - ______X_, - ________, - ________, - ________, - ________, - /* 18 $12 'C0012' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ___XX___, - __XXXX__, - _XXXXXX_, - ___XX___, - ___XX___, - ___XX___, - _XXXXXX_, - __XXXX__, - ___XX___, - ________, - ________, - ________, - ________, - ________, - /* 19 $13 'C0013' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - ________, - _XX__XX_, - _XX__XX_, - ________, - ________, - ________, - ________, - /* 20 $14 'C0014' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - _XXXXXXX, - XX_XX_XX, - XX_XX_XX, - XX_XX_XX, - _XXXX_XX, - ___XX_XX, - ___XX_XX, - ___XX_XX, - ___XX_XX, - ___XX_XX, - ________, - ________, - ________, - ________, - /* 21 $15 'C0015' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - _XXXXX__, - XX___XX_, - _XX_____, - __XXX___, - _XX_XX__, - XX___XX_, - XX___XX_, - _XX_XX__, - __XXX___, - ____XX__, - XX___XX_, - _XXXXX__, - ________, - ________, - ________, - /* 22 $16 'C0016' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - XXXXXXX_, - XXXXXXX_, - XXXXXXX_, - XXXXXXX_, - ________, - ________, - ________, - ________, - /* 23 $17 'C0017' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ___XX___, - __XXXX__, - _XXXXXX_, - ___XX___, - ___XX___, - ___XX___, - _XXXXXX_, - __XXXX__, - ___XX___, - _XXXXXX_, - ________, - ________, - ________, - ________, - /* 24 $18 'C0018' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ___XX___, - __XXXX__, - _XXXXXX_, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ________, - ________, - ________, - ________, - /* 25 $19 'C0019' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - _XXXXXX_, - __XXXX__, - ___XX___, - ________, - ________, - ________, - ________, - /* 26 $1a 'C001a' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - ___XX___, - ____XX__, - XXXXXXX_, - ____XX__, - ___XX___, - ________, - ________, - ________, - ________, - ________, - ________, - /* 27 $1b 'C001b' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - __XX____, - _XX_____, - XXXXXXX_, - _XX_____, - __XX____, - ________, - ________, - ________, - ________, - ________, - ________, - /* 28 $1c 'C001c' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - ________, - XX______, - XX______, - XX______, - XXXXXXX_, - ________, - ________, - ________, - ________, - ________, - ________, - /* 29 $1d 'C001d' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - __X_X___, - _XX_XX__, - XXXXXXX_, - _XX_XX__, - __X_X___, - ________, - ________, - ________, - ________, - ________, - ________, - /* 30 $1e 'C001e' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ___X____, - __XXX___, - __XXX___, - _XXXXX__, - _XXXXX__, - XXXXXXX_, - XXXXXXX_, - ________, - ________, - ________, - ________, - ________, - /* 31 $1f 'C001f' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - XXXXXXX_, - XXXXXXX_, - _XXXXX__, - _XXXXX__, - __XXX___, - __XXX___, - ___X____, - ________, - ________, - ________, - ________, - ________, - /* 32 $20 'C0020' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 33 $21 'C0021' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ___XX___, - __XXXX__, - __XXXX__, - __XXXX__, - ___XX___, - ___XX___, - ___XX___, - ________, - ___XX___, - ___XX___, - ________, - ________, - ________, - ________, - /* 34 $22 'C0022' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - _XX__XX_, - _XX__XX_, - _XX__XX_, - __X__X__, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 35 $23 'C0023' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - _XX_XX__, - _XX_XX__, - XXXXXXX_, - _XX_XX__, - _XX_XX__, - _XX_XX__, - XXXXXXX_, - _XX_XX__, - _XX_XX__, - ________, - ________, - ________, - ________, - /* 36 $24 'C0024' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ___XX___, - ___XX___, - _XXXXX__, - XX___XX_, - XX____X_, - XX______, - _XXXXX__, - _____XX_, - _____XX_, - X____XX_, - XX___XX_, - _XXXXX__, - ___XX___, - ___XX___, - ________, - ________, - /* 37 $25 'C0025' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - XX____X_, - XX___XX_, - ____XX__, - ___XX___, - __XX____, - _XX_____, - XX___XX_, - X____XX_, - ________, - ________, - ________, - ________, - /* 38 $26 'C0026' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - __XXX___, - _XX_XX__, - _XX_XX__, - __XXX___, - _XXX_XX_, - XX_XXX__, - XX__XX__, - XX__XX__, - XX__XX__, - _XXX_XX_, - ________, - ________, - ________, - ________, - /* 39 $27 'C0027' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - __XX____, - __XX____, - __XX____, - _XX_____, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 40 $28 'C0028' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ____XX__, - ___XX___, - __XX____, - __XX____, - __XX____, - __XX____, - __XX____, - __XX____, - ___XX___, - ____XX__, - ________, - ________, - ________, - ________, - /* 41 $29 'C0029' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - __XX____, - ___XX___, - ____XX__, - ____XX__, - ____XX__, - ____XX__, - ____XX__, - ____XX__, - ___XX___, - __XX____, - ________, - ________, - ________, - ________, - /* 42 $2a 'C002a' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - _XX__XX_, - __XXXX__, - XXXXXXXX, - __XXXX__, - _XX__XX_, - ________, - ________, - ________, - ________, - ________, - ________, - /* 43 $2b 'C002b' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - ___XX___, - ___XX___, - _XXXXXX_, - ___XX___, - ___XX___, - ________, - ________, - ________, - ________, - ________, - ________, - /* 44 $2c 'C002c' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ___XX___, - ___XX___, - ___XX___, - __XX____, - ________, - ________, - ________, - /* 45 $2d 'C002d' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - ________, - ________, - XXXXXXX_, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 46 $2e 'C002e' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ___XX___, - ___XX___, - ________, - ________, - ________, - ________, - /* 47 $2f 'C002f' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ______X_, - _____XX_, - ____XX__, - ___XX___, - __XX____, - _XX_____, - XX______, - X_______, - ________, - ________, - ________, - ________, - /* 48 $30 'C0030' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - __XXX___, - _XX_XX__, - XX___XX_, - XX___XX_, - XX_X_XX_, - XX_X_XX_, - XX___XX_, - XX___XX_, - _XX_XX__, - __XXX___, - ________, - ________, - ________, - ________, - /* 49 $31 'C0031' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ___XX___, - __XXX___, - _XXXX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - _XXXXXX_, - ________, - ________, - ________, - ________, - /* 50 $32 'C0032' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - _XXXXX__, - XX___XX_, - _____XX_, - ____XX__, - ___XX___, - __XX____, - _XX_____, - XX______, - XX___XX_, - XXXXXXX_, - ________, - ________, - ________, - ________, - /* 51 $33 'C0033' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - _XXXXX__, - XX___XX_, - _____XX_, - _____XX_, - __XXXX__, - _____XX_, - _____XX_, - _____XX_, - XX___XX_, - _XXXXX__, - ________, - ________, - ________, - ________, - /* 52 $34 'C0034' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ____XX__, - ___XXX__, - __XXXX__, - _XX_XX__, - XX__XX__, - XXXXXXX_, - ____XX__, - ____XX__, - ____XX__, - ___XXXX_, - ________, - ________, - ________, - ________, - /* 53 $35 'C0035' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XXXXXXX_, - XX______, - XX______, - XX______, - XXXXXX__, - _____XX_, - _____XX_, - _____XX_, - XX___XX_, - _XXXXX__, - ________, - ________, - ________, - ________, - /* 54 $36 'C0036' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - __XXX___, - _XX_____, - XX______, - XX______, - XXXXXX__, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - _XXXXX__, - ________, - ________, - ________, - ________, - /* 55 $37 'C0037' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XXXXXXX_, - XX___XX_, - _____XX_, - _____XX_, - ____XX__, - ___XX___, - __XX____, - __XX____, - __XX____, - __XX____, - ________, - ________, - ________, - ________, - /* 56 $38 'C0038' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - _XXXXX__, - XX___XX_, - XX___XX_, - XX___XX_, - _XXXXX__, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - _XXXXX__, - ________, - ________, - ________, - ________, - /* 57 $39 'C0039' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - _XXXXX__, - XX___XX_, - XX___XX_, - XX___XX_, - _XXXXXX_, - _____XX_, - _____XX_, - _____XX_, - ____XX__, - _XXXX___, - ________, - ________, - ________, - ________, - /* 58 $3a 'C003a' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ___XX___, - ___XX___, - ________, - ________, - ________, - ___XX___, - ___XX___, - ________, - ________, - ________, - ________, - ________, - /* 59 $3b 'C003b' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ___XX___, - ___XX___, - ________, - ________, - ________, - ___XX___, - ___XX___, - __XX____, - ________, - ________, - ________, - ________, - /* 60 $3c 'C003c' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - _____XX_, - ____XX__, - ___XX___, - __XX____, - _XX_____, - __XX____, - ___XX___, - ____XX__, - _____XX_, - ________, - ________, - ________, - ________, - /* 61 $3d 'C003d' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - _XXXXXX_, - ________, - ________, - _XXXXXX_, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 62 $3e 'C003e' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - _XX_____, - __XX____, - ___XX___, - ____XX__, - _____XX_, - ____XX__, - ___XX___, - __XX____, - _XX_____, - ________, - ________, - ________, - ________, - /* 63 $3f 'C003f' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - _XXXXX__, - XX___XX_, - XX___XX_, - ____XX__, - ___XX___, - ___XX___, - ___XX___, - ________, - ___XX___, - ___XX___, - ________, - ________, - ________, - ________, - /* 64 $40 'C0040' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - _XXXXX__, - XX___XX_, - XX___XX_, - XX_XXXX_, - XX_XXXX_, - XX_XXXX_, - XX_XXX__, - XX______, - _XXXXX__, - ________, - ________, - ________, - ________, - /* 65 $41 'C0041' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ___X____, - __XXX___, - _XX_XX__, - XX___XX_, - XX___XX_, - XXXXXXX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - ________, - ________, - ________, - ________, - /* 66 $42 'C0042' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XXXXXX__, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XXXXX__, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - XXXXXX__, - ________, - ________, - ________, - ________, - /* 67 $43 'C0043' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - __XXXX__, - _XX__XX_, - XX____X_, - XX______, - XX______, - XX______, - XX______, - XX____X_, - _XX__XX_, - __XXXX__, - ________, - ________, - ________, - ________, - /* 68 $44 'C0044' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XXXXX___, - _XX_XX__, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XX_XX__, - XXXXX___, - ________, - ________, - ________, - ________, - /* 69 $45 'C0045' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XXXXXXX_, - _XX__XX_, - _XX___X_, - _XX_X___, - _XXXX___, - _XX_X___, - _XX_____, - _XX___X_, - _XX__XX_, - XXXXXXX_, - ________, - ________, - ________, - ________, - /* 70 $46 'C0046' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XXXXXXX_, - _XX__XX_, - _XX___X_, - _XX_X___, - _XXXX___, - _XX_X___, - _XX_____, - _XX_____, - _XX_____, - XXXX____, - ________, - ________, - ________, - ________, - /* 71 $47 'C0047' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - __XXXX__, - _XX__XX_, - XX____X_, - XX______, - XX______, - XX_XXXX_, - XX___XX_, - XX___XX_, - _XX__XX_, - __XXX_X_, - ________, - ________, - ________, - ________, - /* 72 $48 'C0048' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XXXXXXX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - ________, - ________, - ________, - ________, - /* 73 $49 'C0049' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - __XXXX__, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - __XXXX__, - ________, - ________, - ________, - ________, - /* 74 $4a 'C004a' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ___XXXX_, - ____XX__, - ____XX__, - ____XX__, - ____XX__, - ____XX__, - XX__XX__, - XX__XX__, - XX__XX__, - _XXXX___, - ________, - ________, - ________, - ________, - /* 75 $4b 'C004b' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XXX__XX_, - _XX__XX_, - _XX__XX_, - _XX_XX__, - _XXXX___, - _XXXX___, - _XX_XX__, - _XX__XX_, - _XX__XX_, - XXX__XX_, - ________, - ________, - ________, - ________, - /* 76 $4c 'C004c' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XXXX____, - _XX_____, - _XX_____, - _XX_____, - _XX_____, - _XX_____, - _XX_____, - _XX___X_, - _XX__XX_, - XXXXXXX_, - ________, - ________, - ________, - ________, - /* 77 $4d 'C004d' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XX___XX_, - XXX_XXX_, - XXXXXXX_, - XXXXXXX_, - XX_X_XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - ________, - ________, - ________, - ________, - /* 78 $4e 'C004e' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XX___XX_, - XXX__XX_, - XXXX_XX_, - XXXXXXX_, - XX_XXXX_, - XX__XXX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - ________, - ________, - ________, - ________, - /* 79 $4f 'C004f' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - _XXXXX__, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - _XXXXX__, - ________, - ________, - ________, - ________, - /* 80 $50 'C0050' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XXXXXX__, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XXXXX__, - _XX_____, - _XX_____, - _XX_____, - _XX_____, - XXXX____, - ________, - ________, - ________, - ________, - /* 81 $51 'C0051' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - _XXXXX__, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX_X_XX_, - XX_XXXX_, - _XXXXX__, - ____XX__, - ____XXX_, - ________, - ________, - /* 82 $52 'C0052' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XXXXXX__, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XXXXX__, - _XX_XX__, - _XX__XX_, - _XX__XX_, - _XX__XX_, - XXX__XX_, - ________, - ________, - ________, - ________, - /* 83 $53 'C0053' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - _XXXXX__, - XX___XX_, - XX___XX_, - _XX_____, - __XXX___, - ____XX__, - _____XX_, - XX___XX_, - XX___XX_, - _XXXXX__, - ________, - ________, - ________, - ________, - /* 84 $54 'C0054' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - _XXXXXX_, - _XXXXXX_, - _X_XX_X_, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - __XXXX__, - ________, - ________, - ________, - ________, - /* 85 $55 'C0055' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - _XXXXX__, - ________, - ________, - ________, - ________, - /* 86 $56 'C0056' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - _XX_XX__, - __XXX___, - ___X____, - ________, - ________, - ________, - ________, - /* 87 $57 'C0057' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX_X_XX_, - XX_X_XX_, - XX_X_XX_, - XXXXXXX_, - XXX_XXX_, - _XX_XX__, - ________, - ________, - ________, - ________, - /* 88 $58 'C0058' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XX___XX_, - XX___XX_, - _XX_XX__, - _XXXXX__, - __XXX___, - __XXX___, - _XXXXX__, - _XX_XX__, - XX___XX_, - XX___XX_, - ________, - ________, - ________, - ________, - /* 89 $59 'C0059' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - __XXXX__, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - __XXXX__, - ________, - ________, - ________, - ________, - /* 90 $5a 'C005a' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XXXXXXX_, - XX___XX_, - X____XX_, - ____XX__, - ___XX___, - __XX____, - _XX_____, - XX____X_, - XX___XX_, - XXXXXXX_, - ________, - ________, - ________, - ________, - /* 91 $5b 'C005b' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - __XXXX__, - __XX____, - __XX____, - __XX____, - __XX____, - __XX____, - __XX____, - __XX____, - __XX____, - __XXXX__, - ________, - ________, - ________, - ________, - /* 92 $5c 'C005c' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - X_______, - XX______, - XXX_____, - _XXX____, - __XXX___, - ___XXX__, - ____XXX_, - _____XX_, - ______X_, - ________, - ________, - ________, - ________, - /* 93 $5d 'C005d' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - __XXXX__, - ____XX__, - ____XX__, - ____XX__, - ____XX__, - ____XX__, - ____XX__, - ____XX__, - ____XX__, - __XXXX__, - ________, - ________, - ________, - ________, - /* 94 $5e 'C005e' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ___X____, - __XXX___, - _XX_XX__, - XX___XX_, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 95 $5f 'C005f' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - XXXXXXXX, - ________, - ________, - /* 96 $60 'C0060' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - __XX____, - __XX____, - ___XX___, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 97 $61 'C0061' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - _XXXX___, - ____XX__, - _XXXXX__, - XX__XX__, - XX__XX__, - XX__XX__, - _XXX_XX_, - ________, - ________, - ________, - ________, - /* 98 $62 'C0062' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XXX_____, - _XX_____, - _XX_____, - _XXXX___, - _XX_XX__, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XXXXX__, - ________, - ________, - ________, - ________, - /* 99 $63 'C0063' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - _XXXXX__, - XX___XX_, - XX______, - XX______, - XX______, - XX___XX_, - _XXXXX__, - ________, - ________, - ________, - ________, - /* 100 $64 'C0064' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ___XXX__, - ____XX__, - ____XX__, - __XXXX__, - _XX_XX__, - XX__XX__, - XX__XX__, - XX__XX__, - XX__XX__, - _XXX_XX_, - ________, - ________, - ________, - ________, - /* 101 $65 'C0065' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - _XXXXX__, - XX___XX_, - XXXXXXX_, - XX______, - XX______, - XX___XX_, - _XXXXX__, - ________, - ________, - ________, - ________, - /* 102 $66 'C0066' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - __XXX___, - _XX_XX__, - _XX__X__, - _XX_____, - XXXX____, - _XX_____, - _XX_____, - _XX_____, - _XX_____, - XXXX____, - ________, - ________, - ________, - ________, - /* 103 $67 'C0067' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - _XXX_XX_, - XX__XX__, - XX__XX__, - XX__XX__, - XX__XX__, - XX__XX__, - _XXXXX__, - ____XX__, - XX__XX__, - _XXXX___, - ________, - /* 104 $68 'C0068' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XXX_____, - _XX_____, - _XX_____, - _XX_XX__, - _XXX_XX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - XXX__XX_, - ________, - ________, - ________, - ________, - /* 105 $69 'C0069' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ___XX___, - ___XX___, - ________, - __XXX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - __XXXX__, - ________, - ________, - ________, - ________, - /* 106 $6a 'C006a' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - _____XX_, - _____XX_, - ________, - ____XXX_, - _____XX_, - _____XX_, - _____XX_, - _____XX_, - _____XX_, - _____XX_, - _XX__XX_, - _XX__XX_, - __XXXX__, - ________, - /* 107 $6b 'C006b' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XXX_____, - _XX_____, - _XX_____, - _XX__XX_, - _XX_XX__, - _XXXX___, - _XXXX___, - _XX_XX__, - _XX__XX_, - XXX__XX_, - ________, - ________, - ________, - ________, - /* 108 $6c 'C006c' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - __XXX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - __XXXX__, - ________, - ________, - ________, - ________, - /* 109 $6d 'C006d' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - XXX_XX__, - XXXXXXX_, - XX_X_XX_, - XX_X_XX_, - XX_X_XX_, - XX_X_XX_, - XX___XX_, - ________, - ________, - ________, - ________, - /* 110 $6e 'C006e' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - XX_XXX__, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - ________, - ________, - ________, - ________, - /* 111 $6f 'C006f' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - _XXXXX__, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - _XXXXX__, - ________, - ________, - ________, - ________, - /* 112 $70 'C0070' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - XX_XXX__, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XXXXX__, - _XX_____, - _XX_____, - XXXX____, - ________, - /* 113 $71 'C0071' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - _XXX_XX_, - XX__XX__, - XX__XX__, - XX__XX__, - XX__XX__, - XX__XX__, - _XXXXX__, - ____XX__, - ____XX__, - ___XXXX_, - ________, - /* 114 $72 'C0072' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - XX_XXX__, - _XXX_XX_, - _XX__XX_, - _XX_____, - _XX_____, - _XX_____, - XXXX____, - ________, - ________, - ________, - ________, - /* 115 $73 'C0073' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - _XXXXX__, - XX___XX_, - _XX_____, - __XXX___, - ____XX__, - XX___XX_, - _XXXXX__, - ________, - ________, - ________, - ________, - /* 116 $74 'C0074' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ___X____, - __XX____, - __XX____, - XXXXXX__, - __XX____, - __XX____, - __XX____, - __XX____, - __XX_XX_, - ___XXX__, - ________, - ________, - ________, - ________, - /* 117 $75 'C0075' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - XX__XX__, - XX__XX__, - XX__XX__, - XX__XX__, - XX__XX__, - XX__XX__, - _XXX_XX_, - ________, - ________, - ________, - ________, - /* 118 $76 'C0076' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - __XXXX__, - ___XX___, - ________, - ________, - ________, - ________, - /* 119 $77 'C0077' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - XX___XX_, - XX___XX_, - XX_X_XX_, - XX_X_XX_, - XX_X_XX_, - XXXXXXX_, - _XX_XX__, - ________, - ________, - ________, - ________, - /* 120 $78 'C0078' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - XX___XX_, - _XX_XX__, - __XXX___, - __XXX___, - __XXX___, - _XX_XX__, - XX___XX_, - ________, - ________, - ________, - ________, - /* 121 $79 'C0079' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - _XXXXXX_, - _____XX_, - ____XX__, - XXXXX___, - ________, - /* 122 $7a 'C007a' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - XXXXXXX_, - XX__XX__, - ___XX___, - __XX____, - _XX_____, - XX___XX_, - XXXXXXX_, - ________, - ________, - ________, - ________, - /* 123 $7b 'C007b' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ____XXX_, - ___XX___, - ___XX___, - ___XX___, - _XXX____, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ____XXX_, - ________, - ________, - ________, - ________, - /* 124 $7c 'C007c' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ________, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ________, - ________, - ________, - ________, - /* 125 $7d 'C007d' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - _XXX____, - ___XX___, - ___XX___, - ___XX___, - ____XXX_, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - _XXX____, - ________, - ________, - ________, - ________, - /* 126 $7e 'C007e' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - _XXX_XX_, - XX_XXX__, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 127 $7f 'C007f' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ___X____, - __XXX___, - _XX_XX__, - XX___XX_, - XX___XX_, - XX___XX_, - XXXXXXX_, - ________, - ________, - ________, - ________, - ________, - /* 128 $80 'C0080' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - __XXXX__, - _XX__XX_, - XX____X_, - XX______, - XX______, - XX______, - XX____X_, - _XX__XX_, - __XXXX__, - ____XX__, - _____XX_, - _XXXXX__, - ________, - ________, - /* 129 $81 'C0081' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XX__XX__, - ________, - ________, - XX__XX__, - XX__XX__, - XX__XX__, - XX__XX__, - XX__XX__, - XX__XX__, - _XXX_XX_, - ________, - ________, - ________, - ________, - /* 130 $82 'C0082' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ____XX__, - ___XX___, - __XX____, - ________, - _XXXXX__, - XX___XX_, - XXXXXXX_, - XX______, - XX______, - XX___XX_, - _XXXXX__, - ________, - ________, - ________, - ________, - /* 131 $83 'C0083' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ___X____, - __XXX___, - _XX_XX__, - ________, - _XXXX___, - ____XX__, - _XXXXX__, - XX__XX__, - XX__XX__, - XX__XX__, - _XXX_XX_, - ________, - ________, - ________, - ________, - /* 132 $84 'C0084' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XX__XX__, - ________, - ________, - _XXXX___, - ____XX__, - _XXXXX__, - XX__XX__, - XX__XX__, - XX__XX__, - _XXX_XX_, - ________, - ________, - ________, - ________, - /* 133 $85 'C0085' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - _XX_____, - __XX____, - ___XX___, - ________, - _XXXX___, - ____XX__, - _XXXXX__, - XX__XX__, - XX__XX__, - XX__XX__, - _XXX_XX_, - ________, - ________, - ________, - ________, - /* 134 $86 'C0086' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - __XXX___, - _XX_XX__, - __XXX___, - ________, - _XXXX___, - ____XX__, - _XXXXX__, - XX__XX__, - XX__XX__, - XX__XX__, - _XXX_XX_, - ________, - ________, - ________, - ________, - /* 135 $87 'C0087' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - __XXXX__, - _XX__XX_, - _XX_____, - _XX_____, - _XX__XX_, - __XXXX__, - ____XX__, - _____XX_, - __XXXX__, - ________, - ________, - ________, - /* 136 $88 'C0088' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ___X____, - __XXX___, - _XX_XX__, - ________, - _XXXXX__, - XX___XX_, - XXXXXXX_, - XX______, - XX______, - XX___XX_, - _XXXXX__, - ________, - ________, - ________, - ________, - /* 137 $89 'C0089' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XX___XX_, - ________, - ________, - _XXXXX__, - XX___XX_, - XXXXXXX_, - XX______, - XX______, - XX___XX_, - _XXXXX__, - ________, - ________, - ________, - ________, - /* 138 $8a 'C008a' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - _XX_____, - __XX____, - ___XX___, - ________, - _XXXXX__, - XX___XX_, - XXXXXXX_, - XX______, - XX______, - XX___XX_, - _XXXXX__, - ________, - ________, - ________, - ________, - /* 139 $8b 'C008b' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - _XX__XX_, - ________, - ________, - __XXX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - __XXXX__, - ________, - ________, - ________, - ________, - /* 140 $8c 'C008c' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ___XX___, - __XXXX__, - _XX__XX_, - ________, - __XXX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - __XXXX__, - ________, - ________, - ________, - ________, - /* 141 $8d 'C008d' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - _XX_____, - __XX____, - ___XX___, - ________, - __XXX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - __XXXX__, - ________, - ________, - ________, - ________, - /* 142 $8e 'C008e' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - XX___XX_, - ________, - ___X____, - __XXX___, - _XX_XX__, - XX___XX_, - XX___XX_, - XXXXXXX_, - XX___XX_, - XX___XX_, - XX___XX_, - ________, - ________, - ________, - ________, - /* 143 $8f 'C008f' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - __XXX___, - _XX_XX__, - __XXX___, - ________, - __XXX___, - _XX_XX__, - XX___XX_, - XX___XX_, - XXXXXXX_, - XX___XX_, - XX___XX_, - XX___XX_, - ________, - ________, - ________, - ________, - /* 144 $90 'C0090' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ___XX___, - __XX____, - _XX_____, - ________, - XXXXXXX_, - _XX__XX_, - _XX_____, - _XXXXX__, - _XX_____, - _XX_____, - _XX__XX_, - XXXXXXX_, - ________, - ________, - ________, - ________, - /* 145 $91 'C0091' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - XX__XX__, - _XXX_XX_, - __XX_XX_, - _XXXXXX_, - XX_XX___, - XX_XX___, - _XX_XXX_, - ________, - ________, - ________, - ________, - /* 146 $92 'C0092' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - __XXXXX_, - _XX_XX__, - XX__XX__, - XX__XX__, - XXXXXXX_, - XX__XX__, - XX__XX__, - XX__XX__, - XX__XX__, - XX__XXX_, - ________, - ________, - ________, - ________, - /* 147 $93 'C0093' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ___X____, - __XXX___, - _XX_XX__, - ________, - _XXXXX__, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - _XXXXX__, - ________, - ________, - ________, - ________, - /* 148 $94 'C0094' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XX___XX_, - ________, - ________, - _XXXXX__, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - _XXXXX__, - ________, - ________, - ________, - ________, - /* 149 $95 'C0095' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - _XX_____, - __XX____, - ___XX___, - ________, - _XXXXX__, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - _XXXXX__, - ________, - ________, - ________, - ________, - /* 150 $96 'C0096' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - __XX____, - _XXXX___, - XX__XX__, - ________, - XX__XX__, - XX__XX__, - XX__XX__, - XX__XX__, - XX__XX__, - XX__XX__, - _XXX_XX_, - ________, - ________, - ________, - ________, - /* 151 $97 'C0097' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - _XX_____, - __XX____, - ___XX___, - ________, - XX__XX__, - XX__XX__, - XX__XX__, - XX__XX__, - XX__XX__, - XX__XX__, - _XXX_XX_, - ________, - ________, - ________, - ________, - /* 152 $98 'C0098' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XX___XX_, - ________, - ________, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - _XXXXXX_, - _____XX_, - ____XX__, - _XXXX___, - ________, - /* 153 $99 'C0099' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - XX___XX_, - ________, - _XXXXX__, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - _XXXXX__, - ________, - ________, - ________, - ________, - /* 154 $9a 'C009a' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - XX___XX_, - ________, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - _XXXXX__, - ________, - ________, - ________, - ________, - /* 155 $9b 'C009b' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ___XX___, - ___XX___, - __XXXX__, - _XX__XX_, - _XX_____, - _XX_____, - _XX_____, - _XX__XX_, - __XXXX__, - ___XX___, - ___XX___, - ________, - ________, - ________, - ________, - /* 156 $9c 'C009c' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - __XXX___, - _XX_XX__, - _XX__X__, - _XX_____, - XXXX____, - _XX_____, - _XX_____, - _XX_____, - _XX_____, - XXX__XX_, - XXXXXX__, - ________, - ________, - ________, - ________, - /* 157 $9d 'C009d' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - _XX__XX_, - _XX__XX_, - __XXXX__, - ___XX___, - _XXXXXX_, - ___XX___, - _XXXXXX_, - ___XX___, - ___XX___, - ___XX___, - ________, - ________, - ________, - ________, - /* 158 $9e 'C009e' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - XXXXX___, - XX__XX__, - XX__XX__, - XXXXX___, - XX___X__, - XX__XX__, - XX_XXXX_, - XX__XX__, - XX__XX__, - XX__XX__, - XX___XX_, - ________, - ________, - ________, - ________, - /* 159 $9f 'C009f' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ____XXX_, - ___XX_XX, - ___XX___, - ___XX___, - ___XX___, - _XXXXXX_, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - XX_XX___, - _XXX____, - ________, - ________, - /* 160 $a0 'C00a0' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ___XX___, - __XX____, - _XX_____, - ________, - _XXXX___, - ____XX__, - _XXXXX__, - XX__XX__, - XX__XX__, - XX__XX__, - _XXX_XX_, - ________, - ________, - ________, - ________, - /* 161 $a1 'C00a1' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ____XX__, - ___XX___, - __XX____, - ________, - __XXX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - __XXXX__, - ________, - ________, - ________, - ________, - /* 162 $a2 'C00a2' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ___XX___, - __XX____, - _XX_____, - ________, - _XXXXX__, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - _XXXXX__, - ________, - ________, - ________, - ________, - /* 163 $a3 'C00a3' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ___XX___, - __XX____, - _XX_____, - ________, - XX__XX__, - XX__XX__, - XX__XX__, - XX__XX__, - XX__XX__, - XX__XX__, - _XXX_XX_, - ________, - ________, - ________, - ________, - /* 164 $a4 'C00a4' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - _XXX_XX_, - XX_XXX__, - ________, - XX_XXX__, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - ________, - ________, - ________, - ________, - /* 165 $a5 'C00a5' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - _XXX_XX_, - XX_XXX__, - ________, - XX___XX_, - XXX__XX_, - XXXX_XX_, - XXXXXXX_, - XX_XXXX_, - XX__XXX_, - XX___XX_, - XX___XX_, - XX___XX_, - ________, - ________, - ________, - ________, - /* 166 $a6 'C00a6' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - __XXXX__, - _XX_XX__, - _XX_XX__, - __XXXXX_, - ________, - _XXXXXX_, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 167 $a7 'C00a7' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - __XXX___, - _XX_XX__, - _XX_XX__, - __XXX___, - ________, - _XXXXX__, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 168 $a8 'C00a8' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - __XX____, - __XX____, - ________, - __XX____, - __XX____, - _XX_____, - XX______, - XX___XX_, - XX___XX_, - _XXXXX__, - ________, - ________, - ________, - ________, - /* 169 $a9 'C00a9' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - ________, - XXXXXXX_, - XX______, - XX______, - XX______, - XX______, - ________, - ________, - ________, - ________, - ________, - /* 170 $aa 'C00aa' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - ________, - XXXXXXX_, - _____XX_, - _____XX_, - _____XX_, - _____XX_, - ________, - ________, - ________, - ________, - ________, - /* 171 $ab 'C00ab' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - XX______, - XX______, - XX____X_, - XX___XX_, - XX__XX__, - ___XX___, - __XX____, - _XX_____, - XX_XXX__, - X____XX_, - ____XX__, - ___XX___, - __XXXXX_, - ________, - ________, - /* 172 $ac 'C00ac' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - XX______, - XX______, - XX____X_, - XX___XX_, - XX__XX__, - ___XX___, - __XX____, - _XX__XX_, - XX__XXX_, - X__XXXX_, - __XXXXX_, - _____XX_, - _____XX_, - ________, - ________, - /* 173 $ad 'C00ad' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ___XX___, - ___XX___, - ________, - ___XX___, - ___XX___, - ___XX___, - __XXXX__, - __XXXX__, - __XXXX__, - ___XX___, - ________, - ________, - ________, - ________, - /* 174 $ae 'C00ae' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - __XX_XX_, - _XX_XX__, - XX_XX___, - _XX_XX__, - __XX_XX_, - ________, - ________, - ________, - ________, - ________, - ________, - /* 175 $af 'C00af' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - XX_XX___, - _XX_XX__, - __XX_XX_, - _XX_XX__, - XX_XX___, - ________, - ________, - ________, - ________, - ________, - ________, - /* 176 $b0 'C00b0' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ___X___X, - _X___X__, - ___X___X, - _X___X__, - ___X___X, - _X___X__, - ___X___X, - _X___X__, - ___X___X, - _X___X__, - ___X___X, - _X___X__, - ___X___X, - _X___X__, - ___X___X, - _X___X__, - /* 177 $b1 'C00b1' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - _X_X_X_X, - X_X_X_X_, - _X_X_X_X, - X_X_X_X_, - _X_X_X_X, - X_X_X_X_, - _X_X_X_X, - X_X_X_X_, - _X_X_X_X, - X_X_X_X_, - _X_X_X_X, - X_X_X_X_, - _X_X_X_X, - X_X_X_X_, - _X_X_X_X, - X_X_X_X_, - /* 178 $b2 'C00b2' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - XX_XXX_X, - _XXX_XXX, - XX_XXX_X, - _XXX_XXX, - XX_XXX_X, - _XXX_XXX, - XX_XXX_X, - _XXX_XXX, - XX_XXX_X, - _XXX_XXX, - XX_XXX_X, - _XXX_XXX, - XX_XXX_X, - _XXX_XXX, - XX_XXX_X, - _XXX_XXX, - /* 179 $b3 'C00b3' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - /* 180 $b4 'C00b4' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - XXXXX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - /* 181 $b5 'C00b5' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - XXXXX___, - ___XX___, - XXXXX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - /* 182 $b6 'C00b6' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - XXXX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - /* 183 $b7 'C00b7' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - ________, - ________, - XXXXXXX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - /* 184 $b8 'C00b8' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - XXXXX___, - ___XX___, - XXXXX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - /* 185 $b9 'C00b9' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - XXXX_XX_, - _____XX_, - XXXX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - /* 186 $ba 'C00ba' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - /* 187 $bb 'C00bb' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - XXXXXXX_, - _____XX_, - XXXX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - /* 188 $bc 'C00bc' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - XXXX_XX_, - _____XX_, - XXXXXXX_, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 189 $bd 'C00bd' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - XXXXXXX_, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 190 $be 'C00be' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - XXXXX___, - ___XX___, - XXXXX___, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 191 $bf 'C00bf' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - ________, - ________, - XXXXX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - /* 192 $c0 'C00c0' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XXXXX, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 193 $c1 'C00c1' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - XXXXXXXX, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 194 $c2 'C00c2' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - ________, - ________, - XXXXXXXX, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - /* 195 $c3 'C00c3' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XXXXX, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - /* 196 $c4 'C00c4' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - ________, - ________, - XXXXXXXX, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 197 $c5 'C00c5' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - XXXXXXXX, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - /* 198 $c6 'C00c6' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XXXXX, - ___XX___, - ___XXXXX, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - /* 199 $c7 'C00c7' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XXX, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - /* 200 $c8 'C00c8' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XXX, - __XX____, - __XXXXXX, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 201 $c9 'C00c9' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - __XXXXXX, - __XX____, - __XX_XXX, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - /* 202 $ca 'C00ca' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - XXXX_XXX, - ________, - XXXXXXXX, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 203 $cb 'C00cb' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - XXXXXXXX, - ________, - XXXX_XXX, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - /* 204 $cc 'C00cc' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XXX, - __XX____, - __XX_XXX, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - /* 205 $cd 'C00cd' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - XXXXXXXX, - ________, - XXXXXXXX, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 206 $ce 'C00ce' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - XXXX_XXX, - ________, - XXXX_XXX, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - /* 207 $cf 'C00cf' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - XXXXXXXX, - ________, - XXXXXXXX, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 208 $d0 'C00d0' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - XXXXXXXX, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 209 $d1 'C00d1' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - XXXXXXXX, - ________, - XXXXXXXX, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - /* 210 $d2 'C00d2' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - ________, - ________, - XXXXXXXX, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - /* 211 $d3 'C00d3' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XXXXXX, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 212 $d4 'C00d4' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XXXXX, - ___XX___, - ___XXXXX, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 213 $d5 'C00d5' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - ___XXXXX, - ___XX___, - ___XXXXX, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - /* 214 $d6 'C00d6' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - ________, - ________, - __XXXXXX, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - /* 215 $d7 'C00d7' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - XXXXXXXX, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - __XX_XX_, - /* 216 $d8 'C00d8' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - XXXXXXXX, - ___XX___, - XXXXXXXX, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - /* 217 $d9 'C00d9' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - XXXXX___, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 218 $da 'C00da' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ___XXXXX, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - /* 219 $db 'C00db' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - /* 220 $dc 'C00dc' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - ________, - ________, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - /* 221 $dd 'C00dd' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - XXXX____, - XXXX____, - XXXX____, - XXXX____, - XXXX____, - XXXX____, - XXXX____, - XXXX____, - XXXX____, - XXXX____, - XXXX____, - XXXX____, - XXXX____, - XXXX____, - XXXX____, - XXXX____, - /* 222 $de 'C00de' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ____XXXX, - ____XXXX, - ____XXXX, - ____XXXX, - ____XXXX, - ____XXXX, - ____XXXX, - ____XXXX, - ____XXXX, - ____XXXX, - ____XXXX, - ____XXXX, - ____XXXX, - ____XXXX, - ____XXXX, - ____XXXX, - /* 223 $df 'C00df' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - XXXXXXXX, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 224 $e0 'C00e0' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - _XXX_XX_, - XX_XXX__, - XX_XX___, - XX_XX___, - XX_XX___, - XX_XXX__, - _XXX_XX_, - ________, - ________, - ________, - ________, - /* 225 $e1 'C00e1' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - _XXXX___, - XX__XX__, - XX__XX__, - XX__XX__, - XX_XX___, - XX__XX__, - XX___XX_, - XX___XX_, - XX___XX_, - XX__XX__, - ________, - ________, - ________, - ________, - /* 226 $e2 'C00e2' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - XXXXXXX_, - XX___XX_, - XX___XX_, - XX______, - XX______, - XX______, - XX______, - XX______, - XX______, - XX______, - ________, - ________, - ________, - ________, - /* 227 $e3 'C00e3' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - XXXXXXX_, - _XX_XX__, - _XX_XX__, - _XX_XX__, - _XX_XX__, - _XX_XX__, - _XX_XX__, - _XX_XX__, - ________, - ________, - ________, - ________, - /* 228 $e4 'C00e4' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - XXXXXXX_, - XX___XX_, - _XX_____, - __XX____, - ___XX___, - __XX____, - _XX_____, - XX___XX_, - XXXXXXX_, - ________, - ________, - ________, - ________, - /* 229 $e5 'C00e5' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - _XXXXXX_, - XX_XX___, - XX_XX___, - XX_XX___, - XX_XX___, - XX_XX___, - _XXX____, - ________, - ________, - ________, - ________, - /* 230 $e6 'C00e6' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XXXXX__, - _XX_____, - _XX_____, - XX______, - ________, - ________, - ________, - /* 231 $e7 'C00e7' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - _XXX_XX_, - XX_XXX__, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ________, - ________, - ________, - ________, - /* 232 $e8 'C00e8' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - _XXXXXX_, - ___XX___, - __XXXX__, - _XX__XX_, - _XX__XX_, - _XX__XX_, - __XXXX__, - ___XX___, - _XXXXXX_, - ________, - ________, - ________, - ________, - /* 233 $e9 'C00e9' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - __XXX___, - _XX_XX__, - XX___XX_, - XX___XX_, - XXXXXXX_, - XX___XX_, - XX___XX_, - _XX_XX__, - __XXX___, - ________, - ________, - ________, - ________, - /* 234 $ea 'C00ea' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - __XXX___, - _XX_XX__, - XX___XX_, - XX___XX_, - XX___XX_, - _XX_XX__, - _XX_XX__, - _XX_XX__, - _XX_XX__, - XXX_XXX_, - ________, - ________, - ________, - ________, - /* 235 $eb 'C00eb' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ___XXXX_, - __XX____, - ___XX___, - ____XX__, - __XXXXX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - _XX__XX_, - __XXXX__, - ________, - ________, - ________, - ________, - /* 236 $ec 'C00ec' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - _XXXXXX_, - XX_XX_XX, - XX_XX_XX, - XX_XX_XX, - _XXXXXX_, - ________, - ________, - ________, - ________, - ________, - ________, - /* 237 $ed 'C00ed' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ______XX, - _____XX_, - _XXXXXX_, - XX_XX_XX, - XX_XX_XX, - XXXX__XX, - _XXXXXX_, - _XX_____, - XX______, - ________, - ________, - ________, - ________, - /* 238 $ee 'C00ee' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ___XXX__, - __XX____, - _XX_____, - _XX_____, - _XXXXX__, - _XX_____, - _XX_____, - _XX_____, - __XX____, - ___XXX__, - ________, - ________, - ________, - ________, - /* 239 $ef 'C00ef' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - _XXXXX__, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - XX___XX_, - ________, - ________, - ________, - ________, - /* 240 $f0 'C00f0' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - XXXXXXX_, - ________, - ________, - XXXXXXX_, - ________, - ________, - XXXXXXX_, - ________, - ________, - ________, - ________, - ________, - /* 241 $f1 'C00f1' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ___XX___, - ___XX___, - _XXXXXX_, - ___XX___, - ___XX___, - ________, - ________, - XXXXXXXX, - ________, - ________, - ________, - ________, - /* 242 $f2 'C00f2' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - __XX____, - ___XX___, - ____XX__, - _____XX_, - ____XX__, - ___XX___, - __XX____, - ________, - _XXXXXX_, - ________, - ________, - ________, - ________, - /* 243 $f3 'C00f3' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ____XX__, - ___XX___, - __XX____, - _XX_____, - __XX____, - ___XX___, - ____XX__, - ________, - _XXXXXX_, - ________, - ________, - ________, - ________, - /* 244 $f4 'C00f4' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ____XXX_, - ___XX_XX, - ___XX_XX, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - /* 245 $f5 'C00f5' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - ___XX___, - XX_XX___, - XX_XX___, - XX_XX___, - _XXX____, - ________, - ________, - ________, - ________, - /* 246 $f6 'C00f6' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ___XX___, - ___XX___, - ________, - _XXXXXX_, - ________, - ___XX___, - ___XX___, - ________, - ________, - ________, - ________, - ________, - /* 247 $f7 'C00f7' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - _XXX_XX_, - XX_XXX__, - ________, - _XXX_XX_, - XX_XXX__, - ________, - ________, - ________, - ________, - ________, - ________, - /* 248 $f8 'C00f8' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - __XXX___, - _XX_XX__, - _XX_XX__, - __XXX___, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 249 $f9 'C00f9' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ___XX___, - ___XX___, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 250 $fa 'C00fa' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ___XX___, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 251 $fb 'C00fb' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ____XXXX, - ____XX__, - ____XX__, - ____XX__, - ____XX__, - ____XX__, - XXX_XX__, - _XX_XX__, - _XX_XX__, - __XXXX__, - ___XXX__, - ________, - ________, - ________, - ________, - /* 252 $fc 'C00fc' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - XX_XX___, - _XX_XX__, - _XX_XX__, - _XX_XX__, - _XX_XX__, - _XX_XX__, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 253 $fd 'C00fd' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - _XXX____, - XX_XX___, - __XX____, - _XX_____, - XX__X___, - XXXXX___, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - /* 254 $fe 'C00fe' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - _XXXXX__, - _XXXXX__, - _XXXXX__, - _XXXXX__, - _XXXXX__, - _XXXXX__, - _XXXXX__, - ________, - ________, - ________, - ________, - ________, - /* 255 $ff 'C00ff' */ - /* width 8, bbx 0, bby -4, bbw 8, bbh 16 */ - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, - ________, -}; - -/*/ character width for each encoding */ -const unsigned char __font_widths__[] = { - 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, - 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, - 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, - 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, - 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, - 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, - 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, - 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, -}; - -/*/ character encoding for each index entry */ -const unsigned short __font_index__[] = { - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, - 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, - 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, - 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, - 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, - 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, - 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, - 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, - 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, - 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, - 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, - 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, - 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, - 247, 248, 249, 250, 251, 252, 253, 254, 255, -}; - -static unsigned int font_mask[8] = {128, 64, 32, 16, 8, 4, 2, 1}; -/* bitmap font structure */ -struct font boot_font = {.width = 8, - .height = 16, - .chars = 256, - .font_bitmap = (void *) __font_bitmap__, - .mask = font_mask, - .cursor_bitmap = (void *) __cursor__bitmap}; diff --git a/kernel/kernel/fonts/u_vga16.bdf.gz b/kernel/kernel/fonts/u_vga16.bdf.gz new file mode 100644 index 0000000000000000000000000000000000000000..59b9c3f2891aa11d4cb3f83e45058825a58f94d4 GIT binary patch literal 40987 zcmXUsWk6I-yEF^Z-JR0iAsrF|N-bR?NUd~tcS{M)TV{deZ+ zd2-I2UD{X>h#l*~76IAa%)`!tPng@%+7_Yf(Ij=RHCYI>Ia14ZQf1Px5M}m?|KA5j zvf}v}M)#95r>VFDr3@3|x9M|+*(DF}RU!2~kIV=J!SV!54|I*Tb-Cq^8W&-NE>xlq zt?-+DedxAG zkLpAKQ%}Xe4}v`vshev(k|nB!S3R4LH)nxO=d%|kP3MtM4rS-?hxOal^8nAgCz;KK z-!jbS0lKFFdRmWjxSQ~b$L%ubRoYD(MYs$U9x3y%zb-N`*R$+CA`@^Q+4FGSu}iz^ zxp`&Kz5J+re|8=KpZh}@dYZGI7%2MhOV4c~eE#W*b~8Wl?(vTdZ&c4R^~_<<)6JaO z(+`{;xGk*A8Q$$7}^OT8)t9i&Y$^-za;8$m`&Btq*g%+2mN7CzaTKFf& zJDw|OPAPm2zVYPpU~;ejxa8^n-60_Gp)m6ABJIki|NXAk?~}M+x5G~@|2hut8=hX+ z!zXqNp6+}fn>I?upfV#f>B1Xki88_;;rp|b50`7717W|!g?ATkVOpN&PY+)3r{jSV zIAjxk%p-Hu^GDbhVCRv_KeeJ`^CFF60sq`HyUv(_UGzg#!pzF!R7kdl793+%dI zez=FX1wPz4C-U?>tWKifc`4i^_|;V|dGdsq+rakB;S)s6G!>GVg>hfCUkyDI*l9oDc#_`~k;TGL@qz`Y6VvBP-<7gn2z zd;Syvzq?(R>3(?Fwt^?_Y{opnYagM0d5Xim=il#KcnM#_n_va!`KftL-rw(kc2*Z@ z-Ua4HiYx|7=B<(Y-2E$)p@BE569`R@n53NDEgIX+;g-4GY-asL*3oj$hhJPnANTJb zB&`s_62*chfSs-RyW;I(g*0yLPx0Q(*k*0!;BtVT{{5Tp=_#P4!U`voH^li0BMu{` z0rJ+*JD?4Ib@XK^^}NV&{%B=8^xm6#D6%XvvcnGj6}!B7wTVzi>6*}Tjq3bivEEni z1)eIjjZvz;#fi)NiQOvX!X4J)kRLK&8G-Yul=8q|#vD6Pb=;kO=-*J}B3s!jxY_X0 zV`Sx8o^M0k*A=9~!@Y^e-7<7fSDHcI3s>v;od+M7pSNGD-zQU*S8Z(f1T69z;ol4^ zULAewf0eA`JSdG98QjdrWopVZ1LNrtCEM|{Ip^@PTX6qvXBg3y5ZOX{c$-$%Xk}1+ z7Ls@+1%I5&eHpRhxh4GTRLljC7^aC!8+-78LMI!!~sj{$R>V6k0zxHXpeo&=zb^ z#nc_*$kL;{VGIgpEKQVgX6OS^vBf8w$Qvrv6HN2@; z6MG(h`}9ifBQC&bdoKMPOwnEX^1QUfdMkr#IluSrcAX&nOM{0Vwk%@&p!~XWA#+|p zlfNb+E7q$R*waeLqY=yl)5S}p1S8gqVc%Ib4JfK55VPpAdfB`;#=5;|X^}V_eWw%f z1Lx$L1D8At`R2;6?S#D$Ie0&G?)+@Yin@PugPZ52!nXOBwL%l@=!Yx6!<<}i}p zHfm)78e08=fKN1<-u*;vQLb?w)iQw7axX$mAn}xX(}w|{j5j-7X*AnpA*Z|}S0|$p zkg>agndt#`*kX2rs~Bbcoz@Aib6%vbla`|Vo$|HWTS%|Y`Ebe8LC#fGhjeF=-^Rtj z^-=S$`}5pL2)eo7;h}4Dm1n@3*UdS=l|OA_km}ytHs9?IC=nIm7A4PVcnlQ7Et3SV zXfom~s{Hun2;T(s;Vz0hRPjTcz}{yW%kVA9+)I}+cONt5C7t`PmoDQ2*A0~)7a(`8 z+%|+ziNo!*NQXvmiASjg@l;37O$k8WT9{7k(RD!$Zmk%u$npX~aO6E)3^!49S##Ks zFm{}?``7&++T2K88mGIdyg(=JCfdBnk>pk#U5Wu_Y$6~2iDueI^ztpoTaZ*I!;1VXrraOISb@|o60 zZilIcS9c>PE+_8G>{vV9TZgbm{66K84wW5Bi1hm4`3#+OnUbZ1w`@yF9=I|7+%xJZ z`xCoZ0EEW0p1Ms_NMgsyNePIQGRjqrbuY~cG7S{`p#)=}MAxn_J>jSPX@F}14f%w_ zFrn=qPxz5^o?GaEO(0(#82|WjPg)A9)6xPFsDJR<5M-SSmv{?1Cc16V?VjgZ$-BsD zgl+hXllD=DrnTkzJhj|^g0i$w$uB^;`b9Zq4E^qSpmXe6o=p#@UOo1==&<8cW^QWT z+Z}1C>BXigmtGkO&#@CNPlf|~H8YbVS5r)@8kxs2|2rSe&8 z_ub}N8Qj4yuS+3c=5ZOo{2O1sYd4pH1Huv(vtdqJcl76v3;yE!C$oP){nWsrDcQKw zFL*it&lvSpLI-U4b`S`GWl1-ioZ3ng=602BJIu-M*R) z<3o7c*sLv!m#S@@xE9laKSLt9K<+xhEw$2g1{It06!BC!=%uL_RYBw<& z$WU~zM#67iQdGsfyJKzZjsXkLwey4u8sF=gr_7wl`LaV?S{^ef-!?Dvh4jUns8k2o zsXG7V_RWyYEYBIwg8cLD_E#Rm?iCtJbea_w4X%91OsGGT2~|0X!WL zzBNrnG%qEy?`(8N2Gj~rRi~QHkvABN_F6pMS0l46=zF9JG~p<~r*6zdD+TrPIo)XEGOCA0;TNQMpB&`(9z*QAW_<>Ac^l1b2efnIRvex|eP5{loM zSRFWdT#O8Cr`x%(=5z+}e7}5T;*#D?v8mjxVZTbDg3K97#khH{=nYP0l+CLqOhEoh zg>}Zr%m6E`go)J1x=zXP=~-?qnvwpMB$nBwrdMBYCa%Jx8tjKjkp6?u6+lL{QtgV2%6(dd?ax zhLYiYA62Tkv($<5Hm4$#FlJJGPy6XFy}Sw;_}54dtTC_YT+HNYvK`9{GEX>r8oK>o zwrKjE+gU_$_NWF3{eJ&z057uwvh|@-ldM@~+AUigy{7zkn8gGgr#O01-Ahq)q>=vd zJ+0M@LQV+zX1hqynFFJ4i^!syzc8=L|+$!KXu%P%C>)h4LgZB2tjNi_P0A zxAL!=I+{i$&v8mHV|x{Z%Pt^)6WtWk0qf4OGMBO$qVY+TkumGlP(7(% zx#V_^AV-Fyu3Rwce3uQ=WC0nt-@k0HWhwLa7)g#h@Z5bbB8G0khQ3`uHKL>Gj zK%g+^mJrCWy|?8oz4Rnq-4nmFDo3LLjXcP^vh{Vz1eJVlN+d|PS(kzF?S%p6K#m1k z@I|82%g!ZCbtj|Bo%#+`=V(PT@H?A96FCVh_u?qK1)_tfd>>3^lLN9RRObE&9F*?U z*@}jYaj%+tR4GPVyuEz4Zig7#fpZe_JiY#gQBK97Q5mk?DC>-pRz4DW`X*S|c7@>ao`3ApFkiubdb6 zLklk#pv7My9`Xc55!3~luAr{+Yf~=ct3NPpJBHMko)M_Vl`m`LC%>ngbo%EbY!zij zz;sD&JbV_Jh|iXcS;VpZjIh{tW2Q`l@oxVJOlzrlPIeJEM~I;{D=Y_v)eTs`m>@hh z^otYwW1IEr3)ulN$ReD5VyaE2@x<1^ZUqLxa+Enp7fg1?+T?cJZPRPn{f^YFL8_9{Vebxe@2R%r zQJj>+ca)W{tx4US!;KDI@pOCItK5viAZR?vR)-!s0BKied)Lh zwKH|mP;2o~{x%tlj*G{AHfx3BkW5S$m^J9=R-ZN!5r5>(7b?36`*B&c`Pu{lyebW= zTAiH;7M4W-Q-fQ6sn_Vv`InfHaPq1ijkj60)?nY+H5YmFhRS-YW}$Ei{@ln#-_$Tk zGw9g0#&Mlp93e-kdHMkgQ}KOtS{8G~%qNx?H10V3imn3I@-0w&7vv}6{zeBQ(f!Rs zCtxS-(E64*9}mrWw{%yhw2c2F)@5od1?|Ld7f~a9#WVt@|B+At7}^EQ)yP*p392T>lSDWx0&P(?PW2#3vx8xzFgfc>o{!T z!S!^m$xKu88h0D**pJ_q@28pc_iFw)3(TXigs+ua*f2=Y^f`vD0$3UK?PsS`Jc^qe z+EOAp&W-xF0g~AAB8fRO0-)T?PX=*~Ib*mJd%>xH96dBm=y7Lg=viK8)&3>fI@h3>$fbv+!Y&Rov7l*=G0OUIBB4>YdxO|&=G_9-n*&-oW zI&`s`x)j~KB4&if-& zmT+7QWBJfujBxLa8acs3^3t+Rij(MVX9!k{lfavFO0p-b|qeUl#W^ zGhIi!SjzY9w1r(CY{=1bN}XnQW@EhJ;9>mC(r#YmmcgCCZ1rD|`ewUMu`l2Ln$20Y z8LaN=jX--m&i?Jrn~(^f%fju@f~%^wk?=iGA8Sjlgez)BN?ZPB7=-kx`Xa>5Q_L|& z>Mie%e^61Mq#MmH^`&5`{`2SRX3Qi9Y(~?9yJzjFbhbfrPPmXi*4O*1QBZYz;Wcr` z6o+8M{7~;(x(}XZ;CM;>&p#x45nYK8YD76;+Xt?pwRmlXcq<{h|JvP`p%ZIDg zh#}c>)4bdSi%%@$)}Tg{VOfL>hZ}8NYq)#?JK}3>@C!rX4qdT3Hcyn*?LABLM{7P7 zdHqyE<*WinFcvub%C=W_^dELaLo9ub+43IC8r9>hkH(5EZ|mYV7)C8m;S*}RLX_RC z`t7W4j0P)WdPw$G_44|gr(21;1>|DuPb9*Sxru_HxRWQ|N!5CcnO5azA^_ORe>7q{U1eO_N_M^Ey5@9woL7?h9x;UT{iB>J+#wfw_c6IpNo?#Sd&FLm4f$I2Y2 zp8W2SpF&oxg-aW9*+QGnZ}KczEN)M_eeqP@q=t`Z-7hjFTzsfI^zqznvC%Rm){3_^ z7vk>Iai*B|o2kLjUia1d?mnj8sLxLn&wR`hUXvXR7{gm`=$hZG9xSu5(I@Cts7T6$ z$pB&u_xS{qP$vg*&5gTCw9pCaR$GaaXr1V}iM7bSeH?IA-iFMT@cG!1DffFpfI)Wj z6M^saSDg|pK(&ibUizt+gx}@%Osa`4=dYiuvU3;r|5k|*Ni(Qm9~m%S^+cvpdTbSo zAB3!9xvE7x8&R~NY3YVbLP#%nJ2AG8Q*6N616@r`!M0CIlx$lIK56`_1mBoZzbquL z3Dk!HCroP5;Axk_3eU>Q^mY5^Ep6$Edn~C>)nIhZKbzF6>fX-R(RZ3L{xqFDY(g#l zOhR#%#Sp!}6r#!(Wqo_m|GXDKlOX~mK3SaC@0siP6BuhTvuJbFJeBL2gGqdmx2c1h z2{OBnqfkH%BrQnB{!4&5nWrdVmP6xi_X|>0(dS)%^f-{LDZo`kPF{A$a$2J=ozdiv z_K&0Q4-$B{YijHg@A0l&m%=)7cpulgEYJ31n7PX>^7@C7xXP|4qUFAQ++5KSFD8LC zj3KH z`y99BkFRqh1;Nhud;mcAAPdi21P7#rcD5crK`ZJy9aQ)(qhJ73A38zXDlH-Wuzb1e zoh6I{EQ>FwlI-7bB?1wb!2_Q0#(inX0h5x(A!|x3v6P- zz5J2Qp;mD-u)L3}6FvP&I*Bi-0T9GM-4df6M-6_WOFw*l+B^Jp8jyksnOwZ8!0tcW zxK${b`PJ(n1PM-Qaj;+L$$9@Bv!-^sSh=un=m$~AnmW?gJQHrGe;Rzw{WiEd_z9@I z75}&II0I|^qxCdZar-zPr4 znKjH3jnnV|#zn_aHdDe*m#tW(u|DrSH(EtOW0^*CO>>`%3!k>*ESQV68w>UpOZjc5 zozATV56KSMjAUg3$&AqnFOcca>Zr@OS$)C^~(oHfYMiyyQVVKfN7G~ODAJIfc z96OpaKP2m|5gQh#Xsvo+iRYtXT**3%(-s?i`Pm)p_);A#O#gO&sYSL$66kHheaCJ! zKo>C!D?gRZXmymUoTzuEpT*(M+4>}wO3)of4UjJYN9ZyGw{_B>m}tf zF#WdF&=gk-UHL$dd)9ALVC;r&9(;AcY*0bsYve|&l(gxj7AHk6`}>a(V;~7gtjdWt zD5>M4?|-?%J;FebxkdS0C%@fl?iF)^H0^ErS+DY50w|EmDZrMOw2bONBH2S~R9f$U zak>M&Ag4j+?u|8iri{x96{Ew!>7QN8l4~i}AK8oDT0h9%nTVlZ>>#?u8vpUee;}rt zQ2YbS)wAmYJ&7mnGu-%U);1KLfl)0FDkj)PyAr&~ua^$0UeXnVD_AJ_yZ|XaCbn zauNcMGrZv@f&W%DrM%y$&(ci49X#1nhjwJTi4#kdhUNIsEjJP)00T73hvLJ34uoOfFp1JxGrL zhV|3S6J8{~*85VuGJf!|U57lO4Z7!M+HXBHEUQ(#6&z?c`$7d+-)7!W5(3lEEc+h9 zge4En9~v^TUJH{BkluqD=Dufaz+uD#ngl2>W;t(t%7}(_0&sRq^50w}5lV#jSiYXp z3hC!V#s<1$Y?C;^>PNPGs1PQUv)l6!S-;!!Cywpu!0FaBdX3+CQ!|nNFY(n5zZ8y5 ziRBJbNZBvGR=*43eP4~?DsHC*cqvZr)`coJgpHabjWdl;7=GMo6v*?;FhRwSs(zz0 zzPz!WYSS+~o9!ZY>?hhv=Hw0~IVSl2_;U#Fko7Xvhtz$hWWwYf+M?pgQl}Vl4x@)^ zP7|zL(~)0kELO!f`<79=sS{+w<=2?Nh=}62p`a1cY+DmERh>30nV=n|=pI|3i?zWM z0VdC@&m4k7yc2jBjX-FQz=aNCv?c}eF+g_(W1IDASE=QBa|{5zzCEJ0wr-C1mqUv+ zAflC``^@Xa)VTVeSPqC-;g{ei^d16{h4V;9P>6OfF}N zMG4D!-owd_DIu=4w;%A_+WDobfXdfu^tZU3lkaTG#Eb6(w(nLN$Lk;&tND#_lP^Ae zE1L0O#JWfY(EQ&M&Lthbjr@3jY}NFiJE=GG1z?kHGC7yNsRTVhS9fm z5t$|4UCJ@AQ*Q~8295(0iAWBUuPp@KPk`BroJlLx(%aOXj~A`t-Iwn4aoFu=s_Hcf=jk$>!rq(Gm;b-y zkI-2ZaO!(|_k6;s+C8x}ugtJ^3IyDdXf*QWPFn-Nmc$rKbT>Vd!xExRhK zEb!eYk{A44y3J1Q>(FxMH+`g>Uug9c^2pUhkm@`j+aAhXL&Scp9R2a_pfs41=e&+l zAM6gMFF@-^HW<2n^qtfm)oj!0?U>VC0=IASyFb@w@BWn&HygK6>!{nKZuY4w|5_Sf z!>K4~3^_-Ns7DN;C^PZKJOU*@6V!y9f2a0gvilfmxr*~2yHKE5gX?I0Q_D)Fn@yPH zk@dER3Rg5{H|D_2k<%|>*2NbBloP*icw0{ujrtZHB9&k%(vd5DHarTbo7Qo?f}h{; zc$8>cxoftchL>;Uh>EQ5J4&OtJB$G)FzffhVhVLS?k6Tu?~kZw>Oxt*%Q}cOo(CX9Gnn61 z7X{68rihqr7*tv)RL{L86|4PaiEL|padQxm5QfCOxH{MFu=2mf&=S4fnG5XiQ-B#Y zSjshBJD>mCu6EiGeJz;C&CK4|Vz@K$*AN{nT}hNWx)8!Q>Te_6ToK>9<#AXoN*vQ= zxR(1x8ml5UxGGn`DR)1k;g7{A7|YS4$wFY^*WQ)Is0kLBdHC-!@fiIYG?%zgZtSXj zSVZoxSc`<6V!8G-V`P<*Bm>s@dBJ9nKkz-{HWE1$uV{&YZ0#afaI_pm9%GD9;DOf^ z8|;?$&O_Gx#tjx=86oN}^6~k#d?F%xo#$C@nOyWH4+W|G+S$+!NL#6GU93ulIL6uW zs&o`Uzb(eJ86p~6iT}Ks*RZ34DsELDn#IHv2ML{xczG>Opg_qnD zlEaD>?Uy(R3`2fzQb=*Oj_2lf_en|R)}#`g`46H+Bu=iMwnHUITOZtvb0T>A4_ijG zJZGHEXrLQw$GyzinfuyN$p}T6rK{3q3@L+-VV$5XA_x}n87iQwGw|-BAvC1JGJcu& z)6Yltm{uhe6ig{0R<_pI=}v6|F*hG+sYf^1<=p8vJbm@I1nwaHO99UpNAG5#&Z?h8 z`^{YvB^2&kM$NW2TX?-S6|E040&9qhsnC|elj<=SAHf-6wWA57z3vPIf407iTqb z*QD0f_|X4{ByXhu;P2JVm7AiAO4vKLmVVw91VkEyDej*+s(iGls6X+_r;k!@~hSzq7xbg#)sN=;AFpCkM_dN|)$|Geev{o+i35Nv48SI@%Lt zNs2cuuEXG+l48Yt3dq(M)`g1P1-3j&6e^7nJ(~}rj69KVKDs-1o|i?=Bu2}}9F0W1 zWLQaT1F;?A@AhkE>LbX0beMw@PYW);R(M)p&*TXT(l93(L?5O}#XDMYAV|p0ZhS5Ty9;Q4PRO26 zSfc)fcGo}M&`Qg+&6GWCeUi^`jFj0348hKh-J*Pvwv3lbAskw#`e!b$18wx?@m*?e z`XSuAnR>3!_cJ&AxIyB@*a~meg6pq>&8na2}mC8sQYzP8Fiq%)FC;Z*p8B3TtoU{5AA&Y$o0>8 znftmzhAhKK$(hdGlcChsd0F^_M)4wA*k$+ONiPi9hyPJ#7L;tde4a*7HDRNHM$Zy? zrVjYO{z>Na`6-T-qQ(+Sp@1cIppYaML0SU`2JRx0pzwa&!A%{I(&rjp>eVK<*RP+5 z(@=O9OTRV<-%{988zQpGDfRO^eCJ{&1bhKvok>*3qzBKR?77SkN4sXqEsP`8!@OD( ztMTM-M$A|ueQSV|4PwLP zpJ6orO*edwsauwXQQY|p{yWQVvjiTOA+wq{5l5jpvEdhqHUkCA`5EVDV`{vqpO^uWh1p} zE?XY5tai70)An+e=E5Z^r5-(-%po_LJ>8GG8@&?Zl4ZGt$}i$SZF!Ruio zL^&QuFm0HkONoIc=JH7s2Hwt6E`4LXpUFG!2vO-p=e-fPC}=s?L%D~m$ni$x_K{`O zAMBSVePRn(*jgpjv z)l~Ap6@qA*z`_1^LKDpEx`QfX&rK#@#GE&{SAA&x_qb(Wx;oqyuI*D4NR??qLW&sb zynPYy9CvcEUu194Yb;=-`jUZmKy+;S6{DT@AH{}CeQ6gG#p@94H}Vk_tf_CGWT?>D zhE33niV*R@>?8wbpTkjFW%29~@V&iPaJu?S=!_8hX2n>W9{K$a~=*ePv6FQuMNhu*)k=8no~LLDC8&&e1?MDa{EhmDUx z$(LB$Rm?S2AU5BO-043`aoU*>|JBK-aHOBgZgLGWAoI=2{RBN7%k@~PIAx!CrT04$ zaZWc^dUet?Mq87wzc-U0*~0>7Z`~m7`&bBr4PzXVxi-y!IOeJpN9SQ5$m+a*%SdL} z`d=ciS`htPGsQB31m2fHulT-s?+*NV(Tk6!ZN@#)QdzriIg@HRUKoHp%9Huq*Tej^ zgE)FWTVFPWyoVCX0YdY7^}Pc`3+O4H9@ZEbQ{)TfcT6<>O7#>ETb3 zt?{Ak9xLhs+v7SKXy+_ao_-_01HaEi)*i80mT#RBrfI19UA&GjE^n`c3wsZq1u-Uz zGTD|;Yw;%8`oU-y05$C4+nK^g^TMuCw3a@>Fpo?Z{MZ+SI>G!f93`quChODMXa*yB zSzFekYr)&=oYci|tpmJ{r2I8$it@Dq&?s9~nP2j}BcSR&R+XFmT;@UdJN&^`MEQ(Z zE@ogkmjw1!D5yyZ2O2wp>BKPqKd{puYxR<6N`f$}v`j`UQE@lbem=}ez>qiGhN4PQ zV*Lm_2D^XltBMDAqE)IZIkq=&`)N9psINotz9kf{zJv>~hQJNhKbzaEBY znx-xY{jiX5U=5K~E8EiA3KFAJ6(2=_@5FV8+?f*+-T96-+=Sv|gt` zh*7|q+LneUh@&^wEqf%=*7IfL&-BHavcFI%gy+2+FQwD#d3ia?A-nJ@-X5gxuBuYJ zf7Mn<%AY#7vlQ-4C9O1DVt@zkb8N?M0;!x`vOY`Eyjdt3g(o4TIA)W_V2KAUNU2 zjS?!QROXXkUeT179!B>al|?Vk1heewwhoV>-`5c8j1v{H`iOWpm1;x@&c(FZgrOxl zvgZKs3x-vzl5KS}5Efx1pMa3p^sP0oB!A#LVgtxfHBYBWWNIL={))fwOyk^8`pWzK zT~&8dQ70Tf3gzlDTn>`{hmUou*wN?ty;$Ty$=e${Y#TcOBj7zP2_lVWf6U{|zy`%R z&jS)JM>6H9=zoa<968=bWY&JW2&QQh3}&JDWSanRpJ1$Zx-(x%Om2Mht2Qi=TYi z{3#*At6S|=M7EuyEF`QN$a)sv_<$%C0@}X3`WcT}e3=|{ApvKdm6(d(m%xub7x&sO z4oyj99?9Ke@K#$Lv19APfH4lsd@Uq;CGHId)?#`YIG`;Zo!|nBkJjqOJ>XZuslbcn zvahhHCB#6m4piFs@pQq!T9#d0L^4g2f(e=H>reDnf<#K#yh~374*#q3@2KTDd@tj! zv*ORN=-6mKhzN^AOCG{rFxDWl_`AHuJ?doeE|=8T>mg9F(QK1@`iJHhAc^rZMq8Ev zGERKiVx^Nok209m`63~<>Zss^GEKK{v!~RHcdPQwcmYoJI_Z5S8$ODsfV1De#Z}I5 zB+Wd}Kol?dvIHCsKf&aC&zWY9C(5yOvEmeo3W+E33bmB&wH+qDYZ*|qLe`|GGw0a- zq~_jLMd?l@u{3AcB9ZU$jRpWS?&_&|7zkpjNo;`-w|^JtYJL?=dvuQYrjRD&D4UC* zkh|FUzQ6V#Rur17OWXA$)@VA#!IuG&-&Y$wv0t~`5o)*QRXuVAKZkCJ`JikAU7r3U zyGRCe6FpHXS(0c$&J^0A;xZzrZGIH>A8%~^$)ktmov5DmozvxYSo zyR%l-5O<<1NjgoEqpa>%L=W#X$j`t~Q)Qq6FR@R)Dx=#{t( z-m+Fdep1wxq<#Yt^v+!}`&to*zQPat0TH%BUGJj`r(iH9!{e0_)0joUw7|$E26J~w zeVvJY{1i&z_l?W|5eBt)L=MSCLF^3CW?beNy!HMe{%s1#Pli(5J}M1S&=#>JbBcZ0 zzSw890(Fip;@3PO_{ws5>ev8Ahgb)G?(w3rf@BhyWGqrJ=eYHZt%;f5E;RMumpr zxX-77qV{sNKPc#1F40Yva&wFw?9&^jR6!tB5*=9)+Tx<@88h)ysRl3dUw<@>5u5x? zH#(1m{>k8~Zz+`ZG+owGZ#>#}EiI(RR<_q;TGjWo4#+}Tyt+I{e|}R8JhAKgz65CVP@Gih#1u zmy^Dt`pWp09UV`ER5T)(4FP3@Pb&N-4bMwm)lT*lQGsd1`c)bN!)Gha{d9}kuSj?z zJ6ZZv7IYsF@bv{DQAv$2rPCr{N9>yD2>5xgyMq`J!X{YxH4_?sva=x!V#q4wk7s@f z3wr;oOxJ{LjY|1m$op(RBl6mhQxiD&9rEFF?SRmJB7w%OoD7QE{!L#89{~+z1l&9* z9EApKtS{Q$5r|+xlA6n8KDqAsQV@bPK6@{H3<3GG<8P;w+&9nI0+IbfQa8apN0wWh z2q=FrGy@vmnTx(B{Rp^nhf;#15+OVoLAp=X`4^I%MJ+NQw<|l9q8x$fKj(Et@`*$ycBD-+52y?H7n+AzA@S+49 z_jMLx*W2G2Iifh4+bxZYEyRY)5%oRS2&*0s%eBBPP?nHKubT9i2I{@dc`o90E9$_x@ga;q~$0p>k@R*Ah1msub|=h9Mj(>BxYf`TC%uzaVPZx zkqk{h=qiqerEXrg>!CE4^FCKXQBJstU~57#BBEc&Q4O*$`mjp>b0sQ>#A+iX^>n78 zAxa$yR2=A<@UqO zvt^7(bKgsnhtL0M&kF7m9G7K?nkmzP{t$v09z&SJ=z6tWqOXv%-w#htGzj#|1{FlT zE*5!DlopDGk}J`waHn{jOxN=sH~e+OsKTA99|IMN*K6+NfR)$Be$=lR@m!!=1jmIXhs<#Shm=iHerI@oe0ED4Xfh{!=l1bHK1y($X4p_-Qca#bEJgw)k#LG*Z2XQqK`#hvtf$L zn!a`yfsV~q$dPPMxp$n_)@b5YII?v#kUst9zU0qRSL!ou zD@heK5%zjP2Mze2IRqND|BM5z62(&=DqiIv6VD=AWsBQ23n>!b>*xOMRA_<+^x-v= zw()}&DXt&JVQn9eQu866`x=KfUKTWj?Zt-xGMg~fXeNajGg?Z_Rlnw`@r0RN29Y@d z09`f=trb#|&_6N~~mixz*OvSEm@zP)>^!REPC4 z|GwOP2Buk8zs~CuMne&FoM5b?pO*3@*Xeq1){i#)R=yGWU!9=~1Qlo$$qRT@w}y?O zbS=2R3QQ15z0Rrh1339%$EojR83hmsVrCK61-q(JIn*MP*W#j3MYS>ElzT`;TO|Zn zyQ}vGXQ}TPm{K>`d`CiRzF7JYC9K<$6QP&$_PNb6yWSL?4CI2cvViWZO4W^dlqxWa z30fKlWJp}3*Nd~7a~!6KQfS4QOhsG$$OW3M`VQMj#KH>*16vqqYQ>E^dVsV&l=ogu zXi!SfVWN#G;X0b!9OM133_#v()~)T8Qc2s*=)9|-4EpnMVRo%PD|+RtjOrKidgnd* zjtp*HeeE7IDBcJLlriOfQg z>(TLAF0)b)Bn0g_S^;nOLX{SE567)8-SrScs?LPSYZ@v;I1!E88u7OG+*`Q_58>eF z!KEMdBVnkY7m*HUtGZRk;e^p0lk-+Ks39f8NZ?n#OKbWzGUBuv_`^p>u{HQ4!!yRV zj#51gEcq=I!H#*#oJV7l_cY%E6-BgaRL5Z>+OReWU9XiJ`6;D*%b4bc5T47;HviJ1@&S1LPPTVjIk(RV zrvRYGk=eoWu=-t>Pzzjx|6!am*V6i}BN&T@7)fi7=KJO}L3y9Afp_K%5s|AL(-qCP zaAl%BTvQA=e-x9SN*#427%sf2MQ)cN24CC^lL<|80JHxvWn^(gsA&xsnCINye#i=L zQx|`NpRz3;d+Xgo8`;+V}jNLdpl64;*4>CbxEgtLi11->jE+kO1-J9ClV$NfKA zk&!h?OU?4iiu+2r?x~VQ=k}_>9$zvtAtGgVhS+ah2@Ssl4)eYF`Vl_JmnPLFOo!o0 zw$LzQMd`Yz?&a`~^JQ!oP|7;D3SK0&)e9*GZ@EP8oeabu-&i0FZR?By{}`@K!_+$d zwB2-V!TceBOcv7m=>m_8W&GHTUG_FA?85u^-S{74WQ&IvYx-6!!x8uWnQkv&LW%s& zMqkl}y!+nDyl|yHfqX^zYx&B$WmRr+1#U$b_AzgIDW_4;9 zgDYmc4s!TPgt5@P;7QE(GjF+PEn{{17%tQUkWOZK$t=-xo17%g01He7hzT33^c>nk7;>q~u+ZAgF z64q)I(^W~Usa|?;NvdoqFU8N0ZwN|pq$Wn}g}#{9El?{r@tO9GQH~x&g!yesP9kKS{o{UFG9tJTYli`~-=GhR~T8%*w$i#Z1 z{uDhD#_}0frML5p?yrvQdH&-mAv04xxq5@yFii6-_C)=suI0<-)u88VoRPWy(O3sf zpE@DH%B#-iX>jmQeII6$TjTI&^bEw+VU<%#0JyuONIe!C^EgvzJbb>)x&E}CDY3Ez zQA_V9inr%m^BPwV>kDEZ7b4FecpnD&s%8;0+(?c_mQR<)T8*$y_Ya(7p^wxl?MK&} z4EY$L-+jjdx|DOfA_w!eoRmkV$=RDX2bC}8cS>aOnb8u+U>Puj8#`G&A3LXCf0mPC zAsjOgLXXGLg_-pf7-iSo9jfd9=~w9sjTxBFZcpQ(dJM^-MIKB2A5~Ww z7UlD`mkx=gLt+8x5Trpml}_oFk_JIiN?1Zdly2$nZb76=L_$)!q@)D?v;5vK?{)2m zd8XscnVI{q;C&uVuBa4ucJ)AJ+iQ+hM9WUpyrQYbnfo!k4MY%tta|O* zj-t6DDqR#|!&ofB7k_&Y4zSdw*BnmBDE8|~1Mfc&wyIue!b%GtReF5=q)Fv@<2im9 zYcYqSFUIJd@7~7sGk|!ytNV&)cHf!TW-L?);*qpve5`IHhLKs z4>P2a&7^oa(E{eH*$+oOf!HfAt!86L$mNf=Ku5Lx@-18PInR_6MsSM=O(H8jPOWTX z?4z<0sIb7a#P^AA@OoOXBc^DTJ!5&M8V#kbs3}mPT-cjec8CE?n93a*a!OT_CK66D z**!$HI3oL`0X@?qNvRb(^RDcUOua8Q@ohz5pbmY9D_N|31mAu8l!xv12 zXglqT!7XfzGuPwP%@ud-BmP5DmD+B<@Ok~87Awo)x>v7b>0X<64KJ4{oGLd=>GQ7Z zVGVheuZ1JIhm_K4Q0aKezv%6P+zp&bDxdkX%0qT)#TCZRVgtyRC7QY5GSeCI?bN<} zTl>5c6W8W}cfYS-KSY{~-E0b;*`{h(NZ^>|7%}(Ww&F+pZOIOumJAsYJpoqW5SKdB-s1B^J6B8sp zyh}fj3Zw`t^kv}7qQKxZ#6R9*e8<-dpnI7ae36qEW$<&`&gfmd6Irb$H;v))k$$R3 zO$t5~6S=V9yXnW)y{h>%Uh^OsWMsXtrjIhk3o=hJm&uK2yUtjh9imw2yBG3}EKweeH1TYl5m>LPT zLl@2lMPzthkq}9c&{!9ay8fYpiYLf?QND>2+WTH}00(|+Q^X0QYy8Oc+j_0J?fJLg ztd-!h2Fe`V49IkYA@^q{(`KBrp1%1S1t62YaVN30xupe0NG48Y(4}m#9FHNlp`Uf(-(XjLzTVr zzjDvGdf9+TPjGnU;1Z1tl9;AFKiv7D5&Qvg9-PS0m{ej1z?O6opPGnP)EdWyQ%cHSE8~HCH}okl zHH&N~Wu%27cl^~bAQ|rh0#>pDlq`v_dnm5R(9kiy#I45R1hysz12;Esq*s>g) z5C_g17+0fwQ&=0q#K-XQGgasi3mez`yiLSFeSgVX*csjEU#{;Z%9ba!^ZH@02aIW!0k zeGC9Fmfm)H0wi2y@^^flhL^+28DjKOkk4jp&mHx9&eFx9%RDLnoHloT7>#bD=6H&* zcNq#Yzf(q+lOd-|8>rc*>h}agRJ*4W45q$M)8+IMA;?w-Cu;&hEc^NPcP#_a9I3y= zwRn&}yW^avPzRdTwj>lq&7qUOt4k^=93g&ceI+fp}biCXq5x z8|LH9{^kHxS}+dFM)U>)7-`GQ-o~y929xhDHd_VQ5A>DU&OE>YyIIXDWHRetAt1Zc z$JRx08u`#iZU|WLOey!vjeaOVeBySJ^lVIe){kl2ZI*EV&N%A>F8|M{$ z9$z3n*!0DOEal-lm|_!#DY0n)jaK|5sioR&fGqS*8!PXaDMrMU4i99@&Dp(e5K{k*_{j@rx8%a(0j^p1d~_$&w_e^hC^y2E7&-ur|3 zY}rQr^sAd73mmVPSK*<|Ue75kd7@M8hW#o?4bCrlfphg5fDdee z+P}iB;pAV3f$%~DJG(C^;1^16h%sLc#J{34GRh(+a-@V&1b}hBl$1X)pdfo{R9yHe zzrPO^ocbBP`!DW~-voo=*JBtlz$=<-PADM{)(-eF1-`$rMY75PF4>EAy4^NM~?ye|` z>uQ_@2|+Mpc%s$W)+#yh93ud5M;7{#Cz(hmx7b1W@PKkWeJ)T42{Wgks>CGbrf^!x zbs_&OISob$k;7;Mr9^nZIK28QrBgZ3W#g$me zaKyiA+}|a1LTqU5n%%^EI8AGy!m#JKf&{UzKkce5;wVdnVe(bMPPu!G+0F)BWUo%s?wOXgB?I&RcMkKw(!};>bNhim5EhU@KAQ!`JUAK38 zTLla5{oc`PMehAPn6lWgpV#88*r@|tC0;Mi%pFym96i{prGIqb3G(sAes?ecYpEGq z+*u?Rm6$5o)K&oGa(=kH0OT#N6ficd5>bHdDBa)?mciWxt?ygu#c@}1a5NJJq8M7=Ga?!#hhV_3w=D%Zt4V~00C;)z zQ{}(7*HLNX|ICGKCV*5GjFCjp3!g}$8Vg{y6Rb|~iC%pW%88}MlW3?~6H=)NAUXc+VpRp&Ho?Eb;iwdAyiC8?{h+Vz{AbVDiL&h<#Gf%e}9;a?|(+LRj#6FOs51bsZ$DQv2TI{?~Upo&KKm zt@dC~95(Ce!3{92st;c^hMi0@Y9r)wt6biO)|h@khIhf`Jpd-ja{_d0it%w5p<$~n z;E)T9kB12X$5(^5{zlWIH$!0u!TM^Xdm0C+YR<+w4WfLOTn}QZqM8uhfA~r=dL>-= zen#`=^elhB0VrXdy`>UPZ&l(?6hu%Zy5<;$iGg6i?~U8onZdq+)m7VD$Q#~Q(CYC* z6&#av2XTVI$xQTvT&n;BR&;b&TPGq_Z^QMMcm>ap1%_(y9H4%fh4Aw z^>&f_3*CMuI8h+fNBQJ+f)$iwD$y1aI&NuV+B=NSMBt%G(0|WspM%Tivn?kRy0fOz z8YDsJ4Ne&9-yvxr!JD*%5fpKhnkA>?;5|%vA#2qQ%etRgMTVNF;Q)l%_6d1dh!M6nILHTGKE8R4zVLgVjsM#1=Cf$(SI7kO} zSxp&+(MyK}ogBO#P{*2vXWBcWYn@k5ye9ub{5PCf@|#$JZ{ z_+8AHBe9&!WK#Bch~O9%>IzSg8n@Vy{U-gWmY?$!=rf>B@A9CCE_#lH-Y z-Zib1#Z-Cqo=O;cQ>Veq4&!!2k6oq9$QgdW?pl=E5dzIMrB)rr|JOQ#k`p(w&Dyz>RALuwaNl3mfd zFAduqr|E&UMfY{+Q8O!+n!djC_TtYx1pAf`eXd;>J$My9B3fq90n4RH%9^e05AiszL(FJ-u*%cU+_$>1Fk_` z-&a6JZM-bN4sjYq@IEr3Yn=ZGeA%akW2fk0P{(H^a4Ly1Q6)eSM15FQJ82UwX#>PW z<6gtU1ysEhh+dng=%h1_b}{!}#V}gSA-MLmm!36=-F1*i`_wZd&c}>f6SG@bN zSVrg*fb#?_hm(Wv1bNVZ;i%!r|C-VBZtHc-3Q~h!#R%V*1`G4uI)gn)hnmc7%kk#? z{=D7WcAHdxex`^A#6Wmh8igCAi9#rtT<2V|^+3=XN>fX@B$vmCz&w(taW`bs5Lcl+ zHG{&3!w~{#P`4O6*Iqqv6`fpY)#25H8jhBXAfYDtHogRG7cKCDjAd0CsFCz`N)nJE z_pB3P6!(5PzIm48PCQ`!!T3GTBkRDFNH47%c1$|R8(#cd2~7VFe^>6Xh*gO=0O)Kyp&pNEVJ*l_YayO^3{xLyeG2!=HG zaMg?bg9WN6PE#kU^s)wsm=T4xe0gs%>5lv}RQp#vt=Bb^y(>Aii%IBpe`J@~H$r3N z+Yej%Jbr5gxNCK-sQ(xzYI`nv=xRoS15e06_?uzs6KvGgWFlz4GA;?f1Y02&YPZiY7CeteUO-pnLRN-wP(JwxQW!ILAB@N^h3jtdESz z+16I1Vu99feB|?p|LypTVG$F_NPYT9mqIr+VP^UQ)eqdM}BwP zysNO9=l1Yabs>`jS_!d#e&=pT9E~oYJ%`P&9NaGJv)GdGA#w-G?$K-N*4vvq9=TBe zUrL*CR6$k(FW_mAhZfNxPcm4IqP&)pLr#V3@c_}6`g10j1P5(gxSVfKty6HT$4qm* z4~R-Dd=qY#EHs@=bI655HYO6CwRH6@MX6pK%6}J|2vIP%I_km6K#N z#y1tHT5J;j4LxwDyWpQZ{Qhl_sB!=C)j2b5ViW4eKD5@D_2WM-cO=gZeo(x)84w(0 z=9ex|M1whXt86-3qV)owzPp=+jY+KvKi8Vc^)8NToh9 z^Z$_YYknKrQYnURFGxW(ok(Wd`^U}uj?bHdU|RP+2FCAgnayNX;gFcU%?o0*W9Q|QG>TH^cA(|HT2HiJHW?#7k}_v&1r!?Y4I=9 zK=yFMXF?}S;Hfy1EdHBm^lS+m?&lNP`X6x%cRu>E^lwFP*lkbWJ3pWOy?#rpBneOZ zYLD8}o}M&Rnba@Ga;%a+h3RKF%k(iyB%VV$-kH;SP`6Bn=8%?*8Qb;jO<%U)l`35J zny~R&4_~??m|2DJ1+**>F!IC?UPv=c>;M|*K9jMPcI7~&In31IzF0ecntz=4i*R%I z@GHvZY^zhzN4q4<=ShERm~;MiXs!PGD`;NSet3JdhJ)pLB1FsHy0U8gl|If;cG&%W zUkCdOqJ}r|)tJ9YXJWq6%Y17=_pE{ZJ~??g%hag2Tcy4+fBb|;UsgJUp2_lE2?tSv z+@nTJzcTetauzk1znS{6By^6T334orm_cRgB=|h~p>7Er!V)_AOiWRVP6W96Ic%Bx zP3RVG2}9pVN>)*jdVez|#eaFEpNlCDl~ji7^08xX2Oi^!~ zR?J=$MIKVWhOEj!TtS6(c|Z!ZCDGBWQg~BcPPS_Gfn4d+wZf3 z%PfeT$TMtyZGr;b{i_=x<}d2}RvtW2^ct4GXea{V#dbxHhjq@Gu0`hdBPnTt!_FlX zRC|+Tl^@ZIt^zjD@=BREO>a9;P`_J?)~QUY?~Q48MsTqNZV41^OZ7U`-iSJ$rrGv8 z0b(YA}i*^UuUTB77Qr=$QFMWhB9wU0?rWoBOZA8O`umrbSvcp7i?L!&1U06mACv z=2eOXp-&XNY@548qVd&Yvmi)~Spp(HWzGVFj02m4{#Mt2|Dj?waXkCQSIk#C^KrMeSUvgpdb>O=E9KL6Trx zBAPcHY9jr^CL(7i?e=R>y#8$Z=wD$@TtiT!-jM(4yLYt z+g$sW7jrdD<#|cksm;}@%-fX23D-^>tSWY3iSo=KDf}hH+!(4n0zfuFQ+Md}0Fvt> zx&{*FpbZ|Xpq4G$5TAk`T`jUH#wIU+T9J}YS@Y7jC;ps9NjYalK2QvZ6v6OqHg8$% zXH{pB!N|5#SPU?H3Z9baIG>So4tsuCVjzeoV!Nqd9t?I=r1@W^>$;_U-)XWZue9|e zY8KBz_lw?j(IvXbkfQZ-U%w2PoSW%6HJVBIOsnAdDInmq3j1wy8ci>Sq`G+R2>O2Y zs@wiqtp-E59(X9GkJU|8-Ayu!xnY$*OTd$K$(Z4f)#yk=MPgeHwoJKUcrZ-)Otf4* z>(`f2DZhEz{TD`G927gYuq1?jK2q0gKY_1m0 zZ=G^AJQQ}w;}=rl`YcbPb!eW~`gF?w(^-_QmN5(aH}^tGwx(#x*Z7mT8Y!;=#*){G zkO88{QfOz?_YB|AP5wRPg}DhFdsdh`ue@xS_C;YwKy%h7xvScr%^_1}aLzz&xS{@g zj!ILU8z-NI(j4(T`_QtuPE+7+_;KZ!*W<^KN2|6bkLywy`Y{38`0lVbuP&Hy7M|>sblm(WE0(lA{;oYqRz0 z(|>=;Z54N(_C~^|LBpv$o|g1qV_nv?q_cfHm-{-4+LfURr-g8r~tOE$L zm!fmQO9e;oc!6QOE)IBs?v7gHOd%o3fiDzV0S5Nut;lqm{MhxBQD?MY(ei@tuncs2 zBv1#QX0UQqN{F*L@*&9|NEN|Qao^4jPI%)9~oq1UgF zK$mw-#h+4$_PIn$d1PZ%uhc)5Oi4=KP@^F$B2thvSe|)V9&8M19OeuiIxqGZnV1}Q zF0@PAB}p3`yFKA|7o(VaPA%_KGM^CnbXW`WIWDm+J#eejyVF2tSCwcH2To!O6$VnP z0i~0|DEa;QX1khcoSthYe0&FKq`+X?SNT+-T<(<(IL5B1b=r31Z8F;P;K3@n8m^KxRYU+O`Z+2FSiWa1>W?5F@{ zY=k!P*8HV^dh~)Q1hok}_fEP{s_q2HrgObWL{4ZLG;2^o#z@!Wi0U`kKbrMHki4Xb z(~d{SxF0=dtU(P`x8K=!xEw)+Ow2_yK61mKd(Xtvufn2^PKv@ILLi9S2>&>Gphasqi6MNQPp=(B41 zwy8xc%AelxDI+TCp9dAI-*XwSh+aw5#5ru#p$2rXZ&Lq#@R>>hfVpHw0gHAL)V?;M zfe=K33NwRAQ8*$@l3K%CZC}9*efS3#3F%+3R)8{-<)yv~AZ_#o zKXpDBU6!|V;G8CdhI-oudef^S?;E-gtN$0yYg=%K;)+Mo-XQy%tyB5CVehe=9Y3j=&#+7uo0$qwzR5+j!3{r;ns?Za3ILj^&E zsLyU_)l=NHP3GuU_Y@LOt9RmXd#ryJ{N9J(F9LroEBY+i_z&6+UG zn27)JLjO6O69+Rp&B*B;<;};kk1i9q?%^RW+tw+vzjgVXW-n^L^6d_6ptwSyJQQu0 z^_YL3b!K=6$PW#6Yy!KK`~7hQ*uWYaggJ$#2moynfk z!jUCqeU!@ywMs2@^ZluT68}P~gRWGyXOWw>Tf6M9nTf6G2~1N3G?>t*@_zL0r&T|FCuKgS^<<!Y8kQU_D zO8isz0tlBi3ACDLtP;}e88vel1GQy!_xH!d3NCamD87N*1p3vDn{Nte40IylOy5P> znhn}#+)6K(D>O3}5_YT>Ay8%*i3cz&SIv3QvraCC1=^#6Bk6me~hw4`0i^I+c-xlxGhe;@$i{ZFK%hkUsBqH7p!<_@<;zeQR{zUfwbh=wpr8!qH%2cW>HtR{aP4h zHi-w(iuuo`P6t&!N%k$Ko&;e!t6SXh$Re(ugo&`-Jdnfa;mZ1TtWVQ49-2K`ctUGIJ1dGXN zZS_=rrF=VFFFf7< ztFMt-zW|xB{jO9*BRLnKo%!yjQd{aKcVRkEH6V(UwOTUcr#1W(_w$YeHR%U!K&HSI zS9AISJE8GK-GoFfMYPVNI3tY@cE3a#JI^pfdR&xG*ayE&mPP~!BYu19%9tZU6;hGlCc~LMbH5r zvG+t@yoObJO^BZep^U+mPYw$EGz={si+;`wvT?CnOc?5sDp(2)#~JP7ZQ?uZvr&5T zzyNO8``tuE9&h0gwTD~tf$IaVk$LiL)5V!Dn70e4NcZ0v$v3M{GxNF=NLNXXQqOad z9_^&x2OFsqn%tWJb`5HOF}5G;Ss4PPY5Cr*z6v>MxMs>9-r27hg71;(%Uy2g?!Z#r#}9xh zE74?XDZ6_vMJe|?1d+UOpAO`nq(b1(6&oe2Y|W3ggB9+kqHj;=$U$80P6?09-S-A# zz5F00;tN!b*V7o%$07iYHJYAYL8>qL;_={&h#FL|Q^NKlM;D- zN|WJyV4kI8tRv%0q0+J&`~5N?>F;J$?bS;?tOUG^(7K(SmaJsa5jkk5Nv7!m4HY>_ z(l4=(IL?)4@b?Y8Dm_$Q=qg`JzU?`E;V;Z5On>-W7+&c$R(bA)PkSH{fC?kX5D71P znDNePsLN@mf$Og6t#|uVHf~60!ldv^YGy{7kf-wKBVXB=h78Xk%0uSbSGDD5m8%uI zKQ4~X&axgSy^!ptRysM-iT)Jc;#XAPnG*lHmE64YOIjGClKOkNs{!td_hj^H;m%W zj7ly?%At|+TlOzKl7!!m?|JkrtiwpabpyVvU+{k0$|3jy{A0w*zSt;Mq7CQASy%bD z|BUNV>mHUiJvwZdgP70$m^{8XUU!+m6Dk9nc8fPoZiXJ;P+sCFuG0J5h}JvHx~Qb&^9RBme(B@0 zGM@;qoTat%WQw;Pru3~-h;VDi7cVm1wzDXnn``(Tie(~w(?0YgmWm~6pwKr}?-dS7 z1T(VpYa^tFN09T`Z&;v_OYvfJ#i{$`O`|&a=X(>+cNN#CF#3+oAmaq_Ow~ ziGa4ZZEtUwv*E2!4VKe*7AiUCs4=xH>mP94mx4k9V{%@xSu|V_x&!$~0~Ke;q_v%5 zCqzCmI@J_Qv)Pus<(JAUfsf|V#|hKQB)U zfu$!%0dy)i4Ju7D3Tv?v524VuxPkzi{@t6OgF-(Z!eDRcRBt>@!pn24cMINi#^6aR zKAaS;{5;9|X|HJR&kRy|Kbzf7(E?XBf%*t@EAcdAJC$ydI zRvqTPk}>VSF6>tub558)O}1YolHo}Gg3u`Bv0FsJOz?k_$^ZpQku|C3lk?oOTz#bQ z)=E=wvvs|ke@GL}pw+%_c*IGR#UH5ir+Cp8dSWgvgLoSE=J||`aA#$KKCm@uAIhOd zIrk!50q_(rda^SovQ}YpP0!;!WGS)shy9R(lrC$wk8oq-oXpy5Bmeig3G-W60=iLU z!OCZu8KQZh(zxP98fbTTw^$W6{j?~><0&!@0zCH6`8E< ztW)RMhAis!!C;lbL%afX!Tfit69-wjc<4;XNcM?qU!hN|%>H`)$KO($Yivx@qSR6s zet*FdX^ywUj(u1XyMTF|)Zbw8ghim4&D4l%ve@+G=l;SJL+MsP%#Htgm7!Zf{xsdV~Y{_dShnP#6@PfEz zaVWdM&*56{rna_W8+m2*sl22|GCt3iA6$Ze#?u<`=6ysEuM*Y0OrY|fP;#8vBbKv= zPh|Y8X(Lpc-5L|2FPG}9%CQo$bb&xXhXe}~p}{WAa@13J($k3(6ik~wTXFHD1_c)g zCmEq@9$AhPsKMZUnB=2v5~ZMQkP>D*A~my?s*RPRQCh*T!y8x8fP>E}eIG^QGhA~V zOmvdH>pBtfkv0WX#{3E_4 z@eKkwTFh(`o($N=yn+y>psq$j2JET31&!_Bpr0t1e?vQjm~r4D2y0Qrc<5-@rSB}z z;tVFjQpplY%Ry)oV=3@yVt47=xIq^$g3sp~A7oBc;_kq&~)bBk$jAPv3fRDH-rgPakg@yFf08(aF zG>)^84sCTr#C{W`3%(!L6k757&s4|E=*~H4-@1pfTWDs|9~Hg*nJr4||51Y=jGLWN zwdHAN+fDQ*!X0vb_32=!&5^*jSlOFovk0LaFWkfm%+`(oKObXmtbGZU?P(ttsLKx~ zEOE&9F0MD;aa?0BpWES(@TF!`9Qz@hIQ9n%>&-O_iAk?AF?@JO(Ni0>cHjI?5C*P38@g0zSRN3lQ(3nX_K35uP<-Z$??CYAeo6S{n#VmrswwcxA~Gq48bL8`gsD)uaVvy-xjExm zD2T66v}(-=JL>zC&@r5DniEL;0}wrivjT&;5lm6_aKayb7(4s#!-AmOdh6!q=Hxnj zXa<9YOSAtpoI9UX#j)>pGoV5CMKn@(RPrs9=}Xr~exp+&_OBPmfB*R0ovmaAGWK{; zawzQAUS$R8MN0+h_p-c051|TgmqkTRmW$T>7FdO6^nbx%p+PA~EnEY8)+eu>QCVja zC~R|Be3br5C>CtcNdQdh5vBK@^9zB|A%fARHR7*K$UV_exgT^;%RXsh+pXQ;*S~?q z+?TUvuN}4Tcvlt(u_pLT`~=-4!bAwujI>S@vq^>7p*@W{NdM3CoPQ$Pmw205;B?e? zd7g7y>_PJW_Ra0t@x{vR%|q;2d)3Ev!3<>`g9P8u%E{aw^yqT_<3&YXh#vRJ!9pej z5RMs1or!IY+D_xnHYsrBP9MjfabHT_ z>xamz;_v#M#1XJHkY5Y?au}+7tm;D6!}`w88lln$;+j8iZ95ePjNVNlcw(L=rj&;b z4+!x*&oQ@Z-DyiVszSW3z>-<_reJ11k~%;|6w}Gn-R>rAAOZ?#taX~H92{xq_wgyr zu6=Vjh;0bgc=>t#y#tN>xYzn4TTR5Zl)|bAMaEFgEfKe0u;V%<_v!P7Z#rwh!3i$V ztzi9KF?NPk4cbG^A)YgWfyGe+i(|D%SQBuF=mv;(<6Mbx(^arSLnxCxOK*5Wt@2^4 z#s5;$?~PB!#|rUYcxZ)sB3neIA&@!|OenX?D_j0j&X#v(gAfpm@a3G{&fbJO;51_d z7a_?m{JMOUs*e#O1c5iTp2)x07Tr>m(!!LaLL_->ltc#G6$FqHDgoAt+F$niiNS`V z_!lW45e>7n4oQrGxS~T{KdmxN^Y$baf$r3au$(#M1;IohKp~-zlkW|g7&8Q@D#*;I zdtxT?JKAnip#Z6nM?*YU&-6~saYK*4OKY+_I!rHi+vI%<0~7#?da=UZ4Syp4;(%Gk zn0;-2OwPoBitvT7lYP`{GZu2t%?&`i2=dqC>fbuXl|GXzeSL>;7Q(XwJNYhM8E?T+ zL5pw+Q;hc{@h85VcdUV5&l|<8*?l=5XJ~VaS4)pEd?{ zy@Ly%q)17Kw#e%^M)1x&=o%#+**(kr#4xo9AYAD6ON_v6% z8ROZ`tfLMVmm%g7HhLwq#xJhn$ol zw0qFM`IEN=2O9OhyPXd&%3H5>YSY|Lz5VpNgwyP9Y$|aQ_@|v8LW9Zn&P>gYDA`U* zW_JDPW_(#tO4fUTu*VKj>M!d7vgyEbnber(o8>seou@(fVXWM_W1X=i)9?z5ShXr? zdZpLQyk98<#I%w#tvq8#>4gvK19ckJ_9ag5(wRS@mc-+EIK7WxJFGv*pAw{zx2-Ow z_$T(%WVGC8t@Ng*De;0&5Lb@pU*TDQtnr9RXN%A`x6<;G*8DKPoEV{iux8eg&ws?m z3JP@%r=%OV5Py+uMn!nUUtTzNEIqZxPmcJ1YzDlQS4U#6V}`@m`ngmR9=ZPPqkGx2 ze_3}^=YQ*mnhUMaUYerUz$DvS{algbIgcrt)J|}@u{d1t_3W~i0509q>YGU#E=}cb z>hs+yCULZb6fPYA8*EJ*Q?>hkV_GYpXMR4u=~W|0Z$tj8)68@5A<`}T5BSIRP_c!? z7A#pQ)DLWW8JZue;nG{8%=iN;Z^j-|K9Hq5&7fN{ZU6ZER6cLwyo-#k4viqd^eq|{ zcFv)ZK*!q}^xP%_q)avai8kofXxea;RTzB>o!JNFsqh(ml&FTdt5LJ0my1d;kRQhs zX1zM9(bndF=@$OJ%a^?Dd*d?)4j@zB{B7xsx%Jbb{>as4-{XdheIB+N=Nh5$Rl`FUWl8E-24PK*&4KicHUfQO z>mN7scoW_#Dy-A8N92Y?rC3KjHYLoR0rwM*mNH>=%KA zb+Q^YP2h;h(MO_kfQ!M|nR{;I?)h zNGo4JEHWHkW$;9_82C*>&P9e!y62p9Bv^@{Fm>u9X|%#D=4j@5UVL@vh(QT#>ib(= zG_H@~iVAg59Pz(l&+><i$;kSNdx}R#h^Xe<0lqBOQOGJO5_bR55*5hSYg>oCG?$F|NQ)t`u>B67Lzun@0o@gQ=fSA zzzFcbe0F&Zt| zBOG9`r)azp$r#MqXTHI*s%cWTM#>fIfDcIWh<-CAMTzzAWmWB*&*EdyNZJyqn$)easXaEdC4=rBPFrT=nNek?)@7 zO9_b+0Wa2Vm6(lVP8@PMWfh{TP#jr2clv??EIsLlr*7cM$*Ru9k`Ed2X7gOJe_Kp z$wJR>cVJ*BR5^!2|IZ58=6r@0>i9}3ZXyi)bn9mbKXgq8?6A<+mQ)VOG^A{Hu{eFu z=CmOe=8jDJtvy5`hZilKs@cBE@(6~VPNbQ%ZAeoyqXDF^OxnJt;n5nzT`YOq-%XS- zf#o8US?k)kVC!{L1hKE(le5Sn;vk4Lol3#q3cQuw`;f^A5g}BPrMLO}?TE~&NT9l+ zR`E>wr(Qxymb{ejuS$VAG3GnIcpt~tijt6=xDAcU5iBe?@RG7>wCGbpe_gDDJR5lSuvZN~hWQi!z>f9ym!?jYy>b->!QN(<-_>Id3_@Wn)Cf5vcBOErB~bDGjKE7XPdY5 zSI8MZ6|6h73{;%G-+$9n6|nT1bn{u{r%xfPouzn;Rnx|M@t&A>$(vmz&i4KA1=_FW zH}(&@DOs)mNpoNuM3OzKNdWHrGccmtuT+;F5|qcMaOMrd4j*EfFkF2zk^ZyFGzil{%QmiX2Y5>NgN~7E-so}9>fs1{ zRMf}4gD^jNar=NTNyX?IE9vkDZg|o*Z{-MRF`IA*9)xI$)l{#f9C%yLW#$`TWNte- zyyLnuxt8jB3D+jqj#XKTJc^eId5rrq@FmQ?NgV1zK~I8yR$5FeR}faLxZ6Q}^6-cQy>9Bs#$fv;jsKiBZ^sEZJ|~*d=`(7Yb?Jt&Cx; zh~Hg-fkV)0j&JVU@z|g}MddV0FICK1cD5#*2zz%2sf&>{6i|gnnq@K(;73_a5p3`( zKv*p!N4qDBzlCy#hzgw`nSjb-@e2oorGHA29iV=AX z$B|s6r3uA*w z8DqYCx`<&R&JcS0EzqT?Fk32ewdyY(-*z6`ho$@qR`u#(wCNat1D{` zs){61#`F)xmC}(y$4wg|T;535L00chC}Yq8+XER@tw6@h*W%21w>1gx-$h(r9bX=N zL6cOZur5i>3B5}XNtS_L4c(aNjAGq2SG~V`fX|y@s-4ep9TD)YA`3;TDW~-VJF&PPy%zof?NUe;h=R-(Jb3qT zt<>3BGCHf#U&r9{WOj5Jk&|(lXamn49Us9`<&+sW^~_auM7IO=SkifA;2Noo+r*+d z6IzvH#mh~qP>&Nx!-x(RzA&UB1cXSi&i8ZNecwo?6G8~6`^x6bf1cQM?Fy|RN2pHh z(MlM3lh`{1ztKVKS?XwkF@onz(>3PrD*}k?04{dRW(uP=vhb$HJ8JbGlx5 z1fE=g+s(g_Bu@ZVQ_@ma4KtSJe?~vDIGm1Iv5=TBOMwi~h<)^4o>}Uvv3e8Y`YXSW zfe>keteMO$!;|OdbLAhZ4(6|kpph4Ko)2;{QF3m2dG~U8^!MviDC#}&>#vN){#Rlh|XVT)E!*VKln`(+xwala=kaT(McO-qDxUt;$OsF_Mjr zbCW3OJL*n2<{xl=5m;cJ2mr%Y;o-V*4zB4(UvE-a4S}MFNH>4OR;Sb9LG1PR)>{f& ztZbr5)V~959t5bPN+fl`xu*Gi{=Y2L;{#{c+vdK9;JBg-bf49$!>$sXMW}VQ+Mc4M zE!WIU&u=}m7NnM|VhmJq`Cmy_9tial#gWY!s}_-Ct!SflL~wT~!%8TWn($Nl8JdGd_nbSr6>#d7IDDHEFKpBiUnj zPYyOm>gypIb3I4fp+_02o@Dx4mDH^Q%a(1obsWxD?av%5_)}Y+%k|o-5sg$Wf0Nn~ zZGN@g!kk<XuYmQVGg*B1a9O zrS2=+dM7la;@Af&J7?>GDFFpw*?IpvO64iSH3h|mJB7SEZdb~w>*iqkbv9p8eu_SfuHh)Ai*3dtu&{25`D)zTR6nzodL1$qJO%tyW@PGW? z*weZ#dS&x8hS7syreE%@etT`i7EHXnd|sq?XnR#Ix~A(LDfyK)wK3x%`AC6YZrT_3 zuk$&u9wM*-nN1n`;MjlyzF5DnGwb;Mt)Hw1RS;ICRoc3Kv4JYa6uA1B_#qp*bRj59 zkcGnsn&~c_+$;$K!QU&h2iFi@4$^T8K=n09uOH~XXZg0tbhDZc2^kouGi4u}!hKL? zqWu3DUTEzRXLyR?AC7oRiLerUSo&*%RfWo~`=G5iZAfJ2p@ zJSd2#wKSP?Rd`3-yyf$5Fo>~+GJJ;CqNnU>U_lSg*;A?K@k_oQbFrA`D>&6!UTp(+ z-GUS$W|O*Xvl zI_Bpff09R?*qs_V+z3fPdW`2`$)z&`B{?W4Z1f;a>f4-tY{v^H*+a@t;gg9#ggkckONc3$Un za!ArOJnijC1#48WyRXLlxLK%1_*mu0t>cbBy;It6<$@VCmW^`~Yo4s?r@f4sYY<}z z(d|zas~n!|;$|~?*~Q5$3$X__={#_iiz(y96ZoxrTEUbx`2rca=8ajq6_V5w3!cQr zm+rVskkb((ihiTmLUYKjz%9gIO$%xbH@g@Qjk`U?*l6d2jZajiq*yeMvZMO); zP=rPGcAwr{y-EqP3dF26`T7ktKXCUx1&6n*^NP0Wny=gS^Y$%HmmYUALOBJ^*WsPH zh)Yg&BzqS~kGJ>)3Y(%KfVf)180G}Wsm6CX?ue<#8)rs9uTGfWB!%JUw-)J6-YYRn=9a)d>`v+VW!`xFkcm*taN2K^nRXxqhVyqU4|Vlh%5t9;zypI=VI>3qz{vZ#Y8uOaJi}k5_UL6VABY>1j-0Rc!5+JqFQtjcDc^MtvL_|?ms`~c z&C@CoFj`KWJl73cddR_MF2VWn{6W)rp(gRRp=Xrw*oD|1Q|#82E#1oY(NN#EWLS~M3#)!pHz0_eAB;47QaQ7EnGKf`1T2ad2Ch>(n!#(}8dhJ}hJ1sMLd2ug@np9Iqs>)1t#g;~ z_OV&SYeZwg1F6$@W7l z-mS1fL8}Bog$ox~?xDVuWwH{PcVx9T{$=)i|R(s>QACINkC#66+1K(2Q@|2jw z-DYBUULe6 zLL?I2S7gswe+gy%YN5;$0VFnB@{G#DoS#+zlfkE&q!jnS7+w?oQHN<76NduoQWWR0 zxACgimT<9vVU{^Upb>59mt@nk*GRgDj&ZBjU5K((`fiViybm?ddXEj6!E;m7C|>>} zZH*1VdUJTpUUf8Md#oteAY}?v1PKl8^A=0?Q0@%BLx?`mP?YNozo@O^yRcj>bQs7Q zM39H+j8{qmweQ2qFH0FN0Gl`m7(2Fo8CVA-NotVqFUHSA9yL;FQwA)+AS;5b#FK4h zc2|uj$08Yj+Ue|w=}Rys74J70_6+FJq;C(_I7>qa3b67;p)S|9(D!U^-jax#=-+|( z@#QyV;(%m&W;&05OzgJ7vv1vF8w~rQB}Xff>j|%$1#f2jX;atIKNTHLYTL?++wb7s z2v=K_;t!y*4E%sudayD*6@IUVA83^|!@rKpppj;m9+7?@T`Hi=C8G=xghqv_em8%h znO0j-K?K%%7E8d~6{5`T%J0!=%kxoEvr6hSFvi+&ju|k)SP;CB039#pj4|)3`+1$3 zWdKNv)t14~j|)cX92UAHgie9E3Q3UBP`>qLLt3Trv3y{R?)Tp9nK+b2v!DvZHd7PP zuFoP^(#YcN<)G9C4VFRL_7;bzol4_G7CS16!$quUahcT4bG$owD|H4%gMZE#Biz-0 z4sJI8Ux^&z&ml3>A<&K$w%RY@2p3Bf&H8=iWIN*r-ZYs(1Z^4Y@k{n8(#`JLO{*Zq zFJ(XQ_-1U94ofT3S^JLGB6%n`J7&9vJ`oM{5E)Srwax^6o>7yxde|?J^;I>cHtBA? zC{S{f+m=poM+q}J#4TcFVw^Z^WLakKPHfn~V3`d|4^g5^j7S`=ymNH{9exzjg3T5F z7Mu)h2_a&2G2p6h5}v1Z;>}bxJ!B)iIpQ*NRGrQRSut%el&#pL;?yp2pYS6DA9oWO zgYt~=EVGBhZyV#bmw=l_fV;^lTnom&n(O~nWMYFg$y9q!E0qCVjY|04%u79}yX(Ov zJW);gYhYPY(cPHG7mhn$`Za6IWoR@$sn(9-R!%=E%lnthGyZKh0p={Ze3$FrjNcLd zu5za!F^XK@IJ^IiKR+3cOmNOQAJ87c_*>fpsCDhw&oW7;Pk4h!YT01Sn9g$P%eSsL z)Nc(K8jC=~HVzqdH3XpH^bn!%I;*9l*KYT^Oim$z4J|@l|0)S7=f*jDX7k67C@0g6 ztMC|TmP@u+mdOSjlLY`Ku5HYAmusHl4ILPS5WS%>lWX!!Zg4uaUItO{!rtIxt{A;G zIm3YD0x%RCi363mr03lR`vgYVm)BqVq1)R1JH4VPs+W;6wCFG-GT~vbdr@Z8OHK-{ zC%aQ}r2L*~&XjEU^vbN&9d7nkpzSVYyI07gB*=AE}jWTy@G*g@lq~fE>n)4*;9(BWs2@F3h z6IDl?JrfTUzCBzPh9Y1$@eX%Vm@Sc#iVSN~BBnb-C$z@7JnO4Hqow<-Ez47FgPU~d zP6g3X=T}lyqTbHk-pSJgAL{R?1KoYyVwyr?B>rVzdloy*Tj^v)Z-%I;kXy$DY-4Io zJ}Cw=q*d9b2~JKrzdAyEcyK!FdE+~blX`S$DNlF+I&cT&6FJ$zp(`+PH7a)|$%`l{JV{IT$wz*K!If5AGegi8OPu9-&2rX-c7#`@exCZ8yT{nc?wFdNOS+zp3KGO zmKY0|gsLn+4m8}eOfPjiCUiDU0d7_PHM)NtL+#OhOp8+7{A&g6-TJUIN??%8E|MPd z4%ZOx^*VV@OfJ?6@_*`%w@r_4a5lmr%k)a*>n&y#~gBO4GwD8BEz8t9#AHAv5v+*NcrWK>`~a$)0-&a>v15B*v-&ZPX!8cWVRf3amx2C3 zIK$z{<@_#?*%j}V>BnCJSUs}Fa>l9GNcSPS4g(B5T>veh-xpxt0Y7%xG=`tgY5_PT zB}38cP0B=Y1xDoBtwQzqjda#>PzgvdW^HiEpQOlWrtA)IZFwb9pI@<;Wpl)sD=y>a zZxz<>^ns=6vT3)6+107dY3va~$50f{d~dQEEOFRAMpU0FeG1WOdZKAI5wzf8@`iEh za4jO%D(NZMAJK(x5aAuhVIZ(Y)CC~HXoF#V1+U-#RA^Wi?%fXdY(PT@AC`~HlxLYu zvc^P*nE)<}_sT||;RaAeCam{CDJTRI???YWbOwuPqH5Kv;(*vPanhRRH;E$t>LX-) z{aZ+cCw56-%!y!_qa8YLo_bS}AIlz~^;Fo5Zzml{#T)|EuGH804WVO3j1 zB%aysY+5QMUTjWB9EE%;aa!US^8b2=yQW!Ue2D<}(<}!ZzKV9j-*-Lyfl>VKY10z= zmOUb}K~tE}bhKZ1#pfCAqO!Jcl{yh)LY*8;VBFAKQhY2xfXY#Cjr(XFBGS!ROiG^w z0)&Tb;ve2xJ+Z5sHBXH($9kdNJOG!t`WaDAr!bk~Yq(6Ew(~>`+|Q`aW&b4Sip!8p577B;MpA&D9M5F(@>-_1 z=KNHK1|4QZwbQu&Pxmtp-K95No0k60v>*DwFiy>!dI9VJpkE1}B=*=AnLs!Bi(IcF zD88ZtBzz0h!8`=mKox=)e;Jv;QDhJ-Iv;%=( z6xg$1)m#B1+ygD75~&|{6w<|rqEBl2s1R7Yf-8Gu$@lmS$9tlU%N~H;5HQ)2^en2} zS7TmhhPoMW6$+*}w;)G3K&6k+R`lJ48UI`Ne7gNmJwt0h zZ~X&X-X@|&ekI67`|6>_tsVid*~`dy;!BEfy$i-RwV+w{K?xduM%p(r1-p_14d%xb zf~-;9EKG9XmDb&=ca{8e?u5GQVaMwx>l%k+a#98FByvvs_?9!1b}r)RB&An(6-8wR tvUU%;<36&54JWF4igcff)_6NfMj{VRM*>dAG^55I`#A?8`40z@z25); literal 0 HcmV?d00001 diff --git a/kernel/kernel/tty/vt/vterm.cpp b/kernel/kernel/tty/vt/vterm.cpp index e271559d0..bfbb2f219 100644 --- a/kernel/kernel/tty/vt/vterm.cpp +++ b/kernel/kernel/tty/vt/vterm.cpp @@ -98,6 +98,70 @@ struct vterm_message struct vterm_message *next; }; +enum Gx_type +{ + Gx_LATIN1, + Gx_GRAPH, +}; + +/* Translations from Linux */ +static const unsigned short translations[][256] = { + /* 8-bit Latin-1 mapped to Unicode -- trivial mapping */ + [Gx_LATIN1] = {0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, + 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, 0x0010, 0x0011, 0x0012, 0x0013, + 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, + 0x001e, 0x001f, 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, 0x0030, 0x0031, + 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, + 0x003c, 0x003d, 0x003e, 0x003f, 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, + 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, + 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, 0x0060, 0x0061, 0x0062, 0x0063, + 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, + 0x006e, 0x006f, 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, 0x0080, 0x0081, + 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, 0x0088, 0x0089, 0x008a, 0x008b, + 0x008c, 0x008d, 0x008e, 0x008f, 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, + 0x0096, 0x0097, 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, + 0x00a0, 0x00a1, 0x00a2, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, 0x00a8, 0x00a9, + 0x00aa, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af, 0x00b0, 0x00b1, 0x00b2, 0x00b3, + 0x00b4, 0x00b5, 0x00b6, 0x00b7, 0x00b8, 0x00b9, 0x00ba, 0x00bb, 0x00bc, 0x00bd, + 0x00be, 0x00bf, 0x00c0, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x00c7, + 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf, 0x00d0, 0x00d1, + 0x00d2, 0x00d3, 0x00d4, 0x00d5, 0x00d6, 0x00d7, 0x00d8, 0x00d9, 0x00da, 0x00db, + 0x00dc, 0x00dd, 0x00de, 0x00df, 0x00e0, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, + 0x00e6, 0x00e7, 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef, + 0x00f0, 0x00f1, 0x00f2, 0x00f3, 0x00f4, 0x00f5, 0x00f6, 0x00f7, 0x00f8, 0x00f9, + 0x00fa, 0x00fb, 0x00fc, 0x00fd, 0x00fe, 0x00ff}, + /* VT100 graphics mapped to Unicode */ + [Gx_GRAPH] = {0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008, 0x0009, + 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, 0x0010, 0x0011, 0x0012, 0x0013, + 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, + 0x001e, 0x001f, 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x2192, 0x2190, 0x2191, 0x2193, 0x002f, 0x2588, 0x0031, + 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, 0x0038, 0x0039, 0x003a, 0x003b, + 0x003c, 0x003d, 0x003e, 0x003f, 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, + 0x0046, 0x0047, 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059, + 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x00a0, 0x25c6, 0x2592, 0x2409, 0x240c, + 0x240d, 0x240a, 0x00b0, 0x00b1, 0x2591, 0x240b, 0x2518, 0x2510, 0x250c, 0x2514, + 0x253c, 0x23ba, 0x23bb, 0x2500, 0x23bc, 0x23bd, 0x251c, 0x2524, 0x2534, 0x252c, + 0x2502, 0x2264, 0x2265, 0x03c0, 0x2260, 0x00a3, 0x00b7, 0x007f, 0x0080, 0x0081, + 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, 0x0088, 0x0089, 0x008a, 0x008b, + 0x008c, 0x008d, 0x008e, 0x008f, 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, + 0x0096, 0x0097, 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, + 0x00a0, 0x00a1, 0x00a2, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, 0x00a8, 0x00a9, + 0x00aa, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af, 0x00b0, 0x00b1, 0x00b2, 0x00b3, + 0x00b4, 0x00b5, 0x00b6, 0x00b7, 0x00b8, 0x00b9, 0x00ba, 0x00bb, 0x00bc, 0x00bd, + 0x00be, 0x00bf, 0x00c0, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x00c7, + 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf, 0x00d0, 0x00d1, + 0x00d2, 0x00d3, 0x00d4, 0x00d5, 0x00d6, 0x00d7, 0x00d8, 0x00d9, 0x00da, 0x00db, + 0x00dc, 0x00dd, 0x00de, 0x00df, 0x00e0, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, + 0x00e6, 0x00e7, 0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef, + 0x00f0, 0x00f1, 0x00f2, 0x00f3, 0x00f4, 0x00f5, 0x00f6, 0x00f7, 0x00f8, 0x00f9, + 0x00fa, 0x00fb, 0x00fc, 0x00fd, 0x00fe, 0x00ff}, +}; + #define MAX_ARGS 4 struct vterm { @@ -125,6 +189,8 @@ struct vterm bool numlck; unsigned long *dirty_row_bitmap; unsigned int bitmap_size; + enum Gx_type gx[2]; + u8 charset; // Buffer used for any multibyte buffering for utf8 char multibyte_buffer[10]; @@ -220,6 +286,7 @@ static void draw_char(uint32_t c, unsigned int x, unsigned int y, struct framebu { struct font *font = get_font_data(); + c = font->utf2char(c); if (c >= font->chars) c = '?'; @@ -397,6 +464,8 @@ void vterm_set_char(utf32_t c, unsigned int x, unsigned int y, struct color fg, { struct console_cell *cell = &vterm->cells[y * vterm->columns + x]; vterm_dirty_cell(x, y, vterm); + if (c < 256) + c = translations[vterm->gx[vterm->charset]][c]; cell->codepoint = c; cell->fg = fg; cell->bg = bg; @@ -412,9 +481,17 @@ bool vterm_putc(utf32_t c, struct vterm *vt) if (c == '\a') return false; - if (c == '\016' || c == '\017') + if (c == '\016') + { + /* Shift-out */ + vt->charset = 1; + return false; + } + + if (c == '\017') { - // Shift in and shift out do nothing right now + /* Shift-in */ + vt->charset = 0; return false; } @@ -1003,6 +1080,18 @@ void vterm::do_generic_escape(char escape) cursor_y = saved_y; break; } + + case ')': { + /* TODO: Do properly. This is quite awkward to do because parsing ESC ( 0 isn't trivial + * since all other ESC's don't take any sort of arguments. */ + gx[1] = Gx_GRAPH; + break; + } + + default: { + // pr_info("vterm: Unhandled ESC %c\n", escape); + break; + } } } diff --git a/kernel/scripts/bdf2c.py b/kernel/scripts/bdf2c.py new file mode 100755 index 000000000..6b74e3251 --- /dev/null +++ b/kernel/scripts/bdf2c.py @@ -0,0 +1,353 @@ +#!/bin/python3 +# Copyright (c) 2024 Pedro Falcato +# This file is part of Onyx, and is released under the terms of the GPLv2 License +# check LICENSE at the root directory for more information +# +# SPDX-License-Identifier: GPL-2.0-only +# +import io +import sys +import gzip + +defines = """#define ________ 0x00 +#define _______X 0x01 +#define ______X_ 0x02 +#define ______XX 0x03 +#define _____X__ 0x04 +#define _____X_X 0x05 +#define _____XX_ 0x06 +#define _____XXX 0x07 +#define ____X___ 0x08 +#define ____X__X 0x09 +#define ____X_X_ 0x0A +#define ____X_XX 0x0B +#define ____XX__ 0x0C +#define ____XX_X 0x0D +#define ____XXX_ 0x0E +#define ____XXXX 0x0F +#define ___X____ 0x10 +#define ___X___X 0x11 +#define ___X__X_ 0x12 +#define ___X__XX 0x13 +#define ___X_X__ 0x14 +#define ___X_X_X 0x15 +#define ___X_XX_ 0x16 +#define ___X_XXX 0x17 +#define ___XX___ 0x18 +#define ___XX__X 0x19 +#define ___XX_X_ 0x1A +#define ___XX_XX 0x1B +#define ___XXX__ 0x1C +#define ___XXX_X 0x1D +#define ___XXXX_ 0x1E +#define ___XXXXX 0x1F +#define __X_____ 0x20 +#define __X____X 0x21 +#define __X___X_ 0x22 +#define __X___XX 0x23 +#define __X__X__ 0x24 +#define __X__X_X 0x25 +#define __X__XX_ 0x26 +#define __X__XXX 0x27 +#define __X_X___ 0x28 +#define __X_X__X 0x29 +#define __X_X_X_ 0x2A +#define __X_X_XX 0x2B +#define __X_XX__ 0x2C +#define __X_XX_X 0x2D +#define __X_XXX_ 0x2E +#define __X_XXXX 0x2F +#define __XX____ 0x30 +#define __XX___X 0x31 +#define __XX__X_ 0x32 +#define __XX__XX 0x33 +#define __XX_X__ 0x34 +#define __XX_X_X 0x35 +#define __XX_XX_ 0x36 +#define __XX_XXX 0x37 +#define __XXX___ 0x38 +#define __XXX__X 0x39 +#define __XXX_X_ 0x3A +#define __XXX_XX 0x3B +#define __XXXX__ 0x3C +#define __XXXX_X 0x3D +#define __XXXXX_ 0x3E +#define __XXXXXX 0x3F +#define _X______ 0x40 +#define _X_____X 0x41 +#define _X____X_ 0x42 +#define _X____XX 0x43 +#define _X___X__ 0x44 +#define _X___X_X 0x45 +#define _X___XX_ 0x46 +#define _X___XXX 0x47 +#define _X__X___ 0x48 +#define _X__X__X 0x49 +#define _X__X_X_ 0x4A +#define _X__X_XX 0x4B +#define _X__XX__ 0x4C +#define _X__XX_X 0x4D +#define _X__XXX_ 0x4E +#define _X__XXXX 0x4F +#define _X_X____ 0x50 +#define _X_X___X 0x51 +#define _X_X__X_ 0x52 +#define _X_X__XX 0x53 +#define _X_X_X__ 0x54 +#define _X_X_X_X 0x55 +#define _X_X_XX_ 0x56 +#define _X_X_XXX 0x57 +#define _X_XX___ 0x58 +#define _X_XX__X 0x59 +#define _X_XX_X_ 0x5A +#define _X_XX_XX 0x5B +#define _X_XXX__ 0x5C +#define _X_XXX_X 0x5D +#define _X_XXXX_ 0x5E +#define _X_XXXXX 0x5F +#define _XX_____ 0x60 +#define _XX____X 0x61 +#define _XX___X_ 0x62 +#define _XX___XX 0x63 +#define _XX__X__ 0x64 +#define _XX__X_X 0x65 +#define _XX__XX_ 0x66 +#define _XX__XXX 0x67 +#define _XX_X___ 0x68 +#define _XX_X__X 0x69 +#define _XX_X_X_ 0x6A +#define _XX_X_XX 0x6B +#define _XX_XX__ 0x6C +#define _XX_XX_X 0x6D +#define _XX_XXX_ 0x6E +#define _XX_XXXX 0x6F +#define _XXX____ 0x70 +#define _XXX___X 0x71 +#define _XXX__X_ 0x72 +#define _XXX__XX 0x73 +#define _XXX_X__ 0x74 +#define _XXX_X_X 0x75 +#define _XXX_XX_ 0x76 +#define _XXX_XXX 0x77 +#define _XXXX___ 0x78 +#define _XXXX__X 0x79 +#define _XXXX_X_ 0x7A +#define _XXXX_XX 0x7B +#define _XXXXX__ 0x7C +#define _XXXXX_X 0x7D +#define _XXXXXX_ 0x7E +#define _XXXXXXX 0x7F +#define X_______ 0x80 +#define X______X 0x81 +#define X_____X_ 0x82 +#define X_____XX 0x83 +#define X____X__ 0x84 +#define X____X_X 0x85 +#define X____XX_ 0x86 +#define X____XXX 0x87 +#define X___X___ 0x88 +#define X___X__X 0x89 +#define X___X_X_ 0x8A +#define X___X_XX 0x8B +#define X___XX__ 0x8C +#define X___XX_X 0x8D +#define X___XXX_ 0x8E +#define X___XXXX 0x8F +#define X__X____ 0x90 +#define X__X___X 0x91 +#define X__X__X_ 0x92 +#define X__X__XX 0x93 +#define X__X_X__ 0x94 +#define X__X_X_X 0x95 +#define X__X_XX_ 0x96 +#define X__X_XXX 0x97 +#define X__XX___ 0x98 +#define X__XX__X 0x99 +#define X__XX_X_ 0x9A +#define X__XX_XX 0x9B +#define X__XXX__ 0x9C +#define X__XXX_X 0x9D +#define X__XXXX_ 0x9E +#define X__XXXXX 0x9F +#define X_X_____ 0xA0 +#define X_X____X 0xA1 +#define X_X___X_ 0xA2 +#define X_X___XX 0xA3 +#define X_X__X__ 0xA4 +#define X_X__X_X 0xA5 +#define X_X__XX_ 0xA6 +#define X_X__XXX 0xA7 +#define X_X_X___ 0xA8 +#define X_X_X__X 0xA9 +#define X_X_X_X_ 0xAA +#define X_X_X_XX 0xAB +#define X_X_XX__ 0xAC +#define X_X_XX_X 0xAD +#define X_X_XXX_ 0xAE +#define X_X_XXXX 0xAF +#define X_XX____ 0xB0 +#define X_XX___X 0xB1 +#define X_XX__X_ 0xB2 +#define X_XX__XX 0xB3 +#define X_XX_X__ 0xB4 +#define X_XX_X_X 0xB5 +#define X_XX_XX_ 0xB6 +#define X_XX_XXX 0xB7 +#define X_XXX___ 0xB8 +#define X_XXX__X 0xB9 +#define X_XXX_X_ 0xBA +#define X_XXX_XX 0xBB +#define X_XXXX__ 0xBC +#define X_XXXX_X 0xBD +#define X_XXXXX_ 0xBE +#define X_XXXXXX 0xBF +#define XX______ 0xC0 +#define XX_____X 0xC1 +#define XX____X_ 0xC2 +#define XX____XX 0xC3 +#define XX___X__ 0xC4 +#define XX___X_X 0xC5 +#define XX___XX_ 0xC6 +#define XX___XXX 0xC7 +#define XX__X___ 0xC8 +#define XX__X__X 0xC9 +#define XX__X_X_ 0xCA +#define XX__X_XX 0xCB +#define XX__XX__ 0xCC +#define XX__XX_X 0xCD +#define XX__XXX_ 0xCE +#define XX__XXXX 0xCF +#define XX_X____ 0xD0 +#define XX_X___X 0xD1 +#define XX_X__X_ 0xD2 +#define XX_X__XX 0xD3 +#define XX_X_X__ 0xD4 +#define XX_X_X_X 0xD5 +#define XX_X_XX_ 0xD6 +#define XX_X_XXX 0xD7 +#define XX_XX___ 0xD8 +#define XX_XX__X 0xD9 +#define XX_XX_X_ 0xDA +#define XX_XX_XX 0xDB +#define XX_XXX__ 0xDC +#define XX_XXX_X 0xDD +#define XX_XXXX_ 0xDE +#define XX_XXXXX 0xDF +#define XXX_____ 0xE0 +#define XXX____X 0xE1 +#define XXX___X_ 0xE2 +#define XXX___XX 0xE3 +#define XXX__X__ 0xE4 +#define XXX__X_X 0xE5 +#define XXX__XX_ 0xE6 +#define XXX__XXX 0xE7 +#define XXX_X___ 0xE8 +#define XXX_X__X 0xE9 +#define XXX_X_X_ 0xEA +#define XXX_X_XX 0xEB +#define XXX_XX__ 0xEC +#define XXX_XX_X 0xED +#define XXX_XXX_ 0xEE +#define XXX_XXXX 0xEF +#define XXXX____ 0xF0 +#define XXXX___X 0xF1 +#define XXXX__X_ 0xF2 +#define XXXX__XX 0xF3 +#define XXXX_X__ 0xF4 +#define XXXX_X_X 0xF5 +#define XXXX_XX_ 0xF6 +#define XXXX_XXX 0xF7 +#define XXXXX___ 0xF8 +#define XXXXX__X 0xF9 +#define XXXXX_X_ 0xFA +#define XXXXX_XX 0xFB +#define XXXXXX__ 0xFC +#define XXXXXX_X 0xFD +#define XXXXXXX_ 0xFE +#define XXXXXXXX 0xFF"""; + +def handle_startchar(font_file, line: str): + fontname = line.split(' ')[1] + font_file.write('\t/* ' + fontname.strip() + ' */\n') + +def write_bitmap(file, line: str): + to_write = bin(int(line,base=16)).removeprefix('0b').zfill(8).replace('0', '_').replace('1', 'X') + ',\n' + file.write('\t' + to_write) + +def main(): + charmap = {} + opener = open + if sys.argv[1].endswith(".gz"): + opener = gzip.open + font = opener(sys.argv[1], 'rt') + with open(sys.argv[2], 'w') as file: + file.write('#include \n\n') + file.write(defines) + + file.write('\n' + 'static const unsigned char font_bitmap[] = {\n') + yres = 0 + inbitmap = False + bitmap_row = 0 + chars = 0 + top_encoding = 0 + + for line in font: + if line.startswith('STARTCHAR'): + handle_startchar(file, line) + chars += 1 + if line.startswith('ENCODING'): + top_encoding = int(line.split(' ')[1]) + file.write('\t/* U+' + hex(top_encoding).removeprefix('0x') + ' */\n') + charmap[top_encoding] = chars + if line.startswith('BBX'): + yres = int(line.split(' ')[2]) + if line.startswith('BITMAP'): + inbitmap = True + bitmap_row = 0 + elif inbitmap: + write_bitmap(file, line) + bitmap_row += 1 + if bitmap_row == yres: + inbitmap = False + + file.write('\n};\n\n') + + file.write("static const unsigned short char_indices[] = {\n"); + + for (key, val) in charmap.items(): + file.write(f" [{key}] = {val},\n"); + + file.write("};\n\n"); + + file.write(f""" +static unsigned int utf_to_char(unsigned int c) +{{ + if (c >= {top_encoding + 1}) + return 0; + unsigned short index = char_indices[c] - 1; + + if (index == (unsigned short) -1) + return 0; + return index; +}}\n +"""); + + file.write(f""" +static const unsigned char __cursor__bitmap[] = {{ + XXXXXXXX, XXXXXXXX, XXXXXXXX, XXXXXXXX, XXXXXXXX, XXXXXXXX, XXXXXXXX, XXXXXXXX, + XXXXXXXX, XXXXXXXX, XXXXXXXX, XXXXXXXX, XXXXXXXX, XXXXXXXX, ________, ________, +}}; +static const unsigned int font_mask[8] = {{128, 64, 32, 16, 8, 4, 2, 1}}; +/* bitmap font structure */ +struct font boot_font = {{.width = 8, + .height = {yres}, + .chars = {chars}, + .font_bitmap = (void *) font_bitmap, + .mask = (void *) font_mask, + .cursor_bitmap = (void *) __cursor__bitmap, + .utf2char = utf_to_char}};\n +"""); + + + +main() From 960589a9009cad7de0feae02fcf5faf64527fbe2 Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Tue, 24 Sep 2024 03:10:38 +0100 Subject: [PATCH 08/76] vterm: Implement bright colours using bold Signed-off-by: Pedro Falcato --- kernel/kernel/tty/vt/vterm.cpp | 57 +++++++++++++++++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/kernel/kernel/tty/vt/vterm.cpp b/kernel/kernel/tty/vt/vterm.cpp index bfbb2f219..7fe0b15d8 100644 --- a/kernel/kernel/tty/vt/vterm.cpp +++ b/kernel/kernel/tty/vt/vterm.cpp @@ -61,6 +61,17 @@ const struct color color_table[] = { {204, 204, 204} /* White */ }; +const struct color bright_color_table[] = { + {102, 102, 102}, /* Black */ + {241, 76, 76}, /* Red */ + {22, 198, 12}, /* Green */ + {245, 245, 67}, /* Yellow */ + {59, 142, 234}, /* Blue */ + {214, 112, 214}, /* Magenta */ + {41, 184, 219}, /* Cyan */ + {242, 242, 242}, /* White */ +}; + #define VTERM_CONSOLE_CELL_DIRTY (1 << 0) #define VTERM_CONSOLE_CELL_CONTINUATION (1 << 1) @@ -191,6 +202,7 @@ struct vterm unsigned int bitmap_size; enum Gx_type gx[2]; u8 charset; + bool bold; // Buffer used for any multibyte buffering for utf8 char multibyte_buffer[10]; @@ -828,6 +840,32 @@ void vterm_blink_thread(void *ctx) } } +static void vterm_make_bold(struct vterm *vt) +{ + if (!vt->bold) + return; + for (int i = 0; i < 8; i++) + { + if (same_colour(&vt->fg, &color_table[i])) + { + vt->fg = bright_color_table[i]; + break; + } + } +} + +static void vterm_undo_bold(struct vterm *vt) +{ + for (int i = 0; i < 8; i++) + { + if (same_colour(&vt->fg, &bright_color_table[i])) + { + vt->fg = color_table[i]; + break; + } + } +} + void vterm_ansi_do_sgr(unsigned long n, struct vterm *vt) { switch (n) @@ -836,15 +874,19 @@ void vterm_ansi_do_sgr(unsigned long n, struct vterm *vt) vt->bg = default_bg; vt->fg = default_fg; vt->reversed = false; + vt->bold = false; + vterm_undo_bold(vt); break; } case ANSI_SGR_REVERSE: { if (vt->reversed) return; + vterm_undo_bold(vt); struct color temp = vt->bg; vt->bg = vt->fg; vt->fg = temp; + vterm_make_bold(vt); vt->reversed = true; break; } @@ -852,9 +894,11 @@ void vterm_ansi_do_sgr(unsigned long n, struct vterm *vt) case ANSI_SGR_NOREVERSE: { if (!vt->reversed) return; + vterm_undo_bold(vt); struct color temp = vt->bg; vt->bg = vt->fg; vt->fg = temp; + vterm_make_bold(vt); vt->reversed = false; break; } @@ -866,6 +910,7 @@ void vterm_ansi_do_sgr(unsigned long n, struct vterm *vt) case ANSI_SGR_DEFAULTFG: { vt->fg = default_fg; + vterm_make_bold(vt); break; } @@ -893,6 +938,12 @@ void vterm_ansi_do_sgr(unsigned long n, struct vterm *vt) break; } + case ANSI_SGR_BOLD: { + vt->bold = true; + vterm_make_bold(vt); + break; + } + default: { if (n >= ANSI_SGR_SETBGMIN && n <= ANSI_SGR_SETBGMAX) { @@ -902,8 +953,12 @@ void vterm_ansi_do_sgr(unsigned long n, struct vterm *vt) else if (n >= ANSI_SGR_SETFGMIN && n <= ANSI_SGR_SETFGMAX) { int index = n - ANSI_SGR_SETFGMIN; - vt->fg = color_table[index]; + vt->fg = vt->bold ? bright_color_table[index] : color_table[index]; } +#if 0 + else + pr_info("vterm: unimplemented CSI %lu m\n", n); +#endif } } } From 0624d8975b94b8b617f99376122cab1c1cde0879 Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Tue, 24 Sep 2024 17:02:20 +0100 Subject: [PATCH 09/76] vterm: Optimize down-scrolling After up-scrolling got optimized recently, less was noticeably slower when scrolling up. Rectify this. Signed-off-by: Pedro Falcato --- kernel/kernel/tty/vt/vterm.cpp | 55 ++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/kernel/kernel/tty/vt/vterm.cpp b/kernel/kernel/tty/vt/vterm.cpp index 7fe0b15d8..dad8d5d0a 100644 --- a/kernel/kernel/tty/vt/vterm.cpp +++ b/kernel/kernel/tty/vt/vterm.cpp @@ -280,6 +280,7 @@ void vterm_send_message(struct vterm *vterm, unsigned long message, void *ctx) static inline void vterm_dirty_cell(unsigned int x, unsigned int y, struct vterm *vt) { struct console_cell *cell = &vt->cells[y * vt->columns + x]; + CHECK(y < vt->rows); vterm_set_dirty(cell); vt->dirty_row_bitmap[y / LONG_SIZE_BITS] |= (1UL << (y % LONG_SIZE_BITS)); } @@ -445,19 +446,35 @@ static void vterm_scroll(struct framebuffer *fb, struct vterm *vt) static void __vterm_scroll_down(struct framebuffer *fb, struct vterm *vt, unsigned int nr, unsigned int top, unsigned int bottom) { - unsigned int src, clear, dst; - src = clear = top * vt->columns; - dst = (top + nr) * vt->columns; + if (top + nr >= bottom) + nr = (bottom - top) - 1; - memmove(vt->cells + dst, vt->cells + src, - (bottom - top - nr) * vt->columns * sizeof(struct console_cell)); - for (unsigned int i = 0; i < vt->columns * nr; i++) + unsigned int dest, end; + dest = (top + nr) * vt->columns; + end = bottom * vt->columns; + unsigned int x = vt->columns - 1, y = bottom - 1; + + if (bottom > vt->rows || top >= bottom || nr < 1) + return; + + for (unsigned int i = 0; i < end - dest; i++) { - struct console_cell *c = &vt->cells[clear + i]; - c->codepoint = ' '; - c->bg = vt->bg; - c->fg = vt->fg; + struct console_cell *src = vt->cells + ((bottom - nr) * vt->columns) - i - 1; + struct console_cell *dst = vt->cells + end - i - 1; + if (dst->codepoint != src->codepoint || !same_colour(&dst->bg, &src->bg) || + !same_colour(&dst->fg, &src->fg)) + vterm_dirty_cell(x, y, vt); + dst->bg = src->bg; + dst->fg = src->fg; + dst->codepoint = src->codepoint; + if (x-- == 0) + { + x = vt->columns - 1; + y--; + } } + + vterm_clear_range(vt, 0, top, 0, top + nr); } static void vterm_scroll_down(struct framebuffer *fb, struct vterm *vt) @@ -1074,10 +1091,7 @@ void vterm::do_ri() { vterm_dirty_cell(cursor_x, cursor_y, this); if (cursor_y == top) - { vterm_scroll_down(fb, this); - vterm_flush_all(this); - } else if (cursor_y) cursor_y--; } @@ -1089,10 +1103,7 @@ void vterm::do_nl() if (cursor_y == bottom) { if (cursor_y <= bottom && cursor_y >= top) - { vterm_scroll(fb, this); - vterm_flush_all(this); - } cursor_y--; } } @@ -1373,7 +1384,6 @@ void vterm::do_csi_command(char escape) case ANSI_SCROLL_DOWN: { for (unsigned long i = 0; i < args[0]; i++) vterm_scroll_down(fb, this); - vterm_flush_all(this); break; } @@ -1515,7 +1525,6 @@ void vterm::insert_lines(unsigned long nr) nr = possible_lines_to_insert; __vterm_scroll_down(fb, this, nr, cursor_y, bottom); - vterm_flush_all(this); } void vterm::delete_lines(unsigned long nr) @@ -1700,7 +1709,6 @@ static int vterm_write_con(const char *buffer, size_t size, unsigned int flags, size_t i = 0; const char *data = (const char *) buffer; - bool did_scroll = false; for (; i < size; i++) { @@ -1730,18 +1738,13 @@ static int vterm_write_con(const char *buffer, size_t size, unsigned int flags, vterm_putc('\r', vt); } - if (vterm_putc(codepoint, vt)) - did_scroll = true; - + vterm_putc(codepoint, vt); /* We sub a 1 because we're incrementing on the for loop */ i += codepoint_length - 1; } } - if (!did_scroll) - vterm_flush(vt); - else - vterm_flush_all(vt); + vterm_flush(vt); update_cursor(vt); if (has_lock) From b283574496ab78ec48e663fac8c4644b4d8e589b Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Thu, 11 Jul 2024 03:44:19 +0100 Subject: [PATCH 10/76] fbdev: Add simple fbdev implementation Add an fbdev implementation as desired by Xorg. Not-Yet-Signed-off-by: Pedro Falcato --- .../onyx/riscv/include/platform/pgtable.h | 2 +- kernel/include/onyx/vm.h | 5 +- .../onyx/x86/include/platform/pgtable.h | 2 +- kernel/kernel/fbpriv.h | 280 ++++++++++++++++++ kernel/kernel/framebuffer.cpp | 115 ++++++- 5 files changed, 397 insertions(+), 7 deletions(-) create mode 100644 kernel/kernel/fbpriv.h diff --git a/kernel/include/onyx/riscv/include/platform/pgtable.h b/kernel/include/onyx/riscv/include/platform/pgtable.h index 25a65eb64..5d749fb29 100644 --- a/kernel/include/onyx/riscv/include/platform/pgtable.h +++ b/kernel/include/onyx/riscv/include/platform/pgtable.h @@ -375,7 +375,7 @@ static inline pte_t pte_wrprotect(pte_t pte) static inline pgprot_t calc_pgprot(u64 phys, u64 prots) { - bool special_mapping = phys == (u64) page_to_phys(vm_get_zero_page()); + bool special_mapping = phys == (u64) page_to_phys(vm_get_zero_page()) || prots & VM_PFNMAP; pgprotval_t page_prots = (prots & VM_EXEC ? _PAGE_EXEC : 0) | (prots & VM_WRITE ? _PAGE_WRITE : 0) | (prots & (VM_READ | VM_WRITE) ? _PAGE_READ : 0) | diff --git a/kernel/include/onyx/vm.h b/kernel/include/onyx/vm.h index 8a3d93b78..40ec25de9 100644 --- a/kernel/include/onyx/vm.h +++ b/kernel/include/onyx/vm.h @@ -58,6 +58,7 @@ __BEGIN_CDECLS #define VM_DONT_MAP_OVER (1 << 8) #define VM_NOFLUSH (1 << 9) #define VM_SHARED (1 << 10) +#define VM_PFNMAP (1 << 11) /* Internal flags used by the mm code */ #define __VM_CACHE_TYPE_REGULAR 0 @@ -87,8 +88,6 @@ static inline unsigned long vm_prot_to_cache_type(uint64_t prot) #define PHYS_TO_VIRT(x) (void *) ((uintptr_t) (x) + PHYS_BASE) -#define VM_PFNMAP (1 << 1) - struct vm_object; struct vm_pf_context; @@ -767,7 +766,7 @@ struct page *vmalloc_to_pages(void *ptr); */ static inline bool vma_is_pfnmap(struct vm_area_struct *vma) { - return vma == NULL; + return vma == NULL || vma->vm_flags & VM_PFNMAP; } void vm_do_mmu_mprotect(struct mm_address_space *as, void *address, size_t nr_pgs, int old_prots, diff --git a/kernel/include/onyx/x86/include/platform/pgtable.h b/kernel/include/onyx/x86/include/platform/pgtable.h index db8f0e66b..c4731692e 100644 --- a/kernel/include/onyx/x86/include/platform/pgtable.h +++ b/kernel/include/onyx/x86/include/platform/pgtable.h @@ -422,7 +422,7 @@ static inline pgprot_t calc_pgprot(u64 phys, u64 prot) bool readable = prot & (VM_READ | VM_WRITE) || !noexec; unsigned int cache_type = vm_prot_to_cache_type(prot); uint8_t caching_bits = cache_to_paging_bits(cache_type); - bool special_mapping = phys == (u64) page_to_phys(vm_get_zero_page()); + bool special_mapping = phys == (u64) page_to_phys(vm_get_zero_page()) || prot & VM_PFNMAP; pgprotval_t page_prots = (noexec ? _PAGE_NX : 0) | (global ? _PAGE_GLOBAL : 0) | (user ? _PAGE_USER : 0) | diff --git a/kernel/kernel/fbpriv.h b/kernel/kernel/fbpriv.h new file mode 100644 index 000000000..3c53e9829 --- /dev/null +++ b/kernel/kernel/fbpriv.h @@ -0,0 +1,280 @@ +/* + * copied from from linux kernel 2.2.4 + * removed internal stuff (#ifdef __KERNEL__) + */ + +#ifdef HAVE_XORG_CONFIG_H +#include +#endif + +#ifndef _LINUX_FB_H +#define _LINUX_FB_H + +#ifndef __onyx__ + +#include + +#else + +#include + +#endif + +/* Definitions of frame buffers */ + +#define FB_MAJOR 29 + +#define FB_MODES_SHIFT 5 /* 32 modes per framebuffer */ +#define FB_NUM_MINORS 256 /* 256 Minors */ +#define FB_MAX (FB_NUM_MINORS / (1 << FB_MODES_SHIFT)) +#define GET_FB_IDX(node) (MINOR(node) >> FB_MODES_SHIFT) + +/* ioctls + 0x46 is 'F' */ +#define FBIOGET_VSCREENINFO 0x4600 +#define FBIOPUT_VSCREENINFO 0x4601 +#define FBIOGET_FSCREENINFO 0x4602 +#define FBIOGETCMAP 0x4604 +#define FBIOPUTCMAP 0x4605 +#define FBIOPAN_DISPLAY 0x4606 +/* 0x4607-0x460B are defined below */ +/* #define FBIOGET_MONITORSPEC 0x460C */ +/* #define FBIOPUT_MONITORSPEC 0x460D */ +/* #define FBIOSWITCH_MONIBIT 0x460E */ +#define FBIOGET_CON2FBMAP 0x460F +#define FBIOPUT_CON2FBMAP 0x4610 +#define FBIOBLANK 0x4611 + +#define FB_TYPE_PACKED_PIXELS 0 /* Packed Pixels */ +#define FB_TYPE_PLANES 1 /* Non interleaved planes */ +#define FB_TYPE_INTERLEAVED_PLANES 2 /* Interleaved planes */ +#define FB_TYPE_TEXT 3 /* Text/attributes */ + +#define FB_AUX_TEXT_MDA 0 /* Monochrome text */ +#define FB_AUX_TEXT_CGA 1 /* CGA/EGA/VGA Color text */ +#define FB_AUX_TEXT_S3_MMIO 2 /* S3 MMIO fasttext */ +#define FB_AUX_TEXT_MGA_STEP16 3 /* MGA Millenium I: text, attr, 14 reserved bytes */ +#define FB_AUX_TEXT_MGA_STEP8 4 /* other MGAs: text, attr, 6 reserved bytes */ + +#define FB_VISUAL_MONO01 0 /* Monochr. 1=Black 0=White */ +#define FB_VISUAL_MONO10 1 /* Monochr. 1=White 0=Black */ +#define FB_VISUAL_TRUECOLOR 2 /* True color */ +#define FB_VISUAL_PSEUDOCOLOR 3 /* Pseudo color (like atari) */ +#define FB_VISUAL_DIRECTCOLOR 4 /* Direct color */ +#define FB_VISUAL_STATIC_PSEUDOCOLOR 5 /* Pseudo color readonly */ + +#define FB_ACCEL_NONE 0 /* no hardware accelerator */ +#define FB_ACCEL_ATARIBLITT 1 /* Atari Blitter */ +#define FB_ACCEL_AMIGABLITT 2 /* Amiga Blitter */ +#define FB_ACCEL_S3_TRIO64 3 /* Cybervision64 (S3 Trio64) */ +#define FB_ACCEL_NCR_77C32BLT 4 /* RetinaZ3 (NCR 77C32BLT) */ +#define FB_ACCEL_S3_VIRGE 5 /* Cybervision64/3D (S3 ViRGE) */ +#define FB_ACCEL_ATI_MACH64GX 6 /* ATI Mach 64GX family */ +#define FB_ACCEL_DEC_TGA 7 /* DEC 21030 TGA */ +#define FB_ACCEL_ATI_MACH64CT 8 /* ATI Mach 64CT family */ +#define FB_ACCEL_ATI_MACH64VT 9 /* ATI Mach 64CT family VT class */ +#define FB_ACCEL_ATI_MACH64GT 10 /* ATI Mach 64CT family GT class */ +#define FB_ACCEL_SUN_CREATOR 11 /* Sun Creator/Creator3D */ +#define FB_ACCEL_SUN_CGSIX 12 /* Sun cg6 */ +#define FB_ACCEL_SUN_LEO 13 /* Sun leo/zx */ +#define FB_ACCEL_IMS_TWINTURBO 14 /* IMS Twin Turbo */ +#define FB_ACCEL_3DLABS_PERMEDIA2 15 /* 3Dlabs Permedia 2 */ +#define FB_ACCEL_MATROX_MGA2064W 16 /* Matrox MGA2064W (Millenium) */ +#define FB_ACCEL_MATROX_MGA1064SG 17 /* Matrox MGA1064SG (Mystique) */ +#define FB_ACCEL_MATROX_MGA2164W 18 /* Matrox MGA2164W (Millenium II) */ +#define FB_ACCEL_MATROX_MGA2164W_AGP 19 /* Matrox MGA2164W (Millenium II) */ +#define FB_ACCEL_MATROX_MGAG100 20 /* Matrox G100 (Productiva G100) */ +#define FB_ACCEL_MATROX_MGAG200 21 /* Matrox G200 (Myst, Mill, ...) */ +#define FB_ACCEL_SUN_CG14 22 /* Sun cgfourteen */ +#define FB_ACCEL_SUN_BWTWO 23 /* Sun bwtwo */ +#define FB_ACCEL_SUN_CGTHREE 24 /* Sun cgthree */ +#define FB_ACCEL_SUN_TCX 25 /* Sun tcx */ +#define FB_ACCEL_MATROX_MGAG400 26 /* Matrox G400 */ +#define FB_ACCEL_NV3 27 /* nVidia RIVA 128 */ +#define FB_ACCEL_NV4 28 /* nVidia RIVA TNT */ +#define FB_ACCEL_NV5 29 /* nVidia RIVA TNT2 */ +#define FB_ACCEL_CT_6555x 30 /* C&T 6555x */ +#define FB_ACCEL_3DFX_BANSHEE 31 /* 3Dfx Banshee */ +#define FB_ACCEL_ATI_RAGE128 32 /* ATI Rage128 family */ + +struct fb_fix_screeninfo +{ + char id[16]; /* identification string eg "TT Builtin" */ + char *smem_start; /* Start of frame buffer mem */ + /* (physical address) */ + uint32_t smem_len; /* Length of frame buffer mem */ + uint32_t type; /* see FB_TYPE_* */ + uint32_t type_aux; /* Interleave for interleaved Planes */ + uint32_t visual; /* see FB_VISUAL_* */ + uint16_t xpanstep; /* zero if no hardware panning */ + uint16_t ypanstep; /* zero if no hardware panning */ + uint16_t ywrapstep; /* zero if no hardware ywrap */ + uint32_t line_length; /* length of a line in bytes */ + char *mmio_start; /* Start of Memory Mapped I/O */ + /* (physical address) */ + uint32_t mmio_len; /* Length of Memory Mapped I/O */ + uint32_t accel; /* Type of acceleration available */ + uint16_t reserved[3]; /* Reserved for future compatibility */ +}; + +/* Interpretation of offset for color fields: All offsets are from the right, + * inside a "pixel" value, which is exactly 'bits_per_pixel' wide (means: you + * can use the offset as right argument to <<). A pixel afterwards is a bit + * stream and is written to video memory as that unmodified. This implies + * big-endian byte order if bits_per_pixel is greater than 8. + */ +struct fb_bitfield +{ + uint32_t offset; /* beginning of bitfield */ + uint32_t length; /* length of bitfield */ + uint32_t msb_right; /* != 0 : Most significant bit is */ + /* right */ +}; + +#define FB_NONSTD_HAM 1 /* Hold-And-Modify (HAM) */ + +#define FB_ACTIVATE_NOW 0 /* set values immediately (or vbl) */ +#define FB_ACTIVATE_NXTOPEN 1 /* activate on next open */ +#define FB_ACTIVATE_TEST 2 /* don't set, round up impossible */ +#define FB_ACTIVATE_MASK 15 +/* values */ +#define FB_ACTIVATE_VBL 16 /* activate values on next vbl */ +#define FB_CHANGE_CMAP_VBL 32 /* change colormap on vbl */ +#define FB_ACTIVATE_ALL 64 /* change all VCs on this fb */ + +#define FB_ACCELF_TEXT 1 /* text mode acceleration */ + +#define FB_SYNC_HOR_HIGH_ACT 1 /* horizontal sync high active */ +#define FB_SYNC_VERT_HIGH_ACT 2 /* vertical sync high active */ +#define FB_SYNC_EXT 4 /* external sync */ +#define FB_SYNC_COMP_HIGH_ACT 8 /* composite sync high active */ +#define FB_SYNC_BROADCAST 16 /* broadcast video timings */ + /* vtotal = 144d/288n/576i => PAL */ + /* vtotal = 121d/242n/484i => NTSC */ +#define FB_SYNC_ON_GREEN 32 /* sync on green */ + +#define FB_VMODE_NONINTERLACED 0 /* non interlaced */ +#define FB_VMODE_INTERLACED 1 /* interlaced */ +#define FB_VMODE_DOUBLE 2 /* double scan */ +#define FB_VMODE_MASK 255 + +#define FB_VMODE_YWRAP 256 /* ywrap instead of panning */ +#define FB_VMODE_SMOOTH_XPAN 512 /* smooth xpan possible (internally used) */ +#define FB_VMODE_CONUPDATE 512 /* don't update x/yoffset */ + +struct fb_var_screeninfo +{ + uint32_t xres; /* visible resolution */ + uint32_t yres; + uint32_t xres_virtual; /* virtual resolution */ + uint32_t yres_virtual; + uint32_t xoffset; /* offset from virtual to visible */ + uint32_t yoffset; /* resolution */ + + uint32_t bits_per_pixel; /* guess what */ + uint32_t grayscale; /* != 0 Graylevels instead of colors */ + + struct fb_bitfield red; /* bitfield in fb mem if true color, */ + struct fb_bitfield green; /* else only length is significant */ + struct fb_bitfield blue; + struct fb_bitfield transp; /* transparency */ + + uint32_t nonstd; /* != 0 Non standard pixel format */ + + uint32_t activate; /* see FB_ACTIVATE_* */ + + uint32_t height; /* height of picture in mm */ + uint32_t width; /* width of picture in mm */ + + uint32_t accel_flags; /* acceleration flags (hints) */ + + /* Timing: All values in pixclocks, except pixclock (of course) */ + uint32_t pixclock; /* pixel clock in ps (pico seconds) */ + uint32_t left_margin; /* time from sync to picture */ + uint32_t right_margin; /* time from picture to sync */ + uint32_t upper_margin; /* time from sync to picture */ + uint32_t lower_margin; + uint32_t hsync_len; /* length of horizontal sync */ + uint32_t vsync_len; /* length of vertical sync */ + uint32_t sync; /* see FB_SYNC_* */ + uint32_t vmode; /* see FB_VMODE_* */ + uint32_t reserved[6]; /* Reserved for future compatibility */ +}; + +struct fb_cmap +{ + uint32_t start; /* First entry */ + uint32_t len; /* Number of entries */ + uint16_t *red; /* Red values */ + uint16_t *green; + uint16_t *blue; + uint16_t *transp; /* transparency, can be NULL */ +}; + +struct fb_con2fbmap +{ + uint32_t console; + uint32_t framebuffer; +}; + +struct fb_monspecs +{ + uint32_t hfmin; /* hfreq lower limit (Hz) */ + uint32_t hfmax; /* hfreq upper limit (Hz) */ + uint16_t vfmin; /* vfreq lower limit (Hz) */ + uint16_t vfmax; /* vfreq upper limit (Hz) */ + unsigned dpms : 1; /* supports DPMS */ +}; + +#if 1 + +#define FBCMD_GET_CURRENTPAR 0xDEAD0005 +#define FBCMD_SET_CURRENTPAR 0xDEAD8005 + +#endif + +#if 1 /* Preliminary */ + +/* + * Hardware Cursor + */ + +#define FBIOGET_FCURSORINFO 0x4607 +#define FBIOGET_VCURSORINFO 0x4608 +#define FBIOPUT_VCURSORINFO 0x4609 +#define FBIOGET_CURSORSTATE 0x460A +#define FBIOPUT_CURSORSTATE 0x460B + +struct fb_fix_cursorinfo +{ + uint16_t crsr_width; /* width and height of the cursor in */ + uint16_t crsr_height; /* pixels (zero if no cursor) */ + uint16_t crsr_xsize; /* cursor size in display pixels */ + uint16_t crsr_ysize; + uint16_t crsr_color1; /* colormap entry for cursor color1 */ + uint16_t crsr_color2; /* colormap entry for cursor color2 */ +}; + +struct fb_var_cursorinfo +{ + uint16_t width; + uint16_t height; + uint16_t xspot; + uint16_t yspot; + uint8_t data[1]; /* field with [height][width] */ +}; + +struct fb_cursorstate +{ + int16_t xoffset; + int16_t yoffset; + uint16_t mode; +}; + +#define FB_CURSOR_OFF 0 +#define FB_CURSOR_ON 1 +#define FB_CURSOR_FLASH 2 + +#endif /* Preliminary */ + +#endif /* _LINUX_FB_H */ diff --git a/kernel/kernel/framebuffer.cpp b/kernel/kernel/framebuffer.cpp index a5e1a5ba2..bbe273049 100644 --- a/kernel/kernel/framebuffer.cpp +++ b/kernel/kernel/framebuffer.cpp @@ -5,11 +5,17 @@ */ #include +#include #include +#include +#include +#include -struct framebuffer *primary_fb = NULL; +#include "fbpriv.h" -struct framebuffer *get_primary_framebuffer(void) +struct framebuffer *primary_fb = nullptr; + +struct framebuffer *get_primary_framebuffer() { return primary_fb; } @@ -18,3 +24,108 @@ void set_framebuffer(struct framebuffer *fb) { primary_fb = fb; } + +int fbdev_do_fscreeninfo(void *argp) +{ + fb_fix_screeninfo info; + strcpy(info.id, "bootfb"); + info.accel = FB_ACCEL_NONE; + info.line_length = primary_fb->pitch; + info.xpanstep = 0; + info.ypanstep = 0; + info.ywrapstep = 0; + info.visual = FB_VISUAL_TRUECOLOR; + info.type = FB_TYPE_PACKED_PIXELS; + info.type_aux = 0; + info.smem_start = (char *) primary_fb->framebuffer_phys; + info.smem_len = primary_fb->framebuffer_size; + info.mmio_start = info.smem_start; + info.mmio_len = info.smem_len; + info.reserved[0] = 0; + + return copy_to_user(argp, &info, sizeof(info)); +} + +int fbdev_do_vscreeninfo(void *argp) +{ + fb_var_screeninfo info; + info.xres = primary_fb->width; + info.yres = primary_fb->height; + info.bits_per_pixel = primary_fb->bpp; + info.xres_virtual = info.xres; + info.yres_virtual = info.yres; + info.yoffset = 0; + info.xoffset = 0; + info.pixclock = 25000000 / info.xres * 2000 / info.yres; + info.left_margin = (info.xres / 8) & 0xf8; + info.hsync_len = info.left_margin; + info.red.offset = primary_fb->color.red_shift; + info.red.length = 8; + info.green.offset = primary_fb->color.green_shift; + info.green.length = 8; + info.blue.offset = primary_fb->color.blue_shift; + info.blue.length = 8; + info.transp.offset = primary_fb->color.resv_shift; + info.transp.length = 8; + info.red.msb_right = 0; + info.green.msb_right = 0; + info.blue.msb_right = 0; + info.transp.msb_right = 0; + info.grayscale = 0; + info.nonstd = 0; + info.activate = FB_ACTIVATE_NOW; + info.vsync_len = 10; + info.upper_margin = 32; + info.lower_margin = 16; + info.right_margin = 0; + info.sync = 0; + info.accel_flags = 0; + info.vmode = FB_VMODE_NONINTERLACED; + + return copy_to_user(argp, &info, sizeof(info)); +} + +unsigned int fbdev_ioctl(int request, void *argp, struct file *file) +{ + switch (request) + { + case FBIOGET_FSCREENINFO: + return fbdev_do_fscreeninfo(argp); + case FBIOGET_VSCREENINFO: + return fbdev_do_vscreeninfo(argp); + case FBIOPUT_VSCREENINFO: + case FBIOPUTCMAP: + return 0; // noop + } + + return -ENOTTY; +} + +void *fbdev_mmap(struct vm_area_struct *area, struct file *node) +{ + area->vm_flags |= VM_PFNMAP; + area->vm_obj = vmo_create(0x1000, nullptr); + if (!area->vm_obj) + return NULL; + vmo_assign_mapping(area->vm_obj, area); + return __map_pages_to_vaddr(area->vm_mm, (void *) area->vm_start, + (void *) primary_fb->framebuffer_phys, + area->vm_end - area->vm_start, area->vm_flags); +} + +const file_ops fbdev_fops = {.read = nullptr, // TODO + .ioctl = fbdev_ioctl, + .mmap = fbdev_mmap}; + +/** + * @brief Initialize fb0 + * + */ +void fbdev_init() +{ + auto ex = dev_register_chardevs(0, 1, 0, &fbdev_fops, "fb0"); + + ex.unwrap()->show(0644); +} + +INIT_LEVEL_CORE_KERNEL_ENTRY(fbdev_init); From bc8834575301c7d75f4442054dbcf9d0288632dc Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Tue, 22 Oct 2024 23:52:07 +0100 Subject: [PATCH 11/76] namei: Add proper d_path implementation Add an RCU-based d_path implementation. dentry names should be RCU-protected in order for this to be truly safe. Signed-off-by: Pedro Falcato --- kernel/include/onyx/dentry.h | 7 +- kernel/include/onyx/mount.h | 3 + kernel/kernel/fs/Makefile | 2 +- kernel/kernel/fs/d_path.c | 125 +++++++++++++++++++++++++++++++++++ kernel/kernel/fs/dentry.cpp | 85 +----------------------- kernel/kernel/fs/file.cpp | 16 +++-- kernel/kernel/fs/mount.c | 2 +- kernel/kernel/net/unix.cpp | 17 +++-- kernel/kernel/process.cpp | 29 ++++++-- 9 files changed, 179 insertions(+), 107 deletions(-) create mode 100644 kernel/kernel/fs/d_path.c diff --git a/kernel/include/onyx/dentry.h b/kernel/include/onyx/dentry.h index 3da2cd8ab..662373ad9 100644 --- a/kernel/include/onyx/dentry.h +++ b/kernel/include/onyx/dentry.h @@ -19,11 +19,14 @@ #include #include #include +#include #ifdef __cplusplus #include #endif +struct path; + __BEGIN_CDECLS #define INLINE_NAME_MAX 40 @@ -85,7 +88,9 @@ struct dentry *dentry_create(const char *name, struct inode *inode, struct dentr = 0 #endif ); -char *dentry_to_file_name(struct dentry *dentry); +char *d_path(const struct path *path, char *buf, unsigned int buflen); + +extern seqlock_t rename_lock; /** * @brief Finish a VFS lookup diff --git a/kernel/include/onyx/mount.h b/kernel/include/onyx/mount.h index 99e2a42b5..d632b9b2d 100644 --- a/kernel/include/onyx/mount.h +++ b/kernel/include/onyx/mount.h @@ -10,6 +10,7 @@ #include #include +#include struct dentry; struct superblock; @@ -47,6 +48,8 @@ int do_mount(const char *source, const char *target, const char *fstype, unsigne struct mount *mnt_traverse(struct dentry *mountpoint); +extern seqlock_t mount_lock; + __END_CDECLS #endif diff --git a/kernel/kernel/fs/Makefile b/kernel/kernel/fs/Makefile index 95213c47a..627f202a5 100644 --- a/kernel/kernel/fs/Makefile +++ b/kernel/kernel/fs/Makefile @@ -1,6 +1,6 @@ fs-y:= anon_inode.o block.o dentry.o dev.o file.o null.o partition.o pipe.o poll.o pseudo.o \ superblock.o sysfs.o tmpfs.o vfs.o zero.o buffer.o inode.o namei.o filemap.o writeback.o readahead.o \ - flock.o mount.o + flock.o mount.o d_path.o include kernel/fs/ext2/Makefile include kernel/fs/block/Makefile diff --git a/kernel/kernel/fs/d_path.c b/kernel/kernel/fs/d_path.c new file mode 100644 index 000000000..d1cff67a2 --- /dev/null +++ b/kernel/kernel/fs/d_path.c @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2024 Pedro Falcato + * This file is part of Onyx, and is released under the terms of the GPLv2 License + * check LICENSE at the root directory for more information + * + * SPDX-License-Identifier: GPL-2.0-only + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct rbuf +{ + char *buf; + int len; +}; + +static char *rbuf_path(struct rbuf *rbuf) +{ + if (rbuf->len < 0) + return ERR_PTR(-ENAMETOOLONG); + return rbuf->buf; +} + +static void prepend_char(struct rbuf *rbuf, char c) +{ + rbuf->len--; + if (rbuf->len >= 0) + { + rbuf->buf--; + *rbuf->buf = c; + } +} + +static void prepend_str(struct rbuf *rbuf, const char *str, size_t len) +{ + rbuf->len -= len; + if (rbuf->len >= 0) + { + rbuf->buf -= len; + memcpy(rbuf->buf, str, len); + } +} + +static void prepend_dentry(struct rbuf *rbuf, struct dentry *dentry) +{ + /* TODO: we should RCU-protect d_name */ + spin_lock(&dentry->d_lock); + prepend_str(rbuf, dentry->d_name, dentry->d_name_length); + spin_unlock(&dentry->d_lock); +} + +static void walk_path(const struct path *path, const struct path *root, struct rbuf *rbuf) +{ + /* TODO: While d_parent on .. Just Works, we don't need to keep track of the struct mnt. This + * will need to be changed once that changes (mnt should keep track of mnt_parent). + **/ + struct dentry *dentry = path->dentry; + + prepend_char(rbuf, '\0'); + while (dentry != NULL && dentry != root->dentry) + { + if (dentry->d_flags & DENTRY_FLAG_MOUNT_ROOT) + goto skip; + prepend_dentry(rbuf, dentry); + prepend_char(rbuf, '/'); + skip: + dentry = dentry->d_parent; + } + + if (*rbuf->buf != '/') + prepend_char(rbuf, '/'); +} + +static char *__d_path(const struct path *path, const struct path *root, char *buf, + unsigned int buflen) +{ + struct rbuf rbuf0 = {buf + buflen, buflen}, rbuf1; + unsigned int seq = 0, m_seq = 0; + rcu_read_lock(); + +retry_mnt: + read_seqbegin_or_lock(&mount_lock, &m_seq); + +retry: + read_seqbegin_or_lock(&rename_lock, &seq); + rbuf1 = rbuf0; + walk_path(path, root, &rbuf1); + + if (read_seqretry(&rename_lock, seq)) + { + seq = 1; + goto retry; + } + + done_seqretry(&rename_lock, seq); + + if (read_seqretry(&mount_lock, m_seq)) + { + m_seq = 1; + goto retry_mnt; + } + + done_seqretry(&mount_lock, m_seq); + rcu_read_unlock(); + return rbuf_path(&rbuf1); +} + +char *d_path(const struct path *path, char *buf, unsigned int buflen) +{ + struct path root = get_filesystem_root(); + char *ret = __d_path(path, &root, buf, buflen); + path_put(&root); + return ret; +} diff --git a/kernel/kernel/fs/dentry.cpp b/kernel/kernel/fs/dentry.cpp index 9d4f42739..1518cbbff 100644 --- a/kernel/kernel/fs/dentry.cpp +++ b/kernel/kernel/fs/dentry.cpp @@ -36,7 +36,7 @@ static struct slab_cache *dentry_cache; /* rename_lock is held (write!) throughout a *dcache-level* rename. This protects against hashtable * entries going bad, and against ->d_parent being changed. It's held in read-mode when traversing * the dcache hashtable. */ -static seqlock_t rename_lock; +seqlock_t rename_lock; fnv_hash_t hash_dentry(dentry *&d) { @@ -692,89 +692,6 @@ struct path_element struct list_head node; }; -char *dentry_to_file_name(struct dentry *dentry) -{ - /* Calculate the initial length as / + the null terminator */ - size_t buf_len = 2; - char *buf = nullptr; - char *s = nullptr; - struct path p = get_filesystem_root(); - auto fs_root = p.dentry; - - if (fs_root == dentry) - { - path_put(&p); - return strdup("/"); - } - - dget(fs_root); - path_put(&p); - - auto d = dentry; - struct list_head element_list; - INIT_LIST_HEAD(&element_list); - - /* Get another ref here to have prettier code */ - dget(d); - - /* TODO: Is this logic safe from race conditions? */ - while (d != fs_root && d != nullptr) - { - path_element *p = new path_element; - if (!p) - goto error; - p->d = d; - /* Add 1 to the len because of the separator */ - buf_len += d->d_name_length + 1; - list_add(&p->node, &element_list); - - if (d->d_flags & DENTRY_FLAG_MOUNT_ROOT) - { - // HACK! - d = dentry_parent(d); - while (d && d->d_flags & DENTRY_FLAG_MOUNTPOINT) - d = dentry_parent(d); - } - else - d = dentry_parent(d); - } - - /* Remove one from the end to avoid trailing slashes */ - buf_len--; - - buf = (char *) malloc(buf_len); - if (!buf) - goto error; - buf[0] = '/'; - s = &buf[1]; - - list_for_every_safe (&element_list) - { - auto elem = container_of(l, struct path_element, node); - auto dent = elem->d; - memcpy(s, dent->d_name, dent->d_name_length); - s += dent->d_name_length; - *s++ = '/'; - dput(dent); - delete elem; - } - - buf[buf_len - 1] = '\0'; - dput(fs_root); - return buf; - -error: - dput(fs_root); - list_for_every_safe (&element_list) - { - auto elem = container_of(l, struct path_element, node); - dput(elem->d); - delete elem; - } - - return nullptr; -} - void dentry_shrink_subtree(struct dentry *dentry); void dentry_do_unlink(dentry *entry) diff --git a/kernel/kernel/fs/file.cpp b/kernel/kernel/fs/file.cpp index 55e1b164a..19488c514 100644 --- a/kernel/kernel/fs/file.cpp +++ b/kernel/kernel/fs/file.cpp @@ -1537,28 +1537,30 @@ int sys_fchdir(int fildes) int sys_getcwd(char *path, size_t size) { + char pathbuf[PATH_MAX]; + size_t pathlen = 0; if (size == 0 && path != nullptr) return -EINVAL; struct path cwd = get_current_directory(); - char *name = dentry_to_file_name(cwd.dentry); + char *name = d_path(&cwd, pathbuf, PATH_MAX); path_put(&cwd); - if (!name) - return -errno; + if (IS_ERR(name)) + return PTR_ERR(name); - if (strlen(name) + 1 > size) + pathlen = pathbuf + PATH_MAX - name; + if (pathlen > size) { free(name); return -ERANGE; } - if (copy_to_user(path, name, strlen(name) + 1) < 0) + if (copy_to_user(path, name, pathlen) < 0) { free(name); return -errno; } - - return strlen(name); + return pathlen - 1; } int get_dirfd(int dirfd, struct path *cwd) diff --git a/kernel/kernel/fs/mount.c b/kernel/kernel/fs/mount.c index 43462e930..b9ff46822 100644 --- a/kernel/kernel/fs/mount.c +++ b/kernel/kernel/fs/mount.c @@ -38,7 +38,7 @@ static inline struct blockdev *blkdev_get_dev(struct file *f) static struct list_head mount_hashtable[MT_HASH_SIZE]; static struct list_head mp_hashtable[MT_HASH_SIZE]; -static seqlock_t mount_lock; +seqlock_t mount_lock; static void mnt_init(struct mount *mnt, unsigned long flags) { diff --git a/kernel/kernel/net/unix.cpp b/kernel/kernel/net/unix.cpp index 9669e4a4c..977e94031 100644 --- a/kernel/kernel/net/unix.cpp +++ b/kernel/kernel/net/unix.cpp @@ -1204,6 +1204,7 @@ ssize_t un_socket::recvmsg(struct msghdr *msg, int flags) int un_get_name(sockaddr_un *addr, socklen_t *addrlen, const un_name &name) { + char pathbuf[PATH_MAX]; addr->sun_family = AF_UNIX; if (name.is_anon()) @@ -1213,15 +1214,19 @@ int un_get_name(sockaddr_un *addr, socklen_t *addrlen, const un_name &name) } else if (name.is_fs_sock_) { - auto filename = dentry_to_file_name(name.dentry_); - if (!filename) - return -errno; - size_t copied = strlcpy(addr->sun_path, filename, sizeof(addr->sun_path)); + /* TODO: Fix this path garbage. It will work for now, but unix sockets need to keep struct + * path's, not dentries. + **/ + struct path path; + path_init(&path); + path.dentry = name.dentry_; + char *p = d_path(&path, pathbuf, PATH_MAX); + if (IS_ERR(p)) + return PTR_ERR(p); + size_t copied = strlcpy(addr->sun_path, p, sizeof(addr->sun_path)); auto len = cul::clamp(copied, sizeof(addr->sun_path)); - *addrlen = sizeof(sa_family_t) + len; - free(filename); } else { diff --git a/kernel/kernel/process.cpp b/kernel/kernel/process.cpp index e8e31d5d7..238825157 100644 --- a/kernel/kernel/process.cpp +++ b/kernel/kernel/process.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -1266,16 +1267,19 @@ ssize_t process::query_vm_regions(void *ubuf, ssize_t len, unsigned long what, s { scoped_mutex g{address_space->vm_lock}; size_t needed_len = 0; + char pathbuf[PATH_MAX]; vm_for_every_region(*address_space, [&](vm_area_struct *region) -> bool { needed_len += sizeof(onx_process_vm_region); if (is_file_backed(region)) { - auto path = dentry_to_file_name(region->vm_file->f_dentry); - - needed_len += strlen(path) + 1; - free(path); + char *path = d_path(®ion->vm_file->f_path, pathbuf, PATH_MAX); + /* Path too long? Ignore */ + if (IS_ERR(path)) + needed_len = 1; + else + needed_len = pathbuf + PATH_MAX - path; } if (needed_len % alignof(onx_process_vm_region)) @@ -1332,9 +1336,20 @@ ssize_t process::query_vm_regions(void *ubuf, ssize_t len, unsigned long what, s if (is_file_backed(region)) { - auto path = dentry_to_file_name(region->vm_file->f_dentry); - strcpy(reg->name, path); - reg->size += strlen(path) + 1; + char *path = d_path(®ion->vm_file->f_path, pathbuf, PATH_MAX); + /* Path too long? Ignore */ + if (IS_ERR(path)) + reg->name[0] = '\0'; + else + { + /* FIXME: This is a vulnerability. d_path can change between the initial scan and + * writing. Please fix this!! + **/ + strcpy(reg->name, path); + reg->size += strlen(path); + } + + reg->size++; } if (reg->size % alignof(onx_process_vm_region)) From f38c5c8e61a2f7dcb354d37df2c5d8861aa77649 Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Sun, 10 Nov 2024 03:53:37 +0000 Subject: [PATCH 12/76] smp: Fix sync_call double-free There was a subtle race condition in sync_call logic where two cpus could observe waiting_for_completion = 0 and double-free the control block. Fix it trivially. Signed-off-by: Pedro Falcato --- kernel/kernel/smp.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/kernel/kernel/smp.cpp b/kernel/kernel/smp.cpp index 1464970aa..ba9bdacbf 100644 --- a/kernel/kernel/smp.cpp +++ b/kernel/kernel/smp.cpp @@ -99,13 +99,15 @@ void sync_call_cntrlblk::complete(unsigned int cpu) #ifdef DEBUG_SMP_SYNC_CALL mask.remove_cpu_atomic(cpu); #endif - waiting_for_completion--; - if (flags & SYNC_CALL_NOWAIT && waiting_for_completion.load(mem_order::acquire) == 0) + if (flags & SYNC_CALL_NOWAIT && waiting_for_completion.sub_fetch(1, mem_order::seq_cst) == 0) { // Free the control block, no one is waiting for us ctlblk_pool.free(this); + return; } + + waiting_for_completion--; } void sync_call_cntrlblk::wait(sync_call_func local, void *context) From b81ea4adb6a1aacaa4a0b4c3e39b3f86528e3ee5 Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Sun, 10 Nov 2024 03:55:03 +0000 Subject: [PATCH 13/76] mm/reclaim: Avoid LRU lock horribleness by batching unrefs This is slightly faster and also avoids LRU locking issues inside free_page. Signed-off-by: Pedro Falcato --- kernel/kernel/mm/reclaim.c | 39 +++++++++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/kernel/kernel/mm/reclaim.c b/kernel/kernel/mm/reclaim.c index 199b9f549..69d2bc90e 100644 --- a/kernel/kernel/mm/reclaim.c +++ b/kernel/kernel/mm/reclaim.c @@ -437,12 +437,35 @@ static void isolate_pages(struct page_lru *lru, enum lru_state list, struct list list_splice(&rotate_list, &lru->lru_lists[list]); } +struct pagebatch +{ + struct page *batch[32]; + int nr; +}; + +static bool page_batch_add(struct pagebatch *batch, struct page *page) +{ + batch->batch[batch->nr++] = page; + return batch->nr == 32; +} + +static void page_unref_batch(struct pagebatch *batch) +{ + /* LRU lock *is not held* */ + for (int i = 0; i < batch->nr; i++) + page_unref(batch->batch[i]); + batch->nr = 0; +} + static unsigned long shrink_page_list(struct reclaim_data *data, struct page_lru *lru, struct list_head *page_list) { DEFINE_LIST(rotate_list); DEFINE_LIST(activate_list); + struct pagebatch free_batch; unsigned long freedp = 0; + + free_batch.nr = 0; list_for_every_safe (page_list) { struct page *page = container_of(l, struct page, lru_node); @@ -473,7 +496,12 @@ static unsigned long shrink_page_list(struct reclaim_data *data, struct page_lru list_add_tail(&page->lru_node, &lru->lru_lists[LRU_INACTIVE_FILE + page_to_state(page)]); page_set_lru(page); inc_page_stat(page, NR_INACTIVE_FILE + page_to_state(page)); - page_unref(page); + if (page_batch_add(&free_batch, page)) + { + spin_unlock(&lru->lock); + page_unref_batch(&free_batch); + spin_lock(&lru->lock); + } } list_for_every_safe (&activate_list) @@ -484,11 +512,16 @@ static unsigned long shrink_page_list(struct reclaim_data *data, struct page_lru inc_page_stat(page, NR_ACTIVE_FILE + page_to_state(page)); page_clear_referenced(page); list_add_tail(&page->lru_node, &lru->lru_lists[LRU_ACTIVE_FILE + page_to_state(page)]); - page_unref(page); + if (page_batch_add(&free_batch, page)) + { + spin_unlock(&lru->lock); + page_unref_batch(&free_batch); + spin_lock(&lru->lock); + } } spin_unlock(&lru->lock); - + page_unref_batch(&free_batch); out: return freedp; } From 6c492da42ff9afc038c7a93c3604e816fc212a50 Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Sun, 10 Nov 2024 03:56:41 +0000 Subject: [PATCH 14/76] ubsan: Do a WARN() when not panicking WARN() will give us a nice backtrace + register dump. Do it. Signed-off-by: Pedro Falcato --- kernel/kernel/ubsan.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/kernel/kernel/ubsan.cpp b/kernel/kernel/ubsan.cpp index f1d23b12f..2d5bbb572 100644 --- a/kernel/kernel/ubsan.cpp +++ b/kernel/kernel/ubsan.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 - 2023 Pedro Falcato + * Copyright (c) 2022 - 2024 Pedro Falcato * This file is part of Onyx, and is released under the terms of the GPLv2 License * check LICENSE at the root directory for more information * @@ -12,6 +12,7 @@ #include #include +#include #include #include #include @@ -181,6 +182,8 @@ void ubsan_exit_kunit() static void ubsan_report_end() { + if (!die_on_every_ubsan) + WARN_ON(1); printk("=================================================================================\n"); if (die_on_every_ubsan) [[unlikely]] ubsan_abort(); From c9280a9e27dd18bea363ddf87650ed57bbbb0317 Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Sun, 10 Nov 2024 03:57:18 +0000 Subject: [PATCH 15/76] pipe: Handle named pipe O_NONBLOCK Named pipe O_NONBLOCK is required by POSIX to not block in the open, e.g don't wait for a reader and just give me a darn open pipe. Signed-off-by: Pedro Falcato --- kernel/kernel/fs/pipe.cpp | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/kernel/kernel/fs/pipe.cpp b/kernel/kernel/fs/pipe.cpp index a032192da..5e37c21ff 100644 --- a/kernel/kernel/fs/pipe.cpp +++ b/kernel/kernel/fs/pipe.cpp @@ -926,7 +926,7 @@ int named_pipe_open(struct file *f) int pipe::open_named(struct file *filp) { scoped_mutex g{pipe_lock}; - ssize_t st; + ssize_t st = 0; // As per standard named pipe behavior, block until a peer shows up if ((filp->f_flags & O_RDWRMASK) == O_RDONLY) @@ -934,7 +934,8 @@ int pipe::open_named(struct file *filp) reader_count++; wake_all(&write_queue); COMPILER_BARRIER(); - st = wait_for_event_mutex_interruptible(&read_queue, writer_count != 0, &pipe_lock); + if (!(filp->f_flags & O_NONBLOCK)) + st = wait_for_event_mutex_interruptible(&read_queue, writer_count != 0, &pipe_lock); } else if ((filp->f_flags & O_RDWRMASK) == O_WRONLY) { @@ -942,9 +943,11 @@ int pipe::open_named(struct file *filp) wake_all(&read_queue); COMPILER_BARRIER(); // Use a lambda to go around the multiple wait_for_event problem - st = [&]() REQUIRES(pipe_lock) -> ssize_t { - return wait_for_event_mutex_interruptible(&write_queue, reader_count != 0, &pipe_lock); - }(); + if (!(filp->f_flags & O_NONBLOCK)) + st = [&]() REQUIRES(pipe_lock) -> ssize_t { + return wait_for_event_mutex_interruptible(&write_queue, reader_count != 0, + &pipe_lock); + }(); } else if ((filp->f_flags & O_RDWRMASK) == O_RDWR) { From eb46afd60c6fab9d476f93c1efdbbd9cb260fb4a Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Sun, 10 Nov 2024 04:00:48 +0000 Subject: [PATCH 16/76] mm/kasan: Add physical page shadowing This lets us use the page allocator directly in the SLAB allocator, for KASAN builds too. Signed-off-by: Pedro Falcato --- kernel/arch/x86_64/mmu.cpp | 128 +++++++++++++++++++++++++++++++++ kernel/include/onyx/mm/kasan.h | 3 +- kernel/kernel/mm/asan/asan.cpp | 10 ++- kernel/kernel/mm/bootmem.cpp | 10 +++ kernel/kernel/mm/pagealloc.cpp | 15 ++++ 5 files changed, 164 insertions(+), 2 deletions(-) diff --git a/kernel/arch/x86_64/mmu.cpp b/kernel/arch/x86_64/mmu.cpp index 69230ea39..2f9833211 100644 --- a/kernel/arch/x86_64/mmu.cpp +++ b/kernel/arch/x86_64/mmu.cpp @@ -1092,4 +1092,132 @@ int mmu_map_kasan_shadow(void *shadow_start, size_t pages) return 0; } +static void pte_kasan_range(pte_t *pte, unsigned long start, unsigned long end) +{ + unsigned long zero_shadow = VA2PA(zero_shadow_map); + unsigned long next_start; + for (; start < end; pte++, start = next_start) + { + next_start = min(pte_addr_end(start), end); + pte_t old = *pte; + if (pte_addr(old) != zero_shadow) + { + CHECK(pte_write(old)); + continue; + } + + void *shadow = alloc_boot_page(1, 0); + memset(PHYS_TO_VIRT(shadow), 0, PAGE_SIZE); + set_pte(pte, pte_mkpte((u64) shadow, + __pgprot(_PAGE_WRITE | _PAGE_PRESENT | _PAGE_NX | _PAGE_GLOBAL))); + } +} + +#define SHADOW(type, hwtype) (VA2PA(shadow_##hwtype)) + +static void pmd_kasan_range(pmd_t *pmd, unsigned long start, unsigned long end) +{ + pte_t *pte; + unsigned long next_start; + for (; start < end; pmd++, start = next_start) + { + next_start = min(pmd_addr_end(start), end); + if (pmd_none(*pmd)) + continue; + DCHECK(!pmd_huge(*pmd)); + pte = pte_offset(pmd, start); + if (pmd_addr(*pmd) == SHADOW(pte, pt)) + { + pte_t *newpte = (pte_t *) alloc_boot_page(1, 0); + memcpy(PHYS_TO_VIRT(newpte), shadow_pt, PAGE_SIZE); + set_pmd(pmd, pmd_mkpmd((unsigned long) newpte, __pgprot(KERNEL_PGTBL))); + pte = pte_offset(pmd, start); + } + + CHECK(pmd_val(*pmd) & (_PAGE_WRITE | _PAGE_PRESENT)); + pte_kasan_range(pte, start, next_start); + } +} + +static void pud_kasan_range(pud_t *pud, unsigned long start, unsigned long end) +{ + pmd_t *pmd; + unsigned long next_start; + for (; start < end; pud++, start = next_start) + { + next_start = min(pud_addr_end(start), end); + if (pud_none(*pud)) + continue; + DCHECK(!pud_huge(*pud)); + pmd = pmd_offset(pud, start); + if (pud_addr(*pud) == SHADOW(pmd, pd)) + { + pmd_t *newpmd = (pmd_t *) alloc_boot_page(1, 0); + memcpy(PHYS_TO_VIRT(newpmd), shadow_pd, PAGE_SIZE); + set_pud(pud, pud_mkpud((unsigned long) newpmd, __pgprot(KERNEL_PGTBL))); + pmd = pmd_offset(pud, start); + } + + CHECK(pud_val(*pud) & (_PAGE_WRITE | _PAGE_PRESENT)); + pmd_kasan_range(pmd, start, next_start); + } +} + +static void p4d_kasan_range(p4d_t *p4d, unsigned long start, unsigned long end) +{ + pud_t *pud; + unsigned long next_start; + for (; start < end; p4d++, start = next_start) + { + next_start = min(p4d_addr_end(start), end); + if (WARN_ON(p4d_none(*p4d))) + continue; + DCHECK(!p4d_huge(*p4d)); + pud = pud_offset(p4d, start); + if (p4d_addr(*p4d) == SHADOW(pud, pdpt)) + { + pud_t *newpud = (pud_t *) alloc_boot_page(1, 0); + memcpy(PHYS_TO_VIRT(newpud), shadow_pdpt, PAGE_SIZE); + set_p4d(p4d, p4d_mkp4d((unsigned long) newpud, __pgprot(KERNEL_PGTBL))); + pud = pud_offset(p4d, start); + } + + CHECK(p4d_val(*p4d) & (_PAGE_WRITE | _PAGE_PRESENT)); + pud_kasan_range(pud, start, next_start); + } +} + +static void pgd_kasan_range(pgd_t *pgd, unsigned long start, unsigned long end) +{ + p4d_t *p4d; + unsigned long next_start; + for (; start < end; pgd++, start = next_start) + { + next_start = min(pgd_addr_end(start), end); + if (WARN_ON(pgd_none(*pgd))) + continue; + p4d = p4d_offset(pgd, start); + if (pml5_present() && pgd_addr(*pgd) == SHADOW(p4d, pml4)) + { + p4d_t *newp4d = (p4d_t *) alloc_boot_page(1, 0); + memcpy(PHYS_TO_VIRT(newp4d), shadow_pml4, PAGE_SIZE); + set_pgd(pgd, pgd_mkpgd((unsigned long) newp4d, __pgprot(KERNEL_PGTBL))); + p4d = p4d_offset(pgd, start); + } + + p4d_kasan_range(p4d, start, next_start); + } +} + +void init_shadow_for_phys(unsigned long addr, size_t len) +{ + unsigned long start = (unsigned long) kasan_get_ptr((unsigned long) PHYS_TO_VIRT(addr)); + unsigned long end = ALIGN_TO(start + (len >> 3), PAGE_SIZE); + pr_info("kasan: initializing shadow for [%lx, %lx] (%lx, %lx)\n", addr, addr + len - 1, start, + end - 1); + pgd_t *pgd = pgd_offset(&kernel_address_space, start); + pgd_kasan_range(pgd, start, end); + mmu_invalidate_range(start, (end - start) >> PAGE_SHIFT, &kernel_address_space); +} + #endif diff --git a/kernel/include/onyx/mm/kasan.h b/kernel/include/onyx/mm/kasan.h index e20c77f49..378524324 100644 --- a/kernel/include/onyx/mm/kasan.h +++ b/kernel/include/onyx/mm/kasan.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 - 2022 Pedro Falcato + * Copyright (c) 2019 - 2024 Pedro Falcato * This file is part of Onyx, and is released under the terms of the GPLv2 License * check LICENSE at the root directory for more information * @@ -18,6 +18,7 @@ __BEGIN_CDECLS void kasan_init(); +void kasan_page_alloc_init(); int kasan_alloc_shadow(unsigned long addr, size_t size, bool accessible); void kasan_set_state(unsigned long *ptr, size_t size, int state); diff --git a/kernel/kernel/mm/asan/asan.cpp b/kernel/kernel/mm/asan/asan.cpp index 4d479c5a0..5e6a5fad1 100644 --- a/kernel/kernel/mm/asan/asan.cpp +++ b/kernel/kernel/mm/asan/asan.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 - 2023 Pedro Falcato + * Copyright (c) 2019 - 2024 Pedro Falcato * This file is part of Onyx, and is released under the terms of the GPLv2 License * check LICENSE at the root directory for more information * @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -705,3 +706,10 @@ size_t kasan_get_redzone_size(size_t objsize) else return 1024; } + +void init_shadow_for_phys(unsigned long addr, size_t len); + +void kasan_page_alloc_init() +{ + for_every_phys_region(init_shadow_for_phys); +} diff --git a/kernel/kernel/mm/bootmem.cpp b/kernel/kernel/mm/bootmem.cpp index 9df4eb7cc..2a6e55434 100644 --- a/kernel/kernel/mm/bootmem.cpp +++ b/kernel/kernel/mm/bootmem.cpp @@ -24,6 +24,7 @@ struct memory_range size_t size; }; +static unsigned long phys_range_seq = 0; memory_range phys_ranges[DEFAULT_NR_MEMORY_RANGES]; unsigned int nr_phys_ranges = 0; @@ -32,8 +33,14 @@ unsigned int nr_resv_ranges = 0; void for_every_phys_region(void (*callback)(unsigned long start, size_t size)) { +again: for (unsigned int i = 0; i < nr_phys_ranges; i++) + { + unsigned long seq = phys_range_seq; callback(phys_ranges[i].start, phys_ranges[i].size); + if (phys_range_seq != seq) + goto again; + } } static void __bootmem_add_range(unsigned long start, size_t size) @@ -66,6 +73,7 @@ void bootmem_add_range(unsigned long start, size_t size) // We need to run this because we might already have memory reservations registered bootmem_re_reserve_memory(); + phys_range_seq++; } static void bootmem_remove_range(unsigned int index) @@ -73,6 +81,7 @@ static void bootmem_remove_range(unsigned int index) auto tail_ranges = nr_phys_ranges - index - 1; memmove(&phys_ranges[index], &phys_ranges[index + 1], tail_ranges * sizeof(memory_range)); nr_phys_ranges--; + phys_range_seq++; } static void bootmem_add_reserve(unsigned long start, size_t size) @@ -160,6 +169,7 @@ void bootmem_reserve(unsigned long start, size_t size) bootmem_add_reserve(start, size); bootmem_reserve_memory_ranges(start, size); + phys_range_seq++; } void *alloc_boot_page(size_t nr_pages, long flags) diff --git a/kernel/kernel/mm/pagealloc.cpp b/kernel/kernel/mm/pagealloc.cpp index 84959a25d..08c669947 100644 --- a/kernel/kernel/mm/pagealloc.cpp +++ b/kernel/kernel/mm/pagealloc.cpp @@ -18,6 +18,7 @@ #include #include +#include #include #include #include @@ -506,6 +507,9 @@ void page_node::add_region(uintptr_t base, size_t size) unsigned long end = cul::clamp(start + size, zone->end) + 1; unsigned long nr_pages = (end - start) >> PAGE_SHIFT; printf("pagealloc: Adding [%016lx, %016lx] to zone %s\n", start, end - 1, zone->name); +#ifdef CONFIG_KASAN + kasan_set_state((unsigned long *) PHYS_TO_VIRT(base), size, 1); +#endif page_zone_add_region(start, nr_pages, zone); nr_global_pages.add_fetch(nr_pages, mem_order::release); start = end; @@ -590,6 +594,9 @@ void page_init(size_t memory_size, unsigned long maxpfn) __kbrk(PHYS_TO_VIRT(ptr), (void *) ((unsigned long) PHYS_TO_VIRT(ptr) + needed_memory)); page_allocate_pagemap(maxpfn); +#ifdef CONFIG_KASAN + kasan_page_alloc_init(); +#endif for_every_phys_region([](unsigned long start, size_t size) { /* page_add_region can't return an error value since it halts @@ -772,6 +779,10 @@ __always_inline void prepare_pages_after_alloc(struct page *page, unsigned int o auto pages = pow2(order); +#ifdef CONFIG_KASAN + kasan_set_state((unsigned long *) PAGE_TO_VIRT(page), (1UL << (order + PAGE_SHIFT)), 0); +#endif + if (page_should_zero(flags)) { memset(PAGE_TO_VIRT(page), 0, 1UL << (order + PAGE_SHIFT)); @@ -893,6 +904,10 @@ void page_node::free_page(struct page *p) if (page_flag_set(p, PAGE_FLAG_ANON)) dec_page_stat(p, NR_ANON); +#ifdef CONFIG_KASAN + kasan_set_state((unsigned long *) PAGE_TO_VIRT(p), PAGE_SIZE, 1); +#endif + /* Reset the page */ p->flags = 0; p->owner = nullptr; From cc3627abd6818e702e6ad862149194ca8c136a1f Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Sun, 10 Nov 2024 04:02:15 +0000 Subject: [PATCH 17/76] vfs: Implement FINBIO for all file descriptors This was previously implemented for sockets, should be for every fd. Signed-off-by: Pedro Falcato --- kernel/kernel/fs/vfs.cpp | 27 +++++++++++++++++++++++---- kernel/kernel/net/socket.cpp | 14 -------------- 2 files changed, 23 insertions(+), 18 deletions(-) diff --git a/kernel/kernel/fs/vfs.cpp b/kernel/kernel/fs/vfs.cpp index 293d7cc61..42b295a93 100644 --- a/kernel/kernel/fs/vfs.cpp +++ b/kernel/kernel/fs/vfs.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 - 2023 Pedro Falcato + * Copyright (c) 2016 - 2024 Pedro Falcato * This file is part of Onyx, and is released under the terms of the GPLv2 License * check LICENSE at the root directory for more information * @@ -32,6 +32,8 @@ #include #include +#include + struct file *fs_root = nullptr; struct file *mount_list = nullptr; @@ -357,10 +359,27 @@ ssize_t read_vfs(size_t offset, size_t len, void *buffer, struct file *file) return read_iter_vfs(file, offset, &iter, 0); } -int ioctl_vfs(int request, char *argp, struct file *this_) +int ioctl_vfs(int request, char *argp, struct file *file) { - if (this_->f_ino->i_fops->ioctl != nullptr) - return this_->f_ino->i_fops->ioctl(request, (void *) argp, this_); + switch (request) + { + case FIONBIO: { + int on; + + if (copy_from_user(&on, argp, sizeof(on)) < 0) + return -EFAULT; + + if (on) + file->f_flags |= O_NONBLOCK; + else + file->f_flags &= ~O_NONBLOCK; + + return 0; + } + } + + if (file->f_ino->i_fops->ioctl != nullptr) + return file->f_ino->i_fops->ioctl(request, (void *) argp, file); return -ENOTTY; } diff --git a/kernel/kernel/net/socket.cpp b/kernel/kernel/net/socket.cpp index b659b57cb..0ea909d02 100644 --- a/kernel/kernel/net/socket.cpp +++ b/kernel/kernel/net/socket.cpp @@ -340,20 +340,6 @@ unsigned int socket_ioctl(int request, void *argp, struct file *file) { switch (request) { - case FIONBIO: { - int on; - - if (copy_from_user(&on, argp, sizeof(on)) < 0) - return -EFAULT; - - if (on) - file->f_flags |= O_NONBLOCK; - else - file->f_flags &= ~O_NONBLOCK; - - return 0; - } - #ifdef CONFIG_NET case SIOCGIFNAME: { return do_siocgifname((struct ifreq *) argp); From d03fabad81ac89b411a02bf0aefbf39378f77834 Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Sun, 10 Nov 2024 04:04:21 +0000 Subject: [PATCH 18/76] mm/slab: Fix mutex declaration Global mutexen should be declared with DECLARE_MUTEX so internal members get initialized properly. Fixes a crash when cache_list_lock is contended. Signed-off-by: Pedro Falcato --- kernel/kernel/mm/slab.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/kernel/mm/slab.c b/kernel/kernel/mm/slab.c index 753de0bab..00d920154 100644 --- a/kernel/kernel/mm/slab.c +++ b/kernel/kernel/mm/slab.c @@ -19,7 +19,7 @@ #include #include -static struct mutex cache_list_lock; +static DECLARE_MUTEX(cache_list_lock); static struct list_head cache_list GUARDED_BY(cache_list_lock) = LIST_HEAD_INIT(cache_list); #define KMEM_CACHE_KEEP_THRESHOLD 131072 From 22af172e02fa3ae2cd98eb3fad364e37ae8f92b1 Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Sun, 10 Nov 2024 04:06:55 +0000 Subject: [PATCH 19/76] mm/slab: Switch to the page allocator by default Results in instability sometimes (to be addressed, related to memory fragmentation and higher order page allocation). Signed-off-by: Pedro Falcato --- kernel/kernel/mm/slab.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/kernel/kernel/mm/slab.c b/kernel/kernel/mm/slab.c index 00d920154..358b8ab8f 100644 --- a/kernel/kernel/mm/slab.c +++ b/kernel/kernel/mm/slab.c @@ -124,7 +124,7 @@ struct slab_cache *kmem_cache_create(const char *name, size_t size, size_t align #else c->redzone = 0; #endif - c->flags = flags | KMEM_CACHE_VMALLOC; + c->flags = flags; c->bufctl_off = 0; // Minimum object alignment is 16 @@ -529,6 +529,7 @@ NO_ASAN static struct slab *kmem_cache_create_slab(struct slab_cache *cache, uns asan_poison_shadow((unsigned long) ptr, redzone, KASAN_LEFT_REDZONE); #endif ptr += redzone; + CHECK(((unsigned long) ptr % cache->alignment) == 0); if (cache->ctor) cache->ctor(ptr); struct bufctl *ctl = (struct bufctl *) (ptr + cache->bufctl_off); @@ -1540,8 +1541,8 @@ void kmalloc_init() { // We start at 16 bytes size_t size = 1UL << (4 + i); - unsigned int flags = KMEM_CACHE_VMALLOC; -#if 0 + unsigned int flags = 0; +#if 1 // TODO: Toggling VMALLOC only for larger sizes is not working well... // at least for will-it-scale/page_fault1, it results in major performance regressions. // Is this a TLB issue? Maybe? From 9131269d31e6a862247bba50036ca9b75e712422 Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Sun, 10 Nov 2024 04:07:42 +0000 Subject: [PATCH 20/76] mm/slab: Don't WARN when magazine refilling fails halfway through Signed-off-by: Pedro Falcato --- kernel/kernel/mm/slab.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/kernel/mm/slab.c b/kernel/kernel/mm/slab.c index 358b8ab8f..a12d2ac3f 100644 --- a/kernel/kernel/mm/slab.c +++ b/kernel/kernel/mm/slab.c @@ -803,7 +803,7 @@ static int kmem_cache_alloc_refill_mag(struct slab_cache *cache, * use __GFP_ATOMIC for the first slab, and anything else is very much extra, thus hardly * "__GFP_ATOMIC". */ flags &= ~__GFP_ATOMIC; - flags |= __GFP_NOWAIT; + flags |= __GFP_NOWAIT | __GFP_NOWARN; list_add_tail(&s->slab_list_node, &allocated_slabs); } From 4cb053fca2df562b649ebf3529f4a59b60cf0204 Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Sun, 10 Nov 2024 04:08:21 +0000 Subject: [PATCH 21/76] mm/slab: Fix KASAN freeing We were passing the bufctl as if it were the object, which fired assertions when dumping the quarantine to the slab allocator. Signed-off-by: Pedro Falcato --- kernel/kernel/mm/slab.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/kernel/kernel/mm/slab.c b/kernel/kernel/mm/slab.c index a12d2ac3f..eb83ff19c 100644 --- a/kernel/kernel/mm/slab.c +++ b/kernel/kernel/mm/slab.c @@ -1669,12 +1669,13 @@ void kmem_free_kasan(void *ptr) { struct slab_cache *cache; struct slab *slab = kmem_pointer_to_slab(ptr); + struct bufctl *buf = ptr; assert(slab != NULL); cache = slab->cache; - kmem_bufctl_from_ptr(cache, ptr)->flags = 0; + buf->flags = 0; spin_lock(&cache->lock); - kmem_free_to_slab(cache, slab, ptr); + kmem_free_to_slab(cache, slab, kmem_bufctl_to_ptr(cache, buf)); spin_unlock(&cache->lock); } From af291c9cab1baebfeac417546e4dc2f3407fad93 Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Sun, 10 Nov 2024 04:09:17 +0000 Subject: [PATCH 22/76] mm/slab: Move preemption reenabling in reclaim down Enabling preemption is tricky because we can stall doing $STUFF while other cpus are stuck waiting for us. This results in a deadlock. Move preemption down to after cpus are unfrozen again. Signed-off-by: Pedro Falcato --- kernel/kernel/mm/slab.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/kernel/mm/slab.c b/kernel/kernel/mm/slab.c index eb83ff19c..bcacd52d4 100644 --- a/kernel/kernel/mm/slab.c +++ b/kernel/kernel/mm/slab.c @@ -1767,7 +1767,6 @@ void slab_shrink_caches(unsigned long target_freep) struct slab_rendezvous rndvz; sched_disable_preempt(); kmem_slab_freeze_start(&rndvz); - sched_enable_preempt(); list_for_every (&cache_list) { @@ -1783,6 +1782,7 @@ void slab_shrink_caches(unsigned long target_freep) } kmem_slab_freeze_end(&rndvz); + sched_enable_preempt(); list_for_every (&cache_list) { From babef634301959d4defcdb663c77eddefe9eb2ce Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Sun, 10 Nov 2024 04:11:58 +0000 Subject: [PATCH 23/76] namei: Fix namei_lookup_parentat symlink traversal When traversing symlinks such as: /symlink -> /path/to/file it erroneously backed out and returned the path string related to /symlink; however, even worse, because we had consumed that path element before, callers would see an empty string. This is obviously wrong. Signed-off-by: Pedro Falcato --- kernel/kernel/fs/namei.cpp | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/kernel/kernel/fs/namei.cpp b/kernel/kernel/fs/namei.cpp index a4d523268..70e7e4acb 100644 --- a/kernel/kernel/fs/namei.cpp +++ b/kernel/kernel/fs/namei.cpp @@ -159,9 +159,7 @@ static int dentry_follow_symlink(nameidata &data, dentry *symlink, unsigned int auto target_str = readlink_vfs(&f); if (!target_str) - { return -errno; - } /* Empty symlinks = -ENOENT. See nameitests for more info. */ if (target_str[0] == '\0') @@ -324,9 +322,9 @@ static int namei_resolve_path(nameidata &data) { #define NAMEI_DEBUG 0 #if NAMEI_DEBUG - printk("pdepth %d %s %s\n", data.pdepth, data.paths[data.pdepth].view.data(), - data.paths[data.pdepth].token_type == fs_token_type::LAST_NAME_IN_PATH ? "last" - : "regular"); + pr_info("pdepth %d %s %s\n", data.pdepth, data.paths[data.pdepth].view.data(), + data.paths[data.pdepth].token_type == fs_token_type::LAST_NAME_IN_PATH ? "last" + : "regular"); #endif auto &path = data.paths[data.pdepth]; if (path.token_type == fs_token_type::LAST_NAME_IN_PATH) @@ -725,6 +723,13 @@ static int do_lookup_parent_last(nameidata &data) if (st == 0) { + bool finished_path = true; + for (int i = 0; i < data.pdepth; i++) + if (data.paths[i].token_type != fs_token_type::LAST_NAME_IN_PATH) + finished_path = false; + if (finished_path) + return 0; + if (data.pdepth > 0) { data.pdepth--; @@ -790,6 +795,7 @@ static int namei_lookup_parentat(int dirfd, const char *name, unsigned int flags return st; DCHECK(!path_is_null(&namedata.cur)); + DCHECK(namedata.paths[namedata.pdepth].token_type != fs_token_type::LAST_NAME_IN_PATH); *outn = namedata.paths[namedata.pdepth]; *parent = namedata.getcur(); return 0; From 1645cc6d46ad20fc54871a10383a276642bad4ee Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Sun, 10 Nov 2024 04:14:13 +0000 Subject: [PATCH 24/76] wait_queue: Fix current state setting when waiting Previously, we only set_current_state once, when entering wait_for(). This is wrong and resulted in busy loops. Fix it. While we're at it, set up some next patches by adding an optional "DO NOT DEQUEUE" flag. Signed-off-by: Pedro Falcato --- kernel/include/onyx/wait_queue.h | 7 ++++--- kernel/kernel/wait_queue.cpp | 26 ++++++++------------------ 2 files changed, 12 insertions(+), 21 deletions(-) diff --git a/kernel/include/onyx/wait_queue.h b/kernel/include/onyx/wait_queue.h index 8424a433b..f489e9b00 100644 --- a/kernel/include/onyx/wait_queue.h +++ b/kernel/include/onyx/wait_queue.h @@ -18,7 +18,8 @@ #include #include -#define WQ_TOKEN_EXCLUSIVE (1u << 0) +#define WQ_TOKEN_EXCLUSIVE (1u << 0) +#define WQ_TOKEN_NO_DEQUEUE (1u << 1) /* Return values for wait_queue_token::wake */ #define WQ_WAKE_DO_NOT_WAKE -1 @@ -125,9 +126,9 @@ bool signal_is_pending(); goto out_final; \ init_wq_token(&token); \ \ - set_current_state(state); \ while (true) \ { \ + set_current_state(state); \ token.thread = get_current_thread(); \ wait_queue_add(wq, &token); \ if (cond) \ @@ -156,9 +157,9 @@ bool signal_is_pending(); goto out_final; \ init_wq_token(&token); \ \ - set_current_state(state); \ while (true) \ { \ + set_current_state(state); \ token.thread = get_current_thread(); \ wait_queue_add(wq, &token); \ if (cond) \ diff --git a/kernel/kernel/wait_queue.cpp b/kernel/kernel/wait_queue.cpp index 9df15b598..4459f67b4 100644 --- a/kernel/kernel/wait_queue.cpp +++ b/kernel/kernel/wait_queue.cpp @@ -29,23 +29,11 @@ void wait_queue_wait(struct wait_queue *queue) sched_yield(); } -struct wait_queue_token *wait_queue_wake_unlocked(struct wait_queue *queue) +struct wait_queue_token *wait_queue_wake_unlocked(struct wait_queue_token *token) { - MUST_HOLD_LOCK(&queue->lock); - assert(list_is_empty(&queue->token_list) == false); - - struct list_head *token_lh = list_first_element(&queue->token_list); - - assert(token_lh != NULL); - - struct wait_queue_token *token = container_of(token_lh, struct wait_queue_token, token_node); - - list_remove(token_lh); - - list_assert_correct(&queue->token_list); - + if (!(token->flags & WQ_TOKEN_NO_DEQUEUE)) + list_remove(&token->token_node); token->signaled = true; - return token; } @@ -59,7 +47,8 @@ void wait_queue_wake(struct wait_queue *queue) return; } - struct wait_queue_token *t = wait_queue_wake_unlocked(queue); + struct wait_queue_token *t = wait_queue_wake_unlocked( + list_first_entry(&queue->token_list, struct wait_queue_token, token_node)); if (t->callback) t->callback(t->context, t); @@ -71,11 +60,12 @@ void wait_queue_wake(struct wait_queue *queue) void wait_queue_wake_all(struct wait_queue *queue) { + struct wait_queue_token *waiter, *next; unsigned long cpu_flags = spin_lock_irqsave(&queue->lock); - while (!list_is_empty(&queue->token_list)) + list_for_each_entry_safe (waiter, next, &queue->token_list, token_node) { - struct wait_queue_token *t = wait_queue_wake_unlocked(queue); + struct wait_queue_token *t = wait_queue_wake_unlocked(waiter); if (t->callback) t->callback(t->context, t); From ac44d25119e23f601b3d547114fcc0205f88d19b Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Sun, 10 Nov 2024 04:15:45 +0000 Subject: [PATCH 25/76] process: Serialize waiting for children under children_lock This fixes a race condition trivially exposed by the previous commit. Signed-off-by: Pedro Falcato --- kernel/kernel/process.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/kernel/kernel/process.cpp b/kernel/kernel/process.cpp index 238825157..20baf9600 100644 --- a/kernel/kernel/process.cpp +++ b/kernel/kernel/process.cpp @@ -327,8 +327,6 @@ void process_remove_from_list(process *process); template static void for_every_child(process *proc, Callable cb) { - scoped_lock g{proc->children_lock}; - for (process *p = proc->children; p != nullptr; p = p->next_sibbling) { if (cb(p) == false) @@ -596,9 +594,12 @@ pid_t sys_wait4(pid_t pid, int *wstatus, int options, rusage *usage) return -EINVAL; wait_info w{pid, (unsigned int) options}; + spin_lock(¤t->children_lock); + + int st = wait_for_event_locked_interruptible( + ¤t->wait_child_event, wait_handle_processes(current, w), ¤t->children_lock); - int st = - wait_for_event_interruptible(¤t->wait_child_event, wait_handle_processes(current, w)); + spin_unlock(¤t->children_lock); #if 0 printk("st %d w.status %d\n", st, w.status); @@ -969,12 +970,13 @@ void process_kill_other_threads(void) { current->remove_thread(current_thread); current_thread->owner = nullptr; - scoped_lock g{current->signal_lock}; + spin_lock(¤t->parent->children_lock); current->exit_code = exit_code; /* Finally, wake up any possible concerned parents */ wait_queue_wake_all(¤t->parent->wait_child_event); current->signal_group_flags |= SIGNAL_GROUP_EXIT; + spin_unlock(¤t->parent->children_lock); } kernel_raise_signal(SIGCHLD, parent, 0, &info); From 417209a9883b871d1964d16735d4bc9fd88be823 Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Sun, 10 Nov 2024 04:16:18 +0000 Subject: [PATCH 26/76] process: Inherit umask We were not inheriting umask, which is obviously wrong and broken. Signed-off-by: Pedro Falcato --- kernel/kernel/process.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/kernel/kernel/process.cpp b/kernel/kernel/process.cpp index 20baf9600..82f81b7ad 100644 --- a/kernel/kernel/process.cpp +++ b/kernel/kernel/process.cpp @@ -207,6 +207,7 @@ process *process_create(const std::string_view &cmd_line, ioctx *ctx, process *p if (copy_file_descriptors(proc, ctx) < 0) return nullptr; + proc->ctx.umask = READ_ONCE(ctx->umask); } else { From 6ef2bc263c8400e6082437e98775bf6760147b48 Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Sun, 10 Nov 2024 04:17:02 +0000 Subject: [PATCH 27/76] mm/swap: Small fix plus debugging Swapping isn't quite stable. Add a small fix and some more debug checks. Signed-off-by: Pedro Falcato --- kernel/kernel/mm/swap.c | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/kernel/kernel/mm/swap.c b/kernel/kernel/mm/swap.c index ddcc488ab..6800b9373 100644 --- a/kernel/kernel/mm/swap.c +++ b/kernel/kernel/mm/swap.c @@ -522,11 +522,27 @@ static void swap_final_put(swp_entry_t swp) return; } - __swap_add_counter(-1); *map = 0; spin_unlock(&bg->lock); } +static u8 swap_get_map(swp_entry_t swp) +{ + struct swap_area *sa = swap_areas[SWP_TYPE(swp)]; + unsigned long eff_off = SWP_OFFSET(swp) - sa->swap_off; + struct swap_block_group *bg = &sa->block_groups[eff_off / MAX_BLOCK_GROUP_SIZE]; + u8 *map; + u8 count; + + spin_lock(&bg->lock); + + map = bg->start + (eff_off % MAX_BLOCK_GROUP_SIZE); + count = *map; + spin_unlock(&bg->lock); + + return count; +} + void swap_unset_swapcache(swp_entry_t swp) { struct swap_area *sa = swap_areas[SWP_TYPE(swp)]; @@ -565,6 +581,8 @@ void swap_inc_map(struct page *page) __swap_inc_map(swpval_to_swp_entry(page->priv)); } +void dump_page(struct page *page); + static int swap_add_to_swapcache(struct page *page) { struct page *result; @@ -583,7 +601,13 @@ static int swap_add_to_swapcache(struct page *page) /* WARN if a page was stale in the swap cache */ if (WARN_ON(result != page)) + { + pr_warn("swap: swap space %lu had a stale swapcache entry at %lx (%p vs %p)\n", + SWP_TYPE(entry), SWP_OFFSET(entry), result, page); + dump_page(result); + pr_warn("swap: swap map entry %hhx\n", swap_get_map(entry)); return -EINVAL; + } return 0; } @@ -611,6 +635,7 @@ int swap_add(struct page *page) if (err) { swap_put_page(page); + page_clear_swap(page); return err; } From a8641fc42bc51d7345c4ed7c5e82bda7d83f0800 Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Sun, 10 Nov 2024 04:18:12 +0000 Subject: [PATCH 28/76] poll: Fix missing wakeups Signed-off-by: Pedro Falcato --- kernel/include/onyx/poll.h | 11 +++++++---- kernel/kernel/fs/poll.cpp | 5 +++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/kernel/include/onyx/poll.h b/kernel/include/onyx/poll.h index 74a884766..1e5ee0b85 100644 --- a/kernel/include/onyx/poll.h +++ b/kernel/include/onyx/poll.h @@ -232,7 +232,7 @@ class poll_table void signal() { - signaled = true; + WRITE_ONCE(signaled, true); } bool may_queue() const @@ -245,13 +245,16 @@ class poll_table is_queueing = false; } - bool was_signaled() const + bool was_signaled() { - return signaled; + bool res = READ_ONCE(signaled); + if (res) + WRITE_ONCE(signaled, false); + return res; } /* timeout in ms - negative means infinite, 0 means don't sleep */ - sleep_result sleep_poll(hrtime_t timeout, bool timeout_valid) const; + sleep_result sleep_poll(hrtime_t timeout, bool timeout_valid); }; void poll_wait_helper(void *poll_file, struct wait_queue *q); diff --git a/kernel/kernel/fs/poll.cpp b/kernel/kernel/fs/poll.cpp index f4fdc7791..f11c00f8b 100644 --- a/kernel/kernel/fs/poll.cpp +++ b/kernel/kernel/fs/poll.cpp @@ -23,6 +23,7 @@ void poll_file_entry::wait_on() wait_token.thread = get_current_thread(); wait_token.context = f; wait_token.callback = wake_callback; + wait_token.flags = WQ_TOKEN_NO_DEQUEUE; wait_queue_add(queue, &wait_token); } @@ -49,7 +50,7 @@ void poll_file::wait(wait_queue *queue) file->wait_on(); } -sleep_result poll_table::sleep_poll(hrtime_t timeout, bool timeout_valid) const +sleep_result poll_table::sleep_poll(hrtime_t timeout, bool timeout_valid) { if (timeout == 0 && timeout_valid) return sleep_result::timeout; @@ -75,7 +76,7 @@ sleep_result poll_table::sleep_poll(hrtime_t timeout, bool timeout_valid) const else sched_sleep(timeout); - if (signaled) + if (was_signaled()) return sleep_result::woken_up; else if (signal_is_pending()) return sleep_result::signal; From 97f4c5778d0cd2ffc5235aba6969493bc960d590 Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Sun, 10 Nov 2024 04:19:00 +0000 Subject: [PATCH 29/76] path: Const-ify a function's arguments Signed-off-by: Pedro Falcato --- kernel/include/onyx/path.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/include/onyx/path.h b/kernel/include/onyx/path.h index c6982a3c3..aea05ddcc 100644 --- a/kernel/include/onyx/path.h +++ b/kernel/include/onyx/path.h @@ -37,12 +37,12 @@ static inline void path_init(struct path *p) p->mount = NULL; } -static inline bool path_is_null(struct path *p) +static inline bool path_is_null(const struct path *p) { return !p->dentry && !p->mount; } -static inline bool path_is_equal(struct path *p1, struct path *p2) +static inline bool path_is_equal(const struct path *p1, const struct path *p2) { return p1->mount == p2->mount && p1->dentry == p2->dentry; } From a3a8493fa8396f6fc75a5bc7ca361df799574e8b Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Sun, 10 Nov 2024 04:19:34 +0000 Subject: [PATCH 30/76] tmpfs: Fix inode leak Fix inode leak by correctly putting the dentry. Signed-off-by: Pedro Falcato --- kernel/kernel/fs/tmpfs.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kernel/kernel/fs/tmpfs.cpp b/kernel/kernel/fs/tmpfs.cpp index cc2486ed5..f3ef8dcd2 100644 --- a/kernel/kernel/fs/tmpfs.cpp +++ b/kernel/kernel/fs/tmpfs.cpp @@ -112,8 +112,10 @@ int tmpfs_unlink(const char *name, int flags, struct dentry *dir) return -ENOTEMPTY; } + /* One ref for its tmpfs existence, one ref for dentry_lookup_internal */ + DCHECK(READ_ONCE(child->d_ref) >= 2); + dput(child); dput(child); - return 0; } From 09182a667034d9369ca3c614e67c0f3aa657e4c5 Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Sun, 10 Nov 2024 04:20:33 +0000 Subject: [PATCH 31/76] vterm: Further reduce console_cell Reduce console_cell's size by using a clever bitfield. Signed-off-by: Pedro Falcato --- kernel/kernel/tty/vt/vterm.cpp | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/kernel/kernel/tty/vt/vterm.cpp b/kernel/kernel/tty/vt/vterm.cpp index dad8d5d0a..266cd6498 100644 --- a/kernel/kernel/tty/vt/vterm.cpp +++ b/kernel/kernel/tty/vt/vterm.cpp @@ -41,10 +41,16 @@ struct tty; struct color { - uint8_t r; - uint8_t g; - uint8_t b; - uint8_t a; + union { + struct + { + uint8_t r; + uint8_t g; + uint8_t b; + uint8_t a; + }; + u32 rgba; + }; }; static struct color default_fg = {204, 204, 204}; @@ -77,25 +83,25 @@ const struct color bright_color_table[] = { struct console_cell { - uint32_t codepoint; + uint32_t codepoint : 31; + u32 dirty : 1; struct color bg; struct color fg; - uint32_t flags; }; static inline void vterm_set_dirty(struct console_cell *c) { - c->flags |= VTERM_CONSOLE_CELL_DIRTY; + c->dirty = VTERM_CONSOLE_CELL_DIRTY; } static inline void vterm_clear_dirty(struct console_cell *c) { - c->flags &= ~VTERM_CONSOLE_CELL_DIRTY; + c->dirty &= ~VTERM_CONSOLE_CELL_DIRTY; } static inline bool vterm_is_dirty(struct console_cell *c) { - return c->flags & VTERM_CONSOLE_CELL_DIRTY; + return c->dirty; } #define VTERM_MESSAGE_FLUSH 1 @@ -367,7 +373,7 @@ void vterm_flush_all(struct vterm *vterm) static inline bool same_colour(const struct color *c1, const struct color *c2) { - return c1->a == c2->a && c1->r == c2->r && c1->g == c2->g && c1->b == c2->b; + return c1->rgba == c2->rgba; } static void vterm_clear_range(struct vterm *vt, unsigned int start_x, unsigned int start_y, @@ -950,8 +956,8 @@ void vterm_ansi_do_sgr(unsigned long n, struct vterm *vt) } case ANSI_SGR_BLINKOFF: { - if (vt->blink_thread) - thread_destroy(vt->blink_thread); + // if (vt->blink_thread) + // thread_destroy(vt->blink_thread); break; } @@ -1148,8 +1154,8 @@ void vterm::do_generic_escape(char escape) } case ')': { - /* TODO: Do properly. This is quite awkward to do because parsing ESC ( 0 isn't trivial - * since all other ESC's don't take any sort of arguments. */ + /* TODO: Do properly. This is quite awkward to do because parsing ESC ( 0 isn't + * trivial since all other ESC's don't take any sort of arguments. */ gx[1] = Gx_GRAPH; break; } From 3278acf1b9218e2440b8e19edbcd287ce7567121 Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Sun, 10 Nov 2024 04:21:52 +0000 Subject: [PATCH 32/76] ext2: Various fixes Including the really important symlink fix that fixes exposing unfinished symlinks in the filesystem. Signed-off-by: Pedro Falcato --- kernel/kernel/fs/ext2/ext2.cpp | 2 +- kernel/kernel/fs/ext2/ext2.h | 9 --- kernel/kernel/fs/ext2/ext2_ll.cpp | 8 +-- kernel/kernel/fs/ext2/symlink.cpp | 94 +++++++++++++++++++++++++------ 4 files changed, 79 insertions(+), 34 deletions(-) diff --git a/kernel/kernel/fs/ext2/ext2.cpp b/kernel/kernel/fs/ext2/ext2.cpp index fe5b76a8c..25fe722b8 100644 --- a/kernel/kernel/fs/ext2/ext2.cpp +++ b/kernel/kernel/fs/ext2/ext2.cpp @@ -904,7 +904,7 @@ struct inode *ext2_mkdir(struct dentry *dentry, mode_t mode, struct dentry *dir) */ void ext2_superblock::error(const char *str) const { - printk("ext2_error: %s\n", str); + pr_err("ext2_error: %s\n", str); sb->s_state = EXT2_ERROR_FS; block_buf_dirty(sb_bb); diff --git a/kernel/kernel/fs/ext2/ext2.h b/kernel/kernel/fs/ext2/ext2.h index 9f513cd5c..badd22a2c 100644 --- a/kernel/kernel/fs/ext2/ext2.h +++ b/kernel/kernel/fs/ext2/ext2.h @@ -662,13 +662,4 @@ inode *ext2_get_inode(ext2_superblock *sb, uint32_t inode_num); inode *ext2_create_file(const char *name, mode_t mode, dev_t dev, dentry *dir); int ext2_unlink(const char *name, int flags, dentry *dir); -/** - * @brief Detects if a symlink is a fast symlink - * - * @param inode Pointer to ext2_inode struct - * @param fs Pointer to ext2_superblock struct - * @return True if a fast symlink, else false. - */ -bool ext2_is_fast_symlink(struct ext2_inode *inode, struct ext2_superblock *fs); - #endif diff --git a/kernel/kernel/fs/ext2/ext2_ll.cpp b/kernel/kernel/fs/ext2/ext2_ll.cpp index 8a3d71f69..e132b5135 100644 --- a/kernel/kernel/fs/ext2/ext2_ll.cpp +++ b/kernel/kernel/fs/ext2/ext2_ll.cpp @@ -85,14 +85,11 @@ ext2_inode *ext2_superblock::get_inode(ext2_inode_no inode) const } ext2_inode *ino = (ext2_inode *) malloc(inode_size); - if (!ino) return nullptr; ext2_inode *on_disk = (ext2_inode *) ((char *) block_buf_data(buf) + off); - - memcpy(ino, on_disk, inode_size); - + memcpy(ino, on_disk, min(inode_size, (u16) sizeof(struct ext2_inode))); return ino; } @@ -126,8 +123,7 @@ void ext2_superblock::update_inode(const ext2_inode *ino, ext2_inode_no inode_no } ext2_inode *on_disk = (ext2_inode *) ((char *) block_buf_data(buf) + off); - - memcpy(on_disk, ino, inode_size); + memcpy(on_disk, ino, min(inode_size, (u16) sizeof(struct ext2_inode))); block_buf_dirty(buf); diff --git a/kernel/kernel/fs/ext2/symlink.cpp b/kernel/kernel/fs/ext2/symlink.cpp index 839432764..69153f0f5 100644 --- a/kernel/kernel/fs/ext2/symlink.cpp +++ b/kernel/kernel/fs/ext2/symlink.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017 - 2021 Pedro Falcato + * Copyright (c) 2017 - 2024 Pedro Falcato * This file is part of Onyx, and is released under the terms of the GPLv2 License * check LICENSE at the root directory for more information * @@ -11,6 +11,7 @@ #include #include +#include #include #include @@ -25,15 +26,16 @@ * @param fs Pointer to ext2_superblock struct * @return True if a fast symlink, else false. */ -bool ext2_is_fast_symlink(struct ext2_inode *inode, struct ext2_superblock *fs) +bool ext2_is_fast_symlink(struct inode *inode, struct ext2_inode *e2inode, + struct ext2_superblock *fs) { /* Essentially, we're comparing the extended attribute blocks * with the inode's i_blocks, and if it's zero we know the inode isn't storing * the link in filesystem blocks, so we look to the ext2_inode->i_data. */ - int ea_blocks = inode->i_file_acl ? (fs->block_size >> 9) : 0; - return (inode->i_blocks - ea_blocks == 0 && EXT2_CALCULATE_SIZE64(inode) <= 60); + int ea_blocks = e2inode->i_file_acl ? (fs->block_size >> 9) : 0; + return (inode->i_blocks - ea_blocks == 0 && inode->i_size <= 60); } #define EXT2_FAST_SYMLINK_SIZE 60 @@ -78,14 +80,10 @@ char *ext2_read_symlink(struct inode *ino, struct ext2_superblock *fs) { auto raw = ext2_get_inode_from_node(ino); - if (ext2_is_fast_symlink(raw, fs)) - { + if (ext2_is_fast_symlink(ino, raw, fs)) return ext2_do_fast_symlink(raw); - } else - { return ext2_do_slow_symlink(ino); - } } char *ext2_readlink(struct file *f) @@ -113,11 +111,11 @@ int ext2_set_symlink(inode *ino, const char *dest) unsigned long old = thread_change_addr_limit(VM_KERNEL_ADDR_LIMIT); // TODO: Kind of dumb that it's not a const void *, fix? - ssize_t read = file_write_cache((void *) dest, length, ino, 0); + ssize_t read = file_write_cache((void *) dest, length - 1, ino, 0); thread_change_addr_limit(old); - if (read != (ssize_t) length) + if (read != (ssize_t) length - 1) return -errno; } @@ -128,18 +126,78 @@ int ext2_set_symlink(inode *ino, const char *dest) inode *ext2_symlink(struct dentry *dentry, const char *dest, struct dentry *dir) { - auto inode = ext2_create_file(dentry->d_name, S_IFLNK | S_IRWXG | S_IRWXO | S_IRWXU, 0, dir); + struct inode *vfs_ino = dir->d_inode; + struct ext2_superblock *fs = ext2_superblock_from_inode(vfs_ino); + uint32_t inumber = 0; + struct inode *ino = nullptr; + unsigned long old = 0; + struct creds *c = nullptr; + + if (WARN_ON(dentry->d_name_length == 0)) + return errno = EIO, nullptr; + + auto res = fs->allocate_inode(); + if (res.has_error()) + { + errno = -res.error(); + return nullptr; + } + + auto p = res.value(); + inumber = p.first; + + struct ext2_inode *inode = p.second; + struct ext2_inode *dir_inode = ext2_get_inode_from_node(vfs_ino); + if (!inode) return nullptr; - if (auto st = ext2_set_symlink(inode, dest); st < 0) + memset(inode, 0, sizeof(struct ext2_inode)); + inode->i_ctime = inode->i_atime = inode->i_mtime = (uint32_t) clock_get_posix_time(); + + c = creds_get(); + + inode->i_uid = c->euid; + inode->i_gid = c->egid; + + creds_put(c); + inode->i_mode = EXT2_INO_TYPE_SYMLINK | (S_IRWXG | S_IRWXO | S_IRWXU); + + ino = ext2_fs_ino_to_vfs_ino(inode, inumber, fs); + if (!ino) { - ext2_unlink(dentry->d_name, 0, dir); - inode_dec_nlink(inode); - inode_unref(inode); + errno = ENOMEM; + goto free_ino_error; + } + + fs->update_inode(inode, inumber); + fs->update_inode(dir_inode, vfs_ino->i_inode); + + old = thread_change_addr_limit(VM_KERNEL_ADDR_LIMIT); + + if (auto st = ext2_set_symlink(ino, dest); st < 0) + { + pr_err("set symlink err %d\n", st); errno = -st; - return nullptr; + goto free_ino_error; } - return inode; + pr_info("creating symlink %s (len %zu)\n", dentry->d_name, dentry->d_name_length); + if (int st = ext2_add_direntry(dentry->d_name, inumber, inode, vfs_ino, fs); st < 0) + { + thread_change_addr_limit(old); + printk("ext2 error %d\n", st); + errno = EINVAL; + goto free_ino_error; + } + + inode_inc_nlink(ino); + + thread_change_addr_limit(old); + superblock_add_inode(vfs_ino->i_sb, ino); + return ino; + +free_ino_error: + inode_unref(ino); + return nullptr; } From 58a9b84c0529c0955680da0dcd891668ff3e0f21 Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Sun, 10 Nov 2024 04:23:02 +0000 Subject: [PATCH 33/76] signal: Fix warning For some reason... Signed-off-by: Pedro Falcato --- kernel/kernel/signal.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/kernel/kernel/signal.cpp b/kernel/kernel/signal.cpp index 3c92995f4..c6840b44d 100644 --- a/kernel/kernel/signal.cpp +++ b/kernel/kernel/signal.cpp @@ -725,11 +725,9 @@ int signal_send_all(int signal, int flags, siginfo_t *info) pid::auto_pid process_get_pgrp(process *p) { scoped_lock g{p->pgrp_lock}; - auto pg = p->process_group; - + CHECK(pg.get() != nullptr); pg->ref(); - return pg; } From d4efd1779e9db0a1de5ee89cdf7fbd7766b2795a Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Sun, 10 Nov 2024 04:23:48 +0000 Subject: [PATCH 34/76] radix: Make the slab cache VMALLOC While fragmentation issues exist, it's best we don't do high order allocations with GFP_ATOMIC (we also _definitely_ shouldn't be doing GFP_ATOMIC, TOFIX). Signed-off-by: Pedro Falcato --- kernel/kernel/radix.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/kernel/radix.cpp b/kernel/kernel/radix.cpp index 0fce3f054..891c3ae25 100644 --- a/kernel/kernel/radix.cpp +++ b/kernel/kernel/radix.cpp @@ -51,7 +51,7 @@ static slab_cache *node_cache; __init static void radix_init_slab() { node_cache = kmem_cache_create("radix_tree_node", sizeof(radix_tree_node), 0, - KMEM_CACHE_HWALIGN, nullptr); + KMEM_CACHE_HWALIGN | KMEM_CACHE_VMALLOC, nullptr); CHECK(node_cache != nullptr); } From 23acc10213f7c13082acfb77687027df6e1927a7 Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Sun, 10 Nov 2024 04:24:58 +0000 Subject: [PATCH 35/76] kcsan: Enable weak memory instrumentation Signed-off-by: Pedro Falcato --- kernel/include/onyx/kcsan.h | 21 ++++++++++++++++++++- kernel/include/onyx/scheduler.h | 4 ++++ kernel/kernel/kcsan/core.c | 23 +++-------------------- kernel/kernel/kcsan/kcsan.h | 4 ++-- 4 files changed, 29 insertions(+), 23 deletions(-) diff --git a/kernel/include/onyx/kcsan.h b/kernel/include/onyx/kcsan.h index a20e172b2..42ef94691 100644 --- a/kernel/include/onyx/kcsan.h +++ b/kernel/include/onyx/kcsan.h @@ -46,6 +46,25 @@ void __kcsan_check_access(const volatile void *ptr, size_t size, int type); #define __KCSAN_BARRIER_TO_SIGNAL_FENCE_rmb __ATOMIC_ACQUIRE #define __KCSAN_BARRIER_TO_SIGNAL_FENCE_release __ATOMIC_RELEASE +struct kcsan_scoped_access +{ + union { + struct list_head list; /* scoped_accesses list */ + /* + * Not an entry in scoped_accesses list; stack depth from where + * the access was initialized. + */ + int stack_depth; + }; + + /* Access information. */ + const volatile void *ptr; + size_t size; + int type; + /* Location where scoped access was set up. */ + unsigned long ip; +}; + struct kcsan_ctx { int disable_count; /* disable counter */ @@ -79,7 +98,7 @@ struct kcsan_ctx /* List of scoped accesses; likely to be empty. */ struct list_head scoped_accesses; - +#define CONFIG_KCSAN_WEAK_MEMORY 1 #ifdef CONFIG_KCSAN_WEAK_MEMORY /* * Scoped access for modeling access reordering to detect missing memory diff --git a/kernel/include/onyx/scheduler.h b/kernel/include/onyx/scheduler.h index 3fda63826..e57eb9c85 100644 --- a/kernel/include/onyx/scheduler.h +++ b/kernel/include/onyx/scheduler.h @@ -87,6 +87,7 @@ typedef struct thread #endif #ifdef CONFIG_KCSAN struct kcsan_ctx kcsan_ctx; + int kcsan_stack_depth; #endif /* And arch dependent stuff in this ifdef */ #ifdef __x86_64__ @@ -107,6 +108,9 @@ typedef struct thread fs{}, gs{} #endif { +#ifdef CONFIG_KCSAN + kcsan_stack_depth = 0; +#endif } /** diff --git a/kernel/kernel/kcsan/core.c b/kernel/kernel/kcsan/core.c index f113cc09e..e6c926eda 100644 --- a/kernel/kernel/kcsan/core.c +++ b/kernel/kernel/kcsan/core.c @@ -225,24 +225,6 @@ __always_inline struct kcsan_ctx *get_ctx(void) __always_inline void check_access(const volatile void *ptr, size_t size, int type, unsigned long ip); -struct kcsan_scoped_access { - union { - struct list_head list; /* scoped_accesses list */ - /* - * Not an entry in scoped_accesses list; stack depth from where - * the access was initialized. - */ - int stack_depth; - }; - - /* Access information. */ - const volatile void *ptr; - size_t size; - int type; - /* Location where scoped access was set up. */ - unsigned long ip; -}; - /* Check scoped accesses; never inline because this is a slow-path! */ static noinline void kcsan_check_scoped_accesses(void) { @@ -417,7 +399,7 @@ void kcsan_restore_irqtrace(struct task_struct *task) __always_inline int get_kcsan_stack_depth(void) { #if CONFIG_KCSAN_WEAK_MEMORY - return current->kcsan_stack_depth; + return get_current_thread() ? get_current_thread()->kcsan_stack_depth : 0; #else BUILD_BUG(); return 0; @@ -427,7 +409,8 @@ __always_inline int get_kcsan_stack_depth(void) __always_inline void add_kcsan_stack_depth(int val) { #if CONFIG_KCSAN_WEAK_MEMORY - current->kcsan_stack_depth += val; + if (get_current_thread()) + get_current_thread()->kcsan_stack_depth += val; #else BUILD_BUG(); #endif diff --git a/kernel/kernel/kcsan/kcsan.h b/kernel/kernel/kcsan/kcsan.h index 930db8f79..3628cfd83 100644 --- a/kernel/kernel/kcsan/kcsan.h +++ b/kernel/kernel/kcsan/kcsan.h @@ -16,9 +16,9 @@ #define CONFIG_KCSAN_NUM_WATCHPOINTS 64 #define CONFIG_KCSAN_UDELAY_TASK 80 #define CONFIG_KCSAN_UDELAY_INTERRUPT 20 -#define CONFIG_KCSAN_SKIP_WATCH 4000 +#define CONFIG_KCSAN_SKIP_WATCH 800 #define CONFIG_KCSAN_IGNORE_ATOMICS 0 -#define CONFIG_KCSAN_WEAK_MEMORY 0 +#define CONFIG_KCSAN_WEAK_MEMORY 1 #define UL(a) ((a) + 0UL) #define ULL(a) ((a) + 0ULL) From 4769e946d22d844616ca865937024abd09b205e4 Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Sun, 10 Nov 2024 04:26:03 +0000 Subject: [PATCH 36/76] memory: Add zap_page_range Add a zapping function (unmaps, but does not tear down page tables). Signed-off-by: Pedro Falcato --- kernel/include/onyx/pgtable.h | 1 + kernel/kernel/mm/memory.c | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/kernel/include/onyx/pgtable.h b/kernel/include/onyx/pgtable.h index 1e8e73391..2100bde56 100644 --- a/kernel/include/onyx/pgtable.h +++ b/kernel/include/onyx/pgtable.h @@ -232,6 +232,7 @@ static inline unsigned long pte_addr_end(unsigned long addr) pte_t pte_get(struct mm_address_space *mm, unsigned long addr); pte_t *ptep_get_locked(struct mm_address_space *mm, unsigned long addr, struct spinlock **lock); int pgtable_prealloc(struct mm_address_space *mm, unsigned long virt); +int zap_page_range(unsigned long start, unsigned long end, struct vm_area_struct *vma); __END_CDECLS #endif diff --git a/kernel/kernel/mm/memory.c b/kernel/kernel/mm/memory.c index 27bd22519..c2ac0c8fd 100644 --- a/kernel/kernel/mm/memory.c +++ b/kernel/kernel/mm/memory.c @@ -760,6 +760,26 @@ int vm_mmu_unmap(struct mm_address_space *mm, void *addr, size_t pages, struct v return 0; } +int zap_page_range(unsigned long start, unsigned long end, struct vm_area_struct *vma) +{ + struct mm_address_space *mm = vma->vm_mm; + struct unmap_info unmap_info; + unmap_info.vma = vma; + unmap_info.mm = mm; + unmap_info.kernel = 0; + unmap_info.full = 0; + unmap_info.freepgtables = 0; + tlbi_tracker_init(&unmap_info.tlbi); + + spin_lock(&mm->page_table_lock); + pgd_unmap_range(&unmap_info, pgd_offset(mm, start), start, end); + spin_unlock(&mm->page_table_lock); + + if (tlbi_active(&unmap_info.tlbi)) + tlbi_end_batch(&unmap_info.tlbi); + return 0; +} + bool paging_write_protect(void *addr, struct mm_address_space *mm) { spin_lock(&mm->page_table_lock); From c9d58f2dcdc7c91612fc5a823f352895561f86c9 Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Sun, 10 Nov 2024 04:29:32 +0000 Subject: [PATCH 37/76] mm: Add madvise Add madvise, as specified by Linux. Currently we only implement MADV_DONTNEED (as required by libjemalloc, etc). Signed-off-by: Pedro Falcato --- kernel/arch/riscv64/syscall_table.json | 21 ++++++ kernel/arch/x86_64/syscall_table.json | 21 ++++++ kernel/kernel/mm/Makefile | 2 +- kernel/kernel/mm/madvise.c | 98 ++++++++++++++++++++++++++ musl | 2 +- 5 files changed, 142 insertions(+), 2 deletions(-) create mode 100644 kernel/kernel/mm/madvise.c diff --git a/kernel/arch/riscv64/syscall_table.json b/kernel/arch/riscv64/syscall_table.json index a309eca71..0d17049a1 100644 --- a/kernel/arch/riscv64/syscall_table.json +++ b/kernel/arch/riscv64/syscall_table.json @@ -2795,5 +2795,26 @@ ] ], "return_type": "int" + }, + { + "name": "madvise", + "nr": 160, + "nr_args": 3, + "args": [ + [ + "void *", + "addr" + ], + [ + "size_t", + "length" + ], + [ + "int", + "advice" + ] + ], + "return_type": "int", + "abi": "c" } ] diff --git a/kernel/arch/x86_64/syscall_table.json b/kernel/arch/x86_64/syscall_table.json index 133deb966..4f00d9bb1 100644 --- a/kernel/arch/x86_64/syscall_table.json +++ b/kernel/arch/x86_64/syscall_table.json @@ -2839,5 +2839,26 @@ ] ], "return_type": "int" + }, + { + "name": "madvise", + "nr": 160, + "nr_args": 3, + "args": [ + [ + "void *", + "addr" + ], + [ + "size_t", + "length" + ], + [ + "int", + "advice" + ] + ], + "return_type": "int", + "abi": "c" } ] diff --git a/kernel/kernel/mm/Makefile b/kernel/kernel/mm/Makefile index 3301a421d..aeb810808 100644 --- a/kernel/kernel/mm/Makefile +++ b/kernel/kernel/mm/Makefile @@ -1,4 +1,4 @@ -mm-y:= bootmem.o page.o pagealloc.o vm_object.o vm.o vmalloc.o reclaim.o anon.o mincore.o page_lru.o swap.o rmap.o slab_cache_pool.o +mm-y:= bootmem.o page.o pagealloc.o vm_object.o vm.o vmalloc.o reclaim.o anon.o mincore.o page_lru.o swap.o rmap.o slab_cache_pool.o madvise.o mm-$(CONFIG_KUNIT)+= vm_tests.o mm-$(CONFIG_X86)+= memory.o mm-$(CONFIG_RISCV)+= memory.o diff --git a/kernel/kernel/mm/madvise.c b/kernel/kernel/mm/madvise.c new file mode 100644 index 000000000..00a4425f5 --- /dev/null +++ b/kernel/kernel/mm/madvise.c @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2024 Pedro Falcato + * This file is part of Onyx, and is released under the terms of the GPLv2 License + * check LICENSE at the root directory for more information + * + * SPDX-License-Identifier: GPL-2.0-only + */ + +#include + +#include +#include +#include +#include + +#include + +static bool madvise_needs_write(int advice) +{ + return true; +} + +static bool madvise_valid_advice(int advice) +{ + switch (advice) + { + case MADV_DONTNEED: + return true; + } + + return false; +} + +static int do_madvise_vma(struct vm_area_struct *vma, unsigned long start, unsigned long end, + int advice) +{ + switch (advice) + { + case MADV_DONTNEED: + return zap_page_range(start, end, vma); + } + + WARN_ON(1); + return -ENOSYS; +} + +static int do_madvise_walk(struct mm_address_space *mm, unsigned long start, size_t len, int advice) +{ + unsigned long limit = start + len; + unsigned long last_vma_end = start; + int ret = -ENOMEM; + struct vm_area_struct *vma; + + MA_STATE(mas, &mm->region_tree, start, limit - 1); + + mas_for_each(&mas, vma, limit - 1) + { + /* Break if we see a gap between VMAs, or if this vma is beyond limit */ + if (vma->vm_start >= limit) + break; + + if (vma->vm_start != last_vma_end) + { + ret = -ENOMEM; + break; + } + + ret = do_madvise_vma(vma, max(vma->vm_start, start), min(limit, vma->vm_end), advice); + if (ret) + break; + last_vma_end = vma->vm_end; + } + + return ret; +} + +int sys_madvise(void *addr, size_t len, int advice) +{ + int ret; + unsigned long start = (unsigned long) addr; + /* TODO: Remove this open coding */ + struct mm_address_space *mm = get_current_thread()->aspace; + + if (!madvise_valid_advice(advice)) + return -EINVAL; + if (start & (PAGE_SIZE - 1)) + return -EINVAL; + + len = ALIGN_TO(len, PAGE_SIZE); + + if (start + len <= start) + return -EINVAL; + + mutex_lock(&mm->vm_lock); + ret = do_madvise_walk(mm, start, len, advice); + mutex_unlock(&mm->vm_lock); + return ret; +} diff --git a/musl b/musl index b1f7fbcfa..35aac5715 160000 --- a/musl +++ b/musl @@ -1 +1 @@ -Subproject commit b1f7fbcfa806ead2af7ef209203bb94d7ed4c8b8 +Subproject commit 35aac5715b84bbe506923552247ea84406b2f24c From 57005260a5a8bf9ac50d562f82e945ff847edb06 Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Mon, 11 Nov 2024 20:24:26 +0000 Subject: [PATCH 38/76] io-queue: Reinitialize the list in submit_batch Reinitialize the list when splicing the requests. This fixes crashes when unplugging a blk plug (when we have requests from different io queues, aka CPU migration, aka upcoming work). Signed-off-by: Pedro Falcato --- kernel/include/onyx/list.h | 6 ++++++ kernel/kernel/fs/block/io-queue.cpp | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/kernel/include/onyx/list.h b/kernel/include/onyx/list.h index 04ce27cc8..7de253643 100644 --- a/kernel/include/onyx/list.h +++ b/kernel/include/onyx/list.h @@ -206,6 +206,12 @@ static inline void list_splice_tail(struct list_head *src, struct list_head *dst list_splice_internal(src, dst->prev, dst); } +static inline void list_splice_tail_init(struct list_head *src, struct list_head *dst) +{ + list_splice_tail(src, dst); + INIT_LIST_HEAD(src); +} + static inline int list_is_head(const struct list_head *list, const struct list_head *head) { return list == head; diff --git a/kernel/kernel/fs/block/io-queue.cpp b/kernel/kernel/fs/block/io-queue.cpp index 71d7db2aa..d744b0016 100644 --- a/kernel/kernel/fs/block/io-queue.cpp +++ b/kernel/kernel/fs/block/io-queue.cpp @@ -115,7 +115,7 @@ void io_queue::__restart_queue() void io_queue::submit_batch(struct list_head *req_list, u32 nr_reqs) { scoped_lock g{lock_}; - list_splice_tail(req_list, &req_list_); + list_splice_tail_init(req_list, &req_list_); if (nr_entries_ - used_entries_ > 0) __restart_queue(); } From 13001bf79e6d701300fc1b8bfaf24d0d1f7bb5a3 Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Mon, 11 Nov 2024 20:25:58 +0000 Subject: [PATCH 39/76] mm/slab: Correct pass gfp flags to alloc_pages Signed-off-by: Pedro Falcato --- kernel/kernel/mm/slab.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/kernel/mm/slab.c b/kernel/kernel/mm/slab.c index bcacd52d4..7068b5b7a 100644 --- a/kernel/kernel/mm/slab.c +++ b/kernel/kernel/mm/slab.c @@ -496,7 +496,7 @@ NO_ASAN static struct slab *kmem_cache_create_slab(struct slab_cache *cache, uns { unsigned int order = pages2order(slab_size >> PAGE_SHIFT); slab_size = 1UL << (order + PAGE_SHIFT); - pages = alloc_pages(order, PAGE_ALLOC_NO_ZERO | PAGE_ALLOC_CONTIGUOUS); + pages = alloc_pages(order, flags | PAGE_ALLOC_NO_ZERO | PAGE_ALLOC_CONTIGUOUS); if (!pages) return NULL; start = (char *) PAGE_TO_VIRT(pages); From 2f7ed11a0631996d292f04f07ec0a35b77189a08 Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Mon, 11 Nov 2024 20:27:24 +0000 Subject: [PATCH 40/76] mm/pagealloc: Fix buddy merging for highmem zones We were checking the page's pfn against start _addresses_, which meant that a good portion of the zone's pages didn't actually get merged up, which led to lots of failures when doing higher order allocation, even when lots of memory was free. Signed-off-by: Pedro Falcato --- kernel/kernel/mm/pagealloc.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/kernel/kernel/mm/pagealloc.cpp b/kernel/kernel/mm/pagealloc.cpp index 08c669947..62b036d9b 100644 --- a/kernel/kernel/mm/pagealloc.cpp +++ b/kernel/kernel/mm/pagealloc.cpp @@ -374,18 +374,19 @@ __always_inline struct page *get_buddy(struct page *page, unsigned int order, pa { unsigned long pfn = page_to_pfn(page); unsigned long pfn2 = pfn ^ (1UL << order); + unsigned long addr2 = pfn2 << PAGE_SHIFT; // Check if we can indeed merge with a buddy. if so // 1) the buddy is not past maxpfn (phys_to_page_mayfail) // 2) the buddy is free and the same order as us // 3) the buddy is in the same zone - struct page *p = phys_to_page_mayfail(pfn2 << PAGE_SHIFT); + struct page *p = phys_to_page_mayfail(addr2); if (!p) [[unlikely]] return nullptr; if (!(p->flags & PAGE_BUDDY) || p->priv != order) return nullptr; - if (pfn2 < zone->start || pfn2 > zone->end) + if (addr2 < zone->start || addr2 > zone->end) return nullptr; return p; } From 77a71d8df821fd2d497dc7843b5dab07b5f2ae03 Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Mon, 11 Nov 2024 20:30:02 +0000 Subject: [PATCH 41/76] mm: Add reclaim for order > 0 allocations This should aid in finding order > 0 allocations, even under memory pressure. Signed-off-by: Pedro Falcato --- kernel/include/onyx/page.h | 9 ++++ kernel/kernel/mm/pagealloc.cpp | 75 ++++++++++++++++++++++++++++++++++ kernel/kernel/mm/reclaim.c | 9 +++- 3 files changed, 92 insertions(+), 1 deletion(-) diff --git a/kernel/include/onyx/page.h b/kernel/include/onyx/page.h index 167f5c629..7253ddadf 100644 --- a/kernel/include/onyx/page.h +++ b/kernel/include/onyx/page.h @@ -534,6 +534,15 @@ static inline uint64_t get_kernel_phys_offset() */ unsigned long pages_under_high_watermark(); +/** + * @brief Calculate a free page target (for reclaim) + * + * @param gfp GFP used for the failed allocation/reclaim + * @param order Order allocation that failed + * @return Free page target. If 0, probably shouldn't reclaim. + */ +unsigned long page_reclaim_target(gfp_t gfp, unsigned int order); + /** * @brief Drain pages from all zones' pcpu caches * diff --git a/kernel/kernel/mm/pagealloc.cpp b/kernel/kernel/mm/pagealloc.cpp index 62b036d9b..976a2b365 100644 --- a/kernel/kernel/mm/pagealloc.cpp +++ b/kernel/kernel/mm/pagealloc.cpp @@ -316,6 +316,40 @@ struct page *page_zone_alloc(struct page_zone *zone, unsigned int gfp_flags, uns return page_zone_alloc_core(zone, gfp_flags, order); } +static bool page_zone_may_alloc(struct page_zone *zone, gfp_t gfp, unsigned int order) +{ + bool may = false; + unsigned long flags = spin_lock_irqsave(&zone->lock); + bool may_use_reserves = gfp & __GFP_ATOMIC; + unsigned long free_pages = zone->total_pages - zone->used_pages; + + if (free_pages < (1UL << order)) + goto out; + + if (!may_use_reserves && zone->min_watermark > free_pages - (1UL << order)) + goto out; + + if (!list_is_empty(&zone->pages[order])) [[likely]] + { + may = true; + goto out; + } + + /* Ok, this order has no pages, see if we could split other higher order ones */ + for (int i = order + 1; i < PAGEALLOC_NR_ORDERS; i++) + { + if (!list_is_empty(&zone->pages[i])) + { + may = true; + goto out; + } + } + +out: + spin_unlock_irqrestore(&zone->lock, flags); + return may; +} + static void page_zone_add(unsigned long start, unsigned int order, struct page_zone *zone) { scoped_lock g{zone->lock}; @@ -1017,3 +1051,44 @@ void page_accumulate_stats(unsigned long pages[PAGE_STATS_MAX]) return true; }); } + +/** + * @brief Calculate a free page target (for reclaim) + * + * @param gfp GFP used for the failed allocation/reclaim + * @param order Order allocation that failed + * @return Free page target. If 0, probably shouldn't reclaim. + */ +unsigned long page_reclaim_target(gfp_t gfp, unsigned int order) +{ + bool may = false; + unsigned long free_target = pages_under_high_watermark(); + if (free_target > 0) + return free_target; + + /* Everything is over the high watermark. Check if we indeed can accomplish this allocation. + * This does a slight emulation of alloc_page logic paths. + */ + int zone = ZONE_NORMAL; + + if (gfp & PAGE_ALLOC_4GB_LIMIT) + zone = ZONE_DMA32; + + while (zone >= 0) + { + may = page_zone_may_alloc(&main_node.zones[zone], gfp, order); + if (may) + break; + zone--; + } + + if (may) + return 0; + + /* We are above the high watermark, however we can't allocate this order. Start freeing pages, + * as a fixed % of total pages, scaled by order (capped to 3). We heuristically pick 1.5% of + * total pages. + */ + free_target = (nr_global_pages / 66) * cul::max(order, 3U); + return free_target; +} diff --git a/kernel/kernel/mm/reclaim.c b/kernel/kernel/mm/reclaim.c index 69d2bc90e..5eb9fceca 100644 --- a/kernel/kernel/mm/reclaim.c +++ b/kernel/kernel/mm/reclaim.c @@ -606,6 +606,13 @@ static void shrink_page_zones(struct reclaim_data *data, struct page_node *node) if (freep <= zone->low_watermark) target = zone->high_watermark - freep; + /* This logic is weird and leaky (we don't get nearly as many details from + * page_reclaim_target as we'd wish), but it should do the job. Get 1.5 * max(order, 3)% of + * the zone free. + */ + if (target == 0 && data->failed_order > 0) + target = (zone->total_pages / 66) * max(data->failed_order, 3); + if (target == 0) continue; @@ -627,7 +634,7 @@ int page_do_reclaim(struct reclaim_data *data) int max_tries = data->attempt > 0 ? 5 : 3; int nr_tries = 0; - while ((free_target = pages_under_high_watermark()) > 0) + while ((free_target = page_reclaim_target(data->gfp_flags, data->failed_order)) > 0) { if (nr_tries == max_tries) return -1; From dfaac1b926e42cdb5d82d80a0045b3417991ce79 Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Mon, 11 Nov 2024 20:31:18 +0000 Subject: [PATCH 42/76] file: Rework fd table allocations to avoid allocating under fdlock Signed-off-by: Pedro Falcato --- kernel/kernel/fs/file.cpp | 138 ++++++++++++++++++++++++++------------ 1 file changed, 95 insertions(+), 43 deletions(-) diff --git a/kernel/kernel/fs/file.cpp b/kernel/kernel/fs/file.cpp index 19488c514..54e396cd8 100644 --- a/kernel/kernel/fs/file.cpp +++ b/kernel/kernel/fs/file.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017 - 2023 Pedro Falcato + * Copyright (c) 2017 - 2024 Pedro Falcato * This file is part of Onyx, and is released under the terms of the GPLv2 License * check LICENSE at the root directory for more information * @@ -39,7 +39,7 @@ * * @return Pointer to struct fd_table, or nullptr */ -fd_table *fdtable_alloc(); +static fd_table *fdtable_alloc(); /** * @brief Free a struct fd_table @@ -364,38 +364,65 @@ struct file *get_file_description(int fd) return __get_file_description(fd, get_current_process()); } -int copy_file_descriptors(struct process *process, struct ioctx *ctx) +static int dup_fdtable(struct ioctx *ctx, struct fd_table *table) { - fd_table *table = fdtable_alloc(); - if (!table) - return -ENOMEM; + struct fd_table *oldt = ctx->table; + unsigned int nr_fds = oldt->file_desc_entries; - scoped_lock g{ctx->fdlock}; - - fd_table *oldt = ctx->table; + /* Release the ioctx lock, required for the next few allocations */ + spin_unlock(&ctx->fdlock); - table->file_desc = (file **) malloc(oldt->file_desc_entries * sizeof(void *)); - table->file_desc_entries = oldt->file_desc_entries; + table->file_desc = (file **) kmalloc(nr_fds * sizeof(void *), GFP_KERNEL); + table->file_desc_entries = nr_fds; if (!table->file_desc) - { return -ENOMEM; - } - table->cloexec_fds = (unsigned long *) malloc(table->file_desc_entries / 8); + table->cloexec_fds = (unsigned long *) kmalloc(nr_fds / 8, GFP_KERNEL); if (!table->cloexec_fds) { - free(table->file_desc); + kfree(table->file_desc); return -ENOMEM; } - table->open_fds = (unsigned long *) malloc(table->file_desc_entries / 8); + table->open_fds = (unsigned long *) kmalloc(nr_fds / 8, GFP_KERNEL); if (!table->open_fds) { - free(table->file_desc); - free(table->cloexec_fds); + kfree(table->file_desc); + kfree(table->cloexec_fds); return -ENOMEM; } + spin_lock(&ctx->fdlock); + return 0; +} + +int copy_file_descriptors(struct process *process, struct ioctx *ctx) +{ + int err; + struct fd_table *oldt; + struct fd_table *table = fdtable_alloc(); + if (!table) + return -ENOMEM; + + spin_lock(&ctx->fdlock); + + for (;;) + { + err = dup_fdtable(ctx, table); + /* dup_fdtable drops the lock and doesn't re-lock on error */ + if (err) + return err; + if (table->file_desc_entries >= ctx->table->file_desc_entries) + break; + + /* Need to re-alloc everything, so free the old data */ + kfree(table->cloexec_fds); + kfree(table->file_desc); + kfree(table->open_fds); + table->file_desc_entries = 0; + } + + oldt = ctx->table; memcpy(table->cloexec_fds, oldt->cloexec_fds, table->file_desc_entries / 8); memcpy(table->open_fds, oldt->open_fds, table->file_desc_entries / 8); @@ -406,8 +433,8 @@ int copy_file_descriptors(struct process *process, struct ioctx *ctx) fd_get(table->file_desc[i]); } + spin_unlock(&ctx->fdlock); rcu_assign_pointer(process->ctx.table, table); - return 0; } @@ -417,20 +444,20 @@ int allocate_file_descriptor_table(struct process *process) if (!table) return -ENOMEM; - table->file_desc = (file **) zalloc(FILE_DESCRIPTOR_GROW_NR * sizeof(void *)); + table->file_desc = (file **) kcalloc(FILE_DESCRIPTOR_GROW_NR, sizeof(void *), GFP_KERNEL); if (!table->file_desc) return -ENOMEM; table->file_desc_entries = FILE_DESCRIPTOR_GROW_NR; - table->cloexec_fds = (unsigned long *) zalloc(FILE_DESCRIPTOR_GROW_NR / 8); + table->cloexec_fds = (unsigned long *) kcalloc(FILE_DESCRIPTOR_GROW_NR / 8, 1, GFP_KERNEL); if (!table->cloexec_fds) { free(table->file_desc); return -ENOMEM; } - table->open_fds = (unsigned long *) zalloc(FILE_DESCRIPTOR_GROW_NR / 8); + table->open_fds = (unsigned long *) kcalloc(FILE_DESCRIPTOR_GROW_NR / 8, 1, GFP_KERNEL); if (!table->open_fds) { free(table->file_desc); @@ -457,32 +484,43 @@ static void defer_free_fd_table_rcu(fd_table *table_) #define FD_ENTRIES_TO_FDSET_SIZE(x) ((x) / 8) /* Enlarges the file descriptor table by FILE_DESCRIPTOR_GROW_NR(64) entries */ -int enlarge_file_descriptor_table(struct process *process, unsigned int new_size) +static int enlarge_fdtable(struct process *process, unsigned int new_size) { + int err = -ENOMEM; struct fd_table *oldt = process->ctx.table; - fd_table *table = fdtable_alloc(); - if (!table) - return -ENOMEM; - unsigned int old_nr_fds = oldt->file_desc_entries; + struct fd_table *table = nullptr; + struct file **ftable = nullptr; + unsigned long *cloexec_fds = nullptr; + unsigned long *open_fds = nullptr; new_size = ALIGN_TO(new_size, FILE_DESCRIPTOR_GROW_NR); - if (new_size > INT_MAX || new_size >= process->get_rlimit(RLIMIT_NOFILE).rlim_cur) - { - fdtable_free(table); return -EMFILE; - } - unsigned int new_nr_fds = new_size; + /* Can't allocate with the fdlock held... */ + spin_unlock(&process->ctx.fdlock); - struct file **ftable = (file **) zalloc(new_nr_fds * sizeof(void *)); - unsigned long *cloexec_fds = (unsigned long *) zalloc(FD_ENTRIES_TO_FDSET_SIZE(new_nr_fds)); - /* We use zalloc here to implicitly zero free fds */ - unsigned long *open_fds = (unsigned long *) zalloc(FD_ENTRIES_TO_FDSET_SIZE(new_nr_fds)); + table = fdtable_alloc(); + if (!table) + goto error; + + ftable = (struct file **) kcalloc(new_size, sizeof(void *), GFP_KERNEL); + cloexec_fds = (unsigned long *) kcalloc(FD_ENTRIES_TO_FDSET_SIZE(new_size), 1, GFP_KERNEL); + /* We use kcalloc here to implicitly zero free fds */ + open_fds = (unsigned long *) kcalloc(FD_ENTRIES_TO_FDSET_SIZE(new_size), 1, GFP_KERNEL); if (!ftable || !cloexec_fds || !open_fds) goto error; + spin_lock(&process->ctx.fdlock); + if (process->ctx.table != oldt) + { + /* Someone changed the fd table while we were gone, retry */ + err = -EAGAIN; + goto error_nolock; + } + + DCHECK(process->ctx.table->file_desc_entries == old_nr_fds); /* Note that we use old_nr_fds for these copies specifically as to not go * out of bounds. */ @@ -490,7 +528,7 @@ int enlarge_file_descriptor_table(struct process *process, unsigned int new_size memcpy(cloexec_fds, oldt->cloexec_fds, FD_ENTRIES_TO_FDSET_SIZE(old_nr_fds)); memcpy(open_fds, oldt->open_fds, FD_ENTRIES_TO_FDSET_SIZE(old_nr_fds)); - table->file_desc_entries = new_nr_fds; + table->file_desc_entries = new_size; rcu_assign_pointer(table->file_desc, ftable); rcu_assign_pointer(table->cloexec_fds, cloexec_fds); rcu_assign_pointer(table->open_fds, open_fds); @@ -501,13 +539,15 @@ int enlarge_file_descriptor_table(struct process *process, unsigned int new_size return 0; error: + spin_lock(&process->ctx.fdlock); +error_nolock: if (table) fdtable_free(table); free(ftable); free(cloexec_fds); free(open_fds); - return -ENOMEM; + return err; } void process_destroy_file_descriptors(process *process) @@ -580,8 +620,10 @@ int alloc_fd(int fdbase) /* TODO: Make it so we can enlarge it directly to the size we want */ int new_entries = table->file_desc_entries + FILE_DESCRIPTOR_GROW_NR; - if (int st = enlarge_file_descriptor_table(current, new_entries); st < 0) + if (int st = enlarge_fdtable(current, new_entries); st < 0) { + if (st == -EAGAIN) + continue; return st; } } @@ -870,6 +912,7 @@ int sys_dup23_internal(int oldfd, int newfd, int dupflags, unsigned int flags) // printk("pid %d oldfd %d newfd %d\n", get_current_process()->pid, oldfd, newfd); struct process *current = get_current_process(); struct ioctx *ioctx = ¤t->ctx; + struct fd_table *table; if (newfd < 0 || oldfd < 0) return -EBADF; @@ -884,19 +927,28 @@ int sys_dup23_internal(int oldfd, int newfd, int dupflags, unsigned int flags) scoped_lock g{ioctx->fdlock}; +retry: + table = rcu_dereference(ioctx->table); if ((unsigned int) newfd >= ioctx->table->file_desc_entries) { - int st = enlarge_file_descriptor_table(current, (unsigned int) newfd + 1); + int st = enlarge_fdtable(current, (unsigned int) newfd + 1); if (st < 0) { + if (st == -EAGAIN) + { + /* EAGAIN = someone touched the fd table while allocating, retry */ + goto retry; + } + // open() expects EMFILE, dup2/3 expects EBADF if (st == -EMFILE) st = -EBADF; return st; } + + table = rcu_dereference(ioctx->table); } - fd_table *table = rcu_dereference(ioctx->table); if (oldfd == newfd) return flags & DUP23_DUP3 ? -EINVAL : oldfd; @@ -2112,9 +2164,9 @@ void file_free(struct file *file) * * @return Pointer to struct fd_table, or nullptr */ -fd_table *fdtable_alloc() +static fd_table *fdtable_alloc() { - auto table = (fd_table *) kmem_cache_alloc(fdtable_cache, 0); + auto table = (fd_table *) kmem_cache_alloc(fdtable_cache, GFP_KERNEL); if (table) memset(table, 0, sizeof(*table)); return table; From be3112c3f3a618f6114b470971ef0c3f874754af Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Mon, 11 Nov 2024 20:31:54 +0000 Subject: [PATCH 43/76] process: Allocating struct process with GFP_KERNEL Signed-off-by: Pedro Falcato --- kernel/kernel/process.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kernel/kernel/process.cpp b/kernel/kernel/process.cpp index 82f81b7ad..39f2c83eb 100644 --- a/kernel/kernel/process.cpp +++ b/kernel/kernel/process.cpp @@ -173,10 +173,11 @@ process *process_create(const std::string_view &cmd_line, ioctx *ctx, process *p assert(process_ids != nullptr); } - auto p = make_unique(); + unique_ptr p{(struct process *) kmalloc(sizeof(struct process), GFP_KERNEL)}; if (!p) return errno = ENOMEM, nullptr; + new (p.get()) process; auto proc = p.get(); /* TODO: idm_get_id doesn't wrap? POSIX COMPLIANCE */ From 9ec9e901b5837f937d24a0d558f2a8fcd3442dfe Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Mon, 11 Nov 2024 20:32:49 +0000 Subject: [PATCH 44/76] ext2: Add GFP_NOFS to most big allocations Also, while we're at it, remove a few debugging leftover messages, and correctly pass errors through when add_dirent fails. Signed-off-by: Pedro Falcato --- kernel/kernel/fs/ext2/ext2.cpp | 3 +-- kernel/kernel/fs/ext2/ext2_ll.cpp | 11 ++++++----- kernel/kernel/fs/ext2/symlink.cpp | 5 +---- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/kernel/kernel/fs/ext2/ext2.cpp b/kernel/kernel/fs/ext2/ext2.cpp index 25fe722b8..0eb6b4639 100644 --- a/kernel/kernel/fs/ext2/ext2.cpp +++ b/kernel/kernel/fs/ext2/ext2.cpp @@ -580,8 +580,7 @@ struct inode *ext2_create_file(const char *name, mode_t mode, dev_t dev, struct if (int st = ext2_add_direntry(name, inumber, inode, vfs_ino, fs); st < 0) { thread_change_addr_limit(old); - printk("ext2 error %d\n", st); - errno = EINVAL; + errno = -st; goto free_ino_error; } diff --git a/kernel/kernel/fs/ext2/ext2_ll.cpp b/kernel/kernel/fs/ext2/ext2_ll.cpp index e132b5135..0c33ef7a6 100644 --- a/kernel/kernel/fs/ext2/ext2_ll.cpp +++ b/kernel/kernel/fs/ext2/ext2_ll.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -204,9 +205,9 @@ int ext2_add_direntry(const char *name, uint32_t inum, struct ext2_inode *ino, i ext2_superblock *fs) { uint8_t *buffer; - uint8_t *buf = buffer = (uint8_t *) zalloc(fs->block_size); + uint8_t *buf = buffer = (uint8_t *) kcalloc(fs->block_size, 1, GFP_NOFS); if (!buf) - return errno = ENOMEM, -1; + return -ENOMEM; if (inum == 0) panic("Bad inode number passed to ext2_add_direntry"); @@ -343,7 +344,7 @@ int ext2_remove_direntry(uint32_t inum, struct inode *dir, struct ext2_superbloc { int st = -ENOENT; uint8_t *buf_start; - uint8_t *buf = buf_start = (uint8_t *) zalloc(fs->block_size); + uint8_t *buf = buf_start = (uint8_t *) kcalloc(fs->block_size, 1, GFP_NOFS); if (!buf) return errno = ENOMEM, -1; @@ -411,7 +412,7 @@ int ext2_retrieve_dirent(inode *inode, const char *name, ext2_superblock *fs, ext2_dirent_result *res) { int st = -ENOENT; - char *buf = static_cast(zalloc(fs->block_size)); + char *buf = static_cast(kcalloc(fs->block_size, 1, GFP_NOFS)); if (!buf) return -ENOMEM; @@ -549,7 +550,7 @@ int ext2_dir_empty(struct inode *ino) struct ext2_superblock *fs = ext2_superblock_from_inode(ino); int st = 1; - char *buf = (char *) zalloc(fs->block_size); + char *buf = (char *) kcalloc(fs->block_size, 1, GFP_NOFS); if (!buf) return -ENOMEM; diff --git a/kernel/kernel/fs/ext2/symlink.cpp b/kernel/kernel/fs/ext2/symlink.cpp index 69153f0f5..37174d4f5 100644 --- a/kernel/kernel/fs/ext2/symlink.cpp +++ b/kernel/kernel/fs/ext2/symlink.cpp @@ -177,17 +177,14 @@ inode *ext2_symlink(struct dentry *dentry, const char *dest, struct dentry *dir) if (auto st = ext2_set_symlink(ino, dest); st < 0) { - pr_err("set symlink err %d\n", st); errno = -st; goto free_ino_error; } - pr_info("creating symlink %s (len %zu)\n", dentry->d_name, dentry->d_name_length); if (int st = ext2_add_direntry(dentry->d_name, inumber, inode, vfs_ino, fs); st < 0) { thread_change_addr_limit(old); - printk("ext2 error %d\n", st); - errno = EINVAL; + errno = -st; goto free_ino_error; } From f4f9a2d04e708ad384369cb881a7172b52b4b355 Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Sun, 17 Nov 2024 18:35:46 +0000 Subject: [PATCH 45/76] readahead: Add block device support for readahead Signed-off-by: Pedro Falcato --- kernel/include/onyx/block.h | 1 + kernel/include/onyx/buffer.h | 1 + kernel/kernel/fs/block.cpp | 7 +- kernel/kernel/fs/buffer.cpp | 176 ++++++++++++++++++++++++++++++++++- kernel/kernel/fs/filemap.cpp | 2 +- kernel/kernel/fs/readahead.c | 5 + 6 files changed, 185 insertions(+), 7 deletions(-) diff --git a/kernel/include/onyx/block.h b/kernel/include/onyx/block.h index 5c888d325..17b874050 100644 --- a/kernel/include/onyx/block.h +++ b/kernel/include/onyx/block.h @@ -212,5 +212,6 @@ int block_set_bsize(struct blockdev *bdev, unsigned int block_size); int bdev_do_open(struct blockdev *bdev, bool exclusive); void bdev_release(struct blockdev *bdev); unsigned int bdev_sector_size(struct blockdev *bdev); +u64 bdev_get_size(struct blockdev *bdev); __END_CDECLS #endif diff --git a/kernel/include/onyx/buffer.h b/kernel/include/onyx/buffer.h index 6cbea2ca3..480773ac0 100644 --- a/kernel/include/onyx/buffer.h +++ b/kernel/include/onyx/buffer.h @@ -49,6 +49,7 @@ struct block_buf #define BLOCKBUF_FLAG_WRITEBACK (1 << 1) #define BLOCKBUF_FLAG_UPTODATE (1 << 2) #define BLOCKBUF_FLAG_AREAD (1 << 3) +#define BLOCKBUF_FLAG_HOLE (1 << 4) static inline bool bb_test_and_set(struct block_buf *buf, unsigned int flag) { diff --git a/kernel/kernel/fs/block.cpp b/kernel/kernel/fs/block.cpp index 6d6649b47..8c44461e4 100644 --- a/kernel/kernel/fs/block.cpp +++ b/kernel/kernel/fs/block.cpp @@ -40,6 +40,11 @@ struct hd_geometry static int block_reread_parts(struct blockdev *bdev); +u64 bdev_get_size(struct blockdev *bdev) +{ + return bdev->nr_sectors * bdev->sector_size; +} + unsigned int blkdev_ioctl(int request, void *argp, struct file *f) { auto d = (blockdev *) f->f_ino->i_helper; @@ -47,7 +52,7 @@ unsigned int blkdev_ioctl(int request, void *argp, struct file *f) switch (request) { case BLKGETSIZE64: { - u64 len = d->nr_sectors * d->sector_size; + u64 len = bdev_get_size(d); return copy_to_user(argp, &len, sizeof(u64)); } diff --git a/kernel/kernel/fs/buffer.cpp b/kernel/kernel/fs/buffer.cpp index 31026126d..85e1733ca 100644 --- a/kernel/kernel/fs/buffer.cpp +++ b/kernel/kernel/fs/buffer.cpp @@ -151,7 +151,7 @@ bool page_has_writeback_bufs(struct page *p) struct block_buf *page_add_blockbuf(struct page *page, unsigned int page_off) { assert(page->flags & PAGE_FLAG_BUFFER); - DCHECK(page_locked(page)); + CHECK_PAGE(page_locked(page), page); auto buf = (struct block_buf *) kmem_cache_alloc(buffer_cache, GFP_KERNEL); if (!buf) @@ -255,9 +255,6 @@ void page_destroy_block_bufs(struct page *page) ssize_t bbuffer_readpage(struct page *p, size_t off, struct inode *ino) { - p->flags |= PAGE_FLAG_BUFFER; - p->priv = 0; - auto blkdev = reinterpret_cast(ino->i_helper); DCHECK(blkdev != nullptr); @@ -291,6 +288,9 @@ ssize_t bbuffer_readpage(struct page *p, size_t off, struct inode *ino) if (iost < 0) return iost; + if (!page_test_set_buffer(p)) + goto skip_setup; + for (size_t i = 0; i < nr_blocks; i++) { struct block_buf *b; @@ -307,7 +307,10 @@ ssize_t bbuffer_readpage(struct page *p, size_t off, struct inode *ino) curr_off += block_size; } - p->flags |= PAGE_FLAG_UPTODATE; +skip_setup: + page_set_uptodate(p); + for (struct block_buf *b = (struct block_buf *) p->priv; b; b = b->next) + bb_test_and_set(b, BLOCKBUF_FLAG_UPTODATE); return PAGE_SIZE; } @@ -411,6 +414,168 @@ static ssize_t buffer_directio(struct file *filp, size_t off, iovec_iter *iter, return to_read; } +static void buffer_readpages_endio(struct bio_req *bio) NO_THREAD_SAFETY_ANALYSIS +{ + for (size_t i = 0; i < bio->nr_vecs; i++) + { + struct page_iov *iov = &bio->vec[i]; + DCHECK(page_locked(iov->page)); + struct block_buf *head = (struct block_buf *) iov->page->priv; + + spin_lock(&head->pagestate_lock); + bool uptodate = true; + + for (struct block_buf *b = head; b != nullptr; b = b->next) + { + if (b->page_off >= iov->page_off && + b->page_off + b->block_size <= iov->page_off + iov->length) + { + bb_clear_flag(b, BLOCKBUF_FLAG_AREAD); + CHECK(bb_test_and_set(b, BLOCKBUF_FLAG_UPTODATE)); + continue; + } + + if (!bb_test_flag(b, BLOCKBUF_FLAG_UPTODATE)) + uptodate = false; + } + + spin_unlock(&head->pagestate_lock); + + if (uptodate) + { + if ((bio->flags & BIO_STATUS_MASK) == BIO_REQ_DONE) + page_test_set_flag(iov->page, PAGE_FLAG_UPTODATE); + unlock_page(iov->page); + } + } +} + +static int buffer_readpages(struct readpages_state *state, + struct inode *ino) NO_THREAD_SAFETY_ANALYSIS +{ + blockdev *blkdev = reinterpret_cast(ino->i_helper); + int st; + struct page *page; + unsigned int nr_ios = 0; + auto block_size = blkdev->block_size; + u64 nblocks = blkdev->nr_sectors / (block_size / blkdev->sector_size); + + while ((page = readpages_next_page(state))) + { + const unsigned long pgoff = page->pageoff; + nr_ios = 0; + auto nr_blocks = PAGE_SIZE / block_size; + size_t starting_block_nr = (pgoff << PAGE_SHIFT) / block_size; + size_t curr_off = 0; + + if (!page_test_set_flag(page, PAGE_FLAG_BUFFER)) + goto skip_setup; + + for (size_t i = 0; i < nr_blocks; i++) + { + struct block_buf *b; + if (!(b = page_add_blockbuf(page, curr_off))) + { + page_destroy_block_bufs(page); + st = -ENOMEM; + goto out_err; + } + + b->block_nr = starting_block_nr + i; + if (b->block_nr >= nblocks) + bb_test_and_set(b, BLOCKBUF_FLAG_HOLE | BLOCKBUF_FLAG_UPTODATE); + b->block_size = block_size; + b->dev = blkdev; + curr_off += block_size; + } + + if (starting_block_nr + nr_blocks <= nblocks) + { + /* Fast, simple case. Fire off a single BIO for this whole contiguous page. This makes + * it so we can fire off larger BIOs for, e.g, NVMe, which then increases the chance of + * it getting merged with other bios, etc. + */ + struct block_buf *b = (struct block_buf *) page->priv; + struct bio_req *bio = bio_alloc(GFP_NOFS, 1); + if (!bio) + { + st = -ENOMEM; + goto out_err; + } + + bb_test_and_set(b, BLOCKBUF_FLAG_AREAD); + + bio->sector_number = b->block_nr * (block_size / blkdev->sector_size); + bio->flags = BIO_REQ_READ_OP; + bio->b_end_io = buffer_readpages_endio; + bio_push_pages(bio, page, 0, PAGE_SIZE); + st = bio_submit_request(blkdev, bio); + bio_put(bio); + + if (st < 0) + { + bb_clear_flag(b, BLOCKBUF_FLAG_AREAD); + goto out_err; + } + + nr_ios++; + goto end_read; + } + + skip_setup: + for (struct block_buf *b = (struct block_buf *) page->priv; b != nullptr; b = b->next) + { + sector_t block = b->block_nr; + if (bb_test_flag(b, BLOCKBUF_FLAG_UPTODATE)) + continue; + if (bb_test_flag(b, BLOCKBUF_FLAG_HOLE)) + continue; + if (!bb_test_and_set(b, BLOCKBUF_FLAG_AREAD)) + continue; + CHECK(!bb_test_flag(b, BLOCKBUF_FLAG_UPTODATE)); + + struct bio_req *bio = bio_alloc(GFP_NOFS, 1); + if (!bio) + { + bb_clear_flag(b, BLOCKBUF_FLAG_AREAD); + st = -ENOMEM; + goto out_err; + } + + /* Note: We do not need to ref, we hold the lock, no one can throw this page away + * while locked (almost like an implicit reference). */ + bio->sector_number = block * (block_size / blkdev->sector_size); + bio->flags = BIO_REQ_READ_OP; + bio->b_end_io = buffer_readpages_endio; + bio_push_pages(bio, page, b->page_off, b->block_size); + st = bio_submit_request(blkdev, bio); + bio_put(bio); + + if (st < 0) + { + bb_clear_flag(b, BLOCKBUF_FLAG_AREAD); + goto out_err; + } + + nr_ios++; + } + + end_read: + if (nr_ios == 0) + unlock_page(page); + page_unref(page); + } + + return 0; +out_err: + /* On error, release the page we're holding. We do not unlock it if we submitted any IOs for the + * page, the endio page will do it for us. */ + if (nr_ios == 0) + unlock_page(page); + page_unref(page); + return st; +} + static int block_prepare_write(struct inode *ino, struct page *page, size_t page_off, size_t offset, size_t len) { @@ -437,6 +602,7 @@ struct file_ops buffer_ops = { .writepages = filemap_writepages, .fsyncdata = filemap_writepages, .directio = buffer_directio, + .readpages = buffer_readpages, }; struct block_buf *sb_read_block(const struct superblock *sb, unsigned long block) diff --git a/kernel/kernel/fs/filemap.cpp b/kernel/kernel/fs/filemap.cpp index db7cbebfe..f00c71161 100644 --- a/kernel/kernel/fs/filemap.cpp +++ b/kernel/kernel/fs/filemap.cpp @@ -80,7 +80,7 @@ int filemap_find_page(struct inode *ino, size_t pgoff, unsigned int flags, struc page_promote_referenced(p); } - if (!(flags & (FIND_PAGE_NO_READPAGE | FIND_PAGE_NO_RA)) && ra_state && !S_ISBLK(ino->i_mode)) + if (!(flags & (FIND_PAGE_NO_READPAGE | FIND_PAGE_NO_RA)) && ra_state) { rw_lock_read(&ino->i_pages->truncate_lock); /* If we found PAGE_FLAG_READAHEAD, kick off more IO */ diff --git a/kernel/kernel/fs/readahead.c b/kernel/kernel/fs/readahead.c index 2f458c463..be85aea25 100644 --- a/kernel/kernel/fs/readahead.c +++ b/kernel/kernel/fs/readahead.c @@ -68,6 +68,8 @@ static void readpages_finish(struct readpages_state *state) NO_THREAD_SAFETY_ANA } } +u64 bdev_get_size(struct blockdev *bdev); + static int filemap_do_readahead(struct inode *inode, struct readahead_state *ra_state, unsigned long pgoff) NO_THREAD_SAFETY_ANALYSIS { @@ -76,6 +78,9 @@ static int filemap_do_readahead(struct inode *inode, struct readahead_state *ra_ size_t endpg; struct blk_plug plug; + if (S_ISBLK(inode->i_mode)) + size = bdev_get_size(inode->i_helper); + /* Do basic bounds checks on our readahead window */ if (!size) return 0; From 832c44549d7de62959e2c9ce3defdf139cbe2405 Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Sun, 17 Nov 2024 18:38:14 +0000 Subject: [PATCH 46/76] vm_obj: Properly truncate pages Set owner to NULL, as is required for truncation detection. This fixes fsx (ext2) kernel crashes on a sufficiently long-running process, when priv = 0 and ext2_writepage. Signed-off-by: Pedro Falcato --- kernel/kernel/mm/vm_object.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/kernel/kernel/mm/vm_object.cpp b/kernel/kernel/mm/vm_object.cpp index 23d66cb33..e89413283 100644 --- a/kernel/kernel/mm/vm_object.cpp +++ b/kernel/kernel/mm/vm_object.cpp @@ -240,6 +240,11 @@ static void vm_obj_truncate_out(struct vm_object *obj, struct page *const *batch CHECK(pg->owner == obj); int st = obj->vm_pages.store(pg->pageoff, 0); CHECK(st == 0); + /* Once we release this lock, other people trying to get a stable reference to this page + * will re-check owner under the page lock. If it observes a pg->owner == original_vmobj, + * it'll keep going without realizing this page is being/was truncated. So we need to + * WRITE_ONCE pg->owner here. */ + WRITE_ONCE(pg->owner, nullptr); } spin_unlock(&obj->page_lock); @@ -570,6 +575,7 @@ bool vm_obj_remove_page(struct vm_object *obj, struct page *page) obj->vm_pages.store(page_pgoff(page), 0); if (page_test_swap(page)) swap_unset_swapcache(swpval_to_swp_entry(page->priv)); + /* We do not need to reset owner here, because we're the only reference */ ret = true; out: spin_unlock(&obj->page_lock); From 75cf5cf93d71317462e47fadab7f82771f0d676d Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Sun, 17 Nov 2024 18:39:41 +0000 Subject: [PATCH 47/76] filemap: Add a retry loop for truncation When we seem to find a truncated page, retry the lookup. Signed-off-by: Pedro Falcato --- kernel/kernel/fs/filemap.cpp | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/kernel/kernel/fs/filemap.cpp b/kernel/kernel/fs/filemap.cpp index f00c71161..d19131e88 100644 --- a/kernel/kernel/fs/filemap.cpp +++ b/kernel/kernel/fs/filemap.cpp @@ -26,7 +26,9 @@ int filemap_find_page(struct inode *ino, size_t pgoff, unsigned int flags, struc { struct page *p = nullptr; int st = 0; - vmo_status_t vst = vmo_get(ino->i_pages, pgoff << PAGE_SHIFT, 0, &p); + vmo_status_t vst = VMO_STATUS_OK; +retry: + vst = vmo_get(ino->i_pages, pgoff << PAGE_SHIFT, 0, &p); if (vst != VMO_STATUS_OK) { if (vst == VMO_STATUS_BUS_ERROR) @@ -108,6 +110,14 @@ int filemap_find_page(struct inode *ino, size_t pgoff, unsigned int flags, struc rw_lock_read(&ino->i_pages->truncate_lock); lock_page(p); + if (p->owner != ino->i_pages) + { + unlock_page(p); + page_unref(p); + rw_unlock_read(&ino->i_pages->truncate_lock); + goto retry; + } + if (!page_flag_set(p, PAGE_FLAG_UPTODATE)) { ssize_t st2 = ino->i_fops->readpage(p, pgoff << PAGE_SHIFT, ino); @@ -126,7 +136,15 @@ int filemap_find_page(struct inode *ino, size_t pgoff, unsigned int flags, struc } if (flags & FIND_PAGE_LOCK) + { lock_page(p); + if (p->owner != ino->i_pages) + { + unlock_page(p); + page_unref(p); + goto retry; + } + } out: if (st == 0) @@ -498,7 +516,8 @@ void filemap_clear_dirty(struct page *page) REQUIRES(page) { /* Clear the dirty flag for IO */ struct vm_object *obj = page_vmobj(page); - __atomic_and_fetch(&page->flags, ~PAGE_FLAG_DIRTY, __ATOMIC_RELEASE); + if (!page_test_clear_dirty(page)) + return; { scoped_lock g{obj->page_lock}; From 9cdc3f4195727317b1806b20482efd891ec1cc9b Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Sun, 17 Nov 2024 18:41:11 +0000 Subject: [PATCH 48/76] namei: Actually enforce umask on file creation umask was being ignored in O_CREAT and other file creations. This is obviously wrong and defeats the purpose of umask. Signed-off-by: Pedro Falcato --- kernel/include/onyx/process.h | 12 ++++++++++++ kernel/kernel/fs/file.cpp | 5 ----- kernel/kernel/fs/namei.cpp | 4 +++- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/kernel/include/onyx/process.h b/kernel/include/onyx/process.h index 81d663738..686c4243d 100644 --- a/kernel/include/onyx/process.h +++ b/kernel/include/onyx/process.h @@ -328,6 +328,18 @@ __attribute__((pure)) static inline struct process *get_current_process() return (thread == NULL) ? NULL : (struct process *) thread->owner; } +static inline mode_t get_current_umask() +{ + if (unlikely(!get_current_process())) + return 0; + return get_current_process()->ctx.umask; +} + +static inline mode_t do_umask(mode_t mode) +{ + return mode & ~get_current_umask(); +} + /** * @brief Get the number of active processes * diff --git a/kernel/kernel/fs/file.cpp b/kernel/kernel/fs/file.cpp index 54e396cd8..0b0c588ec 100644 --- a/kernel/kernel/fs/file.cpp +++ b/kernel/kernel/fs/file.cpp @@ -743,11 +743,6 @@ void handle_open_flags(struct file *fd, int flags) fd->f_seek = fd->f_ino->i_size; } -static inline mode_t get_current_umask() -{ - return get_current_process()->ctx.umask; -} - bool may_noatime(file *f) { creds_guard g; diff --git a/kernel/kernel/fs/namei.cpp b/kernel/kernel/fs/namei.cpp index 70e7e4acb..e3de86b2c 100644 --- a/kernel/kernel/fs/namei.cpp +++ b/kernel/kernel/fs/namei.cpp @@ -485,7 +485,8 @@ static int do_creat(dentry *dir, struct inode *inode, struct dentry *dentry, mod DCHECK(d_is_negative(dentry)); - struct inode *new_inode = inode->i_fops->creat(dentry, (int) mode | S_IFREG, dir); + struct inode *new_inode = + inode->i_fops->creat(dentry, do_umask((int) (mode & ~S_IFMT) | S_IFREG), dir); if (!new_inode) return -errno; @@ -861,6 +862,7 @@ static expected namei_create_generic(int dirfd, const char goto put_unlock_err; } + mode = do_umask(mode); switch (mode & S_IFMT) { case S_IFREG: From 9c59d43b258d67cc06127775bf31a1a9cf8f51fd Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Sun, 17 Nov 2024 18:44:18 +0000 Subject: [PATCH 49/76] block: Add write_begin and write_end Add two new callbacks that allow us to do more intelligent page cache accesses when doing writes. Also add a sample implementation for block devices. Signed-off-by: Pedro Falcato --- kernel/include/onyx/mm/vm_object.h | 6 + kernel/kernel/fs/block.cpp | 198 +++++++++++++++++++++++++++++ kernel/kernel/fs/filemap.cpp | 83 ++++++++---- 3 files changed, 260 insertions(+), 27 deletions(-) diff --git a/kernel/include/onyx/mm/vm_object.h b/kernel/include/onyx/mm/vm_object.h index fd62f106e..6b25cb0cd 100644 --- a/kernel/include/onyx/mm/vm_object.h +++ b/kernel/include/onyx/mm/vm_object.h @@ -47,11 +47,17 @@ static inline int vmo_status_to_errno(vmo_status_t st) struct page; struct vm_object; +struct file; + struct vm_object_ops { void (*free_page)(struct vm_object *vmo, struct page *page); void (*truncate_partial)(struct vm_object *vmobj, struct page *page, size_t offset, size_t len); ssize_t (*writepage)(struct vm_object *vm_obj, struct page *page, size_t off); + int (*write_begin)(struct file *file, struct vm_object *vm_obj, off_t offset, size_t len, + struct page **page); + int (*write_end)(struct file *file, struct vm_object *vm_obj, off_t offset, + unsigned int written, unsigned int to_write, struct page *page); }; #define VMO_FLAG_LOCK_FUTURE_PAGES (1 << 0) diff --git a/kernel/kernel/fs/block.cpp b/kernel/kernel/fs/block.cpp index 8c44461e4..3bed1c9ca 100644 --- a/kernel/kernel/fs/block.cpp +++ b/kernel/kernel/fs/block.cpp @@ -96,8 +96,206 @@ struct block_inode static unique_ptr create(const struct blockdev *dev, flush::writeback_dev *wbdev); }; +static void buffer_write_readpages_endio(struct bio_req *bio) NO_THREAD_SAFETY_ANALYSIS +{ + for (size_t i = 0; i < bio->nr_vecs; i++) + { + struct page_iov *iov = &bio->vec[i]; + DCHECK(page_locked(iov->page)); + struct block_buf *head = (struct block_buf *) iov->page->priv; + + spin_lock(&head->pagestate_lock); + bool uptodate = true; + + for (struct block_buf *b = head; b != nullptr; b = b->next) + { + if (b->page_off == iov->page_off) + { + bb_clear_flag(b, BLOCKBUF_FLAG_AREAD); + CHECK(bb_test_and_set(b, BLOCKBUF_FLAG_UPTODATE)); + wake_address(b); + continue; + } + + if (!bb_test_flag(b, BLOCKBUF_FLAG_UPTODATE)) + uptodate = false; + } + + spin_unlock(&head->pagestate_lock); + + if (uptodate) + { + if ((bio->flags & BIO_STATUS_MASK) == BIO_REQ_DONE) + page_set_uptodate(iov->page); + } + } +} + +static int block_readpage_write(struct vm_object *vm_obj, off_t offset, size_t len, + struct page *page) +{ + struct blockdev *bdev = (struct blockdev *) vm_obj->ino->i_helper; + unsigned int page_off = offset - (page->pageoff << PAGE_SHIFT); + unsigned int page_len = min(len, PAGE_SIZE - page_off); + int st; + + auto nr_blocks = PAGE_SIZE / bdev->block_size; + size_t starting_block_nr = (page->pageoff << PAGE_SHIFT) / bdev->block_size; + size_t curr_off = 0; + auto block_size = bdev->block_size; + u64 nblocks = bdev->nr_sectors / (block_size / bdev->sector_size); + + if (!page_test_set_flag(page, PAGE_FLAG_BUFFER)) + goto skip_setup; + + for (size_t i = 0; i < nr_blocks; i++) + { + struct block_buf *b; + if (!(b = page_add_blockbuf(page, curr_off))) + { + page_destroy_block_bufs(page); + st = -ENOMEM; + goto out_err; + } + + b->block_nr = starting_block_nr + i; + if (b->block_nr >= nblocks) + { + bb_test_and_set(b, BLOCKBUF_FLAG_HOLE); + bb_test_and_set(b, BLOCKBUF_FLAG_UPTODATE); + } + + b->block_size = bdev->block_size; + b->dev = bdev; + curr_off += bdev->block_size; + } +skip_setup: + + for (struct block_buf *buf = (struct block_buf *) page->priv; buf; buf = buf->next) + { + /* Go through the page, read-in block buffers. If we _fully_ overwrite a block, don't bring + * that one in. */ + if (buf->page_off >= page_off && buf->page_off + buf->block_size <= page_len) + continue; + sector_t block = buf->block_nr; + if (bb_test_flag(buf, BLOCKBUF_FLAG_UPTODATE)) + continue; + if (bb_test_flag(buf, BLOCKBUF_FLAG_HOLE)) + continue; + if (!bb_test_and_set(buf, BLOCKBUF_FLAG_AREAD)) + continue; + CHECK(!bb_test_flag(buf, BLOCKBUF_FLAG_UPTODATE)); + + struct bio_req *bio = bio_alloc(GFP_NOFS, 1); + if (!bio) + { + bb_clear_flag(buf, BLOCKBUF_FLAG_AREAD); + st = -ENOMEM; + goto out_err; + } + + /* Note: We do not need to ref, we hold the lock, no one can throw this page away + * while locked (almost like an implicit reference). */ + bio->sector_number = block * (block_size / bdev->sector_size); + bio->flags = BIO_REQ_READ_OP; + bio->b_end_io = buffer_write_readpages_endio; + bio_push_pages(bio, page, buf->page_off, buf->block_size); + st = bio_submit_request(bdev, bio); + bio_put(bio); + + if (st < 0) + { + bb_clear_flag(buf, BLOCKBUF_FLAG_AREAD); + goto out_err; + } + } + + for (struct block_buf *buf = (struct block_buf *) page->priv; buf; buf = buf->next) + { + if (bb_test_flag(buf, BLOCKBUF_FLAG_AREAD)) + wait_for( + buf, + [](void *ptr) -> bool { + struct block_buf *b = (struct block_buf *) ptr; + return !bb_test_flag(b, BLOCKBUF_FLAG_AREAD); + }, + 0, 0); + } + + return 0; + +out_err: + return st; +} + +static int block_write_begin(struct file *filp, struct vm_object *vm_obj, off_t offset, size_t len, + struct page **ppage) +{ + struct page *page; + int st = filemap_find_page(filp->f_ino, offset >> PAGE_SHIFT, + FIND_PAGE_ACTIVATE | FIND_PAGE_NO_READPAGE | FIND_PAGE_LOCK, &page, + &filp->f_ra_state); + if (st < 0) + return st; + + if (!page_test_uptodate(page)) + { + st = block_readpage_write(vm_obj, offset, len, page); + if (st < 0) + goto err_read_page; + } + + *ppage = page; + return 0; + +err_read_page: + unlock_page(page); + page_unref(page); + return st; +} + +static int __block_write_end(struct file *file, struct vm_object *vm_obj, off_t offset, + unsigned int written, unsigned int to_write, struct page *page) +{ + unsigned int page_off = offset - (page->pageoff << PAGE_SHIFT); + bool uptodate = true; + CHECK(page_test_buffer(page)); + struct block_buf *head = (struct block_buf *) page->priv; + spin_lock(&head->pagestate_lock); + + for (struct block_buf *buf = (struct block_buf *) page->priv; buf; buf = buf->next) + { + if (buf->page_off <= page_off && buf->page_off + buf->block_size >= offset + written) + { + /* Fully contained in the write, mark uptodate */ + bb_test_and_set(buf, BLOCKBUF_FLAG_UPTODATE); + wake_address(buf); + } + + if (!bb_test_flag(buf, BLOCKBUF_FLAG_UPTODATE)) + uptodate = false; + } + + if (uptodate && !page_test_uptodate(page)) + page_set_uptodate(page); + spin_unlock(&head->pagestate_lock); + return 0; +} + +static int block_write_end(struct file *file, struct vm_object *vm_obj, off_t offset, + unsigned int written, unsigned int to_write, struct page *page) +{ + __block_write_end(file, vm_obj, offset, written, to_write, page); + unlock_page(page); + page_unref(page); + /* Block devices don't need to set i_size */ + return 0; +} + static const struct vm_object_ops block_vm_obj_ops = { .free_page = buffer_free_page, + .write_begin = block_write_begin, + .write_end = block_write_end, }; /** diff --git a/kernel/kernel/fs/filemap.cpp b/kernel/kernel/fs/filemap.cpp index d19131e88..633233a1a 100644 --- a/kernel/kernel/fs/filemap.cpp +++ b/kernel/kernel/fs/filemap.cpp @@ -394,6 +394,49 @@ ssize_t file_write_cache(void *buffer, size_t len, struct inode *ino, size_t off return file_write_cache_unlocked(buffer, len, ino, offset); } +static int default_write_begin(struct file *filp, struct vm_object *vm_obj, off_t off, size_t len, + struct page **ppage) +{ + struct page *page = nullptr; + struct inode *ino = vm_obj->ino; + int st = filemap_find_page(filp->f_ino, off >> PAGE_SHIFT, FIND_PAGE_ACTIVATE, &page, + &filp->f_ra_state); + + if (st < 0) + return st; + + auto cache_off = off % PAGE_SIZE; + size_t aligned_off = off & -PAGE_SIZE; + auto rest = PAGE_SIZE - cache_off; + + if (rest > len) + rest = len; + + lock_page(page); + + if (st = ino->i_fops->prepare_write(ino, page, aligned_off, cache_off, rest); st < 0) + { + unlock_page(page); + page_unpin(page); + return st; + } + + *ppage = page; + return 0; +} + +static int default_write_end(struct file *file, struct vm_object *vm_obj, off_t offset, + unsigned int written, unsigned int to_write, struct page *page) +{ + struct inode *ino = vm_obj->ino; + unlock_page(page); + page_unref(page); + + if (written > 0 && (size_t) offset + written > ino->i_size && !S_ISBLK(ino->i_mode)) + inode_set_size(ino, offset + written); + return 0; +} + /** * @brief Write to a generic file (using the page cache) using iovec_iter * @@ -406,6 +449,7 @@ ssize_t file_write_cache(void *buffer, size_t len, struct inode *ino, size_t off ssize_t filemap_write_iter(struct file *filp, size_t off, iovec_iter *iter, unsigned int flags) { struct inode *ino = filp->f_ino; + struct vm_object *vm_obj = ino->i_pages; if (filp->f_flags & O_DIRECT) return filemap_do_direct(filp, off, iter, DIRECT_IO_WRITE); @@ -416,48 +460,33 @@ ssize_t filemap_write_iter(struct file *filp, size_t off, iovec_iter *iter, unsi while (!iter->empty()) { - struct page *page = nullptr; - int st2 = filemap_find_page(filp->f_ino, off >> PAGE_SHIFT, FIND_PAGE_ACTIVATE, &page, - &filp->f_ra_state); + int st2; + struct page *page; + st2 = (vm_obj->ops->write_begin ?: default_write_begin)(filp, vm_obj, off, iter->bytes, + &page); if (st2 < 0) return st ?: st2; - void *buffer = PAGE_TO_VIRT(page); - - auto cache_off = off % PAGE_SIZE; - size_t aligned_off = off & -PAGE_SIZE; - auto rest = PAGE_SIZE - cache_off; - - if (rest > iter->bytes) - rest = iter->bytes; - - lock_page(page); - - if (st2 = ino->i_fops->prepare_write(ino, page, aligned_off, cache_off, rest); st2 < 0) - { - unlock_page(page); - page_unpin(page); - return st ?: st2; - } + void *buffer = PAGE_TO_VIRT(page); + unsigned int page_off = off - (page->pageoff << PAGE_SHIFT); + unsigned int len = min(iter->bytes, PAGE_SIZE - page_off); /* copy_from_iter advances the iter automatically */ - ssize_t copied = copy_from_iter(iter, (u8 *) buffer + cache_off, rest); + ssize_t copied = copy_from_iter(iter, (u8 *) buffer + page_off, len); if (copied > 0) filemap_mark_dirty(page, off >> PAGE_SHIFT); - unlock_page(page); - page_unpin(page); - + st2 = (vm_obj->ops->write_end ?: default_write_end)(filp, vm_obj, off, + copied > 0 ? copied : 0, len, page); if (copied <= 0) return st ?: copied; + if (st2 < 0) + return st ?: st2; /* note: if copied < rest, we either faulted or ran out of len. in any case, it's handled */ off += copied; st += copied; - - if (off > ino->i_size && !S_ISBLK(ino->i_mode)) - inode_set_size(ino, off); } return st; From aa6ac1458559f80c5b137ed15c1e3b19bbcd9bda Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Sun, 17 Nov 2024 18:45:26 +0000 Subject: [PATCH 50/76] file: Fix getcwd() freeing a stack buffer This is obviously wrong and resulted in crashes. Signed-off-by: Pedro Falcato --- kernel/kernel/fs/file.cpp | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/kernel/kernel/fs/file.cpp b/kernel/kernel/fs/file.cpp index 0b0c588ec..21e8cfeb7 100644 --- a/kernel/kernel/fs/file.cpp +++ b/kernel/kernel/fs/file.cpp @@ -1597,16 +1597,11 @@ int sys_getcwd(char *path, size_t size) pathlen = pathbuf + PATH_MAX - name; if (pathlen > size) - { - free(name); return -ERANGE; - } if (copy_to_user(path, name, pathlen) < 0) - { - free(name); return -errno; - } + return pathlen - 1; } From 6fdf56ad15838db390e987ec779136cceda676da Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Sun, 17 Nov 2024 18:45:52 +0000 Subject: [PATCH 51/76] page: Add BUFFER flag accessors Signed-off-by: Pedro Falcato --- kernel/include/onyx/page.h | 1 + 1 file changed, 1 insertion(+) diff --git a/kernel/include/onyx/page.h b/kernel/include/onyx/page.h index 7253ddadf..f494a7c07 100644 --- a/kernel/include/onyx/page.h +++ b/kernel/include/onyx/page.h @@ -622,6 +622,7 @@ PAGEFLAG_OPS(active, ACTIVE); PAGEFLAG_OPS(lru, LRU); PAGEFLAG_OPS(uptodate, UPTODATE); PAGEFLAG_OPS(dirty, DIRTY); +PAGEFLAG_OPS(buffer, BUFFER); struct vm_object *page_vmobj(struct page *page); unsigned long page_pgoff(struct page *page); From 5c99607b2d319e9054731522ade7b189a6b13eb5 Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Sun, 17 Nov 2024 18:46:18 +0000 Subject: [PATCH 52/76] block: Add BLKFLSBUF ioctl BLKFLSBUF flushes all buffers and pages related to this block device. Signed-off-by: Pedro Falcato --- kernel/kernel/fs/block.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/kernel/kernel/fs/block.cpp b/kernel/kernel/fs/block.cpp index 3bed1c9ca..997e8abea 100644 --- a/kernel/kernel/fs/block.cpp +++ b/kernel/kernel/fs/block.cpp @@ -71,6 +71,18 @@ unsigned int blkdev_ioctl(int request, void *argp, struct file *f) return copy_to_user(argp, &d->sector_size, sizeof(unsigned int)); } + case BLKFLSBUF: { + if (!is_root_user()) + return -EACCES; + /* Synchronize the block device's page cache and truncate all the pages out! */ + if (int st = filemap_fdatasync(d->b_ino, 0, -1UL); st < 0) + return st; + + if (int st = vmo_punch_range(d->b_ino->i_pages, 0, -1UL); st < 0) + return st; + return 0; + } + default: return -EINVAL; } From 8d059cc783829718a6425a8024a78016f0eb29cb Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Sun, 17 Nov 2024 18:47:00 +0000 Subject: [PATCH 53/76] ext2: Replace a CHECK with CHECK_PAGE Signed-off-by: Pedro Falcato --- kernel/kernel/fs/ext2/ext2.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/kernel/kernel/fs/ext2/ext2.cpp b/kernel/kernel/fs/ext2/ext2.cpp index 0eb6b4639..92d721942 100644 --- a/kernel/kernel/fs/ext2/ext2.cpp +++ b/kernel/kernel/fs/ext2/ext2.cpp @@ -164,11 +164,10 @@ static ssize_t ext2_writepage(struct vm_object *obj, page *page, size_t off) REQ buf = buf->next; } - unlock_page(page); - /* For this to have been a valid dirty page, we must've been able to submit more than 0 ios (a * page full of zero blocks cannot be dirty, as prepare_write must be called). */ - CHECK(nr_ios > 0); + CHECK_PAGE(nr_ios > 0, page); + unlock_page(page); return PAGE_SIZE; } From d3be07f9e377334a7d952044527e5b50475272f6 Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Sun, 17 Nov 2024 18:47:18 +0000 Subject: [PATCH 54/76] ext2: Simply WARN on i_blocks != 0 This stops bad, inconsistent filesystems from crashing the whole system. Signed-off-by: Pedro Falcato --- kernel/kernel/fs/ext2/inode.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/kernel/fs/ext2/inode.cpp b/kernel/kernel/fs/ext2/inode.cpp index ff613a42a..3d99bf560 100644 --- a/kernel/kernel/fs/ext2/inode.cpp +++ b/kernel/kernel/fs/ext2/inode.cpp @@ -257,7 +257,7 @@ int ext2_free_space(size_t new_len, inode *ino); void ext2_free_inode_space(inode *inode_, ext2_superblock *fs) { ext2_free_space(0, inode_); - assert(inode_->i_blocks == 0); + WARN_ON(inode_->i_blocks != 0); } struct ext2_block_coords From 1edeb532a88f99a2aa72fbcb796dd75b7d2e6356 Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Sun, 17 Nov 2024 18:48:17 +0000 Subject: [PATCH 55/76] nvme: Add block request dumping on error Plus the generic bits in fs/block Signed-off-by: Pedro Falcato --- kernel/drivers/nvme/nvme.cpp | 2 ++ kernel/include/onyx/bdev_base_types.h | 2 ++ kernel/kernel/fs/block/request.cpp | 19 +++++++++++++++++++ 3 files changed, 23 insertions(+) diff --git a/kernel/drivers/nvme/nvme.cpp b/kernel/drivers/nvme/nvme.cpp index fcdbcc2a1..c393e8a5b 100644 --- a/kernel/drivers/nvme/nvme.cpp +++ b/kernel/drivers/nvme/nvme.cpp @@ -833,6 +833,8 @@ bool nvme_device::nvme_queue::handle_cq() NVME_CQE_STATUS_DNR(cqe->dw3) ? ", do not repeat" : ""); pr_err("nvme%u: Related SQE dump: %*ph\n", dev_->device_index_, (int) sizeof(nvmesqe), &command->cmd); + if (NVME_CQE_STATUS_DNR(cqe->dw3)) + blk_request_dump(command->req, KERN_ERR); command->req->r_flags |= BIO_REQ_EIO; } diff --git a/kernel/include/onyx/bdev_base_types.h b/kernel/include/onyx/bdev_base_types.h index 9741e6eec..48a40eb9c 100644 --- a/kernel/include/onyx/bdev_base_types.h +++ b/kernel/include/onyx/bdev_base_types.h @@ -89,6 +89,8 @@ static inline void *b_request_to_data(struct request *req) return req + 1; } +void blk_request_dump(struct request *req, const char *log_lvl); + struct blk_plug { struct list_head request_list; diff --git a/kernel/kernel/fs/block/request.cpp b/kernel/kernel/fs/block/request.cpp index 0f2d7ee2f..4b3dee905 100644 --- a/kernel/kernel/fs/block/request.cpp +++ b/kernel/kernel/fs/block/request.cpp @@ -280,6 +280,24 @@ bool blk_merge_plug(struct blk_plug *plug, struct bio_req *bio) return false; } +void blk_request_dump(struct request *req, const char *log_lvl) +{ + printk("%srequest %p: flags %x sectors %llu nsectors %llu nr sgls %zu\n", log_lvl, req, + req->r_flags, (unsigned long long) req->r_sector, (unsigned long long) req->r_nsectors, + req->r_nr_sgls); + printk("%s bdev %p queue %p\n", log_lvl, req->r_bdev, req->r_queue); + + for_every_bio(req, [log_lvl](struct bio_req *bio) -> bool { + printk("bio flags %x sector %llu b_end_io %ps\n", bio->flags, + (unsigned long long) bio->sector_number, bio->b_end_io); + for (size_t i = 0; i < bio->nr_vecs; i++) + printk("%spage_iov %zu: page %p (pfn %lu), length %x, page_off %u\n", log_lvl, i, + bio->vec[i].page, page_to_pfn(bio->vec[i].page), bio->vec[i].length, + bio->vec[i].page_off); + return true; + }); +} + #ifdef CONFIG_KUNIT static blockdev test_bdev; @@ -421,6 +439,7 @@ static struct queue_properties nvme_queue_properties() qp.inter_sgl_boundary_mask = PAGE_SIZE - 1; qp.max_sectors_per_request = 0xffff; qp.dma_address_mask = 3; + qp.max_sgl_desc_length = PAGE_SIZE; return qp; } From 799f645ea7829b3a23dc3a693470288b49663996 Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Sun, 17 Nov 2024 18:48:42 +0000 Subject: [PATCH 56/76] nvme: Actually batch sqe's instead of calling device_io_submit This makes us ring the nvme doorbell once per pull, instead of once per sqe, which results in a nice speedup, particularly in VMs. Signed-off-by: Pedro Falcato --- kernel/drivers/nvme/nvme.cpp | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/kernel/drivers/nvme/nvme.cpp b/kernel/drivers/nvme/nvme.cpp index c393e8a5b..a58ea281b 100644 --- a/kernel/drivers/nvme/nvme.cpp +++ b/kernel/drivers/nvme/nvme.cpp @@ -974,12 +974,26 @@ int nvme_device::nvme_queue::pull_sq() if (!req) break; - if (int st = device_io_submit(req); st < 0) + nvmecmd *cmd = &request_to_pdu(req)->cmd; + if (int st = prepare_nvme_request(req->r_flags & BIO_REQ_OP_MASK, cmd, req, + (nvme_namespace *) req->r_bdev->device_info); + st < 0) { - printf("nvme: device_io_submit failed with err %d, unpulling sqe\n", st); + pr_err("nvme: prepare_nvme_request failed with err %d, unpulling sqe\n", st); unpull_seq(req); break; } + + int cid = allocate_cid(); + DCHECK(cid >= 0); + DCHECK(((sq_tail_ + 1) % sq_size_) != sq_head_); + + auto next_entry = sq_tail_; + + sq_tail_ = (sq_tail_ + 1) % sq_size_; + cmd->cmd.cdw0.cid = cid; + queued_commands_[cmd->cmd.cdw0.cid] = cmd; + memcpy(&sq_[next_entry], &cmd->cmd, sizeof(nvmesqe)); } /* Queue is full, ring the doorbell */ From d06d9db759b3fe63eee6d459671ac8c7f80d4b9a Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Sun, 17 Nov 2024 18:55:48 +0000 Subject: [PATCH 57/76] github: Update {download, upload}-artifact to v4 Signed-off-by: Pedro Falcato --- .github/workflows/main.yml | 46 +++++++++++++++++------------------ .github/workflows/nightly.yml | 38 ++++++++++++++--------------- 2 files changed, 42 insertions(+), 42 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f50f5661d..a04bed73d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -28,7 +28,7 @@ jobs: - name: Download the x86_64-onyx-linux toolchain - uses: actions/download-artifact@v2.0.8 + uses: actions/download-artifact@v4 with: # Artifact name name: x86_64-onyx-linux @@ -54,19 +54,19 @@ jobs: ./scripts/create_disk_image.sh disk-image.img --bootable efi - name: Upload a Build Artifact(Onyx.iso) - uses: actions/upload-artifact@v2.1.4 + uses: actions/upload-artifact@v4 with: name: Onyx ISO path: Onyx.iso - name: Upload a Build Artifact(kernel/vmonyx) - uses: actions/upload-artifact@v2.1.4 + uses: actions/upload-artifact@v4 with: name: vmonyx path: kernel/vmonyx - name: Upload a Build Artifact(Disk image) - uses: actions/upload-artifact@v2.1.4 + uses: actions/upload-artifact@v4 with: name: Disk-image.img path: disk-image.img @@ -89,7 +89,7 @@ jobs: # Runs a single command using the runners shell - name: Download the onyx-llvm-linux toolchain - uses: actions/download-artifact@v2.0.8 + uses: actions/download-artifact@v4 with: # Artifact name name: onyx-llvm-linux @@ -115,19 +115,19 @@ jobs: ./scripts/create_disk_image.sh disk-image-llvm.img --bootable efi - name: Upload a Build Artifact(Onyx.iso) - uses: actions/upload-artifact@v2.1.4 + uses: actions/upload-artifact@v4 with: name: Onyx-iso-llvm path: Onyx.iso - name: Upload a Build Artifact(kernel/vmonyx) - uses: actions/upload-artifact@v2.1.4 + uses: actions/upload-artifact@v4 with: name: vmonyx-llvm path: kernel/vmonyx - name: Upload a Build Artifact(disk-image-llvm) - uses: actions/upload-artifact@v2.1.4 + uses: actions/upload-artifact@v4 with: name: Disk-image-llvm.img path: disk-image-llvm.img @@ -147,7 +147,7 @@ jobs: - name: Download the riscv64-onyx-linux toolchain - uses: actions/download-artifact@v2.0.8 + uses: actions/download-artifact@v4 with: # Artifact name name: riscv64-onyx-linux @@ -176,7 +176,7 @@ jobs: zstd -T0 -15 riscv64-onyx-image.tar - name: Upload the riscv64 boot image - uses: actions/upload-artifact@v2.1.4 + uses: actions/upload-artifact@v4 with: name: Onyx boot image (riscv64) path: riscv64-onyx-image.tar.zst @@ -199,7 +199,7 @@ jobs: # Runs a single command using the runners shell - name: Download the onyx-llvm-linux toolchain - uses: actions/download-artifact@v2.0.8 + uses: actions/download-artifact@v4 with: # Artifact name name: onyx-llvm-linux @@ -229,7 +229,7 @@ jobs: zstd -T0 -15 riscv64-onyx-image-llvm.tar - name: Upload the riscv64 boot image (LLVM) - uses: actions/upload-artifact@v2.1.4 + uses: actions/upload-artifact@v4 with: name: Onyx boot image (riscv64) (llvm) path: riscv64-onyx-image-llvm.tar.zst @@ -295,7 +295,7 @@ jobs: zstd -T0 -15 minimal-sysroot-${{ matrix.target_arch }}.tar - name: Upload the minimal sysroot - uses: actions/upload-artifact@v2.1.4 + uses: actions/upload-artifact@v4 with: name: minimal-sysroot-${{ matrix.target_arch }} path: minimal-sysroot-${{ matrix.target_arch }}.tar.zst @@ -322,7 +322,7 @@ jobs: submodules: "recursive" - name: Download the minimal sysroot - uses: actions/download-artifact@v2.0.8 + uses: actions/download-artifact@v4 - name: Extract minimal sysroot run: | @@ -413,7 +413,7 @@ jobs: mv ${{ env.toolchain_id_no_os }} toolchain_binaries-${{ matrix.toolchain }} - name: Upload the toolchain - uses: actions/upload-artifact@v2.1.4 + uses: actions/upload-artifact@v4 with: name: ${{ env.toolchain_id }} path: ${{ env.toolchain_id }}.tar.zst @@ -434,7 +434,7 @@ jobs: - name: Download the arm64-onyx-linux toolchain - uses: actions/download-artifact@v2.0.8 + uses: actions/download-artifact@v4 with: # Artifact name name: arm64-onyx-linux @@ -463,7 +463,7 @@ jobs: zstd -T0 -15 arm64-onyx-image.tar - name: Upload the arm64 boot image - uses: actions/upload-artifact@v2.1.4 + uses: actions/upload-artifact@v4 with: name: Onyx boot image (arm64) path: arm64-onyx-image.tar.zst @@ -487,7 +487,7 @@ jobs: # Runs a single command using the runners shell - name: Download the onyx-llvm-linux toolchain - uses: actions/download-artifact@v2.0.8 + uses: actions/download-artifact@v4 with: # Artifact name name: onyx-llvm-linux @@ -517,7 +517,7 @@ jobs: zstd -T0 -15 arm64-onyx-image-llvm.tar - name: Upload the arm64 boot image (LLVM) - uses: actions/upload-artifact@v2.1.4 + uses: actions/upload-artifact@v4 with: name: Onyx boot image (arm64) (llvm) path: arm64-onyx-image-llvm.tar.zst @@ -537,7 +537,7 @@ jobs: - name: Download the x86_64-onyx-linux toolchain - uses: actions/download-artifact@v2.0.8 + uses: actions/download-artifact@v4 with: # Artifact name name: x86_64-onyx-linux @@ -582,7 +582,7 @@ jobs: # Runs a single command using the runners shell - name: Download the onyx-llvm-linux toolchain - uses: actions/download-artifact@v2.0.8 + uses: actions/download-artifact@v4 with: # Artifact name name: onyx-llvm-linux @@ -625,7 +625,7 @@ jobs: - name: Download the riscv64-onyx-linux toolchain - uses: actions/download-artifact@v2.0.8 + uses: actions/download-artifact@v4 with: # Artifact name name: riscv64-onyx-linux @@ -669,7 +669,7 @@ jobs: # Runs a single command using the runners shell - name: Download the onyx-llvm-linux toolchain - uses: actions/download-artifact@v2.0.8 + uses: actions/download-artifact@v4 with: # Artifact name name: onyx-llvm-linux diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 64d238f66..05a421121 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -24,7 +24,7 @@ jobs: python-version: '3.11.0' # Require the same version of python as Onyx builds, in order to ease the build - name: Download the x86_64-onyx-linux toolchain - uses: actions/download-artifact@v2.0.8 + uses: actions/download-artifact@v4 with: # Artifact name name: x86_64-onyx-linux @@ -51,19 +51,19 @@ jobs: ./scripts/create_disk_image.sh disk-image.img --bootable efi - name: Upload a Build Artifact(Onyx.iso) - uses: actions/upload-artifact@v2.1.4 + uses: actions/upload-artifact@v4 with: name: Onyx ISO path: Onyx.iso - name: Upload a Build Artifact(kernel/vmonyx) - uses: actions/upload-artifact@v2.1.4 + uses: actions/upload-artifact@v4 with: name: vmonyx path: kernel/vmonyx - name: Upload a Build Artifact(Disk image) - uses: actions/upload-artifact@v2.1.4 + uses: actions/upload-artifact@v4 with: name: Disk-image.img path: disk-image.img @@ -86,7 +86,7 @@ jobs: # Runs a single command using the runners shell - name: Download the onyx-llvm-linux toolchain - uses: actions/download-artifact@v2.0.8 + uses: actions/download-artifact@v4 with: # Artifact name name: onyx-llvm-linux @@ -113,19 +113,19 @@ jobs: ./scripts/create_disk_image.sh disk-image-llvm.img --bootable efi - name: Upload a Build Artifact(Onyx.iso) - uses: actions/upload-artifact@v2.1.4 + uses: actions/upload-artifact@v4 with: name: Onyx-iso-llvm path: Onyx.iso - name: Upload a Build Artifact(kernel/vmonyx) - uses: actions/upload-artifact@v2.1.4 + uses: actions/upload-artifact@v4 with: name: vmonyx-llvm path: kernel/vmonyx - name: Upload a Build Artifact(disk-image-llvm) - uses: actions/upload-artifact@v2.1.4 + uses: actions/upload-artifact@v4 with: name: Disk-image-llvm.img path: disk-image-llvm.img @@ -145,7 +145,7 @@ jobs: - name: Download the riscv64-onyx-linux toolchain - uses: actions/download-artifact@v2.0.8 + uses: actions/download-artifact@v4 with: # Artifact name name: riscv64-onyx-linux @@ -174,7 +174,7 @@ jobs: zstd -T0 -15 riscv64-onyx-image.tar - name: Upload the riscv64 boot image - uses: actions/upload-artifact@v2.1.4 + uses: actions/upload-artifact@v4 with: name: Onyx boot image (riscv64) path: riscv64-onyx-image.tar.zst @@ -197,7 +197,7 @@ jobs: # Runs a single command using the runners shell - name: Download the onyx-llvm-linux toolchain - uses: actions/download-artifact@v2.0.8 + uses: actions/download-artifact@v4 with: # Artifact name name: onyx-llvm-linux @@ -227,7 +227,7 @@ jobs: zstd -T0 -15 riscv64-onyx-image-llvm.tar - name: Upload the riscv64 boot image (LLVM) - uses: actions/upload-artifact@v2.1.4 + uses: actions/upload-artifact@v4 with: name: Onyx boot image (riscv64) (llvm) path: riscv64-onyx-image-llvm.tar.zst @@ -292,7 +292,7 @@ jobs: zstd -T0 -15 minimal-sysroot-${{ matrix.target_arch }}.tar - name: Upload the minimal sysroot - uses: actions/upload-artifact@v2.1.4 + uses: actions/upload-artifact@v4 with: name: minimal-sysroot-${{ matrix.target_arch }} path: minimal-sysroot-${{ matrix.target_arch }}.tar.zst @@ -319,7 +319,7 @@ jobs: submodules: "recursive" - name: Download the minimal sysroot - uses: actions/download-artifact@v2.0.8 + uses: actions/download-artifact@v4 - name: Extract minimal sysroot run: | @@ -410,7 +410,7 @@ jobs: mv ${{ env.toolchain_id_no_os }} toolchain_binaries-${{ matrix.toolchain }} - name: Upload the toolchain - uses: actions/upload-artifact@v2.1.4 + uses: actions/upload-artifact@v4 with: name: ${{ env.toolchain_id }} path: ${{ env.toolchain_id }}.tar.zst @@ -430,7 +430,7 @@ jobs: - name: Download the arm64-onyx-linux toolchain - uses: actions/download-artifact@v2.0.8 + uses: actions/download-artifact@v4 with: # Artifact name name: arm64-onyx-linux @@ -459,7 +459,7 @@ jobs: zstd -T0 -15 arm64-onyx-image.tar - name: Upload the arm64 boot image - uses: actions/upload-artifact@v2.1.4 + uses: actions/upload-artifact@v4 with: name: Onyx boot image (arm64) path: arm64-onyx-image.tar.zst @@ -483,7 +483,7 @@ jobs: # Runs a single command using the runners shell - name: Download the onyx-llvm-linux toolchain - uses: actions/download-artifact@v2.0.8 + uses: actions/download-artifact@v4 with: # Artifact name name: onyx-llvm-linux @@ -513,7 +513,7 @@ jobs: zstd -T0 -15 arm64-onyx-image-llvm.tar - name: Upload the arm64 boot image (LLVM) - uses: actions/upload-artifact@v2.1.4 + uses: actions/upload-artifact@v4 with: name: Onyx boot image (arm64) (llvm) path: arm64-onyx-image-llvm.tar.zst From 6dcb98aa8aeede18b61a5dc7478a22c4ac9ecb3b Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Fri, 29 Nov 2024 18:56:36 +0000 Subject: [PATCH 58/76] Compile time fixen Signed-off-by: Pedro Falcato --- kernel/Makefile | 2 +- kernel/include/onyx/packetbuf.h | 24 +++++++++++++++++++++++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/kernel/Makefile b/kernel/Makefile index ac4209406..2aca9944b 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -31,7 +31,7 @@ CFLAGS:=$(CFLAGS) -MMD -ffreestanding -fbuiltin -Wall -Wextra -fstack-protector- ifneq ($(ONYX_USING_CLANG), yes) # TODO: Find suitable alternatives for clang -CFLAGS:=$(CFLAGS) -Wno-format-truncation -Wshadow-compatible-local +CFLAGS:=$(CFLAGS) -Wno-format-truncation -Wshadow-compatible-local -Wno-narrowing # Note: We need to define _GLIBCXX_INCLUDE_NEXT_C_HEADERS so the shenanigans in the # compiler (in this case, only GCC)'s stdlib.h C++ wrappers get bypassed. If we let stdlib.h include cstdlib, diff --git a/kernel/include/onyx/packetbuf.h b/kernel/include/onyx/packetbuf.h index d74915ddc..6e5bea9c5 100644 --- a/kernel/include/onyx/packetbuf.h +++ b/kernel/include/onyx/packetbuf.h @@ -30,6 +30,7 @@ struct vm_object; struct tcp_packetbuf_info { u32 seq, seq_len; + u8 ack : 1, syn : 1, fin : 1, rst : 1; }; struct packetbuf; @@ -101,7 +102,10 @@ struct packetbuf unsigned int zero_copy : 1; int domain; - struct list_head list_node; + union { + struct list_head list_node; + struct bst_node bst_node; + }; union { struct inet_route route; @@ -121,6 +125,7 @@ struct packetbuf data{}, tail{}, end{}, buffer_start{}, csum_offset{nullptr}, csum_start{nullptr}, header_length{}, gso_size{}, gso_flags{}, needs_csum{0}, zero_copy{0}, domain{0} { + route = {}; } /** @@ -378,6 +383,23 @@ static inline void *pbf_put(struct packetbuf *pbf, unsigned int size) return ret; } +/** + * @brief Get a header, and advance head by size. + * + * @param pbf Packetbuf + * @param size The length of the data. + * + * @return void* The address of the header. + */ +static inline void *pbf_pull(struct packetbuf *pbf, unsigned int size) +{ + if (unlikely(size > (pbf->tail - pbf->data))) + return NULL; + void *ret = pbf->data; + pbf->data += size; + return ret; +} + /** * @brief Calculates the total length of the buffer. * From 81c624cd02b372476301e21cdaede0dfa3978603 Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Fri, 29 Nov 2024 19:06:19 +0000 Subject: [PATCH 59/76] kernlog: Make printk_buf's initialization constinit This fixes certain weird bugs that only seemed to happen on EFI in some cases. Signed-off-by: Pedro Falcato --- kernel/kernel/kernlog.cpp | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/kernel/kernel/kernlog.cpp b/kernel/kernel/kernlog.cpp index a10fec6c1..47315946f 100644 --- a/kernel/kernel/kernlog.cpp +++ b/kernel/kernel/kernlog.cpp @@ -72,12 +72,16 @@ struct printk_header struct printk_buf { - char _log_buf[LOG_BUF_SIZE]; - size_t log_tail = 0; - size_t log_head = 0; - unsigned int msg_seq = 0; + char _log_buf[LOG_BUF_SIZE]{}; + size_t log_tail; + size_t log_head; + unsigned int msg_seq; static constexpr size_t logmask = LOG_BUF_SIZE - 1; + constexpr printk_buf() : log_tail{0}, log_head{0}, msg_seq{0} + { + } + /* Note: The functions below all require the printk_lock (or some other kind of mutual * exclusion) */ @@ -243,7 +247,7 @@ u32 printk_buf::find_and_print(char *buf, size_t *psize, u32 initial_seq, u32 fl return seen; } -static struct printk_buf printk_buf; +static constinit struct printk_buf printk_buf; static struct spinlock printk_lock; #define MAX_LINE 1024 static char flush_buf[MAX_LINE]; From 91eabf55d3da5d07139d673f2bdb4602217620b2 Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Fri, 29 Nov 2024 19:18:55 +0000 Subject: [PATCH 60/76] slab: Add krealloc and kreallocarray Signed-off-by: Pedro Falcato --- kernel/include/onyx/mm/slab.h | 4 ++++ kernel/kernel/mm/slab.c | 15 ++++++++++----- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/kernel/include/onyx/mm/slab.h b/kernel/include/onyx/mm/slab.h index be5d16be5..5c70dead4 100644 --- a/kernel/include/onyx/mm/slab.h +++ b/kernel/include/onyx/mm/slab.h @@ -117,6 +117,10 @@ void kfree(void *ptr); void *kcalloc(size_t nr, size_t size, int flags); +void *krealloc(void *ptr, size_t size, int flags); + +void *kreallocarray(void *ptr, size_t m, size_t n, int flags); + /** * @brief Purge a cache * This function goes through every free slab and gives it back to the page allocator. diff --git a/kernel/kernel/mm/slab.c b/kernel/kernel/mm/slab.c index 7068b5b7a..1456e39ef 100644 --- a/kernel/kernel/mm/slab.c +++ b/kernel/kernel/mm/slab.c @@ -1623,10 +1623,10 @@ void *calloc(size_t nr, size_t size) return kcalloc(nr, size, GFP_ATOMIC); } -void *realloc(void *ptr, size_t size) +void *krealloc(void *ptr, size_t size, int flags) { if (!ptr) - return malloc(size); + return kmalloc(size, flags); struct slab *old_slab = kmem_pointer_to_slab(ptr); @@ -1642,7 +1642,7 @@ void *realloc(void *ptr, size_t size) return ptr; } - void *newbuf = malloc(size); + void *newbuf = kmalloc(size, flags); if (!newbuf) return NULL; __memcpy(newbuf, ptr, old_slab->cache->objsize); @@ -1650,17 +1650,22 @@ void *realloc(void *ptr, size_t size) return newbuf; } +void *realloc(void *ptr, size_t size) +{ + return krealloc(ptr, size, GFP_ATOMIC); +} + int posix_memalign(void **pptr, size_t align, size_t len) { *pptr = NULL; return -1; } -void *reallocarray(void *ptr, size_t m, size_t n) +void *kreallocarray(void *ptr, size_t m, size_t n, int flags) { if (array_overflows(m, n)) return NULL; - return realloc(ptr, n * m); + return krealloc(ptr, n * m, flags); } #ifdef CONFIG_KASAN From fb3a3b4bdb02dcf29c06f9361ef2f3c41f9a29b6 Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Fri, 29 Nov 2024 19:22:01 +0000 Subject: [PATCH 61/76] x86: Rework some efi booting bits Don't believe GRUB's load bias (it seems to lie in some versions, fml). Also rework some bits to be slightly safer when it comes to avoiding a crash (namely, wrt running on wrong page tables). Signed-off-by: Pedro Falcato --- kernel/arch/x86_64/boot.S | 27 +++++++++++++++++---------- kernel/arch/x86_64/efistub/early.cpp | 4 +--- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/kernel/arch/x86_64/boot.S b/kernel/arch/x86_64/boot.S index 9e507bc14..80edde080 100644 --- a/kernel/arch/x86_64/boot.S +++ b/kernel/arch/x86_64/boot.S @@ -240,8 +240,6 @@ ENTRY(efi_entry_multiboot2_64) cmove 8(%rdi), %rdx cmpl $MULTIBOOT_TAG_TYPE_EFI64_IH, (%rdi) cmove 8(%rdi), %rcx - cmpl $MULTIBOOT_TAG_TYPE_LOAD_BASE_ADDR, (%rdi) - cmovel 8(%rdi), %esi /* Increment the pointer by size (rdi + 4) */ addl 4(%rdi), %edi @@ -260,22 +258,34 @@ ENTRY(efi_entry_multiboot2_64) push %rdx mov %rcx, %rdi - xchg %rsi, %rdx - sub $PHYS_BASE, %rdx + lea efi_entry_multiboot2_64(%rip), %rcx + movabs $efi_entry_multiboot2_64, %rsi + movabs $KERNEL_VIRTUAL_BASE, %rdx + sub %rdx, %rsi + sub %rsi, %rcx + mov %rcx, %rdx + lea kernel_phys_offset(%rip), %r9 + mov %rdx, (%r9) lea efi_state(%rip), %rcx mov $pml4, %r8 add %rdx, %r8 - lea kernel_phys_offset(%rip), %r9 - mov %rdx, (%r9) + mov (%rsp), %rsi + push %r8 call efi_handoff + pop %rdi + pop %r12 + lea x86_stack_top(%rip), %rsp + call efi_switch_mmu + call x86_efi_switch_tables + /* We're in pure 64-bit mode, paging enabled, with LA57 if it exists, ints off */ mov $x86_start, %rax mov $efi_entry_mb2, %rdx /* First arg = multiboot2 info */ mov %rbx, %rdi /* Second arg = EFI_SYSTEM_TABLE */ - pop %rsi + mov %r12, %rsi jmp *%rax 99: hlt @@ -295,9 +305,6 @@ END(efi_entry_multiboot2_64) ENTRY(x86_efi_switch_tables) /* We're still running in identity mode */ lea efi_gdtr(%rip), %rax - /* Fixup the gdtr's gdt address */ - lea efi_gdt_begin(%rip), %rdi - mov %rdi, 2(%rax) lgdt (%rax) mov $KERNEL_DS, %ax diff --git a/kernel/arch/x86_64/efistub/early.cpp b/kernel/arch/x86_64/efistub/early.cpp index a46565dca..2a02c4500 100644 --- a/kernel/arch/x86_64/efistub/early.cpp +++ b/kernel/arch/x86_64/efistub/early.cpp @@ -171,7 +171,7 @@ BOOT_SECTION static void setup_mmu(PML *page_tables, unsigned long phys_base, extern "C" void x86_efi_enable_57_mmu(PML *); -static void efi_switch_mmu(PML *page_tables) +extern "C" void efi_switch_mmu(PML *page_tables) { auto cr4 = x86_read_cr4(); // Note: Some people in Tianocore thought it's a brilliant idea to break backwards compatibility @@ -258,6 +258,4 @@ extern "C" BOOT_SECTION void efi_handoff(EFI_HANDLE image_handle, EFI_SYSTEM_TAB __asm__ __volatile__("int3"); __asm__ __volatile__("cli"); - x86_efi_switch_tables(); - efi_switch_mmu(page_tables); } From f49b6f647ec28bdbb5b20478aae14cc5a09d4f77 Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Fri, 29 Nov 2024 19:23:18 +0000 Subject: [PATCH 62/76] efi: SetVirtualAddressMap() fixes Map boot services eagerly (although we should end up unmapping this, Eventually, after we add proper reservation and late freeing of boot services memory). Also create our own memmap instead of passing the firmware's, which ended up causing crashes on some hardware. Signed-off-by: Pedro Falcato --- kernel/drivers/firmware/efi/efi.cpp | 60 +++++++++++++++++++++++++++-- 1 file changed, 57 insertions(+), 3 deletions(-) diff --git a/kernel/drivers/firmware/efi/efi.cpp b/kernel/drivers/firmware/efi/efi.cpp index 2e84db0ab..0034bd204 100644 --- a/kernel/drivers/firmware/efi/efi.cpp +++ b/kernel/drivers/firmware/efi/efi.cpp @@ -9,6 +9,7 @@ #include +#include #include #include #include @@ -89,7 +90,7 @@ unsigned long efi_memory_desc_flags_to_vm(uint64_t attributes) */ void efi_remap_efi_region(EFI_MEMORY_DESCRIPTOR &desc) { - pr_info("Remapping [%016lx, %016lx]\n", desc.PhysicalStart, + pr_warn("Remapping [%016lx, %016lx]\n", desc.PhysicalStart, desc.PhysicalStart + (desc.NumberOfPages << PAGE_SHIFT) - 1); bool mapping_over_null = desc.PhysicalStart == 0; @@ -137,6 +138,47 @@ static void efi_print_info() st->FirmwareRevision & 0xffff); } +static void efi_dump_mem_desc(const EFI_MEMORY_DESCRIPTOR *desc) +{ + pr_debug( + "Descriptor type %u physical start %lx virtual start %lx nr_pages %lx attributes %08lx\n", + desc->Type, desc->PhysicalStart, desc->VirtualStart, desc->NumberOfPages, desc->Attribute); +} + +#define EFI_RESERVED_TYPE 0 +#define EFI_LOADER_CODE 1 +#define EFI_LOADER_DATA 2 +#define EFI_BOOT_SERVICES_CODE 3 +#define EFI_BOOT_SERVICES_DATA 4 +#define EFI_RUNTIME_SERVICES_CODE 5 +#define EFI_RUNTIME_SERVICES_DATA 6 +#define EFI_CONVENTIONAL_MEMORY 7 +#define EFI_UNUSABLE_MEMORY 8 +#define EFI_ACPI_RECLAIM_MEMORY 9 +#define EFI_ACPI_MEMORY_NVS 10 +#define EFI_MEMORY_MAPPED_IO 11 +#define EFI_MEMORY_MAPPED_IO_PORT_SPACE 12 +#define EFI_PAL_CODE 13 +#define EFI_PERSISTENT_MEMORY 14 +#define EFI_UNACCEPTED_MEMORY 15 +#define EFI_MAX_MEMORY_TYPE 16 + +static bool should_map_efi(const EFI_MEMORY_DESCRIPTOR *desc) +{ + if (desc->Attribute & EFI_MEMORY_RUNTIME) + return true; + + /* Some runtime services like touching boot services memory on SVAM. Let's map it just for them, + * as a hack. */ + switch (desc->Type) + { + case EFI_BOOT_SERVICES_CODE: + case EFI_BOOT_SERVICES_DATA: + return true; + } + + return false; +} /** * @brief Initializes EFI * @@ -163,12 +205,24 @@ void efi_init(EFI_SYSTEM_TABLE *system_table, EFI_MEMORY_DESCRIPTOR *descriptors size_t nr_descriptors = mmap_size / descriptor_size; EFI_MEMORY_DESCRIPTOR *desc = descriptors; + EFI_MEMORY_DESCRIPTOR *map = NULL; + size_t nr_maps = 0; + for (size_t i = 0; i < nr_descriptors; desc = (EFI_MEMORY_DESCRIPTOR *) ((char *) desc + descriptor_size), i++) { - if (desc->Attribute & EFI_MEMORY_RUNTIME) +#ifdef CONFIG_EFI_DUMP_MEMMAP + efi_dump_mem_desc(desc); +#endif + if (should_map_efi(desc)) { + nr_maps++; + EFI_MEMORY_DESCRIPTOR *newmap = + (EFI_MEMORY_DESCRIPTOR *) kreallocarray(map, nr_maps, descriptor_size, GFP_ATOMIC); + CHECK(newmap != NULL); efi_remap_efi_region(*desc); + memcpy(&newmap[nr_maps - 1], desc, descriptor_size); + map = newmap; } } @@ -176,7 +230,7 @@ void efi_init(EFI_SYSTEM_TABLE *system_table, EFI_MEMORY_DESCRIPTOR *descriptors efi_guard g; EFI_STATUS st = g.system_table()->RuntimeServices->SetVirtualAddressMap( - mmap_size, descriptor_size, descriptor_version, descriptors); + nr_maps, descriptor_size, descriptor_version, map); assert(st == EFI_SUCCESS); } From e729c1e50839d2a8031ceb385c5c1b0f862cf8be Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Fri, 29 Nov 2024 19:26:01 +0000 Subject: [PATCH 63/76] namei: Rework dot-dot following on mounts We had a nasty hack with regards to .. following and mount roots. Fix it by doing proper mount parents. This also fixes '..' on mounts, which ended up broken and causing mount reference leaks. Signed-off-by: Pedro Falcato --- kernel/include/onyx/dentry.h | 3 + kernel/include/onyx/mount.h | 3 + kernel/kernel/fs/d_path.c | 32 +++++- kernel/kernel/fs/dentry.cpp | 4 +- kernel/kernel/fs/dev.cpp | 9 +- kernel/kernel/fs/mount.c | 50 +++++---- kernel/kernel/fs/namei.cpp | 190 ++++++++++++++++++++++++++++------- kernel/kernel/fs/tmpfs.cpp | 11 +- 8 files changed, 232 insertions(+), 70 deletions(-) diff --git a/kernel/include/onyx/dentry.h b/kernel/include/onyx/dentry.h index 662373ad9..c6fbc9778 100644 --- a/kernel/include/onyx/dentry.h +++ b/kernel/include/onyx/dentry.h @@ -295,6 +295,9 @@ bool dentry_does_not_have_parent(dentry *dir, dentry *to_not_have); void dentry_do_unlink(dentry *entry); void dentry_rename(dentry *dent, const char *name, dentry *parent, dentry *dst); void dentry_move(dentry *target, dentry *new_parent); +dentry *__dentry_try_to_open(std::string_view name, dentry *dir, bool lock_ino); +dentry *dentry_open_from_cache(dentry *dent, std::string_view name); +dentry *dentry_wait_for_pending(dentry *dent); #endif diff --git a/kernel/include/onyx/mount.h b/kernel/include/onyx/mount.h index d632b9b2d..ac214952a 100644 --- a/kernel/include/onyx/mount.h +++ b/kernel/include/onyx/mount.h @@ -22,6 +22,7 @@ struct mount struct dentry *mnt_root; struct superblock *mnt_sb; struct dentry *mnt_point; + struct mount *mnt_parent; unsigned int mnt_flags; /* TODO: percpu */ unsigned long mnt_count; @@ -29,6 +30,8 @@ struct mount struct rcu_head mnt_rcu; struct list_head mnt_mp_node; struct list_head mnt_node; + struct list_head mnt_submounts; + struct list_head mnt_submount_node; }; static inline void mnt_get(struct mount *mnt) diff --git a/kernel/kernel/fs/d_path.c b/kernel/kernel/fs/d_path.c index d1cff67a2..069a02d6b 100644 --- a/kernel/kernel/fs/d_path.c +++ b/kernel/kernel/fs/d_path.c @@ -60,21 +60,47 @@ static void prepend_dentry(struct rbuf *rbuf, struct dentry *dentry) spin_unlock(&dentry->d_lock); } +static bool follow_mount_up(struct mount *mnt, struct path *out) +{ + struct dentry *dentry = mnt->mnt_root, *mountpoint; + + while (mnt->mnt_parent) + { + mountpoint = mnt->mnt_point; + mnt = mnt->mnt_parent; + if (mnt->mnt_root != dentry) + { + out->mount = mnt; + out->dentry = mountpoint; + return true; + } + } + + return false; +} + static void walk_path(const struct path *path, const struct path *root, struct rbuf *rbuf) { /* TODO: While d_parent on .. Just Works, we don't need to keep track of the struct mnt. This * will need to be changed once that changes (mnt should keep track of mnt_parent). **/ struct dentry *dentry = path->dentry; + struct mount *mnt = path->mount; prepend_char(rbuf, '\0'); while (dentry != NULL && dentry != root->dentry) { - if (dentry->d_flags & DENTRY_FLAG_MOUNT_ROOT) - goto skip; + if (dentry == mnt->mnt_root) + { + struct path p; + if (!follow_mount_up(mnt, &p)) + break; + dentry = p.dentry; + mnt = p.mount; + } + prepend_dentry(rbuf, dentry); prepend_char(rbuf, '/'); - skip: dentry = dentry->d_parent; } diff --git a/kernel/kernel/fs/dentry.cpp b/kernel/kernel/fs/dentry.cpp index 1518cbbff..5166f56e0 100644 --- a/kernel/kernel/fs/dentry.cpp +++ b/kernel/kernel/fs/dentry.cpp @@ -101,7 +101,7 @@ static dentry *d_lookup_internal(dentry *dent, std::string_view name) return nullptr; } -static dentry *dentry_open_from_cache(dentry *dent, std::string_view name) +dentry *dentry_open_from_cache(dentry *dent, std::string_view name) { unsigned int old; struct dentry *found; @@ -561,7 +561,7 @@ static expected dentry_create_pending_lookup(const char *name, in return dentry_add_to_cache_careful(dent, parent); } -static dentry *__dentry_try_to_open(std::string_view name, dentry *dir, bool lock_ino) +dentry *__dentry_try_to_open(std::string_view name, dentry *dir, bool lock_ino) { DCHECK(dentry_is_dir(dir)); if (auto d = dentry_open_from_cache(dir, name); d) diff --git a/kernel/kernel/fs/dev.cpp b/kernel/kernel/fs/dev.cpp index 1afa45845..53f1d4173 100644 --- a/kernel/kernel/fs/dev.cpp +++ b/kernel/kernel/fs/dev.cpp @@ -611,8 +611,13 @@ static off_t devfs_getdirent(struct dirent *buf, off_t off, struct file *file) { /* .. */ auto parent = dentry_parent(dent); - put_dentry_to_dirent(buf, parent, ".."); - dput(parent); + if (parent) + { + put_dentry_to_dirent(buf, parent, ".."); + dput(parent); + } + else + put_dentry_to_dirent(buf, dent, ".."); } else { diff --git a/kernel/kernel/fs/mount.c b/kernel/kernel/fs/mount.c index b9ff46822..173e8f418 100644 --- a/kernel/kernel/fs/mount.c +++ b/kernel/kernel/fs/mount.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -46,6 +47,8 @@ static void mnt_init(struct mount *mnt, unsigned long flags) mnt->mnt_flags = flags; mnt->mnt_point = mnt->mnt_root = NULL; mnt->mnt_sb = NULL; + mnt->mnt_parent = NULL; + INIT_LIST_HEAD(&mnt->mnt_submounts); } static unsigned int mnt_hashbucket(struct mount *mnt) @@ -169,21 +172,16 @@ static int mnt_commit(struct mount *mnt, const char *target) * flags on a dentry, etc. */ if (strcmp(target, "/")) { - struct file *filp = open_vfs(AT_FDCWD, target); - if (!filp) - return -errno; - if (!dentry_is_dir(filp->f_dentry)) - { - fd_put(filp); - return -ENOTDIR; - } + struct path mountpoint; + int err; + + err = path_openat(AT_FDCWD, target, LOOKUP_MUST_BE_DIR, &mountpoint); + if (err < 0) + return err; - mnt->mnt_point = filp->f_dentry; - dget(mnt->mnt_point); - /* Another hack... */ - mnt->mnt_root->d_parent = mnt->mnt_point; - /* TODO: This isn't quite safe when we get proper mnt putting and umount */ - fd_put(filp); + /* Path reference gets dilluted into these two members */ + mnt->mnt_point = mountpoint.dentry; + mnt->mnt_parent = mountpoint.mount; } else { @@ -201,6 +199,9 @@ static int mnt_commit(struct mount *mnt, const char *target) list_add_tail(&mnt->mnt_mp_node, &mp_hashtable[mnt_mp_hashbucket(mnt)]); list_add_tail(&mnt->mnt_node, &mount_hashtable[mnt_hashbucket(mnt)]); + if (mnt->mnt_parent) + list_add_tail(&mnt->mnt_submount_node, &mnt->mnt_parent->mnt_submounts); + /* Ref up for the mount root */ dget(mnt->mnt_root); @@ -337,6 +338,12 @@ static bool attempt_disconnect(struct mount *mount) struct dentry *mp = mount->mnt_point; list_remove(&mount->mnt_mp_node); list_remove(&mount->mnt_node); + if (mount->mnt_parent) + { + list_remove(&mount->mnt_submount_node); + mnt_put(mount->mnt_parent); + } + ok = true; /* Check if we have nothing mounted at mp anymore. If so, unset DENTRY_FLAG_MOUNTPOINT. @@ -375,8 +382,6 @@ static int do_umount_path(struct path *path, int flags) WARN_ON(mount->mnt_root->d_ref != 1); - /* Undo our fake d_parent... */ - mount->mnt_root->d_parent = NULL; /* Finally, put our root */ dput(mount->mnt_root); @@ -391,18 +396,21 @@ static int do_umount_path(struct path *path, int flags) int sys_umount2(const char *utarget, int flags) { + int err = -EINVAL; + const char *target; + if (!is_root_user()) return -EPERM; - const char *target = strcpy_from_user(utarget); + + target = strcpy_from_user(utarget); if (!target) return -errno; if (flags & ~UMOUNT_NOFOLLOW) - return -EINVAL; + goto out; struct path path; - int err = - path_openat(AT_FDCWD, target, - LOOKUP_MUST_BE_DIR | (flags & UMOUNT_NOFOLLOW ? LOOKUP_NOFOLLOW : 0), &path); + err = path_openat(AT_FDCWD, target, + LOOKUP_MUST_BE_DIR | (flags & UMOUNT_NOFOLLOW ? LOOKUP_NOFOLLOW : 0), &path); if (err < 0) goto out; diff --git a/kernel/kernel/fs/namei.cpp b/kernel/kernel/fs/namei.cpp index e3de86b2c..b10f188a4 100644 --- a/kernel/kernel/fs/namei.cpp +++ b/kernel/kernel/fs/namei.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -200,6 +201,122 @@ static int dentry_follow_symlink(nameidata &data, dentry *symlink, unsigned int #define NAMEI_NO_FOLLOW_SYM (1U << 1) #define NAMEI_ALLOW_NEGATIVE (1U << 2) +static void follow_mount_up(struct mount *mnt, struct path *out) +{ + struct dentry *dentry = mnt->mnt_root, *mountpoint; + + while (mnt->mnt_parent) + { + mountpoint = mnt->mnt_point; + mnt = mnt->mnt_parent; + if (mnt->mnt_root != dentry) + { + out->mount = mnt; + out->dentry = mountpoint; + return; + } + } + + /* Should not be hittable, I think... */ + CHECK(0); +} + +static bool finish_mount_up(struct path *path, unsigned int seq) +{ + mnt_get(path->mount); + smp_mb(); + + if (path->mount->mnt_flags & MNT_DOOMED) + { + mnt_put(path->mount); + goto retry; + } + + /* I don't think there's a way this can fail, if we grabbed the mount itself */ + dget(path->dentry); + + if (read_seqretry(&mount_lock, seq)) + goto retry; + + return true; +retry: + return false; +} + +static int mount_dotdot(struct mount *mnt, struct path *path) +{ + rcu_read_lock(); + + for (;;) + { + unsigned int seq = read_seqbegin(&mount_lock); + follow_mount_up(mnt, path); + + /* Commit this follow_up by grabbing a reference to mount and mountpoint */ + if (finish_mount_up(path, seq)) + break; + } + + rcu_read_unlock(); + return 0; +} + +static int do_dotdot(nameidata &data, struct path *out) +{ + struct path *curr = &data.cur; + if (curr->dentry == curr->mount->mnt_root) + { + /* We're the mount's root? Gotta take things in a different way. */ + return mount_dotdot(curr->mount, out); + } + + struct dentry *dentry = dentry_parent(curr->dentry); + if (!dentry) + { + /* /.. = right where we are */ + return 0; + } + + struct path p = {dentry, curr->mount}; + mnt_get(curr->mount); + *out = p; + return 0; +} + +static int __namei_walk_component(std::string_view v, nameidata &data, struct path *out, + unsigned int flags) +{ + if (!v.compare(".")) + { + *out = data.cur; + path_get(out); + return 0; + } + + if (!v.compare("..")) + return do_dotdot(data, out); + + struct dentry *dent = dentry_open_from_cache(data.cur.dentry, v); + + if (dent) + { + if (dent->d_flags & DENTRY_FLAG_PENDING) + dent = dentry_wait_for_pending(dent); + } + + if (!dent) + { + dent = __dentry_try_to_open(v, data.cur.dentry, !(flags & DENTRY_LOOKUP_UNLOCKED)); + if (!dent) + return -errno; + } + + struct path p = {.dentry = dent, .mount = data.cur.mount}; + mnt_get(data.cur.mount); + *out = p; + return 0; +} + static int namei_walk_component(std::string_view v, nameidata &data, unsigned int flags = 0) { const bool is_last_name = @@ -207,9 +324,7 @@ static int namei_walk_component(std::string_view v, nameidata &data, unsigned in const bool dont_follow_last = data.lookup_flags & LOOKUP_NOFOLLOW; const bool unlocked_lookup = flags & NAMEI_UNLOCKED; - auto_dentry dwrapper; - dentry *new_found; - + struct path path; file f; f.f_ino = data.cur.dentry->d_inode; @@ -224,42 +339,33 @@ static int namei_walk_component(std::string_view v, nameidata &data, unsigned in /* Stop from escaping the chroot */ return 0; } - else - { - dwrapper = dentry_lookup_internal(v, data.cur.dentry, - unlocked_lookup ? DENTRY_LOOKUP_UNLOCKED : 0); - if (!dwrapper) - { - DCHECK(errno != 0); - return -errno; - } + + path_init(&path); + int err = __namei_walk_component(v, data, &path, unlocked_lookup ? DENTRY_LOOKUP_UNLOCKED : 0); + if (err < 0) + return err; #if 0 - printk("Lookup %s found %p%s\n", v.data(), new_found, - d_is_negative(new_found) ? " (negative)" : ""); + pr_warn("Lookup %s found %p%s\n", v.data(), path.dentry, + d_is_negative(path.dentry) ? " (negative)" : ""); #endif - } - new_found = dwrapper.get_dentry(); - struct mount *mnt = data.cur.mount; + struct mount *mnt = path.mount; - if (d_is_negative(new_found)) + if (d_is_negative(path.dentry)) { /* Check if the caller tolerates negative dentries as the lookup result. This only applies * for the last name. For !last_name, negative is always ENOENT */ if (!is_last_name || !(flags & NAMEI_ALLOW_NEGATIVE)) - return -ENOENT; + { + err = -ENOENT; + goto err_out; + } } - else if (dentry_is_symlink(new_found)) + else if (dentry_is_symlink(path.dentry)) { if (flags & NAMEI_NO_FOLLOW_SYM) - { - /* Save parent and location for the caller */ - struct path p = path{dwrapper.release(), mnt}; - mnt_get(mnt); - data.setcur(p); - return 0; - } + goto out; /* POSIX states that paths that end in a trailing slash are required to be the same as * /. For example: open("/usr/bin/") == open("/usr/bin/."). Therefore, we have to * special case that. @@ -270,7 +376,10 @@ static int namei_walk_component(std::string_view v, nameidata &data, unsigned in // printk("Following symlink for path elem %s\n", v.data()); if (is_last_name && (data.lookup_flags & LOOKUP_FAIL_IF_LINK)) - return -ELOOP; + { + err = -ELOOP; + goto err_out; + } else if (is_last_name && !should_follow_symlink) { // printk("Cannot follow symlink. Trailing slash: %s\n", must_be_dir ? "yes" : @@ -278,26 +387,31 @@ static int namei_walk_component(std::string_view v, nameidata &data, unsigned in } else [[likely]] { - return dentry_follow_symlink(data, new_found); + err = dentry_follow_symlink(data, path.dentry); + path_put(&path); + return err; } } - else if (dentry_is_mountpoint(new_found)) + else if (dentry_is_mountpoint(path.dentry)) { - struct mount *new_mount = mnt_traverse(new_found); + struct mount *new_mount = mnt_traverse(path.dentry); if (new_mount) { - dwrapper = new_mount->mnt_root; - dget(dwrapper.get_dentry()); + struct dentry *d = new_mount->mnt_root; + dget(d); mnt = new_mount; - new_found = dwrapper.get_dentry(); + data.setcur((struct path){d, mnt}); + path_put(&path); + return 0; } } - if (mnt == data.cur.mount) - mnt_get(mnt); - data.setcur(path{dwrapper.release(), mnt}); - +out: + data.setcur(path); return 0; +err_out: + path_put(&path); + return err; } /** diff --git a/kernel/kernel/fs/tmpfs.cpp b/kernel/kernel/fs/tmpfs.cpp index f3ef8dcd2..5070c9d21 100644 --- a/kernel/kernel/fs/tmpfs.cpp +++ b/kernel/kernel/fs/tmpfs.cpp @@ -192,10 +192,13 @@ off_t tmpfs_getdirent(struct dirent *buf, off_t off, struct file *file) { /* .. */ auto parent = dentry_parent(dent); - if (!parent) // We're root, so use ourselves - parent = dent; - put_dentry_to_dirent(buf, parent, ".."); - dput(parent); + if (parent) + { + put_dentry_to_dirent(buf, parent, ".."); + dput(parent); + } + else + put_dentry_to_dirent(buf, dent, ".."); } else { From f8cacb40ab3ff6214684500054367428c2ca301f Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Fri, 29 Nov 2024 21:33:56 +0000 Subject: [PATCH 64/76] block: Turn off TSA for write_begin and write_end Signed-off-by: Pedro Falcato --- kernel/kernel/fs/block.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/kernel/kernel/fs/block.cpp b/kernel/kernel/fs/block.cpp index 997e8abea..8c0e35bc5 100644 --- a/kernel/kernel/fs/block.cpp +++ b/kernel/kernel/fs/block.cpp @@ -241,7 +241,7 @@ static int block_readpage_write(struct vm_object *vm_obj, off_t offset, size_t l } static int block_write_begin(struct file *filp, struct vm_object *vm_obj, off_t offset, size_t len, - struct page **ppage) + struct page **ppage) NO_THREAD_SAFETY_ANALYSIS { struct page *page; int st = filemap_find_page(filp->f_ino, offset >> PAGE_SHIFT, @@ -295,7 +295,8 @@ static int __block_write_end(struct file *file, struct vm_object *vm_obj, off_t } static int block_write_end(struct file *file, struct vm_object *vm_obj, off_t offset, - unsigned int written, unsigned int to_write, struct page *page) + unsigned int written, unsigned int to_write, + struct page *page) NO_THREAD_SAFETY_ANALYSIS { __block_write_end(file, vm_obj, offset, written, to_write, page); unlock_page(page); From d1607c46ca932d594f3c163b3d163e55880de37e Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Fri, 29 Nov 2024 21:42:32 +0000 Subject: [PATCH 65/76] filemap: Turn off TSA for write_begin/end/write_iter Signed-off-by: Pedro Falcato --- kernel/kernel/fs/filemap.cpp | 108 +++++++++++++++++++++++------------ 1 file changed, 71 insertions(+), 37 deletions(-) diff --git a/kernel/kernel/fs/filemap.cpp b/kernel/kernel/fs/filemap.cpp index 633233a1a..d75fb87f4 100644 --- a/kernel/kernel/fs/filemap.cpp +++ b/kernel/kernel/fs/filemap.cpp @@ -50,7 +50,7 @@ int filemap_find_page(struct inode *ino, size_t pgoff, unsigned int flags, struc } /* Let's allocate a new page */ - p = alloc_page(GFP_KERNEL); + p = alloc_page(PAGE_ALLOC_NO_ZERO | GFP_KERNEL); if (!p) return -ENOMEM; p->owner = ino->i_pages; @@ -395,7 +395,7 @@ ssize_t file_write_cache(void *buffer, size_t len, struct inode *ino, size_t off } static int default_write_begin(struct file *filp, struct vm_object *vm_obj, off_t off, size_t len, - struct page **ppage) + struct page **ppage) NO_THREAD_SAFETY_ANALYSIS { struct page *page = nullptr; struct inode *ino = vm_obj->ino; @@ -426,7 +426,8 @@ static int default_write_begin(struct file *filp, struct vm_object *vm_obj, off_ } static int default_write_end(struct file *file, struct vm_object *vm_obj, off_t offset, - unsigned int written, unsigned int to_write, struct page *page) + unsigned int written, unsigned int to_write, + struct page *page) NO_THREAD_SAFETY_ANALYSIS { struct inode *ino = vm_obj->ino; unlock_page(page); @@ -446,7 +447,8 @@ static int default_write_end(struct file *file, struct vm_object *vm_obj, off_t * @param flags Flags * @return Written bytes, or negative error code */ -ssize_t filemap_write_iter(struct file *filp, size_t off, iovec_iter *iter, unsigned int flags) +ssize_t filemap_write_iter(struct file *filp, size_t off, iovec_iter *iter, + unsigned int flags) NO_THREAD_SAFETY_ANALYSIS { struct inode *ino = filp->f_ino; struct vm_object *vm_obj = ino->i_pages; @@ -697,7 +699,7 @@ static int filemap_mkwrite_private(struct vm_pf_context *ctx, static int vm_prepare_write(struct inode *inode, struct page *p) REQUIRES(p) { - DCHECK(page_locked(p)); + DCHECK_PAGE(page_locked(p), p); /* Correctness: We set the i_size before truncating pages from the page cache, so this should * not race... I think? */ @@ -723,30 +725,32 @@ static int filemap_mkwrite_shared(struct vm_pf_context *ctx, static int filemap_fault(struct vm_pf_context *ctx) NO_THREAD_SAFETY_ANALYSIS { - struct vm_area_struct *region = ctx->entry; + struct vm_area_struct *vma = ctx->entry; struct fault_info *info = ctx->info; struct page *page = nullptr; - struct inode *ino = region->vm_file->f_ino; + struct inode *ino = vma->vm_file->f_ino; int st = 0; - unsigned long pgoff = (ctx->vpage - region->vm_start) >> PAGE_SHIFT; - bool needs_invalidate = false; + unsigned long pgoff = (ctx->vpage - vma->vm_start) >> PAGE_SHIFT; + pte_t *ptep; + pte_t oldpte = ctx->oldpte; + struct spinlock *lock; /* We need to lock the page in case we're mapping it (that is, it's either a read-fault on * a private region, or any fault on a MAP_SHARED). */ - bool locked = (vma_private(region) && !ctx->info->write) || vma_shared(region); + bool locked = (vma_private(vma) && !ctx->info->write) || vma_shared(vma); /* Permission checks have already been handled before .fault() */ /* If a page was present, use that as the CoW source */ - if (vma_private(region) && ctx->mapping_info & PAGE_PRESENT) + if (vma_private(vma) && pte_present(oldpte)) { - page = phys_to_page(MAPPING_INFO_PADDR(ctx->mapping_info)); - DCHECK(info->write && !(ctx->mapping_info & PAGE_WRITABLE)); + page = phys_to_page(pte_addr(oldpte)); + DCHECK(info->write && !pte_write(oldpte)); } if (!page) { - unsigned long fileoff = (region->vm_offset >> PAGE_SHIFT) + pgoff; + unsigned long fileoff = (vma->vm_offset >> PAGE_SHIFT) + pgoff; if (ino->i_size <= (fileoff << PAGE_SHIFT)) { info->signal = VM_SIGBUS; @@ -754,24 +758,13 @@ static int filemap_fault(struct vm_pf_context *ctx) NO_THREAD_SAFETY_ANALYSIS } unsigned ffp_flags = FIND_PAGE_ACTIVATE | (locked ? FIND_PAGE_LOCK : 0); - st = filemap_find_page(region->vm_file->f_ino, fileoff, ffp_flags, &page, - ®ion->vm_file->f_ra_state); + st = filemap_find_page(vma->vm_file->f_ino, fileoff, ffp_flags, &page, + &vma->vm_file->f_ra_state); if (st < 0) goto err; } -#ifdef FILEMAP_PARANOID - if (ctx->mapping_info & PAGE_PRESENT) - { - unsigned long mapped = MAPPING_INFO_PADDR(ctx->mapping_info); - unsigned long fetched = (unsigned long) page_to_phys(page); - if (mapped != fetched) - panic("%s[%d]: filemap: Mapped page %lx != fetched %lx %s\n", - get_current_process()->name.data(), get_current_process()->pid_, mapped, fetched, - amap ? "from amap" : "from filemap"); - } -#endif if (!info->write) { /* Write-protect the page */ @@ -779,7 +772,7 @@ static int filemap_fault(struct vm_pf_context *ctx) NO_THREAD_SAFETY_ANALYSIS } else { - if (vma_private(region)) + if (vma_private(vma)) { DCHECK(!locked); st = filemap_mkwrite_private(ctx, page); @@ -791,22 +784,63 @@ static int filemap_fault(struct vm_pf_context *ctx) NO_THREAD_SAFETY_ANALYSIS /* We should invalidate the TLB if we had a mapping before. Note: I don't like that * we're mapping *over* the page, again. But it is what it is, and currently the code is * a little cleaner. */ - needs_invalidate = ctx->mapping_info & PAGE_PRESENT; page = ctx->page; DCHECK(page != nullptr); } - if (!vm_map_page(region->vm_mm, ctx->vpage, (u64) page_to_phys(page), ctx->page_rwx, - ctx->entry)) + if (pgtable_prealloc(vma->vm_mm, ctx->vpage) < 0) goto enomem; - if (needs_invalidate) - vm_invalidate_range(ctx->vpage, 1); + ptep = ptep_get_locked(vma->vm_mm, ctx->vpage, &lock); + if (ptep->pte != oldpte.pte) + { + /* Have to retry. Either this page is going away, or someone else nicely handled it for us. + */ + goto out_unlock_pte; + } - /* Only unref if this page is not new. When we allocate a new page - because of CoW, - * amap_add 'adopts' our reference. This works because amaps are inherently region-specific, - * and we have the address_space locked. - */ + if (ctx->page_rwx & VM_WRITE && !pte_none(oldpte) && + pte_addr(oldpte) == (unsigned long) page_to_phys(page)) + { + /* Okay, logic is simple in case we're just toggling the W bit. This can happen for various + * reasons, including mkwrite_private deciding we don't need to CoW, or a shared fault. In + * this case, we can avoid doing a TLB shootdown. Doing a local TLB invalidation is okay. It + * might result in spurious faults for other threads, but it's just way faster than + * purposefully doing IPIs. + */ + set_pte(ptep, pte_mkwrite(oldpte)); + tlbi_upgrade_pte_prots(vma->vm_mm, ctx->vpage); + } + else + { + /* New page. Just Map It. Sucks that we're copying this around... */ + struct page *oldp = NULL; + if (!pte_present(oldpte)) + increment_vm_stat(vma->vm_mm, resident_set_size, PAGE_SIZE); + + page_add_mapcount(page); + set_pte(ptep, pte_mkpte((u64) page_to_phys(page), + calc_pgprot((u64) page_to_phys(page), ctx->page_rwx))); + + if (unlikely(pte_present(oldpte) && !pte_special(oldpte))) + oldp = phys_to_page(pte_addr(oldpte)); + + /* We did our page table thing, now release the lock. We're going to need to IPI and it's + * best we do it with no spinlock held. + */ + spin_unlock(lock); + + if (pte_present(oldpte)) + vm_invalidate_range(ctx->vpage, 1); + /* After the IPI we can sub the mapcount - which may involve some freeing here... */ + if (oldp) + page_sub_mapcount(oldp); + goto out; + } + +out_unlock_pte: + spin_unlock(lock); +out: if (locked) unlock_page(page); page_unref(page); From e6983a91cef7c013f21a88b13709ad531ad86e3b Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Fri, 29 Nov 2024 22:44:56 +0000 Subject: [PATCH 66/76] ext2: buffer related fixes Data corruption fix in cases where a buffer could be both in the disk's page cache and a file's disk cache (this fix is not perfect). Zero out file holes now that filemap feeds us unzeroed pages. Signed-off-by: Pedro Falcato --- kernel/include/onyx/page.h | 6 ++++++ kernel/kernel/fs/buffer.cpp | 21 +++++++++++++++++++++ kernel/kernel/fs/ext2/ext2.cpp | 5 ++++- kernel/kernel/fs/ext2/inode.cpp | 2 +- 4 files changed, 32 insertions(+), 2 deletions(-) diff --git a/kernel/include/onyx/page.h b/kernel/include/onyx/page.h index f494a7c07..79674d008 100644 --- a/kernel/include/onyx/page.h +++ b/kernel/include/onyx/page.h @@ -627,6 +627,12 @@ PAGEFLAG_OPS(buffer, BUFFER); struct vm_object *page_vmobj(struct page *page); unsigned long page_pgoff(struct page *page); +static inline void page_zero_range(struct page *page, unsigned int off, unsigned int len) +{ + u8 *ptr = (u8 *) PAGE_TO_VIRT(page); + memset(ptr + off, 0, len); +} + __END_CDECLS #endif diff --git a/kernel/kernel/fs/buffer.cpp b/kernel/kernel/fs/buffer.cpp index 85e1733ca..c5dbb0505 100644 --- a/kernel/kernel/fs/buffer.cpp +++ b/kernel/kernel/fs/buffer.cpp @@ -800,6 +800,25 @@ void block_buf_tear_down_assoc(struct vm_object *object) } } +static void bforget(struct block_buf *buf) +{ + /* De-dirty the buffer (and page) if possible */ + struct page *page = buf->this_page; + spin_lock(&buf->pagestate_lock); + bb_clear_flag(buf, BLOCKBUF_FLAG_DIRTY); + bool isdirty = false; + for (struct block_buf *b = (struct block_buf *) page->priv; b; b = b->next) + { + if (bb_test_flag(b, BLOCKBUF_FLAG_DIRTY)) + isdirty = true; + } + + if (!isdirty) + page_clear_dirty(page); + spin_unlock(&buf->pagestate_lock); + block_buf_put(buf); +} + /** * @brief Forget a block_buf's inode * This will remove it from the assoc list @@ -823,6 +842,8 @@ void block_buf_forget_inode(struct block_buf *buf) buf->assoc_buffers_obj = nullptr; break; } + + bforget(buf); } void buffer_free_page(struct vm_object *vmo, struct page *page) diff --git a/kernel/kernel/fs/ext2/ext2.cpp b/kernel/kernel/fs/ext2/ext2.cpp index 92d721942..40ecb829b 100644 --- a/kernel/kernel/fs/ext2/ext2.cpp +++ b/kernel/kernel/fs/ext2/ext2.cpp @@ -206,7 +206,7 @@ int ext2_map_page(struct page *page, size_t off, struct inode *ino) if (block == EXT2_ERR_INV_BLOCK) { // Zero the block, since it's a hole - memset((char *) PAGE_TO_VIRT(page) + curr_off, 0, sb->block_size); + page_zero_range(page, b->page_off, sb->block_size); bb_test_and_set(b, BLOCKBUF_FLAG_UPTODATE); } else @@ -236,6 +236,9 @@ ssize_t ext2_readpage(struct page *page, size_t off, struct inode *ino) for (struct block_buf *b = (struct block_buf *) page->priv; b != nullptr; b = b->next) { sector_t block = b->block_nr; + if (bb_test_flag(b, BLOCKBUF_FLAG_UPTODATE)) + continue; + if (block != EXT2_ERR_INV_BLOCK) { /* TODO: Coalesce reads */ diff --git a/kernel/kernel/fs/ext2/inode.cpp b/kernel/kernel/fs/ext2/inode.cpp index 3d99bf560..1a9176349 100644 --- a/kernel/kernel/fs/ext2/inode.cpp +++ b/kernel/kernel/fs/ext2/inode.cpp @@ -351,7 +351,7 @@ int ext2_truncate_branch(ext2_block_no block, ext2_block_coords &curr_coords, st return EXT2_TRUNCATED_PARTIALLY; /* Truncated fully, we can free this block */ /* Note: we must "forget" the inode block buf */ - block_buf_forget_inode(buf); + block_buf_forget_inode(buf.release()); sb->free_block(block); ino->i_blocks -= sb->block_size >> 9; return EXT2_TRUNCATED_FULLY; From 4c643a496e07488dfe4c6f9e4687213db1ec0c86 Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Fri, 29 Nov 2024 22:47:15 +0000 Subject: [PATCH 67/76] anon: Handle page faults under the page table lock Signed-off-by: Pedro Falcato --- kernel/kernel/mm/anon.cpp | 61 +++++++++++++-------------------------- 1 file changed, 20 insertions(+), 41 deletions(-) diff --git a/kernel/kernel/mm/anon.cpp b/kernel/kernel/mm/anon.cpp index 874f8579d..b3d16a7db 100644 --- a/kernel/kernel/mm/anon.cpp +++ b/kernel/kernel/mm/anon.cpp @@ -20,11 +20,14 @@ const struct vm_operations anon_vmops = {.fault = vm_anon_fault}; int vm_anon_fault(struct vm_pf_context *ctx) { - struct vm_area_struct *region = ctx->entry; + struct vm_area_struct *vma = ctx->entry; struct fault_info *info = ctx->info; - struct page *page = nullptr, *oldp = nullptr; - bool needs_invd = false; + struct page *page = nullptr; + pte_t *ptep; + struct spinlock *lock; + /* pte_present is done in do_wp_page, not here. */ + CHECK(!pte_present(ctx->oldpte)); /* Permission checks have already been handled before .fault() */ if (!info->write) { @@ -32,42 +35,15 @@ int vm_anon_fault(struct vm_pf_context *ctx) page = vm_get_zero_page(); /* Write protect the page and don't bother flushing the TLB */ ctx->page_rwx &= ~VM_WRITE; - ctx->page_rwx |= VM_NOFLUSH; - goto map; } else { - bool copy_old = false; - if (pte_present(ctx->oldpte)) - { - oldp = phys_to_page(pte_addr(ctx->oldpte)); - DCHECK(info->write && !pte_write(ctx->oldpte)); - if (oldp != vm_get_zero_page()) - copy_old = true; - needs_invd = true; - - if (copy_old && 0 && page_flag_set(oldp, PAGE_FLAG_ANON) && page_mapcount(oldp) == 1) - { - /* If this is an anon page *and* mapcount = 1, avoid allocating a new page. Since - * mapcount = 1 (AND *ANON*), no one else can grab a ref. */ - /* TODO: We might be able to explore this - we may avoid the TLB shootdown and just - * change prots, but it would require significant code refactoring as-is. */ - /* TODO: checking mapcount = 1 probably isn't this easy once we get swapping, - * because refs may come and go. Will we need the page lock? */ - page = oldp; - page_ref(page); - goto map; - } - - /* oldp's mapcount will be decremented in vm_map_page */ - } - struct anon_vma *anon = anon_vma_prepare(ctx->entry); if (!anon) return -ENOMEM; - /* Allocate a brand-new (possibly zero-filled) page */ - page = alloc_page((copy_old ? PAGE_ALLOC_NO_ZERO : 0) | GFP_KERNEL); + /* Allocate a brand-new, zero-filled page */ + page = alloc_page(GFP_KERNEL); if (!page) goto enomem; page_set_anon(page); @@ -75,19 +51,22 @@ int vm_anon_fault(struct vm_pf_context *ctx) page->pageoff = ctx->vpage; page_add_lru(page); page_set_dirty(page); - - if (copy_old) - copy_page_to_page(page_to_phys(page), page_to_phys(oldp)); - goto map; } -map: - if (!vm_map_page(region->vm_mm, ctx->vpage, (u64) page_to_phys(page), ctx->page_rwx, - ctx->entry)) + if (pgtable_prealloc(vma->vm_mm, ctx->vpage) < 0) goto enomem; - if (needs_invd) - vm_invalidate_range(ctx->vpage, 1); + ptep = ptep_get_locked(vma->vm_mm, ctx->vpage, &lock); + if (ptep->pte != ctx->oldpte.pte) + goto out; + + increment_vm_stat(vma->vm_mm, resident_set_size, PAGE_SIZE); + page_add_mapcount(page); + set_pte(ptep, pte_mkpte((u64) page_to_phys(page), + calc_pgprot((u64) page_to_phys(page), ctx->page_rwx))); + +out: + spin_unlock(lock); /* The mapcount holds the only reference we need for anon pages... */ if (info->write) page_unref(page); From 435cae0bbb553fb19659d7ecee2adf7477595cb9 Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Fri, 29 Nov 2024 22:48:48 +0000 Subject: [PATCH 68/76] vm: Replace the vm_lock with a vm rwlock Should radically improve page fault scalability. Signed-off-by: Pedro Falcato --- kernel/include/onyx/mm_address_space.h | 4 +- kernel/kernel/mm/madvise.c | 4 +- kernel/kernel/mm/mincore.cpp | 7 ++- kernel/kernel/mm/vm.cpp | 65 +++++++++++--------------- kernel/kernel/process.cpp | 2 +- 5 files changed, 37 insertions(+), 45 deletions(-) diff --git a/kernel/include/onyx/mm_address_space.h b/kernel/include/onyx/mm_address_space.h index 24aed1079..27759c5ed 100644 --- a/kernel/include/onyx/mm_address_space.h +++ b/kernel/include/onyx/mm_address_space.h @@ -12,7 +12,7 @@ #include #include -#include +#include #include @@ -46,7 +46,7 @@ struct mm_address_space struct maple_tree region_tree; unsigned long start CPP_DFLINIT; unsigned long end CPP_DFLINIT; - struct mutex vm_lock CPP_DFLINIT; + struct rwlock vm_lock CPP_DFLINIT; /* mmap(2) base */ void *mmap_base CPP_DFLINIT; diff --git a/kernel/kernel/mm/madvise.c b/kernel/kernel/mm/madvise.c index 00a4425f5..b61f837d3 100644 --- a/kernel/kernel/mm/madvise.c +++ b/kernel/kernel/mm/madvise.c @@ -91,8 +91,8 @@ int sys_madvise(void *addr, size_t len, int advice) if (start + len <= start) return -EINVAL; - mutex_lock(&mm->vm_lock); + rw_lock_read(&mm->vm_lock); ret = do_madvise_walk(mm, start, len, advice); - mutex_unlock(&mm->vm_lock); + rw_unlock_read(&mm->vm_lock); return ret; } diff --git a/kernel/kernel/mm/mincore.cpp b/kernel/kernel/mm/mincore.cpp index 8e2b81392..110aab9de 100644 --- a/kernel/kernel/mm/mincore.cpp +++ b/kernel/kernel/mm/mincore.cpp @@ -9,6 +9,9 @@ #include #include +#undef REQUIRES_SHARED +#define REQUIRES_SHARED(...) + // TODO: Export this stuff in some header, and avoid sticking everything into vm.cpp vm_area_struct *vm_search(struct mm_address_space *mm, void *addr, size_t length) REQUIRES_SHARED(mm->vm_lock); @@ -17,13 +20,13 @@ static long do_pagemap(struct mm_address_space *as, unsigned long start, unsigne u64 *pagemap) { - scoped_mutex g{as->vm_lock}; + scoped_rwlock g{as->vm_lock}; long pfns_processed = 0; vm_area_struct *vma; unsigned long index = start; void *entry_; - mt_for_each(&as->region_tree, entry_, index, end) + mt_for_each (&as->region_tree, entry_, index, end) { vma = (vm_area_struct *) entry_; if (vma->vm_start > end) diff --git a/kernel/kernel/mm/vm.cpp b/kernel/kernel/mm/vm.cpp index ca038d772..a7938a51a 100644 --- a/kernel/kernel/mm/vm.cpp +++ b/kernel/kernel/mm/vm.cpp @@ -71,6 +71,14 @@ static bool limits_are_contained(struct vm_area_struct *reg, unsigned long start unsigned long limit); static bool vm_mapping_is_cow(struct vm_area_struct *entry); +/* TODO */ +#undef REQUIRES_SHARED +#undef REQUIRES +#undef EXCLUDES +#define REQUIRES_SHARED(...) +#define REQUIRES(...) +#define EXCLUDES(...) + vm_area_struct *vm_search(struct mm_address_space *mm, void *addr, size_t length) REQUIRES_SHARED(mm->vm_lock); static void vma_pre_adjust(struct vm_area_struct *vma); @@ -295,8 +303,6 @@ void do_vm_unmap(struct mm_address_space *as, void *range, size_t pages) struct vm_area_struct *entry = vm_find_region(as, range); assert(entry != nullptr); - MUST_HOLD_MUTEX(&entry->vm_mm->vm_lock); - vm_mmu_unmap(entry->vm_mm, range, pages, entry); } @@ -342,8 +348,6 @@ bool vm_mapping_requires_write_protect(struct vm_area_struct *reg) static void vma_destroy(struct vm_area_struct *region) { - MUST_HOLD_MUTEX(®ion->vm_mm->vm_lock); - /* First, unref things */ if (region->vm_file) { @@ -416,7 +420,7 @@ int vm_clone_as(mm_address_space *addr_space, mm_address_space *original) original = get_current_address_space(); if (!original) original = &kernel_address_space; - scoped_mutex g{original->vm_lock}; + scoped_rwlock g{original->vm_lock}; return paging_clone_as(addr_space, original); } @@ -512,7 +516,7 @@ int vm_fork_address_space(struct mm_address_space *addr_space) EXCLUDES(addr_spa EXCLUDES(get_current_address_space()->vm_lock) { struct mm_address_space *current_mm = get_current_address_space(); - scoped_mutex g{current_mm->vm_lock}; + scoped_rwlock g{current_mm->vm_lock}; #ifdef CONFIG_DEBUG_ADDRESS_SPACE_ACCT mmu_verify_address_space_accounting(get_current_address_space()); @@ -553,7 +557,7 @@ int vm_fork_address_space(struct mm_address_space *addr_space) EXCLUDES(addr_spa assert(addr_space->active_mask.is_empty()); - mutex_init(&addr_space->vm_lock); + rwlock_init(&addr_space->vm_lock); validate_mm_tree(addr_space); return 0; } @@ -569,17 +573,10 @@ void vm_change_perms(void *range, size_t pages, int perms) NO_THREAD_SAFETY_ANAL { struct mm_address_space *as; bool kernel = is_higher_half(range); - bool needs_release = false; - if (kernel) - as = &kernel_address_space; - else - as = get_current_process()->get_aspace(); + DCHECK(!kernel); - if (mutex_owner(&as->vm_lock) != get_current_thread()) - { - needs_release = true; - mutex_lock(&as->vm_lock); - } + as = &kernel_address_space; + rw_lock_write(&as->vm_lock); for (size_t i = 0; i < pages; i++) { @@ -589,9 +586,7 @@ void vm_change_perms(void *range, size_t pages, int perms) NO_THREAD_SAFETY_ANAL } vm_invalidate_range((unsigned long) range, pages); - - if (needs_release) - mutex_unlock(&as->vm_lock); + rw_unlock_write(&as->vm_lock); } static bool vma_can_merge_into(struct vm_area_struct *vma, size_t size, int vm_flags, @@ -829,7 +824,7 @@ void *vm_mmap(void *addr, size_t length, int prot, int flags, struct file *file, if (off & (PAGE_SIZE - 1)) return ERR_PTR(-EINVAL); - scoped_mutex g{mm->vm_lock}; + scoped_rwlock g{mm->vm_lock}; /* Calculate the pages needed for the overall size */ size_t pages = vm_size_to_pages(length); @@ -1136,7 +1131,7 @@ int vm_mprotect(struct mm_address_space *as, void *__addr, size_t size, int prot unsigned long limit = addr + size; VMA_ITERATOR(vmi, as, addr, limit); - scoped_mutex g{as->vm_lock}; + scoped_rwlock g{as->vm_lock}; /* Note: vm_munmap has some vma detaching logic for the simple fact that POSIX does not * allow for a partial unmap in case of an error. Whereas this is not the case for mprotect. @@ -1239,7 +1234,7 @@ uint64_t sys_brk(void *newbrk) { mm_address_space *as = get_current_address_space(); - scoped_mutex g{as->vm_lock}; + scoped_rwlock g{as->vm_lock}; if (newbrk == nullptr) { @@ -1458,14 +1453,14 @@ int vm_handle_page_fault(struct fault_info *info) if (irq_is_disabled()) panic("Page fault while IRQs were disabled\n"); - /* Surrender immediately if there's no user address space or the fault was inside vm code */ - if (!as || mutex_holds_lock(&as->vm_lock)) + /* Surrender immediately if there's no user address space */ + if (!as) { info->signal = VM_SIGSEGV; return -1; } - scoped_mutex g{as->vm_lock}; + scoped_rwlock g{as->vm_lock}; struct vm_area_struct *entry = vm_find_region(as, (void *) info->fault_address); if (!entry) @@ -1554,7 +1549,7 @@ void vm_destroy_addr_space(struct mm_address_space *mm) bool free_pgd = true; /* First, iterate through the maple tree and free/unmap stuff */ - scoped_mutex g{mm->vm_lock}; + scoped_rwlock g{mm->vm_lock}; vm_area_struct *entry; void *entry_; @@ -1972,7 +1967,7 @@ int vm_create_address_space(struct mm_address_space *mm) assert(mm->active_mask.is_empty() == true); - mutex_init(&mm->vm_lock); + rwlock_init(&mm->vm_lock); return 0; } @@ -2008,8 +2003,6 @@ void *vm_get_fallback_pgd() void vm_remove_region(struct mm_address_space *as, struct vm_area_struct *region) REQUIRES(as->vm_lock) { - MUST_HOLD_MUTEX(&as->vm_lock); - void *ret = mtree_erase(&as->region_tree, region->vm_start); CHECK(ret == region); } @@ -2020,8 +2013,6 @@ int __vm_munmap(struct mm_address_space *as, void *__addr, size_t size) REQUIRES unsigned long limit = ALIGN_TO(((unsigned long) __addr) + size, PAGE_SIZE); struct list_head list = LIST_HEAD_INIT(list); - MUST_HOLD_MUTEX(&as->vm_lock); - struct vm_area_struct *vma = vm_search(as, (void *) addr, PAGE_SIZE); if (!vma) return -EINVAL; @@ -2093,7 +2084,7 @@ int __vm_munmap(struct mm_address_space *as, void *__addr, size_t size) REQUIRES */ int vm_munmap(struct mm_address_space *as, void *__addr, size_t size) { - scoped_mutex g{as->vm_lock}; + scoped_rwlock g{as->vm_lock}; auto addr = (unsigned long) __addr; if (addr < as->start || addr > as->end) @@ -2165,8 +2156,6 @@ static int __vm_expand_mapping(struct vm_area_struct *region, size_t new_size) static int vm_expand_mapping(struct mm_address_space *as, struct vm_area_struct *region, size_t new_size) REQUIRES(as->vm_lock) { - MUST_HOLD_MUTEX(&as->vm_lock); - if (!vm_can_expand(as, region, new_size)) return -1; @@ -2241,7 +2230,7 @@ void *sys_mremap(void *old_address, size_t old_size, size_t new_size, int flags, bool fixed = flags & MREMAP_FIXED; bool wants_create_new_mapping_of_pages = old_size == 0 && may_move; void *ret = MAP_FAILED; - scoped_mutex g{current->address_space->vm_lock}; + scoped_rwlock g{current->address_space->vm_lock}; auto as = current->get_aspace(); /* TODO: Unsure on what to do if new_size > old_size */ @@ -2419,7 +2408,7 @@ int get_phys_pages(void *_addr, unsigned int flags, struct page **pages, size_t return ret; } - scoped_mutex g{as->vm_lock}; + scoped_rwlock g{as->vm_lock}; size_t pages_gotten = 0; @@ -2512,7 +2501,7 @@ int sys_msync(void *ptr, size_t length, int flags) return -EINVAL; /* Hogging the vm_lock is bad mkay, todo... */ - scoped_mutex g{as->vm_lock}; + scoped_rwlock g{as->vm_lock}; struct vm_area_struct *vma = vm_search(as, (void *) addr, length); diff --git a/kernel/kernel/process.cpp b/kernel/kernel/process.cpp index 39f2c83eb..dfba72cf5 100644 --- a/kernel/kernel/process.cpp +++ b/kernel/kernel/process.cpp @@ -1269,7 +1269,7 @@ ssize_t process::query(void *ubuf, ssize_t len, unsigned long what, size_t *howm ssize_t process::query_vm_regions(void *ubuf, ssize_t len, unsigned long what, size_t *howmany, void *arg) { - scoped_mutex g{address_space->vm_lock}; + scoped_rwlock g{address_space->vm_lock}; size_t needed_len = 0; char pathbuf[PATH_MAX]; From 2f2f6103315bd39fbd76cba10684cd74ba8376f8 Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Fri, 29 Nov 2024 22:49:41 +0000 Subject: [PATCH 69/76] printk: Add ipv6 address printing with %pI6 Signed-off-by: Pedro Falcato --- kernel/lib/libk/stdio/sprintf.c | 96 +++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/kernel/lib/libk/stdio/sprintf.c b/kernel/lib/libk/stdio/sprintf.c index 2b5fffd9d..727689e33 100644 --- a/kernel/lib/libk/stdio/sprintf.c +++ b/kernel/lib/libk/stdio/sprintf.c @@ -16,6 +16,8 @@ #include #include +#include + #ifdef DO_STREAMS struct stream { @@ -400,6 +402,46 @@ static int dump_buffer(struct stream *stream, void *ptr, struct printf_specifier return written; } +static void find_v6_zeroes_range(struct in6_addr *addr, int *start, int *end) +{ + int best_start = -1, best_end = -1; + int curr_start = -1, curr_end = -1; + bool in_zeroes = false; + for (int i = 0; i < 8; i++) + { + if (addr->s6_addr16[i] == 0) + { + if (!in_zeroes) + curr_start = i; + curr_end = i; + in_zeroes = true; + } + else + { + /* The use of the symbol "::" MUST be used to its maximum capability (longest sequence + * wins). It also MUST NOT be used to compress a single field. [...] the first sequence + * of zero bits MUST be shortened (when the length is equal) -- rfc5952, non-verbatim */ + if (in_zeroes && curr_end - curr_start > 1 && + curr_end - curr_start > best_end - best_start) + { + best_start = curr_start; + best_end = curr_end; + } + + in_zeroes = false; + } + } + + if (in_zeroes && curr_end - curr_start > 1 && curr_end - curr_start > best_end - best_start) + { + best_start = curr_start; + best_end = curr_end; + } + + *start = best_start; + *end = best_end; +} + static int print_ipaddr(struct stream *stream, void *ptr, struct printf_specifier *spec, const char **pfmt) { @@ -440,6 +482,60 @@ static int print_ipaddr(struct stream *stream, void *ptr, struct printf_specifie *pfmt = ++fmt; return printf_do_string(stream, buf, spec->fwidth, spec->precision, spec->flags); } + else if (*fmt == '6') + { + /* IPv6 address printing as per rfc5952 */ + int compr_start, compr_end; + struct in6_addr *addr = ptr; + *pfmt = ++fmt; + char buf[sizeof("0000:0000:0000:0000:0000:0000:0000:0000")]; + int j = 0; + find_v6_zeroes_range(addr, &compr_start, &compr_end); + for (int i = 0; i < 8; i++) + { + int k = 4; + u16 val = ntohs(addr->s6_addr16[i]); + char tmp[5]; + + if (i >= compr_start && i <= compr_end) + { + /* Add the extra ':' to indicate a compressed ipv6 addr range */ + if (i == compr_end) + { + buf[j++] = ':'; + /* Add another ':' if nothing comes after us */ + if (compr_end == 7) + buf[j++] = ':'; + } + + continue; + } + + if (i > 0) + buf[j++] = ':'; + + /* Leading zeros MUST be suppressed */ + if (val > 0) + { + while (val) + { + /* rfc5952 demands lowercase hex digits */ + tmp[k--] = digits[val % 16]; + val /= 16; + } + } + else + { + tmp[k--] = '0'; + } + + while (k < 4) + buf[j++] = tmp[++k]; + } + + buf[j] = '\0'; + return printf_do_string(stream, buf, spec->fwidth, spec->precision, spec->flags); + } return 0; } From b398676fa0ff1af4dcad49c31cbc77ce09644627 Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Fri, 29 Nov 2024 22:50:12 +0000 Subject: [PATCH 70/76] sched: Add job stealing Add job stealing from other cores. Radically improves performance in e.g a make -j4 gcc build. Signed-off-by: Pedro Falcato --- kernel/kernel/sched/scheduler.cpp | 49 +++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/kernel/kernel/sched/scheduler.cpp b/kernel/kernel/sched/scheduler.cpp index 23c3739a8..a9d08f97a 100644 --- a/kernel/kernel/sched/scheduler.cpp +++ b/kernel/kernel/sched/scheduler.cpp @@ -212,6 +212,48 @@ void sched_unlock(thread *thread, unsigned long cpu_flags) PER_CPU_VAR(long runnable_delta) = 0; +extern void sched_idle(void *); + +static thread_t *sched_steal_job(unsigned int cpu) +{ + for (unsigned int i = 0; i < get_nr_cpus(); i++) + { + if (i == cpu) + continue; + if (other_cpu_get(tasks_in_queues, i) <= 1) + continue; + struct spinlock *sched_lock = get_per_cpu_ptr_any(scheduler_lock, i); + thread **thread_queues = (thread **) get_per_cpu_ptr_any(thread_queues_head, i); + if (spin_try_lock(sched_lock)) + continue; + + for (int j = NUM_PRIO - 1; j >= 0; j--) + { + /* If this queue has a thread, we found a runnable thread! */ + if (thread_queues[j]) + { + thread_t *ret = thread_queues[j]; + if (ret->entry == sched_idle) + continue; + /* Advance the queue by one */ + thread_queues[j] = ret->next_prio; + if (thread_queues[j]) + ret->prev_prio = nullptr; + ret->next_prio = nullptr; + other_cpu_add(tasks_in_queues, -1, i); + add_per_cpu(tasks_in_queues, 1); + ret->cpu = cpu; + spin_unlock(sched_lock); + return ret; + } + } + + spin_unlock(sched_lock); + } + + return nullptr; +} + thread_t *__sched_find_next(unsigned int cpu) { thread_t *current_thread = get_current_thread(); @@ -252,6 +294,13 @@ thread_t *__sched_find_next(unsigned int cpu) { thread_t *ret = thread_queues[i]; + if (ret->entry == sched_idle) + { + thread_t *stolen = sched_steal_job(cpu); + if (stolen) + return stolen; + } + /* Advance the queue by one */ thread_queues[i] = ret->next_prio; if (thread_queues[i]) From f08b2ceecb8a11cc9a76b989aa1cc2413f583fec Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Fri, 29 Nov 2024 22:51:15 +0000 Subject: [PATCH 71/76] netif: Rework the RX signaling and handling Signed-off-by: Pedro Falcato --- kernel/include/onyx/net/netif.h | 1 + kernel/kernel/net/netif.cpp | 37 +++++++++++++-------------------- 2 files changed, 15 insertions(+), 23 deletions(-) diff --git a/kernel/include/onyx/net/netif.h b/kernel/include/onyx/net/netif.h index 511432048..cb2facd3d 100644 --- a/kernel/include/onyx/net/netif.h +++ b/kernel/include/onyx/net/netif.h @@ -28,6 +28,7 @@ struct netif; #define NETIF_LOOPBACK (1 << 5) #define NETIF_HAS_RX_AVAILABLE (1 << 6) #define NETIF_DOING_RX_POLL (1 << 7) +#define NETIF_SCHEDULED (1 << 8) struct packetbuf; diff --git a/kernel/kernel/net/netif.cpp b/kernel/kernel/net/netif.cpp index ff1bd6a0f..68faa8260 100644 --- a/kernel/kernel/net/netif.cpp +++ b/kernel/kernel/net/netif.cpp @@ -110,7 +110,8 @@ void netif_register_loopback_route6(struct netif *netif) { struct inet6_route route; - route.mask = IN6ADDR_LOOPBACK_INIT; + route.mask.s6_addr32[0] = route.mask.s6_addr32[1] = route.mask.s6_addr32[2] = + route.mask.s6_addr32[3] = ~0; route.dest = IN6ADDR_LOOPBACK_INIT; route.gateway = IN6ADDR_ANY_INIT; @@ -276,19 +277,9 @@ INIT_LEVEL_CORE_PERCPU_CTOR(init_rx_queues); void netif_signal_rx(netif *nif) { - unsigned int flags, og_flags; - - do - { - flags = nif->flags; - og_flags = flags; - - flags |= NETIF_HAS_RX_AVAILABLE; - - } while (!__atomic_compare_exchange_n(&nif->flags, &og_flags, flags, false, __ATOMIC_ACQUIRE, - __ATOMIC_RELAXED)); - - if (og_flags & NETIF_HAS_RX_AVAILABLE) + unsigned int flags = + __atomic_fetch_or(&nif->flags, NETIF_HAS_RX_AVAILABLE | NETIF_SCHEDULED, __ATOMIC_SEQ_CST); + if (flags & NETIF_SCHEDULED) return; auto queue = get_per_cpu_ptr(rx_queue); @@ -319,19 +310,19 @@ void netif_do_rxpoll(netif *nif) int netif_do_rx() { auto queue = get_per_cpu_ptr(rx_queue); - DEFINE_LIST(to_rx); + unsigned long flags = spin_lock_irqsave(&queue->lock); + while (!list_is_empty(&queue->to_rx_list)) { - scoped_lock g{queue->lock}; - list_splice(&queue->to_rx_list, &to_rx); + struct netif *nif = list_first_entry(&queue->to_rx_list, struct netif, rx_queue_node); + list_remove(&nif->rx_queue_node); + spin_unlock_irqrestore(&queue->lock, flags); + netif_do_rxpoll(nif); + flags = spin_lock_irqsave(&queue->lock); + atomic_and_relaxed(nif->flags, ~NETIF_SCHEDULED); } - list_for_every_safe (&to_rx) - { - netif *n = container_of(l, netif, rx_queue_node); - list_remove(&n->rx_queue_node); - netif_do_rxpoll(n); - } + spin_unlock_irqrestore(&queue->lock, flags); return 0; } From 7ebad86e8fa38d8c50a0c567a8b9b62fa43f2901 Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Fri, 29 Nov 2024 23:32:06 +0000 Subject: [PATCH 72/76] tty: Add a proper implementation of TIOCSPGRP Signed-off-by: Pedro Falcato --- kernel/include/onyx/pid.h | 5 +++++ kernel/kernel/tty/tty.cpp | 22 +++++++++++++++++++--- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/kernel/include/onyx/pid.h b/kernel/include/onyx/pid.h index 2d15c0870..83eb7b392 100644 --- a/kernel/include/onyx/pid.h +++ b/kernel/include/onyx/pid.h @@ -111,6 +111,11 @@ struct pid : public refcountable bool is_orphaned_and_has_stopped_jobs(process *ignore) const; int kill_pgrp(int sig, int flags, siginfo_t *info) const; + + bool is(enum pid_type type) + { + return !list_is_empty(&member_list[type]); + } }; static inline pid::auto_pid pid_create(process *leader) diff --git a/kernel/kernel/tty/tty.cpp b/kernel/kernel/tty/tty.cpp index 6ee531e72..044d9e92b 100644 --- a/kernel/kernel/tty/tty.cpp +++ b/kernel/kernel/tty/tty.cpp @@ -568,6 +568,21 @@ unsigned int do_tty_cnotty(tty *tty) return 0; } +static unsigned int do_tiocspgrp(struct tty *tty, pid_t pid) +{ + struct process *current = get_current_process(); + if (current->ctty != tty || tty->session != current->session) + return -ENOTTY; + + pid::auto_pid p = pid::lookup(pid); + if (!p || !p->is(PIDTYPE_PGRP)) + return -ESRCH; + if (!p->is_in_session(tty->session)) + return -EPERM; + tty->foreground_pgrp = pid; + return 0; +} + unsigned int tty_ioctl(int request, void *argp, struct file *dev) { struct tty *slave_tty; @@ -671,9 +686,10 @@ unsigned int tty_ioctl(int request, void *argp, struct file *dev) case TIOCSPGRP: { scoped_mutex g{tty->lock}; - auto pgrp = get_current_process()->process_group; - tty->foreground_pgrp = pgrp->get_pid(); - return 0; + pid_t pid; + if (copy_from_user(&pid, argp, sizeof(int))) + return -EFAULT; + return do_tiocspgrp(tty, pid); } case TIOCGPGRP: { From 506f65c037de47c569a920f92eda931d39de0124 Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Fri, 29 Nov 2024 23:34:24 +0000 Subject: [PATCH 73/76] exec: Enlargen the stack and max args by quite a lot Signed-off-by: Pedro Falcato --- kernel/kernel/binfmt/exec.cpp | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/kernel/kernel/binfmt/exec.cpp b/kernel/kernel/binfmt/exec.cpp index 6c6b4bff8..816678882 100644 --- a/kernel/kernel/binfmt/exec.cpp +++ b/kernel/kernel/binfmt/exec.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -35,6 +36,9 @@ expected process_copy_envarg(const char **envarg, size_t curren const char **b = envarg; const char *ptr = nullptr; long st; + /* TODO: Take into account rlim_stack */ + unsigned long limit = (8 * DEFAULT_USER_STACK_LEN) / 4; + limit = cul::max(limit, (unsigned long) ARG_MAX); while ((st = get_user64((unsigned long *) b, (unsigned long *) &ptr)) == 0 && ptr != nullptr) { @@ -57,10 +61,10 @@ expected process_copy_envarg(const char **envarg, size_t curren size_t buffer_size = (nr_args + 1) * sizeof(void *) + string_size; // Check if we overflow the ARG_MAX - if (current_size + buffer_size > ARG_MAX) + if (current_size + buffer_size > limit) return unexpected{-E2BIG}; - char *new_ = (char *) zalloc(buffer_size); + char *new_ = (char *) kcalloc(buffer_size, 1, GFP_KERNEL); if (!new_) return unexpected{-ENOMEM}; @@ -328,7 +332,7 @@ int flush_old_exec(struct exec_state *state) vm_set_aspace(state->new_address_space.get()); curr->address_space = cul::move(state->new_address_space); - mutex_init(&curr->address_space->vm_lock); + rwlock_init(&curr->address_space->vm_lock); /* Close O_CLOEXEC files */ file_do_cloexec(&curr->ctx); @@ -501,7 +505,7 @@ int sys_execve(const char *p, const char **argv, const char **envp) current->flags &= ~PROCESS_FORKED; struct stack_info si; - si.length = DEFAULT_USER_STACK_LEN; + si.length = DEFAULT_USER_STACK_LEN * 8; if (process_alloc_stack(&si) < 0) goto error_die_signal; From e0734984c667fdce189a71da2e698c24c05400aa Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Fri, 29 Nov 2024 23:35:25 +0000 Subject: [PATCH 74/76] e1000: Loose fixes Signed-off-by: Pedro Falcato --- kernel/drivers/net/e1000/e1000.cpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/kernel/drivers/net/e1000/e1000.cpp b/kernel/drivers/net/e1000/e1000.cpp index 67a11d4a4..09efc913e 100644 --- a/kernel/drivers/net/e1000/e1000.cpp +++ b/kernel/drivers/net/e1000/e1000.cpp @@ -155,21 +155,20 @@ int e1000_pollrx(netif *nif) { e1000_device *dev = (e1000_device *) nif->priv; + bool found_one = false; uint16_t old_cur = 0; while ((dev->rx_descs[dev->rx_cur].status & RSTA_DD)) { auto &rxd = dev->rx_descs[dev->rx_cur]; - e1000_process_packet(nif, rxd); - dev->rx_descs[dev->rx_cur].status = 0; old_cur = dev->rx_cur; - dev->rx_cur = (dev->rx_cur + 1) % number_rx_desc; - - e1000_write(REG_RXDESCTAIL, old_cur, dev); + found_one = true; } + if (found_one) + e1000_write(REG_RXDESCTAIL, old_cur, dev); return 0; } @@ -608,7 +607,7 @@ unsigned int e1000_device::prepare_extended_descs(packetbuf *buf) if (!xmited) { buffer_start_off = (buf->data - (unsigned char *) buf->buffer_start); - length -= buffer_start_off; + length = buf->tail - buf->data; desc.popts = (buf->needs_csum ? POPTS_TXSM : 0); } From 03304cebd56d07e1eaf76fe9d408f6c8a940443b Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Fri, 29 Nov 2024 23:42:41 +0000 Subject: [PATCH 75/76] x86: Fix CONFIG_VERBOSE_SEGV Signed-off-by: Pedro Falcato --- kernel/arch/x86_64/isr.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kernel/arch/x86_64/isr.cpp b/kernel/arch/x86_64/isr.cpp index ba546ebd1..24b8f31f5 100644 --- a/kernel/arch/x86_64/isr.cpp +++ b/kernel/arch/x86_64/isr.cpp @@ -223,6 +223,8 @@ void stack_segment_fault(struct registers *ctx) } #ifdef CONFIG_VERBOSE_SEGV +#undef REQUIRES_SHARED +#define REQUIRES_SHARED(...) vm_area_struct *vm_search(struct mm_address_space *mm, void *addr, size_t length) REQUIRES_SHARED(mm->vm_lock); @@ -236,7 +238,7 @@ static void attempt_map_pointer(unsigned long word) size_t pos = 0; struct mm_address_space *mm = get_current_address_space(); - scoped_mutex g{mm->vm_lock}; + scoped_rwlock g{mm->vm_lock}; // Lets try to "symbolize" it struct vm_area_struct *vm = vm_search(mm, (void *) word, 1); if (vm) From ae506c2b55cce04d64aa555013ae183245ee2a71 Mon Sep 17 00:00:00 2001 From: Pedro Falcato Date: Fri, 29 Nov 2024 23:43:43 +0000 Subject: [PATCH 76/76] netctl: Add more error checking and more verbosity Signed-off-by: Pedro Falcato --- usystem/network/netctld/include/netctl.hpp | 14 +++- usystem/network/netctld/src/dhcpcd.cpp | 82 ++++++++++++++++------ 2 files changed, 73 insertions(+), 23 deletions(-) diff --git a/usystem/network/netctld/include/netctl.hpp b/usystem/network/netctld/include/netctl.hpp index 5b7fe77b4..d0ef5ed39 100644 --- a/usystem/network/netctld/include/netctl.hpp +++ b/usystem/network/netctld/include/netctl.hpp @@ -1,7 +1,9 @@ /* - * Copyright (c) 2020 Pedro Falcato + * Copyright (c) 2020 - 2024 Pedro Falcato * This file is part of Onyx, and is released under the terms of the MIT License * check LICENSE at the root directory for more information + * + * SPDX-License-Identifier: MIT */ #pragma once @@ -12,6 +14,7 @@ #include #include +#include #include #include @@ -111,7 +114,14 @@ class instance dhcpcd::create_instance(new_name); - netctl::v6::configure_if(*this); + try + { + netctl::v6::configure_if(*this); + } + catch (std::exception& e) + { + std::cout << "netctl ipv6 address configuration failed: " << e.what() << "\n"; + } } ~instance() diff --git a/usystem/network/netctld/src/dhcpcd.cpp b/usystem/network/netctld/src/dhcpcd.cpp index 9089b8a49..5af2f7989 100644 --- a/usystem/network/netctld/src/dhcpcd.cpp +++ b/usystem/network/netctld/src/dhcpcd.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017 - 2022 Pedro Falcato + * Copyright (c) 2017 - 2024 Pedro Falcato * This file is part of Onyx, and is released under the terms of the MIT License * check LICENSE at the root directory for more information * @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -87,8 +88,8 @@ off_t dhcp_add_option(dhcp_packet_t *pkt, off_t off, unsigned char len, const vo off_t dhcp_close_options(dhcp_packet_t *pkt, off_t off) { /* Add the needed padding */ - memset(&pkt->options[off], 0, 3); - off += 3; + // memset(&pkt->options[off], 0, 3); + // off += 3; pkt->options[off] = DHO_END; return off + 1; @@ -101,11 +102,14 @@ bool packet::decode() unsigned char *opt = (unsigned char *) &packet_->options; if (length <= DHCP_FIXED_NON_UDP) + { + fprintf(stderr, "dhcpcd: Bad packet length %zu, ignoring!\n", length); return false; + } if (memcmp(opt, DHCP_OPTIONS_COOKIE, 4) == 1) { - printf("dhcpcd: Bad cookie\n"); + fprintf(stderr, "dhcpcd: Bad cookie, ignoring!\n"); return false; } @@ -114,9 +118,12 @@ bool packet::decode() opt += 4; while (*opt != DHO_END) { - /* Check of OOB */ + /* Check for OOB */ if (opt >= limit) + { + fprintf(stderr, "dhcpcd: Went out of bounds processing options, ignoring!\n"); return false; + } unsigned char type = *opt; opt++; @@ -133,22 +140,36 @@ bool packet::decode() } if (!has_message_type) + { + fprintf(stderr, "dhcpcd: Does not have message type, ignoring!\n"); return false; + } return true; } void instance::send_discover() { + const char *vendor_class_identifier = "Onyx dhcpcd (netctld)"; + char hostname[512]; + uint16_t max_msg_size = htons(576); auto boot_packet = buf; memset(boot_packet, 0, sizeof(dhcp_packet_t)); + if (gethostname(hostname, sizeof(hostname) - 1) < 0) + err(1, "gethostname"); + + for (size_t i = 0; i < strlen(hostname); i++) + hostname[i] = tolower(hostname[i]); + + hostname[sizeof(hostname) - 1] = 0; + memcpy(&boot_packet->chaddr, &mac, 6); boot_packet->xid = xid; boot_packet->hlen = 6; boot_packet->htype = HTYPE_ETHER; boot_packet->op = BOOTREQUEST; - boot_packet->flags = 0; + boot_packet->flags = htons(BOOTP_BROADCAST); off_t off = DHCP_MIN_OPT_OFFSET; memcpy(&boot_packet->options, DHCP_OPTIONS_COOKIE, 4); @@ -156,9 +177,17 @@ void instance::send_discover() unsigned char message_type = DHCPDISCOVER; off = dhcp_add_option(boot_packet, off, 1, &message_type, sizeof(message_type), DHO_DHCP_MESSAGE_TYPE); - unsigned char opts[3] = {DHO_SUBNET_MASK, DHO_ROUTERS, DHO_DOMAIN_NAME_SERVERS}; + unsigned char opts[] = {DHO_SUBNET_MASK, DHO_ROUTERS, DHO_DOMAIN_NAME_SERVERS, + DHO_HOST_NAME, DHO_DOMAIN_NAME, DHO_BROADCAST_ADDRESS, + DHO_NTP_SERVERS}; + off = dhcp_add_option(boot_packet, off, sizeof(opts), &opts, sizeof(opts), + DHO_DHCP_PARAMETER_REQUEST_LIST); + off = dhcp_add_option(boot_packet, off, 2, &max_msg_size, 2, DHO_DHCP_MAX_MESSAGE_SIZE); + off = dhcp_add_option(boot_packet, off, strlen(hostname), hostname, strlen(hostname), + DHO_HOST_NAME); off = - dhcp_add_option(boot_packet, off, 3, &opts, sizeof(opts), DHO_DHCP_PARAMETER_REQUEST_LIST); + dhcp_add_option(boot_packet, off, strlen(vendor_class_identifier), vendor_class_identifier, + strlen(vendor_class_identifier), DHO_VENDOR_CLASS_IDENTIFIER); off = dhcp_close_options(boot_packet, off); if (send(sockfd, boot_packet, DHCP_FIXED_NON_UDP + off, 0) < 0) @@ -237,6 +266,7 @@ std::unique_ptr instance::get_packets(std::function pred auto pdata = message_type->option.data(); + printf("dhcpcd: Got message type %x\n", *pdata); if (*pdata == DHCPOFFER) { if (got_dhcp_offer) @@ -254,6 +284,28 @@ std::unique_ptr instance::get_packets(std::function pred return p; } +static bool check_for_dhcpoffer(dhcpcd::packet *data) +{ + auto message_type = data->get_option(DHO_DHCP_MESSAGE_TYPE, 1); + assert(message_type != nullptr); + + auto pdata = message_type->option.data(); + + if (*pdata != DHCPOFFER) + { + fprintf(stderr, "dhcpcd: Expecting DHCPOFFER, got %x, ignoring\n", *pdata); + return false; + } + + if (!data->get_option(DHO_ROUTERS, 4)) + { + fprintf(stderr, "dhcpcd: DHCPOFFER does not supply DHO_ROUTERS, ignoring\n"); + return false; + } + + return true; +} + int instance::setup_netif() { /* DHCP essentially works like this: @@ -269,19 +321,7 @@ int instance::setup_netif() std::unique_ptr packet; /* If for some reason we can't retrieve a packet, the get_packets will throw an exception */ - while (!(packet = get_packets([](dhcpcd::packet *data) -> bool { - if (!data->get_option(DHO_ROUTERS, 4)) - return false; - - auto message_type = data->get_option(DHO_DHCP_MESSAGE_TYPE, 1); - - assert(message_type != nullptr); - - /* Sanitise the option parameters */ - - auto pdata = message_type->option.data(); - return *pdata == DHCPOFFER; - }))) + while (!(packet = get_packets(check_for_dhcpoffer))) { }