From fc63cfeb28e1d7b59eaafea1d5bef1a50572ab48 Mon Sep 17 00:00:00 2001 From: Emil Kovacev Date: Sat, 30 Sep 2023 00:22:42 -0400 Subject: [PATCH 1/9] Added 0x15 eax=0xe820 call for detecting memory --- src/boot/boot_sect.asm | 6 +++--- src/boot/detect_mem.asm | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) create mode 100644 src/boot/detect_mem.asm diff --git a/src/boot/boot_sect.asm b/src/boot/boot_sect.asm index 21d03c5..73ff87d 100644 --- a/src/boot/boot_sect.asm +++ b/src/boot/boot_sect.asm @@ -6,10 +6,11 @@ begin: mov [BOOT_DRIVE], dl mov bp, 0x9000 mov sp, bp - jmp load_kernel + jmp detect_mem %include "src/boot/gdt.asm" %include "src/boot/enter_pm.asm" +%include "src/boot/detect_mem.asm" [bits 16] load_kernel: @@ -26,7 +27,6 @@ load_kernel: begin_pm: call OS_OFFSET hlt - times 509 - ($ - $$) db 0 ;padding @@ -34,4 +34,4 @@ times 509 - ($ - $$) db 0 ;padding BOOT_DRIVE db 0 ;0x7dfd ;above is data that can always be found at 0x7dfd - n during boot process -dw 0xaa55 ;magic boot sector number \ No newline at end of file +dw 0xaa55 ;magic boot sector number diff --git a/src/boot/detect_mem.asm b/src/boot/detect_mem.asm new file mode 100644 index 0000000..4c90357 --- /dev/null +++ b/src/boot/detect_mem.asm @@ -0,0 +1,19 @@ +;code below is largely taken from osdev.org +;see https://wiki.osdev.org/Detecting_Memory_(x86) for more details + +[bits 16] +detect_mem: + mov di, 0x8004 + xor ebx, ebx + xor bp, bp + mov edx, 0x534D4150 + mov eax, 0xe820 + mov [es:di + 20], dword 1 ; force a valid ACPI 3.X entry + mov ecx, 24 ; ask for 24 bytes + int 0x15 + jmp load_kernel + +[bits 16] +.failed: + stc ; "function unsupported" error exit + ret From eb22ffb1286d0a7b6dc0d2c5155c7ef8fc2657f5 Mon Sep 17 00:00:00 2001 From: Emil Kovacev Date: Sun, 8 Oct 2023 14:57:03 -0400 Subject: [PATCH 2/9] initialize asm BIOS interrupt --- src/boot/boot_sect.asm | 1 + src/boot/detect_mem.asm | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/boot/boot_sect.asm b/src/boot/boot_sect.asm index 73ff87d..9fe4683 100644 --- a/src/boot/boot_sect.asm +++ b/src/boot/boot_sect.asm @@ -14,6 +14,7 @@ begin: [bits 16] load_kernel: + mov dl, [BOOT_DRIVE] ;reset dl mov ah, 2 ;read BIOS chs mov al, 42 ;sectors to read mov cl, 0x02 ;start at sector 2 diff --git a/src/boot/detect_mem.asm b/src/boot/detect_mem.asm index 4c90357..53bb96d 100644 --- a/src/boot/detect_mem.asm +++ b/src/boot/detect_mem.asm @@ -1,18 +1,28 @@ -;code below is largely taken from osdev.org +;code below is largely taken from osdev.org, all of which is on the Public Domain ;see https://wiki.osdev.org/Detecting_Memory_(x86) for more details +;for license information, see https://wiki.osdev.org/OSDev_Wiki:License [bits 16] detect_mem: mov di, 0x8004 xor ebx, ebx xor bp, bp - mov edx, 0x534D4150 + mov edx, 0x0534D4150 mov eax, 0xe820 mov [es:di + 20], dword 1 ; force a valid ACPI 3.X entry mov ecx, 24 ; ask for 24 bytes int 0x15 + jc short .failed + mov edx, 0x0534D4150 + cmp eax, edx + jne short .failed + test ebx, ebx ; ebx = 0 implies list is only 1 entry long (worthless) + je short .failed jmp load_kernel +.next_item: + + [bits 16] .failed: stc ; "function unsupported" error exit From b6eb0ffcce5df38ac06acd21326414e801b0418c Mon Sep 17 00:00:00 2001 From: Emil Kovacev Date: Sun, 8 Oct 2023 14:57:38 -0400 Subject: [PATCH 3/9] init palloc files --- src/lib/stdlib/palloc.c | 10 ++++++++++ src/lib/stdlib/palloc.h | 13 +++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 src/lib/stdlib/palloc.c create mode 100644 src/lib/stdlib/palloc.h diff --git a/src/lib/stdlib/palloc.c b/src/lib/stdlib/palloc.c new file mode 100644 index 0000000..17ec0c7 --- /dev/null +++ b/src/lib/stdlib/palloc.c @@ -0,0 +1,10 @@ +#include "palloc.h" +#include + +void init_pages() { + int len = *(uint32_t *)MEM_STRUCT_ADDR; + Chunk *chunk = (Chunk *)(MEM_STRUCT_ADDR + sizeof(int)); + for (int i = 0; i < len; i++, chunk += sizeof(Chunk)) { + // handle memory chunks + } +} diff --git a/src/lib/stdlib/palloc.h b/src/lib/stdlib/palloc.h new file mode 100644 index 0000000..43140fa --- /dev/null +++ b/src/lib/stdlib/palloc.h @@ -0,0 +1,13 @@ +#include + +// RAM addr of structure describing useable memory +#define MEM_STRUCT_ADDR 0x8000 +#define MEM_FREE 0 +#define MEM_RESERVED 1 + +typedef struct { + uint64_t base; + uint64_t len; + uint32_t type; + uint32_t attrs; // ACPI v3.0 Extended Attributes bitfield +} Chunk; From 5ee27cc4b63d6d0e025a18d3cff498e67c97ed22 Mon Sep 17 00:00:00 2001 From: Emil Kovacev Date: Sun, 8 Oct 2023 14:57:53 -0400 Subject: [PATCH 4/9] init detect mem docs --- docs/boot/detecting_memory.md | 47 +++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 docs/boot/detecting_memory.md diff --git a/docs/boot/detecting_memory.md b/docs/boot/detecting_memory.md new file mode 100644 index 0000000..1740c2b --- /dev/null +++ b/docs/boot/detecting_memory.md @@ -0,0 +1,47 @@ +# Detecting Memory + +## Preliminaries + +Memory in x86 is standardized up to 1MB. Past that, other BIOS processes use +chunks of memory in unpredictable locations. To maximize the amount +of useable memory in our system, we need to query the BIOS. + +## Detecting Memory + +"Low Memory" is memory below 1MB. BIOS has a couple of functions to detect +these addresses, as a contiguous block, but is limited in the amount of +memory we are able to get, so we choose to omit it entirely. + +"Upper Memory" is memory above 1MB, typically up to just below 4GB. While +memory detection can move past that in 64-bit systems, this amount is +sufficient for our use cases. The BIOS function used to detect upper memory +across most devices is `int 0x15 EAX=0xe820`. Each call to this function +stores a structure in memory that looks as follows: + +``` +struct memChunk { + uint64_t base; // base addr of mem. chunk + uint64_t len; // len of mem. chunk + uint32_t type; // what mem. chunk is used for +} +``` + +From there, our program builds an array of these structures in memory, +for use later in our operating system. A dynamic allocator can read this +array after we transition to 32-bit mode to build a dynamic allocator. + +#### Important Addresses + +- `0x1000`: location of the kernel. +- `0x7c00`: location of boot sector. +- `0x9000`: location of stack at boot. +- `0x90000`: location of stack at kernel entry. + +![Memory Layout Diagram](boot_memory_diagram.png) + +#### Further Reading + +[BIOS](https://wiki.osdev.org/BIOS) +[GDT](https://wiki.osdev.org/Global_Descriptor_Table) +[Bootloader Guide](https://wiki.osdev.org/Rolling_Your_Own_Bootloader) +[BIOS int 0x13](https://wiki.osdev.org/Disk_access_using_the_BIOS_(INT_13h)) From 867c4d16ad84e8e1fb9aa6e26498be0b10e57b0f Mon Sep 17 00:00:00 2001 From: Emil Kovacev Date: Thu, 19 Oct 2023 20:09:19 -0400 Subject: [PATCH 5/9] initial palloc commit --- Makefile | 2 +- src/boot/detect_mem.asm | 59 +++++++++++++++++++++++++++++++---------- src/lib/stdlib/palloc.c | 45 ++++++++++++++++++++++++++++--- src/lib/stdlib/palloc.h | 16 +++++++++-- src/os/main.c | 2 ++ 5 files changed, 103 insertions(+), 21 deletions(-) diff --git a/Makefile b/Makefile index 8fd6871..5a01e04 100644 --- a/Makefile +++ b/Makefile @@ -48,7 +48,7 @@ OBJ_NAMES := src/os/main.o src/os/test.o os_entry.o src/lib/video/VGA_text.o \ src/lib/device/serial.o src/lib/device/ps2.o src/lib/device/keyboard.o \ src/lib/container/ring_buffer.o \ src/lib/stdlib/stdio.o src/lib/stdlib/stdlib.o src/lib/stdlib/string.o \ - src/lib/pit/pit.o + src/lib/pit/pit.o src/lib/stdlib/palloc.o .PHONY: clean qemu test diff --git a/src/boot/detect_mem.asm b/src/boot/detect_mem.asm index 53bb96d..e9361b1 100644 --- a/src/boot/detect_mem.asm +++ b/src/boot/detect_mem.asm @@ -2,28 +2,59 @@ ;see https://wiki.osdev.org/Detecting_Memory_(x86) for more details ;for license information, see https://wiki.osdev.org/OSDev_Wiki:License +num_ent equ 0x8000 ;stores number of entries +struct_base equ 0x8004 ;base ptr of structure + [bits 16] detect_mem: - mov di, 0x8004 - xor ebx, ebx - xor bp, bp + mov di, struct_base + xor ebx, ebx + xor bp, bp + jmp detect_chunk + +[bits 16] +detect_chunk: mov edx, 0x0534D4150 mov eax, 0xe820 mov [es:di + 20], dword 1 ; force a valid ACPI 3.X entry mov ecx, 24 ; ask for 24 bytes int 0x15 - jc short .failed - mov edx, 0x0534D4150 - cmp eax, edx - jne short .failed - test ebx, ebx ; ebx = 0 implies list is only 1 entry long (worthless) - je short .failed - jmp load_kernel -.next_item: - + cmp eax, edx ; on success, eax must have been reset to "SMAP" + jne failed + + cmp bp, 0 + je short check_first + jmp detect_chunk_2 + +detect_chunk_2: + inc bp + add di, 24 + + cmp bp, 10 + jne detect_chunk + + jmp post_process + +[bits 16] +check_first: + jc short failed ; carry set on first call means "unsupported function" + mov edx, 0x0534D4150 ; Some BIOSes apparently trash this register? + test ebx, ebx ; ebx = 0 implies list is only 1 entry long (worthless) + je short failed + jmp short detect_chunk_2 [bits 16] -.failed: - stc ; "function unsupported" error exit +failed: + stc ret + +[bits 16] +skip_ent: + test ebx, ebx ;second 0 indicates end of list + jne short post_process + +[bits 16] +post_process: + mov [num_ent], bp + jmp load_kernel diff --git a/src/lib/stdlib/palloc.c b/src/lib/stdlib/palloc.c index 17ec0c7..bfff277 100644 --- a/src/lib/stdlib/palloc.c +++ b/src/lib/stdlib/palloc.c @@ -1,10 +1,47 @@ #include "palloc.h" #include +#include +#include "stdlib/stdio.h" +#include "stdlib/string.h" +#include "video/VGA_text.h" -void init_pages() { +void handle_chunk(Chunk *c); +void sort_chunks(int len, Chunk *array); + +void init_palloc() { int len = *(uint32_t *)MEM_STRUCT_ADDR; - Chunk *chunk = (Chunk *)(MEM_STRUCT_ADDR + sizeof(int)); - for (int i = 0; i < len; i++, chunk += sizeof(Chunk)) { - // handle memory chunks + Chunk *chunk = (Chunk *)(MEM_STRUCT_ADDR + sizeof(uint32_t)); + + sort_chunks(len, chunk); + + for (int i = 0; i < len; i++, chunk++) { + handle_chunk(chunk); + } +} + +void handle_chunk(Chunk *c) { +} + +// 5, 2 , 4 , 5 , 9 , 3 +// [2, 5], 4 , 5 , 9 , 3 +// 2,[4 , 5], 5 , 9 , 3 +// 2, 4 ,[5 , 5], 9 , 3 +// 2, 4 , 5 ,[5 , 9], 3 +// 2, 4 , 5 , 5 ,[3 , 9] + +// Insertion Sort; Theta(1) space complexity, Theta(1) time +// complexity if chunks are upper bounded by some constant +// (usually about 10) +void sort_chunks(int len, Chunk *array) { + for (int i = 0; i < len; i++) { + for (int j = i+1; j < len; j++) { + if (array[i].base_lower < array[j].base_lower) { + Chunk a = array[i]; + Chunk b = array[j]; + + memcpy(&array[j], &a, sizeof(Chunk)); + memcpy(&array[i], &b, sizeof(Chunk)); + } + } } } diff --git a/src/lib/stdlib/palloc.h b/src/lib/stdlib/palloc.h index 43140fa..ae63279 100644 --- a/src/lib/stdlib/palloc.h +++ b/src/lib/stdlib/palloc.h @@ -5,9 +5,21 @@ #define MEM_FREE 0 #define MEM_RESERVED 1 +#define PAGE_SIZE 4096 + +// for the purposes of our os, which is 32-bit, we +// will rely soley on the lower 32-bits typedef struct { - uint64_t base; - uint64_t len; + uint32_t base_lower; + uint32_t base_upper; + uint32_t len_lower; + uint32_t len_upper; uint32_t type; uint32_t attrs; // ACPI v3.0 Extended Attributes bitfield } Chunk; + + +void init_palloc(); + +// Returns a PAGE_SIZE page of contiguous, useable memory +void *palloc(); diff --git a/src/os/main.c b/src/os/main.c index 8172a40..6825702 100644 --- a/src/os/main.c +++ b/src/os/main.c @@ -4,10 +4,12 @@ #include "pit/pit.h" #include "test.h" #include "video/VGA_text.h" +#include "stdlib/palloc.h" int os_main() { makeInterruptTable(); init_pit(); + init_palloc(); serialInit(); ps2Init(); From b1b56ba1de4cbe359b98d86879afc713dd9306b1 Mon Sep 17 00:00:00 2001 From: Emil Kovacev Date: Sat, 21 Oct 2023 10:51:37 -0400 Subject: [PATCH 6/9] palloc additions --- src/lib/stdlib/palloc.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/lib/stdlib/palloc.c b/src/lib/stdlib/palloc.c index bfff277..7063d07 100644 --- a/src/lib/stdlib/palloc.c +++ b/src/lib/stdlib/palloc.c @@ -1,6 +1,5 @@ #include "palloc.h" #include -#include #include "stdlib/stdio.h" #include "stdlib/string.h" #include "video/VGA_text.h" From e74f95cdd88105cb984dd975f49cbb233fb459e7 Mon Sep 17 00:00:00 2001 From: Emil Kovacev Date: Mon, 12 Feb 2024 15:24:46 -0500 Subject: [PATCH 7/9] formatting --- src/lib/stdlib/palloc.h | 5 ++--- src/os/main.c | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/lib/stdlib/palloc.h b/src/lib/stdlib/palloc.h index ae63279..6805856 100644 --- a/src/lib/stdlib/palloc.h +++ b/src/lib/stdlib/palloc.h @@ -1,13 +1,13 @@ #include -// RAM addr of structure describing useable memory +// RAM addr of structure describing useable memory #define MEM_STRUCT_ADDR 0x8000 #define MEM_FREE 0 #define MEM_RESERVED 1 #define PAGE_SIZE 4096 -// for the purposes of our os, which is 32-bit, we +// for the purposes of our os, which is 32-bit, we // will rely soley on the lower 32-bits typedef struct { uint32_t base_lower; @@ -18,7 +18,6 @@ typedef struct { uint32_t attrs; // ACPI v3.0 Extended Attributes bitfield } Chunk; - void init_palloc(); // Returns a PAGE_SIZE page of contiguous, useable memory diff --git a/src/os/main.c b/src/os/main.c index 6825702..cead3bd 100644 --- a/src/os/main.c +++ b/src/os/main.c @@ -2,9 +2,9 @@ #include "device/serial.h" #include "hard/idt.h" #include "pit/pit.h" +#include "stdlib/palloc.h" #include "test.h" #include "video/VGA_text.h" -#include "stdlib/palloc.h" int os_main() { makeInterruptTable(); From 699370538e967a5dbd80ac66ce5fc2d4a3c74fda Mon Sep 17 00:00:00 2001 From: Emil Kovacev Date: Thu, 2 Nov 2023 22:14:40 -0400 Subject: [PATCH 8/9] uses builtin sort for memory detection --- src/lib/stdlib/palloc.c | 87 ++++++++++++++++++++++++++--------------- src/lib/stdlib/palloc.h | 4 -- 2 files changed, 55 insertions(+), 36 deletions(-) diff --git a/src/lib/stdlib/palloc.c b/src/lib/stdlib/palloc.c index 7063d07..b0c4810 100644 --- a/src/lib/stdlib/palloc.c +++ b/src/lib/stdlib/palloc.c @@ -1,46 +1,69 @@ #include "palloc.h" #include -#include "stdlib/stdio.h" -#include "stdlib/string.h" -#include "video/VGA_text.h" +#include "stdio.h" +#include "string.h" +#include "stdlib.h" + +#define MAX_CHUNKS 100 void handle_chunk(Chunk *c); void sort_chunks(int len, Chunk *array); +void merge_chunks(Chunk *chunks, size_t len); +int compar(const void *ina, const void *inb); void init_palloc() { - int len = *(uint32_t *)MEM_STRUCT_ADDR; - Chunk *chunk = (Chunk *)(MEM_STRUCT_ADDR + sizeof(uint32_t)); + // last entry is repeat of first + size_t len = *(uint32_t *)MEM_STRUCT_ADDR - 1; - sort_chunks(len, chunk); + if (len > MAX_CHUNKS) + len = MAX_CHUNKS; - for (int i = 0; i < len; i++, chunk++) { - handle_chunk(chunk); - } -} + Chunk *chunks = (Chunk *)(MEM_STRUCT_ADDR + sizeof(uint32_t)); -void handle_chunk(Chunk *c) { + isort(chunks, len, sizeof(Chunk), compar); + // merge_chunks(chunks, len); + // remove_duplicates(chunks, len) } -// 5, 2 , 4 , 5 , 9 , 3 -// [2, 5], 4 , 5 , 9 , 3 -// 2,[4 , 5], 5 , 9 , 3 -// 2, 4 ,[5 , 5], 9 , 3 -// 2, 4 , 5 ,[5 , 9], 3 -// 2, 4 , 5 , 5 ,[3 , 9] - -// Insertion Sort; Theta(1) space complexity, Theta(1) time -// complexity if chunks are upper bounded by some constant -// (usually about 10) -void sort_chunks(int len, Chunk *array) { - for (int i = 0; i < len; i++) { - for (int j = i+1; j < len; j++) { - if (array[i].base_lower < array[j].base_lower) { - Chunk a = array[i]; - Chunk b = array[j]; - - memcpy(&array[j], &a, sizeof(Chunk)); - memcpy(&array[i], &b, sizeof(Chunk)); - } - } +// sorts first by type, then by base address +// +// Types: +// - type 1: usable memory +// - type > 1: unusable (some types are reclaimable, but is left unimplemented for now) +int compar(const void *ina, const void *inb) { + Chunk a = *(Chunk *)ina; + Chunk b = *(Chunk *)inb; + + if (a.type < b.type) + return -1; + else if (a.type > b.type) + return 1; + else if (a.base_lower < b.base_lower) + return -1; + else if (a.base_lower == b.base_lower) { + ((Chunk *)inb) -> type = 2; // a bit janky for sure: changes duplicate to unusable type + return -1; } + return 1; } + +// void merge_chunks(Chunk *chunks, size_t len) { +// if (len < 2) +// return; +// +// for (int i = 0; i < len - 1; i++) { +// Chunk curr = chunks[i]; +// Chunk next = chunks[i+1]; +// +// if (curr.base_lower + curr.len_lower < next.base_lower) +// continue; +// +// // check if two chunks intersect (after sort) +// else if (curr.base_lower >= next.base_lower) // This should never happen????? +// return; +// +// else if (curr.base_lower + curr.len_lower >= next.base_lower) { +// +// } +// } +// } diff --git a/src/lib/stdlib/palloc.h b/src/lib/stdlib/palloc.h index 6805856..a2887cf 100644 --- a/src/lib/stdlib/palloc.h +++ b/src/lib/stdlib/palloc.h @@ -2,10 +2,6 @@ // RAM addr of structure describing useable memory #define MEM_STRUCT_ADDR 0x8000 -#define MEM_FREE 0 -#define MEM_RESERVED 1 - -#define PAGE_SIZE 4096 // for the purposes of our os, which is 32-bit, we // will rely soley on the lower 32-bits From efb0742c93453e847543ca5042aadbe34777b018 Mon Sep 17 00:00:00 2001 From: Emil Kovacev Date: Mon, 12 Feb 2024 15:31:21 -0500 Subject: [PATCH 9/9] formatting pt2 --- src/lib/stdlib/palloc.c | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/src/lib/stdlib/palloc.c b/src/lib/stdlib/palloc.c index b0c4810..fe15065 100644 --- a/src/lib/stdlib/palloc.c +++ b/src/lib/stdlib/palloc.c @@ -1,8 +1,10 @@ #include "palloc.h" -#include + #include "stdio.h" -#include "string.h" #include "stdlib.h" +#include "string.h" + +#include #define MAX_CHUNKS 100 @@ -29,7 +31,8 @@ void init_palloc() { // // Types: // - type 1: usable memory -// - type > 1: unusable (some types are reclaimable, but is left unimplemented for now) +// - type > 1: unusable (some types are reclaimable, but is left unimplemented +// for now) int compar(const void *ina, const void *inb) { Chunk a = *(Chunk *)ina; Chunk b = *(Chunk *)inb; @@ -41,7 +44,8 @@ int compar(const void *ina, const void *inb) { else if (a.base_lower < b.base_lower) return -1; else if (a.base_lower == b.base_lower) { - ((Chunk *)inb) -> type = 2; // a bit janky for sure: changes duplicate to unusable type + ((Chunk *)inb)->type = + 2; // a bit janky for sure: changes duplicate to unusable type return -1; } return 1; @@ -50,20 +54,21 @@ int compar(const void *ina, const void *inb) { // void merge_chunks(Chunk *chunks, size_t len) { // if (len < 2) // return; -// +// // for (int i = 0; i < len - 1; i++) { // Chunk curr = chunks[i]; // Chunk next = chunks[i+1]; -// -// if (curr.base_lower + curr.len_lower < next.base_lower) +// +// if (curr.base_lower + curr.len_lower < next.base_lower) // continue; -// +// // // check if two chunks intersect (after sort) -// else if (curr.base_lower >= next.base_lower) // This should never happen????? +// else if (curr.base_lower >= next.base_lower) // This should never +// happen????? // return; -// +// // else if (curr.base_lower + curr.len_lower >= next.base_lower) { -// +// // } // } // }