vm.cc

#include "core/vm.h"

#include <cxx/string.h>
#include <cxx/units.h>
#include <cxx/utility.h>
#include <libkernel/assert.h>
#include <libkernel/print.h>
#include <libkernel/status.h>

#include "core/constants.h"
#include "core/loader.h"
#include "core/slab.h"
#include "core/thread.h"
#include "platform/arch.h"
#include "platform/mmu.h"

using namespace core::mem;
using platform::mmu::PageDescriptor;
using platform::mmu::TableDescriptor;

namespace core::mem {
   Page* pfndb_arr_start;
}
static Page* pfndb_arr_end;
static uint64_t n_freePages;
static uint64_t n_totalPages;
namespace core::mem {
   PageList pfndb_freelist{};
   PageList InitrdVmo{};
   PageList PerfVmo{};
} // namespace core::mem

extern "C" int perf_monitor_start_;
extern "C" int perf_monitor_end_;

//===----------------------------------------------------------------------===//
// Physical memory allocator code + helpers
//===----------------------------------------------------------------------===//

static inline uint64_t page_to_paddr(Page* p) {
   // Calculate i such that pfndb_arr[i] = page, this will give us the paddr
   auto i = p - pfndb_arr_start;
   return i * 4_KiB;
}

void core::mem::init(loader::LoaderData& args) {
   // Since the freelist is with loader addresses, we need to re-create
   // the freelist so the addresses are now within the kernel aspace.
   pfndb_arr_start = reinterpret_cast<Page*>(args.pfndb_range.start + KernelOffset);
   pfndb_arr_end = reinterpret_cast<Page*>(args.pfndb_range.end + KernelOffset);
   n_freePages = 0;
   n_totalPages = 0;
   InitrdVmo = PageList{};
   for (auto* page = pfndb_arr_start; page != pfndb_arr_end; page++) {
      const auto paddr = page_to_paddr(page);
      if (paddr >= args.initrd_range.start && paddr < args.initrd_range.end) {
         InitrdVmo.push_back(page);
         page->flags = 1;
      } else if (cxx::bits<0, 0>(page->flags) == 0) {
         n_freePages++;
         pfndb_freelist.push_back(page);
      }
      n_totalPages++;
   }
   // Print some early statistics about the allocator
   libkernel::log("PMM initialized with %lld/%lld free pages (%lld allocated)\r\n", n_freePages,
                  n_totalPages, n_totalPages - n_freePages);

   // Now populate the performance monitor VMO
   uint64_t perf_start_vaddrs = reinterpret_cast<uint64_t>(&perf_monitor_start_);
   uint64_t perf_end_vaddrs = reinterpret_cast<uint64_t>(&perf_monitor_end_);
   libkernel::log("Perf start: 0x%llX, Perf end: 0x%llX\r\n", perf_start_vaddrs, perf_end_vaddrs);
   for (auto addrs = perf_start_vaddrs; addrs < perf_end_vaddrs; addrs += 4_KiB) {
      auto pfn = traverse(addrs).phys_addr;
      auto page = phys_to_pfn(pfn * 4_KiB);
      PerfVmo.push_back(page);
      page->flags = 1;
   }
}

Result<uint64_t> core::mem::alloc_page(PageList& out) {
   if (pfndb_freelist.empty()) return E::OUT_OF_MEMORY;
   auto page = pfndb_freelist.pop_back();
   out.push_back(page);
   page->flags = 1;
   return page_to_paddr(page);
}

//===----------------------------------------------------------------------===//
// Create explicit virtual memory mappings
//===----------------------------------------------------------------------===//

static Result<void> make_next_level_table(TableDescriptor* table, void* next, uint64_t idx,
                                          PageList& out) {
   if (table[idx].valid) return E::OK;
   auto pa_wrapped = alloc_page(out);
   if (!pa_wrapped) return pa_wrapped.status();
   table[idx] = UnifyDescriptors(TableDescriptor{
         .valid = 1,
         .table_phys_addr = pa_wrapped.unwrap() / 4_KiB,
   });
   platform::arch::flush_tlb_at(reinterpret_cast<uint64_t>(next));
   cxx::memset(next, 0, 4_KiB);
   return E::OK;
}

Result<void> core::mem::map(uint64_t va, PageDescriptor pa, PageList& out) {
   // 1. Calculate all the table indices from the input address
   // See Figure D8-3 Arm Architecture Reference Manual (DDI 0487)
   auto l3idx = cxx::bits<20, 12>(va);
   auto l2idx = cxx::bits<29, 21>(va);
   auto l1idx = cxx::bits<38, 30>(va);
   auto l0idx = cxx::bits<47, 39>(va);
   auto prefix = cxx::bits<63, 48>(va);
   // 2. Check VA and grab all the table addresses via recursive mappings.
   // Note: Some of these tables (excep L0) may not even exist yet, so be careful.
   assert(prefix == 0 || prefix == 0xFFFF, "The given virtual address is non-canonical!");
   auto l0table = reinterpret_cast<TableDescriptor*>(idx_to_va(prefix, 511, 511, 511, 511));
   auto l1table = reinterpret_cast<TableDescriptor*>(idx_to_va(prefix, 511, 511, 511, l0idx));
   auto l2table = reinterpret_cast<TableDescriptor*>(idx_to_va(prefix, 511, 511, l0idx, l1idx));
   auto l3table = reinterpret_cast<PageDescriptor*>(idx_to_va(prefix, 511, l0idx, l1idx, l2idx));
   // 3. Now go through each level, allocating a table if needed
   PageList allocated{};
   Result<void> result{E::OK};
   result = make_next_level_table(l0table, l1table, l0idx, allocated);
   if (!result) return free_pages(cxx::move(allocated)), result.status();
   result = make_next_level_table(l1table, l2table, l1idx, allocated);
   if (!result) return free_pages(cxx::move(allocated)), result.status();
   result = make_next_level_table(l2table, l3table, l2idx, allocated);
   if (!result) return free_pages(cxx::move(allocated)), result.status();
   // 4. Now we have the L3 table, populate it with the final translation
   l3table[l3idx] = pa;
   out.append(cxx::move(allocated));
   return E::OK;
}

PageDescriptor core::mem::traverse(uint64_t va) {
   // 1. Calculate all the table indices from the input address
   // See Figure D8-3 Arm Architecture Reference Manual (DDI 0487)
   auto l3idx = cxx::bits<20, 12>(va);
   auto l2idx = cxx::bits<29, 21>(va);
   auto l1idx = cxx::bits<38, 30>(va);
   auto l0idx = cxx::bits<47, 39>(va);
   auto prefix = cxx::bits<63, 48>(va);
   // 2. Check VA and grab all the table addresses via recursive mappings.
   // Note: Some of these tables (excep L0) may not even exist yet, so be careful.
   assert(prefix == 0 || prefix == 0xFFFF, "The given virtual address is non-canonical!");
   auto l3table = reinterpret_cast<PageDescriptor*>(idx_to_va(prefix, 511, l0idx, l1idx, l2idx));
   return l3table[l3idx];
}

//===----------------------------------------------------------------------===//
// Region::*
//===----------------------------------------------------------------------===//

PageDescriptor Region::get_page_descriptor(uint64_t phys) {
   const uint8_t attr_idx = flags_.is_device ? 0 : 1;
   const uint8_t uxn_table = flags_.executable ? 0 : 1;
   // FIXME(kevin): Add back permissions, the thing is we can only do
   // kernel R/O + user R/O and not kernel R/W + user R/O
   uint8_t ap = 0b01;
   // if (region->flags_.readable && region->flags_.writable) ap = 0b01;
   // if (region->flags_.readable && !region->flags_.writable) ap = 0b11;
   return PageDescriptor{
         .valid = 1,
         .attr_index = attr_idx,
         .ap = ap,
         .phys_addr = phys / 4_KiB,
         .uxn_table = uxn_table,
   };
}

void Region::dump() const {
   char flags[4]{0};
   flags[0] = flags_.readable ? 'R' : '-';
   flags[1] = flags_.writable ? 'W' : '-';
   flags[2] = flags_.executable ? 'X' : '-';
   char const* type = flags_.shared ? "Shared" : flags_.is_device ? "Device" : "Normal";
   char const* grows =
         flags_.lazy_commit ? (flags_.grows_down ? "Can grow down" : "Can grow") : "Fixed";
   libkernel::log(LL::DEBUG, "Region [%s]\r\n", name_);
   libkernel::log(LL::DEBUG, " -> Range 0x%lX-0x%lX (0x%lX bytes)\r\n", base_, base_ + size_,
                  size_);
   libkernel::log(LL::DEBUG, " -> Flags (RWX): %s, Type: %s, %s\r\n", flags, type, grows);
}

Result<void> Region::commit(uint64_t address) {
   // 1. Check to see if the region is eligible for lazy-commit
   if (!(address >= base_ && address < base_ + size_)) return E::INVALID;
   if (!flags_.lazy_commit) return E::INVALID;

   // 2. Handle device lazy-commit differently than normal lazy-commit.
   // a. For device regions, we should check to see if it's a valid DevicePage.
   // b. For normal regions, just allocate the page and then map it.
   if (flags_.is_device) {
      auto page = cxx::align_down(address, 4_KiB);
      for (auto paddr : DevicePages) {
         if (!(paddr >= page && paddr < page + 4_KiB)) continue;
         const auto pd = get_page_descriptor(paddr);
         auto res = map(address, pd, aspace_->mmu_pages_);
         if (!res) return res.status();
         platform::arch::flush_tlb_at(address);
         return E::OK;
      }
      return E::INVALID;
   } else {
      PageList pages{};
      auto phys = core::mem::alloc_page(pages);
      if (!phys) return phys.status();
      const auto virt = cxx::align_down(address, 4_KiB);
      const auto pd = get_page_descriptor(phys.unwrap());
      auto res = map(virt, pd, aspace_->mmu_pages_);
      if (!res) {
         free_pages(cxx::move(pages));
         return res.status();
      }
      pages_->append(cxx::move(pages));
      platform::arch::flush_tlb_at(virt);
   }
   return E::OK;
}

void Region::activate(bool flush) {
   // 1. If the region grows down, allocate the last page first
   const auto grows_down = flags_.lazy_commit && flags_.grows_down;
   auto addr = !grows_down ? base_ : base_ + size_ - 4_KiB;
   // 2. Dump the region
   dump();
   // 3. For each page in the region, allocate it to the `addr` pointer
   for (auto page : *pages_) {
      const auto pd = get_page_descriptor(page_to_paddr(page));
      auto result = map(addr, pd, aspace_->mmu_pages_);
      assert(result, "Failed to map page");
      addr = grows_down ? addr - 4_KiB : addr + 4_KiB;
   }
   if (flush) {
      platform::arch::flush_tlb();
   }
}

//===----------------------------------------------------------------------===//
// AddressSpace::*
//===----------------------------------------------------------------------===//

AddressSpace::AddressSpace(Page* l0table)
      : regions_{}, mmu_pages_{}, l0table_{page_to_paddr(l0table)} {
   mmu_pages_.push_back(l0table);
}

void AddressSpace::activate() {
   assert(&(thread::cur_thread()->aspace) == this);
   for (auto region : regions_) region->activate(false);
   platform::arch::flush_tlb();
}

Result<Region*> AddressSpace::add_region(char const* name, Range range, Region::Flags flags) {
   // 1. Adjust the flags for the region
   if (!flags.is_device) flags.shared = 0;
   if (flags.writable) flags.readable = 1;
   if (!flags.lazy_commit) flags.grows_down = 0;
   // 2a. Allocate the region object
   const auto base = range.start;
   const auto size = range.end - range.start;
   Region* region;
   {
      auto buf = core::slab::alloc<Region>();
      if (!buf) return buf.status();
      region = new (buf.unwrap()) Region(base, size, flags, name);
   }
   // 2b. Allocate the page object for the region
   PageList* pages;
   {
      auto buf = core::slab::alloc<PageList>();
      if (!buf) return slab::free(region), buf.status();
      pages = new (buf.unwrap()) PageList();
   }
   // 2c. Calculate + allocate the pages needed for the region
   const auto n_pages = flags.lazy_commit ? 1 : size / 4_KiB;
   for (auto i = 0UL; i < n_pages; i++) {
      auto res = alloc_page(*pages);
      if (!res) {
         free_pages(cxx::move(*pages));
         slab::free(pages);
         slab::free(region);
         return res.status();
      }
   }
   // 3. Populate the region object and register it
   region->pages_ = pages;
   region->aspace_ = this;
   regions_.push_back(region);
   return region;
}

Result<Region*> AddressSpace::add_region(char const* name, uint64_t base, PageList* vmo,
                                         Region::Flags flags) {
   // 1. Adjust the flags for the region
   flags.shared = 1;
   flags.lazy_commit = 0;
   flags.grows_down = 0;
   if (flags.writable) flags.readable = 1;
   // 2. Allocate the region object
   Region* region;
   {
      auto buf = core::slab::alloc<Region>();
      auto size = vmo->size() * 4_KiB;
      if (!buf) return buf.status();
      region = new (buf.unwrap()) Region(base, size, flags, name);
   }
   // 3. Populate the region object and register it
   region->pages_ = vmo;
   region->aspace_ = this;
   regions_.push_back(region);
   return region;
}