exception.cc

#include "platform/exception.h"

#include <cxx/string.h>
#include <cxx/utility.h>
#include <libkernel/assert.h>
#include <libkernel/print.h>
#include <stdint.h>

#include "core/syscalls.h"
#include "core/thread.h"
#include "core/vm.h"
#include "libkernel/status.h"
#include "platform/arch.h"

using core::thread::tid_t;
using platform::exception::Frame;

// Send = 0, Receive = 1, Reply = 2
DECLARE_KCOUNTERS(all_ipc_perfs, 3)
DECLARE_KCOUNTERS(all_syscalls_perfs, 1)

static const char* const exception_kind_names[] = {
      "Synchronous exception (current EL, SP0)",
      "IRQ exception (current EL, SP0)",
      "FIQ exception (current EL, SP0)",
      "SError exception (current EL, SP0)",
      "Synchronous exception (current EL, SPx)",
      "IRQ exception (current EL, SPx)",
      "FIQ exception (current EL, SPx)",
      "SError exception (current EL, SPx)",
      "Synchronous exception (lower EL, AArch64)",
      "IRQ exception (lower EL, AArch64)",
      "FIQ exception (lower EL, AArch64)",
      "SError exception (lower EL, AArch64)",
      "Synchronous exception (lower EL, AArch32)",
      "IRQ exception (lower EL, AArch32)",
      "FIQ exception (lower EL, AArch32)",
      "SError exception (lower EL, AArch32)",
      // Last entry is an invalid entry
      "Unknown exception class",
};

//===----------------------------------------------------------------------===//
// Public exception API
//===----------------------------------------------------------------------===//

extern "C" void* vector_table;

void platform::exception::init() {
   // Set up the exception vector table
   platform::arch::write_vbar_el1(reinterpret_cast<uintptr_t>(&vector_table));
   asm volatile("isb" ::: "memory");
   libkernel::log("Exception vector table set to 0x%016lx\r\n", &vector_table);
   // Set up the interrupt controller
   setup_gic();
}

//===----------------------------------------------------------------------===//
// Static helper functions
//===----------------------------------------------------------------------===//

static void dump_frame(Frame* frame) {
   // clang-format off
   for (int i = 0; i < 30; i += 4) {
      libkernel::log(
         "x%02d: 0x%016lx x%02d: 0x%016lx x%02d: 0x%016lx  x%02d: 0x%016lx\r\n",
         i, frame->x[i],
         i + 1, frame->x[i + 1],
         i + 2, frame->x[i + 2],
         i + 3, frame->x[i + 3]
      );
   }
   libkernel::log(
      "lr: 0x%016lx  elr: 0x%016lx  sp: 0x%016lx spsr: 0x%016lx\r\n",
      frame->lr, frame->elr_elx, frame->sp_elx, frame->spsr_elx
   );
   // clang-format on
}

static uint64_t dump_table_value(uint64_t va, int level) {
   using core::mem::idx_to_va;

   // 1. Calculate all the table indices from the input address
   // See Figure D8-3 Arm Architecture Reference Manual (DDI 0487)
   auto l3idx = cxx::bits<20, 12>(va);
   auto l2idx = cxx::bits<29, 21>(va);
   auto l1idx = cxx::bits<38, 30>(va);
   auto l0idx = cxx::bits<47, 39>(va);
   auto prefix = cxx::bits<63, 48>(va);

   // 2. Check VA and grab all the table addresses via recursive mappings.
   // Note: Some of these tables (except L0) may not even exist yet, so be careful.
   if (!(prefix == 0 || prefix == 0xFFFF)) {
      libkernel::log(LL::ERROR, "The given virtual address is non-canonical!\r\n");
      return 0;
   }
   auto l0 = reinterpret_cast<uint64_t*>(idx_to_va(prefix, 511, 511, 511, 511));
   auto l1 = reinterpret_cast<uint64_t*>(idx_to_va(prefix, 511, 511, 511, l0idx));
   auto l2 = reinterpret_cast<uint64_t*>(idx_to_va(prefix, 511, 511, l0idx, l1idx));
   auto l3 = reinterpret_cast<uint64_t*>(idx_to_va(prefix, 511, l0idx, l1idx, l2idx));

   // 3. Now return the value depending on the level
   switch (level) {
      case 0:
         return l0[l0idx];
      case 1:
         return l1[l1idx];
      case 2:
         return l2[l2idx];
      case 3:
         return l3[l3idx];
      default:
         return 0xDEADBEEF'CAFEBABE;
   }
}

static void parse_mmu_abort_iss(uint64_t iss, bool is_data_abort, bool lower_el) {
   const auto is_far_valid = (cxx::bits<10, 10>(iss) == 0);
   const auto read_or_write = cxx::bits<6, 6>(iss) == 0b0 ? "read" : "write";
   const auto dfsc = cxx::bits<5, 0>(iss);
   const auto far = platform::arch::read_far_el1();

   const char* lower_el_msg = lower_el ? " (from Lower EL)" : "";
   const char* abort_type_msg = is_data_abort ? "Data Abort" : "Instruction Abort";
   libkernel::log(LL::ERROR, "!! %s Exception%s !!\r\n", abort_type_msg, lower_el_msg);
   if (is_far_valid) libkernel::log(LL::ERROR, "FAR_ELx: 0x%016lx\r\n", far);
   libkernel::log("Failed on memory %s\r\n", read_or_write);

   // Parse the DFSC field of the ISS and dump the reason
   const auto dfsc_hi = cxx::bits<5, 2>(dfsc);
   const auto dfsc_lo = cxx::bits<1, 0>(dfsc);
   const char* msg = "Unhandled";
   bool level = true;

   // clang-format off
   // 1. Parse the HI bits, the LO bits will have the fault level
   switch (dfsc_hi) {
      case 0b0000: msg = "Address size fault"; break;
      case 0b0001: msg = "Translation fault"; break;
      case 0b0010: msg = "Access flag fault"; break;
      case 0b0011: msg = "Permission fault"; break;
      case 0b0101: msg = "Synchronous External abort"; break;
      case 0b0111: msg = "Synchronous parity or ECC error"; break;
      case 0b1001: msg = "Granule Protection Fault"; break;
      default: level = false; break;
   }
   // 2. For reasons with NO level, parse the whole DFSC field
   switch (dfsc) {
      case 0b010000: msg = "Synchronous External abort, no level"; break;
      case 0b010001: msg = "Synchronous Tag Check Fault"; break;
      case 0b100001: msg = "Alignment fault"; break;
      case 0b011000: msg = "Synchronous parity or ECC error"; break;
      case 0b101000: msg = "Granule Protection Fault"; break;
      case 0b110000: msg = "TLB conflict abort"; break;
   }
   // clang-format on

   // Now we may print out the reason
   const char* fmt = level ? "Reason: %s (level %d)\r\n" : "Reason: %s\r\n";
   libkernel::log(fmt, msg, dfsc_lo);

   // Special cases to dump more data
   if (is_far_valid) {
      if (dfsc_hi == 0 || dfsc_hi == 0b0001) {
         libkernel::log("Table value: 0x%016lx\r\n", dump_table_value(far, level));
      }
      if (dfsc == 0b010000) {
         const auto pd = core::mem::traverse(far);
         const auto pa = pd.phys_addr * 4_KiB;
         libkernel::log("Phys Addr: 0x%llX\r\n", pa);
      }
   }
}

//===----------------------------------------------------------------------===//
// Specific exception handler functions, handle_*()
//===----------------------------------------------------------------------===//

void platform::exception::handle_error(Frame* frame, uint64_t kind) {
   const char* kind_name = exception_kind_names[cxx::min(kind, 16UL)];
   libkernel::log(LL::ERROR, "Illegal exception captured: %s\r\n", kind_name);

   const auto esr_el1 = platform::arch::read_esr_el1();
   const auto ec = cxx::bits<31, 26>(esr_el1);
   const auto iss = cxx::bits<24, 0>(esr_el1);
   const auto iss2 = cxx::bits<55, 32>(esr_el1);
   const auto il = cxx::bits<25, 25>(esr_el1);
   libkernel::log("ESR_EL1: 0x%016lx\r\n", esr_el1);
   libkernel::log("EC: 0x%llX, ISS: 0x%llX, ISS2: 0x%llX, IL: %d\r\n", ec, iss, iss2, il);
   const auto cur_thread = core::thread::cur_thread();
   libkernel::log("Current thread: %s\r\n", (cur_thread == nullptr) ? "None" : cur_thread->dump());

   switch (ec) {
      case 0b100000:
         parse_mmu_abort_iss(iss, false, true);
         break;
      case 0b100001:
         parse_mmu_abort_iss(iss, false, false);
         break;
      case 0b100100:
         parse_mmu_abort_iss(iss, true, true);
         break;
      case 0b100101:
         parse_mmu_abort_iss(iss, true, false);
         break;
      default:
         libkernel::log(LL::ERROR, "Unhandled error code.\r\n");
         break;
   }

   // Dump the exception register frame now
   dump_frame(frame);

   // Either go to the _Exit() handler or halt
   if(cur_thread && cur_thread->exit_address != 0 && !cur_thread->thread_error) {
      frame->x[0] = 0xDEAD;
      frame->elr_elx = cur_thread->exit_address;
      core::thread::cur_thread()->set_error_state();
   } else {
      platform::arch::halt();
   }
}

static bool handle_user_mem_fault(uint64_t iss) {
   const auto dfsc = cxx::bits<5, 0>(iss);
   const auto dfsc_hi = cxx::bits<5, 2>(dfsc);
   const auto is_far_valid = (cxx::bits<10, 10>(iss) == 0);
   const auto far = platform::arch::read_far_el1();

   // We must have a valid FAR address
   if (!is_far_valid) return false;
   // Check if the DFSC code is a Translation fault @ 3rd level
   if (dfsc_hi != 1 && dfsc_hi != 2) return false;
   // Check the FAR address, see if the fault is anywhere near
   for (auto region : core::thread::cur_thread()->aspace) {
      if (!region->flags().lazy_commit) continue;
      auto range = region->range();
      if (far >= range.start && far < range.end) {
         if (!region->commit(far)) {
            libkernel::log(LL::ERROR, "Region [%s] from 0x%016X-0x%016X failed to commit\r\n",
                           region->name(), region->range().start, region->range().end);
            return false;
         }
         libkernel::log(LL::DEBUG, "Region [%s] commit 1 page at 0x%llX\r\n", region->name(), far);
         return true;
      }
   }

   // No region found to commit
   return false;
}

static Result<void> handle_syscall(uint16_t raw_id, Frame* frame) {
   if (raw_id >= static_cast<uint16_t>(Syscall::LAST_MEMBER)) {
      libkernel::log(LL::ERROR, "Invalid syscall ID: %d\r\n", raw_id);
      return E::INVALID;
   }

   all_syscalls_perfs[0].start_measure();
   switch (static_cast<Syscall>(raw_id)) {
      case Syscall::THREAD_CREATE:
         core::syscalls::Create(frame->x[0], frame->x[1]);
         break;
      case Syscall::THREAD_MY_TID:
         core::syscalls::MyTid();
         break;
      case Syscall::THREAD_MY_PARENT_TID:
         core::syscalls::MyParentTid();
         break;
      case Syscall::THREAD_YIELD:
         core::syscalls::Yield();
         break;
      case Syscall::THREAD_EXIT:
         core::syscalls::Exit();
         break;
      case Syscall::THREAD_SEND:
         all_ipc_perfs[0].start_measure();
         core::syscalls::Send((tid_t)frame->x[0], (const void*)frame->x[1], (size_t)frame->x[2],
                              (void*)frame->x[3], (size_t)frame->x[4]);
         all_ipc_perfs[0].end_measure();
         break;
      case Syscall::THREAD_RECEIVE:
         all_ipc_perfs[1].start_measure();
         core::syscalls::Receive((tid_t*)frame->x[0], (void*)frame->x[1], (size_t)frame->x[2]);
         all_ipc_perfs[1].end_measure();
         break;
      case Syscall::THREAD_REPLY:
         all_ipc_perfs[2].start_measure();
         core::syscalls::Reply((tid_t)frame->x[0], (void*)frame->x[1], (size_t)frame->x[2]);
         all_ipc_perfs[2].end_measure();
         break;
      case Syscall::AWAIT_EVENT:
         core::syscalls::AwaitEvent((int)frame->x[0]);
         break;
      default:
         return E::INVALID;
   }
   all_syscalls_perfs[0].end_measure();
   core::thread::schedule();
   return E::OK;
}

//===----------------------------------------------------------------------===//
// Extern "C" exception handlers (referenced in assembly)
//===----------------------------------------------------------------------===//

extern "C" void cpp_lower_sync_exc_handler(Frame* frame) {
   // Grab the exception state from the ESR_EL1 register
   const auto esr = platform::arch::read_esr_el1();
   const auto ec = cxx::bits<31, 26>(esr);
   const auto iss = cxx::bits<24, 0>(esr);

   // Determine the exception kind before handling
   if (ec == 0b010101) [[likely]] {
      // Before dispatching the system call handlers, save the thread context
      core::thread::save_thread_state(frame);
      if (handle_syscall(iss, frame)) {
         // After dispatching the system call handlers, restore the thread context
         core::thread::restore_thread_state(frame);
         return;
      }
   } else if (ec == 0b100100) {
      // If the memory fault handler fails, we fallthrough to the base case
      if (handle_user_mem_fault(iss)) return;
   }

   // Decode the exception based off the EC
   handle_error(frame, 8);
}

extern "C" [[noreturn]]
void cpp_illegal_exc_handler(Frame* frame, uint64_t kind) {
   handle_error(frame, kind);
   platform::arch::halt();
}