Nucleus

/*
 * This file contains the page fault handler.  The main handler determines what
 * went wrong and calls the relevant routine to handle it.  That includes:
 * non-present reads, non-present writes, copy-on-write.
 * There is an early handler which just gives out memory when requested.  This
 * is used by the kernel before it has initialised multi-tasking and the virtual
 * file system.
 */

#include <stdint.h>
#include <nucleus/cpu.h>
#include <nucleus/kernel.h>
#include <nucleus/lib.h>
#include <nucleus/memory.h>
#include <nucleus/task.h>
#include "namespace.h"

extern page_t zeroFrame;

VMRegion *find_region(uintptr_t addr);
void copy_page_frame(page_t src, page_t dest);

/* Copy on write */
static void
copy_on_write(VMRegion *region, uintptr_t addr)
{
	Inode *inode;
	Page *page = NULL, *newPage = NULL;
	File *front = region->front,
	     *back  = region->back;
	off_t offset = (PAGE_ADDR(addr) - region->start) + region->offset;

	/* Handle uninitialised anonymous region */
	int private = region->flags & MAP_PRIVATE;
	int anonymous = (region->flags & MAP_ANONYMOUS);
	if (!front && (private || anonymous))
		region->front = front = create_anonymous_file();

	/* Find original page frame */
	if (!page && front) {
		inode = front->inode;
		page = find_page(inode->pages, offset);
	}
	if (!page && back) {
		inode = back->inode;
		page = find_page(inode->pages, offset);
	}
	ASSERT(page);

	/* Determine what to do */
	if (usage(page) > 1 || page->frame == zeroFrame) {
		/* Copy page */
		newPage = create_page(front->inode->pages,
		                      alloc_frame(), offset);
		copy_page_frame(PAGE_ADDR(page->frame),
		                PAGE_ADDR(newPage->frame));
		remove(inode->pages, page);
		page = newPage;
	} else if (back && inode == back->inode && private) {
		/* Page in wrong inode for write */
		add(front->inode->pages, page);
		remove(inode->pages, page);
	}
	return install_page(addr, page, region->prot);
}

/* Handle a non-present read page fault */
static void
not_present_read(VMRegion *region, uintptr_t addr)
{
	Inode *inode;
	Page *page = NULL;
	File *front = region->front,
	     *back  = region->back;
	off_t offset = (PAGE_ADDR(addr) - region->start) + region->offset;

	/* Handle uninitialised anonymous regions */
	if (!front && (region->flags & MAP_ANONYMOUS))
		region->front = front = create_anonymous_file();

	/* Attempt to use front */
	if (front) {
		inode = front->inode;
		page = find_page(inode->pages, offset);
		if (page)
			return install_page(addr, page, region->prot);
		/* Zero-fill if anonymous */
		if (region->flags & MAP_ANONYMOUS) {
			page = create_page(inode->pages, zeroFrame, offset);
			return install_page(addr, page, PROT_READ);
		}
	}

	/* Use back */
	ASSERT(back);
	inode = back->inode;
	page = find_page(inode->pages, offset);
	if (page)
		return install_page(addr, page, PROT_READ);
	/* Create new block cache entry */
	page = create_page(inode->pages, alloc_frame(), offset);
	install_page(addr, page, PROT_READ);
	file_mmap(back, (void *) PAGE_ADDR(addr), PAGE_SIZE, offset);
}

/* Handle a non-present write page fault */
static void
not_present_write(VMRegion *region, uintptr_t addr)
{
	Inode *inode;
	Page *page = NULL, *newPage = NULL;
	File *front = region->front,
	     *back  = region->back;
	off_t offset = (PAGE_ADDR(addr) - region->start) + region->offset;

	/* Handle uninitialised anonymous regions */
	if (!front && ((region->flags & MAP_PRIVATE)
	 || (region->flags & MAP_ANONYMOUS)))
		region->front = front = create_anonymous_file();

	/* Shared region, write through to back */
	if (region->flags & MAP_SHARED) {
		if (region->flags & MAP_ANONYMOUS)
			back = front;
		ASSERT(back);
		inode = back->inode;
		page = find_page(inode->pages, offset);
		if (page)
			return install_page(addr, page, region->prot);
		page = create_page(inode->pages, alloc_frame(), offset);
		install_page(addr, page, region->prot);
		/* Zero-fill if anonymous, otherwise read */
		if (region->flags & MAP_ANONYMOUS)
			memset((void *) PAGE_ADDR(addr), 0, PAGE_SIZE);
		else
			file_mmap(back, (void *) PAGE_ADDR(addr),
		                  PAGE_SIZE, offset);
		return;
	}

	/* Private region, copy to front */
	ASSERT(front);
	inode = front->inode;
	page = find_page(inode->pages, offset);
	if (page)
		return install_page(addr, page, region->prot);
	newPage = create_page(inode->pages, alloc_frame(), offset);
	install_page(addr, newPage, region->prot);

	/* Anonymous region, zero-fill */
	if (region->flags & MAP_ANONYMOUS) {
		memset((void *) PAGE_ADDR(addr), 0, PAGE_SIZE);
		return;
	}

	/* Use back */
	ASSERT(back);
	inode = back->inode;
	page = find_page(inode->pages, offset);
	if (page) {
		copy_page_frame(PAGE_ADDR(page->frame),
		                PAGE_ADDR(newPage->frame));
		remove(inode->pages, page);
	} else {
		file_mmap(back, (void *) PAGE_ADDR(addr), PAGE_SIZE, offset);
	}
}

/* Page fault handler */
void
page_fault_handler(struct InterruptFrame *frame)
{
	uintptr_t addr;
	asm volatile("mov %%cr2, %0" : "=r" (addr));
	uint8_t present = frame->err & (1 << 0);
	uint8_t write   = frame->err & (1 << 1);
	uint8_t user    = frame->err & (1 << 2);
	page_t pg = get_page(addr);

	ASSERT(current && current->vm);

	/* Handle lazy invalidation */
	if (!present && (pg & PTE_PRESENT))
		return flush_tlb(addr);
	if (write && (pg & PTE_WRITE))
		return flush_tlb(addr);

	/* Iterate VM Regions */
	VMRegion *region = find_region(addr);
	if (__builtin_expect(!region, 0)) {
		/* Not in a region */
		panic("Page Fault [%d:%d] (%#.8x -> %#.8x [tbl:%d, pg:%d][%#.8x], %s, %s, %s)",
		      current->tgid, current->tid, frame->eip,
		      addr, (addr >> 12) / 1024, (addr >> 12) % 1024, pg,
		      present ? "present" : "not present",
		      write ? "write" : "read",
		      user ? "user" : "kernel");
	}

	/* Protection violation, kill process */
	if (user && write && !(region->prot & PROT_WRITE))
		panic("Segmentation violation : %#.8x[%#.8x] (%#.8x -> %#.8x)",
		      region, region->prot, frame->eip, addr);

	/* Update paging structures correctly */
	if (present && write)
		return copy_on_write(region, addr);
	if (!present && write)
		return not_present_write(region, addr);
	if (!present && !write)
		return not_present_read(region, addr);
}