[LITMUS^RT] [PATCH] Architecture dependent uncachable control page.
Christopher Kenna
cjk at cs.unc.edu
Sun Oct 7 21:25:54 CEST 2012
While working on an ODROID-X (Samsung Exynos4412 ARM Cortex-A9), I
experienced non-determinism when reading and writing values to the
LITMUS^RT control page. Writes to the control page from user- (kernel-)
space where not always reflected in kernel- (user-) space.
Neither restricting the task to run on a single CPU nor inserting
general memory barriers (with mb()) fixed the problem. Mapping the
control page as uncachable in both kernel and user space did fix the
problem, which is what this patch does.
Also, since vunmap() cannot be called from an interrupt context, I had
to add a workqueue that unmaps and frees the control page when it is no
longer needed. (On my system, exit_litmus() was called from interrupt
context while the kernel reaped the task_struct.)
---
arch/arm/Kconfig | 3 +
include/litmus/litmus.h | 2 +
include/litmus/rt_param.h | 3 +
litmus/ctrldev.c | 150 +++++++++++++++++++++++++++++++++++++++++++--
litmus/litmus.c | 33 ++++++----
5 files changed, 176 insertions(+), 15 deletions(-)
diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig
index fb228ea..662cb7d 100644
--- a/arch/arm/Kconfig
+++ b/arch/arm/Kconfig
@@ -2047,4 +2047,7 @@ config ARCH_HAS_SEND_PULL_TIMERS
config ARCH_HAS_FEATHER_TRACE
def_bool n
+config ARCH_NEEDS_UNCACHED_CONTROL_PAGE
+ def_bool y
+
source "litmus/Kconfig"
diff --git a/include/litmus/litmus.h b/include/litmus/litmus.h
index 807b788..7ec9baa 100644
--- a/include/litmus/litmus.h
+++ b/include/litmus/litmus.h
@@ -12,6 +12,8 @@
extern atomic_t release_master_cpu;
#endif
+void litmus_schedule_deallocation(struct task_struct *t);
+
/* in_list - is a given list_head queued on some list?
*/
static inline int in_list(struct list_head* list)
diff --git a/include/litmus/rt_param.h b/include/litmus/rt_param.h
index fac939d..12d252f 100644
--- a/include/litmus/rt_param.h
+++ b/include/litmus/rt_param.h
@@ -225,6 +225,9 @@ struct rt_param {
/* Pointer to the page shared between userspace and kernel. */
struct control_page * ctrl_page;
+#ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
+ void *ctrl_page_orig;
+#endif
};
/* Possible RT flags */
diff --git a/litmus/ctrldev.c b/litmus/ctrldev.c
index 9969ab1..07c3a68 100644
--- a/litmus/ctrldev.c
+++ b/litmus/ctrldev.c
@@ -3,6 +3,10 @@
#include <linux/fs.h>
#include <linux/miscdevice.h>
#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+#include <linux/kernel.h>
+#include <linux/workqueue.h>
#include <litmus/litmus.h>
@@ -10,32 +14,157 @@
#define CTRL_NAME "litmus/ctrl"
+static struct workqueue_struct *wq_litmus_dealloc;
+
+struct litmus_dealloc_work {
+ struct work_struct work_struct;
+ void *ctrl_page_mem;
+#ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
+ void *ctrl_page_unmap;
+#endif
+};
+
+static void litmus_dealloc(struct work_struct *work_in)
+{
+ struct litmus_dealloc_work *work = container_of(work_in,
+ struct litmus_dealloc_work, work_struct);
+#ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
+ TRACE("vunmap() control page %p.\n", work->ctrl_page_unmap);
+ vunmap(work->ctrl_page_unmap);
+#endif
+ TRACE("freeing ctrl_page %p\n", work->ctrl_page_mem);
+ free_page((unsigned long) work->ctrl_page_mem);
+
+ kfree((void*) work);
+}
+
+void litmus_schedule_deallocation(struct task_struct *t)
+{
+ struct litmus_dealloc_work *work;
+
+ if (NULL == tsk_rt(t)->ctrl_page)
+ return;
+
+ work = kmalloc(sizeof(*work), GFP_ATOMIC);
+ if (!work) {
+ WARN(1, "Could not allocate LITMUS deallocation work.\n");
+ return;
+ }
+
+ INIT_WORK(&work->work_struct, litmus_dealloc);
+
+#ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
+ work->ctrl_page_mem = tsk_rt(t)->ctrl_page_orig;
+ work->ctrl_page_unmap = tsk_rt(t)->ctrl_page;
+#else
+ work->ctrl_page_mem = tsk_rt(t)->ctrl_page;
+#endif
+ queue_work(wq_litmus_dealloc, &work->work_struct);
+}
+
+#ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
+/*
+ * remap_noncached - creates a non-cached memory "shadow mapping"
+ * @addr: memory base virtual address
+ * @len: length to remap
+ *
+ * The caller should vunmap(addr) when the mapping is no longer needed.
+ * The caller should also save the original @addr to free it later.
+ */
+static void * remap_noncached(void *addr, size_t len)
+{
+ void *vaddr;
+ int nr_pages = DIV_ROUND_UP(offset_in_page(addr) + len, PAGE_SIZE);
+ struct page **pages = kmalloc(nr_pages * sizeof(*pages), GFP_KERNEL);
+ void *page_addr = (void *)((unsigned long)addr & PAGE_MASK);
+ int i;
+
+ if (NULL == pages) {
+ TRACE_CUR("No memory!\n");
+ return ERR_PTR(-ENOMEM);
+ }
+
+ for (i = 0; i < nr_pages; i++) {
+ if (is_vmalloc_or_module_addr(page_addr)) {
+ kfree(pages);
+ TRACE_CUR("Remapping vmalloc or module memory?\n");
+ return ERR_PTR(-EINVAL);
+ }
+
+ pages[i] = virt_to_page(page_addr);
+ if (NULL == pages[i]) {
+ kfree(pages);
+ TRACE_CUR("Bad virtual address.\n");
+ return ERR_PTR(-EINVAL);
+ }
+ page_addr += PAGE_SIZE;
+ }
+
+ vaddr = vmap(pages, nr_pages, VM_MAP, pgprot_noncached(PAGE_KERNEL));
+ kfree(pages);
+ if (NULL == vaddr) {
+ TRACE_CUR("vmap() failed.\n");
+ return ERR_PTR(-ENOMEM);
+ }
+ return vaddr + offset_in_page(addr);
+}
+#endif
+
/* allocate t->rt_param.ctrl_page*/
static int alloc_ctrl_page(struct task_struct *t)
{
+ void *mem;
+#ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
+ void *mem_remap;
+#endif
int err = 0;
/* only allocate if the task doesn't have one yet */
if (!tsk_rt(t)->ctrl_page) {
- tsk_rt(t)->ctrl_page = (void*) get_zeroed_page(GFP_KERNEL);
- if (!tsk_rt(t)->ctrl_page)
+ mem = (void*) get_zeroed_page(GFP_KERNEL);
+ if (!mem) {
err = -ENOMEM;
+ goto out;
+ }
+
+#ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
+ mem_remap = remap_noncached(mem, PAGE_SIZE);
+ if (IS_ERR(mem_remap)) {
+ err = PTR_ERR(mem_remap);
+ free_page((unsigned long) mem);
+ goto out;
+ }
+ tsk_rt(t)->ctrl_page_orig = mem;
+ tsk_rt(t)->ctrl_page = mem_remap;
+ TRACE_TASK(t, "ctrl_page_orig = %p\n",
+ tsk_rt(t)->ctrl_page_orig);
+#else
+ tsk_rt(t)->ctrl_page = mem;
+#endif
+
/* will get de-allocated in task teardown */
TRACE_TASK(t, "%s ctrl_page = %p\n", __FUNCTION__,
tsk_rt(t)->ctrl_page);
}
+out:
return err;
}
static int map_ctrl_page(struct task_struct *t, struct vm_area_struct* vma)
{
+ struct page *ctrl;
int err;
- struct page* ctrl = virt_to_page(tsk_rt(t)->ctrl_page);
+#ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
+ /* vm_insert_page() using the "real" vaddr, not the shadow mapping. */
+ ctrl = virt_to_page(tsk_rt(t)->ctrl_page_orig);
+#else
+ ctrl = virt_to_page(tsk_rt(t)->ctrl_page);
+#endif
TRACE_CUR(CTRL_NAME
": mapping %p (pfn:%lx) to 0x%lx (prot:%lx)\n",
- tsk_rt(t)->ctrl_page,page_to_pfn(ctrl), vma->vm_start,
+ tsk_rt(t)->ctrl_page, page_to_pfn(ctrl), vma->vm_start,
vma->vm_page_prot);
/* Map it into the vma. */
@@ -104,7 +233,11 @@ static int litmus_ctrl_mmap(struct file* filp, struct vm_area_struct* vma)
* don't care if it was touched or not. __S011 means RW access, but not
* execute, and avoids copy-on-write behavior.
* See protection_map in mmap.c. */
+#ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
+ vma->vm_page_prot = pgprot_noncached(__S011);
+#else
vma->vm_page_prot = __S011;
+#endif
err = alloc_ctrl_page(current);
if (!err)
@@ -137,11 +270,20 @@ static int __init init_litmus_ctrl_dev(void)
err = misc_register(&litmus_ctrl_dev);
if (err)
printk("Could not allocate %s device (%d).\n", CTRL_NAME, err);
+
+ wq_litmus_dealloc = alloc_workqueue("litmus_dealloc",
+ WQ_NON_REENTRANT|WQ_MEM_RECLAIM, 0);
+ if (NULL == wq_litmus_dealloc) {
+ printk("Could not allocate vunmap workqueue.\n");
+ misc_deregister(&litmus_ctrl_dev);
+ }
return err;
}
static void __exit exit_litmus_ctrl_dev(void)
{
+ flush_workqueue(wq_litmus_dealloc);
+ destroy_workqueue(wq_litmus_dealloc);
misc_deregister(&litmus_ctrl_dev);
}
diff --git a/litmus/litmus.c b/litmus/litmus.c
index 8138432..b48d23e 100644
--- a/litmus/litmus.c
+++ b/litmus/litmus.c
@@ -9,6 +9,7 @@
#include <linux/sched.h>
#include <linux/module.h>
#include <linux/slab.h>
+#include <linux/vmalloc.h>
#include <litmus/litmus.h>
#include <litmus/bheap.h>
@@ -295,12 +296,18 @@ static void reinit_litmus_state(struct task_struct* p, int restore)
{
struct rt_task user_config = {};
void* ctrl_page = NULL;
+#ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
+ void * ctrl_page_orig = NULL;
+#endif
if (restore) {
/* Safe user-space provided configuration data.
* and allocated page. */
user_config = p->rt_param.task_params;
ctrl_page = p->rt_param.ctrl_page;
+#ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
+ ctrl_page_orig = p->rt_param.ctrl_page_orig;
+#endif
}
/* We probably should not be inheriting any task's priority
@@ -315,6 +322,9 @@ static void reinit_litmus_state(struct task_struct* p, int restore)
if (restore) {
p->rt_param.task_params = user_config;
p->rt_param.ctrl_page = ctrl_page;
+#ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
+ p->rt_param.ctrl_page_orig = ctrl_page_orig;
+#endif
}
}
@@ -458,9 +468,13 @@ void litmus_fork(struct task_struct* p)
reinit_litmus_state(p, 0);
/* Don't let the child be a real-time task. */
p->sched_reset_on_fork = 1;
- } else
+ } else {
/* non-rt tasks might have ctrl_page set */
tsk_rt(p)->ctrl_page = NULL;
+#ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
+ tsk_rt(p)->ctrl_page_orig = NULL;
+#endif
+ }
/* od tables are never inherited across a fork */
p->od_table = NULL;
@@ -476,27 +490,24 @@ void litmus_exec(void)
if (is_realtime(p)) {
WARN_ON(p->rt_param.inh_task);
- if (tsk_rt(p)->ctrl_page) {
- free_page((unsigned long) tsk_rt(p)->ctrl_page);
- tsk_rt(p)->ctrl_page = NULL;
- }
+ litmus_schedule_deallocation(p);
+ tsk_rt(p)->ctrl_page = NULL;
+#ifdef CONFIG_ARCH_NEEDS_UNCACHED_CONTROL_PAGE
+ tsk_rt(p)->ctrl_page_orig = NULL;
+#endif
}
}
void exit_litmus(struct task_struct *dead_tsk)
{
+
/* We also allow non-RT tasks to
* allocate control pages to allow
* measurements with non-RT tasks.
* So check if we need to free the page
* in any case.
*/
- if (tsk_rt(dead_tsk)->ctrl_page) {
- TRACE_TASK(dead_tsk,
- "freeing ctrl_page %p\n",
- tsk_rt(dead_tsk)->ctrl_page);
- free_page((unsigned long) tsk_rt(dead_tsk)->ctrl_page);
- }
+ litmus_schedule_deallocation(dead_tsk);
/* main cleanup only for RT tasks */
if (is_realtime(dead_tsk))
--
1.7.9.5
More information about the litmus-dev
mailing list