libtf: implement x86 assembly fiber switching

it's faster and has smaller context in the beginning of the
fiber. it's also required since setjmp() seems to use mangled
pointers in glibc so we could not schedule fibers in non-creation
threads. additionally the setjmp() setup code has a race condition.

2 files changed, 148 insertions, 8 deletions

diff --git a/src/fiber.c b/src/fiber.c
index 3f8bb15..6ddde1b 100644
--- a/src/fiber.c
+++ b/src/fiber.c
@@ -29,7 +29,7 @@ struct tf_fiber {
 	char			data[TF_EMPTY_ARRAY];
 };
 
-#include "uctx-setjmp.h"
+#include "uctx.h"
 
 /* FIXME: should be in thread local storage */
 struct tf_scheduler *__tf_scheduler;
@@ -91,12 +91,13 @@ static void run_fiber(struct tf_scheduler *sched, struct tf_fiber *f)
 	struct tf_fiber *schedf = container_of((void*) tf_get_scheduler(), struct tf_fiber, data);
 
 	sched->active_fiber = f;
-	switch (tf_uctx_transfer(schedf, f, f->wakeup_type)) {
+	tf_uctx_transfer(schedf, f);
+	switch (f->wakeup_type) {
 	case TF_WAKEUP_KILL:
 		tf_fiber_put(f->data);
 		sched->num_fibers--;
 		break;
-	case TF_WAKEUP_IMMEDIATE:
+	case TF_WAKEUP_NONE:
 		break;
 	default:
 		TF_BUG_ON("bad scheduler call from fiber");
@@ -222,15 +223,19 @@ int tf_schedule(void)
 	struct tf_fiber *f = sched->active_fiber;
 
 	if (unlikely(f->timeout_change)) {
-		if (f->timeout_change & TF_TIMEOUT_CHANGE_NEW_VALUE)
+		if (f->timeout_change & TF_TIMEOUT_CHANGE_NEW_VALUE) {
+			if (tf_mtime_diff(f->timeout, tf_mtime()) <= 0) {
+				f->timeout_change = TF_TIMEOUT_CHANGE;
+				return TF_WAKEUP_TIMEOUT;
+			}
 			tf_heap_change(&f->heap_node, &sched->heap, f->timeout);
-		else
+		} else
 			tf_heap_delete(&f->heap_node, &sched->heap);
 		f->timeout_change = 0;
 	}
 	f->wakeup_type = TF_WAKEUP_NONE;
-
-	return tf_uctx_transfer(f, schedf, TF_WAKEUP_IMMEDIATE);
+	tf_uctx_transfer(f, schedf);
+	return f->wakeup_type;
 }
 
 void tf_wakeup(struct tf_fiber *fiber, int wakeup_type)
@@ -250,7 +255,8 @@ void tf_exit(void)
 	struct tf_fiber *schedf = container_of((void*) sched, struct tf_fiber, data);
 
 	tf_heap_delete(&f->heap_node, &sched->heap);
-	tf_uctx_transfer(f, schedf, TF_WAKEUP_KILL);
+	f->wakeup_type = TF_WAKEUP_KILL;
+	tf_uctx_transfer(f, schedf);
 	TF_BUG_ON(1);
 }
 
diff --git a/src/uctx.h b/src/uctx.h
new file mode 100644
index 0000000..03750e9
--- /dev/null
+++ b/src/uctx.h
@@ -0,0 +1,134 @@
+/* uctx.h - assembly stack switching
+ *
+ * Copyright (C) 2009 Timo Teräs <timo.teras@iki.fi>
+ * All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 or later as
+ * published by the Free Software Foundation.
+ *
+ * See http://www.gnu.org/ for details.
+ */
+
+#include <libtf/fiber.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#ifdef VALGRIND
+#include <valgrind/valgrind.h>
+#endif
+
+#define STACK_GUARD		0xbad57ac4
+
+struct tf_uctx {
+	int *stack_guard;
+	void *alloc;
+	void *current_sp;
+#ifdef VALGRIND
+	unsigned int stack_id;
+#endif
+	struct tf_fiber fiber;
+};
+
+#if defined(__i386__)
+
+#define STACK_GROWS_UP
+
+#else
+#error Your architecture is not supported. Send email to timo.teras@iki.fi.
+#endif
+
+#if defined(STACK_GROWS_UP)
+static inline void *stack_pointer(void *base, size_t size)
+{
+	return base + size;
+}
+
+static inline void *stack_guard(void *base, size_t size)
+{
+	return base;
+}
+
+static inline void *stack_push(void **stackptr, size_t size)
+{
+	(*stackptr) -= size;
+	return (*stackptr);
+}
+#else
+#error Stack growing direction undefined.
+#endif
+
+static inline void stack_push_ptr(void **stackptr, void *ptr)
+{
+	*(void**) stack_push(stackptr, sizeof(ptr)) = ptr;
+}
+
+
+static inline
+struct tf_fiber *tf_uctx_create(tf_fiber_proc fiber_main, int private_size)
+{
+	size_t size = TF_STACK_SIZE;
+	struct tf_uctx *uctx;
+	void *stack, *stack_base;
+
+	/* Allocate new stack */
+	stack_base = malloc(size);
+	if (stack_base == NULL)
+		return NULL;
+
+	stack = stack_pointer(stack_base, size);
+	private_size += sizeof(struct tf_uctx);
+
+	/* Construct inital frame for call the main function and if it
+	 * happens to return, it'll jump back to tf_exit() which kills
+	 * the fiber (cdecl calling convetion assumed) */
+	uctx = stack_push(&stack, TF_ALIGN(private_size, 64));
+	stack_push_ptr(&stack, uctx->fiber.data);
+	stack_push_ptr(&stack, &tf_exit);
+	stack_push_ptr(&stack, fiber_main);
+	uctx->current_sp = stack;
+
+#ifdef VALGRIND
+	uctx->stack_id = VALGRIND_STACK_REGISTER(stack_base, size);
+#endif
+	uctx->alloc = stack_base;
+	uctx->stack_guard = stack_guard(stack_base, size);
+	*uctx->stack_guard = STACK_GUARD;
+
+	return &uctx->fiber;
+}
+
+static inline
+void tf_uctx_destroy(struct tf_fiber *fiber)
+{
+	struct tf_uctx *uctx = container_of(fiber, struct tf_uctx, fiber);
+#ifdef VALGRIND
+	VALGRIND_STACK_DEREGISTER(uctx->stack_id);
+#endif
+	free(uctx->alloc);
+}
+
+#define switch_fiber(oldspptr, newsp)		\
+	__asm__ __volatile__ (			\
+		"push	%%ebp		;"	\
+		"call	.+5		;"	\
+		"add	$0x9,(%%esp)	;"	\
+		"mov	%%esp, (%0)	;"	\
+		"mov	%1, %%esp	;"	\
+		"ret			;"	\
+		"pop	%%ebp"			\
+		:				\
+		: "a"(oldspptr), "d"(newsp)	\
+		: "%ebx", "%ecx", "%esi", "%edi", "memory", "cc");
+
+static inline
+void tf_uctx_transfer(struct tf_fiber *from, struct tf_fiber *to)
+{
+
+	struct tf_uctx *ufrom = container_of(from, struct tf_uctx, fiber);
+	struct tf_uctx *uto   = container_of(to,   struct tf_uctx, fiber);
+
+	/* Switch stack pointers */
+	TF_BUG_ON(*ufrom->stack_guard != STACK_GUARD);
+	switch_fiber(&ufrom->current_sp, uto->current_sp);
+}