libtf: major redesign startedHEADmaster

the idea is to make libtf completely multi-threaded. meaning each
fiber can be running concurrently in separate thread. quite a bit
of framework is added for this and some atomic helpers are already
introduced. however, io polling is busy polling now (will be soon
in own thread) and timeouts are still more or less broken. oh, and
the multithreading core is not there yet. basically we are currently
mostly broken ;)

7 files changed, 179 insertions, 128 deletions

diff --git a/include/libtf/atomic.h b/include/libtf/atomic.h
index ec9f1e0..8ebe5f8 100644
--- a/include/libtf/atomic.h
+++ b/include/libtf/atomic.h
@@ -13,7 +13,14 @@
 #ifndef TF_ATOMIC_H
 #define TF_ATOMIC_H
 
-#define tf_atomic_inc(var) __sync_add_and_fetch(&(var), 1)
-#define tf_atomic_dec(var) __sync_add_and_fetch(&(var), -1)
+#define tf_atomic_inc(var) \
+	__sync_add_and_fetch(&(var), 1)
+#define tf_atomic_dec(var) \
+	__sync_add_and_fetch(&(var), -1)
+
+#define tf_atomic_cmpxchg(ptr, old, new) \
+	__sync_bool_compare_and_swap(ptr, old, new)
+#define tf_atomic_xchg(ptr, new) \
+	((typeof(*(ptr)))__sync_lock_test_and_set(ptr, new))
 
 #endif
diff --git a/include/libtf/defines.h b/include/libtf/defines.h
index 8e39c7e..44ead8a 100644
--- a/include/libtf/defines.h
+++ b/include/libtf/defines.h
@@ -57,6 +57,7 @@
 #define attribute_warn_unused_result	__attribute__((warn_unused_result))
 #define attribute_deprecated		__attribute__((deprecated))
 
+/* FIXME: glibc fprintf requires 8kB on stack */
 #define TF_BUG_ON(cond) if (unlikely(cond)) { \
 	fprintf(stderr, "BUG: failure at %s:%d/%s(): %s!\n",	\
 		__FILE__, __LINE__, __func__, #cond);		\
@@ -68,14 +69,19 @@
 
 #define TF_EMPTY_ARRAY 0
 
+#define TF_BIT(n)		(1 << (n))
+
 #ifndef TF_STACK_SIZE
-#define TF_STACK_SIZE		4096
+/* FIXME: glibc fprintf requires 8kB on stack */
+#define TF_STACK_SIZE		(4*4096)
 #endif
 
 /* Monotonic time */
 typedef uint32_t tf_mtime_t;
 typedef int32_t tf_mtime_diff_t;
 
+tf_mtime_t tf_mtime_now(void);
+
 static inline
 tf_mtime_diff_t tf_mtime_diff(tf_mtime_t a, tf_mtime_t b)
 {
diff --git a/include/libtf/fiber.h b/include/libtf/fiber.h
index d5c6153..f97e963 100644
--- a/include/libtf/fiber.h
+++ b/include/libtf/fiber.h
@@ -16,64 +16,127 @@
 #include <errno.h>
 #include <libtf/defines.h>
 #include <libtf/heap.h>
+#include <libtf/list.h>
 
-/* Fiber wakeup reasons */
-#define TF_WAKEUP_NONE		0
-#define TF_WAKEUP_IMMEDIATE	-EAGAIN
-#define TF_WAKEUP_KILL		-EINTR
-#define TF_WAKEUP_TIMEOUT	-ETIME
-#define TF_WAKEUP_THIS_TIMEOUT	-ETIMEDOUT
-#define TF_WAKEUP_FD		-EIO
+/* Inter-fibre calls */
+struct tf_fiber;
+struct tf_ifc;
+typedef void (*tf_ifc_handler_t)(void *fiber, struct tf_ifc *msg);
 
-/* Fiber management */
-struct tf_scheduler;
-typedef void (*tf_fiber_proc)(void *fiber);
+struct tf_ifc {
+	union {
+		struct tf_ifc		*next;
+		struct tf_list_node	list;
+	};
+	tf_ifc_handler_t		handler;
+	struct tf_fiber *		sender;
+};
 
-void *__tf_fiber_create(tf_fiber_proc fiber_main, int private_size);
-void *tf_fiber_create(tf_fiber_proc fiber_main, int private_size);
-void *tf_fiber_get(void *data);
-void tf_fiber_put(void *data);
-void __tf_fiber_wakeup(void *data, int wakeup_type);
-void __tf_fiber_wakeup_heapnode(struct tf_heap_node *node);
-int  __tf_fiber_schedule(void);
-int  __tf_fiber_bind_scheduler(struct tf_scheduler *sched);
-int  __tf_fiber_release_scheduler(struct tf_scheduler *sched);
+void tf_ifc_queue(void *fiber, struct tf_ifc *msg, tf_ifc_handler_t handler);
+void tf_ifc_complete(void *fiber, struct tf_ifc *msg, tf_ifc_handler_t handler);
+void tf_ifc_process(void);
+void tf_ifc_process_unordered(void);
 
-void tf_fiber_exit(void) attribute_noreturn;
-void tf_fiber_kill(void *fiber);
-int  tf_fiber_yield(void);
+/* Timeouts */
+struct tf_timeout_manager {
+	struct tf_heap_head		heap;
+	unsigned int			num_fibers;
+};
+
+struct tf_timeout_client {
+	struct tf_timeout_manager *	manager;
+	tf_mtime_t			value;
+	tf_mtime_t			latched;
+	struct tf_heap_node		heap_node;
+	struct tf_ifc			ifc;
+};
 
-/* Scheduling and fiber management */
 struct tf_timeout {
-	tf_mtime_t	saved_timeout;
-	unsigned int	timeout_change;
+	tf_mtime_t			saved_timeout;
+	tf_mtime_t			my_timeout;
 };
 
-#define tf_timed(func, timeout)						\
+static inline int tf_timeout_expired(struct tf_timeout *to)
+{
+	return tf_mtime_diff(to->my_timeout, tf_mtime_now()) <= 0;
+}
+
+struct tf_timeout_fiber *tf_timeout_fiber_create(void);
+void tf_timeout_adjust(struct tf_timeout_client *tc);
+void tf_timeout_delete(struct tf_timeout_client *tc);
+
+#define tf_timed(__to, func, timeout)					\
 	({								\
-		struct tf_timeout __timeout;				\
-		tf_timeout_push(&__timeout, timeout);			\
-		tf_timeout_pop(&__timeout, (func));			\
+		tf_timeout_push(__to, timeout);				\
+		tf_timeout_pop(__to, (func));				\
 	})
 
-void tf_timeout_push(struct tf_timeout *timeout, tf_mtime_diff_t milliseconds);
-int __tf_timeout_pop(struct tf_timeout *timeout, int err);
+/* Fibres and their management */
+typedef void *tf_uctx_t;
+
+struct tf_fiber {
+	unsigned int		ref_count;
+	unsigned int		flags;
+	tf_uctx_t		context;
+	tf_uctx_t		return_context;
+	struct tf_list_node	queue_node;
+	struct tf_list_head	wakeup_q;
+	struct tf_timeout_client timeout;
+	struct tf_ifc *		pending_ifc;
+	char			data[TF_EMPTY_ARRAY];
+};
+
+typedef void (*tf_fiber_proc)(void *fiber);
+
+void *__tf_fiber_create(tf_fiber_proc fiber_main, int private_size);
+void *tf_fiber_create(tf_fiber_proc fiber_main, int private_size);
+void *tf_fiber_get(void *fiber);
+void tf_fiber_put(void *fiber);
+int tf_fiber_run(void *fiber);
+void tf_fiber_kill(void *fiber);
+
+void tf_fiber_wakeup(struct tf_fiber *fiber);
+int  tf_fiber_schedule(void);
 
-static inline int tf_timeout_pop(struct tf_timeout *timeout, int err)
+void tf_fiber_exit(void) attribute_noreturn;
+
+static inline struct tf_fiber *tf_vmach_get_current_fiber(void)
 {
-	if (unlikely(timeout->timeout_change))
-		return __tf_timeout_pop(timeout, err);
+	extern __thread struct tf_fiber *tf_current_fiber;
+	return tf_current_fiber;
+}
+
+static inline void tf_timeout_push(struct tf_timeout *to, tf_mtime_diff_t ms)
+{
+	struct tf_fiber *f = tf_vmach_get_current_fiber();
+
+	to->saved_timeout = f->timeout.value;
+	to->my_timeout = tf_mtime_now() + ms;
+	if (f->timeout.value == 0 ||
+	    tf_mtime_diff(to->my_timeout, f->timeout.value) < 0)
+		f->timeout.value = to->my_timeout;
+}
+
+static inline int tf_timeout_pop(struct tf_timeout *to, int err)
+{
+	struct tf_fiber *f = tf_vmach_get_current_fiber();
+
+	f->timeout.value = to->saved_timeout;
 	return err;
 }
 
 static inline
-int tf_msleep(tf_mtime_diff_t milliseconds)
+int tf_msleep(tf_mtime_diff_t ms)
 {
+	struct tf_timeout to;
 	int r;
-	r = tf_timed(__tf_fiber_schedule(), milliseconds);
-	if (r == TF_WAKEUP_THIS_TIMEOUT)
-		r = 0;
-	return r;
+
+	tf_timeout_push(&to, ms);
+	do {
+		r = tf_fiber_schedule();
+	} while (r != -ETIME);
+
+	return tf_timeout_pop(&to, r);
 }
 
 #endif
diff --git a/include/libtf/io.h b/include/libtf/io.h
index 0d34421..d1098e3 100644
--- a/include/libtf/io.h
+++ b/include/libtf/io.h
@@ -21,10 +21,12 @@
 #include <libtf/defines.h>
 
 /* Flags for tf_open_fd() */
-#define TF_FD_AUTOCLOSE			1
-#define TF_FD_STREAM_ORIENTED		2
-#define TF_FD_SET_CLOEXEC		4
-#define TF_FD_ALREADY_NONBLOCKING	8
+#define TF_FD_READ			TF_BIT(0)
+#define TF_FD_WRITE			TF_BIT(1)
+#define TF_FD_AUTOCLOSE			TF_BIT(2)
+#define TF_FD_STREAM_ORIENTED		TF_BIT(3)
+#define TF_FD_SET_CLOEXEC		TF_BIT(4)
+#define TF_FD_ALREADY_NONBLOCKING	TF_BIT(5)
 
 struct tf_sockaddr {
 	union {
@@ -37,23 +39,14 @@ struct tf_sockaddr {
 struct tf_fd {
 	int fd;
 	unsigned int flags;
-	/* Single waiter -- would be relatively trivial to modify to allow
-	 * multiple waiters, if someone actually needs it */
-	unsigned int events;
-	void *waiting_fiber;
+	struct tf_fiber *fiber;
 };
 
-#define TF_POLL_READ	1
-#define TF_POLL_WRITE	2
-
 struct tf_poll_hooks {
-	void (*init)(void);
-	int  (*poll)(tf_mtime_diff_t timeout);
-	void (*close)(void);
-	int  (*fd_created)(struct tf_fd *fd);
-	int  (*fd_destroyed)(struct tf_fd *fd);
-	void (*fd_monitor)(struct tf_fd *fd, int events);
-	void (*fd_unmonitor)(struct tf_fd *fd);
+	void *	(*create)(void);
+	int	(*fd_created)(void *fiber, struct tf_fd *fd);
+	int	(*fd_destroyed)(void *fiber, struct tf_fd *fd);
+	void	(*fd_rearm)(void *fiber, struct tf_fd *fd);
 };
 
 int tf_open_fd(struct tf_fd *fd, int kfd, int flags);
diff --git a/include/libtf/scheduler.h b/include/libtf/scheduler.h
deleted file mode 100644
index db5a823..0000000
--- a/include/libtf/scheduler.h
+++ /dev/null
@@ -1,67 +0,0 @@
-/* scheduler.h - libtf fiber scheduler header
- *
- * Copyright (C) 2009-2010 Timo Teräs <timo.teras@iki.fi>
- * All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 or later as
- * published by the Free Software Foundation.
- *
- * See http://www.gnu.org/ for details.
- */
-
-#ifndef TF_SCHEDULER_H
-#define TF_SCHEDULER_H
-
-#include <libtf/atomic.h>
-#include <libtf/list.h>
-#include <libtf/heap.h>
-#include <libtf/fiber.h>
-
-struct tf_poll_hooks;
-
-struct tf_scheduler {
-	struct tf_list_head	scheduled_q;
-	struct tf_list_head	running_q;
-	struct tf_heap_head	heap;
-	void *			active_fiber;
-	void *			main_fiber;
-	int			num_fibers;
-	tf_mtime_t		scheduler_time;
-	struct tf_poll_hooks *	poller;
-	unsigned long		poll_data[2];
-};
-
-static inline
-struct tf_scheduler *tf_scheduler_get_current(void)
-{
-	extern struct tf_scheduler *__tf_scheduler;
-	TF_BUG_ON(__tf_scheduler == NULL);
-	return __tf_scheduler;
-}
-
-static inline
-tf_mtime_t tf_scheduler_get_mtime(void)
-{
-	return tf_scheduler_get_current()->scheduler_time;
-}
-
-struct tf_scheduler *tf_scheduler_create(void);
-int  tf_scheduler_enable(struct tf_scheduler *);
-void tf_scheduler_disable(void);
-
-static inline struct tf_scheduler *
-tf_scheduler_get(struct tf_scheduler *s)
-{
-	tf_fiber_get(s);
-	return s;
-}
-
-static inline void
-tf_scheduler_put(struct tf_scheduler *s)
-{
-	tf_fiber_put(s);
-}
-
-#endif
-
diff --git a/include/libtf/tf.h b/include/libtf/tf.h
index e613f18..d052ee7 100644
--- a/include/libtf/tf.h
+++ b/include/libtf/tf.h
@@ -14,7 +14,7 @@
 #define TF_H
 
 #include <libtf/fiber.h>
-#include <libtf/scheduler.h>
+#include <libtf/vmach.h>
 #include <libtf/io.h>
 
 #endif
diff --git a/include/libtf/vmach.h b/include/libtf/vmach.h
new file mode 100644
index 0000000..d302366
--- /dev/null
+++ b/include/libtf/vmach.h
@@ -0,0 +1,49 @@
+/* vmach.h - "virtual" machine and cpu contexts
+ *
+ * Copyright (C) 2009 Timo Teräs <timo.teras@iki.fi>
+ * All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 or later as
+ * published by the Free Software Foundation.
+ *
+ * See http://www.gnu.org/ for details.
+ */
+
+#ifndef TF_VMACH_H
+#define TF_VMACH_H
+
+#include <libtf/fiber.h>
+
+struct tf_vmach {
+	struct tf_poll_hooks *		poll_ops;
+	void *				poll_fiber;
+	void *				timeout_fiber;
+	struct tf_fiber			startup_fiber;
+	int				num_user_fibers;
+	struct tf_list_head		run_q;
+};
+
+struct tf_vcpu {
+	struct tf_vmach *		machine;
+};
+
+static inline struct tf_vmach *tf_vmach_get_current(void)
+{
+	extern __thread struct tf_vmach *tf_current_vmach;
+	return tf_current_vmach;
+}
+
+static inline struct tf_vcpu *tf_vmach_get_current_cpu(void)
+{
+	extern __thread struct tf_vcpu *tf_current_vcpu;
+	return tf_current_vcpu;
+}
+
+void tf_vmach_start(void);
+void tf_vmach_stop(void);
+
+void tf_vmach_run(struct tf_vmach *vm, struct tf_fiber *f);
+void tf_vmach_run_dedicated(struct tf_vmach *vm, struct tf_fiber *f);
+
+#endif