libtf: fix stack frame creation and valgrind issues

debugging looks now better. there was also some valgrind issues
that needed attention.

2 files changed, 40 insertions, 42 deletions

diff --git a/src/fiber.c b/src/fiber.c
index 3f58b6d..e507815 100644
--- a/src/fiber.c
+++ b/src/fiber.c
@@ -39,6 +39,15 @@ struct tf_fiber *tf_fiber_get_current(void)
 	return container_of(data, struct tf_fiber, data);
 }
 
+static void tf_fiber_main(void *user_data, void *arg)
+{
+	tf_fiber_proc proc = (tf_fiber_proc) arg;
+	struct tf_fiber *f = user_data;
+
+	proc(f->data);
+	tf_fiber_exit();
+}
+
 void *__tf_fiber_create(tf_fiber_proc fiber_main, int private_size)
 {
 	struct tf_fiber *fiber;
@@ -47,8 +56,7 @@ void *__tf_fiber_create(tf_fiber_proc fiber_main, int private_size)
 		TF_STACK_SIZE,
 		sizeof(struct tf_fiber) + private_size,
 		offsetof(struct tf_fiber, context),
-		fiber_main, offsetof(struct tf_fiber, data),
-		tf_fiber_exit);
+		tf_fiber_main, fiber_main);
 	if (fiber == NULL)
 		return NULL;
 
@@ -61,15 +69,12 @@ void *__tf_fiber_create(tf_fiber_proc fiber_main, int private_size)
 	return fiber->data;
 }
 
-void *tf_fiber_create(
-	struct tf_scheduler *sched,
-	tf_fiber_proc fiber_main, int private_size)
+void *tf_fiber_create(tf_fiber_proc fiber_main, int private_size)
 {
 	struct tf_fiber *fiber;
+	struct tf_scheduler *sched;
 
-	if (sched == NULL)
-		sched = tf_scheduler_get_current();
-
+	sched = tf_scheduler_get_current();
 	if (tf_heap_prealloc(&sched->heap, sched->num_fibers + 1) < 0)
 		return NULL;
 
@@ -94,15 +99,19 @@ void *tf_fiber_get(void *data)
 static void __tf_fiber_destroy(struct tf_fiber *fiber)
 {
 	struct tf_scheduler *sched = fiber->scheduler;
-	int main_fiber;
+	int main_fiber, num_fibers;
+
+	/* decrease first the number of fibers as we might be
+	 * killing the scheduler it self */
+	num_fibers = --sched->num_fibers;
 
 	main_fiber = (fiber->context.alloc == NULL);
 	tf_heap_delete(&fiber->heap_node, &sched->heap);
 	tf_uctx_destroy(&fiber->context);
 	if (main_fiber)
 		free(fiber);
-	sched->num_fibers--;
-	if (sched->num_fibers == 1) {
+
+	if (num_fibers == 1) {
 		/* FIXME: Use proper fiber event*/
 		__tf_fiber_wakeup(sched->main_fiber, TF_WAKEUP_IMMEDIATE);
 	}
@@ -132,28 +141,10 @@ void __tf_fiber_wakeup_heapnode(struct tf_heap_node *node)
 			  TF_WAKEUP_TIMEOUT);
 }
 
-static void __tf_fiber_schedule_next(void)
-{
-	struct tf_scheduler *sched = tf_scheduler_get_current();
-	struct tf_fiber *f = tf_fiber_get_current();
-	struct tf_fiber *nf;
-
-	/* Figure out the next fibre to run */
-	if (unlikely(tf_list_empty(&sched->scheduled_q))) {
-		tf_list_splice_tail(&sched->running_q,
-				    &sched->scheduled_q);
-		TF_BUG_ON(tf_list_empty(&sched->scheduled_q));
-	}
-	nf = tf_list_entry(tf_list_pop(&sched->scheduled_q),
-			   struct tf_fiber, queue_node);
-	sched->active_fiber = nf->data;
-	tf_uctx_transfer(&f->context, &nf->context);
-}
-
 int __tf_fiber_schedule(void)
 {
 	struct tf_scheduler *sched = tf_scheduler_get_current();
-	struct tf_fiber *f = tf_fiber_get_current();
+	struct tf_fiber *f = tf_fiber_get_current(), *nf;
 	int wakeup;
 
 	if (unlikely(f->timeout_change)) {
@@ -168,7 +159,16 @@ int __tf_fiber_schedule(void)
 		f->timeout_change = 0;
 	}
 
-	__tf_fiber_schedule_next();
+	/* Figure out the next fibre to run */
+	if (unlikely(tf_list_empty(&sched->scheduled_q))) {
+		tf_list_splice_tail(&sched->running_q,
+				    &sched->scheduled_q);
+		TF_BUG_ON(tf_list_empty(&sched->scheduled_q));
+	}
+	nf = tf_list_entry(tf_list_pop(&sched->scheduled_q),
+			   struct tf_fiber, queue_node);
+	sched->active_fiber = nf->data;
+	tf_uctx_transfer(&f->context, &nf->context);
 
 	wakeup = f->wakeup_type;
 	f->wakeup_type = TF_WAKEUP_NONE;
diff --git a/src/uctx.h b/src/uctx.h
index e27a53c..5eca6be 100644
--- a/src/uctx.h
+++ b/src/uctx.h
@@ -37,15 +37,15 @@ struct tf_uctx {
 	do {						\
 		unsigned eax, ebx, ecx, edx, esi, edi;	\
 		__asm__ __volatile__ (			\
-		"push	%%ebp				\n" \
 		"call	1f				\n" \
 		"1:					\n" \
 		"addl	$2f-1b, (%%esp)			\n" \
+		"push	%%ebp				\n" \
 		"movl	%%esp, %[prev_sp]		\n" \
 		"movl	%[next_sp], %%esp		\n" \
+		"pop	%%ebp				\n" \
 		"ret					\n" \
 		"2:					\n" \
-		"pop	%%ebp				\n" \
 		: [prev_sp] "=m"(prev->current_sp),	\
 		  "=a"(eax), "=b"(ebx), "=c"(ecx),	\
 		  "=d"(edx), "=S"(esi), "=D"(edi)	\
@@ -101,8 +101,8 @@ tf_uctx_create_embedded(
 	size_t stack_size,
 	size_t private_size,
 	off_t uctx_offset,
-	void (*stack_frame_main)(void*), off_t main_argument_offset,
-	void (*stack_frame_return)(void))
+	void (*stack_frame_main)(void*, void*),
+	void *main_argument)
 {
 	size_t size = TF_STACK_SIZE;
 	void *user_data;
@@ -117,20 +117,18 @@ tf_uctx_create_embedded(
 	/* Create initial stack frame (cdecl convention) */
 	stack = stack_pointer(stack_base, size);
 	user_data = stack_push(&stack, TF_ALIGN(private_size, 64));
+	stack_push_ptr(&stack, main_argument);
+	stack_push_ptr(&stack, user_data);
 	stack_push_ptr(&stack, NULL);
-	stack_push_ptr(&stack, NULL);
-	stack_push_ptr(&stack, NULL);
-	stack_push_ptr(&stack, NULL);
-	stack_push_ptr(&stack, user_data + main_argument_offset);
-	stack_push_ptr(&stack, stack_frame_return);
-	stack_push_ptr(&stack, stack_frame_main);
+	stack_push_ptr(&stack, stack_frame_main);	/* eip */
+	stack_push_ptr(&stack, NULL);			/* ebp */
 
 	uctx = user_data + uctx_offset;
 	*uctx = (struct tf_uctx) {
 		.stack_guard = stack_guard(stack_base, size),
 		.alloc = stack_base,
 		.current_sp = stack,
-		.stack_id = VALGRIND_STACK_REGISTER(stack_base, size),
+		.stack_id = VALGRIND_STACK_REGISTER(stack_base, stack_base+size),
 	};
 	*uctx->stack_guard = STACK_GUARD;