Fixes to get 3 threads working. Fixes to get threaded command working.

Removed some of the temporary debuggery in qtimers that only worked in 1
thread.  Initialised the mqueue mutex.  Stopped peer_index from
zeroizing its mutex after it had been initialized.

9 files changed, 57 insertions, 32 deletions

diff --git a/bgpd/bgp_main.c b/bgpd/bgp_main.c
index ac7d959d..45ea0f59 100644
--- a/bgpd/bgp_main.c
+++ b/bgpd/bgp_main.c
@@ -348,7 +348,8 @@ bgp_exit (int status)
   qexit (status);
 }
 
-/* threaded command*/
+/* Threaded command.  If present must be the first command in the
+ * configuration file.  If not the first command it will log and abort. */
 DEFUN_HID_CALL (threaded,
        threaded_cmd,
        "threaded",
@@ -360,17 +361,41 @@ DEFUN_HID_CALL (threaded,
   return CMD_SUCCESS;
 }
 
+/* If neither the command line nor the first command enabled pthreads
+ * then disable pthreads.  This is a call back after processing the
+ * first command in the configuration file
+ */
+static void after_first_cmd()
+{
+  if (!qpthreads_enabled)
+    init_second_stage(0);
+}
+
+/* Enable or disables pthreads.  Create the nexus(es). Perform
+ * any post nexus creation initialization. The nexus(es) need
+ * to be created as soon as we know the pthread state so that
+ * the message queues are available for the configuration data.
+ */
 static void
 init_second_stage(int pthreads)
 {
   qlib_init_second_stage(pthreads);
   bgp_peer_index_mutex_init();
 
+  /* Make nexus for main thread, always needed */
+  cli_nexus = qpn_init_new(cli_nexus, 1); /* main thread */
+
   /* if using pthreads create additional nexus */
   if (qpthreads_enabled)
     {
-      bgp_nexus = qpn_init_new(cli_nexus, 0);
-      routing_nexus = qpn_init_new(cli_nexus, 0);
+      bgp_nexus = qpn_init_new(bgp_nexus, 0);
+      routing_nexus = qpn_init_new(routing_nexus, 0);
+    }
+  else
+    {
+      /* we all share the single nexus and single thread */
+      bgp_nexus = cli_nexus;
+      routing_nexus = cli_nexus;
     }
 
   /* Nexus hooks.
@@ -417,11 +442,6 @@ main (int argc, char **argv)
   zlog_default = openzlog (progname, ZLOG_BGP,
 			   LOG_CONS|LOG_NDELAY|LOG_PID, LOG_DAEMON);
 
-  /* Make nexus for main thread, always needed */
-  cli_nexus = qpn_init_new(cli_nexus, 1); /* main thread */
-  bgp_nexus = cli_nexus;        /* use main thread for now */
-  routing_nexus = cli_nexus;    /* use main thread for now */
-
   /* BGP master init. */
   bgp_master_init ();
 
@@ -522,10 +542,10 @@ main (int argc, char **argv)
   sort_node ();
 
   /* Parse config file. */
-  vty_read_config (config_file, config_default);
+  vty_read_config_first_cmd_special (config_file, config_default, after_first_cmd);
 
   /* Start execution only if not in dry-run mode */
-  if(dryrun)
+  if (dryrun)
     return(0);
 
   /* only the calling thread survives in the child after a fork
@@ -543,10 +563,6 @@ main (int argc, char **argv)
   /* Process ID file creation. */
   pid_output (pid_file);
 
-  /* stage 2 initialisation, if not already done */
-  if (!qpthreads_enabled)
-    init_second_stage(0);
-
   /* Make bgp vty socket. */
   vty_serv_sock (vty_addr, vty_port, BGP_VTYSH_PATH);
 
diff --git a/bgpd/bgp_peer_index.c b/bgpd/bgp_peer_index.c
index 87b761df..39b4f69e 100644
--- a/bgpd/bgp_peer_index.c
+++ b/bgpd/bgp_peer_index.c
@@ -111,8 +111,6 @@ bgp_peer_index_init(void* parent)
 
   bgp_peer_id_count = 0 ;
 
-  /* OK up to creation of qpthreads !                                   */
-  bgp_peer_index_mutex  = NULL ;
 } ;
 
 /*------------------------------------------------------------------------------
diff --git a/lib/command.c b/lib/command.c
index 574e45bf..59cbbe52 100644
--- a/lib/command.c
+++ b/lib/command.c
@@ -2298,10 +2298,11 @@ cmd_execute_command_strict (vector vline, struct vty *vty,
 
 /* Configration make from file. */
 int
-config_from_file (struct vty *vty, FILE *fp)
+config_from_file (struct vty *vty, FILE *fp, void (*after_first_cmd)(void))
 {
   int ret;
   vector vline;
+  int first_cmd = 1;
 
   while (fgets (vty->buf, VTY_BUFSIZ, fp))
     {
@@ -2313,6 +2314,13 @@ config_from_file (struct vty *vty, FILE *fp)
       /* Execute configuration command : this is strict match */
       ret = cmd_execute_command_strict (vline, vty, NULL);
 
+      /* special handling for after the first command */
+      if (first_cmd && after_first_cmd)
+        {
+          after_first_cmd();
+          first_cmd = 0;
+        }
+
       /* Try again with setting node to CONFIG_NODE */
       while (ret != CMD_SUCCESS && ret != CMD_WARNING
 	     && ret != CMD_ERR_NOTHING_TODO && vty_get_node(vty) != CONFIG_NODE)
diff --git a/lib/command.h b/lib/command.h
index c0d79f94..b1cf18a4 100644
--- a/lib/command.h
+++ b/lib/command.h
@@ -356,7 +356,7 @@ extern void cmd_free_strvec (vector);
 extern vector cmd_describe_command (vector, int, int *status);
 extern char **cmd_complete_command (vector, int, int *status);
 extern const char *cmd_prompt (enum node_type);
-extern int config_from_file (struct vty *, FILE *);
+extern int config_from_file (struct vty *, FILE *, void (*)(void));
 extern enum node_type node_parent (enum node_type);
 extern int cmd_execute_command (vector, struct vty *, struct cmd_element **,
     qpn_nexus, int);
diff --git a/lib/qlib_init.c b/lib/qlib_init.c
index e6bce633..67391395 100644
--- a/lib/qlib_init.c
+++ b/lib/qlib_init.c
@@ -25,6 +25,7 @@
 #include "qpthreads.h"
 #include "thread.h"
 #include "privs.h"
+#include "mqueue.h"
 
 /*==============================================================================
  * Quagga Library Initialise/Closedown
@@ -75,6 +76,7 @@ qlib_init_second_stage(int pthreads)
   memory_init_r();
   thread_init_r();
   zprivs_init_r();
+  mqueue_initialise();
 }
 
 
diff --git a/lib/qpnexus.c b/lib/qpnexus.c
index a350a274..731190b6 100644
--- a/lib/qpnexus.c
+++ b/lib/qpnexus.c
@@ -134,9 +134,6 @@ qpn_exec(qpn_nexus qpn)
  *
  *   5) Timers -- qtimers
  *
- *   6) Legacy threads.  To deal with legacy timer mechanism.
- *
- *
  */
 static void*
 qpn_start(void* arg)
diff --git a/lib/qtimers.c b/lib/qtimers.c
index 6c6c1406..dcce24b9 100644
--- a/lib/qtimers.c
+++ b/lib/qtimers.c
@@ -87,9 +87,6 @@ qtimer_cmp(qtimer* a, qtimer* b)        /* the heap discipline  */
   return 0 ;
 } ;
 
-/* Kill this */
-qtimer_pile our_pile;
-
 /*==============================================================================
  * qtimer_pile handling
  */
@@ -116,9 +113,6 @@ qtimer_pile_init_new(qtimer_pile qtp)
 
   heap_init_new_backlinked(&qtp->timers, 0, (heap_cmp*)qtimer_cmp,
                                                  offsetof(qtimer_t, backlink)) ;
-
-  /* TODO: kill this */
-  our_pile = qtp;
   return qtp ;
 } ;
 
@@ -322,7 +316,6 @@ qtimer_set(qtimer qtr, qtime_mono_t when, qtimer_action* action)
 
   qtp = qtr->pile ;
   dassert(qtp != NULL) ;
-  assert(qtp == our_pile);
   qtimer_pile_verify(qtp) ;     /* TODO: remove after debuggery */
 
   qtr->time = when ;
@@ -356,7 +349,6 @@ qtimer_unset(qtimer qtr)
       qtimer_pile qtp = qtr->pile ;
       dassert(qtp != NULL) ;
 
-      assert(qtp == our_pile);
       qtimer_pile_verify(qtp) ;     /* TODO: remove after debuggery */
 
       heap_delete_item(&qtp->timers, qtr) ;
diff --git a/lib/vty.c b/lib/vty.c
index 4797f923..e651e445 100644
--- a/lib/vty.c
+++ b/lib/vty.c
@@ -2649,7 +2649,7 @@ uty_timeout (struct vty *vty)
 
 /* Read up configuration file from file_name. */
 static void
-vty_read_file (FILE *confp)
+vty_read_file (FILE *confp, void (*after_first_cmd)(void))
 {
   int ret;
   struct vty *vty;
@@ -2658,7 +2658,7 @@ vty_read_file (FILE *confp)
   vty->node = CONFIG_NODE;
 
   /* Execute configuration file */
-  ret = config_from_file (vty, confp);
+  ret = config_from_file (vty, confp, after_first_cmd);
 
   LOCK
 
@@ -2752,6 +2752,15 @@ void
 vty_read_config (char *config_file,
                  char *config_default_dir)
 {
+  vty_read_config_first_cmd_special(config_file, config_default_dir, NULL);
+}
+
+/* Read up configuration file from file_name.
+ * callback after first command */
+void
+vty_read_config_first_cmd_special(char *config_file,
+                 char *config_default_dir, void (*after_first_cmd)(void))
+{
   char cwd[MAXPATHLEN];
   FILE *confp = NULL;
   char *fullpath;
@@ -2840,7 +2849,7 @@ vty_read_config (char *config_file,
         fullpath = config_default_dir;
     }
 
-  vty_read_file (confp);
+  vty_read_file (confp, after_first_cmd);
 
   fclose (confp);
 
@@ -3087,6 +3096,8 @@ vty_event_r (enum event event, int sock, struct vty *vty)
       qps_enable_mode(vty->qf, qps_write_mnum, vty_flush_r) ;
       break;
     case VTY_TIMEOUT_RESET:
+      if (vty->qtr == NULL)
+        break;
       if (vty->v_timeout)
         {
           qtimer_set(vty->qtr, qt_add_monotonic(QTIME(vty->v_timeout)), NULL) ;
diff --git a/lib/vty.h b/lib/vty.h
index ea5df68f..ccb325d1 100644
--- a/lib/vty.h
+++ b/lib/vty.h
@@ -231,6 +231,7 @@ extern int vty_puts(struct vty* vty, const char* str) ;
 extern int vty_out_newline(struct vty *vty) ;
 extern int vty_out_indent(struct vty *vty, int indent) ;
 extern void vty_read_config (char *, char *);
+extern void vty_read_config_first_cmd_special (char *, char *, void (*)(void));
 extern void vty_time_print (struct vty *, int);
 extern void vty_serv_sock (const char *, unsigned short, const char *);
 extern void vty_close (struct vty *);