Fixed problems in mqueue keeping tail pointers correct. Implemented

program terminate code that waits for all sissions to become disabled
before terminating pthreads and running exit code.

6 files changed, 80 insertions, 17 deletions

diff --git a/bgpd/bgp_main.c b/bgpd/bgp_main.c
index 88068017..2916c70f 100644
--- a/bgpd/bgp_main.c
+++ b/bgpd/bgp_main.c
@@ -216,21 +216,25 @@ sigint (void)
 #endif
   zlog_notice ("Terminating on signal");
 
-  /* tell the routing engine */
-  sigterm_enqueue();
-
-  /* TODO: very temporary kludge to test if bgp engine does close */
-  sleep(20);
-
-  /* ask remaining pthreads to die */
-  if (qpthreads_enabled && routing_nexus != NULL)
+  if (!retain_mode)
+    {
+      /* tell the routing engine to send notifies to peers and wait
+       * for all sessions to be disabled */
+      sigterm_enqueue();
+    }
+  else
+    {
+      /* ask remaining pthreads to die */
+      if (qpthreads_enabled && routing_nexus != NULL)
         qpn_terminate(routing_nexus);
 
-  if (qpthreads_enabled && bgp_nexus != NULL)
+      if (qpthreads_enabled && bgp_nexus != NULL)
         qpn_terminate(bgp_nexus);
 
-  if (cli_nexus != NULL)
-      qpn_terminate(cli_nexus);
+      if (cli_nexus != NULL)
+        qpn_terminate(cli_nexus);
+    }
+
 }
 
 /* SIGUSR1 handler. */
@@ -644,7 +648,7 @@ sighup_action(mqueue_block mqb, mqb_flag_t flag)
 {
   if (flag == mqb_action)
     {
-      bgp_terminate ();
+      bgp_terminate (0); /* send notfies */
       bgp_reset ();
     }
 
@@ -665,9 +669,9 @@ sigterm_action(mqueue_block mqb, mqb_flag_t flag)
 {
   if (flag == mqb_action)
     {
-      /* send notify to all peers, unless retaining routes */
-      if (!retain_mode)
-        bgp_terminate();
+      /* send notify to all peers, wiat for alll sessions to be disables
+       * then terminate all pthreads */
+      bgp_terminate(1);
     }
 
   mqb_free(mqb);
diff --git a/bgpd/bgp_peer.c b/bgpd/bgp_peer.c
index 10fedc86..aa1c8841 100644
--- a/bgpd/bgp_peer.c
+++ b/bgpd/bgp_peer.c
@@ -303,6 +303,10 @@ bgp_session_has_disabled(bgp_peer peer)
       bgp_session_enable(peer);
     }
 
+  /* if the program is terminating then see if this was the last session
+   * and if so ... die ....
+   */
+  program_terminate_if_all_disabled();
   return 0;
 }
 
diff --git a/bgpd/bgp_session.c b/bgpd/bgp_session.c
index aa8d4933..0fe7a8dc 100644
--- a/bgpd/bgp_session.c
+++ b/bgpd/bgp_session.c
@@ -350,6 +350,7 @@ bgp_session_disable(bgp_peer peer, bgp_notify notification)
 
   /* Now change to limping state                                        */
   session->state = bgp_session_sLimping;
+  session->defer_enable = 0;
 
   /* Ask the BGP engine to disable the session.
    *
diff --git a/bgpd/bgpd.c b/bgpd/bgpd.c
index 2f50e419..30f32592 100644
--- a/bgpd/bgpd.c
+++ b/bgpd/bgpd.c
@@ -80,6 +80,9 @@ qpn_nexus routing_nexus = NULL;
 /* BGP community-list.  */
 struct community_list_handler *bgp_clist;
 
+/* true while program terminating */
+static int program_terminating = 0;
+
 /* BGP global flag manipulation.  */
 int
 bgp_option_set (int flag)
@@ -4693,13 +4696,16 @@ bgp_init (void)
 }
 
 void
-bgp_terminate (void)
+bgp_terminate (int terminating)
 {
   struct bgp *bgp;
   struct peer *peer;
   struct listnode *node, *nnode;
   struct listnode *mnode, *mnnode;
 
+  program_terminating = terminating;
+
+  /* Disable all peers */
   for (ALL_LIST_ELEMENTS (bm->bgp, mnode, mnnode, bgp))
     for (ALL_LIST_ELEMENTS (bgp->peer, node, nnode, peer))
       bgp_peer_disable(peer, bgp_notify_new(BGP_NOTIFY_CEASE,
@@ -4717,4 +4723,39 @@ bgp_terminate (void)
       work_queue_free (bm->process_rsclient_queue);
       bm->process_rsclient_queue = NULL;
     }
+
+  /* if no sessions were enabled then need to check here */
+  program_terminate_if_all_disabled();
 }
+
+/* If we are terminating the program, and all sessions are disabled
+ * then terminate all threads
+ */
+void
+program_terminate_if_all_disabled(void)
+{
+  struct bgp *bgp;
+  struct peer *peer;
+  struct listnode *node, *nnode;
+  struct listnode *mnode, *mnnode;
+
+  if (!program_terminating)
+    return;
+
+  /* are there any active sessions remaining? */
+  for (ALL_LIST_ELEMENTS (bm->bgp, mnode, mnnode, bgp))
+    for (ALL_LIST_ELEMENTS (bgp->peer, node, nnode, peer))
+      if (bgp_session_is_active(peer->session))
+        return;
+
+  /* ask remaining pthreads to die */
+  if (qpthreads_enabled && routing_nexus != NULL)
+        qpn_terminate(routing_nexus);
+
+  if (qpthreads_enabled && bgp_nexus != NULL)
+        qpn_terminate(bgp_nexus);
+
+  if (cli_nexus != NULL)
+      qpn_terminate(cli_nexus);
+}
+
diff --git a/bgpd/bgpd.h b/bgpd/bgpd.h
index 94a372c1..8cfe65c2 100644
--- a/bgpd/bgpd.h
+++ b/bgpd/bgpd.h
@@ -406,7 +406,7 @@ extern qpn_nexus bgp_nexus;
 extern qpn_nexus routing_nexus;
 
 /* Prototypes. */
-extern void bgp_terminate (void);
+extern void bgp_terminate (int);
 extern void bgp_reset (void);
 
 extern void bgp_zclient_reset (void);                      /* See bgp_zebra ! */
@@ -553,4 +553,6 @@ extern int peer_maximum_prefix_unset (struct peer *, afi_t, safi_t);
 extern int peer_clear (struct peer *);
 extern int peer_clear_soft (struct peer *, afi_t, safi_t, enum bgp_clear_type);
 
+extern void program_terminate_if_all_disabled(void);
+
 #endif /* _QUAGGA_BGPD_H */
diff --git a/lib/mqueue.c b/lib/mqueue.c
index f252586e..3031891e 100644
--- a/lib/mqueue.c
+++ b/lib/mqueue.c
@@ -434,11 +434,15 @@ mqueue_enqueue(mqueue_queue mq, mqueue_block mqb, int priority)
         {
           mqb->next = mq->head ;
           mq->head  = mqb ;
+          /* mq non-empty, enchain at head, therefore tail unaffected */
         }
       else
         {
           mqb->next   = after->next ;
           after->next = mqb ;
+          /* if only have priority messages then fix tail */
+          if (mq->tail == after)
+            mq->tail = mqb;
         }
       mq->tail_priority = mqb ;
     }
@@ -593,6 +597,10 @@ mqueue_dequeue(mqueue_queue mq, int wait, void* arg)
   /* Have something to pull off the queue       */
 
   mq->head = mqb->next ;
+
+  /* fix tails if at tail */
+  if (mqb == mq->tail)
+    mq->tail = NULL ;
   if (mqb == mq->tail_priority)
     mq->tail_priority = NULL ;
 
@@ -644,6 +652,9 @@ mqueue_revoke(mqueue_queue mq, void* arg0)
           if (mq->tail == mqb)
             mq->tail = prev ;
 
+          if (mqb == mq->tail_priority)
+            mq->tail_priority = prev ;
+
           qpt_mutex_unlock(&mq->mutex) ;
             mqb_dispatch_destroy(mqb) ;
           qpt_mutex_lock(&mq->mutex) ;