Revised thread/timer handling, work queue and scheduling.

Updated quagga thread handling to use qtimers when using the new
qpnexus -- so all timers are qtimers in the new scheme.

Updated work queue handling so that each work queue item is a single
malloced structure, not three.  (Only bgpd and zebra use the work
queue system.)

When using qpnexus the background thread queue is no longer a timer
queue, but simply a list of pending background threads.  When a
background thread is waiting on a timer, it is in the qtimer pile,
same like any other thread.

When using qpnexus, the only remaining quagga thread queues are the
event and ready queues.

Revised the qpnexus loop so that only when there is nothing else to
do will it consider the background threads.

Revised write I/O in the BGP Engine so that all writing is via the
connection's write buffer.  Revised the write I/O in the Routeing
Engine, so that it passes groups of updates in a single mqueue
message.  This all reduces the number of TCP packets sent (because
BGP messages are collected together in the connection's write buffer)
and reduces the number of mqueue messages involved.
(No need for TCP_CORK.)

Code and comments review for the new code.

  modified:   bgpd/bgp_advertise.c
  modified:   bgpd/bgp_common.h
  modified:   bgpd/bgp_connection.c
  modified:   bgpd/bgp_connection.h
  modified:   bgpd/bgp_engine.h
  modified:   bgpd/bgp_fsm.c
  modified:   bgpd/bgp_main.c
  modified:   bgpd/bgp_msg_read.c
  modified:   bgpd/bgp_msg_write.c
  modified:   bgpd/bgp_network.c
  modified:   bgpd/bgp_packet.c
  modified:   bgpd/bgp_packet.h
  modified:   bgpd/bgp_peer.c
  modified:   bgpd/bgp_peer_index.h
  modified:   bgpd/bgp_route.c
  modified:   bgpd/bgp_route_refresh.h
  modified:   bgpd/bgp_session.c
  modified:   bgpd/bgp_session.h
  modified:   bgpd/bgpd.c
  new file:   bgpd/bgpd.cx
  modified:   lib/mqueue.h
  modified:   lib/qpnexus.c
  modified:   lib/qpnexus.h
  modified:   lib/qpselect.c
  modified:   lib/qtimers.c
  modified:   lib/qtimers.h
  modified:   lib/sigevent.c
  modified:   lib/stream.c
  modified:   lib/stream.h
  modified:   lib/thread.c
  modified:   lib/thread.h
  modified:   lib/workqueue.c
  modified:   lib/workqueue.h
  modified:   tests/heavy-wq.c
  modified:   zebra/zebra_rib.c

1 files changed, 37 insertions, 26 deletions

diff --git a/lib/qpnexus.h b/lib/qpnexus.h
index a6cad148..d5b7c5a6 100644
--- a/lib/qpnexus.h
+++ b/lib/qpnexus.h
@@ -48,31 +48,27 @@
  */
 
 /* maximum time in seconds to sit in a pselect */
-#define MAX_PSELECT_TIMOUT 10
+#define MAX_PSELECT_WAIT 10
 
 /* signal for message queues */
 #define SIGMQUEUE SIGUSR2
 
 /* number of event hooks */
-#define NUM_EVENT_HOOK 2
-
-/* Work priorities */
-enum qpn_priority
-{
-  qpn_pri_highest = 1,
-
-  qpn_pri_first = 1,
-  qpn_pri_second = 2,
-  qpn_pri_third = 3,
-  qpn_pri_fourth = 4,
-
-  qpn_pri_lowest = 4,
-};
+enum { qpn_hooks_max = 4 } ;
 
 /*==============================================================================
  * Data Structures.
  */
 
+typedef int qpn_hook_function(void) ;
+
+typedef struct qpn_hook_list* qpn_hook_list ;
+struct qpn_hook_list
+{
+  void*     hooks[qpn_hooks_max] ;
+  unsigned  count ;
+} ;
+
 typedef struct qpn_nexus* qpn_nexus ;
 
 struct qpn_nexus
@@ -99,30 +95,45 @@ struct qpn_nexus
   /* qpthread routine, can override */
   void* (*start)(void*);
 
-  /* in-thread initialize, can override.  Called within the thread
-   * after all other initializion just before thread loop */
+  /* in-thread initialise, can override.  Called within the thread
+   * after all other initialisation just before thread loop */
   void (*in_thread_init)(void);
 
-  /* in-thread finalize, can override.  Called within thread
+  /* in-thread finalise, can override.  Called within thread
    * just before thread dies.  Nexus components all exist but
    * thread loop is no longer executed */
   void (*in_thread_final)(void);
 
-  /* thread loop events, can override.  Called before and after message queue,
-   * and before I/O and timers.
-   * Hook should perform all work <= given priority.
-   * Returns the time to try again, 0 means default to maximum.
+  /* in-thread queue(s) of events or other work.
+   *
+   * The hook function(s) are called in the qpnexus loop, at the top of the
+   * loop.  So in addition to the mqueue, I/O, timers and any background stuff,
+   * the thread may have other queue(s) of things to be done.
+   *
+   * Hook function can process some queue(s) of things to be done.  It does not
+   * have to empty its queues, but it MUST only return 0 if all queues are now
+   * empty.
    */
-  qtime_mono_t (*event_hook[NUM_EVENT_HOOK])(enum qpn_priority);
-
+  struct qpn_hook_list foreground ;
+
+  /* in-thread background queue(s) of events or other work.
+   *
+   * The hook functions are called at the bottom of the qpnexus loop, but only
+   * when there is absolutely nothing else to do.
+   *
+   * The hook function should do some unit of background work (if there is any)
+   * and return.  MUST return 0 iff there is no more work to do.
+   */
+  struct qpn_hook_list background ;
 };
 
 /*==============================================================================
  * Functions
  */
 
-extern qpn_nexus qpn_init_new(qpn_nexus qtn, int main_thread);
-extern void qpn_exec(qpn_nexus qtn);
+extern qpn_nexus qpn_init_new(qpn_nexus qpn, int main_thread);
+extern void qpn_add_hook_function(qpn_hook_list list, void* hook) ;
+extern void qpn_exec(qpn_nexus qpn);
 extern void qpn_terminate(qpn_nexus qpn);
 extern qpn_nexus qpn_free(qpn_nexus qpn);