diff -uNr snort-2.3.3/etc/snort.conf snort-2.3.3-spade/etc/snort.conf
--- snort-2.3.3/etc/snort.conf	2005-04-23 21:48:52.000000000 +0200
+++ snort-2.3.3-spade/etc/snort.conf	2006-03-31 14:16:22.000000000 +0200
@@ -637,6 +637,7 @@
 #   Zeno <admin@cgisecurity.com>
 #   Ryan Russell <ryan@securityfocus.com>
 
+include spade.ossim.conf
 
 
 #=========================================
diff -uNr snort-2.3.3/etc/spade.ossim.conf snort-2.3.3-spade/etc/spade.ossim.conf
--- snort-2.3.3/etc/spade.ossim.conf	1970-01-01 01:00:00.000000000 +0100
+++ snort-2.3.3-spade/etc/spade.ossim.conf	2006-03-31 14:16:22.000000000 +0200
@@ -0,0 +1,117 @@
+# Example configuration file for Spade v021026.1 and later
+# use this as your snort config file (-c option) to run Snort Spade-only
+# include it in your snort config file or put lines of this form in it
+
+# set this to a directory Spade can read and write to store its files
+var SPADEDIR /var/log/snort/
+
+# see the Usage.Spade file for the full meaning of and all the options
+#   available for all these lines
+
+# This is the main Spade configuration line; it must appear first.
+# Here are some options for this line:
+# + dest:  the Snort facility that the Spade output should go to
+#   (alert, log, or both)
+# + statefile:  where Spade's persistant data is stored
+# + logfile:  where Spade will store information about its run
+# + Xdports,Xdips,Xsips,Xsports: like below but with global application
+preprocessor spade:  dest=alert logfile=$SPADEDIR/spade.log statefile=$SPADEDIR/spade.rcv
+
+# This line sets up your Spade homenet.  Set this to the network that is
+#   connecting to the larger network at the point Spade is running.
+# It is important to configure this line.
+# Your networks should be like [10.0.0.0/8,192.168.0.0/16] or space separated
+#preprocessor spade-homenet: 192.168.1.0/24
+
+# Turn on some detectors with "spade-detect" lines.  Each of these enables
+#   a cetain type of detector for a certain type of packet.  If you start to
+#   feel overwhelmed, use Xdports, Xdips, Xsips, and/or Xsports on the lines 
+#   below to suppress reports you don't care about, and/or disable some of
+#   your detectors these that you care least about.
+#        These detect packets going to seemingly closed dest ports
+#            You can add thresh=N to override the default reporting threshold.
+#
+#
+#  Closed Dest Ports
+preprocessor spade-detect: type=closed-dport Xsports=80,443 tcpflags=synonly wait=2
+preprocessor spade-detect: type=closed-dport Xsports=53,137,138,139 proto=udp wait=2
+#  Rare but Open Dst Used
+preprocessor spade-detect: type=closed-dport Xsports=80,443 Xdports=80,443 tcpflags=synonly revwaitrpt wait=2
+preprocessor spade-detect: type=closed-dport Xsports=53,137,138,139 proto=udp revwaitrpt wait=2
+#preprocessor spade-detect: type=closed-dport tcpflags=weird thresh=0.5
+#preprocessor spade-detect: type=closed-dport tcpflags=synack 
+#preprocessor spade-detect: type=closed-dport tcpflags=established 
+#preprocessor spade-detect: type=closed-dport tcpflags=teardown 
+#preprocessor spade-detect: type=closed-dport to=nothome tcpflags=synonly wait=5
+#preprocessor spade-detect: type=closed-dport to=nothome tcpflags=weird 
+#preprocessor spade-detect: type=closed-dport to=nothome tcpflags=synack 
+#preprocessor spade-detect: type=closed-dport to=nothome tcpflags=established 
+#preprocessor spade-detect: type=closed-dport to=nothome tcpflags=teardown 
+#preprocessor spade-detect: type=closed-dport to=nothome proto=udp wait=7
+
+
+#  These detect packets going to a seemingly non-live IP
+#
+#
+preprocessor spade-detect: type=dead-dest Xdports=80,443 tcpflags=synack wait=2
+preprocessor spade-detect: type=dead-dest proto=udp Xsports=53 wait=2
+preprocessor spade-detect: type=dead-dest proto=icmp icmptype=noterr wait=2
+#preprocessor spade-detect: type=dead-dest tcpflags=weird wait=2
+#preprocessor spade-detect: type=dead-dest tcpflags=setup wait=2
+#preprocessor spade-detect: type=dead-dest tcpflags=synonly wait=2
+#preprocessor spade-detect: type=dead-dest tcpflags=established wait=5
+#preprocessor spade-detect: type=dead-dest tcpflags=teardown wait=2
+#preprocessor spade-detect: type=dead-dest proto=icmp icmptype=err wait=2
+
+
+
+#        These detect unusual use of a dest port by a source IP
+#            You can add thresh=N to override the default reporting threshold.
+#
+#  Source used Odd Dest Port
+preprocessor spade-detect: type=odd-dport proto=tcp wait=2
+preprocessor spade-detect: type=odd-dport proto=udp Xsports=53,137,138,139 wait=2
+#preprocessor spade-detect: type=odd-dport from=nothome proto=tcp
+#preprocessor spade-detect: type=odd-dport from=nothome proto=udp
+
+
+#        These detect ICMP packets with an unusual type and code
+#            You can add thresh=N to override the default reporting threshold.
+#
+#
+preprocessor spade-detect: type=odd-typecode
+preprocessor spade-detect: type=odd-typecode to=nothome
+
+
+#        These detect unusual connections to a dest IP by a source IP when the
+#            dest port has predictable dest IPs
+#            You can add thresh=N to override the default reporting threshold.
+#
+#   Source Used Odd Dest For Port
+preprocessor spade-detect: type=odd-port-dest proto=tcp Xdports=80,443
+preprocessor spade-detect: type=odd-port-dest proto=udp Xsports=53,137,138,139
+#preprocessor spade-detect: type=odd-port-dest from=nothome proto=tcp Xdports=80
+#preprocessor spade-detect: type=odd-port-dest from=nothome proto=udp Xdports=80
+
+
+# This line causes Spade to adjust the reporting threshold for a given
+#   detector automatically; repeat it for each detector that you want to apply
+#   it to
+# Target is the target rate of alerts for normal circumstances
+#   (0.01= 1% or you can give it an hourly rate)
+# After the first hour (or however long the period is set to with "obsper"),
+#   the initially configured reporting threshold is ignored
+# To use this, you will need to an option of the form id=<label> to the
+#   spade-detect line of the detector you want to be adapted and set
+#   id=<label> below to match
+# This mode is recommended for users getting started that are using absolute
+#   anomaly scores; relative score users might want it as well.
+#preprocessor spade-adapt3: id=<label> target=0.01 obsper=60
+
+# some other possible Spade config lines:
+# offline threshold advising for a detector
+#preprocessor spade-threshadvise: id=<label> target=200 obsper=24
+# periodically report on the anom scores and count of packets seen by a detector
+#preprocessor spade-survey:  id=<label> surveyfile=$SPADEDIR/survey.txt interval=60
+# print out certain all known stats about packet features
+#preprocessor spade-stats: entropy uncondprob condprob
diff -uNr snort-2.3.3/src/Makefile.am snort-2.3.3-spade/src/Makefile.am
--- snort-2.3.3/src/Makefile.am	2004-09-13 19:44:49.000000000 +0200
+++ snort-2.3.3-spade/src/Makefile.am	2006-03-31 14:16:22.000000000 +0200
@@ -45,7 +45,8 @@
 smalloc.h \
 snort_packet_header.h \
 event_queue.c event_queue.h \
-inline.c inline.h
+inline.c inline.h \
+packets.c packets.h
 
 snort_LDADD = output-plugins/libspo.a \
 detection-plugins/libspd.a            \
diff -uNr snort-2.3.3/src/packets.c snort-2.3.3-spade/src/packets.c
--- snort-2.3.3/src/packets.c	1970-01-01 01:00:00.000000000 +0100
+++ snort-2.3.3-spade/src/packets.c	2006-03-31 14:16:22.000000000 +0200
@@ -0,0 +1,265 @@
+/*
+To: snort-devel@lists.sourceforge.net
+From: James Hoagland <hoagland@SiliconDefense.com>
+Cc: hoagland@SiliconDefense.com
+Subject: [Snort-devel] New internal facility: packet cloning
+Date: Sun, 29 Sep 2002 20:31:08 -0700
+
+Greetings all,
+
+I have written some new Snort internal functionality that should help current and future Snort detectors (e.g., preprocessors) do better detection and/or more complete and standard reporting.  That functionality is creating a copy of a Packet.
+
+Some background first, as I understand it (correct me if I am mistaken).  A Packet is the Snort data structure that contains the parsed fields of a packet that libpcap presents to Snort.  It is Snort's standard representation of a packet and so is used throughout the program.  It is given to preprocessors and Snort's signature based detector as one in a stream of packets.  When these detectors wish to report on a packet, they typically pass the Packet to the alert/log facilities (e.g., alert file or database) that the user has configured.  However, sometimes detectors use different output means than the standard ones that the user configured.  For example, spp_portscan and spp_portscan2 and their packet logs.  (There could be others.)
+
+I cannot say for sure why these two do not use the standard output mechanism.  But one barrier is the combination of the fact that the output mechanisms can only output a Packet structure and the fact that the Packet that is given to the detectors does not persist beyond that call into that detector.  Specifically, there is only one Packet that Snort has in memory at a time.  Regardless of the reason, the effect is that the user does not have the control they might like over the place where packets are stored.  For example, they cannot have portscan packets (as reported by spp_portscan) be logged to the database.  So this suggests that in order to give this flexibility to the user, the simplest thing is to provide the detector with a means of holding onto a Packet beyond its invocation.  However, this functionality does not exist in Snort and, given the complexity, it can be daunting for the detector writer to do write it themselves.
+
+The class of detectors that this will help is those that do not want to immediately report on a packet.  Often this will be because they want to wait for future information contained on the packet stream. For example, they want to eliminate a potential false positive.  And we know that false positives can be a barrier to effective use of Snort (and other IDSs).  So, providing the copying functionality, while not getting rid of false positives and promoting more standard packet reporting, can be enabling towards these goals.
+
+
+Okay, enough for the motivational speech. :)  Its working code that gets things done in open source.  So, it is attached.  At least it is working in my tests; I need others to test it our as well since, e.g., I do not have access to FDDI network packet to try it on.  And I am not an expert on the Packet data structure.  But I know much more about it now, given frequent consultation with decode.[ch].
+
+This is what is attached:
+
+1) packets.c: the code that implements ClonePacket() and FreePacket().
+
+2) packet.h:  I'll let you guess what role this serves. :)
+
+3) snort+pclone.patch:  A patch against snort 1.9.0beta6 that puts packets.[ch] into the Snort source.  In addition, the patch includes a hack of a preprocessor, spp_pcopytest, that uses the cloning code. It makes clones of 100 packets on the packet stream at a time before pushing them to the standard output facilities.  For reference, the original packets are spit out as they are received.  The original should match the clone (as they do in all my tests).  Run it as "src/snort -c etc/pcopytest.conf".
+
+Implementation notes.  There were other ways that this can be approached.  My goal was for it to be efficient and with only localized changes (e.g., no changes to Packet).  There are some other notes in the source.
+
+Snort notes.  There might be a couple small snags when using cloned packets.  One that I noticed is an (now) incorrect assumption that there will only be one call to PrintNetData between each packet acquisition.  (The workaround it to manually flush its cache.)  If some output mechanism makes a hardcoded reference to "the" Packet, that will be broken.  And if anything else makes the assumption that PrintNetData did, that would be broken.
+
+
+My main motivation for doing this is that the next generation version of Spade will need this functionality to reduce false positives and to accurately detect new scan types.  I could have this functionality internal to Spade, but I'd rather it be a general Snort internal facility that can be used anywhere in Snort.  If someone can implement this better, go for it.  But the functionality of a Packet persisting across calls to a preprocessor is something that I'll be needing in the not-too-distant future.
+
+If anyone has any questions/concerns/comments/fixes, let me know.
+
+Sorry for the length of this message.
+
+Kind regards,
+
+  Jim
+
+P.s. I do not think that Packet cloning is covered by any US federal or state stature, so at least we are safe that way. :)
+*/
+
+#include <pcap.h>
+#include <stdlib.h>
+#include <string.h>
+#include "decode.h"
+#include "snort.h"
+#include "packets.h"
+
+typedef struct _PacketClone
+{
+    Packet p; /* our packet; the address of this must be the same as
+                 the address of the structure */
+    struct pcap_pkthdr pkth; /* the pcap packet header to be included
+                                in the above packet */
+    u_int8_t *pkt; /* storage space for the packet data */
+    int pkt_alloc_size; /* how much space is allocated there */
+    struct _PacketClone *next;
+} PacketClone;
+
+PacketClone *free_packets; /* freelist of normal sized PacketClones */
+PacketClone *free_oz_packets; /* freelist of oversized PacketClones */
+
+#define ADJUST_PKT_INDEX_FIELD(to_p,from_p,field) \
+    if ((from_p)->field) \
+        (to_p)->field= (void *)((to_p)->pkt + \
+           ((unsigned long)(from_p)->field - (unsigned long)(from_p)->pkt));
+    
+#define ADJUST_OPTIONS_DATA_FIELD(to_p,from_p,field,index) \
+    if ((from_p)->field[index].data) \
+        (to_p)->field[index].data= ((to_p)->pkt + \
+            ((unsigned long)(from_p)->field[index].data - \
+             (unsigned long)(from_p)->pkt));
+
+
+/* the normal (minimum) size packet length to allocate; based on pcap snap length */
+#define NORMAL_ALLOC_PACKETLEN (pv.pkt_snaplen ? pv.pkt_snaplen : SNAPLEN)
+/* sometimes we'll need more that the snap length (e.g., with stream4_reassemble 
+   packets), so here's how we calculate how big; oversized allocations get up to
+   63 bytes extra added on at end to make their reuse more likely. */
+#define PACKET_ALLOC_SIZE(min) \
+    (min < NORMAL_ALLOC_PACKETLEN \
+    ? NORMAL_ALLOC_PACKETLEN \
+    : (((min+63) >> 6) << 6))
+
+/*
+ * Function: ClonePacket(Packet *p)
+ *
+ * Purpose: Make a copy of a decoded packet struct (Packet) so that the
+ *          contents of a Packet can survive beyond a call to pcap for a
+ *          new packet
+ *
+ * Arguments: p   => pointer to the decoded packet struct to clone
+ *
+ * Returns: a pointer to the copied Packet
+ *
+ * Notes: this function and FreePacket participate in a recycling program
+ *        for Packets to minimize malloc calls.  ssnptr and state fields not
+ *        copied but set to NULL instead.
+ */
+Packet *ClonePacket(Packet *p)
+{
+    PacketClone *clone,*prev,*here;
+    Packet *cp;
+    int i;
+    int packet_alloc_size= PACKET_ALLOC_SIZE(p->pkth->len);
+    int oversized= packet_alloc_size > NORMAL_ALLOC_PACKETLEN;
+    
+    /* get storage for the Packet */
+    if (free_packets != NULL || free_oz_packets != NULL)
+    {
+        if (oversized) {
+            /* draw from oversize list else the regular list */
+            if (free_oz_packets != NULL) {
+                prev= NULL;
+                here= free_oz_packets;
+                while (here->next != NULL) {
+                    if (here->pkt_alloc_size >= packet_alloc_size) break;
+                    prev= here;
+                    here= here->next;
+                }
+                clone= here;
+                if (prev != NULL)
+                    prev->next= here->next;
+                else
+                    free_oz_packets= here->next;
+            } else {
+                clone= free_packets;
+                free_packets= free_packets->next;
+            }
+            /* get bigger memory chunk if needed */
+            if (clone->pkt_alloc_size < packet_alloc_size) {
+                free(clone->pkt);
+                clone->pkt= (u_int8_t *)calloc(packet_alloc_size,sizeof(u_int8_t));
+                if (clone->pkt == NULL) /* must be out of memory */
+                {
+                    free(clone);
+                    return NULL;
+                }
+                clone->pkt_alloc_size= packet_alloc_size;
+            }
+        } else {
+            /* draw from regular list else the oversize list */
+            if (free_packets != NULL) {
+                clone= free_packets;
+                free_packets= free_packets->next;
+            } else {
+                clone= free_oz_packets;
+                free_oz_packets= free_oz_packets->next;
+            }
+        }
+    } else {
+        clone= (PacketClone *)calloc(1,sizeof(PacketClone));
+        if (clone == NULL) return NULL; /* must be out of memory */
+        
+        clone->pkt= (u_int8_t *)calloc(packet_alloc_size,sizeof(u_int8_t));
+        if (clone->pkt == NULL) /* must be out of memory */
+        {
+            free(clone);
+            return NULL;
+        }
+        clone->pkt_alloc_size= packet_alloc_size;
+    }
+    
+    /* make a copy of the Packet p into cp */
+    cp= &clone->p;
+    *cp= *p; /* this will get everything except the pointers */
+    
+    /* copy the pcap header */
+    cp->pkth= &clone->pkth; /* we use the PacketClone storage space rather
+                               than malloc */
+    *cp->pkth= *(p->pkth);
+    
+    /* copy the packet data */
+    cp->pkt= clone->pkt;
+    memcpy(cp->pkt,p->pkt,p->pkth->len);
+    
+    /* set the Packet fields which are just pointers into the packet data
+       (i.e., just about all of them) */
+    /* many of these fields are NULL, so let's use some knowledge about 
+       the relationship between fields to avoid trying to adjust; of course
+       this means that this must be maintained */
+    if (p->fddihdr) {
+        ADJUST_PKT_INDEX_FIELD(cp,p,fddihdr);
+        ADJUST_PKT_INDEX_FIELD(cp,p,fddisaps);
+        ADJUST_PKT_INDEX_FIELD(cp,p,fddisna);
+        ADJUST_PKT_INDEX_FIELD(cp,p,fddiiparp);
+        ADJUST_PKT_INDEX_FIELD(cp,p,fddiother);
+    }
+    if (p->trh) {
+        ADJUST_PKT_INDEX_FIELD(cp,p,trh);
+        ADJUST_PKT_INDEX_FIELD(cp,p,trhllc);
+        ADJUST_PKT_INDEX_FIELD(cp,p,trhmr);
+    }
+    ADJUST_PKT_INDEX_FIELD(cp,p,sllh);
+    ADJUST_PKT_INDEX_FIELD(cp,p,pfh);
+    ADJUST_PKT_INDEX_FIELD(cp,p,eh);
+    ADJUST_PKT_INDEX_FIELD(cp,p,vh);
+    ADJUST_PKT_INDEX_FIELD(cp,p,ehllc);
+    ADJUST_PKT_INDEX_FIELD(cp,p,ehllcother);
+    ADJUST_PKT_INDEX_FIELD(cp,p,wifih);
+    ADJUST_PKT_INDEX_FIELD(cp,p,ah);
+    if (p->eplh) {
+        ADJUST_PKT_INDEX_FIELD(cp,p,eplh);
+        ADJUST_PKT_INDEX_FIELD(cp,p,eaph);
+        ADJUST_PKT_INDEX_FIELD(cp,p,eaptype);
+        ADJUST_PKT_INDEX_FIELD(cp,p,eapolk);
+    }
+    if (p->iph) {
+        ADJUST_PKT_INDEX_FIELD(cp,p,iph);
+        ADJUST_PKT_INDEX_FIELD(cp,p,ip_options_data);
+        /* get the ones inside decoded options too */
+        for (i= 0; i < p->ip_option_count; i++)
+            ADJUST_OPTIONS_DATA_FIELD(cp,p,ip_options,i);
+        if (p->tcph) {
+            ADJUST_PKT_INDEX_FIELD(cp,p,tcph);
+            ADJUST_PKT_INDEX_FIELD(cp,p,tcp_options_data);
+            for (i= 0; i < p->tcp_option_count; i++)
+                ADJUST_OPTIONS_DATA_FIELD(cp,p,tcp_options,i);
+        } else if (p->icmph) {
+            ADJUST_PKT_INDEX_FIELD(cp,p,icmph);
+            ADJUST_PKT_INDEX_FIELD(cp,p,orig_iph);
+            ADJUST_PKT_INDEX_FIELD(cp,p,orig_udph);
+            ADJUST_PKT_INDEX_FIELD(cp,p,orig_tcph);
+            ADJUST_PKT_INDEX_FIELD(cp,p,orig_icmph);
+        } else
+            ADJUST_PKT_INDEX_FIELD(cp,p,udph);
+    }
+    ADJUST_PKT_INDEX_FIELD(cp,p,ext); /* is this used anywhere? */
+    ADJUST_PKT_INDEX_FIELD(cp,p,data);
+
+    /* we don't copy these over */
+    cp->ssnptr= NULL;
+
+    return cp;
+}
+
+/*
+ * Function: FreePacket(Packet *p)
+ *
+ * Purpose: Free a Packet created by ClonePacket
+ *
+ * Arguments: p   => pointer to the decoded packet struct to free
+ *
+ * Returns: nada
+ *
+ * Notes: this function and FreePacket participate in a recycling program
+ *        for Packets to minimize malloc calls.
+ */
+void FreePacket(Packet *p)
+{
+    PacketClone *pc= (PacketClone *)p; /* assumes pc has same addr as p */
+    if (pc->pkt_alloc_size > NORMAL_ALLOC_PACKETLEN) {
+        pc->next= free_oz_packets;
+        free_oz_packets= pc;
+    } else {
+        pc->next= free_packets;
+        free_packets= pc;
+    }
+    /* note: pc->pkt remains allocated. */
+}
diff -uNr snort-2.3.3/src/packets.h snort-2.3.3-spade/src/packets.h
--- snort-2.3.3/src/packets.h	1970-01-01 01:00:00.000000000 +0100
+++ snort-2.3.3-spade/src/packets.h	2006-03-31 14:16:22.000000000 +0200
@@ -0,0 +1,11 @@
+/* part of the packet cloning patch by Jim Hoagland, Silicon Defense
+ (hoagland@silicondefense.com).  Hopefully this or equivalent functionality
+ will become a standard part of the Snort source. */
+
+#ifndef __PACKETS_H__
+#define __PACKETS_H__
+
+Packet *ClonePacket(Packet *p);
+void FreePacket(Packet *p);
+
+#endif // __PACKETS_H__
diff -uNr snort-2.3.3/src/plugbase.c snort-2.3.3-spade/src/plugbase.c
--- snort-2.3.3/src/plugbase.c	2005-04-22 21:03:56.000000000 +0200
+++ snort-2.3.3-spade/src/plugbase.c	2006-03-31 14:16:22.000000000 +0200
@@ -62,6 +62,8 @@
 #include "preprocessors/spp_flow.h"
 #include "preprocessors/spp_sfportscan.h"
 #include "preprocessors/spp_xlink2state.h"
+#include "preprocessors/spp_spade.h"
+
 
 /* built-in detection plugins */
 #include "detection-plugins/sp_pattern_match.h"
@@ -161,6 +163,7 @@
     SetupReact();
     SetupRespond();
 #endif
+    SetupSpade();
 }
 
 /****************************************************************************
diff -uNr snort-2.3.3/src/preprocessors/Makefile.am snort-2.3.3-spade/src/preprocessors/Makefile.am
--- snort-2.3.3/src/preprocessors/Makefile.am	2005-04-22 21:03:56.000000000 +0200
+++ snort-2.3.3-spade/src/preprocessors/Makefile.am	2006-03-31 14:16:22.000000000 +0200
@@ -25,6 +25,8 @@
 spp_xlink2state.c spp_xlink2state.h \
 xlink2state.c xlink2state.h \
 str_search.c str_search.h \
+spp_sfportscan.c spp_sfportscan.h \
+spp_spade.c spp_spade.h \
 stream.h
 
 
diff -uNr snort-2.3.3/src/preprocessors/spp_spade.c snort-2.3.3-spade/src/preprocessors/spp_spade.c
--- snort-2.3.3/src/preprocessors/spp_spade.c	1970-01-01 01:00:00.000000000 +0100
+++ snort-2.3.3-spade/src/preprocessors/spp_spade.c	2006-03-31 14:18:53.000000000 +0200
@@ -0,0 +1,7495 @@
+/*********************************************************************
+Spade, a Snort preprocessor plugin to report unusual packets
+Author: James Hoagland, Silicon Defense (hoagland@SiliconDefense.com)
+copyright (c) 2002 by Silicon Defense (http://www.silicondefense.com/)
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.  
+
+Spade description:
+
+SPADE, the Statistical Packet Anomaly Detection Engine, is a Snort
+preprocessor plugin to report packets that are unusual for your network. 
+Port scans and probes tend to be unusual, so this will tend to report them
+(as well as some benign packets that are simply uncommon).
+
+Spade's home page: http://www.silicondefense.com/spice/
+
+Please send complaints, kudos, and especially improvements and bugfixes to
+hoagland@SiliconDefense.com. This is a research project and would love to
+have your feedback.  It is still under active development and may change at
+any time.
+
+This file (snort_spade.c) is part of Spade v030125.1.  It makes netspade
+available via a snort plugin.
+*********************************************************************/
+
+/* Internal version control: Id: snort_spade.c,v 1.1 2004/10/11 14:35:54 jonkman Exp $ */
+ 
+/*! \file snort_spade.c
+ * \brief 
+ *  snort_spade.c makes netspade available to snort
+ * \ingroup snort_spade
+ */
+
+/*! \addtogroup snort_spade Netspade for Snort
+ * \brief this group contains objects in the Snort binding to netspade
+ * @{
+*/
+
+/*! \addtogroup libnetspade */
+
+#define ENABLE_IDMEF 0
+
+
+/* this should be in generators.h */
+#ifndef SPADE_CLOSED_DESTPORT_USED
+#define     SPADE_CLOSED_DESTPORT_USED   1
+#define     SPADE_NONLIVE_DEST_USED   3
+#define     SPADE_SRC_ODD_DESTPORT_USED   4
+#define     SPADE_SRC_ODD_TYPECODE_USED   5
+#define     SPADE_SRC_ODD_PORTDEST_USED   6
+// OSSIM
+#define     SPADE_RARE_OPEN_DESTPORT_USED   101
+#define     SPADE_RARE_DESTPORT_USED   102
+#endif
+ 
+#include "snort.h"
+#include "plugbase.h"
+#include "generators.h"
+#include "util.h"
+#include "parser.h"
+#include "log.h"
+#include "detect.h"
+#if ENABLE_IDMEF
+#include "../output-plugins/spo_idmef.h"
+#endif
+#include "packets.h"
+#include "spp_spade.h"
+#include <string.h>
+#include <stdarg.h>
+#include <signal.h>
+
+/** external globals from parser.c **/
+extern char *file_name;
+extern int file_line;
+
+static void SpadeReportAnom(void *context,spade_report *rpt);
+static void SpadeReportThreshChanged(void *context, char *id,char *mess, int using_corrscore);
+static void SnortSpadeMsgFn(spade_message_type msg_type,const char *msg);
+
+/// our instance of netspade
+netspade *spade;
+
+#define DEST_NOWHERE 0
+#define DEST_ALERT_FACILITY 1
+#define DEST_LOG_FACILITY 2
+#define DEST_IDMEF_FACILITY 4
+
+/// where to send spade alerts
+int spade_alert_dest= DEST_ALERT_FACILITY;
+/// where to send spade threshold adjusted messages
+int spade_adj_dest= DEST_ALERT_FACILITY;
+
+/// the bigger the number, the more debuging statements that are active
+int as_debug= 0; 
+
+/// is threshold adapting enabled?
+int adapt_active= 0;
+/// the number of detectors that have been enabled
+int num_detectors= 0;
+/// how many packets has Snort passed to us
+int pkt_count= 0;
+
+/*************/
+
+/* A call to this function needs to be added to plugbase.c somehow */
+void SetupSpade()
+{
+    /* link the preprocessor keyword list to the init functions in 
+       the preproc list to arrange for modules to run when specified */
+    RegisterPreprocessor("spade", SpadeInit);
+    RegisterPreprocessor("spade-homenet", SpadeHomenetInit);
+    RegisterPreprocessor("spade-detect", SpadeDetectInit);
+    RegisterPreprocessor("spade-stats", SpadeStatInit);
+    RegisterPreprocessor("spade-threshlearn", SpadeThreshadviseInit); /* for backwards compatability */
+    RegisterPreprocessor("spade-threshadvise", SpadeThreshadviseInit);
+    RegisterPreprocessor("spade-adapt", SpadeAdaptInit);
+    RegisterPreprocessor("spade-adapt2", SpadeAdapt2Init);
+    RegisterPreprocessor("spade-adapt3", SpadeAdapt3Init);
+    RegisterPreprocessor("spade-survey", SpadeSurveyInit);
+
+    if (as_debug) printf("Preprocessor: Spade is setup...\n");
+}
+
+
+
+/*========================================================================*/
+/*========================= Spade core routines ==========================*/
+/*========================================================================*/
+
+/* Spade core init function:
+     set up netspade, recover, register the signal handler,
+     register the preprocessor function */
+void SpadeInit(u_char *argsstr)
+{
+    int prob_mode=3,checkpoint_freq=50000,recover;
+    double init_thresh= -1;
+    char statefile[401]= "spade.rcv";
+    char outfile[401]= "-";
+    int use_corrscore= 0;
+    char dest[11]= "alert";
+    char adjdest[11]= "\0";
+    char xsips[401]="",xdips[401]="",xsports[401]="",xdports[401]="";
+    void *args[12];
+
+    args[0]= &init_thresh;
+    args[1]= &statefile;
+    args[2]= &outfile;
+    args[3]= &prob_mode;
+    args[4]= &checkpoint_freq;
+    args[5]= &use_corrscore;
+    args[6]= &dest;
+    args[7]= &adjdest;
+    args[8]= &xsips;
+    args[9]= &xdips;
+    args[10]= &xsports;
+    args[11]= &xdports;
+    fill_args_space_sep(argsstr,"d:thresh;s400:statefile;s400:logfile;"
+            "i:probmode;i:cpfreq;b:-corrscore,corrscore;s10:dest;s10:adjdest;"
+            "s400:Xsips,Xsip,xsips;s400:Xdips,Xdip,xdips;"
+            "s400:Xsports,Xsport,xsports;s400:Xdports,Xdport,xdports",args,SnortSpadeMsgFn);
+
+    if (as_debug) printf("statefile=%s; logfile=%s; cpfreq=%d\n",statefile,outfile,checkpoint_freq);
+
+    LogMessage("Spade is enabled\n");
+    recover= strcmp(statefile,"0") && strcmp(statefile,"/dev/null");
+
+    if (recover) {
+        spade= new_netspade_from_statefile(statefile,SnortSpadeMsgFn,as_debug,&recover);
+        if (recover)
+            LogMessage("    Spade state initialized to what is in %s\n",statefile);
+        else
+            LogMessage("    Could not load Spade state from %s\n",statefile);
+    }
+
+    if (!recover) {
+        spade= new_netspade(SnortSpadeMsgFn,as_debug);
+        LogMessage("    Spade state initialized to a clean slate (no prior knowledge)\n");
+    }
+    if (spade == NULL) {
+        FatalError("Spade initialization failed: out of memory!");
+    }
+
+    netspade_set_checkpointing(spade,statefile,checkpoint_freq);
+    LogMessage("    Spade will record its state to %s after every %d updates\n",statefile,checkpoint_freq);
+    netspade_set_output_file(spade,outfile);
+    LogMessage("    Spade's log is %s\n",outfile);
+
+    if (!strcmp(dest,"log")) {
+        LogMessage("    Spade reports will go to the log facility\n");
+        spade_alert_dest= DEST_LOG_FACILITY;
+    } else if (!strcmp(dest,"both")) {
+        LogMessage("    Spade reports will go to both the alert and log facility\n");
+        spade_alert_dest= DEST_ALERT_FACILITY|DEST_LOG_FACILITY;
+#if ENABLE_IDMEF
+    } else if (!strcmp(dest,"spo_idmef")) {
+        LogMessage("    Spade reports will go directly to the IDMEF facility\n");
+        spade_alert_dest= DEST_IDMEF_FACILITY;
+#endif
+    } else {
+        if (strcmp(dest,"alert"))
+            ErrorMessage("Spade: dest=%s not recognized, using dest=alert\n",dest);
+        LogMessage("    Spade reports will go to the alert facility\n");
+        spade_alert_dest= DEST_ALERT_FACILITY;
+    }
+    if (adjdest[0] == '\0') {
+        spade_adj_dest= spade_alert_dest;
+    } else {
+        if (!strcmp(adjdest,"log")) {
+            LogMessage("    Spade threshold adjusted reports will go to the log facility\n");
+            spade_adj_dest= DEST_LOG_FACILITY;
+        } else if (!strcmp(adjdest,"both")) {
+            LogMessage("    Spade threshold adjusted reports will go to both the alert and log facility\n");
+            spade_adj_dest= DEST_ALERT_FACILITY|DEST_LOG_FACILITY;
+        } else if (!strcmp(adjdest,"none")) {
+            LogMessage("    Spade threshold adjusted reports will not be reported\n");
+            spade_adj_dest= DEST_NOWHERE;
+        } else {
+            if (strcmp(adjdest,"alert"))
+                ErrorMessage("Spade: adjdest=%s not recognized, using adjdest=alert\n",adjdest);
+            LogMessage("    Spade threshold adjusted reports will go to the alert facility\n");
+            spade_adj_dest= DEST_ALERT_FACILITY;
+        }
+    }
+    netspade_set_callbacks(spade,NULL,SpadeReportAnom,((spade_adj_dest == DEST_NOWHERE) ? NULL : SpadeReportThreshChanged),(event_native_copier_t)ClonePacket,(event_native_freer_t)FreePacket);
+    
+    netspade_add_rpt_excludes(spade,xsips,xdips,xsports,xdports);
+    
+    /* at this point we don't know if there are any spade-detect lines, but
+       if this looks like the old form of this line (corrscore, thresh, or
+       probmode is specified).  Our approximation of specified is variance
+       from defaults.  Because our defaults are the same as netspade's
+       default detection mode, if the user did explicitly set things to the
+       default and provides no spade-detect line, the default detector will
+       be enabled in netspade automatically with the configuration we want. */
+       
+    if (prob_mode != 3 || init_thresh != -1 || use_corrscore) {
+        char init_detect_str[100];
+        /* backwards compatability mode */
+        if (prob_mode > 4 || prob_mode < 0) {
+            ErrorMessage("Warning: Spade probabity mode %d undefined, using #3 instead",prob_mode);
+            prob_mode= 3;
+        }
+        if (as_debug) printf("thresh=%f; probmode=%d; corrscore=%d\n",init_thresh,prob_mode,use_corrscore);
+
+        sprintf(init_detect_str,"id=default relscore=0  corrscore=0 thresh=%f probmode=%d corrscore=%d",
+                                                init_thresh,prob_mode,use_corrscore);
+        netspade_new_detector(spade,init_detect_str);
+        
+        LogMessage("    default detector enabled with: %s\n",init_detect_str);
+        num_detectors++;
+
+        if (as_debug) netspade_print_detector_config_details(spade,stdout,"default");
+    }       
+    
+
+   /* Set the preprocessor function into the function list */
+    AddFuncToPreprocList(PreprocSpade);
+    AddFuncToCleanExitList(SpadeCatchSig,NULL);
+    AddFuncToRestartList(SpadeCatchSig,NULL);
+}
+
+
+/*========================================================================*/
+/*========================= SpadeHomenet module ==========================*/
+/*========================================================================*/
+
+/* Set the Spade homenet */
+
+/* snort config file line:
+    preprocessor spade-homenet: {<network>}
+    where <network> is a network in CIDR notation (address/numbits)
+                       or an IP address */
+                                                        
+/* Spade homenet init function:
+     set up the homenet list */
+void SpadeHomenetInit(u_char *args)
+{
+    if (spade == NULL) FatalError("Please initialize Spade with the "
+        "'preprocessor spade:' line before listing spade-homenet: %s(%d)\n",
+        file_name,file_line);
+
+    netspade_set_homenet_from_str(spade,args);
+    LogMessage("    Spade homenet set to: %s\n",args);
+}
+
+
+
+/*========================================================================*/
+/*======================== SpadeDetect module ==========================*/
+/*========================================================================*/
+
+/* enable a Spade detector */                                                     
+void SpadeDetectInit(u_char *args)
+{
+    char *id;
+    if (spade == NULL) FatalError("Please initialize Spade with the "
+        "'preprocessor spade:' line before listing spade-detect: %s(%d)\n",
+        file_name,file_line);
+
+    id= netspade_new_detector(spade,args);
+    LogMessage("    detector %s enabled with: %s\n",id,args);
+    num_detectors++;
+
+    if (as_debug) netspade_print_detector_config_details(spade,stdout,id);
+}
+
+
+/*========================================================================*/
+/*=========================== SpadeStat module ===========================*/
+/*========================================================================*/
+
+/* Whenever SpadeCatchSig is invoked, this module arranges for certain
+   specified statistics to be written to the log file.  The available
+   statistics depend on what is recorded in the tree, which depends on the
+   probability measure used.  There is no good way to have more granularity
+   at present. */
+
+/* snort config file line:
+    preprocessor spade-stats: {<stat-option>}
+    where <stat-option> is one of:
+      "entropy" (to display the known entropies and conditional entropies)
+      "uncondprob" (to display the known non-0 simple (joint) probabilities)
+      "condprob" (to display the known non-0 conditional (joint)
+probabilities) */
+                                                        
+void SpadeStatInit(u_char *args)
+{
+    if (spade == NULL) FatalError("Please initialize Spade with the "
+        "'preprocessor spade:' line before listing spade-stat: %s(%d)\n",
+        file_name,file_line);
+
+    /* parse the argument list from the rules file */
+    netspade_set_output_stats_from_str(spade,args);
+    LogMessage("    Spade will report certain observation statistics to its log file: %s\n",args);
+}
+
+
+
+/*========================================================================*/
+/*======================== SpadeThreshadvise module =======================*/
+/*========================================================================*/
+
+/* Given a packet count and a length of time, this module reports a reporting
+   threshold that would have been effective in producing that number of alerts
+   in that time interval.  The idea is that one might use this as a threshold
+   for future runs.  The module quietly watches the network for the length of
+   time, adding events to the tree and calculating anomaly scores.  When the
+   time period is up, the module calls exit() after reporting the top anomaly
+   scores seen to the log file. */
+   
+   /* Spade threshold learning module init function:
+     set up threshold learning module per args */
+void SpadeThreshadviseInit(u_char *args)
+{
+    char *id;
+    if (spade == NULL) FatalError("Please initialize Spade with the "
+        "'preprocessor spade:' line before listing spade-threshadvise (a.k.a. "
+        "spade-threshlearn): %s(%d)\n",
+        file_name,file_line);
+    
+    id=netspade_setup_detector_advise_from_str(spade,args);
+
+    LogMessage("    Spade threshold advising inited for %s: %s\n",id,args);
+}
+
+/*========================================================================*/
+/*=========================== SpadeAdapt module ==========================*/
+/*========================================================================*/
+
+/* Given a report count target and a length of time, this module tries to keep
+   the reporting threshold at a level that would produce that number of alerts
+   in that time interval based on what was observed in the last interval.  To
+   support this, a list of the most anomalous scores seen in the current
+   interval is maintained.  At the end of the interval, an ideal threshold is
+   calculated based on the interval's scores.  This is combined linearly with
+   the current threshold to produce the threshold for the next interval.  As a
+   default option, the interval can implemented in terms of a count of packets,
+   where this count is the average number of packets seen during the specified
+   time interval length; this tends to make the transitions more smooth and
+   reliable since a more constant number of anomaly scores is used in finding
+   the topmost anamolous ones. */
+
+void SpadeAdaptInit(u_char *args)
+{
+    char *id;
+    if (spade == NULL) FatalError("Please initialize Spade with the "
+        "'preprocessor spade:' line before listing spade-adapt: %s(%d)\n",
+        file_name,file_line);
+    if (adapt_active && num_detectors < 2) {
+        ErrorMessage("Spade threshold adapting repeatedly specified, "
+            "ignoring later specification: %s(%d)\n",file_name,file_line);
+        return;
+    }
+    adapt_active= 1;
+
+    /* parse the argument list from the rules file */
+    id= netspade_setup_detector_adapt_from_str(spade,1,args);
+    
+    LogMessage("    Spade adapt mode 1 inited for %s: %s\n",id,args);
+
+    if (as_debug) netspade_print_detector_config_details(spade,stdout,id);
+}
+
+
+/*========================================================================*/
+/*========================== SpadeAdapt2 module ==========================*/
+/*========================================================================*/
+
+/* Given an hourly alert target count (or target fraction) and a length of
+   time, this module tries to keep the reporting threshold at a level that
+   would produce that number of alerts (or fraction of total reports) in an
+   hour based on what has been observed in the past.  When the report threshold
+   is updated, it is based in equal parts on observations from the short term,
+   middle term, and long term (at least for these that have been observed). 
+   The user can specify the time period for observations, the number of those
+   that make up the short term (NS), the number of short terms that make up the
+   medium term (NM), and the number of medium terms that make up the long term
+   (NL).  The short term component of the threshold is defined to be the
+   average of the kth and (k+1)st highest anomaly scores in the last NS
+   complete periods of observation, where k is number of anamoly reports that
+   should occur in the observation period assuming a uniform rate.  The middle
+   term component is the average of the last NM special short term components. 
+   The special short term components are the ones that are multiples of NS if
+   labeled with the number of observation periods that had completed when it
+   was calculated (i.e., #NS, #2NS, #3NS, etc.); these have the property that
+   they are based entirely on distinct measurements.  The long term component
+   is based on the last NL medium term componenets, including the current one. 
+   For each of the components, if there have been less than the specified
+   number of constituant parts (but there has been at least one complete one),
+   what is observed thus far is used.  To accomadate the varying rates of
+   packets fairly, the observation period is based on a count of packets.  This
+   count is the product of the specified observation period and the average
+   packet rate.
+*/
+
+void SpadeAdapt2Init(u_char *args)
+{
+    char *id;
+    
+    if (spade == NULL) FatalError("Please initialize Spade with the "
+        "'preprocessor spade:' line before listing spade-adapt2: %s(%d)\n",
+        file_name,file_line);
+    if (adapt_active && num_detectors < 2) {
+        ErrorMessage("Spade threshold adapting repeatedly specified, "
+            "ignoring later specification: %s(%d)\n",file_name,file_line);
+        return;
+    }
+    adapt_active= 2;
+
+    /* parse the argument list from the rules file */
+    id= netspade_setup_detector_adapt_from_str(spade,2,args);
+    
+    LogMessage("    Spade adapt mode 2 inited for %s: %s\n",id,args);
+
+    if (as_debug) netspade_print_detector_config_details(spade,stdout,id);
+}
+
+/*========================================================================*/
+/*========================== SpadeAdapt3 module ==========================*/
+/*========================================================================*/
+
+/* Given an hourly alert target count (or target fraction) and a length of
+   time, this module tries to keep the reporting threshold at a level that
+   would produce that number of alerts (or fraction of total reports) in an
+   hour based on what has been observed in the past.  ...
+*/
+
+void SpadeAdapt3Init(u_char *args)
+{
+    char *id;
+    
+    if (spade == NULL) FatalError("Please initialize Spade with the "
+        "'preprocessor spade:' line before listing spade-adapt3: %s(%d)\n",
+        file_name,file_line);
+    if (adapt_active && num_detectors < 2) {
+        ErrorMessage("Spade threshold adapting repeatedly specified, "
+            "ignoring later specification: %s(%d)\n",file_name,file_line);
+        return;
+    }
+    adapt_active= 3;
+
+    /* parse the argument list from the rules file */
+    id= netspade_setup_detector_adapt_from_str(spade,3,args);
+
+    LogMessage("    Spade adapt mode 3 inited for %s: %s\n",id,args);
+}
+
+
+/*========================================================================*/
+/*========================== SpadeSurvey module ==========================*/
+/*========================================================================*/
+
+/* This module surveys the anomoly scores observed across periods of time
+and reports this to a specified survey file.  The period #, the packet
+count, the median score, the 90th percentile score, and the 99th percentile
+score are recorded to the file in tab-delinated format.  Interpolation is
+used between scores if there is no score at exactly the position implied by
+the percentile. */
+
+/* efficiency note:  This use linked list to represent the observed anomoly
+   scores.  While it is necessary to maintain all these scores (the current
+   worst score might end up being the 99th percentile), a different
+   representation (order stat tree?) should be used if the packet count gets
+   high.  */
+
+void SpadeSurveyInit(u_char *args)
+{
+    char *id;
+    
+    if (spade == NULL) FatalError("Please initialize Spade with the "
+        "'preprocessor spade:' line before listing spade-survey: %s(%d)\n",
+        file_name,file_line);
+
+    /* parse the argument list from the rules file */
+    id= netspade_setup_detector_survey_from_str(spade,args);
+
+    LogMessage("    Spade survey mode inited for %s: %s\n",id,args);
+
+    if (as_debug) netspade_print_detector_config_details(spade,stdout,id);
+}
+
+
+/*********************************************************************/
+
+/* Spade core routine that is called with each packet; be efficient! */
+void PreprocSpade(Packet *p)
+{
+    spade_event pkt;
+    pkt_count++;
+    
+    if (p == NULL || p->iph == NULL) return; /* netspade only looks at IP packets for now */
+    
+    pkt.native= p;
+    pkt.time= (time_t)p->pkth->ts.tv_sec;
+    
+    pkt.origin= PKTORIG_TOP;
+    pkt.fldval[IPPROTO]= p->iph->ip_proto;
+    
+    switch (pkt.fldval[IPPROTO]) { /* protocol-specific processing */
+    case IPPROTO_TCP:
+        if (p->tcph == NULL) return;
+        pkt.fldval[TCPFLAGS]= p->tcph->th_flags;
+        break;
+    case IPPROTO_UDP:
+        if (p->udph == NULL) return;
+        break;
+    case IPPROTO_ICMP:
+        if (p->icmph == NULL) return;
+        pkt.fldval[ICMPTYPE]= p->icmph->type;
+        pkt.fldval[ICMPTYPECODE]= (p->icmph->type << 8) | p->icmph->code;
+        break;
+    default:;
+    }
+
+    pkt.fldval[SIP]= ntohl(p->iph->ip_src.s_addr);
+    pkt.fldval[DIP]= ntohl(p->iph->ip_dst.s_addr);
+    pkt.fldval[SPORT]= p->sp;
+    pkt.fldval[DPORT]= p->dp;
+    //pkt.fldval[TTL]= p->iph->ip_ttl;
+    //pkt.fldval[WIN] = p->tcph->th_win;
+
+    netspade_new_pkt(spade,&pkt);
+    
+    if ((p->orig_iph != NULL) && (p->icmph != NULL) && (p->icmph->type == 3)) {
+        pkt.origin= PKTORIG_UNRCH;
+
+        pkt.fldval[IPPROTO]= p->orig_iph->ip_proto;
+        
+        switch (pkt.fldval[IPPROTO]) { /* protocol-specific processing */
+        case IPPROTO_TCP:
+            if (p->orig_tcph == NULL) return;
+            pkt.fldval[TCPFLAGS]= p->orig_tcph->th_flags;
+            break;
+        case IPPROTO_UDP:
+            if (p->orig_udph == NULL) return;
+            break;
+        case IPPROTO_ICMP:
+            if (p->orig_icmph == NULL) return;
+            //pkt.fldval[ICMPTYPE]= p->orig_icmph->type;
+            //pkt.fldval[ICMPTYPECODE]= (p->orig_icmph->type << 8) | p->orig_icmph->code;
+            break;
+        default:;
+        }
+    
+        pkt.fldval[SIP]= ntohl(p->orig_iph->ip_src.s_addr);
+        pkt.fldval[DIP]= ntohl(p->orig_iph->ip_dst.s_addr);
+        pkt.fldval[SPORT]= p->orig_sp;
+        pkt.fldval[DPORT]= p->orig_dp;
+
+        netspade_new_pkt(spade,&pkt);
+    }
+}
+
+
+/*********************************************************************/
+/*********************************************************************/
+
+/* our netspade callback for when there is something anomalous to report */
+static void SpadeReportAnom(void *context,spade_report *rpt) {
+    char message[256];
+    Event event;
+    u_int32_t id;
+    spade_event *pkt= rpt->pkt;
+    Packet *p= pkt->native;
+    double score= spade_report_mainscore(rpt);
+
+    /*
+    sprintf(message,"Spade: %s: %s: %.4f",rpt->detect_type_str,rpt->scope_str,score);
+    */
+
+    /* OSSIM */
+    sprintf(message,"Spade: %s",rpt->detect_type_str);
+    
+    switch (rpt->detect_type) {
+    case SPADE_DR_TYPE_CLOSED_DPORT: id= SPADE_CLOSED_DESTPORT_USED; break;
+    case SPADE_DR_TYPE_DEAD_DEST: id= SPADE_NONLIVE_DEST_USED; break;
+    case SPADE_DR_TYPE_ODD_DPORT: id= SPADE_SRC_ODD_DESTPORT_USED; break;
+    case SPADE_DR_TYPE_ODD_TYPECODE: id= SPADE_SRC_ODD_TYPECODE_USED; break;
+    case SPADE_DR_TYPE_ODD_PORTDEST: id= SPADE_SRC_ODD_PORTDEST_USED; break;
+    default : id= 0;
+    }
+
+    /* OSSIM */
+    if (!strcmp(rpt->detect_type_str,"Rare but open dest port used"))
+      id = SPADE_RARE_OPEN_DESTPORT_USED;
+    else if (!strcmp(rpt->detect_type_str,"Rare dest port used"))
+      id = SPADE_RARE_DESTPORT_USED;
+
+    SetEvent(&event, GENERATOR_SPP_SPADE, id,
+            1, 0, 0, 0);
+
+#if ENABLE_IDMEF
+    if (spade_alert_dest & DEST_IDMEF_FACILITY)
+        SpadeIDMEFDirect(p, rpt->detect_type_str, NULL, &event, score, rpt->detectorid);
+#endif
+    if (spade_alert_dest & DEST_ALERT_FACILITY)
+        CallAlertFuncs(p, message, NULL, &event);
+    if (spade_alert_dest & DEST_LOG_FACILITY)
+        CallLogFuncs(p, message, NULL, &event);
+}   
+
+/* our netspade callback for when there the threshold is adjusted */
+static void SpadeReportThreshChanged(void *context,char *id,char *mess,int using_corrrscore) {
+    char message[100];
+    Event event;
+
+    sprintf(message,"Spade: id=%s: %s",id,mess);
+
+    SetEvent(&event, GENERATOR_SPP_SPADE,
+            SPADE_ANOM_THRESHOLD_ADJUSTED, 1, 0, 0, 0);
+    if (spade_adj_dest & DEST_ALERT_FACILITY)
+        CallAlertFuncs(NULL, message, NULL, &event);
+    if (spade_adj_dest & DEST_LOG_FACILITY)
+        CallLogFuncs(NULL, message, NULL, &event);
+}
+
+static void SnortSpadeMsgFn(spade_message_type msg_type,const char *msg) {
+    char buf[MAX_SPADE_MSG_LEN+1];
+    const char *newmsg;
+    
+    if (!pkt_count)  { // must be a message originating from configuration activity; include conf file name and line #
+        switch (msg_type) {
+        case SPADE_MSG_TYPE_FATAL:
+        case SPADE_MSG_TYPE_WARNING:
+        case SPADE_MSG_TYPE_DEBUG:
+        {
+            int len= strlen(msg);
+            strncpy(buf,msg,MAX_SPADE_MSG_LEN);
+            snprintf(buf+len,MAX_SPADE_MSG_LEN-len,": %s(%d)\n",file_name,file_line);
+            newmsg= (const char *)buf;
+            break;
+        }
+        default:
+            newmsg= msg;
+            break;
+        }
+    } else {
+        newmsg= msg;
+    }
+        
+    switch (msg_type) {
+    case SPADE_MSG_TYPE_FATAL:
+        FatalError("%s", newmsg);
+        break;
+    case SPADE_MSG_TYPE_WARNING:
+        ErrorMessage("%s", newmsg);
+        break;
+    default:
+        LogMessage("%s", newmsg);
+        break;
+    }
+}
+
+
+/*****************************************************
+ * Called on signals
+ *****************************************************/
+void SpadeCatchSig(int signal,void *arg) {
+    if (signal == SIGUSR1) {
+        LogMessage("Spade got SIGUSR1, refreshing its disk state");
+        netspade_dump(spade);
+    } else if (signal == SIGQUIT || signal == SIGHUP || signal == SIGINT) {
+        LogMessage("Spade got shutdown signal, cleaning up");
+        netspade_cleanup(spade);
+    }
+}
+
+/*********************************************************************
+netspade.c, distributed as part of Spade v030125.1
+Author: James Hoagland, Silicon Defense (hoagland@SiliconDefense.com)
+copyright (c) 2002 by Silicon Defense (http://www.silicondefense.com/)
+Released under GNU General Public License, see the COPYING file included
+with the distribution or http://www.silicondefense.com/spice/ for details.
+
+Please send complaints, kudos, and especially improvements and bugfixes to
+hoagland@SiliconDefense.com.  As described in GNU General Public License, no
+warranty is expressed for this program.
+*********************************************************************/
+
+/*! \file netspade.c
+ * \ingroup netspade_layer
+ * \brief 
+ *  netspade.c contains a "class" netspade which applies Spade to a network
+ */
+
+/*! \addtogroup libnetspade Netspade Library
+ * \brief This group contains objects the objects in libnetspade, which
+ * applies Spade to the network packets.
+  @{
+*/
+/*! \addtogroup libspade */
+/*@}*/
+
+/*! \addtogroup netspade_layer Netspade Layer
+ * \brief This group contains objects the objects specific to Netspade, which
+ * applies Spade to the network packets.
+ * \ingroup libnetspade
+  @{
+*/
+
+
+#include <netinet/in.h>
+#include <arpa/inet.h>
+#include <stdlib.h>
+#include <ctype.h>
+#include <string.h>
+#include <stdlib.h>
+#include <math.h>
+
+/// an array mapping a netspade feature number to its name
+const char *featurenames[NETSPADE_NUM_FEATURES+1]= {"sip","dip","sport","dport","proto","tcpflags","icmptype","icmptype+code",NULL};
+
+#define IS_TCP          EVENT_CONDITION_NUM(1) ///< is the packet TCP?
+#define IS_UDP          EVENT_CONDITION_NUM(2) ///< is the packet UDP?
+#define IS_ICMP         EVENT_CONDITION_NUM(3) ///< is the packet ICMP?
+#define IS_UNRCHTCP     EVENT_CONDITION_NUM(5) ///< is the packet from inside a unreachable packet and is TCP?
+#define IS_UNRCHUDP     EVENT_CONDITION_NUM(6) ///< is the packet from inside a unreachable packet and is UDP?
+#define IS_UNRCHICMP    EVENT_CONDITION_NUM(7) ///< is the packet from inside a unreachable packet and is ICMP?
+#define SYNONLY         EVENT_CONDITION_NUM(9) ///< is the packet TCP and only have the SYN flag set (among the 6 flags)?
+#define NORMAL_RST      EVENT_CONDITION_NUM(10) ///< is the packet TCP and a normal RST?
+#define SYNACK          EVENT_CONDITION_NUM(11) ///< is the packet TCP and only have the SYN and ACK flags set?
+#define WEIRDFLAGS      EVENT_CONDITION_NUM(12) ///< is the packet TCP and have an unusual combination of flags?
+#define SETUPFLAGS      EVENT_CONDITION_NUM(13) ///< is the packet TCP and have flags associated with connection setup?
+#define ESTFLAGS        EVENT_CONDITION_NUM(14) ///< is the packet TCP and have flags associated with the established phase of a connection?
+#define TEARDOWNFLAGS   EVENT_CONDITION_NUM(15) ///< is the packet TCP and have flags associated with connection teardown?
+#define SIP_IN_HOMENET      EVENT_CONDITION_NUM(17) ///< is the source IP of the packet in the homenet?
+#define SIP_NOT_IN_HOMENET  EVENT_CONDITION_NUM(18) ///< is the source IP of the packet not in the homenet?
+#define DIP_IN_HOMENET      EVENT_CONDITION_NUM(19) ///< is the dest IP of the packet in the homenet?
+#define DIP_NOT_IN_HOMENET  EVENT_CONDITION_NUM(20) ///< is the dest IP of the packet not in the homenet?
+#define ICMPNOTERR      EVENT_CONDITION_NUM(21) ///< is the packet an ICMP but not indicating an error?
+#define ICMPERR         EVENT_CONDITION_NUM(22) ///< is the packet an ICMP and not indicating an error?
+#define REPR_PKT        EVENT_CONDITION_NUM(23) ///< does this packet fit in the minority of packets that can be considered representative of the sources of packets?
+#define UDPRESP         EVENT_CONDITION_NUM(25) ///< might the packet be a response to a UDP packet?
+#define ICMPRESP        EVENT_CONDITION_NUM(26) ///< might the packet be a response to a ICMP packet?
+#define SYNRESP         EVENT_CONDITION_NUM(27) ///< might the packet be a response to a SYN packet?
+#define ESTRESP         EVENT_CONDITION_NUM(29) ///< might the packet be a response to a ESTFLAGS packet?
+#define TEARDOWNRESP    EVENT_CONDITION_NUM(30) ///< might the packet be a response to a TEARDOWN packet?
+#define SETUPRESP       EVENT_CONDITION_NUM(31) ///< might the packet be a response to a SETUP packet?
+
+#define HOMENET_CONDS (CONDS_PLUS_3CONDS(SIP_IN_HOMENET,SIP_NOT_IN_HOMENET,DIP_IN_HOMENET,DIP_NOT_IN_HOMENET))
+#define TCPFLAG_CONDS (CONDS_PLUS_6CONDS(SYNACK,SYNONLY,NORMAL_RST,WEIRDFLAGS,ESTFLAGS,TEARDOWNFLAGS,SETUPFLAGS))
+#define REPR_PKT_CONDS (CONDS_PLUS_3CONDS(SETUPFLAGS,TEARDOWNFLAGS,IS_UDP,IS_ICMP))
+
+#define SYNRESP_CONDS (CONDS_PLUS_2CONDS(SYNACK,NORMAL_RST,IS_UNRCHTCP))
+#define ESTRESP_CONDS (CONDS_PLUS_2CONDS(ESTFLAGS,TEARDOWNFLAGS,IS_UNRCHTCP))
+#define TEARDOWNRESP_CONDS (CONDS_PLUS_CONDS(TEARDOWNFLAGS,IS_UNRCHTCP))
+#define SETUPRESP_CONDS (CONDS_PLUS_CONDS(SYNRESP_CONDS,ESTRESP_CONDS))
+#define UDPRESP_CONDS (CONDS_PLUS_CONDS(IS_UDP,IS_UNRCHUDP))
+#define ICMPRESP_CONDS (CONDS_PLUS_CONDS(ICMPNOTERR,IS_UNRCHICMP))
+
+
+#define PKT_IP_IN_HOMENET_LIST(pkt,fldname,list,res) {\
+    res= 0; \
+    if (list != NULL) { \
+        ll_net *home; \
+        for (home= list; home != NULL; home= home->next) { \
+            if ((pkt->fldval[fldname] & home->netmask) == home->netaddr) { \
+                res= 1; \
+                break; \
+            } \
+        } \
+    } else { \
+        res= 1; \
+    } \
+}
+
+static void init_netspade_empty(netspade *self,spade_msg_fn msg_callback, int debug_level);
+static netspade_detector *acquire_detector_for_id(netspade *self, char *id);
+static netspade_detector *detector_for_id(netspade *self, char *id);
+static void netspade_detector_dump(netspade_detector *detector);
+static void netspade_detector_cleanup(netspade_detector *detector);
+static void netspade_update_conds_to_calc(netspade *self);
+static event_condition_set netspade_nonstore_conds(netspade *self);
+static event_condition_set flipped_homenet_conds(event_condition_set orig);
+static int do_checkpointing(netspade *self);
+static int do_recovery(netspade *self, char *statefile);
+static void threshold_was_exceeded(void *context, void *mgrref, spade_event *pkt, score_info *score);
+static void canceller_status_report(void *context, spade_report *rpt, port_status_t status);
+static void threshold_was_adjusted(void *context, void *mgrref);
+static void netspade_add_net_to_homenet(netspade *self, char *net_str);
+static char *scope_str_for_cond(event_condition_set cond);
+static void process_netspade_xarg(netspade *self, char *str, features feat, xarg_type_t type);
+static void process_detector_xargs(netspade_detector *d, char *xsips, char *xdips, char *xsports, char *xdports);
+static void process_detector_xarg(netspade_detector *d, char *str, features feat, xarg_type_t type);
+static void process_detector_xarg(netspade_detector *d,char *str,features feat,xarg_type_t type);
+static xfeatval_link *process_xarg(char *str,features feat,xarg_type_t type,spade_msg_fn msg_callback,xfeatval_link **tail);
+static xfeatval_link *new_xfeatval_link(features feat, xarg_type_t type, char *val);
+static int pkt_is_excluded(xfeatval_link *list,spade_event *pkt);
+static int cidr_to_netmask(char *str, u32 *netip, u32 *netmask);
+static void file_print_conds(FILE *file,event_condition_set conds);
+
+void init_netspade(netspade *self,spade_msg_fn msg_callback, int debug_level) {
+    init_netspade_empty(self,msg_callback,debug_level);
+}
+
+static void init_netspade_empty(netspade *self,spade_msg_fn msg_callback, int debug_level) {
+    self->msg_callback= (msg_callback == NULL) ? default_spade_msg_fn : msg_callback;
+    self->debug_level= debug_level;
+
+    self->detectors= NULL;
+    self->detectors_tail= NULL;
+    
+    self->homelist_head= NULL;
+    self->homelist_tail= NULL;
+    
+    self->checkpoint_file= NULL;
+    self->checkpoint_freq= -1;
+
+    self->records_since_checkpoint=0;
+    self->last_time_forwarded= (time_t)0;
+    
+    self->callback_context= NULL;
+    self->exc_callback= NULL;
+    self->adj_callback= NULL;
+    self->pkt_native_copier_callback= NULL;
+    self->pkt_native_freer_callback= NULL;
+
+    self->rpt_exclude_list= NULL;
+    
+    init_event_recorder(&self->recorder);
+    self->recorder_needed_conds= 0;
+    self->nonstore_conds= 0;
+    self->conds_to_calc= 0;
+    
+    self->stats_to_print= 0;
+    self->outfile= NULL;
+    
+    self->detector_id_nonce= 0;
+    self->total_pkts= 0;
+}
+
+int init_netspade_from_statefile(netspade *self,char *statefile,spade_msg_fn msg_callback, int debug_level) {
+    init_netspade_empty(self,msg_callback,debug_level);
+    return do_recovery(self,statefile);
+}
+
+netspade *new_netspade(spade_msg_fn msg_callback, int debug_level) {
+    netspade *new= (netspade *)malloc(sizeof(netspade));
+    init_netspade(new,msg_callback,debug_level);
+    return new;
+}
+
+netspade *new_netspade_from_statefile(char *statefile,spade_msg_fn msg_callback, int debug_level,int *succ) {
+    netspade *new= (netspade *)malloc(sizeof(netspade));
+    *succ= init_netspade_from_statefile(new,statefile,msg_callback,debug_level);
+    return new;
+}
+
+void netspade_set_callbacks(netspade *self,void *context,netspade_exc_callback_t exc_callback,netspade_adj_callback_t adj_callback,event_native_copier_t pkt_native_copier_callback,event_native_freer_t pkt_native_freer_callback) {
+    self->callback_context= context;
+    self->exc_callback= exc_callback;
+    self->adj_callback= adj_callback;
+    self->pkt_native_copier_callback= pkt_native_copier_callback;
+    self->pkt_native_freer_callback= pkt_native_freer_callback;
+}
+
+void netspade_set_checkpointing(netspade *self,char *checkpoint_file,int checkpoint_freq) {
+    self->checkpoint_file= (checkpoint_file == NULL) ? NULL : strdup(checkpoint_file);
+    self->checkpoint_freq= checkpoint_freq;
+}
+
+void netspade_set_homenet_from_str(netspade *self,char *homenet_str) {
+    char *strcopy= (homenet_str == NULL) ? NULL : strdup(homenet_str);
+    char *p= strcopy;
+    int len;
+    char oldchar,*term;
+    if (strcopy == NULL) return;
+
+    while (isspace((int)*p)) p++; /* kill leading whitepspace */
+    if (*p == '[') {
+        char *end;
+        p++;
+        end= strrchr(p, (int)']');
+        if (end != NULL) *end = '\0'; /* null out the end-bracket */
+    }
+    while ((len= terminate_first_tok(p,", \t\n",&p,&oldchar)) > 0) {
+        netspade_add_net_to_homenet(self,p);
+        term= p+len;
+        *term= oldchar;
+        if (oldchar == '\0') { break; } /* Thanks Risto! */
+        p= term+1;
+    }
+
+    free(strcopy);
+
+    if (self->debug_level) {
+        ll_net *n;
+        struct in_addr net;
+        (*self->msg_callback)(SPADE_MSG_TYPE_DEBUG,"Spade home nets are:\n");
+        for (n=self->homelist_head; n != NULL; n=n->next) {
+            net.s_addr= ntohl(n->netaddr);
+            formatted_spade_msg_send(SPADE_MSG_TYPE_DEBUG,self->msg_callback,"\t%s with mask %lx\n",inet_ntoa(net),(u_long)ntohl(n->netmask));
+        }
+    }
+}
+
+void netspade_set_output_stats(netspade *self,int stats_to_print) {
+    self->stats_to_print= stats_to_print;
+}
+
+void netspade_set_output_stats_from_str(netspade *self,char *str) {
+    char *strcopy= strdup(str);
+    char *head= strcopy;
+    char oldchar;
+    int len;
+    char *term;
+    
+    self->stats_to_print= STATS_NONE;
+    
+    while ((len= terminate_first_tok(head,", \t\n",&head,&oldchar)) > 0) {
+        if (!(strcmp(head,"entropy"))) {
+            self->stats_to_print |= STATS_ENTROPY;
+        } else if (!(strcmp(head,"condprob"))) {
+            self->stats_to_print |= STATS_CONDPROB;
+        } else if (!(strcmp(head,"uncondprob"))) {
+            self->stats_to_print |= STATS_UNCONDPROB;
+        } else {
+            // warn
+        }
+        term= head+len;
+        *term= oldchar;
+        head= term+1;
+    }
+    free(strcopy);
+}
+
+int netspade_set_output(netspade *self,char *file,int stats_to_print) {
+    self->outfile= strdup(file);
+    self->stats_to_print= stats_to_print;
+    return 1;
+}
+
+int netspade_set_output_file(netspade *self,char *file) {
+    self->outfile= strdup(file);
+    return 1;
+}
+
+void netspade_add_rpt_excludes(netspade *self,char *xsips,char *xdips,char *xsports,char *xdports) {
+    process_netspade_xarg(self,xsports,SPORT,XARG_TYPE_UINT);
+    process_netspade_xarg(self,xdips,DIP,XARG_TYPE_CIDR);
+    process_netspade_xarg(self,xsips,SIP,XARG_TYPE_CIDR);
+    process_netspade_xarg(self,xdports,DPORT,XARG_TYPE_UINT);
+}
+
+char *netspade_new_detector(netspade *self,char *str) {
+    netspade_detector *new;
+    char *strcopy= strdup(str);
+    char *type;
+    int calcboth;
+    int minobs_prefix_len= -1,entropy_prefix_len=-1;
+    int canceller_timeout_implication;
+    event_condition_set cancel_homenet_conds;
+    feature_list fla[4];
+    char to[8]="home",from[8]="home",protocol[5]="tcp";
+    char tcpflags[21]="synonly";
+    char xsips[401]="",xdips[401]="",xsports[401]="",xdports[401]="";
+    double thresh= 10000000;
+    int wait=0;
+    int relscore=1,minobs=0,probmode=3;
+    int scalefreqmins=240;
+    double scalefactor= 0.98363,scalecutoff= 0.18,scalehalflifehrs=-1;
+    int reverse_reporting=0;
+    double maxentropy= -1;
+    void *args[30];
+    char formatstr[500]="$i:wait;s50:id;i:minobs;"
+                "i:scalefreq;d:scalefactor;d:scalecutoff;d:scalehalflife;"
+                "s400:Xsips,Xsip,xsips;s400:Xdips,Xdip,xdips;"
+                "s400:Xsports,Xsport,xsports;s400:Xdports,Xdport,xdports;"
+                "b:revwaitrpt";
+    char id[51]="\0";
+    char defaultid[31];
+    sprintf(defaultid,"%d",++self->detector_id_nonce);
+    
+    args[0]= &wait;
+    args[1]= &id;
+    args[2]= &minobs;
+    args[3]= &scalefreqmins;
+    args[4]= &scalefactor;
+    args[5]= &scalecutoff;
+    args[6]= &scalehalflifehrs;
+    args[7]= &xsips;
+    args[8]= &xdips;
+    args[9]= &xsports;
+    args[10]= &xdports;
+    args[11]= &reverse_reporting;
+    
+    new= (netspade_detector *)malloc(sizeof(netspade_detector));
+    new->parent= self;
+    init_score_calculator_clear(&new->calculator,&self->recorder);
+    new->store_conds= 0;
+    new->scorecalc_conds= 0;
+    new->thresh_exc_port_impl= PORT_UNKNOWN;
+    PS_INIT_SET_WITH_STRONGER(new->port_report_criterea,PORT_UNKNOWN);
+    new->cancel_closed_conds= EVENT_CONDITION_FALSE;
+    new->cancel_open_conds= EVENT_CONDITION_FALSE;
+    new->report_scope_str= NULL;
+    new->exclude_broadcast_dip= 0;
+    
+    type= extract_str_arg_space_sep(strcopy,"type");
+
+    if (type == NULL) type= "closed-dport";
+    new->detect_type= SPADE_DR_TYPE_NUM4SHORT(type);
+    new->report_detection_type= DEFAULT_DN_TYPE_FOR_DR_TYPE(new->detect_type); // may be overridden below
+    
+    switch (new->detect_type) {
+    case SPADE_DR_TYPE_CLOSED_DPORT: {
+        int corrscore=1;
+        scalefactor= 0.96409; /* this detection type uses a different that normal scaling factor */
+        minobs= -1;
+        minobs_prefix_len= 0;
+        
+        new->thresh_exc_port_impl= PORT_PROBCLOSED;
+        PS_INIT_SET_WITH_STRONGER(new->port_report_criterea,PORT_PROBCLOSED); /* override default default; this will be overriden if wait is set */
+        
+        args[12]= &protocol;
+        args[13]= &to;
+        args[14]= &tcpflags;
+        args[15]= &thresh;
+        args[16]= &relscore;
+        args[17]= &probmode;
+        args[18]= &corrscore;
+        strcat(formatstr,";s4:protocol,proto;s7:to;s20:tcpflags;d:thresh;b:relscore;"
+                          "i:probmode;b:-corrscore,corrscore");
+        fill_args_space_sep(strcopy,formatstr,args,self->msg_callback);
+            
+        if (thresh == 10000000) thresh= relscore ? 0.85 : -1;
+        if (minobs == -1) minobs= relscore ? 400 : 0;
+        score_calculator_set_corrscore(&new->calculator,corrscore);
+
+        fla[0].feat[0]= DIP; fla[0].feat[1]= DPORT;
+        fla[0].feat[2]= SIP; fla[0].feat[3]= SPORT;
+        switch (probmode) {
+            case 0:
+                fla[0].num= 1; fla[0].feat[0]= DIP;
+                fla[1].num= 3; fla[1].feat[0]= SIP; fla[1].feat[1]= DPORT; fla[1].feat[2]= SPORT;
+                fla[2].num= 2; fla[2].feat[0]= SPORT; fla[2].feat[1]= DPORT;
+                fla[3].num= 3; fla[3].feat[0]= DIP; fla[3].feat[1]= SPORT; fla[3].feat[2]= SIP;
+                /* P(dport) * P(sip|dport,sport) * P(sport|dport) * P(dip|sport,sip) */
+                score_calculator_set_features(&new->calculator,4,fla,NULL,featurenames);
+                minobs= 0; /* prob mode 0 disables minobs; what would it mean? */
+                break;
+            case 1: 
+                fla[0].num= 4;
+                score_calculator_set_features(&new->calculator,1,fla,NULL,featurenames);
+                break;
+            case 2:
+                fla[0].num= 3;
+                score_calculator_set_features(&new->calculator,1,fla,NULL,featurenames);
+                break;
+            default:
+                if (probmode < 0 || probmode > 3) formatted_spade_msg_send(SPADE_MSG_TYPE_WARNING,self->msg_callback,"Probability mode %d not valid, using mode 3\n",probmode);
+            case 3:
+                fla[0].num= 2;
+                score_calculator_set_features(&new->calculator,1,fla,NULL,featurenames);
+                break;
+        }
+
+        score_calculator_set_condcutoff(&new->calculator,0);
+        
+        if (!strcmp(to,"any")) {
+            /* no restriction => no conditions to set */
+        } else if (!strcmp(to,"nothome")) {
+            ADD_TO_CONDS(new->scorecalc_conds,DIP_NOT_IN_HOMENET);
+            ADD_TO_CONDS(new->store_conds,DIP_NOT_IN_HOMENET);
+        } else {
+            if (strcmp(to,"home")) {
+                formatted_spade_msg_send(SPADE_MSG_TYPE_WARNING,self->msg_callback,"\"to\" setting %s not valid, using home\n",to);
+            }
+            ADD_TO_CONDS(new->scorecalc_conds,DIP_IN_HOMENET);
+            ADD_TO_CONDS(new->store_conds,DIP_IN_HOMENET);
+        }
+
+        if (!strcmp(protocol,"udp")) {
+            ADD_TO_CONDS(new->store_conds,IS_UDP);
+            ADD_TO_CONDS(new->scorecalc_conds,IS_UDP);
+            //new->cancel_open_conds= IS_UDP; /* waiting to see if might be open is optional */
+            new->cancel_closed_conds= IS_UNRCHUDP; /* waiting for a ICMP UNRCH to be sure it is a probe is optional */
+            //if (!wait) wait= 5;
+       } else {
+            if (strcmp(protocol,"tcp")) {
+                formatted_spade_msg_send(SPADE_MSG_TYPE_WARNING,self->msg_callback,"Protocol %s not valid, using tcp\n",protocol);
+            }
+            ADD_TO_CONDS(new->store_conds,SYNONLY);           /* only store tcp syns */
+            if (!strcmp(tcpflags,"weird")) {
+                ADD_TO_CONDS(new->scorecalc_conds,WEIRDFLAGS);
+                new->cancel_open_conds= EVENT_CONDITION_FALSE; /* not available */
+            } else if (!strcmp(tcpflags,"synack")) {
+                ADD_TO_CONDS(new->scorecalc_conds,SYNACK);
+                new->cancel_closed_conds= NORMAL_RST; /* we need to wait for a RST to be sure it is a probe */
+                if (!wait) wait= 5;
+            } else if (!strcmp(tcpflags,"established")) {
+                ADD_TO_CONDS(new->scorecalc_conds,ESTFLAGS);
+                new->cancel_closed_conds= NORMAL_RST; /* we need to wait for a RST to be sure it is a probe */
+                if (!wait) wait= 5;
+            } else if (!strcmp(tcpflags,"teardown")) {
+                ADD_TO_CONDS(new->scorecalc_conds,TEARDOWNFLAGS);
+                new->cancel_closed_conds= NORMAL_RST; /* we need to wait for a RST to be sure it is a probe */
+                if (!wait) wait= 5;
+            } else {
+                if (strcmp(tcpflags,"synonly")) {
+                    formatted_spade_msg_send(SPADE_MSG_TYPE_WARNING,self->msg_callback,"TCP flags %s not valid, using synonly\n",tcpflags);
+                }
+                ADD_TO_CONDS(new->scorecalc_conds,SYNONLY);
+                new->cancel_open_conds= SYNACK; /* waiting to see if might be open is optional */
+            }
+        }
+        
+        if (!wait) {
+            new->report_detection_type= SPADE_DN_TYPE_ODD_DPORT;
+        } else if (reverse_reporting) {
+            new->report_detection_type= SPADE_DN_TYPE_ODD_OPEN_DPORT;
+        }
+        break;
+    }
+    case SPADE_DR_TYPE_ODD_TYPECODE: {
+        char icmptype[7]="any";
+        thresh=0.9;
+        scalefactor= 0.96409; /* this detection type uses a different that normal scaling factor */
+        minobs=-1; /* this detection type uses a different that normal default minobs */
+        
+        minobs_prefix_len= 0;
+
+        args[12]= &to;
+        args[13]= &thresh;
+        args[14]= &icmptype;        
+        strcat(formatstr,";s7:to;d:thresh;s6:icmptype");
+        fill_args_space_sep(strcopy,formatstr,args,self->msg_callback);
+            
+        fla[0].num= 1; fla[0].feat[0]= ICMPTYPECODE;
+        score_calculator_set_features(&new->calculator,1,fla,NULL,featurenames);
+        score_calculator_set_condcutoff(&new->calculator,0);
+
+        if (!strcmp(to,"any")) {
+            /* no restriction => no conditions to set */
+        } else if (!strcmp(to,"nothome")) {
+            ADD_TO_CONDS(new->scorecalc_conds,DIP_NOT_IN_HOMENET);
+            ADD_TO_CONDS(new->store_conds,DIP_NOT_IN_HOMENET);
+        } else {
+            if (strcmp(to,"home")) {
+                formatted_spade_msg_send(SPADE_MSG_TYPE_WARNING,self->msg_callback,"\"to\" setting %s not valid, using home\n",from);
+            }
+            ADD_TO_CONDS(new->scorecalc_conds,DIP_IN_HOMENET);
+            ADD_TO_CONDS(new->store_conds,DIP_IN_HOMENET);
+        }
+
+        if (!strcmp(icmptype,"noterr")) {
+            ADD_TO_CONDS(new->scorecalc_conds,ICMPNOTERR);
+            ADD_TO_CONDS(new->store_conds,ICMPNOTERR);
+            if (minobs == -1) minobs= 2000;
+        } else if (!strcmp(icmptype,"err")) {
+            ADD_TO_CONDS(new->scorecalc_conds,ICMPERR);
+            ADD_TO_CONDS(new->store_conds,ICMPERR);
+            if (minobs == -1) minobs= 2000;
+        } else {
+            if (strcmp(icmptype,"any")) {
+                formatted_spade_msg_send(SPADE_MSG_TYPE_WARNING,self->msg_callback,"ICMP type %s not valid, using any\n",icmptype);
+            }
+            ADD_TO_CONDS(new->scorecalc_conds,IS_ICMP);
+            ADD_TO_CONDS(new->store_conds,IS_ICMP);
+            if (minobs == -1) minobs= 4000;
+        }
+
+        new->cancel_open_conds= ICMPNOTERR;
+        
+        break;
+    }
+    case SPADE_DR_TYPE_ODD_DPORT: {
+        thresh=0.8;
+        minobs=600; /* this detection type uses a different that normal default minobs */
+        
+        args[12]= &protocol;
+        args[13]= &from;
+        args[14]= &thresh;
+        strcat(formatstr,";s4:protocol,proto;s7:from;d:thresh");
+        fill_args_space_sep(strcopy,formatstr,args,self->msg_callback);
+            
+        fla[0].num= 2; fla[0].feat[0]= SIP; fla[0].feat[1]= DPORT;
+        score_calculator_set_features(&new->calculator,1,fla,NULL,featurenames);
+        score_calculator_set_condcutoff(&new->calculator,1);
+
+        if (!strcmp(from,"any")) {
+            /* no restriction => no conditions to set */
+        } else if (!strcmp(from,"nothome")) {
+            ADD_TO_CONDS(new->scorecalc_conds,SIP_NOT_IN_HOMENET);
+            ADD_TO_CONDS(new->store_conds,SIP_NOT_IN_HOMENET);
+        } else {
+            if (strcmp(from,"home")) {
+                formatted_spade_msg_send(SPADE_MSG_TYPE_WARNING,self->msg_callback,"\"from\" setting %s not valid, using home\n",from);
+            }
+            ADD_TO_CONDS(new->scorecalc_conds,SIP_IN_HOMENET);
+            ADD_TO_CONDS(new->store_conds,SIP_IN_HOMENET);
+        }
+
+        /* this detection type only makes sense for connection-openers */    
+        if (!strcmp(protocol,"udp")) {
+            ADD_TO_CONDS(new->store_conds,IS_UDP);
+            ADD_TO_CONDS(new->scorecalc_conds,IS_UDP);
+            //new->cancel_open_conds= IS_UDP;
+            new->cancel_closed_conds= IS_UNRCHUDP; /* waiting for a ICMP UNRCH is optional */
+        } else {
+            if (strcmp(protocol,"tcp")) {
+                formatted_spade_msg_send(SPADE_MSG_TYPE_WARNING,self->msg_callback,"Protocol %s not valid, using tcp\n",protocol);
+            }
+            ADD_TO_CONDS(new->store_conds,SYNONLY);           /* only store tcp syns */
+            ADD_TO_CONDS(new->scorecalc_conds,SYNONLY);
+            new->cancel_open_conds= SYNACK;
+        }
+        
+        break;
+    }
+    case SPADE_DR_TYPE_ODD_PORTDEST: {
+        thresh=0.9;
+        minobs=-1; /* this detection type uses a different that normal default minobs */
+        maxentropy= 2.5;
+        scalefreqmins= 90; /* override the default defaults */
+        scalefactor= 0.97957;
+        scalecutoff= 0.25;
+        
+        args[12]= &protocol;
+        args[13]= &from;
+        args[14]= &thresh;
+        args[15]= &maxentropy;
+        strcat(formatstr,";s4:protocol,proto;s7:from;d:thresh;d:maxentropy");
+        fill_args_space_sep(strcopy,formatstr,args,self->msg_callback);
+
+        fla[0].num= 3; fla[0].feat[0]= SIP; fla[0].feat[1]= DPORT; fla[0].feat[2]= DIP;
+        score_calculator_set_features(&new->calculator,1,fla,NULL,featurenames);
+        score_calculator_set_condcutoff(&new->calculator,1);
+
+        if (!strcmp(from,"any")) {
+            /* no restriction => no conditions to set */
+        } else if (!strcmp(from,"nothome")) {
+            ADD_TO_CONDS(new->scorecalc_conds,SIP_NOT_IN_HOMENET);
+            ADD_TO_CONDS(new->store_conds,SIP_NOT_IN_HOMENET);
+        } else {
+            if (strcmp(from,"home")) {
+                formatted_spade_msg_send(SPADE_MSG_TYPE_WARNING,self->msg_callback,"\"from\" setting %s not valid, using home\n",from);
+            }
+            ADD_TO_CONDS(new->scorecalc_conds,SIP_IN_HOMENET);
+            ADD_TO_CONDS(new->store_conds,SIP_IN_HOMENET);
+        }
+
+        /* this detection type only makes sense for connection-openers */    
+        if (!strcmp(protocol,"udp")) {
+            ADD_TO_CONDS(new->store_conds,IS_UDP);
+            ADD_TO_CONDS(new->scorecalc_conds,IS_UDP);
+            //new->cancel_open_conds= IS_UDP;
+            new->cancel_closed_conds= IS_UNRCHUDP; /* waiting for a ICMP UNRCH is optional */
+
+            if (minobs == -1) minobs= pow(2,maxentropy) * 200; /* default is 200 times the minimum number of observations need to acheive maxentropy */
+        } else {
+            if (strcmp(protocol,"tcp")) {
+                formatted_spade_msg_send(SPADE_MSG_TYPE_WARNING,self->msg_callback,"Protocol %s not valid, using tcp\n",protocol);
+            }
+            ADD_TO_CONDS(new->store_conds,SYNONLY);           /* only store tcp syns */
+            ADD_TO_CONDS(new->scorecalc_conds,SYNONLY);
+            new->cancel_open_conds= SYNACK;
+
+            if (minobs == -1) minobs= pow(2,maxentropy) * 100; /* default is 100 times the minimum number of observations need to acheive maxentropy */
+        }
+        
+        break;
+    }
+    case SPADE_DR_TYPE_DEAD_DEST: {
+        feature_list cfl;
+        char icmptype[7]="noterr";
+        scalefreqmins= 60; /* override the default defaults */
+        scalefactor= 0.94387;
+        scalecutoff= 0.25;
+        wait= 2;
+        minobs= 2000;
+        minobs_prefix_len= 0;
+        thresh= 1;
+        
+        new->exclude_broadcast_dip= 1;
+        new->thresh_exc_port_impl= PORT_PROBCLOSED;
+        ADD_TO_CONDS(new->scorecalc_conds,DIP_IN_HOMENET);
+        ADD_TO_CONDS(new->store_conds,SIP_IN_HOMENET);
+        ADD_TO_CONDS(new->store_conds,REPR_PKT);
+
+        fla[0].num= 1; fla[0].feat[0]= SIP;
+        cfl.num= 1; cfl.feat[0]= DIP;
+        score_calculator_set_features(&new->calculator,1,fla,&cfl,featurenames);
+        score_calculator_set_corrscore(&new->calculator,1);
+        
+        args[12]= &protocol;
+        args[13]= &tcpflags;        
+        args[14]= &icmptype;        
+        strcat(formatstr,";s4:protocol,proto;s20:tcpflags;s6:icmptype");
+        fill_args_space_sep(strcopy,formatstr,args,self->msg_callback);
+
+        score_calculator_set_condcutoff(&new->calculator,0);
+
+        if (!strcmp(protocol,"udp")) {
+            ADD_TO_CONDS(new->scorecalc_conds,IS_UDP);
+            new->cancel_open_conds= UDPRESP; /* waiting to see if might be live is optional */
+        } else if (!strcmp(protocol,"icmp")) {
+            new->cancel_open_conds= ICMPRESP; /* waiting to see if might be live is optional */
+            if (!strcmp(icmptype,"any")) {
+                ADD_TO_CONDS(new->scorecalc_conds,IS_ICMP);
+            } else if (!strcmp(icmptype,"err")) {
+                ADD_TO_CONDS(new->scorecalc_conds,ICMPERR);
+            } else {
+                if (strcmp(icmptype,"noterr")) {
+                    formatted_spade_msg_send(SPADE_MSG_TYPE_WARNING,self->msg_callback,"ICMP type %s not valid, using noterr\n",icmptype);
+                }
+                ADD_TO_CONDS(new->scorecalc_conds,ICMPNOTERR);
+            }
+        } else {
+            if (strcmp(protocol,"tcp")) {
+                formatted_spade_msg_send(SPADE_MSG_TYPE_WARNING,self->msg_callback,"Protocol %s not valid, using tcp\n",protocol);
+            }
+            if (!strcmp(tcpflags,"weird")) {
+                ADD_TO_CONDS(new->scorecalc_conds,WEIRDFLAGS);
+                new->cancel_open_conds= EVENT_CONDITION_FALSE; /* not available */
+            } else if (!strcmp(tcpflags,"setup")) {
+                ADD_TO_CONDS(new->scorecalc_conds,SETUPFLAGS);
+                new->cancel_open_conds= SETUPRESP;
+            } else if (!strcmp(tcpflags,"synack")) {
+                ADD_TO_CONDS(new->scorecalc_conds,SYNACK);
+                new->cancel_open_conds= ESTRESP;
+            } else if (!strcmp(tcpflags,"established")) {
+                ADD_TO_CONDS(new->scorecalc_conds,ESTFLAGS);
+                new->cancel_open_conds= ESTRESP;
+            } else if (!strcmp(tcpflags,"teardown")) {
+                ADD_TO_CONDS(new->scorecalc_conds,TEARDOWNFLAGS);
+                new->cancel_open_conds= TEARDOWNRESP;
+            } else {
+                if (strcmp(tcpflags,"synonly")) {
+                    formatted_spade_msg_send(SPADE_MSG_TYPE_WARNING,self->msg_callback,"TCP flags %s not valid, using synonly\n",tcpflags);
+                }
+                ADD_TO_CONDS(new->scorecalc_conds,SYNONLY);
+                new->cancel_open_conds= SYNRESP;
+            }
+        }
+
+        break;
+    }
+    default:
+        free(new);
+        formatted_spade_msg_send(SPADE_MSG_TYPE_WARNING,self->msg_callback,"detector type \"%s\" not recognized, not enabling this detector: %s",type,str);
+        return NULL;
+    } 
+
+    /* finish new->store_conds,scorecalc_conds and cancel_open_conds */
+    
+    if (wait > 0 && (CONDS_NOT_FALSE(new->cancel_open_conds) ||  CONDS_NOT_FALSE(new->cancel_closed_conds))) {
+        int canceller_response_implication;
+        int report_timeout;
+        if (CONDS_NOT_FALSE(new->cancel_closed_conds)) { /* waiting is for closed */
+            canceller_timeout_implication= PORT_UNKNOWN;
+            canceller_response_implication= PORT_CLOSED;
+            new->cancel_open_conds= EVENT_CONDITION_FALSE;
+            report_timeout= 0 || reverse_reporting;
+        } else { /* waiting is for open */
+            canceller_timeout_implication= PORT_LIKELYCLOSED;
+            canceller_response_implication= PORT_OPEN;
+            new->cancel_closed_conds= EVENT_CONDITION_FALSE;
+            report_timeout= 1 && !reverse_reporting;
+        }
+        if (report_timeout) {
+            PS_INIT_SET(new->port_report_criterea,canceller_timeout_implication);
+        } else {
+            PS_INIT_SET_WITH_STRONGER(new->port_report_criterea,canceller_response_implication);
+        }
+        new->canceller= new_packet_resp_canceller(wait,&canceller_status_report,new,canceller_timeout_implication);
+    } else {
+        new->canceller= NULL;
+        new->cancel_open_conds= EVENT_CONDITION_FALSE;
+        new->cancel_closed_conds= EVENT_CONDITION_FALSE;
+    }
+    /* check if restrictions on report's homenet, need to flip for cancelling */
+    cancel_homenet_conds= flipped_homenet_conds(new->scorecalc_conds);
+    if (CONDS_NOT_FALSE(new->cancel_open_conds))
+        ADD_TO_CONDS(new->cancel_open_conds,cancel_homenet_conds);
+    if (CONDS_NOT_FALSE(new->cancel_closed_conds))
+        ADD_TO_CONDS(new->cancel_closed_conds,cancel_homenet_conds);
+
+    score_calculator_set_storage_conditions(&new->calculator,new->store_conds);
+    
+    calcboth= 0; /*relscore ? 0 : 1;*/
+    score_calculator_set_relscore(&new->calculator,relscore || calcboth,relscore);
+    score_calculator_set_rawscore(&new->calculator,!relscore || calcboth,!relscore);
+    if (scalehalflifehrs >= 0) // set factor based on halflife and frequency
+        scalefactor= exp((scalefreqmins/(scalehalflifehrs*60))*log(0.5));
+    score_calculator_set_scaling(&new->calculator,scalefreqmins*60,scalefactor,scalecutoff);
+    if (minobs > 0) {
+        if (minobs_prefix_len < 0) minobs_prefix_len+= fla[0].num;
+        score_calculator_set_min_obs(&new->calculator,minobs_prefix_len,minobs);
+    }
+    if (maxentropy >= 0) {
+        if (entropy_prefix_len < 0) entropy_prefix_len+= fla[0].num;
+        score_calculator_set_low_entropy_domain(&new->calculator,entropy_prefix_len,maxentropy);
+    }
+    init_spade_enviro(&new->enviro,thresh,&self->total_pkts);
+    init_score_mgr(&new->mgr, new, &new->enviro, self,
+                threshold_was_exceeded, threshold_was_adjusted,self->msg_callback);
+    
+    process_detector_xargs(new,xsips,xdips,xsports,xdports);
+                
+    if (id[0] == '\0' || detector_for_id(self,id)) {
+        if (id[0] != '\0') formatted_spade_msg_send(SPADE_MSG_TYPE_WARNING,self->msg_callback,"Detector with id=%s already exists; using id=%s instead on: %s",id,defaultid,str);
+        new->id= strdup(defaultid);
+    } else {
+        new->id= strdup(id);
+    }
+    new->last_adj_stats.scored= 0;
+    new->last_adj_stats.reported= 0;
+    
+    /* append new detector to the list */
+    new->next= NULL;
+    if (self->detectors == NULL) { /* first entry */
+        self->detectors= new;
+    } else {
+        self->detectors_tail->next= new;
+    }
+    self->detectors_tail= new;
+    
+    netspade_detector_scope_str(self,new->id);
+                
+    free(strcopy);
+    return new->id;
+}
+
+int netspade_set_detector_scaling(netspade *self,char *detectorid,int scale_freq,double scale_factor,double prune_threshold) {
+    netspade_detector *detector;
+    detector= acquire_detector_for_id(self,detectorid);
+    score_calculator_set_scaling(&detector->calculator,scale_freq,scale_factor,prune_threshold);
+    return 1;
+}
+
+char *netspade_setup_detector_adapt_from_str(netspade *self,int adaptmode,char *str) {
+    char *strcopy= strdup(str);
+    char *detectorid= extract_str_arg_space_sep(strcopy,"id");
+    netspade_detector *detector= acquire_detector_for_id(self,detectorid);
+    score_mgr_setup_adapt_from_str(&detector->mgr,adaptmode,strcopy);
+    free(strcopy);
+    return detector->id;
+}
+
+int netspade_setup_detector_adapt1(netspade *self,char *detectorid,int adapttarget, time_t period, float new_obs_weight, int by_count) {
+    netspade_detector *detector;
+    detector= acquire_detector_for_id(self,detectorid);
+    score_mgr_setup_adapt1(&detector->mgr,adapttarget,period,new_obs_weight,by_count);
+    return 1;
+}
+
+int netspade_setup_detector_adapt2(netspade *self,char *detectorid,double targetspec, double obsper, int NS, int NM, int NL) {
+    netspade_detector *detector;
+    detector= acquire_detector_for_id(self,detectorid);
+    score_mgr_setup_adapt2(&detector->mgr,targetspec,obsper,NS,NM,NL);
+    return 1;
+}
+
+int netspade_setup_detector_adapt3(netspade *self,char *detectorid,double targetspec, double obsper, int NO) {
+    netspade_detector *detector;
+    detector= acquire_detector_for_id(self,detectorid);
+    score_mgr_setup_adapt3(&detector->mgr,targetspec,obsper,NO);
+    return 1;
+}
+
+int netspade_setup_detector_advise(netspade *self,char *detectorid,int obs_size, int obs_secs) {
+    netspade_detector *detector;
+    detector= acquire_detector_for_id(self,detectorid);
+    score_mgr_setup_advise(&detector->mgr,obs_size,obs_secs);
+    return 1;
+}
+
+char *netspade_setup_detector_advise_from_str(netspade *self,char *str) {
+    char *strcopy= strdup(str);
+    char *detectorid= extract_str_arg_space_sep(strcopy,"id");
+    netspade_detector *detector= acquire_detector_for_id(self,detectorid);
+    score_mgr_setup_advise_from_str(&detector->mgr,strcopy);
+    free(strcopy);
+    return detector->id;
+}
+
+int netspade_setup_detector_survey(netspade *self,char *detectorid,char *filename,float interval) {
+    netspade_detector *detector;
+    detector= acquire_detector_for_id(self,detectorid);
+    score_mgr_setup_survey(&detector->mgr,filename,interval);
+    return 1;
+}
+
+char *netspade_setup_detector_survey_from_str(netspade *self,char *str) {
+    char *strcopy= strdup(str);
+    char *detectorid= extract_str_arg_space_sep(strcopy,"id");
+    netspade_detector *detector= acquire_detector_for_id(self,detectorid);
+    score_mgr_setup_survey_from_str(&detector->mgr,strcopy);
+    free(strcopy);
+    return detector->id;
+}
+
+/* called frequently, should be efficient esp for packets we don't care about */
+void netspade_new_pkt(netspade *self,spade_event *pkt) {
+    event_condition_set pkt_conds=0;
+    int write_log= 0;
+    int ip_in_homenet;
+    features orig_sip,orig_dip,orig_sport,orig_dport;
+    netspade_detector *detector;
+    int new_sec= (self->last_time_forwarded < (time_t)pkt->time);
+
+    if (self->last_time_forwarded == 0) { /* first packet */
+        if (self->detectors == NULL)
+            netspade_new_detector(self,"relscore=0 corrscore=0");
+        for (detector= self->detectors; detector != NULL; detector=detector->next) {
+            score_calculator_init_complete(&detector->calculator); /* make sure calculator is all set up */
+        }
+    }
+
+//printf("packet time is %.4f\n",pkt->time);
+
+    /* update packet counts and tell detector of new time */
+    self->total_pkts++;
+    if (new_sec) {
+        for (detector= self->detectors; detector != NULL; detector=detector->next) {
+            if (score_mgr_new_time(&detector->mgr,pkt->time)) write_log=1; /* advising completed */
+            if (detector->canceller != NULL)
+                packet_resp_canceller_new_time(detector->canceller,pkt->time);
+            detector->enviro.now= (time_t)pkt->time;
+        }
+        event_recorder_new_time(&self->recorder,(time_t)pkt->time);
+        self->last_time_forwarded= (time_t)pkt->time;
+    }
+    
+    /* calculate the conditions that this packet satisfies; no need to calculate any conditions we don't care about (i.e., not on recorder_needed_conds or nonstore_conds) */
+    if (!self->nonstore_conds || !self->recorder_needed_conds)
+        netspade_update_conds_to_calc(self);
+    if (pkt->origin == PKTORIG_UNRCH) {
+        orig_sip= DIP;
+        orig_dip= SIP;
+        orig_sport= DPORT;
+        orig_dport= SPORT;
+        if (pkt->fldval[IPPROTO] == IPPROTO_TCP) 
+             ADD_TO_CONDS(pkt_conds,IS_UNRCHTCP);
+        else if (pkt->fldval[IPPROTO] == IPPROTO_UDP) 
+            ADD_TO_CONDS(pkt_conds,IS_UNRCHUDP);
+        else if (pkt->fldval[IPPROTO] == IPPROTO_ICMP) 
+            ADD_TO_CONDS(pkt_conds,IS_UNRCHICMP);
+    } else {
+        orig_sip= SIP;
+        orig_dip= DIP;
+        orig_sport= SPORT;
+        orig_dport= DPORT;
+        if (pkt->fldval[IPPROTO] == IPPROTO_TCP) {
+            u8 tcpflags= pkt->fldval[TCPFLAGS] & 0x3F; /* strip off reserved bits */
+            ADD_TO_CONDS(pkt_conds,IS_TCP);
+            if (tcpflags == 0x02) {
+                ADD_TO_CONDS(pkt_conds,SYNONLY);
+            } else if (tcpflags == 0x12) {
+                ADD_TO_CONDS(pkt_conds,SYNACK);
+            } else {
+                if (!(tcpflags & 0x16)) { /* no syn, no ack, and no rst */
+                    ADD_TO_CONDS(pkt_conds,WEIRDFLAGS);
+                } else if (tcpflags & 0x10) { /* has ack */
+                    event_condition_set tmp= tcpflags & 0x07; /* now strip out A,P,U */
+                    if (tmp != 0x00 && tmp != 0x01 && tmp != 0x04) /* not more than one of F or R */
+                        ADD_TO_CONDS(pkt_conds,WEIRDFLAGS);
+                } else if (!((tcpflags == 0x01) || (tcpflags == 0x02) || (tcpflags == 0x04))) { /* no ack, but no S,F,or R */
+                    ADD_TO_CONDS(pkt_conds,WEIRDFLAGS);
+                }
+            }
+            if (!SOME_CONDS_MET(pkt_conds,WEIRDFLAGS)) {
+                event_condition_set tmp= (pkt->fldval[TCPFLAGS] & 0x07); /* strip off all but S,F,R */
+                switch (tmp) {
+                case 0x00: ADD_TO_CONDS(pkt_conds,ESTFLAGS); break;
+                case 0x02: ADD_TO_CONDS(pkt_conds,SETUPFLAGS); break;
+                case 0x01: ADD_TO_CONDS(pkt_conds,TEARDOWNFLAGS); break;
+                case 0x04: ADD_TO_CONDS(pkt_conds,CONDS_PLUS_CONDS(NORMAL_RST,TEARDOWNFLAGS)); break;
+                default:;
+                }
+            }
+        } else if (pkt->fldval[IPPROTO] == IPPROTO_UDP) {
+            ADD_TO_CONDS(pkt_conds,IS_UDP);
+        } else if (pkt->fldval[IPPROTO] == IPPROTO_ICMP) {
+            ADD_TO_CONDS(pkt_conds,IS_ICMP);
+            if ((pkt->fldval[ICMPTYPE] < 3 || pkt->fldval[ICMPTYPE] > 5) &&
+                pkt->fldval[ICMPTYPE] != 11 && pkt->fldval[ICMPTYPE] != 12) {
+                ADD_TO_CONDS(pkt_conds,ICMPNOTERR);
+            } else {
+                ADD_TO_CONDS(pkt_conds,ICMPERR);
+            }
+        }
+    }
+    if (SOME_CONDS_MET(pkt_conds,REPR_PKT_CONDS))
+        ADD_TO_CONDS(pkt_conds,REPR_PKT);
+    if (SOME_CONDS_MET(pkt_conds,UDPRESP_CONDS))
+        ADD_TO_CONDS(pkt_conds,UDPRESP);
+    if (SOME_CONDS_MET(pkt_conds,ICMPRESP_CONDS))
+        ADD_TO_CONDS(pkt_conds,ICMPRESP);
+    if (SOME_CONDS_MET(pkt_conds,ONLY_CONDS(self->conds_to_calc,CONDS_PLUS_3CONDS(SYNRESP_CONDS,ESTRESP_CONDS,TEARDOWNRESP_CONDS,SETUPRESP_CONDS)))) {
+        /* did that test so only TCP and ICMP unreachable will go in here */
+        if (SOME_CONDS_MET(pkt_conds,SYNRESP_CONDS))
+            ADD_TO_CONDS(pkt_conds,SYNRESP);
+        if (SOME_CONDS_MET(pkt_conds,ESTRESP_CONDS))
+            ADD_TO_CONDS(pkt_conds,ESTRESP);
+        if (SOME_CONDS_MET(pkt_conds,TEARDOWNRESP_CONDS))
+            ADD_TO_CONDS(pkt_conds,TEARDOWNRESP);
+        if (SOME_CONDS_MET(pkt_conds,SETUPRESP_CONDS))
+            ADD_TO_CONDS(pkt_conds,SETUPRESP);
+    }
+
+    
+    if (SOME_CONDS_MET(self->conds_to_calc,CONDS_PLUS_CONDS(DIP_IN_HOMENET,DIP_NOT_IN_HOMENET))) {
+        PKT_IP_IN_HOMENET_LIST(pkt,orig_dip,self->homelist_head,ip_in_homenet);
+        ADD_TO_CONDS(pkt_conds,(ip_in_homenet ? DIP_IN_HOMENET : DIP_NOT_IN_HOMENET));
+    }
+    if (SOME_CONDS_MET(self->conds_to_calc,CONDS_PLUS_CONDS(SIP_IN_HOMENET,SIP_NOT_IN_HOMENET))) {
+        PKT_IP_IN_HOMENET_LIST(pkt,orig_sip,self->homelist_head,ip_in_homenet);
+        ADD_TO_CONDS(pkt_conds,(ip_in_homenet ? SIP_IN_HOMENET : SIP_NOT_IN_HOMENET));
+    }
+    
+    if (SOME_CONDS_MET(pkt_conds,self->nonstore_conds)) { /* might match something to calculate or cancel */
+        /* check for scoring and cancelling in each detector */
+        int portless= (pkt->fldval[IPPROTO] != IPPROTO_TCP) && (pkt->fldval[IPPROTO] != IPPROTO_UDP);
+        for (detector= self->detectors; detector != NULL; detector=detector->next) {
+            if (ALL_CONDS_MET(pkt_conds,detector->scorecalc_conds) && (!detector->exclude_broadcast_dip || ((pkt->fldval[DIP] & 0xFF) != 0xFF))) {
+                score_info score;
+                int enoughobs;
+                score_info *res= score_calculator_calc_event_score(&detector->calculator,pkt,&score,&enoughobs);
+                if (!enoughobs || res != NULL) { // ignore events we decided not to apply this detector to */
+                    detector->enviro.pkt_stats.scored++;
+                    if (enoughobs) { // res != NULL
+                        score_mgr_new_event(&detector->mgr,&score,pkt);
+                    } else {  // !enoughobs
+                        detector->enviro.pkt_stats.insuffobsed++;
+                    }
+                }
+            }
+            if (ALL_CONDS_MET(pkt_conds,detector->cancel_open_conds)) {
+                detector->enviro.pkt_stats.respchecked++;
+                packet_resp_canceller_note_response(detector->canceller,PORT_OPEN,
+                    pkt->fldval[orig_dip],pkt->fldval[orig_dport],
+                    pkt->fldval[orig_sip],pkt->fldval[orig_sport],portless);
+            }
+            if (ALL_CONDS_MET(pkt_conds,detector->cancel_closed_conds)) {
+                detector->enviro.pkt_stats.respchecked++;
+                packet_resp_canceller_note_response(detector->canceller,PORT_CLOSED,
+                    pkt->fldval[orig_dip],pkt->fldval[orig_dport],
+                    pkt->fldval[orig_sip],pkt->fldval[orig_sport],portless);
+            }
+        }
+    }
+    if (SOME_CONDS_MET(pkt_conds,self->recorder_needed_conds)) { /* might match something to record */
+        self->records_since_checkpoint+=
+            event_recorder_new_event(&self->recorder,pkt,pkt_conds);
+    }
+    
+    if (write_log) { /* time to write the log */
+        netspade_write_log(self);
+    }
+    if ((self->checkpoint_freq > 0) && (self->records_since_checkpoint >= self->checkpoint_freq)) { // see if its time to checkpoint
+        do_checkpointing(self); // should report err if returns 0
+        self->records_since_checkpoint= 0;
+    }
+}
+
+
+void netspade_dump(netspade *self) 
+{
+    netspade_detector *detector;
+    netspade_write_log(self);
+    for (detector= self->detectors; detector != NULL; detector=detector->next) {
+        netspade_detector_dump(detector);
+    }
+    if (self->checkpoint_file != NULL) do_checkpointing(self);
+}
+
+void netspade_cleanup(netspade *self) 
+{
+    netspade_detector *detector;
+    netspade_write_log(self);
+    for (detector= self->detectors; detector != NULL; detector=detector->next) {
+        netspade_detector_cleanup(detector);
+    }
+    if (self->checkpoint_file != NULL) do_checkpointing(self);
+}
+
+char *netspade_detector_scope_str(netspade *self,char *id) {
+    int i;
+    char *str,*tail;
+    int numconds= 0;
+    netspade_detector *d= detector_for_id(self,id);
+    if (d == NULL) return NULL;
+    if (d->report_scope_str != NULL) return d->report_scope_str;
+    
+    for (i=0; i < 31; i++)
+        if (SOME_CONDS_MET(d->scorecalc_conds,EVENT_CONDITION_NUM(i))) numconds++;
+    
+    str= (char *)malloc(sizeof(char)*(numconds*(15+2)+1));
+    if (str == NULL) return NULL;
+    
+    tail= str;
+    for (i=31; i >= 0; i--)
+        if (SOME_CONDS_MET(d->scorecalc_conds,EVENT_CONDITION_NUM(i))) {
+            char* new= scope_str_for_cond(EVENT_CONDITION_NUM(i));
+            if (new == NULL) continue; /* nothing to add */
+            if (tail != str) {
+                *tail= ',';
+                *(tail+1)= ' ';
+                tail+= 2;
+            }
+            strcpy(tail,new);
+            tail+= strlen(new);
+        }
+
+   d->report_scope_str= str;
+   return str;         
+}
+
+static void condprinter(FILE *file,event_condition_set conds) {
+    int i,first=1;
+    for (i=31; i >= 0; i--)
+        if (SOME_CONDS_MET(conds,EVENT_CONDITION_NUM(i))) {
+            if (first) {
+                first= 0;
+            } else {
+                fprintf(file,", ");
+            }
+            fprintf(file,"%s",scope_str_for_cond(EVENT_CONDITION_NUM(i)));
+        }
+}
+
+static char *scope_str_for_cond(event_condition_set cond) {
+    switch (cond) {
+        case IS_TCP: return "tcp";
+        case IS_UDP: return "udp";
+        case IS_ICMP: return "icmp";
+        case ICMPNOTERR: return "non-err icmp";
+        case ICMPERR: return "error icmp";
+        case IS_UNRCHTCP: return "returned tcp";
+        case IS_UNRCHUDP: return "returned udp";
+        case IS_UNRCHICMP: return "returned icmp";
+        case SYNONLY: return "syn";
+        case NORMAL_RST: return "rst";
+        case SYNACK: return "synack";
+        case WEIRDFLAGS: return "weird flags";
+        case SETUPFLAGS: return "setup flags";
+        case ESTFLAGS: return "est. flags";
+        case TEARDOWNFLAGS: return "teardown flags";
+        case SIP_IN_HOMENET: return "local source";
+        case SIP_NOT_IN_HOMENET: return "nonlocal source";
+        case DIP_IN_HOMENET: return "local dest";
+        case DIP_NOT_IN_HOMENET: return "nonlocal dest";
+        case UDPRESP: return "udp response";
+        case ICMPRESP: return "icmp response";
+        case SYNRESP: return "syn response";
+        case ESTRESP: return "est. response";
+        case TEARDOWNRESP: return "teardown response";
+        case SETUPRESP: return "setup response";
+        case REPR_PKT: return "representative";
+        default: return NULL;
+    }
+}
+
+static netspade_detector *acquire_detector_for_id(netspade *self,char *id) {
+    netspade_detector *detector;
+    if (id == NULL) {
+        if (self->detectors == NULL) netspade_new_detector(self,"id=default");
+        return self->detectors_tail; /* return most recent detector */
+    }
+    detector= detector_for_id(self,id);
+    if (detector == NULL) {
+        char detect_str[45];
+        sprintf(detect_str,"id=%s",id);
+        netspade_new_detector(self,detect_str);
+        detector= detector_for_id(self,id);
+    }
+    return detector;
+}
+
+static netspade_detector *detector_for_id(netspade *self,char *id) {
+    netspade_detector *detector;
+    for (detector= self->detectors; detector != NULL; detector=detector->next) {
+        if (!strcmp(detector->id,id)) {
+            return detector;
+        }
+    }
+    return NULL;
+}
+
+static void netspade_detector_dump(netspade_detector *detector) {
+    score_mgr_dump(&detector->mgr);
+}
+static void netspade_detector_cleanup(netspade_detector *detector) {
+    score_mgr_cleanup(&detector->mgr);
+}
+
+static void netspade_update_conds_to_calc(netspade *self) {
+    if (!self->recorder_needed_conds)
+        self->recorder_needed_conds= event_recorder_needed_conds(&self->recorder);
+    if (!self->nonstore_conds)
+        self->nonstore_conds= netspade_nonstore_conds(self);
+        
+    self->conds_to_calc= CONDS_PLUS_CONDS(self->recorder_needed_conds,self->nonstore_conds);
+}
+
+static event_condition_set netspade_nonstore_conds(netspade *self) {
+    netspade_detector *detector;
+    event_condition_set nonstore_conds= 0;
+    for (detector= self->detectors; detector != NULL; detector=detector->next) {
+        ADD_TO_CONDS(nonstore_conds,CONDS_PLUS_2CONDS(detector->scorecalc_conds,detector->cancel_open_conds,detector->cancel_closed_conds));
+    }
+    return nonstore_conds;
+}
+
+static event_condition_set flipped_homenet_conds(event_condition_set orig) {
+    event_condition_set flipped=0;
+    if (SOME_CONDS_MET(orig,SIP_IN_HOMENET)) {
+        ADD_TO_CONDS(flipped,DIP_IN_HOMENET);
+    }
+    if (SOME_CONDS_MET(orig,SIP_NOT_IN_HOMENET)) {
+        ADD_TO_CONDS(flipped,DIP_NOT_IN_HOMENET);
+    }
+    if (SOME_CONDS_MET(orig,DIP_IN_HOMENET)) {
+        ADD_TO_CONDS(flipped,SIP_IN_HOMENET);
+    }
+    if (SOME_CONDS_MET(orig,DIP_NOT_IN_HOMENET)) {
+        ADD_TO_CONDS(flipped,SIP_NOT_IN_HOMENET);
+    }
+    return flipped;
+}
+
+
+static int do_checkpointing(netspade *self) {
+    statefile_ref *ref= spade_state_begin_checkpointing(self->checkpoint_file,"netspade",2);
+    if (ref == NULL) return 0;
+    
+    return event_recorder_checkpoint(&self->recorder,ref)
+        /* could checkpoint detectors in here */
+        && spade_state_end_checkpointing(ref);
+}
+
+static int do_recovery(netspade *self,char *statefile) {
+    char *appname;
+    u8 file_app_fvers;
+    
+    statefile_ref *ref= spade_state_begin_recovery(statefile,2,&appname,&file_app_fvers);
+    if (ref == NULL) return 0;
+    if (strcmp(appname,"netspade")) return 0;
+    
+    return event_recorder_merge_recover(&self->recorder,ref)
+        /* would checkpoint detectors in here; if we checkpointed that */
+        && spade_state_end_recovery(ref);
+}
+
+static void threshold_was_exceeded(void *context,void *mgrref,spade_event *pkt,score_info *score) {
+    netspade *self= (netspade *)context;
+    netspade_detector *detector= (netspade_detector *)mgrref;
+    char *id= detector->id;
+    port_status_t port_status= detector->thresh_exc_port_impl;
+    
+    /* first check if this report should be excluded */
+    if (pkt_is_excluded(self->rpt_exclude_list,pkt) ||
+            pkt_is_excluded(detector->rpt_exclude_list,pkt)) {
+        detector->enviro.pkt_stats.excluded++;
+        return;
+    } else {
+        detector->enviro.pkt_stats.nonexcluded++;
+    }
+    
+    if (PS_IN_SET(detector->port_report_criterea,port_status)) {
+        spade_report *rpt= new_spade_report(pkt,score,detector->detect_type,id,SPADE_DN_TYPE_MEDDESCR4NUM(detector->report_detection_type),netspade_detector_scope_str(self,id),&detector->enviro.pkt_stats,port_status);
+        (*(self->exc_callback))(self->callback_context,rpt);
+        free_spade_report(rpt);
+        detector->enviro.pkt_stats.reported++;
+    } else if (detector->canceller != NULL) {
+        /* we didn't meet criterea for reporting yet, and we have a canceller avail, so use it */
+        spade_event *newpkt= spade_event_clone(pkt,self->pkt_native_copier_callback,self->pkt_native_freer_callback);
+        score_info *newscore= score_info_clone(score);
+        spade_report *rpt= new_spade_report(newpkt,newscore,detector->detect_type,id,SPADE_DN_TYPE_MEDDESCR4NUM(detector->report_detection_type),netspade_detector_scope_str(self,id),&detector->enviro.pkt_stats,port_status);
+        packet_resp_canceller_add_report(detector->canceller,rpt);
+        detector->enviro.pkt_stats.waited++;
+    } else {
+        /* drop the report since no canceller available to strengthen it; that was silly */
+    }
+}
+
+static void canceller_status_report(void *context,spade_report *rpt,port_status_t status) {
+    netspade_detector *d= (netspade_detector *)context;
+    netspade *self= d->parent;
+    if (self->debug_level > 1) formatted_spade_msg_send(SPADE_MSG_TYPE_DEBUG,self->msg_callback,"canceller_status_report(%p,%p %8x:%d %8x:%d,%s)\n",d,rpt,rpt->pkt->fldval[SIP],rpt->pkt->fldval[SPORT],rpt->pkt->fldval[DIP],rpt->pkt->fldval[DPORT],PORT_STATUS_AS_STR(status));
+    if (PS_IN_SET(d->port_report_criterea,status)) {
+        /* met one of the critea so report it */
+        rpt->port_status= status;
+        (*(self->exc_callback))(self->callback_context,rpt);
+        if (rpt->stream_stats) {
+            rpt->stream_stats->reported++;
+        }
+    }
+    /* free report and pkt */
+    free_score_info(rpt->score);
+    free_spade_event(rpt->pkt);
+    free_spade_report(rpt);
+}
+
+static void threshold_was_adjusted(void *context,void *mgrref) {
+    char message[85];
+    spade_pkt_stats adj_period_stats;
+    netspade *self= (netspade *)context;
+    netspade_detector *detector= (netspade_detector *)mgrref;
+    int using_corrscore;
+    
+    adj_period_stats.scored= detector->enviro.pkt_stats.scored - detector->last_adj_stats.scored;
+    adj_period_stats.reported= detector->enviro.pkt_stats.reported - detector->last_adj_stats.reported;
+    detector->last_adj_stats= detector->enviro.pkt_stats; // copy over current stats for reference on next call
+    
+    sprintf(message,"Threshold adjusted to %.4f after %d alerts (of %d)",detector->enviro.thresh,adj_period_stats.reported,adj_period_stats.scored);
+
+    using_corrscore= score_calculator_using_corrscore(&detector->calculator);
+    (*(self->adj_callback))(self->callback_context,detector->id,message,using_corrscore);
+}
+
+
+static void netspade_add_net_to_homenet(netspade *self,char *net_str) {
+    ll_net *cur=NULL;
+
+    cur= (ll_net *)malloc(sizeof(ll_net));
+    cur->next= NULL;
+    if (self->homelist_head == NULL) {
+        self->homelist_head= cur;
+    } else {
+        self->homelist_tail->next= cur;
+    }
+    self->homelist_tail= cur;
+    
+    if (!cidr_to_netmask(net_str,&cur->netaddr,&cur->netmask)) {
+        formatted_spade_msg_send(SPADE_MSG_TYPE_FATAL,self->msg_callback,"Could not interpret homenet network: %s",net_str);
+    }
+}
+
+
+
+void netspade_write_log(netspade *self) {
+    FILE *file;
+    netspade_detector *detector;
+
+    if (!strcmp(self->outfile,"-")) {
+        file= stdout;
+    } else {
+        file = fopen(self->outfile, "w");
+        if (!file) formatted_spade_msg_send(SPADE_MSG_TYPE_FATAL,self->msg_callback,"netspade: unable to open %s",self->outfile);
+    }
+
+    fprintf(file,"%ld total packets were processed by spade in this run\n\n",self->total_pkts);
+    for (detector= self->detectors; detector != NULL; detector=detector->next) {
+        spade_pkt_stats *stats= &detector->enviro.pkt_stats;
+        int scored= stats->scored;
+        int was_anom= stats->nonexcluded + stats->excluded;
+        fprintf(file,"** %s: %s (id=%s) **\n",SPADE_DN_TYPE_MEDDESCR4NUM(detector->report_detection_type),detector->report_scope_str,detector->id);
+        fprintf(file,"%d packets were evaluated\n",stats->scored);
+        fprintf(file,"  %d (%.2f%%) packets were considered anomalous\n",was_anom,((was_anom/(float)scored)*100));
+        if (stats->excluded > 0) 
+            fprintf(file,"    %d (%.2f%%) packets were dropped due to configured exclusion\n",stats->excluded,((stats->excluded/(float)scored)*100));
+        if (stats->waited > 0) 
+            fprintf(file,"    %d (%.2f%%) packets were inserted in the wait queue\n",stats->waited,((stats->waited/(float)scored)*100));
+        fprintf(file,"    %d (%.2f%%) packets were reported as alerts\n",stats->reported,((stats->reported/(float)scored)*100));
+        if (stats->insuffobsed > 0) 
+            fprintf(file,"  %d (%.2f%%) packets did not have enough observations\n",stats->insuffobsed,((stats->insuffobsed/(float)scored)*100));
+        if (stats->respchecked > 0) 
+            fprintf(file,"  %d packets were checked against the wait queue\n",stats->respchecked);
+        fprintf(file,"%d observations were stored\n",score_calculator_get_store_count(&detector->calculator));
+        fprintf(file,"%.4f observations are remembered\n",score_calculator_get_obs_count(&detector->calculator));
+        score_mgr_file_print_log(&detector->mgr,file);
+        fprintf(file,"\n");
+    }
+    fflush(file);
+    
+    if (self->stats_to_print != STATS_NONE)
+        event_recorder_write_stats(&self->recorder,file,self->stats_to_print,condprinter);
+        
+    if (file != stdout) {
+        fclose(file);
+    }
+}
+
+void print_conds_line(event_condition_set conds) {
+    file_print_conds(stdout,conds);
+    printf("\n");
+}
+
+void print_conds(event_condition_set conds) {
+    file_print_conds(stdout,conds);
+}
+
+static void file_print_conds(FILE *f,event_condition_set conds) {
+    int first= 1;
+    if (SOME_CONDS_MET(conds,EVENT_CONDITION_FALSE)) {
+        fprintf(f,"FALSE");
+        return;
+    }
+    fprintf(f,"{");
+    if (SOME_CONDS_MET(conds,IS_TCP)) {
+        if (!first) { fprintf(f,","); } first=0;
+        fprintf(f,"TCP");
+    }
+    if (SOME_CONDS_MET(conds,IS_UDP)) {
+        if (!first) { fprintf(f,","); } first=0;
+        fprintf(f,"UDP");
+    }
+    if (SOME_CONDS_MET(conds,IS_ICMP)) {
+        if (!first) { fprintf(f,","); } first=0;
+        fprintf(f,"ICMP");
+    }
+    if (SOME_CONDS_MET(conds,ICMPNOTERR)) {
+        if (!first) { fprintf(f,","); } first=0;
+        fprintf(f,"ICMPNOTERR");
+    }
+    if (SOME_CONDS_MET(conds,ICMPERR)) {
+        if (!first) { fprintf(f,","); } first=0;
+        fprintf(f,"ICMPERR");
+    }
+    if (SOME_CONDS_MET(conds,IS_UNRCHTCP)) {
+        if (!first) { fprintf(f,","); } first=0;
+        fprintf(f,"IS_UNRCHTCP");
+    }
+    if (SOME_CONDS_MET(conds,IS_UNRCHUDP)) {
+        if (!first) { fprintf(f,","); } first=0;
+        fprintf(f,"IS_UNRCHUDP");
+    }
+    if (SOME_CONDS_MET(conds,IS_UNRCHICMP)) {
+        if (!first) { fprintf(f,","); } first=0;
+        fprintf(f,"IS_UNRCHICMP");
+    }
+    if (SOME_CONDS_MET(conds,NORMAL_RST)) {
+        if (!first) { fprintf(f,","); } first=0;
+        fprintf(f,"NORMAL_RST");
+    }
+    if (SOME_CONDS_MET(conds,SYNACK)) {
+        if (!first) { fprintf(f,","); } first=0;
+        fprintf(f,"SYNACK");
+    }
+    if (SOME_CONDS_MET(conds,WEIRDFLAGS)) {
+        if (!first) { fprintf(f,","); } first=0;
+        fprintf(f,"WEIRDFLAGS");
+    }
+    if (SOME_CONDS_MET(conds,SETUPFLAGS)) {
+        if (!first) { fprintf(f,","); } first=0;
+        fprintf(f,"SETUP");
+    }
+    if (SOME_CONDS_MET(conds,ESTFLAGS)) {
+        if (!first) { fprintf(f,","); } first=0;
+        fprintf(f,"EST");
+    }
+    if (SOME_CONDS_MET(conds,TEARDOWNFLAGS)) {
+        if (!first) { fprintf(f,","); } first=0;
+        fprintf(f,"TEARDOWN");
+    }
+    if (SOME_CONDS_MET(conds,SIP_IN_HOMENET)) {
+        if (!first) { fprintf(f,","); } first=0;
+        fprintf(f,"SIP_IS_HOME");
+    }
+    if (SOME_CONDS_MET(conds,SIP_NOT_IN_HOMENET)) {
+        if (!first) { fprintf(f,","); } first=0;
+        fprintf(f,"!SIP_IS_HOME");
+    }
+    if (SOME_CONDS_MET(conds,DIP_IN_HOMENET)) {
+        if (!first) { fprintf(f,","); } first=0;
+        fprintf(f,"DIP_IS_HOME");
+    }
+    if (SOME_CONDS_MET(conds,DIP_NOT_IN_HOMENET)) {
+        if (!first) { fprintf(f,","); } first=0;
+        fprintf(f,"!DIP_IS_HOME");
+    }
+    if (SOME_CONDS_MET(conds,REPR_PKT)) {
+        if (!first) { fprintf(f,","); } first=0;
+        fprintf(f,"REPR_PKT");
+    }
+    if (SOME_CONDS_MET(conds,UDPRESP)) {
+        if (!first) { fprintf(f,","); } first=0;
+        fprintf(f,"UDPRESP");
+    }
+    if (SOME_CONDS_MET(conds,ICMPRESP)) {
+        if (!first) { fprintf(f,","); } first=0;
+        fprintf(f,"ICMPRESP");
+    }
+    if (SOME_CONDS_MET(conds,SYNRESP)) {
+        if (!first) { fprintf(f,","); } first=0;
+        fprintf(f,"SYNRESP");
+    }
+    if (SOME_CONDS_MET(conds,ESTRESP)) {
+        if (!first) { fprintf(f,","); } first=0;
+        fprintf(f,"ESTRESP");
+    }
+    if (SOME_CONDS_MET(conds,TEARDOWNRESP)) {
+        if (!first) { fprintf(f,","); } first=0;
+        fprintf(f,"TEARDOWNRESP");
+    }
+    if (SOME_CONDS_MET(conds,SETUPRESP)) {
+        if (!first) { fprintf(f,","); } first=0;
+        fprintf(f,"SETUPRESP");
+    }
+    fprintf(f,"}");
+}
+
+int netspade_print_detector_config_details(netspade *self,FILE *f,char *id) {
+    netspade_detector *d= detector_for_id(self,id);
+    if (d == NULL) return 0;
+    
+    fprintf(f,"id=%s; detect_type=%s\n",d->id,SPADE_DR_TYPE_SHORT4NUM(d->detect_type));
+    fprintf(f,"scorecalc_conds= ");
+    file_print_conds(f,d->scorecalc_conds);
+    fprintf(f,"; store_conds= ");
+    file_print_conds(f,d->store_conds);
+    fprintf(f,"\n");
+
+    fprintf(f,"calculator=\n");
+    score_calculator_print_config_details(&d->calculator,f,"  ");
+    fprintf(f,"mgr=\n");
+    score_mgr_print_config_details(&d->mgr,f,"  ");
+    
+    fprintf(f,"port_report_criterea=");
+    port_status_set_file_print(d->port_report_criterea,f);
+    fprintf(f,"\n");
+    if (d->canceller != NULL) {
+        fprintf(f,"canceller=\n");
+        packet_resp_canceller_print_config_details(d->canceller,f,"  ");
+        fprintf(f,"cancel_open_conds= ");
+        file_print_conds(f,d->cancel_open_conds);
+        fprintf(f,"; cancel_closed_conds= ");
+        file_print_conds(f,d->cancel_closed_conds);
+        fprintf(f,"\n");
+    }
+    
+    fprintf(f,"report_detection_type=%d\n",d->report_detection_type);
+    return 1;
+}
+
+static void process_netspade_xarg(netspade *self,char *str,features feat,xarg_type_t type) {
+    xfeatval_link *list,*list_tail;
+
+    list= process_xarg(str,feat,type,self->msg_callback,&list_tail);
+    if (list == NULL) return;
+    
+    /* prepend this list onto the detector */
+    list_tail->next= self->rpt_exclude_list;
+    self->rpt_exclude_list= list;
+}
+
+static void process_detector_xargs(netspade_detector *d,char *xsips,char *xdips,char *xsports,char *xdports) {
+    d->rpt_exclude_list= NULL;
+    process_detector_xarg(d,xsports,SPORT,XARG_TYPE_UINT);
+    process_detector_xarg(d,xdips,DIP,XARG_TYPE_CIDR);
+    process_detector_xarg(d,xsips,SIP,XARG_TYPE_CIDR);
+    process_detector_xarg(d,xdports,DPORT,XARG_TYPE_UINT);
+}
+
+static void process_detector_xarg(netspade_detector *d,char *str,features feat,xarg_type_t type) {
+    xfeatval_link *list,*list_tail;
+
+    list= process_xarg(str,feat,type,d->parent->msg_callback,&list_tail);
+    if (list == NULL) return;
+    
+    /* prepend this list onto the detector */
+    list_tail->next= d->rpt_exclude_list;
+    d->rpt_exclude_list= list;
+}
+
+static xfeatval_link *process_xarg(char *str,features feat,xarg_type_t type,spade_msg_fn msg_callback,xfeatval_link **tail) {
+    xfeatval_link *list=NULL,*new;
+    char *val;
+    
+    *tail= NULL;
+
+    if (str == NULL || str[0] == '\0') return NULL;
+    val= strtok(str,",");
+    while (val != NULL) {
+        new= new_xfeatval_link(feat,type,val);
+        if (new == NULL) {
+            if (type == XARG_TYPE_CIDR)
+                formatted_spade_msg_send(SPADE_MSG_TYPE_WARNING,msg_callback,"could not parse %s as a IP or network to exclude for detector %s; skipping it");
+            else
+                formatted_spade_msg_send(SPADE_MSG_TYPE_WARNING,msg_callback,"could not parse %s as an unsigned integer to exclude for detector %s; skipping it");
+            continue;
+        }
+        
+        /* append */
+        if (list == NULL) {
+            list= new;
+        } else {
+            (*tail)->next= new;
+        }
+        *tail= new;
+        
+        val= strtok(NULL,",");
+    }
+    return list;
+}
+
+static xfeatval_link *new_xfeatval_link(features feat,xarg_type_t type,char *val) {
+    xfeatval_link *new= (xfeatval_link *)malloc(sizeof(xfeatval_link));
+    if (new == NULL) return NULL;
+    
+    new->feat= feat;
+    new->type= type;
+    new->next= NULL;
+    
+    if (type == XARG_TYPE_CIDR) {
+        if (!cidr_to_netmask(val,&new->val.cidr.netip,&new->val.cidr.netmask)) {
+            free(new);
+            return NULL;
+        }
+    } else { // XARG_TYPE_UINT
+        if (val[0] < '0' || val[0] > '9') {
+            free(new);
+            return NULL;
+        }
+        new->val.i= atoi(val);
+    }
+    return new;
+}
+
+static int pkt_is_excluded(xfeatval_link *list,spade_event *pkt) {
+    xfeatval_link *x;
+    for (x= list; x != NULL; x= x->next) {
+        u32 pktval= pkt->fldval[x->feat];
+        if ((x->type == XARG_TYPE_UINT)
+            ? (x->val.i == pktval)
+            : ((pktval & x->val.cidr.netmask) == x->val.cidr.netip)) { /* found one to exclude */
+            return 1; 
+        }
+    }
+    return 0;
+}
+
+static int cidr_to_netmask(char *str,u32 *netip,u32 *netmask) {
+    char *sep;
+    int masklen;
+    struct in_addr net;
+    char *strcopy= strdup(str);
+    
+    if ((sep= strchr(strcopy,'/')) == NULL) {
+        masklen= 32;
+    } else { 
+        *sep= '\0'; // null terminate IP
+        masklen = atoi(sep+1);
+    }
+
+    if ((masklen >= 0) && (masklen <= 32)) {
+        *netmask = (~((u32)0))<<(32-masklen);
+    } else {
+        free(strcopy);
+        return 0;
+    }
+
+    /* convert the IP addr into its 32-bit value */
+    if ((net.s_addr = inet_addr(strcopy)) ==-1) {
+        if (!strcmp(strcopy,"any")) {
+            *netmask= 0;
+            *netip= 0;
+        } else {
+            free(strcopy);
+            return 0;
+        }
+    } else {
+        *netip = (ntohl((u_long)net.s_addr) & *netmask);
+    }
+    free(strcopy);
+    return 1;
+}
+
+/*@}*/
+
+/* Id: netspade.c,v 1.1 2004/10/11 14:35:54 jonkman Exp $ */
+/*********************************************************************
+packet_resp_canceller.c, distributed as part of Spade v030125.1
+Author: James Hoagland, Silicon Defense (hoagland@SiliconDefense.com)
+copyright (c) 2002 by Silicon Defense (http://www.silicondefense.com/)
+Released under GNU General Public License, see the COPYING file included
+with the distribution or http://www.silicondefense.com/spice/ for details.
+
+packet_resp_canceller.c implements the "class" packet_resp_canceller which
+  buffers spade reports for a period of time, awaiting information that
+  the port is open
+
+Please send complaints, kudos, and especially improvements and bugfixes to
+hoagland@SiliconDefense.com.  As described in GNU General Public License, no
+warranty is expressed for this program.
+*********************************************************************/
+
+/*! \file packet_resp_canceller.c
+ * \ingroup netspade_layer
+ * \brief 
+ *  packet_resp_canceller.c implements the "class" packet_resp_canceller
+ *  which buffers spade reports for a period of time, awaiting information
+ *  that the port is open
+ */
+
+/*! \addtogroup netspade_layer
+    @{
+*/
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <netinet/in.h>
+
+//static void lt_extract(packet_resp_canceller *self, prc_link *todel, prc_link *prev, u16 hash1, u32 hash2);
+static void ltl_extract(prc_lookup_table *lt, prc_link *todel, prc_link *prev, u16 hash1, u32 hash2);
+static int lt_delete_report(prc_lookup_table *lt, spade_report *rpt);
+static prc_link *new_prc_link(spade_report *rpt);
+static void free_prc_ttl_list(prc_link *head);
+//static void free_prc_link(prc_link *l);
+static prc_lookup_table2 *new_prc_lookup_table2(void);
+static void init_prc_lookup_table2(prc_lookup_table2 *t);
+static void free_prc_lookup_table2(prc_lookup_table2 *t);
+static void free_prc_lookup_table2_clean(prc_lookup_table2 *t);
+
+#define u8_right_rotate(i,bits) ((i >> bits) | ((i & ((1 << bits)-1)) << (8-bits)))
+
+#define calc_hash1(sport,dport) ((sport ^ dport) & LOOKUP_TABLE1_MASK)
+#define calc_hash2(sip,dip,hash) { \
+    u32 _tmp= sip ^ dip; \
+    u8 _tmp1= _tmp & LOOKUP_TABLE2_MASK; \
+    u8 _tmp2= (_tmp & LOOKUP_TABLE2_MASK) >> 8; \
+    u8 _tmp3= (_tmp & LOOKUP_TABLE2_MASK) >> 16; \
+    u8 _tmp4= (_tmp & LOOKUP_TABLE2_MASK) >> 24; \
+    hash= _tmp1 \
+        ^  u8_right_rotate(_tmp2,2) \
+        ^  u8_right_rotate(_tmp3,4) \
+        ^  u8_right_rotate(_tmp4,6); \
+}
+
+/// free list of allocated prc_lookup_table2s
+prc_lookup_table2 *prc_lookup_table2_freelist= NULL;
+/// free list of allocated prc_links
+prc_link *prc_link_freelist= NULL;
+
+//int disp_hashinfo= 0;
+
+void init_packet_resp_canceller(packet_resp_canceller *self,int wait_secs,prc_report_status_fn status_callback,void *callback_context,port_status_t timeout_implication) {
+    int i;
+    
+    self->tt.num_buckets= wait_secs+1;
+    self->tt.last_timeout= (time_t)0;
+    self->tt.arr= (prc_list *)malloc(sizeof(prc_list)*self->tt.num_buckets);
+    for (i= 0; i <= wait_secs; i++) {
+        self->tt.arr[i].head= NULL;
+        self->tt.arr[i].tail= NULL;
+    }
+    
+    for (i= 0; i < LOOKUP_TABLE1_SIZE; i++) self->lt.arr[i]= NULL;
+    
+    self->status_callback= status_callback;
+    self->callback_context= callback_context;
+    self->timeout_implication= timeout_implication;
+}
+
+packet_resp_canceller *new_packet_resp_canceller(int wait_secs,prc_report_status_fn status_callback,void *callback_context,port_status_t timeout_implication) {
+    packet_resp_canceller *new= (packet_resp_canceller *)malloc(sizeof(packet_resp_canceller));
+    init_packet_resp_canceller(new,wait_secs,status_callback,callback_context,timeout_implication);
+    return new;
+}
+
+void free_packet_resp_canceller(packet_resp_canceller *self) {
+    int i;
+    /* free all the prc_link's */
+    for (i=0; i < self->tt.num_buckets; i++)
+        if (self->tt.arr[i].head != NULL)
+            free_prc_ttl_list(self->tt.arr[i].head);
+    
+    /* free the timeout table's array */
+    free(self->tt.arr);
+    
+    /* free all the prc_lookup_table2's */
+    for (i=0; i < LOOKUP_TABLE1_SIZE; i++)
+        if (self->lt.arr[i] != NULL)
+            free_prc_lookup_table2(self->lt.arr[i]);
+
+    /* free ourself */
+    free(self);
+}
+
+void packet_resp_canceller_new_time(packet_resp_canceller *self,time_t now) {
+    int i;
+    prc_link *l;
+    int count= now - self->tt.last_timeout;
+    //if (self->debug_level > 1) printf("packet_resp_canceller_new_time(%p,%d)\n",self,(int)now);
+    if (count > self->tt.num_buckets) count= self->tt.num_buckets;
+    for (i=1; i <= count; i++) {
+        int slot= (self->tt.last_timeout+i) % self->tt.num_buckets;
+        if (self->tt.arr[slot].head != NULL) {
+            for (l= self->tt.arr[slot].head; l != NULL; l= l->ttl_next) {
+                if (l->rpt != NULL) {
+                    /* send the report as closed and delete this from the lookup table */
+                    (*self->status_callback)(self->callback_context,l->rpt,self->timeout_implication);
+                    lt_delete_report(&self->lt,l->rpt);
+                }
+            }
+            free_prc_ttl_list(self->tt.arr[slot].head);
+            self->tt.arr[slot].head= NULL;
+            self->tt.arr[slot].tail= NULL;
+        }
+    }
+    self->tt.last_timeout= now;
+}
+
+void packet_resp_canceller_add_report(packet_resp_canceller *self,spade_report *rpt) {
+    spade_event *pkt= rpt->pkt;
+    u16 hash1;
+    u32 hash2;
+    prc_lookup_table2 *t2;
+    prc_link *new;
+    int slot;
+    /*if (self->debug_level) printf("packet_resp_canceller_add_report(%p,%p %.2f %8x:%d %8x:%d)\n",self,rpt,rpt->pkt->time,rpt->pkt->fldval[SIP],rpt->pkt->fldval[SPORT],rpt->pkt->fldval[DIP],rpt->pkt->fldval[DPORT]);*/
+    
+    if ((pkt->fldval[IPPROTO] == IPPROTO_TCP) || (pkt->fldval[IPPROTO] == IPPROTO_UDP))
+        hash1= calc_hash1(pkt->fldval[SPORT],pkt->fldval[DPORT]);
+    else
+        hash1= calc_hash1(pkt->fldval[SIP],pkt->fldval[DIP]);
+    //if (disp_hashinfo) printf(": addrep hash1(%d,%d) => %d\n",pkt->fldval[SPORT],pkt->fldval[DPORT],hash1);
+    if (self->lt.arr[hash1] == NULL) self->lt.arr[hash1]= new_prc_lookup_table2();
+    t2= self->lt.arr[hash1];
+    calc_hash2(pkt->fldval[SIP],pkt->fldval[DIP],hash2);
+    //if (disp_hashinfo) printf(": addrep hash2(%08x,%08x) => %d\n",pkt->fldval[SIP],pkt->fldval[DIP],hash2);
+    new= new_prc_link(rpt);
+    if (t2->arr[hash2] == NULL) { // first entry here
+        t2->num_used++;
+    } else {
+        new->ltl_next= t2->arr[hash2];
+    }
+    t2->arr[hash2]= new;
+    
+    // append in time table
+    slot= ((int)pkt->time) % self->tt.num_buckets;
+    if (self->tt.arr[slot].tail == NULL) {
+        self->tt.arr[slot].head= new;
+    } else {
+        self->tt.arr[slot].tail->ttl_next= new;
+    }
+    self->tt.arr[slot].tail= new;
+    //fflush(stdout);
+}
+
+void packet_resp_canceller_note_response(packet_resp_canceller *self,port_status_t implied_status,u32 sip,u16 sport,u32 dip,u16 dport,int portless) {
+    prc_link *l,*prev,*next,*newprev;
+    u32 hash2;
+    prc_lookup_table2 *t2;
+    u16 hash1= portless ? calc_hash1(sip,dip) : calc_hash1(sport,dport);
+    //if (disp_hashinfo) printf(": noteresp hash1(%d,%d) => %d\n",sport,dport,hash1);
+    /*if (self->debug_level) printf("packet_resp_canceller_note_response(%p,%s,%8x:%d %8x:%d,%d)\n",self,PORT_STATUS_AS_STR(implied_status),sip,sport,dip,dport,portless);*/
+    
+    if (self->lt.arr[hash1] == NULL) return;
+    t2= self->lt.arr[hash1];
+    calc_hash2(sip,dip,hash2);
+    //if (disp_hashinfo) printf(": noteresp hash2(%08x,%08x) => %d\n",sip,dip,hash2);
+    if (t2->arr[hash2] == NULL) return;
+    for (l= t2->arr[hash2], prev=NULL; l != NULL; prev=newprev,l=next) {
+        next= l->ltl_next; /* in case this link is removed */
+        newprev= l;
+        if (l->rpt == NULL) continue; /* shouldn't happen */
+        if (l->rpt->pkt->fldval[SIP] != sip) continue;
+        if (l->rpt->pkt->fldval[DIP] != dip) continue;
+        if (l->rpt->pkt->fldval[SPORT] != sport) continue;
+        if (l->rpt->pkt->fldval[DPORT] != dport) continue;
+        /* found a match */
+        ltl_extract(&self->lt,l,prev,hash1,hash2);
+        (*self->status_callback)(self->callback_context,l->rpt,implied_status);
+        l->rpt= NULL; /* mark as deleted from lookup table */
+        newprev= prev;
+        /* check for more matches */
+    }
+    //fflush(stdout);
+}
+
+#if 0 // not currently needed
+static void lt_extract(packet_resp_canceller *self,prc_link *todel,prc_link *prev,u16 hash1,u32 hash2) {
+    ltl_extract(&self->lt,todel,prev,hash1,hash2);
+}
+#endif
+
+static void ltl_extract(prc_lookup_table *lt,prc_link *todel,prc_link *prev,u16 hash1,u32 hash2) {
+    if (prev == NULL) {
+        if (todel->ltl_next == NULL) {
+            lt->arr[hash1]->arr[hash2]= NULL;
+            lt->arr[hash1]->num_used--;
+            if (lt->arr[hash1]->num_used == 0) {
+                free_prc_lookup_table2_clean(lt->arr[hash1]);
+                lt->arr[hash1]= NULL;
+            }
+        } else {
+            lt->arr[hash1]->arr[hash2]= todel->ltl_next;
+        }
+    } else {
+        prev->ltl_next= todel->ltl_next;
+    }
+}
+
+static int lt_delete_report(prc_lookup_table *lt,spade_report *rpt) {
+    prc_link *l,*prev;
+    u32 hash2;
+    prc_lookup_table2 *t2;
+    spade_event *pkt= rpt->pkt;
+    u16 hash1;
+    if ((pkt->fldval[IPPROTO] == IPPROTO_TCP) || (pkt->fldval[IPPROTO] == IPPROTO_UDP))
+        hash1= calc_hash1(pkt->fldval[SPORT],pkt->fldval[DPORT]);
+    else
+        hash1= calc_hash1(pkt->fldval[SIP],pkt->fldval[DIP]);
+    
+    if (lt->arr[hash1] == NULL) return 0;
+    t2= lt->arr[hash1];
+    calc_hash2(pkt->fldval[SIP],pkt->fldval[DIP],hash2);
+    if (t2->arr[hash2] == NULL) return 0;
+    for (l= t2->arr[hash2], prev=NULL; l != NULL && l->rpt != rpt; prev=l,l=l->ltl_next);
+    if (l == NULL) return 0; /* no match */
+    ltl_extract(lt,l,prev,hash1,hash2);
+    return 1;
+}
+
+static prc_link *new_prc_link(spade_report *rpt) {
+    prc_link *new;
+    if (prc_link_freelist == NULL) {
+        new= (prc_link *)malloc(sizeof(prc_link));
+        if (new == NULL) return NULL;
+    } else {
+        new= prc_link_freelist;
+        prc_link_freelist= prc_link_freelist->ttl_next;
+    }
+    new->rpt= rpt;
+    new->ltl_next= NULL;    
+    new->ttl_next= NULL;    
+    return new;
+}
+
+static void free_prc_ttl_list(prc_link *head) {
+    prc_link *tail;
+    if (head == NULL) return;
+    for (tail= head; tail->ttl_next != NULL; tail= tail->ttl_next);
+    tail->ttl_next= prc_link_freelist;
+    prc_link_freelist= head;  
+}
+
+#if 0 // not currently needed
+static void free_prc_link(prc_link *l) {
+    if (l == NULL) return;
+    l->ttl_next= prc_link_freelist;
+    prc_link_freelist= l;  
+}
+#endif 
+
+static prc_lookup_table2 *new_prc_lookup_table2() {
+    prc_lookup_table2 *new;
+    
+    if (prc_lookup_table2_freelist == NULL) {
+        new= (prc_lookup_table2 *)malloc(sizeof(prc_lookup_table2));
+        if (new == NULL) return NULL;
+        init_prc_lookup_table2(new);
+    } else {
+        new= prc_lookup_table2_freelist;
+        prc_lookup_table2_freelist= (prc_lookup_table2 *)prc_lookup_table2_freelist->arr[0];
+        /* items on freelist are pre-inited for efficiency except for arr[0] */
+        new->arr[0]= NULL;
+    }
+    return new;
+}
+
+static void init_prc_lookup_table2(prc_lookup_table2 *t) {
+    int i;
+    t->num_used= 0;
+    for (i= 0; i < LOOKUP_TABLE2_SIZE; i++) t->arr[i]= NULL;
+}
+
+static void free_prc_lookup_table2(prc_lookup_table2 *t) {    
+    init_prc_lookup_table2(t);
+    free_prc_lookup_table2_clean(t);
+}
+
+/* t is expected to be cleaned up to to an empty state */
+static void free_prc_lookup_table2_clean(prc_lookup_table2 *t) {    
+    if (t == NULL) return;
+    t->arr[0]= (prc_link *)prc_lookup_table2_freelist;
+    prc_lookup_table2_freelist= t;  
+}
+
+void packet_resp_canceller_print_config_details(packet_resp_canceller *self,FILE *f,char *indent) {
+    fprintf(f,"%swait=%d; timeout_implication=%s\n",indent,self->tt.num_buckets,PORT_STATUS_AS_STR(self->timeout_implication));
+}
+
+/*@}*/
+
+/* Id: packet_resp_canceller.c,v 1.1 2004/10/11 14:35:54 jonkman Exp $ */
+
+/*********************************************************************
+spade_report.c, distributed as part of Spade v030125.1
+Author: James Hoagland, Silicon Defense (hoagland@SiliconDefense.com)
+copyright (c) 2002 by Silicon Defense (http://www.silicondefense.com/)
+Released under GNU General Public License, see the COPYING file included
+with the distribution or http://www.silicondefense.com/spice/ for details.
+
+Please send complaints, kudos, and especially improvements and bugfixes to
+hoagland@SiliconDefense.com.  As described in GNU General Public License, no
+warranty is expressed for this program.
+*********************************************************************/
+
+
+#include <stdlib.h>
+#include <string.h>
+
+/*! \file spade_report.c
+ * \ingroup netspade_layer
+ * \brief 
+ *  spade_report.c contains routines for spade_report allocation, initing, and
+ *  recycling; spade_report reprents a report that Spade is making
+ */
+
+/*! \addtogroup netspade_layer
+    @{
+*/
+
+/* creation and recycling routines for spade_report's */
+spade_report *spade_report_freelist=NULL;
+
+spade_report *new_spade_report(spade_event *pkt,score_info *score, int detect_type, char *detectorid,const char *detect_type_str,char *scope_str,spade_pkt_stats *stream_stats,port_status_t port_status) {
+    spade_report *new;
+    if (spade_report_freelist != NULL) {
+        new= spade_report_freelist;
+        spade_report_freelist= new->next;
+    } else {
+        new= (spade_report *)malloc(sizeof(spade_report));
+    }
+    
+    new->pkt= pkt;
+    new->score=score;
+    new->detect_type= detect_type;
+    new->detectorid= detectorid;
+    new->stream_stats= stream_stats;
+    new->port_status= port_status;
+    new->detect_type_str= detect_type_str;
+    if (scope_str != NULL)
+        strncpy(new->scope_str,scope_str,200);
+    else
+        new->scope_str[0]= '\0';
+    new->next= NULL;
+    return new;
+}
+
+void free_spade_report(spade_report *rpt) {
+    rpt->next= spade_report_freelist;
+    spade_report_freelist= rpt;
+}
+
+void free_spade_reports(spade_report *start) {
+    spade_report *end,*next;
+    for (end= start, next=start->next; next != NULL; end=next,next=next->next);
+    end->next= spade_report_freelist;
+    spade_report_freelist= start;
+}
+
+void port_status_set_file_print(port_status_set_t set,FILE *f) {
+    int first= 1;
+    int i;
+    fprintf(f,"{");
+    if ((set & 0xFFF) == 0xFFF)
+        fprintf(f,"*ALL*");
+    else {
+        for (i=0; i < 16; i++) {
+            if (PS_IN_SET(set,i)) {
+                if (!first) { fprintf(f,","); }
+                first=0;
+                fprintf(f,"%s",PORT_STATUS_AS_STR(i));
+            }
+        }
+    }
+    fprintf(f,"}");
+}
+
+/*@}*/
+
+/* Id: spade_report.c,v 1.1 2004/10/11 14:35:54 jonkman Exp $ */
+/*********************************************************************
+score_mgr.c, distributed as part of Spade v030125.1
+Author: James Hoagland, Silicon Defense (hoagland@SiliconDefense.com)
+copyright (c) 2002 by Silicon Defense (http://www.silicondefense.com/)
+Released under GNU General Public License, see the COPYING file included
+with the distribution or http://www.silicondefense.com/spice/ for details.
+
+Please send complaints, kudos, and especially improvements and bugfixes to
+hoagland@SiliconDefense.com.  As described in GNU General Public License, no
+warranty is expressed for this program.
+*********************************************************************/
+
+
+/*! \file score_mgr.c
+ * \brief 
+ *  score_mgr.c contains the "class" score_mgr which is a spade
+ *  probability table along with a reporting threshold and the
+ *  other accompaniments of applying the Spade approach to somewhere.
+ * \ingroup stmgr
+ */
+
+/*! \addtogroup stmgr Score and threshold management
+ * \brief this group contains objects to manaage anomaly scores and reporting thresholds
+ * \ingroup libspade
+    @{
+*/
+
+#include <stdio.h>
+#include <stdlib.h>
+
+score_mgr *new_score_mgr(void *mgrref,spade_enviro *enviro,void *callback_context,spade_thresh_exceeded_fn_t threshexceeded_callback,spade_thresh_changed_fn_t threshchanged_callback,spade_msg_fn msg_callback) {
+    score_mgr *new= (score_mgr *)malloc(sizeof(score_mgr));
+    init_score_mgr(new,mgrref,enviro,callback_context,threshexceeded_callback,threshchanged_callback,msg_callback);
+    return new;
+}
+
+/* caller must set enviro->thresh already; we'll keep a pointer to the enviro */
+void init_score_mgr(score_mgr *self,void *mgrref,spade_enviro *enviro, void *callback_context, spade_thresh_exceeded_fn_t threshexceeded_callback, spade_thresh_changed_fn_t threshchanged_callback,spade_msg_fn msg_callback) {
+    self->mgrref= mgrref;
+    self->enviro= enviro;
+
+    self->threshexceeded_callback= threshexceeded_callback;
+    self->threshchanged_callback= threshchanged_callback;
+    self->callback_context= callback_context != NULL ? callback_context : self;
+    
+    self->msg_callback= msg_callback;
+   
+    self->adapt_active= 0;
+    self->advise_status= ADVISING_OFF;
+    self->survey_active= 0;
+}
+
+void score_mgr_setup_adapt_from_str(score_mgr *self,int adaptmode,char *str) {
+    self->adapt_active= 1;
+    init_thresh_adapter(&self->adapter,self->msg_callback);
+    thresh_adapter_setup_from_str(&self->adapter,adaptmode,str);
+}
+
+void score_mgr_setup_adapt1(score_mgr *self,int target, time_t period, float new_obs_weight, int by_count) {
+    self->adapt_active= 1;
+    init_thresh_adapter(&self->adapter,self->msg_callback);
+    thresh_adapter_setup_1(&self->adapter,target,period,new_obs_weight,by_count);
+}
+
+void score_mgr_setup_adapt2(score_mgr *self,double targetspec, double obsper, int NS, int NM, int NL) {
+    self->adapt_active= 2;
+    init_thresh_adapter(&self->adapter,self->msg_callback);
+    thresh_adapter_setup_2(&self->adapter,targetspec,obsper,NS,NM,NL);
+}
+
+void score_mgr_setup_adapt3(score_mgr *self,double targetspec, double obsper, int NO) {
+    self->adapt_active= 3;
+    init_thresh_adapter(&self->adapter,self->msg_callback);
+    thresh_adapter_setup_3(&self->adapter,targetspec,obsper,NO);
+}
+
+void score_mgr_setup_adapt4(score_mgr *self,double thresh, double obsper) {
+    self->adapt_active= 4;
+    init_thresh_adapter(&self->adapter,self->msg_callback);
+    thresh_adapter_setup_4(&self->adapter,thresh,obsper);
+}
+
+void score_mgr_setup_advise(score_mgr *self,int obs_size, int obs_secs) {
+    self->advise_status= ADVISING_RUNNING;
+    init_thresh_adviser(&self->adviser,obs_size,obs_secs,self->msg_callback);
+}
+
+void score_mgr_setup_advise_from_str(score_mgr *self,char *str) {
+    self->advise_status= ADVISING_RUNNING;
+    init_thresh_adviser_from_str(&self->adviser,str,self->msg_callback);
+}
+
+void score_mgr_setup_survey(score_mgr *self,char *filename,float interval) {
+    self->survey_active= 1;
+    init_anomscore_surveyer(&self->surveyer,filename,interval,self->msg_callback);
+}
+
+void score_mgr_setup_survey_from_str(score_mgr *self,char *str) {
+    self->survey_active= 1;
+    init_anomscore_surveyer_from_str(&self->surveyer,str,self->msg_callback);
+}
+
+
+
+int score_mgr_new_time(score_mgr *self,time_t time) {
+    int advising_completed= 0;
+    // a new second
+    self->enviro->now= time;
+
+    // tell our helpers
+    if (self->adapt_active) {
+        double new_thresh;
+        if (thresh_adapter_new_time(&self->adapter,self->enviro,&new_thresh)) {
+            // there is a new threshold
+            self->enviro->thresh= new_thresh;
+            if (self->threshchanged_callback != NULL)
+                (*(self->threshchanged_callback))(self->callback_context,self->mgrref);
+        }
+    }
+    if (self->advise_status == ADVISING_RUNNING) {
+        if (thresh_adviser_new_time(&self->adviser,self->enviro)) {
+            // advising period has completed
+            self->advise_status= ADVISING_DONE;
+            advising_completed= 1;
+        }
+    }
+    if (self->survey_active) anomscore_surveyer_new_time(&self->surveyer,self->enviro);
+    return advising_completed;
+}
+
+void score_mgr_new_event(score_mgr *self,score_info *score,spade_event *event) {
+    double mainscore= score_info_mainscore(score);
+
+    //if (self->debug_level > 1) printf("%p: packet #%d: %.4f\n",self,self->enviro->pkt_stats.scored,mainscore);
+    if (self->enviro->thresh >= 0.0 && mainscore >= self->enviro->thresh) {
+        if (self->threshexceeded_callback != NULL)
+            (*(self->threshexceeded_callback))(self->callback_context,self->mgrref,event,score);
+    }
+
+    if (self->adapt_active) thresh_adapter_new_score(&self->adapter,mainscore);
+    if (self->advise_status == ADVISING_RUNNING) thresh_adviser_new_score(&self->adviser,mainscore);
+    if (self->survey_active) anomscore_surveyer_new_score(&self->surveyer,mainscore);
+}
+
+void score_mgr_dump(score_mgr *self) 
+{
+    if (self->survey_active) anomscore_surveyer_flush(&self->surveyer);
+}
+
+void score_mgr_cleanup(score_mgr *self) 
+{
+    score_mgr_dump(self);
+}
+
+void score_mgr_file_print_log(score_mgr *self,FILE *file) {
+    if (self->advise_status != ADVISING_OFF) thresh_adviser_write_advice(&self->adviser,file);
+}
+
+void score_mgr_print_config_details(score_mgr *self, FILE *f, char *indent) {
+    char indent2[100];
+    sprintf(indent2,"%s  ",indent);
+    fprintf(f,"%scurrent thresh=%.4f\n",indent,self->enviro->thresh);
+    if (self->adapt_active) {
+        fprintf(f,"%sadapt_active=%d:\n",indent,self->adapt_active);
+        thresh_adapter_print_config_details(&self->adapter,f,indent2);
+    }
+    if (self->advise_status != ADVISING_OFF) {
+        fprintf(f,"%sadvise_status=%s:\n",indent,self->adapt_active == ADVISING_RUNNING ? "ADVISING_RUNNING" : "ADVISING_DONE");
+        thresh_adviser_print_config_details(&self->adviser,f,indent2);
+    }
+    if (self->survey_active) {
+        fprintf(f,"%ssurvey_active=\n",indent);
+        anomscore_surveyer_print_config_details(&self->surveyer,f,indent2);
+    }
+}
+
+/*@}*/
+/* Id: score_mgr.c,v 1.1 2004/10/11 14:35:54 jonkman Exp $ */
+/*********************************************************************
+score_calculator.c, distributed as part of Spade v030125.1
+Author: James Hoagland, Silicon Defense (hoagland@SiliconDefense.com)
+copyright (c) 2002 by Silicon Defense (http://www.silicondefense.com/)
+Released under GNU General Public License, see the COPYING file included
+with the distribution or http://www.silicondefense.com/spice/ for details.
+
+Please send complaints, kudos, and especially improvements and bugfixes to
+hoagland@SiliconDefense.com.  As described in GNU General Public License, no
+warranty is expressed for this program.
+*********************************************************************/
+
+/*! \file score_calculator.c
+ * \brief 
+ *  score_calculator.c contains a module to store the details about how
+ *  to calculate an anomaly score
+ * \ingroup scoreprod
+ */
+
+/*! \addtogroup scoreprod Anomaly score production
+ * \brief this group contains objects to produce and represent anomaly scores
+ * \ingroup libspade
+    @{
+*/
+
+#include <stdlib.h>
+#include <math.h>
+
+/*#define LOG10 2.30258509299 */
+#define LOG2 0.69314718056
+
+static table_use_specs *new_evfiles_specs(void);
+
+score_calculator *new_score_calculator(int prodcount,feature_list feats[],const char **featurenames,event_condition_set conds,int scale_freq,double scale_factor,double prune_threshold,event_recorder *recorder,feature_list *calc_feats) {
+    score_calculator *new= (score_calculator *)malloc(sizeof(score_calculator));
+    init_score_calculator(new,prodcount,feats,featurenames,conds,scale_freq,scale_factor,prune_threshold,recorder,calc_feats);
+    return new;
+}
+
+void init_score_calculator(score_calculator *self,int prodcount,feature_list feats[],const char **featurenames,event_condition_set conds,int scale_freq,double scale_factor,double prune_threshold,event_recorder *recorder,feature_list *calc_feats) {
+    init_score_calculator_clear(self,recorder);
+    self->prodcount= prodcount;
+    if (prodcount == 1) {
+        self->evfile= event_recorder_new_event_file(self->recorder,&feats[0],featurenames,conds,scale_freq,scale_factor,prune_threshold,0,calc_feats);
+    } else {
+        self->evfiles= event_recorder_new_event_files(self->recorder,prodcount,feats,featurenames,conds,scale_freq,scale_factor,prune_threshold,0);
+    }
+}
+
+score_calculator *new_score_calculator_clear(event_recorder *recorder) {
+    score_calculator *new= (score_calculator *)malloc(sizeof(score_calculator));
+    init_score_calculator_clear(new,recorder);
+    return new;
+}
+
+void init_score_calculator_clear(score_calculator *self,event_recorder *recorder) {
+    self->prodcount= -1;
+    self->evfile= NULL;
+    self->evfiles= NULL;
+    self->cond_prefix_len= 0;
+    self->use_corrscore= 1;
+    self->calc_relscore= 0;
+    self->calc_rawscore= 0;
+    self->mainpref= PREF_NOSCORE;
+    self->min_obs_prefix_len= 0;
+    self->min_obs_count= 0.0;
+    self->max_entropy= -1;
+    self->recorder= recorder;
+    self->evfiles_data= NULL;
+}
+
+void score_calculator_set_features(score_calculator *self,int prodcount,feature_list feats[],feature_list *calc_feats,const char **featurenames) {
+    int i;
+    if (self->evfiles_data == NULL) self->evfiles_data= new_evfiles_specs();
+    self->evfiles_data->prodcount= prodcount;
+    self->evfiles_data->feats= (feature_list*)malloc(sizeof(feature_list)*prodcount);
+    for (i= 0; i < prodcount; i++)
+        self->evfiles_data->feats[i]= feats[i];
+    if (calc_feats == NULL)
+        self->evfiles_data->calc_feats.num= 0;
+    else
+        self->evfiles_data->calc_feats= *calc_feats; /* copy over */
+    self->evfiles_data->featurenames= featurenames;
+}
+
+void score_calculator_set_storage_conditions(score_calculator *self,event_condition_set conds) {
+    if (self->evfiles_data == NULL) self->evfiles_data= new_evfiles_specs();
+    self->evfiles_data->conds= conds;
+}
+
+void score_calculator_set_scaling(score_calculator *self,int scale_freq,double scale_factor,double prune_threshold) {
+    if (self->evfiles_data == NULL) self->evfiles_data= new_evfiles_specs();
+    self->evfiles_data->scale_freq= scale_freq;
+    self->evfiles_data->scale_factor= scale_factor;
+    self->evfiles_data->prune_threshold= prune_threshold;
+}
+
+void score_calculator_init_complete(score_calculator *self) {
+    table_use_specs *d;
+    
+    if (self->evfiles_data == NULL) return; /* nothing to do */
+    d= self->evfiles_data;
+    if (d->feats== NULL) { /* no feature lists provided */
+        d->feats= (feature_list *)malloc(1*sizeof(feature_list));
+        d->feats[0].num= 1;
+        d->feats[0].feat[0]= 0;
+        d->prodcount= 1;
+    }
+    
+    self->prodcount= d->prodcount;
+    if (d->prodcount == 1) {
+        self->evfile= event_recorder_new_event_file(self->recorder,&d->feats[0],d->featurenames,d->conds,d->scale_freq,d->scale_factor,d->prune_threshold,0,(d->calc_feats.num==0 ?NULL:&d->calc_feats));
+    } else {
+        self->evfiles= event_recorder_new_event_files(self->recorder,d->prodcount,d->feats,d->featurenames,d->conds,d->scale_freq,d->scale_factor,d->prune_threshold,0);
+    }
+    free(self->evfiles_data->feats);
+    free(self->evfiles_data);
+    self->evfiles_data= NULL;
+}
+
+void score_calculator_set_condcutoff(score_calculator *self,int cond_prefix_len)
+{
+    self->cond_prefix_len= cond_prefix_len;
+}
+
+void score_calculator_set_relscore(score_calculator *self, int calc_relscore, int rel_is_main) {
+    self->calc_relscore= calc_relscore;
+    if (rel_is_main) self->mainpref= PREF_RELSCORE;
+}
+
+void score_calculator_set_rawscore(score_calculator *self, int calc_rawscore, int raw_is_main) {
+    self->calc_rawscore= calc_rawscore;
+    if (raw_is_main) self->mainpref= PREF_RAWSCORE;
+}
+
+void score_calculator_set_corrscore(score_calculator *self, int use_corrscore) {
+    self->use_corrscore= use_corrscore;
+}
+
+void score_calculator_set_min_obs(score_calculator *self,int featlist_prefix_len,int min_obs_count) {
+    self->min_obs_prefix_len= featlist_prefix_len;
+    self->min_obs_count= (double)min_obs_count;
+}
+
+void score_calculator_set_low_entropy_domain(score_calculator *self, int val_prefix_len, double max_entropy) {
+    self->max_entropy= max_entropy;
+    self->entropy_prefix_len= val_prefix_len;
+}
+
+void score_calculator_cleanup(score_calculator *self) {
+    if (self->prodcount > 0 && self->evfiles != NULL) free(self->evfiles);
+    self->prodcount= -1;
+}
+
+int score_calculator_using_corrscore(score_calculator *self) {
+    return self->use_corrscore || (self->prodcount > 0) || !self->calc_rawscore;
+}
+
+score_info *score_calculator_calc_event_score(score_calculator *self,spade_event *event,score_info* storage,int *enoughobs) {
+    int prodidx;
+    double prob;
+    double rawscore= NO_SCORE;
+    double relscore= NO_SCORE;
+    *enoughobs= 1;
+    
+    if (self->prodcount < 0) score_calculator_init_complete(self);
+            
+    if (self->prodcount > 1) { /* multiply together the straight maximally conditioned probabilities and return absolute score */
+        prob= 1;
+        for (prodidx= 0; prodidx < self->prodcount; prodidx++)
+            prob*= event_recorder_get_condprob(self->recorder,self->evfiles[prodidx],event,-1,1);
+        rawscore= -1*(log(prob)/LOG2);
+    } else {
+        if (self->min_obs_count > 0) {
+            double count= event_recorder_get_count(self->recorder,self->evfile,event,self->min_obs_prefix_len);
+            if ((count+1) < self->min_obs_count) {
+                *enoughobs= 0;
+                return NULL;
+            }
+        }
+        if (self->max_entropy > 0) {
+            double entropy;
+            entropy= event_recorder_get_entropy(self->recorder,self->evfile,event,self->entropy_prefix_len);
+            if (entropy > self->max_entropy) return NULL;
+        }
+        prob= event_recorder_get_condprob(self->recorder,self->evfile,event,self->cond_prefix_len,1);
+        if (self->calc_rawscore) { // calculate raw anomaly score
+            if (self->use_corrscore) { // use the scores that are computed as adverstised
+                rawscore= -1.0*(log(prob)/LOG2);
+            } else { // use the old, incorrectly computed joint score
+                rawscore= -1.0*log(prob/LOG2);
+            }
+        }
+        if (self->calc_relscore) { // calculate relative anomaly score
+            double basecount= event_recorder_get_count(self->recorder,self->evfile,event,self->cond_prefix_len)+1;
+            double ratio= log(prob)/log(1/basecount);
+            relscore= ratio; /* *ratio; */
+        }
+    }
+    if (storage == NULL)
+        return new_score_info(self->mainpref,relscore,rawscore,self->use_corrscore);
+    else {
+        init_score_info(storage,self->mainpref,relscore,rawscore,self->use_corrscore);
+        return storage;
+    }
+}
+
+
+int score_calculator_get_store_count(score_calculator *self) {
+    evfile_ref f= (self->prodcount > 1) ? self->evfiles[0] : self->evfile;
+    return event_recorder_get_store_count(self->recorder,f);
+}
+
+double score_calculator_get_obs_count(score_calculator *self) {
+    evfile_ref f= (self->prodcount > 1) ? self->evfiles[0] : self->evfile;
+    return event_recorder_get_obs_count(self->recorder,f);
+}
+
+static table_use_specs *new_evfiles_specs() {
+    table_use_specs *new= (table_use_specs *)malloc(sizeof(table_use_specs));
+    new->prodcount= 1;
+    new->feats= NULL;
+    new->calc_feats.num= 0;
+    new->featurenames= NULL;
+    new->conds= 0;
+    new->scale_freq= -1;
+    new->scale_factor= 1;
+    new->prune_threshold= 0;
+    return new;
+}
+
+void score_calculator_print_config_details(score_calculator *self,FILE *f,char *indent) {
+    char indent2[100],indent3[100];
+    sprintf(indent2,"%s  ",indent);
+    sprintf(indent3,"%s  ",indent2);
+    if (self->prodcount < 0) score_calculator_init_complete(self);
+
+    fprintf(f,"%sprodcount=%d:\n",indent,self->prodcount);
+    if (self->prodcount == 1) {
+        fprintf(f,"%sevfile:\n",indent2);
+        evfile_print_config_details(self->evfile,f,indent3);
+        
+        fprintf(f,"%scond_prefix_len=%d\n",indent2,self->cond_prefix_len);
+        fprintf(f,"%smainpref=%s; calc_rawscore=%d; calc_relscore=%d; use_corrscore=%d\n",indent2,scorepref_str(self->mainpref),self->calc_rawscore,self->calc_relscore,self->use_corrscore);
+        
+        if (self->min_obs_count > 0)
+            fprintf(f,"%smin_obs_count=%.4f; min_obs_prefix_len=%d\n",indent2,self->min_obs_count,self->min_obs_prefix_len);
+    } else {
+        int i;
+        for (i=0; i < self->prodcount; i++) {
+            fprintf(f,"%sevfiles[%d]:\n",indent2,i);
+            evfile_print_config_details(self->evfiles[i],f,indent3);
+        }
+    }
+}
+
+/*@}*/
+/* Id: score_calculator.c,v 1.1 2004/10/11 14:35:54 jonkman Exp $ */
+/*********************************************************************
+spade_prob_table.c, distributed as part of Spade v030125.1
+Author: James Hoagland, Silicon Defense (hoagland@SiliconDefense.com)
+copyright (c) 2002 by Silicon Defense (http://www.silicondefense.com/)
+Released under GNU General Public License, see the COPYING file included
+with the distribution or http://www.silicondefense.com/spice/ for details.
+
+Please send complaints, kudos, and especially improvements and bugfixes to
+hoagland@SiliconDefense.com.  As described in GNU General Public License, no
+warranty is expressed for this program.
+*********************************************************************/
+
+#include <limits.h>
+#include <stdlib.h>
+#include <math.h>
+
+/*! \file spade_prob_table.c,
+ * \brief 
+ *  spade_prob_table.c contains all the routines to build and maintain the
+ *  tree structure that Spade uses to maintain its probability tables.  It
+ *  also contains the access functions.
+ * \ingroup staterec
+ */
+
+/*! \addtogroup staterec
+    @{
+*/
+
+/* return the standard wait time for an interior node given the counts on 
+   its children */
+#define wait_time(c1,c2) (min_int(max_int(10,ceil(c1>c2?(2*c2-c1):(2*c1-c2))),MAX_U16))
+
+// static mindex find_nexttree_of_type(mindex leaf,features type) {
+#define find_nexttree_of_type_macro(_leaf,_type,_res) { \
+    mindex _t; \
+    _res= TNULL; \
+    for (_t=leafnexttree(_leaf); _t != TNULL; _t=treenext(_t)) { /* make common case quick */ \
+        if (treetype(_t) == _type) { \
+            _res=_t; \
+            break; \
+        } \
+    } \
+}
+
+/* return the leaf below this interior or leaf [encoded] node else TNULL */
+//static mindex find_leaf_in_subtree(dmindex encchild,valtype val) {
+#define find_leaf_in_subtree_macro(_encchild,_val,_res) { \
+    mindex _child; \
+    \
+    _res= TNULL; \
+    while (_encchild != TNULL) { \
+        if (isleaf(_encchild)) { \
+            _child= encleaf2mindex(_encchild); \
+            if (_val == leafvalue(_child)) { /* found the leaf */ \
+                _res= _child; \
+                break; \
+            } else { /* leaf not present */ \
+                break; \
+            } \
+        } else { \
+            _child= _encchild; \
+        } \
+        \
+        if (_val <= intsortpt(_child)) { /* go left */ \
+            _encchild= intleft(_child); \
+        } else { /* go right */ \
+            _encchild= intright(_child); \
+        } \
+    } \
+}
+
+//static mindex find_leaf(mindex tree,valtype val) {
+#define find_leaf_macro(_tree,_val,_res) { \
+    mindex _leaf; \
+    mindex _root=treeroot(_tree); \
+    _res= TNULL; \
+    if (_root != TNULL) { \
+        find_leaf_in_subtree_macro(_root,_val,_leaf); \
+        _res= _leaf; \
+    } \
+}
+
+
+static int min_int(int a, int b);
+static int max_int(int a, int b);
+static double tree_value_prob(mindex tree, valtype val);
+static mindex find_nexttree_of_type(mindex leaf, features type);
+static mindex get_nexttree_of_type(mindex leaf, features type);
+static mindex incr_tree_value_count(mindex tree, valtype newval);
+static mindex increment_value_count(mindex node, valtype val);
+static mindex add_node_above_to_right(mindex node, valtype val);
+static mindex add_node_above_to_left(mindex node, valtype val);
+static mindex add_node_between(mindex node, valtype val);
+static void rebalance_subtree(mindex encnode);
+static int out_of_balance(mindex node);
+static void free_all_in_tree(mindex tree);
+static void free_all_in_subtree(dmindex encnode);
+static void scale_and_prune_tree(mindex tree, double factor, double threshold);
+static dmindex scale_and_prune_subtree(dmindex encnode, double factor, double threshold, double *change, valtype *newrightmost);
+static valtype largest_val(mindex node);
+static mindex dup_intnode(mindex node);
+static mindex find_leaf(mindex tree, valtype val);
+static unsigned int feature_tree_stats(mindex tree, features f, unsigned int *smind, unsigned int *smaxd, float *saved, float *swaved, unsigned int *snum_leaves);
+static unsigned int feature_subtree_stats(mindex encnode, features f, unsigned int *smind, unsigned int *smaxd, float *saved, float *swaved, unsigned int *snum_leaves);
+static unsigned int tree_stats(mindex tree, unsigned int *mind, unsigned int *maxd, float *aved, float *waved);
+static double tree_count(mindex tree);
+static unsigned int num_leaves(mindex tree);
+static unsigned int num_subtree_leaves(mindex encnode);
+static unsigned int tree_depth_total(mindex tree);
+static unsigned int subtree_depth_total(mindex encnode, unsigned int depth);
+static double weighted_tree_depth_total(mindex tree);
+static double weighted_subtree_depth_total(mindex encnode, unsigned int depth);
+static void tree_min_max_depth(mindex tree, unsigned int *mind, unsigned int *maxd);
+static void subtree_min_max_depth(mindex encnode, unsigned int *mind, unsigned int *maxd, unsigned int depth);
+static void write_all_tree_uncond_probs(spade_prob_table *self,FILE *f, mindex tree, int depth, features feats[], valtype vals[], double treesum);
+static void write_all_subtree_uncond_probs(spade_prob_table *self,FILE *f, dmindex encnode, int depth, features feats[], valtype vals[], double treesum);
+static void write_all_tree_cond_probs(spade_prob_table *self,FILE *f, mindex tree, int depth, features feats[], valtype vals[]);
+static void write_all_subtree_cond_probs(spade_prob_table *self,FILE *f, dmindex encnode, int depth, features feats[], valtype vals[], double treesum);
+static void inc_featurecomb(featcomb C, double val, int depth, features feats[]);
+static featcomb create_featurecomb(int depth, double val);
+static void scale_all_featurecomb(featcomb c, double factor);
+static void add_all_tree_entrsum(featcomb c, mindex tree, int depth, features feats[], double totsum);
+static void add_all_subtree_entrsum(featcomb c, dmindex encnode, int depth, features feats[], double treesum, double totsum);
+static void write_all_entropies2(spade_prob_table *self,FILE *f, featcomb c, int depth, features feats[]);
+static void write_feature_names(spade_prob_table *self,FILE *f, int depth, features feats[]);
+static void printtree(spade_prob_table *self,mindex tree, char *ind);
+static void printtree2(spade_prob_table *self,dmindex encnode, char *ind);
+static void printtree2_shallow(dmindex encnode);
+static int sanity_check_tree(mindex tree);
+static int sanity_check_subtree(dmindex encnode);
+static mindex find_leaf2(spade_prob_table *self, features type1, valtype val1, features type2, valtype val2);
+static mindex find_leaf3(spade_prob_table *self, features type1, valtype val1, features type2, valtype val2, features type3, valtype val3);
+static double calc_tree_entropy(mindex tree);
+static double calc_subtree_entropy(mindex node,double prob_base);
+
+
+#ifndef LOG2
+/*#define LOG2 log(2);*/
+#define LOG2 ((double)0.693147180559945)
+#endif
+
+void init_spade_prob_table(spade_prob_table *self,const char **featurenames,int recovering) {
+    int i;
+    if (!recovering) {
+        init_mem();
+    
+        for (i=0; i < MAX_NUM_FEATURES; i++) {
+            self->root[i]= TNULL;
+        }
+    }
+    self->featurenames= featurenames;
+}
+
+spade_prob_table *new_spade_prob_table(const char **featurenames) {
+    spade_prob_table *new= (spade_prob_table *)malloc(sizeof(spade_prob_table));
+    init_spade_prob_table(new,featurenames,0);
+    return new;
+}
+
+int spade_prob_table_is_empty(spade_prob_table *self) {
+    int i;
+    for (i=0; i < MAX_NUM_FEATURES; i++) {
+        if (self->root[i] != TNULL) return 0;
+    }
+    return 1;
+}
+
+static int min_int(int a,int b) {
+    return a < b ? a : b;
+}
+
+static int max_int(int a,int b) {
+    return a > b ? a : b;
+}
+
+void increment_simple_count(spade_prob_table *self,features type1,valtype val1) {
+    if (self->root[type1] == TNULL) {
+        self->root[type1]= new_treeinfo(type1);
+    }
+    incr_tree_value_count(self->root[type1],val1);
+}
+
+/* assumes type1 and type2 are in a consistant order */
+void increment_2joint_count(spade_prob_table *self,features type1,valtype val1,features type2,valtype val2,int skip) {
+    mindex leaf1,tree2;
+    
+    if (skip >= 1) {
+        /* this should always find something and self->root[type1] should be non-NULL since has been marked before */
+        leaf1= find_leaf(self->root[type1],val1);
+    } else {
+        if (self->root[type1] == TNULL) {
+            self->root[type1]= new_treeinfo(type1);
+        }
+        leaf1= incr_tree_value_count(self->root[type1],val1);
+    }
+    tree2= get_nexttree_of_type(leaf1,type2);
+    incr_tree_value_count(tree2,val2);
+}
+
+void increment_3joint_count(spade_prob_table *self,features type1,valtype val1,features type2,valtype val2,features type3,valtype val3,int skip) {
+    mindex leaf1,leaf2,tree2,tree3;
+    
+    if (skip >= 1) {
+        /* this should always find something and self->root[type1] should be non-NULL since has been marked before */
+        leaf1= find_leaf(self->root[type1],val1);
+    } else {
+        if (self->root[type1] == TNULL) {
+            self->root[type1]= new_treeinfo(type1);
+        }
+        leaf1= incr_tree_value_count(self->root[type1],val1);
+    }
+    tree2= get_nexttree_of_type(leaf1,type2);
+    /* skip case: find_leaf should always find something since has been marked before */
+    leaf2= skip >= 2 ? find_leaf(tree2,val2) : incr_tree_value_count(tree2,val2);
+    tree3= get_nexttree_of_type(leaf2,type3);
+    incr_tree_value_count(tree3,val3);
+}
+
+void increment_4joint_count(spade_prob_table *self,features type1,valtype val1,features type2,valtype val2,features type3,valtype val3,features type4,valtype val4,int skip) {
+    mindex leaf1,leaf2,leaf3,tree2,tree3,tree4;
+    
+    if (skip >= 1) {
+        /* this should always find something and self->root[type1] should be non-NULL since has been marked before */
+        leaf1= find_leaf(self->root[type1],val1);
+    } else {
+        if (self->root[type1] == TNULL) {
+            self->root[type1]= new_treeinfo(type1);
+        }
+        leaf1= incr_tree_value_count(self->root[type1],val1);
+    }
+    tree2= get_nexttree_of_type(leaf1,type2);
+    /* skip case: find_leaf should always find something since has been marked before */
+    leaf2= skip >= 2 ? find_leaf(tree2,val2) : incr_tree_value_count(tree2,val2);
+    tree3= get_nexttree_of_type(leaf2,type3);
+    /* skip case: find_leaf should always find something since has been marked before */
+    leaf3= skip >= 3 ? find_leaf(tree3,val3) : incr_tree_value_count(tree3,val3);
+    tree4= get_nexttree_of_type(leaf3,type4);
+    incr_tree_value_count(tree4,val4);
+}
+
+void increment_Njoint_count(spade_prob_table *self,int size,features type[],valtype val[],int skip) {
+    mindex leaf,tree;
+    int i;
+    
+    if (self->root[type[0]] == TNULL) {
+        self->root[type[0]]= new_treeinfo(type[0]);
+    }
+    tree= self->root[type[0]];
+    for (i= 1; i < size; i++) {
+        /* skip case: find_leaf should always find something since has been here before */
+        if (skip >= i) 
+            find_leaf_macro(tree,val[i-1],leaf)
+        else 
+            leaf= incr_tree_value_count(tree,val[i-1]);
+        tree= get_nexttree_of_type(leaf,type[i]);
+    }
+    incr_tree_value_count(tree,val[size-1]);
+}
+
+/*****************************************************/
+
+double prob_simple(spade_prob_table *self,features type1,valtype val1) {
+    if (self->root[type1] == TNULL) return PROBRESULT_NO_RECORD; /* this feature was not counted */
+    return tree_value_prob(self->root[type1],val1);
+}
+
+/* return the probabilty of the value in the tree; assumes tree is not TNULL */
+static double tree_value_prob(mindex tree,valtype val) {
+    mindex root=treeroot(tree);
+    mindex leaf;
+    find_leaf_in_subtree_macro(root,val,leaf);
+    if (leaf == TNULL) return PROBRESULT_NO_RECORD;
+    return leafcount(leaf)/count_or_sum(root);
+}
+
+double prob_cond1(spade_prob_table *self,features type,valtype val,features ctype,valtype cval) {
+    mindex condleaf,tree,leaf;
+    if (self->root[ctype] == TNULL) {
+        return PROBRESULT_NO_RECORD; /* denominator would be 0 */
+    }
+    condleaf= find_leaf(self->root[ctype],cval);
+    if (condleaf == TNULL) return PROBRESULT_NO_RECORD; /* denominator would be 0 */
+    find_nexttree_of_type_macro(condleaf,type,tree);
+    if (tree == TNULL) {
+        return 0.0; /* numerator would be 0 */
+    }
+    leaf= find_leaf(tree,val);
+    if (leaf == TNULL) return 0.0; /* numerator would be 0 */
+    return leafcount(leaf)/leafcount(condleaf);
+}
+
+double prob_cond2(spade_prob_table *self,features type,valtype val,features ctype1,valtype cval1,features ctype2,valtype cval2) {
+    mindex condleaf,leaf,tree;
+    condleaf= find_leaf2(self,ctype1,cval1,ctype2,cval2);
+    if (condleaf == TNULL) return PROBRESULT_NO_RECORD; /* denominator would be 0 */
+    find_nexttree_of_type_macro(condleaf,type,tree);
+    if (tree == TNULL) {
+        return 0.0; /* numerator would be 0 */
+    }
+    leaf= find_leaf(tree,val);
+    if (leaf == TNULL) return 0.0; /* numerator would be 0 */
+    return leafcount(leaf)/leafcount(condleaf);
+}
+
+double prob_cond3(spade_prob_table *self,features type,valtype val,features ctype1,valtype cval1,features ctype2,valtype cval2,features ctype3,valtype cval3) {
+    mindex condleaf,leaf,tree;
+    condleaf= find_leaf3(self,ctype1,cval1,ctype2,cval2,ctype3,cval3);
+    if (condleaf == TNULL) return PROBRESULT_NO_RECORD; /* denominator would be 0 */
+    find_nexttree_of_type_macro(condleaf,type,tree);
+    if (tree == TNULL) {
+        return 0.0; /* numerator would be 0 */
+    }
+    leaf= find_leaf(tree,val);
+    if (leaf == TNULL) return 0.0; /* numerator would be 0 */
+    return leafcount(leaf)/leafcount(condleaf);
+}
+
+
+double prob_2joint(spade_prob_table *self,features type1,valtype val1,features type2,valtype val2) {
+    mindex tree,leaf;
+    double totcount;
+    if (self->root[type1] == TNULL) {
+        return PROBRESULT_NO_RECORD; /* denominator would be 0 */
+    }
+    totcount= tree_count(self->root[type1]);
+    leaf= find_leaf(self->root[type1],val1);
+    if (leaf == TNULL) return 0.0; /* numerator would be 0 */
+    find_nexttree_of_type_macro(leaf,type2,tree);
+    if (tree == TNULL) {
+        return 0.0; /* numerator would be 0 */
+    }
+    leaf= find_leaf(tree,val2);
+    if (leaf == TNULL) return 0.0; /* numerator would be 0 */
+    return leafcount(leaf)/totcount;
+}
+
+double prob_Njoint(spade_prob_table *self,int size,features type[],valtype val[]) {
+    mindex tree=self->root[type[0]],leaf;
+    double totcount;
+    int i;
+    if (tree == TNULL) return PROBRESULT_NO_RECORD; /* denominator would be 0 */
+    totcount= tree_count(tree);
+    for (i=1;i < size; i++) {
+        leaf= find_leaf(tree,val[i-1]);
+        if (leaf == TNULL) return 0.0; /* numerator would be 0 */
+        tree= find_nexttree_of_type(leaf,type[i]);
+        if (tree == TNULL) return 0.0; /* numerator would be 0 */
+    }
+    leaf= find_leaf(tree,val[size-1]);
+    if (leaf == TNULL) return 0.0; /* numerator would be 0 */
+    return leafcount(leaf)/totcount;
+}
+
+double prob_Njoint_Ncond(spade_prob_table *self,int size,features type[],valtype val[],int condbase) {
+    mindex tree=self->root[type[0]],leaf;
+    double basecount=1; /* initialized to keep compiler happy */
+    int i;
+    if (tree == TNULL) return PROBRESULT_NO_RECORD; /* denominator would be 0 */
+    if (condbase == 0) basecount= tree_count(tree);
+    for (i=1;i < size; i++) {
+        find_leaf_macro(tree,val[i-1],leaf);
+        if (condbase == i) basecount= leafcount(leaf);
+        if (leaf == TNULL) {
+            if (condbase < i) return PROBRESULT_NO_RECORD; /* denominator would be 0 */
+            else return 0.0; /* numerator would be 0 */
+        }
+        tree= find_nexttree_of_type(leaf,type[i]);
+        if (tree == TNULL) {
+            if (condbase < i) return PROBRESULT_NO_RECORD; /* denominator would be 0 */
+            else return 0.0; /* numerator would be 0 */
+        }
+    }
+    find_leaf_macro(tree,val[size-1],leaf);
+    if (leaf == TNULL) return 0.0; /* numerator would be 0 */
+    return leafcount(leaf)/basecount;
+}
+
+double prob_Njoint_Ncond_plus_one(spade_prob_table *self,int size,features type[],valtype val[],int condbase) {
+    mindex tree=self->root[type[0]],leaf;
+    double basecount=-1;
+    int i;
+    /* pretend the table has one more observation for numerator and numerator */
+    if (tree == TNULL) return 1; /* natural denominator is 0 */
+    if (condbase == 0) basecount= tree_count(tree)+1;
+    for (i=1;i < size; i++) {
+        find_leaf_macro(tree,val[i-1],leaf);
+        if (condbase == i) basecount= leafcount(leaf)+1;
+        if (leaf == TNULL) {
+            if (condbase < i) return 1; /* natural denominator is 0  */
+            else return 1/basecount; /* natural numerator is 0 */
+        }
+        tree= find_nexttree_of_type(leaf,type[i]);
+        if (tree == TNULL) {
+            if (condbase < i) return 1; /* natural denominator is 0  */
+            else return 1/basecount; /* natural numerator is 0 */
+        }
+    }
+    find_leaf_macro(tree,val[size-1],leaf);
+    if (leaf == TNULL) return 1/basecount; /* natural numerator is 0 */
+    return (leafcount(leaf)+1)/basecount;
+}
+
+/* return what the probability would be if some instance of the indicated feature had a count of 1 */
+double one_prob_simple(spade_prob_table *self,features type1) {
+    mindex root;
+    if (self->root[type1] == TNULL) return PROBRESULT_NO_RECORD; /* denominator would be 0 */
+    root= treeroot(self->root[type1]);
+    return 1/count_or_sum(root);
+}
+
+/*****************************************************/
+
+double jointN_count(spade_prob_table *self,int size,features type[], valtype val[]) {
+    mindex tree=self->root[type[0]],leaf;
+    int i;
+    if (tree == TNULL || treeroot(tree) == TNULL) {
+        return 0.0;
+    }
+    if (size == 0) {
+        return count_or_sum(treeroot(tree));
+    }
+    for (i=1;i < size; i++) {
+        find_leaf_macro(tree,val[i-1],leaf);
+        if (leaf == TNULL) return 0.0;
+        tree= find_nexttree_of_type(leaf,type[i]);
+        if (tree == TNULL) {
+            return 0.0;
+        }
+    }
+    find_leaf_macro(tree,val[size-1],leaf);
+    if (leaf == TNULL) return 0.0;
+    return leafcount(leaf);
+}
+
+/*****************************************************/
+
+double spade_prob_table_entropy(spade_prob_table *self,int depth,features type[], valtype val[]) {
+    mindex tree=self->root[type[0]],leaf;
+    int i;
+    //printf("H(%s",self->featurenames[type[0]]);
+    if (tree == TNULL) {
+        return 0.0;
+    }
+    for (i=1;i <= depth; i++) {
+        find_leaf_macro(tree,val[i-1],leaf);
+        if (leaf == TNULL) return 0.0;
+        tree= find_nexttree_of_type(leaf,type[i]);
+        if (tree == TNULL) {
+            return 0.0;
+        }
+        //printf("=%d,%s",val[i-1],self->featurenames[type[i]]);
+    }
+    //printf(")= ");
+    if (treeH(tree) < 0 || treeH_wait(tree) == 0) {
+        /* need to recalculate entropy */
+        int wait= count_or_sum(treeroot(tree)) * 0.1;
+        treeH_wait(tree)= wait > 10000 ? 10000 : (wait < 100 ? 100 : wait);
+        treeH(tree)= calc_tree_entropy(tree);
+        //printf("*");
+    }
+    //printf("%.4f\n",treeH(tree));
+    return treeH(tree);
+}
+
+static double calc_tree_entropy(mindex tree) {
+    mindex root= treeroot(tree);
+    return calc_subtree_entropy(root,count_or_sum(root));
+}
+
+static double calc_subtree_entropy(mindex node,double prob_base) {
+    double prob;
+    if (isleaf(node)) {
+        prob= leafcount(encleaf2mindex(node))/prob_base;
+        return -1*prob*(log(prob)/LOG2);
+    } else { /* recurse */
+        return calc_subtree_entropy(intleft(node),prob_base) +
+               calc_subtree_entropy(intright(node),prob_base);
+    }
+}
+
+/*****************************************************/
+static mindex find_nexttree_of_type(mindex leaf,features type) {
+    mindex t;
+    for (t=leafnexttree(leaf); t != TNULL; t=treenext(t)) { /* make common case quick */
+        if (treetype(t) == type) {
+            return t;
+        }
+    }
+    return TNULL;
+}
+
+static mindex get_nexttree_of_type(mindex leaf,features type) {
+    mindex t;
+    for (t=leafnexttree(leaf); t != TNULL; t=treenext(t)) { /* make common case quick */
+        if (treetype(t) == type) {
+            return t;
+        }
+    }
+    t= leafnexttree(leaf);
+    if (t == TNULL) {
+        leafnexttree(leaf)= new_treeinfo(type);
+        return leafnexttree(leaf);
+    }
+    for (; t != TNULL; t=treenext(t)) {
+        if (treenext(t) == TNULL) { /* we are at end */
+            treenext(t)= new_treeinfo(type);
+            return treenext(t);
+        }
+    }
+    return TNULL; /* just to keep cc -Wall from complaining :) */
+}
+
+/* increment the count of instance of val in the tree and return the leaf updated */
+static mindex incr_tree_value_count(mindex tree,valtype newval) {
+    mindex root=treeroot(tree);
+    if (treeH_wait(tree)) treeH_wait(tree)--;
+    if (root == TNULL) {
+        mindex newleaf=new_leaf(newval);
+        treeroot(tree)= asleaf(newleaf);
+        return newleaf;
+    }
+    if (isleaf(root)) {
+        mindex leaf= encleaf2mindex(root);
+        valtype curval= leafvalue(leaf);
+        
+        if (curval == newval) {
+            leafcount(leaf)++;
+            return leaf;
+        } else {
+            mindex newleaf= new_leaf(newval); /* count is 1 */
+            mindex node= new_int();
+            intsum(node)= leafcount(leaf)+1;
+            
+            if (curval < newval) {
+                intleft(node)= asleaf(leaf);
+                intright(node)= asleaf(newleaf);
+                intsortpt(node)= curval;
+            } else {
+                intleft(node)= asleaf(newleaf);
+                intright(node)= asleaf(leaf);
+                intsortpt(node)= newval;
+            }
+            /* no rebalancing possible now, so just set wait time to standard */
+            intwait(node)= wait_time(1,leafcount(leaf));
+            treeroot(tree)= node;
+            return newleaf;
+        }
+    } else {
+        return increment_value_count(root,newval);
+    }
+}
+
+/* increment the sum for this interior node and the counts and sums for subtrees containing the given value and return the leaf node for the value */
+static mindex increment_value_count(mindex node,valtype val) {
+    mindex child,res;
+    dmindex encchild;
+
+    /* TODO: optimize by making non-recursive (note: not tail-recursive) */
+    if (val <= intsortpt(node)) { /* going left */
+        encchild= intleft(node);
+    } else { /* going right */
+        encchild= intright(node);
+    }
+    
+    if (isleaf(encchild)) {
+        child= encleaf2mindex(encchild);
+        if (val == leafvalue(child)) { /* found the leaf */
+            intsum(node)++;
+            leafcount(child)++; 
+            res= child;
+        } else { /* need to add the leaf */
+            if (val > leafvalue(child)) { /* higher than right node */
+                res= add_node_above_to_right(node,val);
+            } else if (val <= intsortpt(node)) { /* lower than left node */
+                res= add_node_above_to_left(node,val);
+            } else { /* in between */
+                res= add_node_between(node,val);
+            }
+            /* note: "node" may have different children now */
+        }
+    } else { /* recurse */
+        child= encchild;
+        intsum(node)++;
+        res= increment_value_count(child,val);
+    }
+    
+    intwait(node)--;
+    if (intwait(node) == 0) {/*printf("** rebalancing %X since got to 0 **\n",node);*/rebalance_subtree(node);}
+    
+    return res;
+}
+
+/* conceptually add a node 'node' and with a new leaf for val to right; return new leaf */
+static mindex add_node_above_to_right(mindex node,valtype val) {
+    mindex leaf= new_leaf(val); /* count is 1 */
+    /* to keep things local (esp since don't have parent node), make the new intnode like 'node' */
+    mindex newint= dup_intnode(node);
+    
+    /* now reshape 'node' to have newint on left and leaf on right */
+    intleft(node)= newint;
+    intright(node)= asleaf(leaf);
+    intsum(node)++; /* sum on newint + count on leaf */
+    intsortpt(node)= largest_val(newint);
+
+    rebalance_subtree(node);
+    
+    return leaf;
+}
+
+/* conceptually add a node 'node' and with a new leaf for val to left; return new leaf */
+static mindex add_node_above_to_left(mindex node,valtype val) {
+    mindex leaf= new_leaf(val); /* count is 1 */
+    /* to keep things local (esp since don't have parent node), make the new intnode like 'node' */
+    mindex newint= dup_intnode(node);
+    
+    /* now reshape 'node' to have newint on right and leaf on left */
+    intright(node)= newint;
+    intleft(node)= asleaf(leaf);
+    intsum(node)++; /* sum on newint + count on leaf */
+    intsortpt(node)= val; /* val is largest value on left side */
+
+    rebalance_subtree(node);
+    
+    return leaf;
+}
+
+/*  */
+static mindex add_node_between(mindex node,valtype val) {
+    mindex leaf= new_leaf(val); /* count is 1 */
+
+    mindex newint= new_int();
+    intsortpt(newint)= val; /* val is largest value on left side */
+    intleft(newint)= asleaf(leaf);
+    intright(newint)= intright(node);
+    intsum(newint)= count_or_sum(intright(node))+1;
+    intright(node)= newint;
+    intsum(node)++; /* counts stayed the same except adding 1 */
+    
+    rebalance_subtree(newint);
+    
+    return leaf;
+}
+
+#if 0 /* not currently needed, prob not tested */
+/* regardless of wait counts, start rebalancing this tree from the root */
+static void rebalance_tree(mindex tree) {
+    mindex root=treeroot(tree);
+    if (root == TNULL) return;
+    rebalance_subtree(root);
+}
+#endif
+
+/* if given a leaf, do nothing.  Otherwise consider it time to try to rebalance this tree and recurse on new or moved interior nodes; reset the wait count on the node */
+static void rebalance_subtree(mindex encnode) {
+    mindex node,left,right;
+    int changed;
+
+    if (isleaf(encnode)) return;
+    node= encnode;
+
+    do {
+#ifdef NO_REBALANCE
+break;
+#endif
+/*printf("rebalance_subtree(%X):",encnode);printtree2_shallow(encnode);
+printf("\n",encnode);*/
+        changed= 0;
+        left= intleft(node);
+        right= intright(node);
+        
+        if ((left != TNULL) && (right != TNULL) && out_of_balance(node)) {
+            double lct=count_or_sum(left);
+            double rct=count_or_sum(right);
+            mindex left2,right2,newright,newleft;
+            double l2ct,r2ct,newsum;
+            double unbalanced_amount;
+            if (lct > rct) {
+                if (!isleaf(left)) {
+                    left2= intleft(left);
+                    right2= intright(left);
+                    l2ct= count_or_sum(left2);
+                    r2ct= count_or_sum(right2);
+                    newsum= r2ct+rct; /* sum for right interior node */
+                    unbalanced_amount= lct-rct;
+                    if (fabs(l2ct-newsum) < unbalanced_amount*0.999) { /* if improves balance */
+/*printf("[rotating %X right improves balance (%f,%f) [%f]",node,l2ct,newsum,fabs(l2ct-newsum));
+printf(" < (%f,%f)*0.999 [%f]\n",lct,rct,unbalanced_amount*0.999);*/
+                        /* rotate right */
+                        /* recycle "left" interior node into one for right */ 
+                        newright= left;
+                        intsortpt(newright)= largestval(right2);
+                        intleft(newright)= right2;
+                        intright(newright)= right;
+                        intsum(newright)= newsum;
+                        /* update "node" */
+                        intsortpt(node)= largestval(left2);
+                        intleft(node)= left2;
+                        intright(node)= newright;
+                        /* count stays the same */
+                        rebalance_subtree(newright);
+                        changed= 1;
+                    } else {
+                        mindex pprl,prl,rl,lrl,n;
+                        double rlct;
+                        valtype prl_largest;
+                        /*printf("rotating %X right would not improve balance (%.2f,%.2f) vs (%.2f,%.2f)\n",node,l2ct,newsum,lct,rct);*/
+                        /* find first node on the right edge of the "right2" tree that is smaller than unbalanced_amount (if any); also the parent (plr) and grandparent (pplr) */
+                        for (pprl= left,prl= right2; !isleaf(prl) && count_or_sum(intright(prl)) >= unbalanced_amount; pprl= prl,prl=intright(prl));
+                        if (!isleaf(prl)) {
+                            rl= intright(prl);
+                            /*printf("  so moving right of %X (%X) to left side\n",prl,rl);*/
+                            prl_largest= intsortpt(prl);
+                            rlct= count_or_sum(rl);
+                            lrl= intleft(prl);
+                            
+                            /* intsortpt(pprl) remains same */
+                            intright(pprl)= lrl;
+                            /* rl is now out of right of tree and prl can be recycled */
+                            /* use prl for node on right of "node", containing rl and "right" */
+                            newright= prl;
+                            intsortpt(newright)= largestval(rl);
+                            intleft(newright)= rl;
+                            intright(newright)= right;
+                            intsum(newright)= rct+count_or_sum(rl);
+                            intright(node)= newright;
+                            intsortpt(node)= prl_largest;
+                            /* update sums from left to pprl (inclusive) to reflect loss (rlct) of the node rl */
+                            for (n=left; 1; n=intright(n)) {
+                                intsum(n)-= rlct;
+                                if (n == pprl) break;
+                            }
+                            rebalance_subtree(newright);
+                            changed= 1;
+                        } /*else { printf("  and hit leaf (%X) before hitting node smaller than %.2f\n",encleaf2mindex(prl),unbalanced_amount);}*/
+                    }
+                } /*else {printf("%X cannot be rotated right since left is a leaf node\n",node);}*/
+            } else {
+                if (!isleaf(right)) {
+                    left2= intleft(right);
+                    right2= intright(right);
+                    l2ct= count_or_sum(left2);
+                    r2ct= count_or_sum(right2);
+                    newsum= lct+l2ct; /* sum for right interior node */
+                    unbalanced_amount= rct-lct;
+                    if (fabs(r2ct-newsum) < unbalanced_amount*0.999) { /* if improves balance */
+/*printf("[rotating %X left improves balance (%f,%f) [%f]",node,r2ct,newsum,fabs(r2ct-newsum));
+printf(" < (%f,%f)*0.999 [%f]]\n",rct,lct,unbalanced_amount*0.999);*/
+                        /* rotate left */
+                        /* transform "right" interior node into one for left */
+                        newleft= right;
+                        intsortpt(newleft)= largestval(left);
+                        intleft(newleft)= left;
+                        intright(newleft)= left2;
+                        intsum(newleft)= newsum;
+                        /* update "node" */
+                        intsortpt(node)= largestval(left2);
+                        intleft(node)= newleft;
+                        intright(node)= right2;
+                        /* count stays the same */
+                        rebalance_subtree(newleft);
+                        changed= 1;
+                    } else {
+                        mindex pplr,plr,lr,rlr,n;
+                        double lrct;
+                        valtype lr_largest;
+                        /*printf("rotating %X left would not improve balance (%.2f,%.2f) vs (%.2f,%.2f)\n",node,r2ct,newsum,rct,lct);*/
+                        /* find first node on the left edge of the "left2" tree that is smaller than unbalanced_amount (if any); also the parent (plr) and grandparent (pplr) */
+                        for (pplr= right,plr= left2; !isleaf(plr) && count_or_sum(intleft(plr)) >= unbalanced_amount; pplr= plr,plr=intleft(plr));
+                        if (!isleaf(plr)) {
+                            lr= intleft(plr);
+                            /*printf("  so moving left of %X (%X) to right side\n",plr,lr);*/
+                            lr_largest= intsortpt(plr);
+                            lrct= count_or_sum(lr);
+                            rlr= intright(plr);
+                            
+                            intsortpt(pplr)= largestval(rlr);
+                            intleft(pplr)= rlr;
+                            /* lr is now out of right of tree and plr can be recycled */
+                            /* use plr for node on left of node, containing left and lr */
+                            newleft= plr;
+                            intsortpt(newleft)= largestval(left);
+                            intleft(newleft)= left;
+                            intright(newleft)= lr;
+                            intsum(newleft)= lct+count_or_sum(lr);
+                            intleft(node)= newleft;
+                            intsortpt(node)= lr_largest;
+                            /* update sums from right to pplr (inclusive) to reflect loss (lrct) of the node lr */
+                            for (n=right; 1; n=intleft(n)) {
+                                intsum(n)-= lrct;
+                                if (n == pplr) break;
+                            }
+                            rebalance_subtree(newleft);
+                            changed= 1;
+                        }/* else { printf("  and hit leaf (%X) before hitting node smaller than %.2f\n",encleaf2mindex(plr),unbalanced_amount);}*/
+                    }
+                }/* else {printf("%X cannot be rotated left since right is a leaf node\n",node);}*/
+            }
+        }/* else {printf("%X is not out of balance\n",node);}*/
+    } while (changed);
+    
+    /* note: right and left of node may have changed */
+
+    /* reset the wait count */
+    intwait(node)= wait_time(count_or_sum(intleft(node)),count_or_sum(intright(node)));
+}
+
+static int out_of_balance(mindex node) {
+    double lct=count_or_sum(intleft(node));
+    double rct=count_or_sum(intright(node));
+    return (fabs(lct-rct) > 1) && (((rct>lct)?rct/lct:lct/rct)>=2.0);
+}
+
+static void free_all_in_tree(mindex tree) {
+/*printf("free_all_in_tree(%X)\n",tree);*/
+    if (tree != TNULL) {
+        if (treeroot(tree) != TNULL) free_all_in_subtree(treeroot(tree));
+        free_treeinfo(tree);
+    }
+}
+
+static void free_all_in_subtree(dmindex encnode) {
+    mindex node,t,next;
+/*printf("free_all_in_subtree(%X)\n",encnode);*/
+    if (isleaf(encnode)) {
+        node= encleaf2mindex(encnode);
+        for (t=leafnexttree(node); t != TNULL; t=next) {
+            next= treenext(t);
+            free_all_in_tree(t);
+        }
+        free_leaf(node);
+    } else {
+        node= encnode;
+        if (intleft(node) != TNULL) free_all_in_subtree(intleft(node));
+        if (intright(node) != TNULL) free_all_in_subtree(intright(node));
+        free_int(node);
+    }   
+}
+
+void scale_and_prune_table(spade_prob_table *self,double factor,double threshold) {
+    int i;
+    for (i=0; i < MAX_NUM_FEATURES; i++) {
+        if (self->root[i] != TNULL) scale_and_prune_tree(self->root[i],factor,threshold);
+    }
+}
+
+static void scale_and_prune_tree(mindex tree,double factor,double threshold) {
+    double change;
+    valtype newrightmost;
+    if (treeroot(tree) != TNULL) treeroot(tree)= scale_and_prune_subtree(treeroot(tree),factor,threshold,&change,&newrightmost);
+}
+
+static dmindex scale_and_prune_subtree(dmindex encnode,double factor,double threshold,double *change,valtype *newrightmost) {
+    mindex node,t;
+    int a_leaf= isleaf(encnode);
+
+    /* scale ourselves */
+    if (a_leaf) {
+        node= encleaf2mindex(encnode);
+        leafcount(node)*= factor;
+    } else {
+        node= encnode;
+        intsum(node)*= factor; /* should really get this by adding otherwise there is some drift */
+    }
+    
+    /* if we get too small, delete us and return TNULL and how much weight we had */
+    if (count_or_sum(encnode) < threshold) {
+/*printf("Deleting %X:\n",encnode);printtree2(encnode,"");printf("\n");*/
+        *change= count_or_sum(encnode);
+        *newrightmost= NOT_A_SORTPT; /* we don't have the info */
+        free_all_in_subtree(encnode);
+        return TNULL;
+    }
+    
+    /* scale below us and react to reported changes */
+    if (a_leaf) {
+        *change= 0.0;
+        *newrightmost= NOT_A_SORTPT;
+                
+        for (t=leafnexttree(node); t != TNULL; t=treenext(t)) {
+            scale_and_prune_tree(t,factor,threshold);
+        }
+    } else {
+        dmindex left,right;
+        double mychange,reduced=0.0;
+        left= intleft(node);
+        right= intright(node);
+        
+        if (left != TNULL) {
+            valtype leftnewrightmost; /* we want this to update our sortpt, if there is a change, and the rightmost has changed */
+            intleft(node)= scale_and_prune_subtree(left,factor,threshold,change,&leftnewrightmost);
+            if (*change > 0.0) {
+                reduced+= *change;
+                if (leftnewrightmost != NOT_A_SORTPT) intsortpt(node)= leftnewrightmost; /* there is a new rightmost on left side */
+            }
+        }
+        if (right != TNULL) {
+            intright(node)= scale_and_prune_subtree(right,factor,threshold,&mychange,newrightmost);
+            if (mychange > 0.0) {
+                reduced+= mychange;
+            }
+        } else {
+            *newrightmost= NOT_A_SORTPT;
+        }
+        *change= reduced;
+        
+        left= intleft(node);
+        right= intright(node);
+        if (right == TNULL && left != TNULL) *newrightmost= largestval(left);
+        
+        if (left == TNULL || right == TNULL) { /* at least one child is gone, so delete self and return whats left */
+            free_int(node);
+            if (left == TNULL) {
+                return right; /* might be TNULL */
+            } else {
+                return left;
+            }
+        } else { /* nothing deleted at this level */
+            intsum(node)-= reduced;
+        }
+    }
+    return encnode;
+}
+
+
+/* return the largest value found below this interior node */
+/* note: sometimes called from the macro function largestval(node) */
+static valtype largest_val(mindex node) {
+    dmindex encright= intright(node);
+    if (isleaf(encright)) {
+        return eleafval(encright);
+    } else {
+        return largest_val(encright);
+    }
+}
+
+/* make a new intermediate node identical to the give one */
+static mindex dup_intnode(mindex node) {
+    mindex newint= new_int();
+    intsortpt(newint)= intsortpt(node);
+    intleft(newint)= intleft(node);
+    intright(newint)= intright(node);
+    intsum(newint)= intsum(node);
+    intwait(newint)= intwait(node);
+    return newint;
+}
+
+
+/* return the leaf representing val in the tree else TNULL */
+static mindex find_leaf(mindex tree,valtype val) {
+    mindex leaf;
+    mindex root=treeroot(tree);
+    if (root == TNULL) {
+        return TNULL;
+    }
+    find_leaf_in_subtree_macro(root,val,leaf);
+    return leaf;
+}
+
+/* return the leaf representing val2 in the tree of type2 below the leaf representing val1 top level tree of type1 else TNULL */
+static mindex find_leaf2(spade_prob_table *self,features type1,valtype val1,features type2,valtype val2) {
+    mindex leaf,tree;
+    if (self->root[type1] == TNULL) {
+        return TNULL; /* actually this is an undefined case; this feature was not counted */
+    }
+    leaf= find_leaf(self->root[type1],val1);
+    if (leaf == TNULL) return TNULL;
+    find_nexttree_of_type_macro(leaf,type2,tree);
+    if (tree == TNULL) return TNULL; /* actually this is an undefined case; this feature was not counted */
+    return find_leaf(tree,val2);
+}
+
+/* return the leaf representing val3 in the tree of type3 below the leaf representing val2 in the tree of type2 below the leaf representing val1 top level tree of type1 else TNULL */
+static mindex find_leaf3(spade_prob_table *self,features type1,valtype val1,features type2,valtype val2,features type3,valtype val3) {
+    mindex leaf,tree;
+    if (self->root[type1] == TNULL) {
+        return TNULL; /* actually this is an undefined case; this feature was not counted */
+    }
+    leaf= find_leaf(self->root[type1],val1);
+    if (leaf == TNULL) return TNULL;
+    find_nexttree_of_type_macro(leaf,type2,tree);
+    if (tree == TNULL) return TNULL; /* actually this is an undefined case; this feature was not counted */
+    leaf= find_leaf(tree,val2);
+    if (leaf == TNULL) return TNULL;
+    find_nexttree_of_type_macro(leaf,type3,tree);
+    if (tree == TNULL) return TNULL; /* actually this is an undefined case; this feature was not counted */
+    return find_leaf(tree,val3);
+}
+
+
+/*****************************************************/
+
+float feature_trees_stats(spade_prob_table *self,features f,float *amind,float *amaxd,float *aaved,float *awaved) {
+    unsigned int tot_num_leaves=0,tot_mind=0,tot_maxd=0,tree_count=0;
+    float tot_aved=0.0,tot_waved=0.0;
+    unsigned int sum_num_leaves,sum_mind,sum_maxd;
+    float sum_aved,sum_waved;
+    int i;
+    for (i=0; i < MAX_NUM_FEATURES; i++) {
+        if (self->root[i] != TNULL) {
+            tree_count+= feature_tree_stats(self->root[i],f,&sum_mind,&sum_maxd,&sum_aved,&sum_waved,&sum_num_leaves);
+            tot_num_leaves+= sum_num_leaves;
+            tot_mind+= sum_mind;
+            tot_maxd+= sum_maxd;
+            tot_aved+= sum_aved;
+            tot_waved+= sum_waved;
+        }
+    }
+    if (tree_count == 0) return 0; /* no non-empty trees of this type */
+    *amind= tot_mind/((float)tree_count);
+    *amaxd= tot_maxd/((float)tree_count);
+    *aaved= tot_aved/((float)tree_count);
+    *awaved= tot_waved/((float)tree_count);
+    return tot_num_leaves/((float)tree_count);
+}
+
+static unsigned int feature_tree_stats(mindex tree,features f,unsigned int *smind,unsigned int *smaxd,float *saved,float *swaved,unsigned int *snum_leaves) {
+    unsigned int tree_count= 0;
+    dmindex root= treeroot(tree);
+    if (root != TNULL) {
+        tree_count= feature_subtree_stats(root,f,smind,smaxd,saved,swaved,snum_leaves);
+        if (treetype(tree) == f) {
+            /* gather stats from this tree */
+            unsigned int mind,maxd;
+            float aved,waved;
+            *snum_leaves+= tree_stats(tree,&mind,&maxd,&aved,&waved);
+            *smind+= mind;
+            *smaxd+= maxd;
+            *saved+= aved;
+            *swaved+= waved;
+            tree_count++;
+        }
+    }
+    return tree_count;
+}
+
+static unsigned int feature_subtree_stats(mindex encnode,features f,unsigned int *smind,unsigned int *smaxd,float *saved,float *swaved,unsigned int *snum_leaves) {
+    mindex node,t;
+    unsigned int new_smind,new_smaxd,new_snum_leaves;
+    float new_saved,new_swaved;
+    
+    unsigned int tree_count= 0;
+    *smind= *smaxd= *snum_leaves= 0;
+    *saved= *swaved= 0.0;
+    
+    if (isleaf(encnode)) {
+        node= encleaf2mindex(encnode);
+
+        for (t=leafnexttree(node); t != TNULL; t=treenext(t)) {
+            tree_count+= feature_tree_stats(t,f,&new_smind,&new_smaxd,&new_saved,&new_swaved,&new_snum_leaves);
+            *smind+= new_smind;
+            *smaxd+= new_smaxd;
+            *saved+= new_saved;
+            *swaved+= new_swaved;
+            *snum_leaves+= new_snum_leaves;
+        }
+    } else {
+        node= encnode;
+        if (intleft(node) != TNULL) {
+            tree_count+= feature_subtree_stats(intleft(node),f,&new_smind,&new_smaxd,&new_saved,&new_swaved,&new_snum_leaves);
+            *smind+= new_smind;
+            *smaxd+= new_smaxd;
+            *saved+= new_saved;
+            *swaved+= new_swaved;
+            *snum_leaves+= new_snum_leaves;
+        }
+        if (intright(node) != TNULL) {
+            tree_count+= feature_subtree_stats(intright(node),f,&new_smind,&new_smaxd,&new_saved,&new_swaved,&new_snum_leaves);
+            *smind+= new_smind;
+            *smaxd+= new_smaxd;
+            *saved+= new_saved;
+            *swaved+= new_swaved;
+            *snum_leaves+= new_snum_leaves;
+        }
+    }
+    return tree_count;
+}
+
+static unsigned int tree_stats(mindex tree,unsigned int *mind,unsigned int *maxd,float *aved,float *waved) {
+    unsigned int num_leafs= num_leaves(tree);
+    unsigned int tot= tree_depth_total(tree);
+    double wtot= weighted_tree_depth_total(tree);
+    tree_min_max_depth(tree,mind,maxd);
+    *aved= (float)tot/num_leafs;    
+    *waved= wtot/tree_count(tree);  
+/*printf("tree_stats results for tree %X: min depth=%u; max depth=%u; ave depth=%.2f; w. ave depth=%.2f; # vals repr=%u\n",tree,*mind,*maxd,*aved,*waved,num_leafs);*/
+    return num_leafs;
+}
+
+static double tree_count(mindex tree) {
+    mindex root=treeroot(tree);
+    if (root == TNULL) {
+        return 0.0;
+    }
+    return count_or_sum(root);
+}
+
+static unsigned int num_leaves(mindex tree) {
+    mindex root=treeroot(tree);
+    if (root == TNULL) {
+        return 0;
+    }
+    return num_subtree_leaves(root);
+}
+
+static unsigned int num_subtree_leaves(mindex encnode) {
+    if (isleaf(encnode)) {
+        return 1;
+    } else {
+        int count= 0;
+        if (intleft(encnode) != TNULL) count+=num_subtree_leaves(intleft(encnode));
+        if (intright(encnode) != TNULL) count+=num_subtree_leaves(intright(encnode));
+        return count;
+    }
+}
+
+static unsigned int tree_depth_total(mindex tree) {
+    mindex root=treeroot(tree);
+    if (root == TNULL) {
+        return 0;
+    }
+    return subtree_depth_total(root,0);
+}
+
+static unsigned int subtree_depth_total(mindex encnode,unsigned int depth) {
+    depth++;
+    if (isleaf(encnode)) {
+        return depth;
+    } else {
+        int count= 0;
+        if (intleft(encnode) != TNULL) count+=subtree_depth_total(intleft(encnode),depth);
+        if (intright(encnode) != TNULL) count+=subtree_depth_total(intright(encnode),depth);
+        return count;
+    }
+}
+
+static double weighted_tree_depth_total(mindex tree) {
+    mindex root=treeroot(tree);
+    if (root == TNULL) {
+        return 0;
+    }
+    return weighted_subtree_depth_total(root,0);
+}
+
+static double weighted_subtree_depth_total(mindex encnode,unsigned int depth) {
+    depth++;
+    if (isleaf(encnode)) {
+        return depth*leafnode(encleaf2mindex(encnode)).count;
+    } else {
+        double count= 0;
+        if (intleft(encnode) != TNULL) count+=weighted_subtree_depth_total(intleft(encnode),depth);
+        if (intright(encnode) != TNULL) count+=weighted_subtree_depth_total(intright(encnode),depth);
+        return count;
+    }
+}
+
+
+static void tree_min_max_depth(mindex tree,unsigned int *mind,unsigned int *maxd) {
+    mindex root=treeroot(tree);
+    
+    if (root == TNULL) {
+        *mind= *maxd= 0;
+    } else {
+        *mind= MAX_U32; /* this is the num of leaf vals, so a safe min */
+        *maxd= 0;
+        subtree_min_max_depth(root,mind,maxd,0);
+    }
+}
+
+static void subtree_min_max_depth(mindex encnode,unsigned int *mind,unsigned int *maxd,unsigned int depth) {
+    depth++;
+    if (isleaf(encnode)) {
+        if (*mind > depth) {
+            *mind= depth;
+        }
+        if (*maxd < depth) {
+            *maxd= depth;
+        }
+    } else {
+        if (intleft(encnode) != TNULL) subtree_min_max_depth(intleft(encnode),mind,maxd,depth);
+        if (intright(encnode) != TNULL) subtree_min_max_depth(intright(encnode),mind,maxd,depth);
+    }
+}
+
+/*****************************************************/
+/*****************************************************/
+void spade_prob_table_write_stats(spade_prob_table *self,FILE *file,u8 stats_to_print) {
+    featcomb H;
+
+    if (stats_to_print & STATS_ENTROPY) {
+        H= calc_all_entropies(self);
+        write_all_entropies(self,file,H);
+    }
+    if (stats_to_print & STATS_UNCONDPROB) write_all_uncond_probs(self,file);
+    if (stats_to_print & STATS_CONDPROB) write_all_cond_probs(self,file);
+}
+
+/* print to the given FILE the given features and values in the form <self->featurenames>=<value>, separated by commas; depth is the depth to look in the arrays */
+void write_feat_val_list(spade_prob_table *self,FILE *f,int depth,features feats[],valtype vals[]) {
+    int i;
+    if (depth == 0) return;
+    fprintf(f,"%s=%u",self->featurenames[feats[0]],vals[0]);
+    for (i=1; i < depth; i++) {
+        fprintf(f,",%s=%d",self->featurenames[feats[i]],vals[i]);
+    }
+}
+
+/* write a display of all unconditional probabilities to the given FILE */
+void write_all_uncond_probs(spade_prob_table *self,FILE *f) {
+    int i;
+    features feats[MAX_NUM_FEATURES];
+    valtype vals[MAX_NUM_FEATURES];
+    for (i=0; i < MAX_NUM_FEATURES; i++) {
+        if (self->root[i] != TNULL) write_all_tree_uncond_probs(self,f,self->root[i],0,feats,vals,count_or_sum(tree(self->root[i]).root));
+    }
+}
+
+/* write a display of all uncond probabilities rooted at this tree to the given FILE; depth is the last depth completed */
+static void write_all_tree_uncond_probs(spade_prob_table *self,FILE *f,mindex tree,int depth,features feats[],valtype vals[],double treesum) {
+    dmindex root= treeroot(tree);
+    if (root == TNULL) return;
+    feats[depth]= treetype(tree);
+    depth++;
+    write_all_subtree_uncond_probs(self,f,root,depth,feats,vals,treesum);
+}
+
+/* write a display of all uncond probabilities below this interior or leaf node (as encoded) to the given FILE; depth is the depth that we are at */
+static void write_all_subtree_uncond_probs(spade_prob_table *self,FILE *f,dmindex encnode,int depth,features feats[],valtype vals[],double treesum) {
+    mindex node,t;
+
+    if (isleaf(encnode)) {
+        node= encleaf2mindex(encnode);
+        vals[depth-1]= leafvalue(node);
+        fprintf(f,"P(");
+        write_feat_val_list(self,f,depth,feats,vals);
+        fprintf(f,")= %.12f\n",leafcount(node)/treesum);
+        
+        for (t=leafnexttree(node); t != TNULL; t=treenext(t)) {
+            write_all_tree_uncond_probs(self,f,t,depth,feats,vals,treesum);
+        }
+    } else {
+        node= encnode;
+        if (intleft(node) != TNULL) write_all_subtree_uncond_probs(self,f,intleft(node),depth,feats,vals,treesum);
+        if (intright(node) != TNULL) write_all_subtree_uncond_probs(self,f,intright(node),depth,feats,vals,treesum);
+    }
+}
+
+/*****************************************************/
+
+/* write a display of all conditional probabilities to the given FILE */
+void write_all_cond_probs(spade_prob_table *self,FILE *f) {
+    int i;
+    features feats[MAX_NUM_FEATURES];
+    valtype vals[MAX_NUM_FEATURES];
+    for (i=0; i < MAX_NUM_FEATURES; i++) {
+        if (self->root[i] != TNULL) write_all_tree_cond_probs(self,f,self->root[i],0,feats,vals);
+    }
+}
+
+/* write a display of all conditional probabilities rooted at this tree to the given FILE; depth is the last depth completed */
+static void write_all_tree_cond_probs(spade_prob_table *self,FILE *f,mindex tree,int depth,features feats[],valtype vals[]) {
+    dmindex root= treeroot(tree);
+    if (root == TNULL) return;
+    feats[depth]= treetype(tree);
+    depth++;
+    write_all_subtree_cond_probs(self,f,root,depth,feats,vals,count_or_sum(root));
+}
+
+/* write a display of all conditional probabilities below this interior or leaf node (as encoded) to the given FILE; depth is the depth that we are at */
+static void write_all_subtree_cond_probs(spade_prob_table *self,FILE *f,dmindex encnode,int depth,features feats[],valtype vals[],double treesum) {
+    mindex node,t;
+
+    if (isleaf(encnode)) {
+        node= encleaf2mindex(encnode);
+        vals[depth-1]= leafvalue(node);
+        if (depth > 1) {
+            fprintf(f,"P(%s=%u|",self->featurenames[feats[depth-1]],vals[depth-1]);
+            write_feat_val_list(self,f,depth-1,feats,vals);
+            fprintf(f,")= %.12f\n",leafcount(node)/treesum);
+        }
+        
+        for (t=leafnexttree(node); t != TNULL; t=treenext(t)) {
+            write_all_tree_cond_probs(self,f,t,depth,feats,vals);
+        }
+    } else {
+        node= encnode;
+        if (intleft(node) != TNULL) write_all_subtree_cond_probs(self,f,intleft(node),depth,feats,vals,treesum);
+        if (intright(node) != TNULL) write_all_subtree_cond_probs(self,f,intright(node),depth,feats,vals,treesum);
+    }
+}
+
+/*****************************************************/
+
+#if 0 /* not currently needed, prob not tested */
+static void write_featurecomb(featcomb C,double val,int depth,features feats[]) {
+    int i;
+    featcomb c= C;
+    for (i=0; i < (depth-1); i++) {
+        c= c->next[feats[i]];
+    }
+    c->val[feats[depth-1]]= val;
+}
+#endif
+
+static void inc_featurecomb(featcomb C,double val,int depth,features feats[]) {
+    int i;
+    featcomb c= C;
+    for (i=0; i < (depth-1); i++) {
+        c= c->next[feats[i]];
+    }
+    c->val[feats[depth-1]]+= val;
+}
+
+static featcomb create_featurecomb(int depth,double val) {
+    int i;
+    featcomb root= (featcomb)malloc(sizeof(struct _featcomb));
+    if (root == NULL) return NULL;
+    for (i=0; i < MAX_NUM_FEATURES; i++) {
+        root->val[i]= val;
+        if (depth > 1) {
+            root->next[i]= create_featurecomb(depth-1,val);
+        } else {
+            root->next[i]= NULL;
+        }
+    }
+    return root;
+}
+
+static void scale_all_featurecomb(featcomb c,double factor) {
+    int i;
+    for (i=0; i < MAX_NUM_FEATURES; i++) {
+        c->val[i]*= factor;
+        if (c->next[i] != NULL) scale_all_featurecomb(c->next[i],factor);
+    }
+}
+
+featcomb calc_all_entropies(spade_prob_table *self) {
+    features feats[MAX_NUM_FEATURES];
+    featcomb H= create_featurecomb(MAX_NUM_FEATURES,0.0);
+    int i;
+    for (i=0; i < MAX_NUM_FEATURES; i++) {
+        if (self->root[i] != TNULL) {
+            add_all_tree_entrsum(H,self->root[i],0,feats,tree_count(self->root[i]));
+        }
+    }
+    return H;
+}
+
+static void add_all_tree_entrsum(featcomb c,mindex tree,int depth,features feats[],double totsum) {
+    dmindex root= treeroot(tree);
+    if (root == TNULL) return;
+    feats[depth]= treetype(tree);
+    depth++;
+    add_all_subtree_entrsum(c,root,depth,feats,count_or_sum(root),totsum);
+}
+
+static void add_all_subtree_entrsum(featcomb c,dmindex encnode,int depth,features feats[],double treesum,double totsum) {
+    mindex node,t;
+
+    if (isleaf(encnode)) {
+        double mysumcomp,myprob,condprob;
+        node= encleaf2mindex(encnode);
+        myprob= leafcount(node)/totsum;
+        if (depth > 1) {
+            condprob= leafcount(node)/treesum;
+            mysumcomp= -1*myprob*(log(condprob)/LOG2);
+            /*printf("H[");
+            write_feature_names(stdout,depth,feats);
+            printf("]+=%f (-%f*(log(%f)/log2); myprob=%f/%f\n",mysumcomp,myprob,condprob,leafcount(node),totsum);*/
+        } else {
+            mysumcomp= -1*myprob*(log(myprob)/LOG2);
+        }
+        inc_featurecomb(c,mysumcomp,depth,feats);
+        
+        for (t=leafnexttree(node); t != TNULL; t=treenext(t)) {
+            add_all_tree_entrsum(c,t,depth,feats,totsum);
+        }
+    } else {
+        node= encnode;
+        if (intleft(node) != TNULL) add_all_subtree_entrsum(c,intleft(node),depth,feats,treesum,totsum);
+        if (intright(node) != TNULL) add_all_subtree_entrsum(c,intright(node),depth,feats,treesum,totsum);
+    }
+}
+
+void write_all_entropies(spade_prob_table *self,FILE *f,featcomb c) {
+    int i;
+    features feats[MAX_NUM_FEATURES];
+    for (i=0; i < MAX_NUM_FEATURES; i++) {
+        if (c->val[i] > 0) {
+            fprintf(f,"H(%s)=%.8f\n",self->featurenames[i],c->val[i]);
+        }
+    }
+    for (i=0; i < MAX_NUM_FEATURES; i++) {
+        if (c->next[i] != NULL) {
+            feats[0]= i;
+            write_all_entropies2(self,f,c->next[i],1,feats);
+        }
+    }
+}
+
+static void write_all_entropies2(spade_prob_table *self,FILE *f,featcomb c,int depth,features feats[]) {
+    int i;
+    for (i=0; i < MAX_NUM_FEATURES; i++) {
+        if (c->val[i] > 0) {
+            fprintf(f,"H(%s|",self->featurenames[i]);
+            write_feature_names(self,f,depth,feats);
+            fprintf(f,")=%.8f\n",c->val[i]);
+        }
+    }
+    for (i=0; i < MAX_NUM_FEATURES; i++) {
+        if (c->next[i] != NULL) {
+            feats[depth]= i;
+            write_all_entropies2(self,f,c->next[i],depth+1,feats);
+        }
+    }
+}
+
+/* print to the given FILE the given features separated by commas; depth is the depth to look in the array */
+static void write_feature_names(spade_prob_table *self,FILE *f,int depth,features feats[]) {
+    int i;
+    if (depth == 0) return;
+    fprintf(f,"%s",self->featurenames[feats[0]]);
+    for (i=1; i < depth; i++) {
+        fprintf(f,",%s",self->featurenames[feats[i]]);
+    }
+}
+
+/*****************************************************/
+
+void print_spade_prob_table(spade_prob_table *self) {
+    int i;
+    for (i=0; i < MAX_NUM_FEATURES; i++) {
+        if (self->root[i] != TNULL) {
+            printtree(self,self->root[i],"");
+        }
+    }
+}
+
+static void printtree(spade_prob_table *self,mindex tree,char *ind) {
+    mindex t;
+    for (t=tree; t != TNULL; t=treenext(t)) {
+        printf("%sTree %X of %s: ",ind,t,self->featurenames[treetype(t)]);
+        printtree2(self,treeroot(t),ind);
+        printf("\n");
+    }
+}
+
+static void printtree2(spade_prob_table *self,dmindex encnode,char *ind) {
+    mindex node;
+    char myind[4*MAX_NUM_FEATURES+1];
+    if (encnode == TNULL) {
+        printf("NULL");
+    } else if (isleaf(encnode)) {
+        node=encleaf2mindex(encnode);
+        printf("{%X: %dx%.2f",node,leafvalue(node),leafcount(node));
+        if (leafnexttree(node) != TNULL) {
+            sprintf(myind,"    %s",ind);
+            printf(" ->{{\n");
+            printtree(self,leafnexttree(node),myind);
+            printf("%s}}",ind);
+        }
+        printf("}");
+    } else {
+        node= encnode;
+        printf("[%X: <=%d (%.2f) W=%d ",node,intsortpt(node),intsum(node),intwait(node));
+        printtree2(self,intleft(node),ind);
+        printf(" ");
+        printtree2(self,intright(node),ind);
+        printf("]");
+    }
+}
+
+#if 0 /* not currently needed, prob not tested */
+static void printtree_shallow(spade_prob_table *self,mindex tree) {
+    printf("Tree %X of %s: ",tree,self->featurenames[treetype(tree)]);
+    printtree2_shallow(treeroot(tree));
+    printf("\n");
+}
+#endif
+
+static void printtree2_shallow(dmindex encnode) {
+    mindex node;
+    if (encnode == TNULL) {
+        printf("NULL");
+    } else if (isleaf(encnode)) {
+        node=encleaf2mindex(encnode);
+        printf("{%X: %dx%.2f",node,leafvalue(node),leafcount(node));
+        printf("}");
+    } else {
+        node= encnode;
+        printf("[%X: <=%d (%.2f) ",node,intsortpt(node),intsum(node));
+        printtree2_shallow(intleft(node));
+        printf(" ");
+        printtree2_shallow(intright(node));
+        printf("]");
+    }
+}
+
+/*****************************************************/
+
+int sanity_check_spade_prob_table(spade_prob_table *self) {
+    int i,numerrs=0;
+    for (i=0; i < MAX_NUM_FEATURES; i++) {
+        if (self->root[i] != TNULL) {
+            numerrs+= sanity_check_tree(self->root[i]);
+        }
+    }
+    return numerrs;
+}
+
+static int sanity_check_tree(mindex tree) {
+    dmindex root= treeroot(tree);
+    int numerrs= 0;
+    if (treetype(tree) >= MAX_NUM_FEATURES) {
+        fprintf(stderr,"*** integrity check failure: type of %X is not valid (%d)\n",tree,treetype(tree));
+        numerrs++;
+    }
+    if (treeroot(tree) != TNULL) {
+        numerrs+= sanity_check_subtree(root);
+    }
+    return numerrs;
+}
+
+static int sanity_check_subtree(dmindex encnode) {
+    int numerrs= 0;
+    mindex node,t;
+    double count,sum;
+    
+    if (isleaf(encnode)) {
+        node= encleaf2mindex(encnode);
+        count= leafcount(node);
+        if (count <= 0.0) {
+            fprintf(stderr,"*** integrity check failure: count on leaf %X is negative or 0 (%f)\n",node,count);
+            numerrs++;
+        }
+        for (t=leafnexttree(node); t != TNULL; t=treenext(t)) {
+            /* can check if our count is approx that of the root's child */
+            numerrs+= sanity_check_tree(t);
+        }
+    }  else {
+        dmindex left,right;
+        node= encnode;
+        sum= intsum(node);
+        left= intleft(node);
+        right= intright(node);
+        if ((left != TNULL) && (right != TNULL)) {
+            double lct= count_or_sum(left);
+            double rct= count_or_sum(right);
+            double ratio= (lct+rct)/sum;
+            if (ratio < 0.999 || ratio > 1.001) {
+                fprintf(stderr,"*** integrity check failure: sum on interior node %X (%f) does not match sum/counts on leaves (%f+%f)\n",node,sum,lct,rct);
+                numerrs++;
+            }
+        }
+        if (left == TNULL) {
+            fprintf(stderr,"*** integrity check failure: left of interior node %X is TNULL\n",node);
+            numerrs++;
+        } else {
+            numerrs+= sanity_check_subtree(left);
+        }
+        if (right == TNULL) {
+            fprintf(stderr,"*** integrity check failure: right of interior node %X is TNULL\n",node);
+            numerrs++;
+        } else {
+            numerrs+= sanity_check_subtree(right);
+        }
+        if ((left != TNULL) && (right != TNULL)) {
+            if (largestval(left) != intsortpt(node)) {
+                fprintf(stderr,"*** integrity check failure: sortpoint on interior node %X (%d) does not match largest value on left (%d)\n",node,intsortpt(node),largestval(left));
+                numerrs++;
+            }
+        }   
+    }
+    return numerrs;
+}
+
+
+int spade_prob_table_checkpoint(statefile_ref *s,spade_prob_table *self) {
+    return spade_state_checkpoint_arr(s,self->root,MAX_NUM_FEATURES,sizeof(mindex));
+}
+
+int spade_prob_table_recover(statefile_ref *s,spade_prob_table *self) {
+    return spade_state_recover_arr(s,&self->root,MAX_NUM_FEATURES,sizeof(mindex));
+}
+
+/* Id: spade_prob_table.c,v 1.1 2004/10/11 14:35:54 jonkman Exp $ */
+/*********************************************************************
+spade_prob_table_types.c, distributed as part of Spade v030125.1
+Author: James Hoagland, Silicon Defense (hoagland@SiliconDefense.com)
+copyright (c) 2002 by Silicon Defense (http://www.silicondefense.com/)
+Released under GNU General Public License, see the COPYING file included
+with the distribution or http://www.silicondefense.com/spice/ for details.
+
+Please send complaints, kudos, and especially improvements and bugfixes to
+hoagland@SiliconDefense.com.  As described in GNU General Public License, no
+warranty is expressed for this program.
+*********************************************************************/
+
+/*! \file spade_prob_table_types.c
+ * \brief 
+ *  spade_prob_table_types.c is a module containing the memory management
+ *  for the main 3 data types used by the spade_prob_table "class".
+ * \ingroup staterec
+ */
+
+/*! \addtogroup staterec
+    @{
+*/
+
+#include <stdio.h>
+#include <stdlib.h>
+
+treeroot **ROOT_M;
+intnode **INT_M;
+leafnode **LEAF_M;
+
+mindex root_freelist;
+mindex int_freelist;
+mindex leaf_freelist;
+
+unsigned char ROOT_BLOCK_BITS;
+unsigned char INT_BLOCK_BITS;
+unsigned char LEAF_BLOCK_BITS;
+unsigned int MAX_ROOT_BLOCKS;
+unsigned int MAX_INT_BLOCKS;
+unsigned int MAX_LEAF_BLOCKS;
+
+static void reset_mem();
+
+/* initialize the memory manager */
+void init_mem() {
+    static int spade_prob_table_mem_inited= 0;
+    reset_mem();
+
+    if (spade_prob_table_mem_inited) return; /* already inited */
+    spade_prob_table_mem_inited= 1;
+    
+    allocate_mem_blocks();
+}
+
+static void reset_mem() {
+    ROOT_BLOCK_BITS= DEFAULT_ROOT_BLOCK_BITS;
+    INT_BLOCK_BITS= DEFAULT_INT_BLOCK_BITS;
+    LEAF_BLOCK_BITS= DEFAULT_LEAF_BLOCK_BITS;
+    MAX_ROOT_BLOCKS= DEFAULT_MAX_ROOT_BLOCKS;
+    MAX_INT_BLOCKS= DEFAULT_MAX_INT_BLOCKS;
+    MAX_LEAF_BLOCKS= DEFAULT_MAX_LEAF_BLOCKS;
+
+    root_freelist=TNULL;
+    int_freelist=TNULL;
+    leaf_freelist=TNULL;
+}
+
+void allocate_mem_blocks() {
+    unsigned int i;
+
+    ROOT_M=(treeroot **)malloc(sizeof(treeroot *)*MAX_ROOT_BLOCKS);
+    for (i=0; i < MAX_ROOT_BLOCKS; i++) ROOT_M[i]= NULL;
+    INT_M=  (intnode **)malloc(sizeof(intnode *)*MAX_INT_BLOCKS);
+    for (i=0; i < MAX_INT_BLOCKS; i++) INT_M[i]= NULL;
+    LEAF_M=(leafnode **)malloc(sizeof(leafnode *)*MAX_LEAF_BLOCKS);
+    for (i=0; i < MAX_LEAF_BLOCKS; i++) LEAF_M[i]= NULL;
+}
+
+int reallocate_ptr_array(void ***arrptr,int oldsize,int newsize) {
+    unsigned int i;
+    void **arr= NULL;
+
+    arr= (void **)realloc(arr,sizeof(void *)*newsize);
+    if (arr == NULL) return 0;
+    for (i=oldsize; i < newsize; i++) arr[i]= NULL;
+    *arrptr= arr;
+    return 1;
+}
+
+/* allocate a new treeroot node with the give feature type and return it */
+mindex new_treeinfo(features type) {
+    mindex root;
+    int i,p;
+    if (root_freelist == TNULL) { /* need to allocate a new block */
+        /* find first unused block */
+        for (p=0; p < MAX_ROOT_BLOCKS && (ROOT_M[p] != NULL); p++) {}
+        if (p == MAX_ROOT_BLOCKS) {
+            fprintf(stderr,"exhausted all %d blocks of %d treeroots; exiting; you might want to increase DEFAULT_MAX_ROOT_BLOCKS or DEFAULT_ROOT_BLOCK_BITS in spp_spade.h or wherever it is defined\n",MAX_ROOT_BLOCKS,ROOT_BLOCK_SIZE);
+            printf("next free root: %X; int: %X, leaf: %X\n",root_freelist,int_freelist,leaf_freelist);
+            exit(1);
+        }
+        ROOT_M[p]= (treeroot *)calloc(ROOT_BLOCK_SIZE,sizeof(treeroot));
+        if (ROOT_M[p] == NULL) {
+            fprintf(stderr,"Out of memory! in allocation of new treeroot block; exiting");
+            exit(2);
+        }
+        /* add new slots to freelist */
+        root_freelist= root_index(p,0);
+        for (i=0; i < (ROOT_BLOCK_SIZE-1); i++) {
+#ifdef EXTRA_MARK_FREE
+            ROOT_M[p][i].root= TNULL;
+#endif
+            rfreenext(ROOT_M[p][i])= root_index(p,i+1);
+        }
+#ifdef EXTRA_MARK_FREE
+        ROOT_M[p][ROOT_BLOCK_SIZE-1].root= TNULL;
+#endif
+        rfreenext(ROOT_M[p][ROOT_BLOCK_SIZE-1])= TNULL;
+    }
+    /* give out the head and make its next the new head */
+    root= root_freelist;
+    root_freelist= rfreenext(tree(root_freelist));
+    treetype(root)= type;
+    treeroot(root)= TNULL;
+    treenext(root)= TNULL;
+    treeH(root)= -1;
+    treeH_wait(root)= 0;
+    return root;
+}
+
+/* free the treeroot node given */
+void free_treeinfo(mindex f) {
+#ifdef EXTRA_MARK_FREE
+    treeroot(f)= TNULL;
+#endif
+    /* add it to the start of the list */
+    rfreenext(tree(f))= root_freelist;
+    root_freelist= f;
+}
+
+
+/* allocate a new intnode node and return it */
+mindex new_int() {
+    mindex res;
+    int i,p;
+    if (int_freelist == TNULL) { /* need to allocate a new block */
+        /* find first unused block */
+        for (p=0; p < MAX_INT_BLOCKS && (INT_M[p] != NULL); p++) {}
+        if (p == MAX_INT_BLOCKS) {
+            fprintf(stderr,"exhausted all %d blocks of %d intnodes; exiting; you might want to increase DEFAULT_MAX_INT_BLOCKS or DEFAULT_INT_BLOCK_BITS in spp_spade.h or wherever it is defined\n",MAX_INT_BLOCKS,INT_BLOCK_SIZE);
+            printf("next free root: %X; int: %X, leaf: %X\n",root_freelist,int_freelist,leaf_freelist);
+            exit(1);
+        }
+        INT_M[p]= (intnode *)calloc(INT_BLOCK_SIZE,sizeof(intnode));
+        if (INT_M[p] == NULL) {
+            fprintf(stderr,"Out of memory! in allocation of new intnode block; exiting");
+            exit(2);
+        }
+        /* add new slots to freelist */
+        int_freelist= intnode_index(p,0);
+        for (i=0; i < (INT_BLOCK_SIZE-1); i++) {
+#ifdef EXTRA_MARK_FREE
+            INT_M[p][i].sum= -1;
+#endif
+            ifreenext(INT_M[p][i])= intnode_index(p,i+1);
+        }
+#ifdef EXTRA_MARK_FREE
+        INT_M[p][INT_BLOCK_SIZE-1].sum= -1;
+#endif
+        ifreenext(INT_M[p][INT_BLOCK_SIZE-1])= TNULL;
+    }
+    /* give out the head and make its next the new head */
+    res= int_freelist;
+    int_freelist= ifreenext(intnode(int_freelist));
+    intleft(res)= intright(res)= TNULL;
+    intsum(res)=0;
+    intsortpt(res)= NOT_A_SORTPT;
+    intwait(res)= 999;
+    return res;
+}
+
+/* free the intnode node given */
+void free_int(mindex f) {
+#ifdef EXTRA_MARK_FREE
+    intsum(f)= -1;
+#endif
+    /* add it to the start of the list */
+    ifreenext(intnode(f))= int_freelist;
+    int_freelist= f;
+}
+
+/* allocate a new leafnode node and return it */
+mindex new_leaf(valtype val) {
+    mindex res;
+    int i,p;
+    if (leaf_freelist == TNULL) { /* need to allocate a new block */
+        /* find first unused block */
+        for (p=0; p < MAX_LEAF_BLOCKS && (LEAF_M[p] != NULL); p++) {}
+        if (p == MAX_LEAF_BLOCKS) {
+            fprintf(stderr,"exhausted all %d blocks of %d leafnodes; exiting; you might want to increase DEFAULT_LEAF_ROOT_BLOCKS or DEFAULT_LEAF_BLOCK_BITS in spp_spade.h or wherever it is defined\n",MAX_LEAF_BLOCKS,LEAF_BLOCK_SIZE);
+            printf("next free root: %X; int: %X, leaf: %X\n",root_freelist,int_freelist,leaf_freelist);
+            exit(1);
+        }
+        LEAF_M[p]= (leafnode *)calloc(LEAF_BLOCK_SIZE,sizeof(leafnode));
+        if (LEAF_M[p] == NULL) {
+            fprintf(stderr,"Out of memory! in allocation of new leafnode block; exiting");
+            exit(2);
+        }
+        /* add new slots to freelist */
+        leaf_freelist= leafnode_index(p,0);
+        for (i=0; i < (LEAF_BLOCK_SIZE-1); i++) {
+#ifdef EXTRA_MARK_FREE
+            LEAF_M[p][i].count= -1;
+#endif
+            lfreenext(LEAF_M[p][i])= leafnode_index(p,i+1);
+        }
+#ifdef EXTRA_MARK_FREE
+        LEAF_M[p][LEAF_BLOCK_SIZE-1].count= -1;
+#endif
+        lfreenext(LEAF_M[p][LEAF_BLOCK_SIZE-1])= TNULL;
+    }
+    /* give out the head and make its next the new head */
+    res= leaf_freelist;
+    leaf_freelist= lfreenext(leafnode(leaf_freelist));
+    leafvalue(res)= val;
+    leafcount(res)= 1;
+    leafnexttree(res)= TNULL;
+    return res;
+}
+
+/* free the leafnode node given */
+void free_leaf(mindex f) {
+#ifdef EXTRA_MARK_FREE
+    leafcount(f)= -1;
+#endif
+    /* add it to the start of the list */
+    lfreenext(leafnode(f))= leaf_freelist;
+    leaf_freelist= f;
+}
+
+/* Id: spade_prob_table_types.c,v 1.1 2004/10/11 14:35:54 jonkman Exp $ */
+/*********************************************************************
+spade_state.c, distributed as part of Spade v030125.1
+Author: James Hoagland, Silicon Defense (hoagland@SiliconDefense.com)
+copyright (c) 2002 by Silicon Defense (http://www.silicondefense.com/)
+Released under GNU General Public License, see the COPYING file included
+with the distribution or http://www.silicondefense.com/spice/ for details.
+
+Please send complaints, kudos, and especially improvements and bugfixes to
+hoagland@SiliconDefense.com.  As described in GNU General Public License, no
+warranty is expressed for this program.
+*********************************************************************/
+
+/*! \file spade_state.c
+ * \brief 
+ *  spade_state.c contains a module to assist with checkpoint and recovery
+ *  of a Spade application; it is intimate with the spade_prob_table_types
+ *  module
+ * \ingroup staterec
+ */
+
+/*! \addtogroup staterec
+    @{
+*/
+
+#include <errno.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+
+#define CUR_FVERS 5
+
+/// treeroot structure used in file checkpoint version 4 and earlier
+typedef struct {
+    mindex next;  ///< the next tree root in a list
+    dmindex root; ///< root node of the tree, if top bit is 1, it is a leafnode, otherwise it is a interior node
+    features type;///< the feature that is being represented in this tree
+} upto_v4_treeroot;
+
+
+statefile_ref *spade_state_begin_checkpointing(char *filename,char *appname,u8 app_cur_fvers) {
+    statefile_ref *s= (statefile_ref *)malloc(sizeof(statefile_ref));
+    char v='v';
+    u8 fvers= CUR_FVERS,uc;
+    double d= 1234.56789;
+    u32 l= 0x01020304;
+    u32 i,blocks_used;
+    u8 numfeat= MAX_NUM_FEATURES;
+
+    errno=0;
+    s->f= fopen(filename,"wb");
+    if (errno) {
+        perror(filename);
+        free(s);
+        return NULL;
+    }
+
+    fwrite(&v,sizeof(v),1,s->f);
+    fwrite(&fvers,1,1,s->f);
+    spade_state_checkpoint_str(s,appname);
+    fwrite(&app_cur_fvers,1,1,s->f);
+    
+    uc= sizeof(u16);
+    fwrite(&uc,sizeof(uc),1,s->f);
+    uc= sizeof(u32);
+    fwrite(&uc,sizeof(uc),1,s->f);
+    uc= sizeof(double);
+    fwrite(&uc,sizeof(uc),1,s->f);
+    
+    /* + the integer 0x01020304 (16,909,060) as a 4 byte unsigned int, to indicate the endianness of this file */
+    fwrite(&l,4,1,s->f);
+    /* + the number 1234.56789 as a double (sizeof(double)) [to verify that doubles are binary compatable] */
+    fwrite(&d,sizeof(d),1,s->f);
+
+    fwrite(&numfeat,sizeof(numfeat),1,s->f);
+
+    /* write out tree library state */
+        /* treeroot type state */
+    fwrite(&ROOT_BLOCK_BITS,sizeof(ROOT_BLOCK_BITS),1,s->f);
+    for (blocks_used= 0; ROOT_M[blocks_used] != NULL; blocks_used++) {}
+    fwrite(&blocks_used,sizeof(blocks_used),1,s->f);
+    for (i= 0; i < blocks_used; i++) {
+        fwrite(ROOT_M[i],sizeof(treeroot),ROOT_BLOCK_SIZE,s->f);
+    }
+    fwrite(&root_freelist,sizeof(root_freelist),1,s->f);
+
+        /* intnode type state */
+    fwrite(&INT_BLOCK_BITS,sizeof(INT_BLOCK_BITS),1,s->f);
+    for (blocks_used= 0; INT_M[blocks_used] != NULL; blocks_used++) {}
+    fwrite(&blocks_used,sizeof(blocks_used),1,s->f);
+    for (i= 0; i < blocks_used; i++) {
+        fwrite(INT_M[i],sizeof(intnode),INT_BLOCK_SIZE,s->f);
+    }
+    fwrite(&int_freelist,sizeof(int_freelist),1,s->f);
+
+        /* leafnode type state */
+    fwrite(&LEAF_BLOCK_BITS,sizeof(LEAF_BLOCK_BITS),1,s->f);
+    for (blocks_used= 0; LEAF_M[blocks_used] != NULL; blocks_used++) {}
+    fwrite(&blocks_used,sizeof(blocks_used),1,s->f);
+    for (i= 0; i < blocks_used; i++) {
+        fwrite(LEAF_M[i],sizeof(leafnode),LEAF_BLOCK_SIZE,s->f);
+    }
+    fwrite(&leaf_freelist,sizeof(leaf_freelist),1,s->f);
+
+    return s;
+}
+
+int spade_state_end_checkpointing(statefile_ref *s) {
+    fclose(s->f);
+    free(s);
+    return 1;
+}
+
+int spade_state_checkpoint_str(statefile_ref *s,char *str) {
+    u16 len=strlen(str);
+    fwrite(&len,2,1,s->f);
+    fwrite(str,sizeof(char),strlen(str),s->f);
+    return 1;
+}
+
+int spade_state_checkpoint_arr(statefile_ref *s,void *arr,int len,int elsize) {
+    fwrite(arr,elsize,len,s->f);
+    return 1;
+}
+
+int spade_state_checkpoint_str_arr(statefile_ref *s,char **arr,int len) {
+    int i;
+    for (i=0; i < len; i++)
+        if (!spade_state_checkpoint_str(s,arr[i])) return 0;
+    return 1;
+}
+
+int spade_state_checkpoint_u32(statefile_ref *s,u32 val) {
+    fwrite(&val,4,1,s->f);
+    return 1;
+}
+
+int spade_state_checkpoint_u8(statefile_ref *s,u8 val) {
+    fwrite(&val,1,1,s->f);
+    return 1;
+}
+
+int spade_state_checkpoint_time_t(statefile_ref *s,time_t val) {
+    return spade_state_checkpoint_u32(s,val);
+}
+
+int spade_state_checkpoint_double(statefile_ref *s,double val) {
+    fwrite(&val,sizeof(double),1,s->f);
+    return 1;
+}
+
+int spade_state_end_section(statefile_ref *s) {
+    u16 nul= 0x0000;
+    fwrite(&nul,2,1,s->f);
+    return 1;
+}
+
+#define PREMATURE_END_CHECK(count,minsize) if (count < minsize) { \
+        fprintf(stderr,"Premature end in Spade recovery file %s; not recovering from it\n",filename); \
+        fclose(s->f); \
+        return NULL; \
+    }
+
+#define CORRUPT_FILE_CHECK(testres,whatwentwrong) \
+    if (testres) { \
+        fprintf(stderr,"Corrupt Spade recovery file %s: %s; not recovering from it\n",filename,whatwentwrong); \
+        fclose(s->f); \
+        return NULL; \
+    }
+
+
+statefile_ref *spade_state_begin_recovery(char *filename,int min_app_fvers,char **appname,u8 *file_app_fvers) {
+    statefile_ref *s= (statefile_ref *)malloc(sizeof(statefile_ref));
+    unsigned char uc,fvers;
+    char v;
+    unsigned int i,blocks_used;
+    int count;
+    u8 numfeat;
+    double d;
+    u32 l;
+    
+    init_mem();
+    
+    if (s == NULL) return NULL;
+    
+    errno=0;
+    s->f= fopen(filename,"rb");
+    if (errno) { /* file prob does not exist */
+        return NULL;
+    }
+    
+    count= fread(&v,sizeof(v),1,s->f);
+    PREMATURE_END_CHECK(count,1);
+    if (v == 'v') { /* format version # encoded */
+        count= fread(&fvers,1,1,s->f);
+        PREMATURE_END_CHECK(count,1);
+    } else { /* format version #0; only diff from version 1 is that the version # is listed */
+        fvers= 0;
+    }
+    if (fvers < 4) {
+        fprintf(stderr,"This version of the Spade state recover procedure cannot read file %s since has format version %d; this routine can recover version 4 onwards; sorry\n",filename,fvers);
+        fclose(s->f);
+        return NULL;
+    }
+    if (fvers > CUR_FVERS) {
+        fprintf(stderr,"This version of the Spade state recover procedure cannot read file %s since has format version %d; this routine can only handle up to version %d\n",filename,fvers,CUR_FVERS);
+        fclose(s->f);
+        return NULL;
+    }
+    spade_state_recover_str(s,appname);
+    count= fread(file_app_fvers,1,1,s->f);
+    PREMATURE_END_CHECK(count,1);
+    if (*file_app_fvers < min_app_fvers) {
+        fprintf(stderr,"Spade state recover failed on file %s: application %s version was %d but at least version %d was required\n",filename,*appname,*file_app_fvers,min_app_fvers);
+        fclose(s->f);
+        return NULL;
+    }
+
+    count= fread(&uc,sizeof(uc),1,s->f);
+    PREMATURE_END_CHECK(count,1);
+    if (sizeof(u16) != uc) {
+        fprintf(stderr,"u16 type size from recovery file (%s) (%d bytes) does not match current size (%d bytes)\n",filename,uc,(int)sizeof(u16));
+        fclose(s->f);
+        return NULL;
+    }
+    count= fread(&uc,sizeof(uc),1,s->f);
+    PREMATURE_END_CHECK(count,1);
+    if (sizeof(u32) != uc) {
+        fprintf(stderr,"u32 type size from recovery file (%s) (%d bytes) does not match current size (%d bytes)\n",filename,uc,(int)sizeof(u32));
+        fclose(s->f);
+        return NULL;
+    }
+    count= fread(&uc,sizeof(uc),1,s->f);
+    PREMATURE_END_CHECK(count,1);
+    if (sizeof(double) != uc) {
+        fprintf(stderr,"double type size from recovery file (%s) (%d bytes) does not match current size (%d bytes)\n",filename,uc,(int)sizeof(double));
+        fclose(s->f);
+        return NULL;
+    }
+    
+    /* ========= read in encoding and sanity check things ========== */
+    /* + the integer 0x01020304 (16,909,060) as a 4 byte unsigned int, to indicate the endianness of this file */
+    count= fread(&l,sizeof(l),1,s->f);
+    PREMATURE_END_CHECK(count,1);
+    if (l != 0x01020304) {
+        fprintf(stderr,"recovery file (%s) was produced with a different byte ordering (got 0x%x where expecting 0x01020304); you will need to convert the file\n",filename,l);
+        return NULL;
+    }
+
+    /* + the number 1234.56789 as a double (sizeof(double)) [to verify that doubles are binary compatable] */
+    count= fread(&d,sizeof(d),1,s->f);
+    PREMATURE_END_CHECK(count,1);
+    if (d > 1234.58 || d < 1234.55) {
+        fprintf(stderr,"recovery file (%s) was produced with a different double representation (got %f where expecting something close to 1234.56789); you will need to convert the file\n",filename,d);
+        return NULL;
+    }
+
+    count= fread(&numfeat,1,1,s->f);
+    PREMATURE_END_CHECK(count,1);
+    if (numfeat > MAX_NUM_FEATURES) {
+        fprintf(stderr,"recovery file (%s) has a higher number of features (%d) than we allow (%d); you will need to increase MAX_NUM_FEATURES\n",filename,numfeat,MAX_NUM_FEATURES);
+        return NULL;
+    }
+
+    count= fread(&ROOT_BLOCK_BITS,sizeof(ROOT_BLOCK_BITS),1,s->f);
+    PREMATURE_END_CHECK(count,1);
+    CORRUPT_FILE_CHECK(ROOT_BLOCK_BITS < 3,"stored ROOT_BLOCK_BITS is too small");
+    
+    /* use the max block size for this run unless there is more stored in the file */
+    count= fread(&blocks_used,sizeof(blocks_used),1,s->f);
+    PREMATURE_END_CHECK(count,1);
+    if (blocks_used > DEFAULT_MAX_ROOT_BLOCKS) {
+        reallocate_ptr_array((void ***)&ROOT_M,DEFAULT_MAX_ROOT_BLOCKS,blocks_used);
+        MAX_ROOT_BLOCKS= blocks_used;
+    }
+    
+    if (fvers >= 5) { // can read block of treeroots directly
+        for (i= 0; i < blocks_used; i++) {
+            ROOT_M[i]= (treeroot *)malloc(sizeof(treeroot)*ROOT_BLOCK_SIZE);
+            count= fread(ROOT_M[i],sizeof(treeroot),ROOT_BLOCK_SIZE,s->f);
+            PREMATURE_END_CHECK(count,ROOT_BLOCK_SIZE);
+        }
+    } else { // need to translate treeroot struct from treeroot_orig to treeroot
+        int j;
+        upto_v4_treeroot *origblock= (upto_v4_treeroot *)malloc(sizeof(upto_v4_treeroot)*ROOT_BLOCK_SIZE);
+        for (i= 0; i < blocks_used; i++) {
+            ROOT_M[i]= (treeroot *)malloc(sizeof(treeroot)*ROOT_BLOCK_SIZE);
+            count= fread(origblock,sizeof(upto_v4_treeroot),ROOT_BLOCK_SIZE,s->f);
+            PREMATURE_END_CHECK(count,ROOT_BLOCK_SIZE);
+            for (j=0; j < ROOT_BLOCK_SIZE; j++) {
+                // look at original structure and initialize new from it
+                ROOT_M[i][j].next= origblock[j].next;
+                ROOT_M[i][j].root= origblock[j].root;
+                ROOT_M[i][j].type= origblock[j].type;
+                ROOT_M[i][j].entropy= -1;
+            }
+        }
+        free(origblock);
+    }
+
+    count= fread(&root_freelist,sizeof(root_freelist),1,s->f);
+    PREMATURE_END_CHECK(count,1);
+    
+    
+    count= fread(&INT_BLOCK_BITS,sizeof(INT_BLOCK_BITS),1,s->f);
+    PREMATURE_END_CHECK(count,1);
+    CORRUPT_FILE_CHECK(INT_BLOCK_BITS < 3,"stored INT_BLOCK_BITS is too small");
+    
+    /* use the max block size for this run unless there is more stored in the file */
+    count= fread(&blocks_used,sizeof(blocks_used),1,s->f);
+    PREMATURE_END_CHECK(count,1);
+    if (blocks_used > DEFAULT_MAX_INT_BLOCKS) {
+        reallocate_ptr_array((void ***)&INT_M,DEFAULT_MAX_INT_BLOCKS,blocks_used);
+        MAX_INT_BLOCKS= blocks_used;
+    }
+    
+    for (i= 0; i < blocks_used; i++) {
+        INT_M[i]= (intnode *)malloc(sizeof(intnode)*INT_BLOCK_SIZE);
+        count= fread(INT_M[i],sizeof(intnode),INT_BLOCK_SIZE,s->f);
+        PREMATURE_END_CHECK(count,INT_BLOCK_SIZE);
+    }
+
+    count= fread(&int_freelist,sizeof(int_freelist),1,s->f);
+    PREMATURE_END_CHECK(count,1);
+    
+    
+    count= fread(&LEAF_BLOCK_BITS,sizeof(LEAF_BLOCK_BITS),1,s->f);
+    PREMATURE_END_CHECK(count,1);
+    CORRUPT_FILE_CHECK(LEAF_BLOCK_BITS < 3,"stored LEAF_BLOCK_BITS is too small");
+    
+    /* use the max block size for this run unless there is more stored in the file */
+    count= fread(&blocks_used,sizeof(blocks_used),1,s->f);
+    PREMATURE_END_CHECK(count,1);
+    if (blocks_used > DEFAULT_MAX_LEAF_BLOCKS) {
+        reallocate_ptr_array((void ***)LEAF_M,DEFAULT_MAX_LEAF_BLOCKS,blocks_used);
+        MAX_LEAF_BLOCKS= blocks_used;
+    }
+    
+    for (i= 0; i < blocks_used; i++) {
+        LEAF_M[i]= (leafnode *)malloc(sizeof(leafnode)*LEAF_BLOCK_SIZE);
+        count= fread(LEAF_M[i],sizeof(leafnode),LEAF_BLOCK_SIZE,s->f);
+        PREMATURE_END_CHECK(count,LEAF_BLOCK_SIZE);
+    }
+    
+    count= fread(&leaf_freelist,sizeof(leaf_freelist),1,s->f);
+    PREMATURE_END_CHECK(count,1);
+    
+    return s;
+}
+
+int spade_state_end_recovery(statefile_ref *s) {
+    fclose(s->f);
+    free(s);
+    return 1;
+}
+
+int spade_state_recover_check_end_of_section(statefile_ref *s,int *res) {
+    u16 c;
+    int count= fread(&c,2,1,s->f);
+    if (!count) return 0;
+    
+    *res= (c == 0x0000);
+    if (!*res) fseek(s->f,-2,SEEK_CUR);
+    return 1;
+}
+
+int spade_state_recover_arr(statefile_ref *s,void *arr,int len,int elsize) {
+    int count= fread(arr,elsize,len,s->f);
+    return count == len;
+}
+
+int spade_state_recover_str_arr(statefile_ref *s,char **arr,int len) {
+    int i;
+    for (i=0; i < len; i++)
+        if (!spade_state_recover_str(s,&arr[i])) return 0;
+    return 1;
+}
+
+int spade_state_recover_u32(statefile_ref *s,u32 *val) {
+    int count= fread(val,4,1,s->f);
+    return count == 1;
+}
+
+int spade_state_recover_u8(statefile_ref *s,u8 *val) {
+    int count= fread(val,1,1,s->f);
+    return count == 1;
+}
+
+int spade_state_recover_time_t(statefile_ref *s,time_t *val) {
+    return spade_state_recover_u32(s,(u32 *)val);
+}
+
+int spade_state_recover_double(statefile_ref *s,double *val) {
+    int count= fread(val,sizeof(double),1,s->f);
+    return count == 1;
+}
+
+
+int spade_state_recover_str(statefile_ref *s,char **str) {
+    u16 len;
+    int count= fread(&len,2,1,s->f);
+    if (!count) return 0;
+    *str= (char *)malloc(sizeof(char)*(len+1));
+    if (*str == NULL) return 0;
+    
+    count= fread(*str,sizeof(char),len,s->f);
+    (*str)[count]= '\0';
+    if (count < len) return 0;
+    return 1;
+}
+
+int spade_state_recover_str_to_buff(statefile_ref *s,char *buff,int maxlen) {
+    u16 len,readlen;
+    int count= fread(&len,2,1,s->f);
+    if (!count) return 0;
+    readlen= len > (maxlen-1) ? maxlen-1 : len;
+    count= fread(buff,sizeof(char),readlen,s->f);
+    if (count < readlen) return 0;
+    buff[readlen]= '\0';
+    
+    while (readlen < len) { /* discard extra */
+        char c;
+        count= fread(&c,sizeof(char),1,s->f);
+        if (!count) return 0;
+        readlen++;
+    }
+    return 1;
+}
+
+
+/* Id: spade_state.c,v 1.1 2004/10/11 14:35:54 jonkman Exp $ */
+/*********************************************************************
+thresh_adapter.c, distributed as part of Spade v030125.1
+Author: James Hoagland, Silicon Defense (hoagland@SiliconDefense.com)
+copyright (c) 2002 by Silicon Defense (http://www.silicondefense.com/)
+Released under GNU General Public License, see the COPYING file included
+with the distribution or http://www.silicondefense.com/spice/ for details.
+
+Please send complaints, kudos, and especially improvements and bugfixes to
+hoagland@SiliconDefense.com.  As described in GNU General Public License, no
+warranty is expressed for this program.
+*********************************************************************/
+
+/*! \file thresh_adapter.c
+ * \brief 
+ *  thresh_adapter.c contains a "class" thresh_adapter which implements
+ *  several ways to adapt a Spade threshold to hit a certain target
+ * \ingroup stmgr
+ */
+
+/*! \weakgroup stmgr
+    @{
+*/
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <math.h>
+
+static void thresh_adapter_setup_1_from_str(thresh_adapter *self,char *str);
+static void thresh_adapter_setup_2_from_str(thresh_adapter *self,char *str);
+static void thresh_adapter_setup_3_from_str(thresh_adapter *self,char *str);
+static void thresh_adapter_setup_4_from_str(thresh_adapter *self,char *str);
+static void thresh_adapter_new_pkt_rate(thresh_adapter *self, spade_enviro *enviro);
+static void thresh_adapter_2_new_pkt_rate(thresh_adapter *self, spade_enviro *enviro);
+static void thresh_adapter_3_new_pkt_rate(thresh_adapter *self, spade_enviro *enviro);
+static float grab_new_thresh_1(thresh_adapter *self, spade_enviro *enviro);
+static float grab_new_thresh_2(thresh_adapter *self, spade_enviro *enviro);
+static float calc_new_thresh_2(thresh_adapter *self, spade_enviro *enviro);
+static double thresh_from_obslists(adapt2_data *data);
+static double anom_ave(double a[], int size);
+static void reset_obslist(adapt2_data *data, int slot);
+static float grab_new_thresh_3(thresh_adapter *self, spade_enviro *enviro);
+static float grab_new_thresh_4(thresh_adapter *self,spade_enviro *enviro);
+static void thresh_adapter_1_new_score(thresh_adapter *self, double anom_score);
+static void thresh_adapter_2_new_score(thresh_adapter *self, double anom_score);
+static void thresh_adapter_3_new_score(thresh_adapter *self, double anom_score);
+
+void init_thresh_adapter(thresh_adapter *self,spade_msg_fn msg_callback) {
+    self->adapt_mode= 0;
+    self->period_start= (time_t)0;
+    self->pkt_period_count= 0;
+    self->last_total_stats= 0;
+    self->period_pkt_rate= 100000000.0;
+    self->period_acc_rate= 100000000.0;
+    self->done= 0;
+    self->msg_callback= msg_callback;
+}
+
+thresh_adapter *new_thresh_adapter(spade_msg_fn msg_callback) {
+    thresh_adapter *new= (thresh_adapter *)malloc(sizeof(thresh_adapter));
+    init_thresh_adapter(new,msg_callback);
+    return new;
+}
+
+void thresh_adapter_setup_from_str(thresh_adapter *self,int adaptmode,char *str) {
+    switch (adaptmode) {
+        case 1: thresh_adapter_setup_1_from_str(self,str); break;
+        case 2: thresh_adapter_setup_2_from_str(self,str); break;
+        case 3: thresh_adapter_setup_3_from_str(self,str); break;
+        case 4: thresh_adapter_setup_4_from_str(self,str); break;
+    }
+}
+
+void thresh_adapter_setup_1(thresh_adapter *self,int target,time_t period,float new_obs_weight,int by_count) {
+    adapt1_data *data= &self->d.a1;
+
+    self->adapt_mode= 1;
+    self->adapt_period= period;
+    self->adapt_by_count= by_count;
+    
+    data->target= target;
+    data->period= period;
+    data->new_obs_weight= new_obs_weight;
+    data->by_count= by_count;
+    
+    /* init list to contain 0 and 0; this is to let us assume the list has a
+       bottom and runner-up elsewhere */
+    data->top_list= (ll_double *)malloc(sizeof(ll_double));
+    data->top_list->val= 0.0;
+    data->top_list->next= (ll_double *)malloc(sizeof(ll_double));
+    data->top_list->next->val= 0.0;
+    data->top_list->next->next= NULL;
+    data->top_list_size= 1;
+}
+
+static void thresh_adapter_setup_1_from_str(thresh_adapter *self,char *str) {
+    int target=20,by_count=1;
+    float hours=2,new_obs_weight=0.5;
+    void *args[4];
+    time_t period;
+
+    args[0]= &target;
+    args[1]= &hours;
+    args[2]= &new_obs_weight;
+    args[3]= &by_count;
+    fill_args_space_sep(str,"i:target;f:obsper;f:newweight;b:bycount",args,self->msg_callback);
+
+    period= (time_t)(hours*3600);
+    thresh_adapter_setup_1(self,target,period,new_obs_weight,by_count);
+}
+
+
+void thresh_adapter_setup_2(thresh_adapter *self,double targetspec,double obsper,int NS,int NM,int NL) {
+    int i;
+
+    adapt2_data *data= &self->d.a2;
+
+    self->adapt_mode= 2;
+    self->adapt_period= (time_t)(obsper+0.5);
+    self->adapt_by_count= 1;
+    
+    data->targetspec= targetspec;
+    data->obsper= obsper;
+    data->NS= NS;
+    data->NM= NM;
+    data->NL= NL;
+    
+    /* 10000000 packets per hour is an overestimate but ensures we keep enough during the first period */
+    data->target= (int) floor(0.5+ (targetspec >= 1 ? targetspec*(obsper/3600.0) : ((10000000/3600.0)*obsper)*targetspec));
+    if (data->target==0) data->target= 1; /* ensure at least 1 long */
+
+    data->obslists_head= (dll_double **)malloc(NS * sizeof(dll_double *));
+    data->obslists_tail= (dll_double **)malloc(NS * sizeof(dll_double *));
+    data->obslists_size= (int *)malloc(NS * sizeof(int));
+    for (i= 0; i < NS; i++) {
+        data->obslists_head[i]= new_dll_double(0.0);
+        data->obslists_tail[i]= new_dll_double(0.0);
+        data->obslists_head[i]->next= data->obslists_tail[i];
+        data->obslists_tail[i]->prev= data->obslists_head[i];
+        data->obslists_size[i]= 1;
+    }
+    data->obsper_count= 0;
+    data->recScomps= (double *)malloc(NM * sizeof(double));
+    data->recMcomps= (double *)malloc(NL * sizeof(double));
+    
+    data->mid_anom_comp= 0;
+    data->long_anom_comp= 0;
+
+    data->obslist_new_slot= 0;
+    
+    data->per2_count=0;
+    data->per3_count=0;
+}
+
+static void thresh_adapter_setup_2_from_str(thresh_adapter *self,char *str) {
+    double targetspec=0.01,obsper=15;
+    int NS=4,NM=24,NL=7;
+    void *args[4];
+
+    args[0]= &targetspec;
+    args[1]= &obsper;
+    args[2]= &NS;
+    args[3]= &NM;
+    args[4]= &NL;
+    fill_args_space_sep(str,"d:target;d:obsper;i:NS;i:NM;i:NL",args,self->msg_callback);
+
+    obsper*= 60;
+    thresh_adapter_setup_2(self,targetspec,obsper,NS,NM,NL);
+}
+
+void thresh_adapter_setup_3(thresh_adapter *self,double targetspec,double obsper,int NO) {
+    adapt3_data *data= &self->d.a3;
+
+    self->adapt_mode= 3;
+    self->adapt_period= (time_t)(obsper+0.5);
+    self->adapt_by_count= 1;
+    
+    data->targetspec= targetspec;
+    data->NO= NO;
+    
+    /* 10000 packets per hour is our pure guess as to the rate of packets.
+       Is there a better way to figure out how many packets to note for our
+       first interval when we want a percent of packets? */
+    data->target= (int) floor(0.5+ (targetspec >= 1 ? targetspec*(obsper/3600.0) : ((10000/3600.0)*obsper)*targetspec));
+    if (data->target==0) data->target= 1;
+
+    data->hist= (double *)malloc(sizeof(double)*NO);
+    
+    /* init list to contain 0 and 0; this is to let us assume the list
+       has a bottom and runner-up elsewhere */
+    data->anoms= (ll_double *)malloc(sizeof(ll_double));
+    data->anoms->val= 0.0;
+    data->anoms->next= (ll_double *)malloc(sizeof(ll_double));
+    data->anoms->next->val= 0.0;
+    data->anoms->next->next= NULL;
+    data->anoms_size= 1;
+    data->completed_obs_per= 0;
+    data->obssum= 0;
+}
+
+static void thresh_adapter_setup_3_from_str(thresh_adapter *self,char *str) {
+    double targetspec=0.01,obsper=60;
+    int NO=168;
+    void *args[3];
+
+    args[0]= &targetspec;
+    args[1]= &obsper;
+    args[2]= &NO;
+    fill_args_space_sep(str,"d:target;d:obsper;i:numper",args,self->msg_callback);
+    obsper*= 60;
+    thresh_adapter_setup_3(self,targetspec,obsper,NO);
+}
+
+void thresh_adapter_setup_4(thresh_adapter *self,double thresh,double obsper) {
+    adapt4_data *data= &self->d.a4;
+
+    self->adapt_mode= 4;
+    self->adapt_period= (time_t)(obsper+0.5);
+    self->adapt_by_count= 0;
+    
+    data->thresh= thresh;
+}
+
+static void thresh_adapter_setup_4_from_str(thresh_adapter *self,char *str) {
+    double thresh=0.8,obsper=60;
+    void *args[2];
+
+    args[0]= &thresh;
+    args[1]= &obsper;
+    fill_args_space_sep(str,"d:thresh;d:obsper",args,self->msg_callback);
+    obsper*= 60;
+    thresh_adapter_setup_4(self,thresh,obsper);
+}
+
+
+void thresh_adapter_start_time(thresh_adapter *self,time_t now) {
+    //self->obs_start_time= now;
+    self->period_start= now;
+}
+
+int thresh_adapter_new_time(thresh_adapter *self,spade_enviro *enviro,double *sugg_thresh) {
+    int adapt_now= 0;
+    int new_period= 0;
+    if (self->done) return 0;
+    
+    if (self->period_start == (time_t)0) { /* first time called and start time not given */
+        self->period_start= enviro->now;
+        return 0;
+    }
+
+    while (enviro->now > (self->period_start + self->adapt_period)) {
+        new_period= 1;
+        thresh_adapter_new_pkt_rate(self,enviro);
+    }
+
+    if (self->adapt_by_count && (self->pkt_period_count >= 1)) {
+        if ((*(enviro->total_pkts) - self->last_total_stats) >= self->period_pkt_rate) {
+            adapt_now= 1;
+        }
+    } else {
+        if (new_period) {
+            adapt_now= 1;
+        }
+    }
+    
+    if (adapt_now) {
+        self->last_total_stats= *(enviro->total_pkts);
+        switch (self->adapt_mode) {
+        case 1: *sugg_thresh= grab_new_thresh_1(self,enviro); break;
+        case 2: *sugg_thresh= grab_new_thresh_2(self,enviro); break;
+        case 3: *sugg_thresh= grab_new_thresh_3(self,enviro); break;
+        case 4: *sugg_thresh= grab_new_thresh_4(self,enviro); break;
+        }
+    
+        return 1;
+    }
+    return 0;
+}
+
+static void thresh_adapter_new_pkt_rate(thresh_adapter *self,spade_enviro *enviro) {
+    self->pkt_period_count++;
+    self->period_pkt_rate= *(enviro->total_pkts)/(float)self->pkt_period_count;
+    self->period_acc_rate= enviro->pkt_stats.scored/(float)self->pkt_period_count;
+    self->period_start+= self->adapt_period;
+
+    switch (self->adapt_mode) {
+    //case 1: thresh_adapter_1_new_pkt_rate(self,enviro); break;
+    case 2: thresh_adapter_2_new_pkt_rate(self,enviro); break;
+    case 3: thresh_adapter_3_new_pkt_rate(self,enviro); break;
+    }
+}
+
+static void thresh_adapter_2_new_pkt_rate(thresh_adapter *self,spade_enviro *enviro) {
+    adapt2_data *data= &self->d.a2;
+    dll_double *l;
+    
+    data->target= (int) floor(0.5+ (data->targetspec >= 1 ? data->targetspec*(self->adapt_period/3600.0) : data->targetspec*self->period_acc_rate));
+    if (data->target==0) data->target= 1; /* ensure at least 1 long */
+    
+    //if (self->debug_level) printf("new target is %d\n",data->target);
+    
+    if (data->obsper_count == 0) {
+        data->obsper_count++;
+        data->obslist_new_slot= data->obsper_count % data->NS;
+        if (data->obslists_size[0] > data->target) { /* remove excess */
+            int i;
+            for (i= data->target, l=data->obslists_head[0]; i < data->obslists_size[0]; i++,l=l->next);
+            l->prev->next= NULL;
+            l->prev= NULL;
+            free_dll_double_list(data->obslists_head[0]);
+            data->obslists_head[0]= l;
+        }
+    }
+}
+
+static void thresh_adapter_3_new_pkt_rate(thresh_adapter *self,spade_enviro *enviro) {
+    adapt3_data *data= &self->d.a3;
+    ll_double *prev,*newstart;
+    int i;
+
+    data->target= (int) floor(0.5+ (data->targetspec >= 1 ? data->targetspec*(data->obsper/3600.0) : data->targetspec*self->period_acc_rate));
+    if (data->target==0) data->target= 1;
+    //if (self->debug_level) printf("new target is %d\n",data->target);
+    
+    if (data->completed_obs_per == 0) {
+        if (data->anoms_size > data->target) { /* remove excess */
+            for (i= data->target, prev=data->anoms; (i+1) < data->anoms_size; i++,prev=prev->next);
+            newstart= prev->next;
+            prev->next= NULL;
+            free_ll_double_list(data->anoms);
+            data->anoms= newstart;
+        }
+    }
+}
+
+static float grab_new_thresh_1(thresh_adapter *self,spade_enviro *enviro) {
+    ll_double *l;
+    float new_thresh;
+    adapt1_data *data= &self->d.a1;
+    double obs_thresh= (data->top_list->val + data->top_list->next->val)/2;
+    //if (self->debug_level) printf("observed recent ideal threshold is %.4f\n",obs_thresh);
+    if (enviro->thresh < 0.0) { /* started up with no reporting */
+        new_thresh= obs_thresh;
+    } else {
+        new_thresh= (1-data->new_obs_weight)*enviro->thresh + data->new_obs_weight*obs_thresh;
+    }
+    
+    //if (self->debug_level) printf("new threshold is %.4f\n",new_thresh); 
+    
+    for (l=data->top_list; l != NULL; l=l->next)  l->val= 0.0;
+    
+    return new_thresh;
+}
+
+static float grab_new_thresh_2(thresh_adapter *self,spade_enviro *enviro) {
+    int new_thresh= calc_new_thresh_2(self,enviro);
+    adapt2_data *data= &self->d.a2;
+
+    data->obsper_count++;
+    data->obslist_new_slot= data->obsper_count % data->NS;
+    reset_obslist(data,data->obslist_new_slot);
+
+    return new_thresh;
+}
+
+static float calc_new_thresh_2(thresh_adapter *self,spade_enviro *enviro) {
+    adapt2_data *data= &self->d.a2;
+    double rec_anom_comp= thresh_from_obslists(data);
+
+    //if (self->debug_level) printf("* New recent anom observation (#%d) is %.5f\n",data->obsper_count,rec_anom_comp);
+    if (data->obsper_count < (data->NS-1)) {
+        return rec_anom_comp; /* haven't observed mid or long yet */
+    }
+    if (((data->obsper_count+1) % data->NS) == 0) { /* time to add new mid */
+        data->recScomps[data->per2_count % data->NM]= rec_anom_comp;
+        //if (self->debug_level) printf("data->recScomps[%d]:= %.5f\n",data->per2_count % data->NM,rec_anom_comp);
+        data->per2_count++;
+        data->mid_anom_comp= anom_ave(data->recScomps,((data->per2_count < data->NM)?data->per2_count:data->NM));
+        //if (self->debug_level) printf("** New mid anom component (#%d) is %.5f\n",data->per2_count-1,data->mid_anom_comp);
+        if (data->per2_count < (data->NM-1)) {
+            return (rec_anom_comp+data->mid_anom_comp)/2.0; /* haven't observed long yet */
+        }
+        if ((data->per2_count % data->NM) == 0) { /* time to add new long */
+            data->recMcomps[data->per3_count % data->NL]= data->mid_anom_comp;
+            //if (self->debug_level) printf("data->recMcomps[%d]:= %.5f\n",data->per3_count % data->NL,data->mid_anom_comp);
+            data->per3_count++; 
+            data->long_anom_comp= anom_ave(data->recMcomps,((data->per3_count < data->NL)?data->per3_count:data->NL));
+            //if (self->debug_level) printf("*** New long anom component (#%d) is %.5f\n",data->per3_count-1,data->long_anom_comp);
+        }
+    }
+    if (data->per2_count < data->NM) {
+        return (rec_anom_comp+data->mid_anom_comp)/2.0; /* haven't observed long yet */
+    }
+    return (rec_anom_comp+data->mid_anom_comp+data->long_anom_comp)/3.0;
+}
+
+static double thresh_from_obslists(adapt2_data *data) {
+    dll_double **pos= (dll_double **)malloc(data->NS * sizeof(dll_double *));
+    int i,c,maxpos=-1;
+    double max,last_score=0.0,before_last_score=0.0;
+    if (0) { /*(self->debug_level > 1) {*/
+        dll_double *l;
+        printf("thresh_from_obslists: finding score that is #%d highest in:\n",data->target);
+        for (i= 0; i < data->NS; i++) {
+            printf("  slot %d: %.5f",i,data->obslists_head[i]->val);
+            for (l=data->obslists_head[i]->next; l != NULL; l=l->next) {
+                printf(" -> %.5f",l->val);
+            }
+            printf("\n");
+        }
+    }
+    for (i= 0; i < data->NS; i++) { /* init pos's to be the list tails */
+        pos[i]= data->obslists_tail[i];
+    }
+    for (c= 1; c <= data->target+1; c++) {
+        max= -1;
+        for (i= 0; i < data->NS; i++) {
+            if (pos[i] != NULL) {
+                if (max < pos[i]->val) {
+                    max= pos[i]->val;
+                    maxpos= i;
+                }
+                
+            }
+        }
+        if (max == -1) {/* should only happen if we don't have enough packets recorded */
+            free(pos);
+            return last_score; 
+        }
+        pos[maxpos]= pos[maxpos]->prev; /* we extracted the tail, so put prev here now */
+        before_last_score= last_score;
+        last_score= max; /* in case this is the last */
+    }
+    free(pos);
+    return (before_last_score+last_score)/2.0;
+}
+
+static double anom_ave(double a[],int size) {
+    double sum= 0.0;
+    int i;
+    if (0) { // self->debug_level) {
+        printf("anom_ave: taking average of (%.5f",a[0]);
+        for (i=1; i < size; i++) printf(",%.5f",a[i]);
+        printf(")\n");
+    }
+    for (i=0; i < size; i++) sum+= a[i];
+    return sum/(double)size;
+}
+
+static void reset_obslist(adapt2_data *data,int slot) {
+    dll_double *first= data->obslists_head[slot];
+    dll_double *second= first->next;
+    if (second->next != NULL) free_dll_double_list(second->next);
+    first->val= 0.0;
+    second->val= 0.0;
+    second->next= NULL;
+    data->obslists_tail[slot]= second;
+    data->obslists_size[slot]= 1;
+}
+
+
+static float grab_new_thresh_3(thresh_adapter *self,spade_enviro *enviro) {
+    ll_double *l;
+    int slot;
+    float new_thresh;
+    adapt3_data *data= &self->d.a3;
+    double obs_thresh= (data->anoms->val + data->anoms->next->val)/2;
+    
+    //if (self->debug_level) printf("observed recent ideal threshold for adapt3 is %.4f\n",obs_thresh);
+    
+    slot= data->completed_obs_per % data->NO;
+    data->completed_obs_per++;
+    if (data->completed_obs_per > data->NO) data->obssum-= data->hist[slot]; /* kicking a score out */
+    data->hist[slot]= obs_thresh;
+    data->obssum+= obs_thresh;
+    
+    if (0) { // self->debug_level > 1) {
+        int i;
+        printf("data->hist= [");
+        printf("%.4f",data->hist[0]);
+        for (i= 1; i < data->NO && i < data->completed_obs_per; i++) {
+            printf(",%.4f",data->hist[i]);
+        }
+        printf("]\n");
+    }
+    
+    new_thresh= data->obssum/((data->completed_obs_per >= data->NO)?data->NO:data->completed_obs_per);  
+    //if (self->debug_level) printf("new threshold is %.4f\n",new_thresh); 
+    
+    for (l=data->anoms; l != NULL; l=l->next)  l->val= 0.0;
+    
+    return new_thresh;
+}
+
+static float grab_new_thresh_4(thresh_adapter *self,spade_enviro *enviro) {
+    adapt4_data *data= &self->d.a4;
+    self->done= 1;
+    return data->thresh;
+}
+
+
+
+
+void thresh_adapter_new_score(thresh_adapter *self,double anom_score) {
+    if (self->done) return;
+    switch (self->adapt_mode) {
+    case 1: thresh_adapter_1_new_score(self,anom_score); break;
+    case 2: thresh_adapter_2_new_score(self,anom_score); break;
+    case 3: thresh_adapter_3_new_score(self,anom_score); break;
+    }
+}
+
+static void thresh_adapter_1_new_score(thresh_adapter *self,double anom_score) {
+    ll_double *new,*prev,*l;
+    adapt1_data *data= &self->d.a1;
+        
+    /* add anomaly score to list if it is high enough */
+    if (data->top_list_size <= data->target) {
+        new= (ll_double *)malloc(sizeof(ll_double));
+        data->top_list_size++;
+    } else if (anom_score > data->top_list->val) {
+        if (anom_score < data->top_list->next->val) {
+            data->top_list->val= anom_score; /* can just replace first */
+            return;
+        }
+        new= data->top_list;
+        data->top_list= data->top_list->next;
+    } else {
+        return;
+    }
+    new->val= anom_score;
+    for (prev= data->top_list, l=data->top_list->next; l != NULL && anom_score > l->val; prev=l,l=l->next);
+    /* add between prev and l */
+    prev->next= new;
+    new->next= l;
+}
+
+static void thresh_adapter_2_new_score(thresh_adapter *self,double anom_score) {
+    dll_double *new,*prev,*l;
+    adapt2_data *data= &self->d.a2;
+    double score= anom_score;
+    int slot= data->obslist_new_slot;
+
+    if (data->obslists_size[slot] < data->target) {
+        new= new_dll_double(score);
+        data->obslists_size[slot]++;
+    } else if (score > data->obslists_head[slot]->val) {
+        if (score < data->obslists_head[slot]->next->val) {
+            data->obslists_head[slot]->val= score; /* can just replace first in place*/
+            return;
+        }
+        new= data->obslists_head[slot];
+        new->val= score;
+        data->obslists_head[slot]= data->obslists_head[slot]->next;
+        new->next->prev= NULL;
+    } else {
+        return;
+    }
+    for (l=data->obslists_head[slot]->next; l != NULL && score > l->val; l=l->next);
+    /* add between l->prev and l */
+    prev= (l == NULL) ? data->obslists_tail[slot] : l->prev;
+    prev->next= new;
+    new->prev= prev;
+    new->next= l;
+    if (l == NULL) {
+        data->obslists_tail[slot]= new;
+    } else {
+        l->prev= new;
+    }
+}
+
+
+
+static void thresh_adapter_3_new_score(thresh_adapter *self,double anom_score) {
+    adapt3_data *data= &self->d.a3;
+    ll_double *prev,*next,*new;
+
+    /* add anomaly score to list if it is high enough */
+    if (data->anoms_size <= data->target) {
+        new= new_ll_double(anom_score);
+        data->anoms_size++;
+    } else if (anom_score > data->anoms->val) {
+        if (anom_score < data->anoms->next->val) {
+            data->anoms->val= anom_score; /* can just replace first */
+            return;
+        }
+        new= data->anoms;
+        new->val= anom_score;
+        data->anoms= data->anoms->next;
+    } else {
+        return;
+    }
+    for (prev= data->anoms, next=data->anoms->next; next != NULL && anom_score > next->val; prev=next,next=next->next);
+    /* add between prev and next */
+    prev->next= new;
+    new->next= next;
+}
+
+void thresh_adapter_print_config_details(thresh_adapter *self,FILE *f,char *indent) {
+    char indent2[100];
+    sprintf(indent2,"%s  ",indent);
+    
+    fprintf(f,"%sadapt_mode=%d\n",indent,self->adapt_mode);
+    fprintf(f,"%sadapt_period=%d; adapt_by_count=%d\n",indent,(int)self->adapt_period,self->adapt_by_count);
+
+    switch (self->adapt_mode) {
+    case 1:
+        fprintf(f,"%starget=%d; period=%d; new_obs_weight=%.3f\n",indent,self->d.a1.target,(int)self->d.a1.period,self->d.a1.new_obs_weight);
+        break;
+    case 2:
+        fprintf(f,"%stargetspec=%.3f; obsper=%d\n",indent,self->d.a2.targetspec,(int)self->d.a2.obsper);
+        fprintf(f,"%sNS=%d; NM=%d; NL=%d\n",indent,self->d.a2.NS,self->d.a2.NM,self->d.a2.NL);
+        break;
+    case 3:
+        fprintf(f,"%stargetspec=%.3f; obsper=%d; NO=%d\n",indent,self->d.a3.targetspec,(int)self->d.a3.obsper,self->d.a3.NO);
+        break;
+    case 4:
+        fprintf(f,"%sthresh=%.3f\n",indent,self->d.a4.thresh);
+        break;
+    }
+}
+
+/*@}*/
+/* Id: thresh_adapter.c,v 1.1 2004/10/11 14:35:54 jonkman Exp $ */
+/*********************************************************************
+thresh_adviser.c, distributed as part of Spade v030125.1
+Author: James Hoagland, Silicon Defense (hoagland@SiliconDefense.com)
+copyright (c) 2002 by Silicon Defense (http://www.silicondefense.com/)
+Released under GNU General Public License, see the COPYING file included
+with the distribution or http://www.silicondefense.com/spice/ for details.
+
+Please send complaints, kudos, and especially improvements and bugfixes to
+hoagland@SiliconDefense.com.  As described in GNU General Public License, no
+warranty is expressed for this program.
+*********************************************************************/
+
+/*! \file thresh_adviser.c
+ * \brief 
+ *  thresh_adviser.c contains a "class" thresh_adviser which takes
+ *  observations of Spade anomaly scores and provides a recommendation
+ *  about how to set the threshold to achieve a target rate.
+ * \ingroup stmgr
+ */
+
+/*! \weakgroup stmgr
+    @{
+*/
+
+#include <stdlib.h>
+
+void init_thresh_adviser(thresh_adviser *self,int obs_size,int obs_secs,spade_msg_fn msg_callback) {
+    self->obs_size= obs_size;
+    self->obs_secs= obs_secs;
+    
+    /* init list to contain just 0; this is to let us assume the list is not
+       empty elsewhere */
+    self->top_anom_list= (ll_double *)malloc(sizeof(ll_double));
+    self->top_anom_list->next= NULL;
+    self->top_anom_list->val= 0.0;
+    self->top_anom_list_size= 1;
+    
+    self->obs_start_time= (time_t)0;
+}
+
+void init_thresh_adviser_from_str(thresh_adviser *self,char *str,spade_msg_fn msg_callback) {
+    int obs_size=200,hours=24;
+    void *args[2];
+    time_t obs_secs;
+
+    args[0]= &obs_size;
+    args[1]= &hours;
+    fill_args_space_sep(str,"i:target;i:obsper",args,msg_callback);
+
+    obs_secs= (time_t)(hours*3600);
+    init_thresh_adviser(self,obs_size,obs_secs,msg_callback);
+}
+
+thresh_adviser *new_thresh_adviser(int obs_size,int obs_secs,spade_msg_fn msg_callback) {
+    thresh_adviser *new= (thresh_adviser *)malloc(sizeof(thresh_adviser));
+    init_thresh_adviser(new,obs_size,obs_secs,msg_callback);
+    return new;
+}
+
+void thresh_adviser_reset(thresh_adviser *self,int obs_size,int obs_secs,spade_msg_fn msg_callback) {
+    free_ll_double_list(self->top_anom_list);
+    init_thresh_adviser(self,obs_size,obs_secs,msg_callback);
+}
+
+void thresh_adviser_start_time(thresh_adviser *self,time_t time) {
+    self->obs_start_time= time;
+}
+
+int thresh_adviser_new_time(thresh_adviser *self,spade_enviro *enviro) {
+    if (self->obs_start_time == 0) { /* first time and start time not given */
+        self->obs_start_time= enviro->now;
+    } else if (enviro->now > (self->obs_start_time + self->obs_secs)) {
+        return 1;
+    }
+    return 0;
+}
+
+void thresh_adviser_new_score(thresh_adviser *self,double anom_score) {
+    ll_double *new,*prev,*l;
+    
+    if (self->top_anom_list_size <= self->obs_size) {
+        new= (ll_double *)malloc(sizeof(ll_double));
+        self->top_anom_list_size++;
+    } else if (anom_score > self->top_anom_list->val) {
+        if (self->top_anom_list->next == NULL ||
+            (self->top_anom_list->next != NULL && anom_score < self->top_anom_list->next->val)) {
+            self->top_anom_list->val= anom_score; /* can just replace first */
+            return;
+        }
+        new= self->top_anom_list;
+        self->top_anom_list= self->top_anom_list->next;
+    } else {
+        return;
+    }
+    new->val= anom_score;
+    for (prev= self->top_anom_list, l=self->top_anom_list->next; l != NULL && anom_score > l->val; prev=l,l=l->next);
+    /* add between prev and l */
+    prev->next= new;
+    new->next= l;   
+
+    return; // not done yet
+}
+
+void thresh_adviser_write_advice(thresh_adviser *self,FILE *file) {
+    ll_double *n;
+    double obs_hours= self->obs_secs/3600.0;
+
+    if (!self->obs_size || self->top_anom_list_size <= 1) return;
+
+    fprintf(file,"Threshold learning results: top %d anomaly scores over %.5f hours\n",self->top_anom_list_size-1,obs_hours);
+    fprintf(file,"  Suggested threshold based on observation: %.6f\n",(self->top_anom_list->val+self->top_anom_list->next->val)/2);
+    fprintf(file,"  Top scores: %.5f",self->top_anom_list->next->val);
+    for (n=self->top_anom_list->next->next; n != NULL; n=n->next) {
+        fprintf(file,",%.5f",n->val);
+    }
+    fprintf(file,"\n  First runner up is %.5f, so use threshold between %.5f and %.5f for %.3f packets/hr\n",self->top_anom_list->val,self->top_anom_list->val,self->top_anom_list->next->val,(self->top_anom_list_size/obs_hours));    
+}
+
+void thresh_adviser_print_config_details(thresh_adviser *self, FILE *f, char *indent) {
+    fprintf(f,"%sobs_size=%d; obs_secs=%d\n",indent,self->obs_size,(int)self->obs_secs);
+}
+
+/*@}*/
+/* Id: thresh_adviser.c,v 1.1 2004/10/11 14:35:54 jonkman Exp $ */
+/*********************************************************************
+anomscore_surveyer.c, distributed as part of Spade v030125.1
+Author: James Hoagland, Silicon Defense (hoagland@SiliconDefense.com)
+copyright (c) 2002 by Silicon Defense (http://www.silicondefense.com/)
+Released under GNU General Public License, see the COPYING file included
+with the distribution or http://www.silicondefense.com/spice/ for details.
+
+Please send complaints, kudos, and especially improvements and bugfixes to
+hoagland@SiliconDefense.com.  As described in GNU General Public License, no
+warranty is expressed for this program.
+*********************************************************************/
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+/*! \file anomscore_surveyer.c
+ * \brief 
+ *  anomscore_surveyer.c contains a class to do a survey of anomaly scores
+ * \ingroup stmgr
+ */
+
+/*! \addtogroup stmgr Score and threshold management
+    @{
+*/
+
+static double survey_ostat(anomscore_surveyer *self, double loc);
+
+int init_anomscore_surveyer(anomscore_surveyer *self,char *filename,float interval,spade_msg_fn msg_callback) {
+    if (filename == NULL) filename="-";
+    if (!strcmp(filename,"-")) {
+        self->surveyfile= stdout;
+    } else {
+        self->surveyfile= fopen(filename,"w");
+    }
+    if (!self->surveyfile) return 0;
+
+    self->interval= interval;
+    self->filename= strdup(filename);
+    self->list= NULL;
+    self->list_len= 0;
+    self->period= 1;
+    self->interval_start_time= (time_t)0;
+    self->rec_count= 0;
+    
+    fprintf(self->surveyfile,"%.2f minute interval #\tPacket Count\tMedian Anom\t90th Percentile Anom\t99th Percentile Anom\n",self->interval/60.0);
+
+    return 1;
+}
+
+int init_anomscore_surveyer_from_str(anomscore_surveyer *self,char *str,spade_msg_fn msg_callback) {
+    char filename[400]= "-";
+    float interval=60.0;
+    void *args[2];
+
+    args[0]= &filename;
+    args[1]= &interval;
+    fill_args_space_sep(str,"s400:surveyfile;f:interval",args,msg_callback);
+
+    interval*= 60.0;
+    return init_anomscore_surveyer(self,filename,interval,msg_callback);
+}
+
+anomscore_surveyer *new_anomscore_surveyer(char *filename,float interval,spade_msg_fn msg_callback) {
+    anomscore_surveyer *new= (anomscore_surveyer *)malloc(sizeof(anomscore_surveyer));
+    init_anomscore_surveyer(new,filename,interval,msg_callback);
+    return new;
+}
+
+void anomscore_surveyer_flush(anomscore_surveyer *self) {
+    fflush(self->surveyfile);
+}
+
+void anomscore_surveyer_shutdown(anomscore_surveyer *self) {
+    fclose(self->surveyfile);
+}
+
+void anomscore_surveyer_new_time(anomscore_surveyer *self,spade_enviro *enviro) {
+    while (enviro->now > (self->interval_start_time + self->interval)) {
+        if (self->interval_start_time == 0) { /* first packet */
+            self->interval_start_time= enviro->now;
+        } else {
+            fprintf(self->surveyfile,"%d\t%d\t%.6f\t%.6f\t%.6f\n",self->period,self->rec_count,survey_ostat(self,0.5),survey_ostat(self,0.9),survey_ostat(self,0.99));
+            fflush(self->surveyfile);
+            if (self->list) 
+                free_ll_double_list(self->list);
+            self->list= NULL;
+            self->list_len= 0;
+            self->rec_count=0;
+            self->period++;
+            self->interval_start_time+= (long) self->interval;
+        }
+    }
+}
+
+void anomscore_surveyer_new_score(anomscore_surveyer *self,double anom_score) {
+    ll_double *new,*prev,*next;
+
+    new= new_ll_double(anom_score);
+    
+    if (self->list == NULL) {
+        self->list= new;
+        self->list_len= 1;
+    } else {
+        if (anom_score < self->list->val) { /* add at head */
+            new->next= self->list;
+            self->list= new;
+        } else {
+            for (prev= self->list, next=self->list->next; next != NULL && anom_score > next->val; prev=next,next=next->next);
+            /* add between prev and next */
+            prev->next= new;
+            new->next= next;    
+        }
+        self->list_len++;
+    }
+
+    self->rec_count++;
+}   
+
+static double survey_ostat(anomscore_surveyer *self,double loc) {
+    ll_double *pos;
+    int p;
+    double fromnext;
+    double posnum;
+    
+    if (self->list_len == 0) return 0.0;
+    posnum= loc*(double)self->list_len + (1.0-loc);/* = (self->list_len-1)*loc+1 */
+
+    for (p= 1, pos=self->list; p <= posnum && pos->next != NULL; p++,pos=pos->next);
+    fromnext= posnum-(double)(p-1);
+    if (fromnext == 0.0 || pos->next == NULL) { /* got it exactly */
+        return pos->val;
+    } else {
+        return (pos->val*(1-fromnext))+(pos->next->val*fromnext);
+    }
+}
+
+void anomscore_surveyer_print_config_details(anomscore_surveyer *self,FILE *f,char *indent) {
+    fprintf(f,"%sinterval=%.2f; filename=%s\n",indent,self->interval,self->filename);
+}
+
+/*@}*/
+/* Id: anomscore_surveyer.c,v 1.1 2004/10/11 14:35:54 jonkman Exp $ */
+/*********************************************************************
+strtok.c, distributed as part of Spade v030125.1
+Author: James Hoagland, Silicon Defense (hoagland@SiliconDefense.com)
+copyright (c) 2002 by Silicon Defense (http://www.silicondefense.com/)
+Released under GNU General Public License, see the COPYING file included
+with the distribution or http://www.silicondefense.com/spice/ for details.
+
+Please send complaints, kudos, and especially improvements and bugfixes to
+hoagland@SiliconDefense.com.  As described in GNU General Public License, no
+warranty is expressed for this program.
+*********************************************************************/
+
+
+/*! \file strtok.c
+ * \brief
+ *  strtok.c contains a module providing string parsing functionality
+ * \ingroup libspade_util
+ */
+
+/*! \addtogroup libspade_util
+    @{
+*/
+
+#include <ctype.h>
+#include <string.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+/// a record for an argument we are set to expect
+typedef struct _argspec {
+    /// the next argspec in a linked list
+    struct _argspec *next;
+    /// an indication of what type of argument value we are expecting and how to interpret it
+    /** 'c' indicates a single character, 's' a string, 'd' a double,
+        'f' a float, 'l' a long int, 'i' and int, and 'b' a boolean */
+    char type;
+    /// if we are expecting a sting, the maximum number of charecters that should be read
+    int maxlen;
+    /// for matching unlabeled argument values, this is the position number we are expect our value to be in
+    int argpos;
+    /// up to 5 strings we accept as our argument name; list is terminated by a '\0' in the first char of the first unused array slot
+    char labels[5][50];
+} argspec;
+
+argspec *argspec_freelist= NULL;
+
+static argspec *new_argspec(char format, int argpos);
+static void free_argspecs(argspec *head);
+//static void free_argspec(argspec *spec);
+static argspec *parse_argspec_format(char *format);
+
+int strtok_space_sep(char *str,char **after) {
+    char *p= str;
+    int len= 0;
+    if (p == NULL) return 0;
+    while (*p != '\0' && !isspace((int)*p)) {
+        p++;
+        len++;
+    }
+    while (*p != '\0' && isspace((int)*p)) p++;
+    *after= p;
+    return len;
+}
+
+static argspec *new_argspec(char format,int argpos) {
+    argspec *new;
+    if (argspec_freelist == NULL) {
+        new= (argspec *)malloc(sizeof(argspec));
+    } else {
+        new= argspec_freelist;
+        argspec_freelist= argspec_freelist->next;
+    }
+    new->next= NULL;
+    new->type= format;
+    new->maxlen= 0;
+    new->argpos= argpos;
+    new->labels[0][0]= '\0';
+    return new;
+}
+
+static void free_argspecs(argspec *head) {
+    argspec *last;
+    if (head == NULL) return;
+    for (last= head; last->next != NULL; last=last->next);
+    last->next= argspec_freelist;
+    argspec_freelist= head;
+}
+
+#if 0
+static void free_argspec(argspec *spec) {
+    if (spec == NULL) return;
+    spec->next= argspec_freelist;
+    argspec_freelist= spec;
+}
+#endif
+
+static argspec *parse_argspec_format(char *format) {
+    char errstr[100]= "\0";
+    char *f= format;
+    argspec *spec,*spechead=NULL,*spectail=NULL;
+    int argpos= 0;
+    char *rem;
+    
+    while (f != NULL && *f != '\0') {
+        // parse new arg
+        argpos++;
+        if (*f != 'c' && *f != 's' && *f != 'd' && *f != 'f' && *f != 'l' && *f != 'i' && *f != 'b') {
+            sprintf(errstr,"invalid arg letter: %c",*f);
+            break;
+        }
+        spec= new_argspec(*f,argpos);
+        f++;
+        if (isdigit(*f)) {
+            spec->maxlen= strtol(f,&f,10);
+        }
+        if (*f == ':') {
+            int count= 0;
+            f++;
+            while (isspace((int)*f)) f++;
+            rem= f;
+            f= strchr(rem,';');
+            do {
+                int len= 0;
+                if (*rem == ',') rem++;
+                while (isspace((int)*rem)) rem++;
+                while (*rem != '\0' && !isspace((int)*rem) && *rem != ',' && *rem != ';' && len < 50) {
+                    spec->labels[count][len]= *rem;
+                    rem++;
+                    len++;
+                }
+                spec->labels[count][len]= '\0';
+                while (isspace((int)*rem)) rem++;
+                count++;
+                if (count < 5) spec->labels[count][0]= '\0';
+            } while (count < 5 && *rem == ',');
+        }
+        if (f != NULL) {
+            while (isspace((int)*f)) f++;
+            if (*f == ';') f++;
+            while (isspace((int)*f)) f++;
+        }
+        if (spechead == NULL) {
+            spechead= spectail= spec;
+        } else {
+            spectail->next= spec;
+            spectail= spec;
+        }
+    }
+    if (errstr[0] != '\0') { // an error occurred
+        fprintf(stderr,"syntax error in spec format: %s: \"%s\"",errstr,format);
+        free_argspecs(spechead);
+        return NULL;
+    }
+    return spechead;
+}
+
+int fill_args_space_sep(char *str,char *format,void *args[],spade_msg_fn msg_callback) {
+    int i,arrfilepos;
+    char *next,*seppos,*labelhead,*valhead,*after;
+    char oldchar;
+    int copylen,len,labellen;
+    char *strcopy= strdup(str);
+    char *head= strcopy;
+    int argpos= 0;
+    int count= 0;
+    int no_pos_based= 0;
+    argspec *curspec,*spec;
+    argspec *argspecs;
+    argspec *pb_spec;
+    
+    if (*format == '$') {
+        no_pos_based= 1;
+        format++;
+    }
+    argspecs= parse_argspec_format(format);
+    pb_spec= argspecs;
+
+    while (head != NULL) {
+        argpos++;
+        len= strtok_space_sep(head,&next);
+        if (len == 0) break;
+        after= head+len;
+        oldchar= *after;
+        *after= '\0'; // temporarily null terminate head
+        
+        seppos= strchr(head,'=');
+        if (seppos == NULL) { // boolean label-based spec else position based spec
+            valhead= head;
+            labelhead= head;
+            labellen= len;
+        } else { // label-based spec or else string position based spec that contains a '='
+            /* search for matching spec label */
+            labelhead= head;
+            labellen= seppos-labelhead;
+            valhead= seppos+1;
+        }
+        
+        /* look for label-based-spec */
+        curspec= NULL;
+        for (spec= argspecs; curspec == NULL && spec != NULL; spec= spec->next) {
+            if (seppos == NULL && spec->type != 'b') continue;
+            for (i=0; i < 5; i++) {
+                if (spec->labels[i][0] == '\0') break;
+                if (!strncmp(spec->labels[i],labelhead,labellen) && strlen(spec->labels[i]) == labellen) {
+                    curspec= spec;
+                    break;
+                }
+            }
+        }
+        if (curspec == NULL) { // must be position based spec
+            if (!no_pos_based || pb_spec == NULL) {
+                if (seppos != NULL) { // must be string position based spec that contains a '='
+                    valhead= head; /* reset value start to start of arg */
+                }
+                while ((pb_spec->next != NULL) && (pb_spec->argpos < argpos)) pb_spec=pb_spec->next;
+            }
+            if (no_pos_based || pb_spec == NULL || (seppos != NULL && pb_spec->type != 's') || pb_spec->argpos != argpos) {
+                formatted_spade_msg_send(SPADE_MSG_TYPE_WARNING,msg_callback,"Warning: option \"%s\" not understood in \"%s\"; ignoring it",labelhead,str);
+                *(head+len)= oldchar; // restore orig char
+                head= next;
+                continue; // no match; ignore
+            }
+            curspec= pb_spec;
+        } else {
+            if (seppos == NULL) { /* a boolean with no '=' */
+                valhead= NULL;
+            }
+        }
+            
+        arrfilepos= curspec->argpos-1;
+        switch (curspec->type) {
+            case 'i':
+                *((int *)args[arrfilepos])= atoi(valhead);
+                break;
+            case 'l':
+                *((long *)args[arrfilepos])= atol(valhead);
+                break;
+            case 'f':
+                *((float *)args[arrfilepos])= (float)atof(valhead);
+                break;
+            case 'd':
+                *((double *)args[arrfilepos])= atof(valhead);
+                break;
+            case 'c':
+                *((char *)args[arrfilepos])= *valhead;
+                break;
+            case 'b':
+            {
+                int bool;
+                if (valhead == NULL) {
+                    bool= 1;
+                } else {
+                    if (!strcmp(valhead,"yes") || !strcmp(valhead,"true") || !strcmp(valhead,"on")) {
+                        bool= 1;
+                    } else if  (!strcmp(valhead,"no") || !strcmp(valhead,"false") || !strcmp(valhead,"off")) {
+                        bool= 0;
+                    } else {
+                        bool= atoi(valhead);
+                    }
+                }
+                *((int *)args[arrfilepos])= bool;
+                break;
+            }
+            case 's':
+                copylen= after-valhead;
+                if (curspec->maxlen) {
+                    if (len > curspec->maxlen) copylen=curspec->maxlen;
+                }
+                strncpy((char *)args[arrfilepos],valhead,copylen);
+                ((char *)args[arrfilepos])[copylen]= '\0';
+                break;
+            default:
+                args[arrfilepos]= NULL;
+                break;
+        }
+        
+        *after= oldchar; // restore orig char
+        count++;
+        head= next;
+    }
+    free(strcopy);
+    free_argspecs(argspecs);
+    return count;
+}
+
+char *extract_str_arg_space_sep(char *str,char *argname) {
+    char *after;
+    char *val,*valhead;
+    int vallen,copylen;
+    char *head= str;
+    int arglen= strlen(argname);
+    char *strafter= str+strlen(str);
+    while ((head=strstr(head,argname)) != NULL) {
+        if (head != str && !isspace((int)*(head-1)))
+            continue;
+        after= head+arglen;
+        if (*(head+arglen) != '=')
+            continue;
+            
+        /* found it */
+        /* put in a str */
+        valhead= after+1;
+        after= valhead;
+        while (*after != '\0' && !isspace((int)*after)) after++;
+        vallen= (after-valhead);
+        val= (char *)malloc(sizeof(char)*(vallen+1));
+        strncpy(val,valhead,vallen);
+        *(val+vallen)= '\0';
+        
+        /* now remove it from the str */
+        while (*after != '\0' && isspace((int)*after)) after++;
+        copylen= strafter-after+1; /* includes the terminating \0 */
+        memmove(head,after,copylen);
+        
+        return val;
+    }
+    return NULL;
+}
+
+int terminate_first_tok(char *str,char *sepchars,char **head,char *oldchar) {
+    char *p= str;
+    int len;
+    
+    do {
+        char *sep= strpbrk(p,sepchars);
+        if (sep == NULL) {
+            if (*p == '\0') return 0;
+            len= strlen(p);
+        } else {
+            len= sep- p;
+        }
+        if (len > 0) {
+            if (sep != NULL) {
+                *oldchar= *sep;
+                *sep= '\0';
+            } else {
+                *oldchar= '\0';
+            }
+            *head= p;
+        } else {
+            p++;
+        }
+    } while (len == 0);
+    
+    return len;
+}
+
+/*@}*/
+/* Id: strtok.c,v 1.1 2004/10/11 14:35:54 jonkman Exp $ */
+/*********************************************************************
+dll_double.c, distributed as part of Spade v030125.1
+Author: James Hoagland, Silicon Defense (hoagland@SiliconDefense.com)
+copyright (c) 2002 by Silicon Defense (http://www.silicondefense.com/)
+Released under GNU General Public License, see the COPYING file included
+with the distribution or http://www.silicondefense.com/spice/ for details.
+
+Please send complaints, kudos, and especially improvements and bugfixes to
+hoagland@SiliconDefense.com.  As described in GNU General Public License, no
+warranty is expressed for this program.
+*********************************************************************/
+
+#include <stdlib.h>
+
+/*! \file dll_double.c
+ * \brief 
+ *  dll_double.c contains routines for dll_double allocation, initing,
+ *  and recycling; dll_doubles are doubly linked list of doubles
+ * \ingroup libspade_util
+ */
+
+/*! \addtogroup libspade_util Spade library utilities
+ * \brief this group contains general utility objects in libspade
+ * \ingroup libspade
+    @{
+*/
+
+/// free list of dll_doubles
+dll_double *free_dlink_list= NULL;
+
+/* creation and recycling routines for dll_double's */
+
+dll_double *new_dll_double(double val) {
+    dll_double *link;
+    if (free_dlink_list != NULL) {
+        link= free_dlink_list;
+        free_dlink_list= link->next;
+    } else {
+        link= (dll_double *)malloc(sizeof(dll_double));
+    }
+    link->val= val;
+    link->prev= NULL;
+    link->next= NULL;
+    return link;
+}
+
+void free_dll_double_list(dll_double *start) {
+    dll_double *end;
+    for (end= start; end->next != NULL; end=end->next);
+    end->next= free_dlink_list;
+    free_dlink_list= start;
+}
+
+/*@}*/
+/* Id: dll_double.c,v 1.1 2004/10/11 14:35:54 jonkman Exp $ */
+/*********************************************************************
+ll_double.c, distributed as part of Spade v030125.1
+Author: James Hoagland, Silicon Defense (hoagland@SiliconDefense.com)
+copyright (c) 2002 by Silicon Defense (http://www.silicondefense.com/)
+Released under GNU General Public License, see the COPYING file included
+with the distribution or http://www.silicondefense.com/spice/ for details.
+
+lease send complaints, kudos, and especially improvements and bugfixes to
+hoagland@SiliconDefense.com.  As described in GNU General Public License, no
+warranty is expressed for this program.
+*********************************************************************/
+
+/*! \file ll_double.c
+ * \brief
+ *  contains routines for ll_double allocation, initing, and
+ *  recycling; ll_doubles are singly linked list of doubles
+ * \ingroup libspade_util
+ */
+
+/*! \addtogroup libspade_util
+    @{
+*/
+
+#include <stdlib.h>
+
+/// free list of allocated ll_doubles
+ll_double *free_link_list=NULL;
+
+/* creation and recycling routines for ll_double's */
+
+ll_double *new_ll_double(double val) {
+    ll_double *link;
+    if (free_link_list != NULL) {
+        link= free_link_list;
+        free_link_list= link->next;
+    } else {
+        link= (ll_double *)malloc(sizeof(ll_double));
+    }
+    link->val= val;
+    link->next= NULL;
+    return link;
+}
+
+void free_ll_double_list(ll_double *start) {
+    ll_double *end,*next;
+    for (end= start, next=start->next; next != NULL; end=next,next=next->next);
+    end->next= free_link_list;
+    free_link_list= start;
+}
+
+/*@}*/
+
+/* Id: ll_double.c,v 1.1 2004/10/11 14:35:54 jonkman Exp $ */
+/*********************************************************************
+spade_event.c, distributed as part of Spade v030125.1
+Author: James Hoagland, Silicon Defense (hoagland@SiliconDefense.com)
+copyright (c) 2002 by Silicon Defense (http://www.silicondefense.com/)
+Released under GNU General Public License, see the COPYING file included
+with the distribution or http://www.silicondefense.com/spice/ for details.
+
+Please send complaints, kudos, and especially improvements and bugfixes to
+hoagland@SiliconDefense.com.  As described in GNU General Public License, no
+warranty is expressed for this program.
+*********************************************************************/
+
+#include <stdlib.h>
+
+/*! \file spade_event.c
+ * \brief 
+ *  spade_event.c contains routines for spade_event allocation, initing,
+ *  and recycling; spade_event reprents an event that Spade is processing
+ * \ingroup libspade_misc
+ */
+
+/*! \addtogroup libspade Spade library
+ * \brief This group contains the objects in libspade, which provides core
+ *  routines for Spade-like detection
+*/
+ 
+/*! \addtogroup libspade_misc Spade library misc
+ * \brief this group contains miscellaneous objects in libspade
+ * \ingroup libspade
+ * @{
+*/
+
+/// free list of allocated spade_events
+spade_event *spade_event_freelist=NULL;
+
+/* creation and recycling routines for spade_event's */
+
+spade_event *new_spade_event() {
+    spade_event *new;
+    if (spade_event_freelist != NULL) {
+        new= spade_event_freelist;
+        spade_event_freelist= (spade_event *)new->native;
+    } else {
+        new= (spade_event *)malloc(sizeof(spade_event));
+    }
+    new->native= NULL;
+    new->native_freer= NULL;
+    return new;
+}
+
+spade_event *spade_event_clone(spade_event *e,event_native_copier_t native_copier,event_native_freer_t native_freer) {
+    spade_event *clone= new_spade_event();
+    *clone= *e; /* copy data */
+    if (native_copier != NULL)
+        clone->native= (*native_copier)(e->native);
+    clone->native_freer= native_freer;
+    return clone;
+}
+
+void free_spade_event(spade_event *e) {
+    if (e->native_freer != NULL) (*(e->native_freer))(e->native);
+    e->native= (void *)spade_event_freelist;
+    spade_event_freelist= e;
+}
+
+/*@}*/
+
+/* Id: spade_event.c,v 1.1 2004/10/11 14:35:54 jonkman Exp $ */
+
+/*********************************************************************
+event_recorder.c, distributed as part of Spade v030125.1
+Author: James Hoagland, Silicon Defense (hoagland@SiliconDefense.com)
+copyright (c) 2002 by Silicon Defense (http://www.silicondefense.com/)
+Released under GNU General Public License, see the COPYING file included
+with the distribution or http://www.silicondefense.com/spice/ for details.
+
+event_recorder.c contains the "class" event_recorder which is manages a set
+  of spade_prob_tables to accomplish user specified goals.  This trys to 
+  avoid creating more tables that are needed for the set of goals.
+
+Please send complaints, kudos, and especially improvements and bugfixes to
+hoagland@SiliconDefense.com.  As described in GNU General Public License, no
+warranty is expressed for this program.
+*********************************************************************/
+
+/*! \file event_recorder.c
+ * \brief 
+ *  event_recorder.c contains the "class" event_recorder which is
+ *  manages a set of spade_prob_tables to accomplish user specified goals.
+ *  This trys to avoid creating more tables that are needed for the set of
+ *  goals.
+ * \ingroup staterec
+ */
+
+/*! \addtogroup staterec Spade state recording
+ * \brief this group contains objects the maintain accumulated Spade
+ *  observation state
+ * \ingroup libspade
+    @{
+*/
+
+
+
+#include <stdlib.h>
+#include <string.h>
+
+static evfile *new_evfile(table_mgr *mgr,int feat_depth,feature_list *calc_feats);
+static table_mgr *new_table_mgr(feature_list *feats, const char **featurenames, event_condition_set conds, int scale_freq, double scale_factor, double prune_threshold, time_t curtime);
+static int table_mgr_recover(statefile_ref *ref, table_mgr **mgr);
+static int table_mgr_checkpoint(table_mgr *mgr, statefile_ref *ref);
+static int table_mgr_is_compatable(table_mgr *mgr, feature_list *feats, const char **featurenames, event_condition_set conds, int scale_freq, double scale_factor, double prune_threshold);
+static void table_mgr_new_time(table_mgr *mgr, time_t time);
+static void free_table_mgr(table_mgr *mgr);
+static void table_mgr_write_stats(table_mgr *mgr, FILE *file, u8 stats_to_print,condition_printer_t condprinter);
+static void table_mgr_print_config_details(table_mgr *mgr, FILE *f, char *indent);
+static void file_print_feature_list(feature_list *feats, FILE *f, const char **featurenames);
+
+#define feats_to_calc_with(evf) ( (evf->calc_feats.num > 0) ? (&(evf->calc_feats)) : (&(evf->mgr->feats)) )
+
+#define map_event_to_val_arr(featmap,size,event,val) { \
+    int featidx; \
+    for (featidx= 0; featidx < size; featidx++) { \
+        val[featidx]= event->fldval[featmap[featidx]]; \
+    } \
+}
+
+event_recorder *new_event_recorder() {
+    event_recorder *new= (event_recorder *)malloc(sizeof(event_recorder));
+    init_event_recorder(new);
+    return new;
+}
+
+void init_event_recorder(event_recorder *self) {
+    self->tables= NULL;
+    self->files= NULL;
+    self->curtime= (time_t)0;
+}
+
+int event_recorder_recover(event_recorder **self,statefile_ref *ref) {
+    *self= new_event_recorder();
+    if (*self == NULL) return 0;
+    return event_recorder_merge_recover(*self,ref);
+}
+
+int event_recorder_merge_recover(event_recorder *self,statefile_ref *ref) {
+    int i,count;
+    table_mgr *mgr;
+    if (!spade_state_recover_u32(ref,&count)) return 0;
+    for (i= 0; i < count; i++) {
+        if (!table_mgr_recover(ref,&mgr)) return 0;
+        /* add manager into list by prepending*/
+        mgr->next= self->tables;
+        self->tables= mgr;
+    }
+    return 1;
+}
+
+int event_recorder_checkpoint(event_recorder *self,statefile_ref *ref) {
+    table_mgr *mgr;
+    u32 count= 0;
+    for (mgr= self->tables; mgr != NULL; mgr=mgr->next) count++;
+    
+    spade_state_checkpoint_u32(ref,count);
+    for (mgr= self->tables; mgr != NULL; mgr=mgr->next) {
+        if (!table_mgr_checkpoint(mgr,ref)) return 0;
+    }
+    return 1;
+}
+
+evfile_ref event_recorder_new_event_file(event_recorder *self,feature_list *feats,const char **featurenames,event_condition_set conds,int scale_freq,double scale_factor,double prune_threshold,int fresh_only, feature_list *calc_feats) {
+    table_mgr *mgr=NULL;
+    evfile *eventfile;
+    
+    /**** find a compatable table manager, extending or creating if needed ****/
+    if (!fresh_only) {
+        for (mgr= self->tables; mgr != NULL; mgr=mgr->next) {
+            if (table_mgr_is_compatable(mgr,feats,featurenames,conds,scale_freq,scale_factor,prune_threshold)) break;
+        }
+        if (mgr != NULL) {
+            /* reusing a table manager, but some tweaking may be required */
+            if (mgr->feats.num < feats->num) /* need to extend features */
+                mgr->feats= *feats; /* copy whole struct */
+            if (!mgr->use_count) {
+                mgr->scale_freq= scale_freq;
+                mgr->scale_factor= scale_factor;
+                mgr->prune_threshold= prune_threshold;
+            }
+        }
+    }
+    
+    if (mgr == NULL) {
+        /* NOTE: we could go through tables again looking for compatable shared leading features to save on a table and save double-recording of leading features, but that seeking code is a little hairy and it would require recording "skip" information when getting an event */
+        mgr= new_table_mgr(feats,featurenames,conds,scale_freq,scale_factor,prune_threshold,self->curtime);
+        if (mgr == NULL) return NULL;
+        /* add manager into list by prepending*/
+        mgr->next= self->tables;
+        self->tables= mgr;
+    }
+    mgr->use_count++;
+        
+    /**** find a compatable evfile, creating if needed ****/
+    for (eventfile= self->files; eventfile != NULL; eventfile=eventfile->next) {
+        if (eventfile->mgr != mgr) continue;
+        if (eventfile->feat_depth != feats->num)
+            continue;
+        if ((calc_feats == NULL) && eventfile->calc_feats.num > 0) continue;
+        if (calc_feats != NULL) {
+            int i;
+            if (eventfile->calc_feats.num == 0) /* calc_feats is off */
+                continue;
+            if (eventfile->feat_depth == calc_feats->num)
+                continue;
+            for (i= 0; i < eventfile->feat_depth; i++)
+                if (eventfile->calc_feats.feat[i] != calc_feats->feat[i]) break;
+            if (i < eventfile->feat_depth) continue;
+        }
+        break; // all matched
+    }
+    
+    if (eventfile == NULL) {
+        eventfile= new_evfile(mgr,feats->num,calc_feats);
+        if (eventfile == NULL) return NULL;
+        /* add eventfile into list by prepending*/
+        eventfile->next= self->files;
+        self->files= eventfile;
+    }
+    
+    return (evfile_ref)eventfile;
+}
+
+evfile_ref *event_recorder_new_event_files(event_recorder *self,int howmany,feature_list feats[],const char **featurenames,event_condition_set conds,int scale_freq,double scale_factor,double prune_threshold,int fresh_only) {
+    int i;
+    evfile_ref *arr= (evfile_ref *)malloc(sizeof(evfile_ref)*howmany);
+    if (arr == NULL) return NULL;
+
+    for (i= 0; i < howmany; i++)
+        arr[i]= event_recorder_new_event_file(self,&feats[i],featurenames,conds,scale_freq,scale_factor,prune_threshold,fresh_only,NULL);
+
+    return arr;
+}
+
+void event_recorder_new_time(event_recorder *self, time_t time) {
+    table_mgr *mgr;
+    self->curtime= time;
+    /* check for scaling */
+    for (mgr= self->tables; mgr != NULL; mgr=mgr->next) {
+        if (mgr->use_count) table_mgr_new_time(mgr,time);
+    }
+}
+
+event_condition_set event_recorder_needed_conds(event_recorder *self) {
+    table_mgr *mgr;
+    event_condition_set needed_conds= 0;
+    /* record which conditions we require events for */
+    for (mgr= self->tables; mgr != NULL; mgr=mgr->next)
+        ADD_TO_CONDS(needed_conds,mgr->conds); /* NOTE: some information loss here, so might get passed unneeded events; can represent more precisely as a or-ed list of masks */
+    return needed_conds;
+}
+
+int event_recorder_new_event(event_recorder *self, spade_event *event, event_condition_set matching_conds) {
+    table_mgr *mgr;
+    int updates= 0;
+    for (mgr= self->tables; mgr != NULL; mgr=mgr->next) {
+        if (ALL_CONDS_MET(matching_conds,mgr->conds)) { /* all of mgr's conditions are met by event */
+            u32 val[MAX_NUM_FEATURES];
+            feature_list *l= &mgr->feats;
+            map_event_to_val_arr(l->feat,l->num,event,val);
+            increment_Njoint_count(&mgr->table,l->num,l->feat,val,0);
+            mgr->store_count++;
+            updates++;
+        }
+    }
+    return updates;
+}
+
+void event_recorder_prune_unused(event_recorder *self) {
+    table_mgr *mgr,*prev=NULL;
+    for (mgr= self->tables; mgr != NULL; mgr=mgr->next) {
+        if (!mgr->use_count) {
+            if (prev == NULL)
+                self->tables= mgr->next;
+            else 
+                prev->next= mgr->next;
+            free_table_mgr(mgr);
+        } else {
+            prev= mgr;
+        }
+    }
+}
+
+double event_recorder_get_prob(event_recorder *self,evfile_ref eventfile,spade_event *event,int one_more) {
+    u32 val[MAX_NUM_FEATURES];
+    feature_list *l=  &eventfile->mgr->feats;
+    /* calculate the joint probability to the depth indicated in the evfile */
+    map_event_to_val_arr(feats_to_calc_with(eventfile)->feat,eventfile->feat_depth,event,val);
+    return one_more ?
+        prob_Njoint_Ncond_plus_one(&eventfile->mgr->table,eventfile->feat_depth,l->feat,val,0) :
+        prob_Njoint_Ncond(&eventfile->mgr->table,eventfile->feat_depth,l->feat,val,0);
+}
+
+double event_recorder_get_condprob(event_recorder *self,evfile_ref eventfile,spade_event *event,int condcutoff,int one_more) {
+    u32 val[MAX_NUM_FEATURES];
+    feature_list *l= &eventfile->mgr->feats;
+    if (condcutoff < 0) condcutoff+= eventfile->feat_depth; /* condition cutoff specified from end */
+    /* calculate the joint probability to the depth indicated in the evfile and conditioned to the indicated level */
+    map_event_to_val_arr(feats_to_calc_with(eventfile)->feat,eventfile->feat_depth,event,val);
+    return one_more ?
+        prob_Njoint_Ncond_plus_one(&eventfile->mgr->table,eventfile->feat_depth,l->feat,val,condcutoff) :
+        prob_Njoint_Ncond(&eventfile->mgr->table,eventfile->feat_depth,l->feat,val,condcutoff);
+}
+
+double event_recorder_get_count(event_recorder *self,evfile_ref eventfile,spade_event *event,int featdepth) {
+    u32 val[MAX_NUM_FEATURES];
+    feature_list *l=  &eventfile->mgr->feats;
+    /* calculate the joint probability to the depth indicated in the evfile and conditioned to the indicated level */
+    map_event_to_val_arr(feats_to_calc_with(eventfile)->feat,featdepth,event,val);
+    return jointN_count(&eventfile->mgr->table,featdepth,l->feat,val);
+}
+
+double event_recorder_get_entropy(event_recorder *self,evfile_ref eventfile,spade_event *event,int entropy_prefix_len) {
+    u32 val[MAX_NUM_FEATURES];
+    feature_list *l=  &eventfile->mgr->feats;
+    map_event_to_val_arr(feats_to_calc_with(eventfile)->feat,entropy_prefix_len,event,val);
+    return spade_prob_table_entropy(&eventfile->mgr->table,entropy_prefix_len,l->feat,val);
+}
+
+int event_recorder_get_store_count(event_recorder *self, evfile_ref eventfile) {
+    return eventfile->mgr->store_count;
+}
+
+double event_recorder_get_obs_count(event_recorder *self, evfile_ref eventfile) {
+    return jointN_count(&eventfile->mgr->table,0,eventfile->mgr->feats.feat,NULL);
+}
+
+void event_recorder_write_stats(event_recorder *self,FILE *file,u8 stats_to_print,condition_printer_t condprinter) {
+    table_mgr *mgr;
+    for (mgr= self->tables; mgr != NULL; mgr=mgr->next) {
+        table_mgr_write_stats(mgr,file,stats_to_print,condprinter);
+    }
+}
+
+static evfile *new_evfile(table_mgr *mgr,int feat_depth,feature_list *calc_feats) {
+    evfile *new= (evfile *)malloc(sizeof(evfile));
+    if (new == NULL) return NULL;
+    new->mgr= mgr;
+    new->feat_depth= feat_depth;
+    if (calc_feats == NULL)
+        new->calc_feats.num= 0;
+    else
+        new->calc_feats= *calc_feats; /* copy struct over */
+    new->next= NULL;
+    return new;
+}
+
+static table_mgr *new_table_mgr(feature_list *feats,const char **featurenames,event_condition_set conds,int scale_freq,double scale_factor,double prune_threshold,time_t curtime) {
+    int num_featurenames;
+    table_mgr *new= (table_mgr *)malloc(sizeof(table_mgr));
+    if (new == NULL) return NULL;
+    
+    init_spade_prob_table(&new->table,featurenames,0);
+    new->next= NULL;
+    new->last_scale= (time_t)0;
+    
+    new->feats= *feats; // make copy
+    for (num_featurenames= 0; featurenames[num_featurenames] != NULL; num_featurenames++);
+    new->featurenames= (const char **)malloc(sizeof(const char *)*(num_featurenames+1));
+    if (new->featurenames != NULL) {
+        int i;
+        for (i= 0; featurenames[i] != NULL; i++) {
+            new->featurenames[i]= strdup(featurenames[i]);
+        }
+        new->featurenames[i]= NULL;
+    }
+    
+    new->conds= conds;
+    new->start_time= curtime;
+    new->scale_freq= scale_freq;
+    new->scale_factor= scale_factor;
+    new->prune_threshold= prune_threshold;
+    new->use_count= 0;
+    new->store_count= 0;
+    return new;
+}
+
+static int table_mgr_recover(statefile_ref *ref,table_mgr **mgr) {
+    u8 count= 0;
+
+    feature_list feats;
+    const char **featurenames;
+    event_condition_set conds;
+    int scale_freq;
+    double scale_factor;
+    double prune_threshold;
+    time_t start_time;
+    
+    time_t last_scale;
+
+    if (!(spade_state_recover_u32(ref,(u32 *)&conds)
+        && spade_state_recover_u8(ref,(u8 *)&feats.num)
+        && spade_state_recover_arr(ref,(u8 **)&feats.feat,feats.num,1)
+        && spade_state_recover_u8(ref,(u8 *)&count)
+    )) return 0;
+    
+    featurenames= (const char **)malloc(sizeof(const char *)*(count+1));
+    if (featurenames == NULL) return 0;
+    if (!spade_state_recover_str_arr(ref,(char **)featurenames,count)) return 0;
+    featurenames[count]= NULL;
+    
+    if (!(spade_state_recover_u32(ref,(u32 *)&scale_freq)
+        && spade_state_recover_double(ref,&scale_factor)
+        && spade_state_recover_double(ref,&prune_threshold)
+        && spade_state_recover_time_t(ref,&start_time)
+    )) return 0;
+
+    *mgr= new_table_mgr(&feats,featurenames,conds,scale_freq,scale_factor,prune_threshold,start_time);
+
+    if (!spade_state_recover_time_t(ref,&last_scale)) return 0;
+    /* we choose not to record the recovered last_scale; it would cause repeated immediate scaling to make up for lost time; not want we want most of the time */
+
+    return spade_prob_table_recover(ref,&(*mgr)->table); /* does not effect featurenames of the table */
+}
+
+static int table_mgr_checkpoint(table_mgr *mgr,statefile_ref *ref) {
+    u8 count= 0;
+    for (count= 0; mgr->featurenames[count] != NULL; count++);
+    
+    return spade_state_checkpoint_u32(ref,(u32)mgr->conds)
+        && spade_state_checkpoint_u8(ref,(u8)mgr->feats.num)
+        && spade_state_checkpoint_arr(ref,(u8 *)mgr->feats.feat,mgr->feats.num,1)
+        && spade_state_checkpoint_u8(ref,(u8)count)
+        && spade_state_checkpoint_str_arr(ref,(char **)mgr->featurenames,count)
+    
+        && spade_state_checkpoint_u32(ref,(u32)mgr->scale_freq)
+        && spade_state_checkpoint_double(ref,mgr->scale_factor)
+        && spade_state_checkpoint_double(ref,mgr->prune_threshold)
+        && spade_state_checkpoint_time_t(ref,mgr->start_time)
+        && spade_state_checkpoint_time_t(ref,mgr->last_scale)    
+
+        && spade_prob_table_checkpoint(ref,&mgr->table);
+}
+
+static int table_mgr_is_compatable(table_mgr *mgr,feature_list *feats,const char **featurenames,event_condition_set conds,int scale_freq,double scale_factor,double prune_threshold) {
+    int i,cmp_featlen;
+
+    if (mgr->conds != conds) return 0;
+    
+    if (mgr->use_count) { /* we'll waive these checks if an orphan */
+        //if (mgr->start_time != self->curtime) return 0;
+        if (mgr->scale_freq != scale_freq) return 0;
+        if (mgr->scale_factor != scale_factor) return 0;
+        if (mgr->prune_threshold != prune_threshold) return 0;
+    }
+    
+    for (i= 0; featurenames[i] != NULL; i++)
+        if (strcmp(mgr->featurenames[i],featurenames[i])) return 0;
+
+    if (mgr->feats.num < feats->num) {
+        if (!spade_prob_table_is_empty(&mgr->table)) return 0; /* can only extend if empty */
+        cmp_featlen= mgr->feats.num;
+    } else {
+        cmp_featlen= feats->num;
+    }
+    for (i= 0; i < cmp_featlen; i++)
+        if (mgr->feats.feat[i] != feats->feat[i]) return 0;
+
+    return 1; /* found a compatable table manager */
+}
+
+static void table_mgr_new_time(table_mgr *mgr,time_t time) {
+    if (mgr->scale_freq > 0) {
+        while (time - mgr->last_scale > mgr->scale_freq) {
+            if (mgr->last_scale == (time_t)0) { /* never have scaled before */
+                mgr->last_scale= time;
+            } else {
+                //if (self->debug_level > 1) printf("scaling by %f at time %d; discarding at %f\n",mgr->scale_factor,(int)time,mgr->prune_threshold);
+                scale_and_prune_table(&mgr->table,mgr->scale_factor,mgr->prune_threshold);
+                mgr->last_scale+= mgr->scale_freq;  /* lets pretend we did this right on time */
+                //if (self->debug_level > 1) printf("done with scale/prune\n");
+            }
+        }
+    }
+}
+
+static void free_table_mgr(table_mgr *mgr) {
+    int i;
+    /* need to reset mgr->table */
+    for (i= 0; mgr->featurenames[i] != NULL; i++) {
+        free((char *)mgr->featurenames[i]);
+    }
+    free((char *)mgr->featurenames);
+    free(mgr);
+}
+
+static void table_mgr_write_stats(table_mgr *mgr,FILE *file,u8 stats_to_print,condition_printer_t condprinter) {
+    fprintf(file,"** table for ");
+    if (condprinter != NULL)
+        (*condprinter)(file,mgr->conds);
+    else
+        fprintf(file," %x",mgr->conds);
+    fprintf(file," **\n");
+    fprintf(file,"Recorded is: P(");
+    file_print_feature_list(&mgr->feats,file,mgr->featurenames);
+    fprintf(file,")\n");
+    fprintf(file,"Scaling freqency: %d; Scaling factor: %.5f; Pruning Threshold=%.5f\n",mgr->scale_freq,mgr->scale_factor,mgr->prune_threshold);
+    fprintf(file,"Start time: %d; Last time scaled: %d\n",(int)mgr->start_time,(int)mgr->last_scale);
+    spade_prob_table_write_stats(&mgr->table,file,stats_to_print);
+    fprintf(file,"\n");
+}
+
+void evfile_print_config_details(evfile_ref eventfile,FILE *f,char *indent) {
+    char indent2[100];
+    sprintf(indent2,"%s  ",indent);
+    fprintf(f,"%smgr=\n",indent);
+    table_mgr_print_config_details(eventfile->mgr,f,indent2);
+    fprintf(f,"%sfeat_depth=%d\n",indent,eventfile->feat_depth);
+    if (eventfile->calc_feats.num > 0) {
+        fprintf(f,"%scalc_feats=",indent);
+        file_print_feature_list(&eventfile->calc_feats,f,eventfile->mgr->featurenames);
+        fprintf(f,"\n");
+    }
+}
+
+static void table_mgr_print_config_details(table_mgr *mgr,FILE *f,char *indent) {
+    fprintf(f,"%sfeats=",indent);
+    file_print_feature_list(&mgr->feats,f,mgr->featurenames);
+    fprintf(f,"\n%sconds=%x\n",indent,mgr->conds);
+    fprintf(f,"%sscale_freq=%d; scale_factor=%.5f; prune_threshold=%.5f\n",indent,mgr->scale_freq,mgr->scale_factor,mgr->prune_threshold);
+}
+
+static void file_print_feature_list(feature_list* feats,FILE *f,const char **featurenames) {
+    int i;
+    for (i= 0; i < feats->num; i++) {
+        if (i != 0) fprintf(f,",");
+        fprintf(f,"%s",featurenames[feats->feat[i]]);
+    }
+}
+/* Id: event_recorder.c,v 1.1 2004/10/11 14:35:54 jonkman Exp $ */
+/*********************************************************************
+score_info.c, distributed as part of Spade v030125.1
+Author: James Hoagland, Silicon Defense (hoagland@SiliconDefense.com)
+copyright (c) 2002 by Silicon Defense (http://www.silicondefense.com/)
+Released under GNU General Public License, see the COPYING file included
+with the distribution or http://www.silicondefense.com/spice/ for details.
+
+Please send complaints, kudos, and especially improvements and bugfixes to
+hoagland@SiliconDefense.com.  As described in GNU General Public License, no
+warranty is expressed for this program.
+*********************************************************************/
+
+#include <stdlib.h>
+
+/*! \file score_info.c
+ * \brief 
+ *  score_info.c contains routines for score_info allocation, initing,
+ *  and recycling and access; score_info reprents computed anomaly
+ *  score(s)
+ * \ingroup scoreprod
+ */
+
+/*! \addtogroup scoreprod
+    @{
+*/
+
+
+/// free list of allocated score_infos
+score_info *score_info_freelist=NULL;
+
+/* creation and recycling routines for score_info's */
+
+score_info *new_score_info(scorepref main,double relscore,double rawscore,int corrscore_used) {
+    score_info *new;
+    if (score_info_freelist != NULL) {
+        new= score_info_freelist;
+        score_info_freelist= new->next;
+    } else {
+        new= (score_info *)malloc(sizeof(score_info));
+    }
+    init_score_info(new,main,relscore,rawscore,corrscore_used);
+    return new;
+}
+
+void init_score_info(score_info *i,scorepref main,double relscore,double rawscore,int corrscore_used) {
+    i->main= main;
+    i->relscore= relscore;
+    i->rawscore= rawscore;
+    i->corrscore_used= corrscore_used;
+}
+
+score_info *score_info_clone(score_info *i) {
+    return new_score_info(i->main,i->relscore,i->rawscore,i->corrscore_used);
+}
+
+void free_score_info(score_info *i) {
+    i->next= score_info_freelist;
+    score_info_freelist= i;
+}
+
+void free_score_infos(score_info *start) {
+    score_info *end,*next;
+    for (end= start, next=start->next; next != NULL; end=next,next=next->next);
+    end->next= score_info_freelist;
+    score_info_freelist= start;
+}
+
+double score_info_mainscore(score_info *i) {
+    return i->main == PREF_RAWSCORE ? i->rawscore : i->relscore;
+}
+
+double score_info_relscore(score_info *i) {
+    return i->relscore;
+}
+
+double score_info_rawscore(score_info *i) {
+    return i->rawscore;
+}
+
+int score_info_raw_is_corrscore(score_info *i) {
+    return i->corrscore_used;
+}
+
+scorepref score_info_main_pref(score_info *i) {
+    return i->main;
+}
+
+
+const char *scorepref_str(scorepref pref) {
+    switch (pref) {
+    case PREF_NOSCORE: return "NOSCORE";
+    case PREF_RAWSCORE: return "RAWSCORE";
+    case PREF_RELSCORE: return "RELSCORE";
+    default: return "undefined";
+    }
+}
+
+/*@}*/
+/* Id: score_info.c,v 1.1 2004/10/11 14:35:54 jonkman Exp $ */
+
+/*********************************************************************
+spade_enviro.c, distributed as part of Spade v030125.1
+Author: James Hoagland, Silicon Defense (hoagland@SiliconDefense.com)
+copyright (c) 2002 by Silicon Defense (http://www.silicondefense.com/)
+Released under GNU General Public License, see the COPYING file included
+with the distribution or http://www.silicondefense.com/spice/ for details.
+
+Please send complaints, kudos, and especially improvements and bugfixes to
+hoagland@SiliconDefense.com.  As described in GNU General Public License, no
+warranty is expressed for this program.
+*********************************************************************/
+
+#include <stdlib.h>
+
+/*! \file spade_enviro.c
+ * \brief 
+ *  spade_enviro.c contains routines for spade_enviro allocation,
+ *  and initing
+ * \ingroup stmgr
+ */
+
+/*! \weakgroup stmgr
+    @{
+*/
+
+spade_enviro *new_spade_enviro(double thresh,unsigned long *total_pkts_ptr) {
+    spade_enviro *new= (spade_enviro *)malloc(sizeof(spade_enviro));
+    if (new == NULL) return NULL;
+    init_spade_enviro(new,thresh,total_pkts_ptr);
+    return new;
+}
+
+void init_spade_enviro(spade_enviro *self,double thresh,unsigned long *total_pkts_ptr) {
+    self->now= (time_t)0;
+    self->thresh= thresh;
+    self->total_pkts= total_pkts_ptr;
+    self->pkt_stats.scored= 0;
+    self->pkt_stats.respchecked= 0;
+    self->pkt_stats.excluded= 0;
+    self->pkt_stats.nonexcluded= 0;
+    self->pkt_stats.reported= 0;
+    self->pkt_stats.waited= 0;
+    self->pkt_stats.insuffobsed= 0;
+}
+
+/*@}*/
+/* Id: spade_enviro.c,v 1.1 2004/10/11 14:35:54 jonkman Exp $ */
+/*********************************************************************
+spade_output.c, distributed as part of Spade v030125.1
+Author: James Hoagland, Silicon Defense (hoagland@SiliconDefense.com)
+copyright (c) 2002 by Silicon Defense (http://www.silicondefense.com/)
+Released under GNU General Public License, see the COPYING file included
+with the distribution or http://www.silicondefense.com/spice/ for details.
+
+Please send complaints, kudos, and especially improvements and bugfixes to
+hoagland@SiliconDefense.com.  As described in GNU General Public License, no
+warranty is expressed for this program.
+*********************************************************************/
+
+#include <stdarg.h>
+#include <stdio.h>
+
+/*! \file spade_output.c
+ * \brief 
+ *  spade_output.c contains the definition of default_spade_msg_fn
+ * \ingroup libspade_misc
+ */
+
+/*! \addtogroup libspade_misc
+    @{
+*/
+
+void default_spade_msg_fn(spade_message_type msg_type,const char *str) {
+    switch (msg_type) {
+    case SPADE_MSG_TYPE_FATAL:
+        fprintf(stderr, "%s", str);
+        exit(1);
+    case SPADE_MSG_TYPE_WARNING:
+        fprintf(stderr, "%s", str);
+        break;
+    default:
+        printf("%s", str);
+        break;
+    }
+}
+
+void formatted_spade_msg_send(spade_message_type msg_type,spade_msg_fn msg_fn,const char *format,...) {
+    char buf[MAX_SPADE_MSG_LEN+1];
+    va_list ap;
+    va_start(ap, format);
+    
+    vsnprintf(buf, MAX_SPADE_MSG_LEN, format, ap);
+    (*msg_fn)(msg_type,buf);
+    va_end(ap);
+}
+
+/*@}*/
+
+/* Id: spade_output.c,v 1.1 2004/10/11 14:35:54 jonkman Exp $ */
diff -uNr snort-2.3.3/src/preprocessors/spp_spade.h snort-2.3.3-spade/src/preprocessors/spp_spade.h
--- snort-2.3.3/src/preprocessors/spp_spade.h	1970-01-01 01:00:00.000000000 +0100
+++ snort-2.3.3-spade/src/preprocessors/spp_spade.h	2006-03-31 14:16:22.000000000 +0200
@@ -0,0 +1,2055 @@
+/*********************************************************************
+snort_spade.h, distributed as part of Spade v030125.1
+Author: James Hoagland, Silicon Defense (hoagland@SiliconDefense.com)
+copyright (c) 2002 by Silicon Defense (http://www.silicondefense.com/)
+Released under GNU General Public License, see the COPYING file included
+with the distribution or http://www.silicondefense.com/spice/ for details.
+
+Please send complaints, kudos, and especially improvements and bugfixes to
+hoagland@SiliconDefense.com.  As described in GNU General Public License, no
+warranty is expressed for this program.
+*********************************************************************/
+
+/* Internal version control: Id: snort_spade.h,v 1.1 2004/10/11 14:35:54 jonkman Exp $ */
+
+#ifndef __SNORT_SPADE_H__
+#define __SNORT_SPADE_H__
+
+/*! \file snort_spade.h
+ * \brief 
+ *  snort_spade.h is the header file for snort_spade.c
+ */
+
+/*! \addtogroup snort_spade
+ * @{
+*/
+
+void SetupSpade();
+void SpadeInit(u_char *argsstr);
+void PreprocSpade(Packet *p);
+void SpadeHomenetInit(u_char *args);
+void SpadeDetectInit(u_char *args);
+void SpadeStatInit(u_char *args);
+void SpadeThreshadviseInit(u_char *args);
+void SpadeAdaptInit(u_char *args);
+void SpadeAdapt2Init(u_char *args);
+void SpadeAdapt3Init(u_char *args);
+void SpadeSurveyInit(u_char *args);
+void SpadeCatchSig(int signal, void *arg);
+
+#endif // __SNORT_SPADE_H__
+/*********************************************************************
+netspade.h, distributed as part of Spade v030125.1
+Author: James Hoagland, Silicon Defense (hoagland@SiliconDefense.com)
+copyright (c) 2002 by Silicon Defense (http://www.silicondefense.com/)
+Released under GNU General Public License, see the COPYING file included
+with the distribution or http://www.silicondefense.com/spice/ for details.
+
+Please send complaints, kudos, and especially improvements and bugfixes to
+hoagland@SiliconDefense.com.  As described in GNU General Public License, no
+warranty is expressed for this program.
+*********************************************************************/
+
+/* Internal version control: Id: netspade.h,v 1.1 2004/10/11 14:35:54 jonkman Exp $ */
+
+/*! \file netspade.h
+ * \brief 
+ *  netspade.h is the header file for the netspade "class"
+ * \ingroup netspade_layer
+ */
+
+/*! \addtogroup netspade_layer
+    @{
+*/
+
+#ifndef NETSPADE_H
+#define NETSPADE_H
+
+/*********************************************************************
+netspade_features.h, distributed as part of Spade v030125.1
+Author: James Hoagland, Silicon Defense (hoagland@SiliconDefense.com)
+copyright (c) 2002 by Silicon Defense (http://www.silicondefense.com/)
+Released under GNU General Public License, see the COPYING file included
+with the distribution or http://www.silicondefense.com/spice/ for details.
+
+netspade_features.h is contains the features definitions for netspade
+
+Please send complaints, kudos, and especially improvements and bugfixes to
+hoagland@SiliconDefense.com.  As described in GNU General Public License, no
+warranty is expressed for this program.
+*********************************************************************/
+
+/* Internal version control: Id: netspade_features.h,v 1.1 2004/10/11 14:35:54 jonkman Exp $ */
+
+#ifndef NETSPADE_FEATURES_H
+#define NETSPADE_FEATURES_H
+
+/*********************************************************************
+spade_features.h, distributed as part of Spade v030125.1
+Author: James Hoagland, Silicon Defense (hoagland@SiliconDefense.com)
+copyright (c) 2002 by Silicon Defense (http://www.silicondefense.com/)
+Released under GNU General Public License, see the COPYING file included
+with the distribution or http://www.silicondefense.com/spice/ for details.
+
+Please send complaints, kudos, and especially improvements and bugfixes to
+hoagland@SiliconDefense.com.  As described in GNU General Public License, no
+warranty is expressed for this program.
+*********************************************************************/
+
+/* Internal version control: Id: spade_features.h,v 1.1 2004/10/11 14:35:54 jonkman Exp $ */
+
+#ifndef SPADE_FEATURES_H
+#define SPADE_FEATURES_H
+
+/*! \file spade_features.h
+ * \brief 
+ *  spade_features.h is a set of type declarations around a spade feature
+ * \ingroup staterec
+ */
+
+/*! \addtogroup staterec
+    @{
+*/
+
+#include <limits.h>
+
+#define MAX_U32 0xFFFFFFFF ///< the largest number that can be represented in a u32
+#define MAX_U16 0xFFFF     ///< the largest number that can be represented in a u16
+#define MAX_U8  0xFF       ///< the largest number that can be represented in a u8
+
+/* search to find a type to be u32 */
+// u32 is a unsigned (exactly) 32 bit integer
+#if USHRT_MAX == MAX_U32
+typedef unsigned short u32;
+#elif UINT_MAX == MAX_U32
+typedef unsigned int u32;
+#elif ULONG_MAX == MAX_U32
+typedef unsigned long u32;
+#else
+#error could not find a 4 byte int to be u32 in types.h
+#endif
+
+/* search to find a type to be u16 */
+// u16 is a unsigned (exactly) 16 bit integer
+#if USHRT_MAX == MAX_U16
+typedef unsigned short u16;
+#elif UINT_MAX == MAX_U16
+typedef unsigned int u16;
+#elif ULONG_MAX == MAX_U16
+typedef unsigned long u16;
+#else
+#error could not find a 2 byte int to be u16 in types.h
+#endif
+
+/* search to find a type to be u8 */
+/* u8 is a unsigned (exactly) 8 bit integer */
+#if UCHAR_MAX == MAX_U8
+typedef unsigned char u8;
+#elif USHRT_MAX == MAX_U8
+typedef unsigned short u8;
+#elif UINT_MAX == MAX_U8
+typedef unsigned int u8;
+#else
+#error could not find a 1 byte int to be u8 in types.h
+#endif
+
+
+/// the maximum number of features libspade can handle
+#define MAX_NUM_FEATURES 8
+
+typedef u8 features; ///< type of the index represting the features we are storing the prob table about
+
+/// this represents a sorted list of up to MAX_NUM_FEATURES features
+typedef struct {
+    u8 num;  ///< how many featrues are in the list
+    features feat[MAX_NUM_FEATURES]; ///< 0-based array storing the features
+} feature_list;
+
+#endif // SPADE_FEATURES_H
+
+
+/*! \file netspade_features.h
+ * \ingroup netspade_layer
+ * \brief 
+ *  netspade_features.h is contains the features definitions for netspade
+ */
+
+/*! \addtogroup netspade_layer
+    @{
+*/
+
+
+#define SIP             0 ///< the source IP netspade feature
+#define DIP             1 ///< the dest IP netspade feature
+#define SPORT           2 ///< the source port netspade feature
+#define DPORT           3 ///< the dest port netspade feature
+#define IPPROTO         4 ///< the IP protocol netspade feature
+#define TCPFLAGS        5 ///< the TCP flags netspade feature
+#define ICMPTYPE        6 ///< the ICMP type netspade feature
+#define ICMPTYPECODE    7 ///< the ICMP type&code netspade feature
+
+/// \brief the number of netspade features defined
+/// \note must be no more than MAX_NUM_FEATURES in spade_features.h
+//#define NETSPADE_NUM_FEATURES 6
+#define NETSPADE_NUM_FEATURES 8
+extern const char *featurenames[NETSPADE_NUM_FEATURES+1];
+
+/// used for the IPPROTO file when we don't know the IP protocol
+#define IPPROTO_UNKNOWN (u32)-1;
+
+/// the value for the "origin" field in spade_event when the header was at the top level of the packet
+#define PKTORIG_TOP     1
+/// the value for the "origin" field in spade_event when the header was enclosed in an unreachable packet
+#define PKTORIG_UNRCH   2
+
+/*@}*/
+
+#endif // NETSPADE_FEATURES_H
+
+/*********************************************************************
+spade_detection_types.h, distributed as part of Spade v030125.1
+Author: James Hoagland, Silicon Defense (hoagland@SiliconDefense.com)
+copyright (c) 2002 by Silicon Defense (http://www.silicondefense.com/)
+Released under GNU General Public License, see the COPYING file included
+with the distribution or http://www.silicondefense.com/spice/ for details.
+
+Please send complaints, kudos, and especially improvements and bugfixes to
+hoagland@SiliconDefense.com.  As described in GNU General Public License, no
+warranty is expressed for this program.
+*********************************************************************/
+
+#ifndef SPADE_DETECTION_TYPES_H
+#define SPADE_DETECTION_TYPES_H
+
+
+/*! \file spade_detection_types.h
+ * \ingroup netspade_layer
+ * \brief 
+ *  spade_detection_types.h contains constants, strings, and conversion macros
+ *  for spade detection types and detector types
+ */
+
+/*! \addtogroup netspade_layer
+    @{
+*/
+
+
+/* detector types (each spade detector is of a certain type) */
+#define SPADE_DR_TYPE_UNKNOWN       0
+#define SPADE_DR_TYPE_CLOSED_DPORT  1
+#define SPADE_DR_TYPE_DEAD_DEST     2
+#define SPADE_DR_TYPE_ODD_DPORT     3
+#define SPADE_DR_TYPE_ODD_TYPECODE  4
+#define SPADE_DR_TYPE_ODD_PORTDEST  5
+
+/// map a detector type plus an relative detector number to the corresponding detector number
+#define SPADE_DN_TYPE_FOR_DR_TYPE(num,index) ((num << 8) + index)
+
+/// map detector type number to a short strings representing it
+#define SPADE_DR_TYPE_SHORT4NUM(num) \
+    ((num == SPADE_DR_TYPE_CLOSED_DPORT) ? "closed-dport" \
+    :(num == SPADE_DR_TYPE_DEAD_DEST) ? "dead-dest" \
+    :(num == SPADE_DR_TYPE_ODD_DPORT) ? "odd-dport" \
+    :(num == SPADE_DR_TYPE_ODD_TYPECODE) ? "odd-typecode" \
+    :(num == SPADE_DR_TYPE_ODD_PORTDEST) ? "odd-port-dest" \
+    :"?" \
+    )
+
+/// map short strings representing a detector type to its number
+#define SPADE_DR_TYPE_NUM4SHORT(str) \
+    ((!strcmp(str,"closed-dport")) ? SPADE_DR_TYPE_CLOSED_DPORT \
+    :(!strcmp(str,"dead-dest")) ? SPADE_DR_TYPE_DEAD_DEST \
+    :(!strcmp(str,"odd-dport")) ? SPADE_DR_TYPE_ODD_DPORT \
+    :(!strcmp(str,"odd-typecode")) ? SPADE_DR_TYPE_ODD_TYPECODE \
+    :(!strcmp(str,"odd-port-dest")) ? SPADE_DR_TYPE_ODD_PORTDEST \
+    :SPADE_DR_TYPE_UNKNOWN \
+    )
+
+
+/* detection types (a detector type can have multiple detection types) */
+#define SPADE_DN_TYPE_UNKNOWN       0
+#define SPADE_DN_TYPE_CLOSED_DPORT  ((SPADE_DR_TYPE_CLOSED_DPORT << 8) +0)
+#define SPADE_DN_TYPE_ODD_OPEN_DPORT  ((SPADE_DR_TYPE_CLOSED_DPORT << 8) +1)
+#define SPADE_DN_TYPE_ODD_DPORT  ((SPADE_DR_TYPE_CLOSED_DPORT << 8) +2)
+#define SPADE_DN_TYPE_NONLIVE_DEST  ((SPADE_DR_TYPE_DEAD_DEST << 8) +0)
+#define SPADE_DN_TYPE_SRC_ODD_DPORT ((SPADE_DR_TYPE_ODD_DPORT << 8) +0)
+#define SPADE_DN_TYPE_ODD_TYPECODE  ((SPADE_DR_TYPE_ODD_TYPECODE << 8) +0)
+#define SPADE_DN_TYPE_ODD_PORTDEST_LOWH  ((SPADE_DR_TYPE_ODD_PORTDEST << 8) +0)
+//#define SPADE_DN_TYPE_ODD_PORTDEST_HIGHH  ((SPADE_DR_TYPE_ODD_PORTDEST << 8) +1)
+
+/// map a detector type number to the detection type number
+#define SPADE_DR_TYPE_FOR_DN_TYPE(num) (num >> 8)
+
+/// obtain the default detection type number for a detector type number
+#define DEFAULT_DN_TYPE_FOR_DR_TYPE(num) SPADE_DN_TYPE_FOR_DR_TYPE(num,0)
+
+/// map detction type to very brief strings denoting the detection type
+#define SPADE_DN_TYPE_BRIEF4NUM(num) \
+    ((num == SPADE_DN_TYPE_CLOSED_DPORT) ? "CD" \
+    :(num == SPADE_DN_TYPE_ODD_OPEN_DPORT) ? "ROD" \
+    :(num == SPADE_DN_TYPE_ODD_DPORT) ? "RD" \
+    :(num == SPADE_DN_TYPE_NONLIVE_DEST) ? "DD" \
+    :(num == SPADE_DN_TYPE_SRC_ODD_DPORT) ? "OD" \
+    :(num == SPADE_DN_TYPE_ODD_TYPECODE) ? "OT" \
+    :(num == SPADE_DN_TYPE_ODD_PORTDEST_LOWH) ? "PD" \
+    :"?" \
+    )
+
+/// map detction type very brief strings to the detection type number
+#define SPADE_DN_TYPE_NUM4BRIEF(str) \
+    ((!strcmp(str,"CD")) ? SPADE_DN_TYPE_CLOSED_DPORT \
+    ((!strcmp(str,"ROD")) ? SPADE_DN_TYPE_ODD_OPEN_DPORT \
+    ((!strcmp(str,"RD")) ? SPADE_DN_TYPE_ODD_DPORT \
+    :(!strcmp(str,"DD")) ? SPADE_DN_TYPE_NONLIVE_DEST \
+    :(!strcmp(str,"OD")) ? SPADE_DN_TYPE_SRC_ODD_DPORT \
+    :(!strcmp(str,"OT")) ? SPADE_DN_TYPE_ODD_TYPECODE \
+    :(!strcmp(str,"PD")) ? SPADE_DN_TYPE_ODD_PORTDEST_LOWH \
+    :SPADE_DN_TYPE_UNKNOWN \
+    )
+
+/// map detction type to medium-length strings describing the detection done
+#define SPADE_DN_TYPE_MEDDESCR4NUM(num) \
+    ((num == SPADE_DN_TYPE_CLOSED_DPORT) ? "Closed dest port used" \
+    :(num == SPADE_DN_TYPE_ODD_OPEN_DPORT) ? "Rare but open dest port used" \
+    :(num == SPADE_DN_TYPE_ODD_DPORT) ? "Rare dest port used" \
+    :(num == SPADE_DN_TYPE_NONLIVE_DEST) ? "Non-live dest used" \
+    :(num == SPADE_DN_TYPE_SRC_ODD_DPORT) ? "Source used odd dest port" \
+    :(num == SPADE_DN_TYPE_ODD_TYPECODE) ? "Odd ICMP type/code found" \
+    :(num == SPADE_DN_TYPE_ODD_PORTDEST_LOWH) ? "Source used odd dest for port" \
+    :"?" \
+    )
+
+/// map the medium-length strings describing the detection done to the detection type number
+#define SPADE_DN_TYPE_NUM4MEDDESCR(str) \
+    ((!strcmp(str,"Closed dest port used")) ? SPADE_DN_TYPE_CLOSED_DPORT \
+    :(!strcmp(str,"Rare but open dest port used")) ? SPADE_DN_TYPE_ODD_OPEN_DPORT \
+    :(!strcmp(str,"Rare dest port used")) ? SPADE_DN_TYPE_ODD_DPORT \
+    :(!strcmp(str,"Non-live dest used")) ? SPADE_DN_TYPE_NONLIVE_DEST \
+    :(!strcmp(str,"Source used odd dest port")) ? SPADE_DN_TYPE_SRC_ODD_DPORT \
+    :(!strcmp(str,"Odd ICMP type/code found")) ? SPADE_DN_TYPE_ODD_TYPECODE \
+    :(!strcmp(str,"Source used odd dest for port")) ? SPADE_DN_TYPE_ODD_PORTDEST_LOWH \
+    :SPADE_DN_TYPE_UNKNOWN \
+    )
+    
+/*@}*/
+
+/* Id: spade_detection_types.h,v 1.1 2004/10/11 14:35:54 jonkman Exp $ */
+
+#endif // SPADE_DETECTION_TYPES_H
+
+/*********************************************************************
+score_mgr.h, distributed as part of Spade v030125.1
+Author: James Hoagland, Silicon Defense (hoagland@SiliconDefense.com)
+copyright (c) 2002 by Silicon Defense (http://www.silicondefense.com/)
+Released under GNU General Public License, see the COPYING file included
+with the distribution or http://www.silicondefense.com/spice/ for details.
+
+Please send complaints, kudos, and especially improvements and bugfixes to
+hoagland@SiliconDefense.com.  As described in GNU General Public License, no
+warranty is expressed for this program.
+*********************************************************************/
+
+/* Internal version control: Id: score_mgr.h,v 1.1 2004/10/11 14:35:54 jonkman Exp $ */
+
+#ifndef SCORE_MGR_H
+#define SCORE_MGR_H
+
+/*! \file score_mgr.h
+ * \brief 
+ *  score_mgr.h is the header file for score_mgr.c
+ * \ingroup stmgr
+ */
+
+/*! \weakgroup stmgr
+    @{
+*/
+
+#include <stdio.h>
+
+/*********************************************************************
+thresh_adviser.h, distributed as part of Spade v030125.1
+Author: James Hoagland, Silicon Defense (hoagland@SiliconDefense.com)
+copyright (c) 2002 by Silicon Defense (http://www.silicondefense.com/)
+Released under GNU General Public License, see the COPYING file included
+with the distribution or http://www.silicondefense.com/spice/ for details.
+
+Please send complaints, kudos, and especially improvements and bugfixes to
+hoagland@SiliconDefense.com.  As described in GNU General Public License, no
+warranty is expressed for this program.
+*********************************************************************/
+
+/* Internal version control: Id: thresh_adviser.h,v 1.1 2004/10/11 14:35:54 jonkman Exp $ */
+
+#ifndef THRESH_ADVISER_H
+#define THRESH_ADVISER_H
+
+/*! \file thresh_adviser.h
+ * \brief 
+ *  thresh_adviser.h is the header file for thresh_adviser.c
+ * \ingroup stmgr
+ */
+
+/*! \weakgroup stmgr
+    @{
+*/
+
+#include <stdio.h>
+/*********************************************************************
+spade_enviro.h, distributed as part of Spade v030125.1
+Author: James Hoagland, Silicon Defense (hoagland@SiliconDefense.com)
+copyright (c) 2002 by Silicon Defense (http://www.silicondefense.com/)
+Released under GNU General Public License, see the COPYING file included
+with the distribution or http://www.silicondefense.com/spice/ for details.
+
+Please send complaints, kudos, and especially improvements and bugfixes to
+hoagland@SiliconDefense.com.  As described in GNU General Public License, no
+warranty is expressed for this program.
+*********************************************************************/
+
+/* Internal version control: Id: spade_enviro.h,v 1.1 2004/10/11 14:35:54 jonkman Exp $ */
+
+#ifndef SPADE_ENVIRO_H
+#define SPADE_ENVIRO_H
+
+/*! \file spade_enviro.h
+ * \brief 
+ *  spade_enviro.h contains the type declaration for the spade_enviro
+ *  struct
+ * \ingroup stmgr
+ */
+
+/*! \weakgroup stmgr
+    @{
+*/
+
+
+#include <time.h>
+
+/// various counts of the disposition of events; yes, this should be in netspade, not libspade because it violates abstractions
+typedef struct {
+    int scored;      ///< count of all packets scored 
+    int respchecked; ///< count of all packets checked as a response
+    int reported;    ///< count of all packets reported
+    int excluded;    ///< count of all packets not reported due to configured exclusion
+    int nonexcluded; ///< count of all packets that were anomalous but not excluded
+    int waited;      ///< count of all packets added to the wait queue
+    int insuffobsed; ///< count of packets with insufficient obsercations
+} spade_pkt_stats;
+
+
+/// shared state between libspade and its user
+typedef struct {
+    time_t now; ///< the time of the last packet added
+
+    /// the threshold at which anomolous events are reported
+    double thresh;
+    
+    unsigned long *total_pkts; ///< pointer to read-only count of all packets seen
+    spade_pkt_stats pkt_stats; ///< packet disposition counts
+} spade_enviro;
+
+spade_enviro *new_spade_enviro(double thresh,unsigned long *total_pkts_ptr);
+void init_spade_enviro(spade_enviro *self,double thresh,unsigned long *total_pkts_ptr);
+
+/*@}*/
+#endif // SPADE_ENVIRO_H
+
+/*********************************************************************
+ll_double.h, distributed as part of Spade v030125.1
+Author: James Hoagland, Silicon Defense (hoagland@SiliconDefense.com)
+copyright (c) 2002 by Silicon Defense (http://www.silicondefense.com/)
+Released under GNU General Public License, see the COPYING file included
+with the distribution or http://www.silicondefense.com/spice/ for details.
+
+Please send complaints, kudos, and especially improvements and bugfixes to
+hoagland@SiliconDefense.com.  As described in GNU General Public License, no
+warranty is expressed for this program.
+*********************************************************************/
+
+#ifndef LL_DOUBLE_H
+#define LL_DOUBLE_H
+
+/*! \file ll_double.h
+ * \brief
+ *  ll_double.h is the header file for ll_double.c
+ * \ingroup libspade_util
+ */
+
+/*! \addtogroup libspade_util
+    @{
+*/
+
+/// a link in a singly linked list of doubles
+typedef struct _ll_double {
+    double val; ///< the value
+    struct _ll_double *next; ///< the next in the list
+} ll_double;
+
+ll_double *new_ll_double(double val);
+void free_ll_double_list(ll_double *start);
+
+/*@}*/
+#endif  /* ! LL_DOUBLE_H */
+
+/* Id: ll_double.h,v 1.1 2004/10/11 14:35:54 jonkman Exp $ */
+
+/*********************************************************************
+spade_output.h, distributed as part of Spade v030125.1
+Author: James Hoagland, Silicon Defense (hoagland@SiliconDefense.com)
+copyright (c) 2002 by Silicon Defense (http://www.silicondefense.com/)
+Released under GNU General Public License, see the COPYING file included
+with the distribution or http://www.silicondefense.com/spice/ for details.
+
+Please send complaints, kudos, and especially improvements and bugfixes to
+hoagland@SiliconDefense.com.  As described in GNU General Public License, no
+warranty is expressed for this program.
+*********************************************************************/
+
+/* Internal version control: Id: spade_output.h,v 1.1 2004/10/11 14:35:54 jonkman Exp $ */
+
+#ifndef SPADE_OUTPUT_H
+#define SPADE_OUTPUT_H
+
+/*! \file spade_output.h
+ * \brief 
+ *  spade_output.h contains some declarations regarding Spade
+ *  message output
+ * \ingroup libspade_misc
+ */
+
+/** \addtogroup libspade_misc
+    @{
+*/
+
+#include <stdarg.h>
+
+/// the max length of a message from Spade
+#define MAX_SPADE_MSG_LEN 1000
+
+/// enum listing the different messages types from Spade to the user
+typedef enum {
+    SPADE_MSG_TYPE_STATUS,  ///< message is providing routine status information, e.g., from starting up
+    SPADE_MSG_TYPE_INFO,    ///< the message is merely informational
+    SPADE_MSG_TYPE_DEBUG,   ///< it is a debugging message
+    SPADE_MSG_TYPE_WARNING, ///< the message is a warning
+    SPADE_MSG_TYPE_FATAL    ///< fatal error
+} spade_message_type;
+
+/// function type for a spade message callback function
+typedef void (*spade_msg_fn)(spade_message_type msg_type,const char *str);
+
+void default_spade_msg_fn(spade_message_type msg_type,const char *str);
+
+void formatted_spade_msg_send(spade_message_type msg_type,spade_msg_fn msg_fn,const char *format,...);
+
+/*@}*/
+
+#endif // SPADE_OUTPUT_H
+
+
+/// representation of a threshold adviser
+typedef struct {
+    int obs_size;  ///< the number of anomalous packets desired
+    time_t obs_secs; ///< how long to observe for
+    /// head of a linked list of the highest anomaly scores we've seen
+    /** this list can be up to tl_obs_size+1 long and is ordered by increasing score; the list is initialized to 0 -> 0 in case we never see enough packets */
+    ll_double *top_anom_list; 
+    int top_anom_list_size; ///< the number of scores on the list (0-based)
+    time_t obs_start_time; ///< the start time of the observation, set after the first packet we see
+} thresh_adviser;
+
+
+void init_thresh_adviser(thresh_adviser *self, int obs_size, int obs_secs, spade_msg_fn msg_callback);
+void init_thresh_adviser_from_str(thresh_adviser *self, char *str, spade_msg_fn msg_callback);
+thresh_adviser *new_thresh_adviser(int obs_size, int obs_secs, spade_msg_fn msg_callback);
+void thresh_adviser_reset(thresh_adviser *self, int obs_size, int obs_secs, spade_msg_fn msg_callback);
+void thresh_adviser_start_time(thresh_adviser *self, time_t time);
+int thresh_adviser_new_time(thresh_adviser *self, spade_enviro *enviro);
+void thresh_adviser_new_score(thresh_adviser *self, double anom_score);
+void thresh_adviser_write_advice(thresh_adviser *self, FILE *file);
+
+void thresh_adviser_print_config_details(thresh_adviser *self, FILE *f, char *indent);
+
+/*@}*/
+#endif // THRESH_ADVISER_H
+
+/*********************************************************************
+thresh_adapter.h, distributed as part of Spade v030125.1
+Author: James Hoagland, Silicon Defense (hoagland@SiliconDefense.com)
+copyright (c) 2002 by Silicon Defense (http://www.silicondefense.com/)
+Released under GNU General Public License, see the COPYING file included
+with the distribution or http://www.silicondefense.com/spice/ for details.
+
+thresh_adapter.h is the header file for thresh_adapter.c
+
+Please send complaints, kudos, and especially improvements and bugfixes to
+hoagland@SiliconDefense.com.  As described in GNU General Public License, no
+warranty is expressed for this program.
+*********************************************************************/
+
+/* Internal version control: Id: thresh_adapter.h,v 1.1 2004/10/11 14:35:54 jonkman Exp $ */
+
+#ifndef THRESH_ADAPTER_H
+#define THRESH_ADAPTER_H
+
+/*! \file thresh_adapter.h
+ * \brief 
+ *  thresh_adapter.h is the header file for thresh_adapter.c
+ * \ingroup stmgr
+ */
+
+/*! \weakgroup stmgr
+    @{
+*/
+
+/*********************************************************************
+dll_double.h, distributed as part of Spade v030125.1
+Author: James Hoagland, Silicon Defense (hoagland@SiliconDefense.com)
+copyright (c) 2002 by Silicon Defense (http://www.silicondefense.com/)
+Released under GNU General Public License, see the COPYING file included
+with the distribution or http://www.silicondefense.com/spice/ for details.
+
+Please send complaints, kudos, and especially improvements and bugfixes to
+hoagland@SiliconDefense.com.  As described in GNU General Public License, no
+warranty is expressed for this program.
+*********************************************************************/
+
+#ifndef DLL_DOUBLE_H
+#define DLL_DOUBLE_H
+
+/*! \file dll_double.h
+ * \brief 
+ *  dll_double.h is the header file for dll_double.c.
+ * \ingroup libspade_util
+ */
+
+/*! \addtogroup libspade_util
+    @{
+*/
+
+/// a link in a doubly linked list of doubles
+typedef struct _dll_double {
+    double val; ///< the value
+    struct _dll_double *prev; ///< the previous item on the linked list
+    struct _dll_double *next; ///< the next item on the linked list
+} dll_double;
+
+dll_double *new_dll_double(double val);
+void free_dll_double_list(dll_double *start);
+
+
+#endif  /* ! DLL_DOUBLE_H */
+
+
+/* Id: dll_double.h,v 1.1 2004/10/11 14:35:54 jonkman Exp $ */
+ 
+ 
+ 
+
+#include <stdio.h>
+#include <time.h>
+
+/// structure to hold the data for adapt mode #1
+typedef struct {
+    /// the number of alerts that is ideal for the given length of time
+    int target;
+    /// the length of time in which to ideally produce the given number of alerts; also the interval at which to adjust the report threshold
+    time_t period;
+    /// the weight to give to the new observation ideal cutoff in determining the new weight
+    float new_obs_weight;
+    /// adapt by count or by time only
+    int by_count;
+    
+    /// the head of the list of anomaly scores.
+    ll_double *top_list;
+    /// the current size of this list (0-based)
+    int top_list_size;
+} adapt1_data;
+
+/// structure to hold the data for adapt mode #2
+typedef struct {
+    /// the specification of the target
+    double targetspec;
+    /// the observation period
+    double obsper;
+    /// the number of short periods in a medium period
+    int NS;
+    /// the number of medium periods in a long period
+    int NM;
+    /// the number of long periods to average over
+    int NL;
+    
+    /// the current target based on targetspec
+    int target;
+    /// latest medium term component
+    double mid_anom_comp;
+    /// latest long term component
+    double long_anom_comp;
+    /// an array of heads of observation linked lists
+    dll_double **obslists_head;
+    /// an array of tails of the observation linked lists
+    dll_double **obslists_tail;
+    /// an array of the (0-based) size of these lists
+    int *obslists_size;
+    /// the number of complete observation periods
+    int obsper_count;
+    /// arrays of short and medium term components used for calculating other components
+    double *recScomps,*recMcomps;
+
+    /// the start time of the current observation period
+    time_t obsper_start;
+    /// which obslist to add to, aka, obsper_count % NS
+    int obslist_new_slot;
+
+    // the count of period 2 instances
+    int per2_count;
+    // the count of period 3 instances
+    int per3_count; 
+} adapt2_data;
+
+/// structure to hold the data for adapt mode #3
+typedef struct {
+    /// the specification of the target
+    double targetspec;
+    /// the observation period
+    double obsper;
+    /// the number of observation period results to average together
+    int NO;
+    
+    /// the current target based on targetspec
+    int target;
+    /// array of past observations
+    double *hist;
+    /// a linked list of anomaly scores from the current period
+    ll_double *anoms;
+    /// (0-based) size of this list
+    int anoms_size;
+    /// number of completed observation periods
+    int completed_obs_per;
+    
+    double obssum; ///< the sum of all current elements in the array
+} adapt3_data;
+ 
+/// structure to hold the data for adapt mode #4
+typedef struct {
+    double thresh; ///< the threshold to change to
+} adapt4_data;
+
+/// the representation of a threshold adapter
+typedef struct {
+    /// the adapt mode number
+    int adapt_mode;
+    /// the adapt-type specific storage; the union is selected by adapt_mode
+    union {
+        adapt1_data a1;
+        adapt2_data a2;
+        adapt3_data a3;
+        adapt4_data a4;
+    } d;
+    
+    /// adapt by count or by time only
+    int adapt_by_count;
+    
+    /// the total count of packets at the start of the observation period
+    int last_total_stats;
+    /// how often (in secs) to recalc packet rate and to trigger adapting
+    time_t adapt_period; 
+    /// how many packet periods have occurred
+    int pkt_period_count;
+    /// the time this period started
+    time_t period_start;
+    /// the rate of all packets in the adapt period; used to decide when to adapt
+    float period_pkt_rate;
+    /// rate of accepted packets in the adapt period; sometimes used to determine target rate of reports
+    float period_acc_rate; 
+    /// are we done adapting?
+    int done;
+    /// the function to call to provide a message to the user
+    spade_msg_fn msg_callback;
+} thresh_adapter;
+
+
+void init_thresh_adapter(thresh_adapter *self,spade_msg_fn msg_callback);
+thresh_adapter *new_thresh_adapter(spade_msg_fn msg_callback);
+void thresh_adapter_setup_from_str(thresh_adapter *self,int adaptmode,char *str);
+void thresh_adapter_setup_1(thresh_adapter *self, int target, time_t period, float new_obs_weight, int by_count);
+void thresh_adapter_setup_2(thresh_adapter *self, double targetspec, double obsper, int NS, int NM, int NL);
+void thresh_adapter_setup_3(thresh_adapter *self, double targetspec, double obsper, int NO);
+void thresh_adapter_setup_4(thresh_adapter *self, double thresh, double obsper);
+void thresh_adapter_start_time(thresh_adapter *self, time_t now);
+int thresh_adapter_new_time(thresh_adapter *self, spade_enviro *enviro, double *sugg_thresh);
+void thresh_adapter_new_score(thresh_adapter *self, double anom_score);
+
+void thresh_adapter_print_config_details(thresh_adapter *self,FILE *f,char *indent);
+
+/*@}*/
+#endif // THRESH_ADAPTER_H
+
+/*********************************************************************
+anomscore_surveyer.h, distributed as part of Spade v030125.1
+Author: James Hoagland, Silicon Defense (hoagland@SiliconDefense.com)
+copyright (c) 2002 by Silicon Defense (http://www.silicondefense.com/)
+Released under GNU General Public License, see the COPYING file included
+with the distribution or http://www.silicondefense.com/spice/ for details.
+
+anomscore_surveyer.h is the header file for anomscore_surveyer.c
+
+Please send complaints, kudos, and especially improvements and bugfixes to
+hoagland@SiliconDefense.com.  As described in GNU General Public License, no
+warranty is expressed for this program.
+*********************************************************************/
+
+/* Internal version control: Id: anomscore_surveyer.h,v 1.1 2004/10/11 14:35:54 jonkman Exp $ */
+
+#ifndef ANOMSCORE_SURVEYER_H
+#define ANOMSCORE_SURVEYER_H
+
+/*! \file anomscore_surveyer.h
+ * \brief 
+ *  anomscore_surveyer.h is the header file for anomscore_surveyer.c
+ * \ingroup stmgr
+ */
+
+/*! \weakgroup stmgr
+    @{
+*/
+
+
+
+
+#include <time.h>
+#include <stdio.h>
+
+/// represents an instance of an anomscore_surveyer
+typedef struct {
+    /// the number of seconds in the survey interval
+    float interval;
+    /// the survey file name
+    char *filename;
+
+    /// the survey log file handle
+    FILE *surveyfile;
+    /// the list of anomaly scores for the survey
+    ll_double *list;
+    /// the length of the list (1-based)
+    int list_len;
+    /// the suvery period number (starts with 1)
+    int period;
+    
+    /// the start time for this survey interval
+    time_t interval_start_time;
+    /// the number of packets seen in this survey period so far
+    int rec_count;
+} anomscore_surveyer;
+
+
+int init_anomscore_surveyer(anomscore_surveyer *self, char *filename, float interval,spade_msg_fn msg_callback);
+int init_anomscore_surveyer_from_str(anomscore_surveyer *self,char *str,spade_msg_fn msg_callback);
+anomscore_surveyer *new_anomscore_surveyer(char *filename, float interval,spade_msg_fn msg_callback);
+void anomscore_surveyer_flush(anomscore_surveyer *self);
+void anomscore_surveyer_shutdown(anomscore_surveyer *self);
+void anomscore_surveyer_new_time(anomscore_surveyer *self, spade_enviro *enviro);
+void anomscore_surveyer_new_score(anomscore_surveyer *self, double anom_score);
+
+void anomscore_surveyer_print_config_details(anomscore_surveyer *self,FILE *f,char *indent);
+
+/*@}*/
+#endif // ANOMSCORE_SURVEYER_H
+
+/*********************************************************************
+spade_state.h, distributed as part of Spade v030125.1
+Author: James Hoagland, Silicon Defense (hoagland@SiliconDefense.com)
+copyright (c) 2002 by Silicon Defense (http://www.silicondefense.com/)
+Released under GNU General Public License, see the COPYING file included
+with the distribution or http://www.silicondefense.com/spice/ for details.
+
+Please send complaints, kudos, and especially improvements and bugfixes to
+hoagland@SiliconDefense.com.  As described in GNU General Public License, no
+warranty is expressed for this program.
+*********************************************************************/
+
+#ifndef SPADE_STATE_H
+#define SPADE_STATE_H
+
+/*! \file spade_state.h
+ * \brief 
+ *  spade_state.h is the header file for spade_state.c.
+ * \ingroup staterec
+ */
+
+/*! \addtogroup staterec
+    @{
+*/
+
+#include <stdio.h>
+#include <time.h>
+
+/// a handle for ths user on a currently active state recovery file
+typedef struct {
+    FILE *f; ///< the file pointer
+} statefile_ref;
+
+
+statefile_ref *spade_state_begin_checkpointing(char *filename, char *appname, u8 app_cur_fvers);
+int spade_state_end_checkpointing(statefile_ref *s);
+int spade_state_checkpoint_str(statefile_ref *s, char *str);
+int spade_state_checkpoint_arr(statefile_ref *s, void *arr, int len, int elsize);
+int spade_state_checkpoint_str_arr(statefile_ref *s, char **arr, int len);
+int spade_state_checkpoint_u32(statefile_ref *s, u32 val);
+int spade_state_checkpoint_u8(statefile_ref *s, u8 val);
+int spade_state_checkpoint_time_t(statefile_ref *s, time_t val);
+int spade_state_checkpoint_double(statefile_ref *s,double val);
+int spade_state_end_section(statefile_ref *s);
+
+statefile_ref *spade_state_begin_recovery(char *filename, int min_app_fvers, char **appname, u8 *file_app_fvers);
+int spade_state_end_recovery(statefile_ref *s);
+int spade_state_recover_check_end_of_section(statefile_ref *s, int *res);
+int spade_state_recover_arr(statefile_ref *s, void *arr, int len, int elsize);
+int spade_state_recover_str_arr(statefile_ref *s, char **arr, int len);
+int spade_state_recover_u32(statefile_ref *s, u32 *val);
+int spade_state_recover_u8(statefile_ref *s, u8 *val);
+int spade_state_recover_time_t(statefile_ref *s, time_t *val);
+int spade_state_recover_double(statefile_ref *s, double *val);
+int spade_state_recover_str(statefile_ref *s, char **str);
+int spade_state_recover_str_to_buff(statefile_ref *s, char *buff, int maxlen);
+
+#endif // SPADE_STATE_H
+
+/* Id: spade_state.h,v 1.1 2004/10/11 14:35:54 jonkman Exp $ */
+//
+/*********************************************************************
+score_info.h, distributed as part of Spade v030125.1
+Author: James Hoagland, Silicon Defense (hoagland@SiliconDefense.com)
+copyright (c) 2002 by Silicon Defense (http://www.silicondefense.com/)
+Released under GNU General Public License, see the COPYING file included
+with the distribution or http://www.silicondefense.com/spice/ for details.
+
+Please send complaints, kudos, and especially improvements and bugfixes to
+hoagland@SiliconDefense.com.  As described in GNU General Public License, no
+warranty is expressed for this program.
+*********************************************************************/
+
+/* Internal version control: Id: score_info.h,v 1.1 2004/10/11 14:35:54 jonkman Exp $ */
+
+#ifndef SCORE_INFO_H
+#define SCORE_INFO_H
+
+/*! \file score_info.h
+ * \brief 
+ *  score_info.h contains the type declaration for the score_info
+ *  struct and the interface to the associated functions
+ * \ingroup scoreprod
+ */
+
+/*! \addtogroup scoreprod
+    @{
+*/
+
+/// a special double indicating that there is no anomaly score
+#define NO_SCORE (double)-1
+
+/// enum containing the possible preferences among types of anomaly scores
+typedef enum {PREF_NOSCORE,PREF_RAWSCORE,PREF_RELSCORE} scorepref;
+
+const char *scorepref_str(scorepref pref);
+
+/// stores information about a anomaly scoring result
+typedef struct _score_info {
+    scorepref main; ///< the preference for which available score to use as the main score
+    double relscore; ///< the relative anomaly score, or NO_SCORE
+    double rawscore; ///< the raw anomaly score, or NO_SCORE
+    int corrscore_used; ///< if the raw anomaly score was computed, was it computed correctly
+    
+    struct _score_info *next; ///< the next score_info in a list of them
+} score_info;
+
+score_info *new_score_info(scorepref main, double relscore, double rawscore, int corrscore_used);
+void init_score_info(score_info *i, scorepref main, double relscore, double rawscore, int corrscore_used);
+score_info *score_info_clone(score_info *i);
+void free_score_info(score_info *i);
+void free_score_infos(score_info *start);
+
+double score_info_mainscore(score_info *i);
+double score_info_relscore(score_info *i);
+double score_info_rawscore(score_info *i);
+int score_info_raw_is_corrscore(score_info *i);
+scorepref score_info_main_pref(score_info *i);
+
+/*@}*/
+#endif // SCORE_INFO_H
+
+/*********************************************************************
+spade_event.h, distributed as part of Spade v030125.1
+Author: James Hoagland, Silicon Defense (hoagland@SiliconDefense.com)
+copyright (c) 2002 by Silicon Defense (http://www.silicondefense.com/)
+Released under GNU General Public License, see the COPYING file included
+with the distribution or http://www.silicondefense.com/spice/ for details.
+
+Please send complaints, kudos, and especially improvements and bugfixes to
+hoagland@SiliconDefense.com.  As described in GNU General Public License, no
+warranty is expressed for this program.
+*********************************************************************/
+
+/* Internal version control: Id: spade_event.h,v 1.1 2004/10/11 14:35:54 jonkman Exp $ */
+
+#ifndef SPADE_EVENT_H
+#define SPADE_EVENT_H
+
+/*! \file spade_event.h
+ * \brief 
+ *  spade_event.h contains the type declaration for the spade_event struct 
+ *  and the interface to the associated functions
+ * \ingroup libspade_misc
+ */
+
+/*! \addtogroup libspade_misc
+    @{
+*/
+
+/*********************************************************************
+spade_prob_table_types.h, distributed as part of Spade v030125.1
+Author: James Hoagland, Silicon Defense (hoagland@SiliconDefense.com)
+copyright (c) 2002 by Silicon Defense (http://www.silicondefense.com/)
+Released under GNU General Public License, see the COPYING file included
+with the distribution or http://www.silicondefense.com/spice/ for details.
+
+Please send complaints, kudos, and especially improvements and bugfixes to
+hoagland@SiliconDefense.com.  As described in GNU General Public License, no
+warranty is expressed for this program.
+*********************************************************************/
+
+#ifndef SPADE_PROB_TABLE_TYPES_H
+#define SPADE_PROB_TABLE_TYPES_H
+
+/*! \file spade_prob_table_types.h
+ * \brief 
+ *  spade_prob_table_types.h is the header file for spade_prob_table_types.c.
+ * \ingroup staterec
+ */
+
+/*! \addtogroup staterec
+    @{
+*/
+
+
+
+/// index type into tree memory block data structures
+typedef u32 mindex;
+
+/// dmindex is a mindex used with top bit indicating if one of two datatypes is present
+typedef u32 dmindex;
+
+/// the type of the values of the features
+/** \note right now, we assume all features can be contained in a u32 and can be sorted as unsigned ints; we may need to extend this someday */
+typedef u32 valtype;
+
+
+/// a tree root
+typedef struct _treeroot {
+    mindex next;      ///< the next tree root in a list
+    dmindex root;     ///< root node of the tree, if top bit is 1, it is a leafnode, otherwise it is a interior node
+    features type;    ///< the feature that is being represented in this tree
+    double entropy;   ///< the last calculated entropy in this tree; < 0 if it has not been calculated
+    u16 entropy_wait; ///< the number of additions to the tree to wait till recalculating the entropy; only valid if entropy >= 0
+} treeroot;
+
+/// an interior node in the tree
+typedef struct _intnode {
+    double sum;     ///< the sum of the counts underneath this node in the tree
+    valtype sortpt; ///< the highest value on the left side of this node
+    dmindex left;   ///< the left node; if top bit is 1, it is a leafnode
+    dmindex right;  ///< the right node; if top bit is 1, it is a leafnode
+    u16 wait;       ///< the number of additions to the subtree to wait before checking for reblancing
+} intnode;
+
+/// a leaf node of the tree
+typedef struct _leafnode {
+    double count;    ///< the count on this node
+    valtype value;   ///< the value this node represents
+    mindex nexttree; ///< the first in a linked list of trees anchored from this leaf node
+} leafnode;
+
+#define isleaf(node) (node & DMINDEXMASK)
+#define asleaf(leaf) (leaf | DMINDEXMASK)
+#define encleaf2mindex(node) (node ^ DMINDEXMASK)
+/* arg is a dmindex; if it denotes a leaf, return the count on that leaf
+   otherwise return the sum on the interior node */ 
+#define count_or_sum(node) (isleaf(node) ? leafnode(encleaf2mindex(node)).count : intnode(node).sum)
+#define eleafval(leaf) leafnode(encleaf2mindex(leaf)).value
+#define largestval(node) (isleaf(node) ? eleafval(node) : largest_val(node))
+#define treetype(t) tree(t).type
+#define treeroot(t) tree(t).root
+#define treenext(t) tree(t).next
+#define treeH(t) tree(t).entropy
+#define treeH_wait(t) tree(t).entropy_wait
+#define intleft(node) intnode(node).left
+#define intright(node) intnode(node).right
+#define intsum(node) intnode(node).sum
+#define intsortpt(node) intnode(node).sortpt
+#define intwait(node) intnode(node).wait
+#define leafcount(leaf) leafnode(leaf).count
+#define leafvalue(leaf) leafnode(leaf).value
+#define leafnexttree(leaf) leafnode(leaf).nexttree
+
+
+/* defaults unless recovering from a checkpoint */
+#define DEFAULT_ROOT_BLOCK_BITS 10
+#define DEFAULT_INT_BLOCK_BITS 9
+#define DEFAULT_LEAF_BLOCK_BITS 10
+
+/* these number of blocks are used
+   unless file recovering from already uses more blocks */
+#define DEFAULT_MAX_ROOT_BLOCKS 4500
+#define DEFAULT_MAX_INT_BLOCKS 12000
+#define DEFAULT_MAX_LEAF_BLOCKS 9000
+
+#define bits2blocksize(b) (1 << b)
+
+#define ROOT_BLOCK_SIZE bits2blocksize(ROOT_BLOCK_BITS)
+#define ROOT_BLOCK_MASK ((1 << ROOT_BLOCK_BITS) -1)
+#define tree(i) ROOT_M[i>>ROOT_BLOCK_BITS][i&ROOT_BLOCK_MASK]
+#define root_index(p,i) ((p<<ROOT_BLOCK_BITS)+i)
+
+#define INT_BLOCK_SIZE bits2blocksize(INT_BLOCK_BITS)
+#define INT_BLOCK_MASK ((1 << INT_BLOCK_BITS) -1)
+#define intnode(i) INT_M[i>>INT_BLOCK_BITS][i&INT_BLOCK_MASK]
+#define intnode_index(p,i) ((p<<INT_BLOCK_BITS)+i)
+
+#define LEAF_BLOCK_SIZE bits2blocksize(LEAF_BLOCK_BITS)
+#define LEAF_BLOCK_MASK ((1 << LEAF_BLOCK_BITS) -1)
+#define leafnode(i) LEAF_M[i>>LEAF_BLOCK_BITS][i&LEAF_BLOCK_MASK]
+#define leafnode_index(p,i) ((p<<LEAF_BLOCK_BITS)+i)
+
+#define rfreenext(n) (n).next
+#define ifreenext(n) (n).left
+#define lfreenext(n) (n).nexttree
+
+/* something of valtype that cannot be a sortpt */
+#define NOT_A_SORTPT ((u32)MAX_U32)
+
+#define TNULL (mindex)-1
+#define DMINDEXMASK ((dmindex)(1 << (sizeof(dmindex)*8-1)))
+
+extern treeroot **ROOT_M;
+extern intnode **INT_M;
+extern leafnode **LEAF_M;
+extern mindex root_freelist;
+extern mindex int_freelist;
+extern mindex leaf_freelist;
+
+void init_mem();
+void allocate_mem_blocks();
+int reallocate_ptr_array(void ***arrptr,int oldsize,int newsize);
+
+mindex new_treeinfo(features type);
+void free_treeinfo(mindex f);
+mindex new_int();
+void free_int(mindex f);
+mindex new_leaf(valtype val);
+void free_leaf(mindex f);
+
+extern unsigned char ROOT_BLOCK_BITS;
+extern unsigned char INT_BLOCK_BITS;
+extern unsigned char LEAF_BLOCK_BITS;
+extern unsigned int MAX_ROOT_BLOCKS;
+extern unsigned int MAX_INT_BLOCKS;
+extern unsigned int MAX_LEAF_BLOCKS;
+
+#endif // SPADE_PROB_TABLE_TYPES_H
+
+/* Id: spade_prob_table_types.h,v 1.1 2004/10/11 14:35:54 jonkman Exp $ */
+
+
+/// function type to make a copy of a spade_event's "native" field
+typedef void *(*event_native_copier_t)(void *native);
+/// function type to free a copy of a spade_event's "native" field
+typedef void (*event_native_freer_t)(void *native);
+
+/// the representation of an event that is being given to libspade
+typedef struct {
+    /// the features, indexed by feature number (libspade-user assigned)
+    valtype fldval[MAX_NUM_FEATURES];
+    /// the time the event occurred
+    double time;
+    /// the origin of the event, in user defined terms
+    u32 origin;
+
+    /// a field the user may use to maintain a reference to its own representation of the event
+    void *native;
+    /// used internally, at times this field may be set to point to a routine to free the native copy
+    event_native_freer_t native_freer;
+} spade_event;
+
+spade_event *new_spade_event(void);
+spade_event *spade_event_clone(spade_event *e, event_native_copier_t native_copier, event_native_freer_t native_freer);
+void free_spade_event(spade_event *e);
+
+/*@}*/
+
+#endif // SPADE_EVENT_H
+
+
+
+
+#define ADVISING_OFF     0 ///< threshold advising status indicating the threshold advising was never on
+#define ADVISING_RUNNING 1 ///< threshold advising status indicating the threshold advising is on and running
+#define ADVISING_DONE    2 ///< threshold advising status indicating the threshold advising was on but has now completed
+
+/// function type for a function to call when a score exceeds the reporting threshold
+typedef void (*spade_thresh_exceeded_fn_t)(void *context,void *targetref,spade_event *event,score_info *score);
+/// function type for a function to call when a reporting threshold changes
+typedef void (*spade_thresh_changed_fn_t)(void *context,void *targetref);
+
+// storage for an instance of a score manager
+typedef struct {
+    void *mgrref; ///< some sort of identifier of this score manager
+    spade_enviro *enviro; ///< environment shared with target user
+    
+    int adapt_active; ///< 0 is there is no adapting; otherwise the adapt method #
+    int advise_status; ///< ADVISING_OFF, ADVISING_RUNNING, or ADVISING_DONE
+    int survey_active; ///< true if survey mode is active
+    
+    thresh_adapter adapter; ///< our threshold adapter module, if threshold adapting is in use
+    thresh_adviser adviser; ///< our threshold advising module, if threshold advising is in use
+    anomscore_surveyer surveyer; ///< our anomaly score surveyer module, if this is on
+        
+    /// function to call when we see a score that exceeds the threshold, or NULL if none
+    spade_thresh_exceeded_fn_t threshexceeded_callback;
+    /// function to call when the reporting threshold changes, or NULL if none
+    spade_thresh_changed_fn_t threshchanged_callback;
+    void *callback_context; ///< user-provided pointer that is returned as first arg in callback
+
+    spade_msg_fn msg_callback; ///< function to call when there is a message for the user
+} score_mgr;
+
+score_mgr *new_score_mgr(void *mgrref, spade_enviro *enviro, void *callback_context, spade_thresh_exceeded_fn_t threshexceeded_callback, spade_thresh_changed_fn_t threshchanged_callback,spade_msg_fn msg_callback);
+void init_score_mgr(score_mgr *self, void *mgrref, spade_enviro *enviro, void *callback_context, spade_thresh_exceeded_fn_t threshexceeded_callback, spade_thresh_changed_fn_t threshchanged_callback,spade_msg_fn msg_callback);
+
+void score_mgr_setup_adapt_from_str(score_mgr *self, int adaptmode, char *str);
+void score_mgr_setup_adapt1(score_mgr *self, int target, time_t period, float new_obs_weight, int by_count);
+void score_mgr_setup_adapt2(score_mgr *self, double targetspec, double obsper, int NS, int NM, int NL);
+void score_mgr_setup_adapt3(score_mgr *self, double targetspec, double obsper, int NO);
+void score_mgr_setup_adapt4(score_mgr *self, double thresh, double obsper);
+void score_mgr_setup_advise(score_mgr *self, int obs_size, int obs_secs);
+void score_mgr_setup_advise_from_str(score_mgr *self, char *str);
+void score_mgr_setup_survey(score_mgr *self, char *filename, float interval);
+void score_mgr_setup_survey_from_str(score_mgr *self, char *str);
+
+int score_mgr_new_time(score_mgr *self, time_t time);
+void score_mgr_new_event(score_mgr *self, score_info *score, spade_event *event);
+
+void score_mgr_dump(score_mgr *self);
+void score_mgr_cleanup(score_mgr *self);
+void score_mgr_file_print_log(score_mgr *self, FILE *file);
+
+void score_mgr_print_config_details(score_mgr *self, FILE *file, char *indent);
+
+/*@}*/
+#endif // SCORE_MGR_H
+
+
+/*********************************************************************
+spade_report.h, distributed as part of Spade v030125.1
+Author: James Hoagland, Silicon Defense (hoagland@SiliconDefense.com)
+copyright (c) 2002 by Silicon Defense (http://www.silicondefense.com/)
+Released under GNU General Public License, see the COPYING file included
+with the distribution or http://www.silicondefense.com/spice/ for details.
+
+spade_report.h contains the type declaration for the spade_report struct 
+  and the interface to the associated functions
+
+Please send complaints, kudos, and especially improvements and bugfixes to
+hoagland@SiliconDefense.com.  As described in GNU General Public License, no
+warranty is expressed for this program.
+*********************************************************************/
+
+/* Internal version control: Id: spade_report.h,v 1.1 2004/10/11 14:35:54 jonkman Exp $ */
+
+#ifndef SPADE_REPORT_H
+#define SPADE_REPORT_H
+
+/*! \file spade_report.h
+ * \ingroup netspade_layer
+ * \brief 
+ *  spade_report.h contains the type declaration for the spade_report struct 
+ *  and the interface to the associated functions
+ */
+
+/*! \addtogroup netspade_layer
+    @{
+*/
+
+
+
+
+#include <stdio.h>
+
+/// a mask for the bits of a port_status_t that represent belief strength
+#define PORT_STRENGTH_MASK 0x003
+/// a mask for the bits of a port_status_t that represent the port status
+#define PORT_BASE_MASK (0xFFF & ~PORT_STRENGTH_MASK)
+
+/// mask off the port status part of a port_status_t
+#define PORT_BASE(status) ((status) & PORT_BASE_MASK)
+/// mask off the belief strength part of a port_status_t
+#define PORT_STRENGTH(status) ((status) & PORT_STRENGTH_MASK)
+
+/// representation of a definate strength belief in port status
+#define PORT_STRENGTH_DEFINATE 0x002
+/// representation of a "likely" strength belief in port status
+#define PORT_STRENGTH_LIKELY 0x001
+/// representation of a "probably" strength belief in port status
+#define PORT_STRENGTH_PROBABLY 0x000
+
+/// port status part of a port_status_t when it is open
+#define PORT_OPEN_BASE (1<<2)
+/// port status part of a port_status_t when it is closed
+#define PORT_CLOSED_BASE (2<<2)
+
+/// an enum of value that represents our preception of a port's status
+/** the integer value is based on a combination of port status and strength of belief */
+typedef enum {
+    /// we don't know the port's status
+    PORT_UNKNOWN=0,
+    /// we think the port is probably open
+    PORT_PROBOPEN       =PORT_OPEN_BASE  |PORT_STRENGTH_PROBABLY,
+    /// we think the port is likely open
+    PORT_LIKELYOPEN     =PORT_OPEN_BASE  |PORT_STRENGTH_LIKELY,
+    /// we think the port is definately open
+    PORT_OPEN           =PORT_OPEN_BASE  |PORT_STRENGTH_DEFINATE,
+    /// we think the port is probably closed
+    PORT_PROBCLOSED     =PORT_CLOSED_BASE|PORT_STRENGTH_PROBABLY,
+    /// we think the port is likely closed
+    PORT_LIKELYCLOSED   =PORT_CLOSED_BASE|PORT_STRENGTH_LIKELY,
+    /// we think the port is definately closed
+    PORT_CLOSED         =PORT_CLOSED_BASE|PORT_STRENGTH_DEFINATE
+} port_status_t;
+
+#define PORT_STATUS_AS_STR(status) (status == PORT_UNKNOWN ? "PORT_UNKNOWN" \
+    : (status == PORT_PROBOPEN) ? "PORT_PROBOPEN" \
+    : (status == PORT_LIKELYOPEN) ? "PORT_LIKELYOPEN" \
+    : (status == PORT_OPEN) ? "PORT_OPEN" \
+    : (status == PORT_PROBCLOSED) ? "PORT_PROBCLOSED" \
+    : (status == PORT_LIKELYCLOSED) ? "PORT_LIKELYCLOSED" \
+    : (status == PORT_PROBOPEN) ? "PORT_PROBOPEN" \
+    : (status == PORT_CLOSED) ? "PORT_CLOSED" \
+    : "UNDEFINED" \
+    )
+
+/// data type for representing a set of port statuses
+/** the value is based on a bit mask, where the bit position is determined by the value of the port_status_t */
+typedef u16 port_status_set_t;
+/// port_status_set_t value when the set is empty
+#define PS_EMPTY_SET (port_status_set_t)0
+/// port_status_set_t value when the set consists of just the given port_status_t value
+#define PS_STATUS_MASK(status) (1 << (status))
+/// port_status_set_t value when the set consists of the given port_status_t value and stronger beliefs
+#define PS_STATUS_MASK_WITH_STRONGER(status) ((status) == PORT_UNKNOWN ? 0xFFF \
+    : PS_STATUS_MASK(status) \
+        | (PORT_STRENGTH(status) < PORT_STRENGTH_LIKELY \
+            ? (PS_STATUS_MASK(PORT_BASE(status)|PORT_STRENGTH_LIKELY)) \
+            : 0 ) \
+        | (PORT_STRENGTH(status) < PORT_STRENGTH_DEFINATE \
+            ? (PS_STATUS_MASK(PORT_BASE(status)|PORT_STRENGTH_DEFINATE)) \
+            : 0 ) \
+    )
+/// initialize a port_status_set_t variable to contatin just the given port_status_t value
+#define PS_INIT_SET(set,status) (set= PS_STATUS_MASK(status))
+/// initialize a port_status_set_t variable to contatin of the given port_status_t value and stronger beliefs
+#define PS_INIT_SET_WITH_STRONGER(set,status) (set= PS_STATUS_MASK_WITH_STRONGER(status))
+/// add the given port_status_t to a port_status_set_t stored in the given variable
+#define PS_ADD_TO_SET(set,status) (set|= PS_STATUS_MASK(status))
+/// add the given port_status_t and stronger beliefs to a port_status_set_t stored in the given variable
+#define PS_ADD_TO_SET_WITH_STRONGER(set,status) (set|= PS_STATUS_MASK_WITH_STRONGER(status))
+/// test if the given port_status_t in the given port_status_set_t
+#define PS_IN_SET(set,status) (set & PS_STATUS_MASK(status))
+
+void port_status_set_file_print(port_status_set_t set,FILE *f);
+
+/// the representation of a Spade report internally and to the libnetspade user
+typedef struct _spade_report {
+    /// the detector type triggering this spade report (from spade_detection_types.h)
+    int detect_type;
+    /// the detector id of the spade detector that is sending this report
+    char *detectorid;
+    /// pointer to the packet (originally provided by the libspade user) this report is about
+    spade_event *pkt;
+    /// pointer to the calculated packet anomaly score(s)
+    score_info *score;
+    /// current port status
+    port_status_t port_status;
+    /// string representing the detection type employed
+    const char *detect_type_str;
+    /// string representing the detectors scope
+    char scope_str[200];
+    /// pointer to stream statistics
+    spade_pkt_stats *stream_stats;
+    /// the next spade report in a list of them
+    struct _spade_report *next;
+} spade_report;
+
+spade_report *new_spade_report(spade_event *pkt,score_info *score, int detect_type, char *detectorid,const char *detect_type_str,char *scope_str,spade_pkt_stats *stream_stats,port_status_t port_status);
+void free_spade_report(spade_report *rpt);
+void free_spade_reports(spade_report *rpt);
+
+#define spade_report_mainscore(rpt) (rpt != NULL ? score_info_mainscore(rpt->score) : NO_SCORE)
+#define spade_report_relscore(rpt) (rpt != NULL ? score_info_relscore(rpt->score) : NO_SCORE)
+#define spade_report_rawscore(rpt) (rpt != NULL ? score_info_rawscore(rpt->score) : NO_SCORE)
+#define spade_report_raw_is_corrscore(rpt) (score_info_raw_is_corrscore(rpt->score))
+#define spade_report_main_pref(rpt) (score_info_main_pref(rpt->score))
+
+/*@}*/
+
+#endif // SPADE_REPORT_H
+
+/*********************************************************************
+score_calculator.h, distributed as part of Spade v030125.1
+Author: James Hoagland, Silicon Defense (hoagland@SiliconDefense.com)
+copyright (c) 2002 by Silicon Defense (http://www.silicondefense.com/)
+Released under GNU General Public License, see the COPYING file included
+with the distribution or http://www.silicondefense.com/spice/ for details.
+
+Please send complaints, kudos, and especially improvements and bugfixes to
+hoagland@SiliconDefense.com.  As described in GNU General Public License, no
+warranty is expressed for this program.
+*********************************************************************/
+
+/* Internal version control: Id: score_calculator.h,v 1.1 2004/10/11 14:35:54 jonkman Exp $ */
+
+#ifndef SCORE_CALCULATOR_H
+#define SCORE_CALCULATOR_H
+
+/*! \file score_calculator.h
+ * \brief 
+ *  score_calculator.h is the header file for score_calculator.c
+ * \ingroup scoreprod
+ */
+
+/*! \addtogroup scoreprod
+    @{
+*/
+
+#include <stdio.h>
+
+/*********************************************************************
+event_recorder.h, distributed as part of Spade v030125.1
+Author: James Hoagland, Silicon Defense (hoagland@SiliconDefense.com)
+copyright (c) 2002 by Silicon Defense (http://www.silicondefense.com/)
+Released under GNU General Public License, see the COPYING file included
+with the distribution or http://www.silicondefense.com/spice/ for details.
+
+Please send complaints, kudos, and especially improvements and bugfixes to
+hoagland@SiliconDefense.com.  As described in GNU General Public License, no
+warranty is expressed for this program.
+*********************************************************************/
+
+/* Internal version control: Id: event_recorder.h,v 1.1 2004/10/11 14:35:54 jonkman Exp $ */
+
+#ifndef EVENT_RECORDER_H
+#define EVENT_RECORDER_H
+
+/*! \file event_recorder.h
+ * \brief 
+ *  event_recorder.h is the header file for event_recorder.c
+ * \ingroup staterec
+ */
+
+/*! \addtogroup staterec
+    @{
+*/
+
+
+/*********************************************************************
+spade_prob_table.h, distributed as part of Spade v030125.1
+Author: James Hoagland, Silicon Defense (hoagland@SiliconDefense.com)
+copyright (c) 2002 by Silicon Defense (http://www.silicondefense.com/)
+Released under GNU General Public License, see the COPYING file included
+with the distribution or http://www.silicondefense.com/spice/ for details.
+
+Please send complaints, kudos, and especially improvements and bugfixes to
+hoagland@SiliconDefense.com.  As described in GNU General Public License, no
+warranty is expressed for this program.
+*********************************************************************/
+
+#ifndef SPADE_PROB_TABLE_H
+#define SPADE_PROB_TABLE_H
+
+/*! \file spade_prob_table.h
+ * \brief 
+ *  spade_prob_table.h is the header file for spade_prob_table.c.
+ * \ingroup staterec
+ */
+
+/*! \addtogroup staterec
+    @{
+*/
+
+
+
+
+
+#include <stdio.h>
+
+/// a spade probability table, represented by some number of nested trees
+typedef struct {
+    mindex root[MAX_NUM_FEATURES]; ///< top level tree roots in an array indexed by the feature type of the top level tree
+    const char **featurenames;     ///< user provided pointer to array of the string names of the features, used for output
+} spade_prob_table;
+
+/// an element in a data structure representing a set of doubles indexed by a list of features
+typedef struct _featcomb {
+    struct _featcomb *next[MAX_NUM_FEATURES]; ///< pointers to the next level of structure, indexed by feature
+    double val[MAX_NUM_FEATURES];  ///< the values stored; indexed by the final feature in the list
+} *featcomb;
+
+#define STATS_NONE          0x00  ///< no statistics
+#define STATS_ENTROPY       0x01  ///< entropy statistics
+#define STATS_UNCONDPROB    0x02  ///< unconditional probabilities
+#define STATS_CONDPROB      0x04  ///< conditional probabilities
+
+#define PROBRESULT_NO_RECORD (double)-1.0 ///< a special probability value denoting the probability denominator was 0
+
+
+void init_spade_prob_table(spade_prob_table *self,const char **featurenames,int recovering);
+spade_prob_table *new_spade_prob_table(const char **featurenames);
+
+int spade_prob_table_is_empty(spade_prob_table *self);
+
+void increment_simple_count(spade_prob_table *self, features type1, valtype val1);
+void increment_2joint_count(spade_prob_table *self, features type1, valtype val1, features type2, valtype val2, int skip);
+void increment_3joint_count(spade_prob_table *self, features type1, valtype val1, features type2, valtype val2, features type3, valtype val3, int skip);
+void increment_4joint_count(spade_prob_table *self, features type1, valtype val1, features type2, valtype val2, features type3, valtype val3, features type4, valtype val4, int skip);
+void increment_Njoint_count(spade_prob_table *self,int size,features type[],valtype val[],int skip);
+
+double prob_simple(spade_prob_table *self, features type1, valtype val1);
+double prob_2joint(spade_prob_table *self, features type1, valtype val1, features type2, valtype val2);
+double prob_Njoint(spade_prob_table *self, int size, features type[], valtype val[]);
+double prob_Njoint_Ncond(spade_prob_table *self, int size, features type[], valtype val[], int condoffset);
+double prob_Njoint_Ncond_plus_one(spade_prob_table *self, int size, features type[], valtype val[], int condoffset);
+double prob_cond1(spade_prob_table *self, features type, valtype val, features ctype, valtype cval);
+double prob_cond2(spade_prob_table *self, features type, valtype val, features ctype1, valtype cval1, features ctype2, valtype cval2);
+double prob_cond3(spade_prob_table *self, features type, valtype val, features ctype1, valtype cval1, features ctype2, valtype cval2, features ctype3, valtype cval3);
+double one_prob_simple(spade_prob_table *self,features type1);
+
+double jointN_count(spade_prob_table *self,int size,features type[], valtype val[]);
+
+double spade_prob_table_entropy(spade_prob_table *self, int size, features type[], valtype val[]);
+
+void scale_and_prune_table(spade_prob_table *self, double factor, double threshold);
+
+float feature_trees_stats(spade_prob_table *self, features f, float *amind, float *amaxd, float *aaved, float *awaved);
+
+void spade_prob_table_write_stats(spade_prob_table *self,FILE *file,u8 stats_to_print);
+void write_feat_val_list(spade_prob_table *self,FILE *f, int depth, features feats[], valtype vals[]);
+void write_all_uncond_probs(spade_prob_table *self, FILE *f);
+void write_all_cond_probs(spade_prob_table *self, FILE *f);
+featcomb calc_all_entropies(spade_prob_table *self);
+void write_all_entropies(spade_prob_table *self,FILE *f, featcomb c);
+
+void print_spade_prob_table(spade_prob_table *self);
+int sanity_check_spade_prob_table(spade_prob_table *self);
+
+
+int spade_prob_table_checkpoint(statefile_ref *s,spade_prob_table *self);
+int spade_prob_table_recover(statefile_ref *s,spade_prob_table *self);
+
+
+#endif // SPADE_PROB_TABLE_H
+
+/* Id: spade_prob_table.h,v 1.1 2004/10/11 14:35:54 jonkman Exp $ */
+
+
+
+
+
+#include <stdio.h>
+
+/// a set of event conditions
+/** An event condition is a boolean condition on a spade_event; it is true
+    if the condition is satisfied and false otherwise.  All regular event
+    conditions are defined by the libspade user.  event_condition_set
+    represents a set of conditions which are simulatneously true for an
+    event.  This is presently represented as a u32 bit-mask, which is
+    operationally very efficient but which has its limits including only
+    being able to represent 32 conditions and permitting only boolean
+    conditions */
+typedef u32 event_condition_set;
+/// macro to get at event condition #n
+#define EVENT_CONDITION_NUM(n) (1 << (n-1))
+/// the special true event condition
+#define EVENT_CONDITION_TRUE 0
+/// the special false event condition
+#define EVENT_CONDITION_FALSE EVENT_CONDITION_NUM(32)
+
+/// are all conditions in the event_condition_set ref met in the testcase event_condition_set
+#define ALL_CONDS_MET(testcase,ref) (((testcase) & (ref)) == (ref))
+/// is at least one of the conditions in ref met in the testcase
+#define SOME_CONDS_MET(testcase,ref) ((testcase) & (ref))
+/// add newconds to event_condition_set var
+#define ADD_TO_CONDS(var,newconds) ((var) |= (newconds))
+/// remove remconds from the event_condition_set var
+#define REMOVE_FROM_CONDS(var,remconds) ((var) &= ~remconds)
+/// compose a condition set (returned) from N event conditions
+#define CONDS_PLUS_CONDS(cond1,cond2) (cond1 | cond2)
+#define CONDS_PLUS_2CONDS(cond1,cond2,cond3) (cond1 | cond2 | cond3)
+#define CONDS_PLUS_3CONDS(cond1,cond2,cond3,cond4) (cond1 | cond2 | cond3 | cond4)
+#define CONDS_PLUS_4CONDS(cond1,cond2,cond3,cond4,cond5) (cond1 | cond2 | cond3 | cond4 | cond5)
+#define CONDS_PLUS_5CONDS(cond1,cond2,cond3,cond4,cond5,cond6) (cond1 | cond2 | cond3 | cond4 | cond5 | cond6)
+#define CONDS_PLUS_6CONDS(cond1,cond2,cond3,cond4,cond5,cond6,cond7) (cond1 | cond2 | cond3 | cond4 | cond5 | cond6 | cond7)
+/// return the event_condition_set formed by removing cond2 from cond1
+#define CONDS_MINUS_CONDS(cond1,cond2) (cond1 & ~cond2)
+/// test if the condition set indicates false
+#define CONDS_NOT_FALSE(conds) (((conds) & EVENT_CONDITION_FALSE) == 0)
+/// return the event_condition_set formed by restricting cond1 to only those conditions in cond2
+#define ONLY_CONDS(origconds,onlyconds) ((origconds) & onlyconds)
+
+
+/// structure containing the elements of a table manager
+typedef struct _table_mgr {
+    spade_prob_table table; ///< the probability table we use
+    struct _table_mgr *next; ///< the next table manager in a linked list of them
+    time_t last_scale; ///< the last time this table was scaled/pruned
+    u32 store_count; ///< the number of events we have stored
+    
+    feature_list feats; ///< the features we store, in order
+    const char **featurenames; ///< direct-index lookup array to map features to their names
+    
+    event_condition_set conds; ///< the event conditions under which this table is used for storage
+    time_t start_time; ///< the time we started recording events
+    int scale_freq; ///< how often we scale/prune, in secs
+    double scale_factor; ///< when we scale, how much do we do so?; this is the multiplier
+    double prune_threshold;  ///< if an observation gets below this size, it gets discarded
+    int use_count; ///< how many event files are using this table manager
+} table_mgr;
+
+/// structure containing the elements on an event file
+typedef struct _evfile {
+    /// the table manager that does the storge for this event file
+    table_mgr *mgr;
+    /// this how many features deep we care about; table manager may store more for other reasons
+    int feat_depth;
+    /// the features we need stored, in order
+    feature_list calc_feats; /*!< \note maybe eventually we should maintain a user-addressable set of these */
+
+    /// the next event file in a linked list of them
+    struct _evfile *next;
+} evfile;
+
+/// the way a event_recorder user refers to a particular event file
+typedef evfile *evfile_ref;
+
+/// the representation for an instance of an event_recorder
+typedef struct {
+    /// linked list of the table managers that are defined
+    table_mgr *tables;
+    /// linked list of the event files that are defined
+    evfile *files;
+    /// the current time
+    time_t curtime;
+} event_recorder;
+
+/// function type that can be called to print the string version of a set of event conditions to a FILE *
+typedef void (*condition_printer_t)(FILE *file,event_condition_set conds);
+
+
+event_recorder *new_event_recorder(void);
+void init_event_recorder(event_recorder *self);
+
+int event_recorder_recover(event_recorder **self, statefile_ref *ref);
+int event_recorder_merge_recover(event_recorder *self, statefile_ref *ref);
+int event_recorder_checkpoint(event_recorder *self, statefile_ref *ref);
+
+evfile_ref event_recorder_new_event_file(event_recorder *self, feature_list *feats, const char **featurenames, event_condition_set conds, int scale_freq, double scale_factor, double prune_threshold, int fresh_only, feature_list *calc_feats);
+evfile_ref *event_recorder_new_event_files(event_recorder *self, int howmany, feature_list feats[], const char **featurenames, event_condition_set conds, int scale_freq, double scale_factor, double prune_threshold, int fresh_only);
+
+void event_recorder_new_time(event_recorder *self, time_t time);
+event_condition_set event_recorder_needed_conds(event_recorder *self);
+int event_recorder_new_event(event_recorder *self, spade_event *event, event_condition_set matching_conds);
+void event_recorder_prune_unused(event_recorder *self);
+
+double event_recorder_get_prob(event_recorder *self, evfile_ref eventfile, spade_event *event,int one_more);
+double event_recorder_get_condprob(event_recorder *self, evfile_ref eventfile, spade_event *event, int condcutoff,int one_more);
+double event_recorder_get_count(event_recorder *self, evfile_ref eventfile, spade_event *event, int featdepth);
+double event_recorder_get_entropy(event_recorder *self,evfile_ref eventfile,spade_event *event,int entropy_prefix_len);
+
+int event_recorder_get_store_count(event_recorder *self, evfile_ref eventfile);
+double event_recorder_get_obs_count(event_recorder *self, evfile_ref eventfile);
+
+void event_recorder_write_stats(event_recorder *self, FILE *file, u8 stats_to_print,condition_printer_t condprinter);
+
+void evfile_print_config_details(evfile_ref eventfile,FILE *f,char *indent);
+#endif // EVENT_RECORDER_H
+
+
+
+/// a structure containing the various components describing how a probability table is going to be used
+typedef struct {
+    int prodcount; ///< the number of conditional probabilities to multiply
+    feature_list *feats; ///< if prodcount > 1, the array of feature lists to use; space is dynamically allocated 
+    feature_list calc_feats; ///< if prodcount=1, the list of features for the table
+    const char **featurenames; ///< direct-index lookup array to map features to their names
+    event_condition_set conds; ///< the event conditions under which the table will be used for storage
+    int scale_freq; ///< how often the table will be scaled/pruned, in secs
+    double scale_factor; ///< when we scale, how much do we do so by
+    double prune_threshold; ///< if an observation gets below this size, it will be discarded
+} table_use_specs;
+
+/// an instance of a score calculator
+typedef struct {
+    int prodcount; ///< the number of conditional probabilities to multiply
+    evfile_ref *evfiles; ///< if prodcount > 1, array of evfile's in the event recorder storing the tables needed for each probability
+    evfile_ref evfile; ///< if we only have one prob, we store store the evfile here
+    int calc_rawscore; ///< should the raw anonamaly score be calculated
+    int calc_relscore; ///< should the relative anonamaly score be calculated
+    scorepref mainpref; ///< the preferred score type to store
+    int use_corrscore; ///< correctly compute the raw anomaly score?
+    int cond_prefix_len; ///< how far into feats should we use as the denominator for calculating the probability
+    int min_obs_prefix_len; ///< if > 0, we require a min observation, this is how far into feats the spec is for
+    double min_obs_count; ///< the minimum observation count
+    double max_entropy; ///< if >= 0, we are using a selection critea based on maximum entropy under the values of a certain feature
+    int entropy_prefix_len; ///< the depth of the run-up to the value field when using max entropy selection criterea
+    table_use_specs *evfiles_data; ///< parameters to evfiles while being set up
+    event_recorder *recorder; ///< a pointer to the event recorder where the events are stored 
+} score_calculator;
+
+score_calculator *new_score_calculator(int prodcount, feature_list prod_cond[], const char **featurenames, event_condition_set conds, int scale_freq, double scale_factor, double prune_threshold, event_recorder *recorder, feature_list *calc_feats);
+void init_score_calculator(score_calculator *self, int prodcount, feature_list prod_cond[], const char **featurenames, event_condition_set conds, int scale_freq, double scale_factor, double prune_threshold, event_recorder *recorder, feature_list *calc_feats);
+score_calculator *new_score_calculator_clear(event_recorder *recorder);
+void init_score_calculator_clear(score_calculator *self, event_recorder *recorder);
+
+void score_calculator_set_features(score_calculator *self, int prodcount, feature_list prod_cond[], feature_list *calc_feats, const char **featurenames);
+void score_calculator_set_storage_conditions(score_calculator *self, event_condition_set conds);
+void score_calculator_set_scaling(score_calculator *self, int scale_freq, double scale_factor, double prune_threshold);
+void score_calculator_init_complete(score_calculator *self);
+
+void score_calculator_set_condcutoff(score_calculator *self, int cond_prefix_len);
+void score_calculator_set_relscore(score_calculator *self, int calc_relscore, int rel_is_main);
+void score_calculator_set_rawscore(score_calculator *self, int calc_rawscore, int raw_is_main);
+void score_calculator_set_corrscore(score_calculator *self, int use_corrscore);
+void score_calculator_set_min_obs(score_calculator *self, int featlist_prefix_len, int min_obs_count);
+void score_calculator_set_low_entropy_domain(score_calculator *self, int val_prefix_len, double max_entropy);
+void score_calculator_cleanup(score_calculator *self);
+
+int score_calculator_using_corrscore(score_calculator *self);
+int score_calculator_using_relscore(score_calculator *self);
+
+score_info *score_calculator_calc_event_score(score_calculator *self, spade_event *event, score_info *storage,int *enoughobs);
+
+int score_calculator_get_store_count(score_calculator *self);
+double score_calculator_get_obs_count(score_calculator *self);
+
+void score_calculator_print_config_details(score_calculator *self,FILE *f,char *indent);
+
+/*@}*/
+#endif // SCORE_CALCULATOR_H
+
+/*********************************************************************
+packet_resp_canceller.h, distributed as part of Spade v030125.1
+Author: James Hoagland, Silicon Defense (hoagland@SiliconDefense.com)
+copyright (c) 2002 by Silicon Defense (http://www.silicondefense.com/)
+Released under GNU General Public License, see the COPYING file included
+with the distribution or http://www.silicondefense.com/spice/ for details.
+
+Please send complaints, kudos, and especially improvements and bugfixes to
+hoagland@SiliconDefense.com.  As described in GNU General Public License, no
+warranty is expressed for this program.
+*********************************************************************/
+
+#ifndef PACKET_RESP_CANCELLER_H
+#define PACKET_RESP_CANCELLER_H
+
+/*! \file packet_resp_canceller.h
+ * \ingroup netspade_layer
+ * \brief 
+ *  packet_resp_canceller.h is the header file for packet_resp_canceller.c.
+ */
+
+/*! \addtogroup netspade_layer
+    @{
+*/
+
+
+#include <time.h>
+
+/// function type for a callback for a packet response canceller to report the status of a report
+typedef void (*prc_report_status_fn)(void *context,spade_report *rpt,port_status_t status);
+
+/// a two-way link used to store reports in the packet response canceller
+typedef struct _prc_link {
+    spade_report *rpt; /*!< the report being stored */
+    struct _prc_link *ltl_next; /*!< next link in a lookup table list */
+    struct _prc_link *ttl_next; /*!< next link in a time table list */
+} prc_link;
+
+/// a list of prc_link's
+typedef struct {
+    prc_link *head;  ///< the head
+    prc_link *tail;  ///< the tail
+} prc_list;
+
+/// the packet response canceller time-based table
+typedef struct {
+    prc_list *arr; ///< an array of buckets, each holding the reports for a given second
+    int num_buckets; ///< how many buckets are there
+    time_t last_timeout; ///< when was the last time a timeout was checked for
+} prc_time_table;
+
+#define LOOKUP_TABLE1_BITS 12 ///< how many bits are in the key to the first level lookup table
+#define LOOKUP_TABLE2_BITS 8 ///< how many bits are in the key to the second level lookup tables
+#define LOOKUP_TABLE1_SIZE (1 << LOOKUP_TABLE1_BITS) ///< the number of elements in the first level lookup hash table
+#define LOOKUP_TABLE2_SIZE (1 << LOOKUP_TABLE2_BITS) ///< the number of elements in the second level lookup hash tables
+#define LOOKUP_TABLE1_MASK (LOOKUP_TABLE1_SIZE -1) ///< mask used in the hash function for the first level lookup table
+#define LOOKUP_TABLE2_MASK (LOOKUP_TABLE2_SIZE -1) ///< mask used in the hash function for the second level lookup tables
+
+/// a second level lookup table
+/** this table maps a report to a unsorted linked list of stored reports */
+typedef struct {
+    prc_link *arr[LOOKUP_TABLE2_SIZE]; ///< a second level hash table, each bucket hold a set reports
+    int num_used; ///< the number of slots in the hash table that are presently used
+} prc_lookup_table2;
+
+/// the first level lookup table
+/** this table maps a report to a second level hash table */
+typedef struct {
+    prc_lookup_table2 *arr[LOOKUP_TABLE1_SIZE]; ///< a second level hash table, each bucket holds a second level lookup table
+} prc_lookup_table;
+
+/// an instance of a packet response canceller, which implements a packet response buffer
+typedef struct {
+    prc_lookup_table lt; ///< the lookup table, used for quick access to a given report
+    prc_time_table tt;  ///< the time-based table, used for quick access to the reports from a given second
+    port_status_t timeout_implication;  ///< the implication for when a report times out
+    prc_report_status_fn status_callback;  ///< the function to call with report status
+    void *callback_context; ///< context to provide with the callback on report status
+} packet_resp_canceller;
+
+
+void init_packet_resp_canceller(packet_resp_canceller *self,int wait_secs,prc_report_status_fn status_callback,void *callback_context,port_status_t timeout_implication);
+packet_resp_canceller *new_packet_resp_canceller(int wait_secs,prc_report_status_fn status_callback,void *callback_context,port_status_t timeout_implication);
+void free_packet_resp_canceller(packet_resp_canceller *self);
+
+void packet_resp_canceller_new_time(packet_resp_canceller *self,time_t time);
+
+void packet_resp_canceller_add_report(packet_resp_canceller *self,spade_report *rpt);
+void packet_resp_canceller_note_response(packet_resp_canceller *self,port_status_t implied_status,u32 sip,u16 sport,u32 dip,u16 dport,int portless);
+
+void packet_resp_canceller_print_config_details(packet_resp_canceller *self,FILE *f,char *indent);
+
+/*@}*/
+
+#endif  /* ! PACKET_RESP_CANCELLER_H */
+
+
+/* Id: packet_resp_canceller.h,v 1.1 2004/10/11 14:35:54 jonkman Exp $ */
+
+
+
+
+
+typedef void (*netspade_exc_callback_t)(void *context,spade_report *rpt);
+typedef void (*netspade_adj_callback_t)(void *context,char *id,char *mess,int using_corrrscore);
+
+/// the type of xarg_type_t node value; either unsigned int or CIDR
+typedef enum {XARG_TYPE_UINT,XARG_TYPE_CIDR} xarg_type_t;
+
+/// an administatively excluded field value; this results from Xdips, Xsports, etc
+typedef struct _xfeatval_link {
+    features feat; ///< the field the restriction is on
+    xarg_type_t type;  /// the type of the value for this restriction
+    union {
+        int i; /// the value to exclude, if type=XARG_TYPE_UINT
+        struct {
+            u32 netip; /// the network IP address in the CIDR
+            u32 netmask;  /// the netmask in the CIDR
+        } cidr; /// the network to exclude, if type=XARG_TYPE_CIDR
+    } val;
+    struct _xfeatval_link *next; /// the next element in a linked list of this type
+} xfeatval_link;
+
+struct _netspade;
+
+/// encapsulates the state specific to a netspade detector
+typedef struct _netspade_detector {
+    /// the next netspade_detector in a linked list of them
+    struct _netspade_detector *next;
+    struct _netspade* parent;
+    /// the id for this detector
+    char *id;
+
+    /// the type of detector this is; value is from spade_detect_types.h
+    int detect_type;
+    /// the packet conditions under which this detector will calculate the score and assess anomalousness
+    event_condition_set scorecalc_conds;
+    /// should this detector ignore packets with destination IPs that are broadcast IP addresses
+    int exclude_broadcast_dip;
+    /// if the detector is doing response waiting, this is the implication of a timeout
+    port_status_t thresh_exc_port_impl;
+
+    /// the packet conditions under which this detector will store an event in the probability table
+    event_condition_set store_conds;
+    /// the packet conditions for which this detector will check for a response in the open direction
+    event_condition_set cancel_open_conds;
+    /// the packet conditions for which this detector will check for a response in the closed direction
+    event_condition_set cancel_closed_conds;
+    /// the criterea required for a anomalous event to be reported (versus dropped)
+    port_status_set_t port_report_criterea;
+    
+    /// the libspade entity which calculates anomaly scores
+    score_calculator calculator;
+    /// the libspade score manager
+    score_mgr mgr;
+    /// if non-NULL, the packet response canceller in use
+    packet_resp_canceller *canceller;
+
+    /// the shared state with the score manager
+    spade_enviro enviro;
+    /// the packet stats as of the last time the anomaly score was adjusted
+    spade_pkt_stats last_adj_stats;
+    
+    /// a linked list of reports to exclude in this detector; suppliments netspade's global list
+    xfeatval_link *rpt_exclude_list;
+
+    /// \brief the detection type we report for this detector
+    /// \note eventually, there may be more than one detection type possible from a given detector, so this will need to be generalized
+    int report_detection_type;
+    /// the string encoding the scope of this detector; used in building reports
+    char *report_scope_str; 
+} netspade_detector;
+
+/// a link in a linked list of networks
+typedef struct _ll_net {
+    u32 netaddr; ///< the network address
+    u32 netmask; ///< the network mask
+    struct _ll_net *next;  ///< the next link in a linked list of networks
+} ll_net;
+
+/// a instance of netspade
+typedef struct _netspade {
+    /// the head of a linked list of detectors contained in this netspade
+    netspade_detector *detectors;
+    /// the tail of the linked list of detectors
+    netspade_detector *detectors_tail;
+    /// the packet conditions under which we need to pass an event to our event_recorder
+    event_condition_set recorder_needed_conds;
+    /// the packet conditions under which we need to do something besides storing a packet
+    event_condition_set nonstore_conds;
+    /// the the packet conditions that we actually need to calculate for any given packet
+    event_condition_set conds_to_calc;
+
+    ll_net *homelist_head; ///< the head a linked list of home networks
+    ll_net *homelist_tail; ///< the tail in a linked list of home networks 
+
+    char *checkpoint_file; ///< the name of the file to checkpoint to
+    int checkpoint_freq; ///< the frequency (in recorded packet counts) with which to checkpoint
+
+    /// records how many things have been recorded since the last checkpoint
+    int records_since_checkpoint;
+    /// the last packet time that we passed along to the enties that need it
+    time_t last_time_forwarded;
+    
+    /// the callback to invoke when there is an anomalous event
+    netspade_exc_callback_t exc_callback;
+    /// the callback to invoke when the threshold has been adjusted in a detector, or NULL if none should be called
+    netspade_adj_callback_t adj_callback;
+    /// the netspade-user-defined context to provide with the callback
+    void *callback_context;
+    
+    /// a pointer to a routine to call to make a copy of the "native" field of a spade_event, or NULL if none is needed
+    /** this is used when a spade_event is being copied for storage in the response buffer (packet reponse canceller) */
+    event_native_copier_t pkt_native_copier_callback;
+    /// a pointer to a routine to call free a copy of the "native" field of a spade_event, or NULL if none is needed
+    /** this is used when a copied spade_event is being freed when it is being removed from the packet reponse canceller */
+    event_native_freer_t pkt_native_freer_callback;
+    
+    xfeatval_link *rpt_exclude_list; ///< a linked list of reports to exclude globally
+
+    event_recorder recorder; ///< our event recorder
+
+    u8 stats_to_print; ///< the statistics to print to the output log file
+    char *outfile; ///< the name of the output log file
+
+    int debug_level;  ///< the debug level we are at (bigger is higher/more verbose)
+    spade_msg_fn msg_callback; ///< the function to call when we have a text message for the user
+    
+    int detector_id_nonce;  ///< the default detector id for the last detector that was set up
+    unsigned long total_pkts; ///< the total number of packets passed to us
+} netspade;
+
+
+void init_netspade(netspade *self, spade_msg_fn msg_callback, int debug_level);
+int init_netspade_from_statefile(netspade *self, char *statefile, spade_msg_fn msg_callback, int debug_level);
+netspade *new_netspade(spade_msg_fn msg_callback, int debug_level);
+netspade *new_netspade_from_statefile(char *statefile, spade_msg_fn msg_callback, int debug_level,int *succ);
+
+void netspade_set_callbacks(netspade *self, void *context, netspade_exc_callback_t exc_callback, netspade_adj_callback_t adj_callback, event_native_copier_t pkt_native_copier_callback, event_native_freer_t pkt_native_freer_callback);
+void netspade_set_checkpointing(netspade *self, char *checkpoint_file, int checkpoint_freq);
+void netspade_set_homenet_from_str(netspade *self, char *homenet_str);
+void netspade_set_output_stats(netspade *self, int stats_to_print);
+void netspade_set_output_stats_from_str(netspade *self, char *str);
+int netspade_set_output(netspade *self, char *file, int stats_to_print);
+int netspade_set_output_file(netspade *self, char *file);
+void netspade_add_rpt_excludes(netspade *self,char *xsips,char *xdips,char *xsports,char *xdports);
+
+char *netspade_new_detector(netspade *self, char *str);
+
+int netspade_set_detector_scaling(netspade *self, char *detectorid, int scale_freq, double scale_factor, double prune_threshold);
+char *netspade_setup_detector_adapt_from_str(netspade *self, int adaptmode, char *str);
+int netspade_setup_detector_adapt1(netspade *self, char *detectorid, int adapttarget, time_t period, float new_obs_weight, int by_count);
+int netspade_setup_detector_adapt2(netspade *self, char *detectorid, double targetspec, double obsper, int NS, int NM, int NL);
+int netspade_setup_detector_adapt3(netspade *self, char *detectorid, double targetspec, double obsper, int NO);
+int netspade_setup_detector_advise(netspade *self, char *detectorid, int obs_size, int obs_secs);
+char *netspade_setup_detector_advise_from_str(netspade *self, char *str);
+int netspade_setup_detector_survey(netspade *self, char *detectorid, char *filename, float interval);
+char *netspade_setup_detector_survey_from_str(netspade *self, char *str);
+
+void netspade_new_pkt(netspade *self, spade_event *pkt);
+
+void netspade_dump(netspade *self);
+void netspade_cleanup(netspade *self);
+void netspade_write_log(netspade *self);
+
+char *netspade_detector_scope_str(netspade *self,char *id);
+
+int netspade_print_detector_config_details(netspade *self,FILE *f,char *id);
+
+void print_conds(event_condition_set conds);
+void print_conds_line(event_condition_set conds);
+
+/*@}*/
+
+#endif // NETSPADE_H
+/*********************************************************************
+strtok.h, distributed as part of Spade v030125.1
+Author: James Hoagland, Silicon Defense (hoagland@SiliconDefense.com)
+copyright (c) 2002 by Silicon Defense (http://www.silicondefense.com/)
+Released under GNU General Public License, see the COPYING file included
+with the distribution or http://www.silicondefense.com/spice/ for details.
+
+Please send complaints, kudos, and especially improvements and bugfixes to
+hoagland@SiliconDefense.com.  As described in GNU General Public License, no
+warranty is expressed for this program.
+*********************************************************************/
+
+#ifndef STRTOK_H
+#define STRTOK_H
+
+/*! \file strtok.h
+ * \brief
+ *  strtok.h is the header file for strtok.c.
+ * \ingroup libspade_util
+ */
+
+/*! \addtogroup libspade_util
+    @{
+*/
+
+
+
+int fill_args_space_sep(char *str,char *format,void *args[],spade_msg_fn msg_callback);
+char *extract_str_arg_space_sep(char *str,char *argname);
+int terminate_first_tok(char *str,char *sepchars,char **head,char *oldchar);
+
+/*@}*/
+#endif // STRTOK_H
+
+/* Id: strtok.h,v 1.1 2004/10/11 14:35:54 jonkman Exp $ */