/*
 * This file is part of libswish3
 * Copyright (C) 2010 Peter Karman
 *
 *  libswish3 is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  libswish3 is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with libswish3; if not, write to the Free Software
 *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 */
 /**
  * This file is automatically generated from the individual src .c and .h files
  * that are part of the libswish3 distribution. The guiding purpose of this file
  * is to allow for easier distribution in language bindings. See the
  * bindings/perl/3.xs file in the libswish3 distribution for one example.
  */

#ifdef HAVE_STDLIB_H
#include <stdlib.h>
#endif
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#include <stdio.h>
#include <locale.h>
#include <stdarg.h>
#include <assert.h>
#include <wchar.h>
#include <limits.h>
#include <err.h> /* conflicts with Perl err.h ? */
#include <errno.h>
#include <string.h>
#include <ctype.h>
#include <wctype.h>
#include <dirent.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <stdint.h>
#include <inttypes.h>
#include <time.h>

#if defined (HAVE_GETRUSAGE) && defined (HAVE_SYS_RESOURCE_H)
#include <sys/time.h>
#include <sys/resource.h>
#endif

#ifdef HAVE_TIMES
#ifdef HAVE_SYS_PARAM_H
#include <sys/param.h>
#endif
#include <sys/times.h>
#endif

#include <zlib.h>

#include <libxml/parserInternals.h>
#include <libxml/parser.h>
#include <libxml/hash.h>
#include <libxml/xmlstring.h>
#include <libxml/HTMLparser.h>
#include <libxml/globals.h>
#include <libxml/xmlerror.h>
#include <libxml/tree.h>
#include <libxml/debugXML.h>
#include <libxml/xmlmemory.h>
#include <libxml/xmlreader.h>
#include <libxml/xmlwriter.h>
#include <libxml/encoding.h>


#define LIBSWISH3_SINGLE_FILE 1



/*************** start libswish3.h ************/
/*
 * This file is part of libswish3
 * Copyright (C) 2007 Peter Karman
 *
 *  libswish3 is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  libswish3 is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with libswish3; if not, write to the Free Software
 *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 */


#ifndef __LIBSWISH3_H__
#define __LIBSWISH3_H__

#ifndef LIBSWISH3_SINGLE_FILE
#include <sys/types.h>
#include <stdint.h>
#include <inttypes.h>
#include <sys/stat.h>
#include <time.h>
#include <libxml/parser.h>
#include <libxml/hash.h>
#include <libxml/xmlstring.h>
#endif

#define SWISH_LIB_VERSION           "0.1.2968"
#define SWISH_VERSION               "3.0.0"
#define SWISH_BUFFER_CHUNK_SIZE     16384
#define SWISH_TOKEN_LIST_SIZE       1024
#define SWISH_MAXSTRLEN             2048
#define SWISH_MAX_HEADERS           6
#define SWISH_RD_BUFFER_SIZE        65536   // used ??
#define SWISH_MAX_WORD_LEN          256
#define SWISH_MIN_WORD_LEN          1
#define SWISH_STACK_SIZE            255  /* starting size for metaname/tag stack */
#define SWISH_CONTRACTIONS          1
#define SWISH_SPECIAL_ARG           1
#define SWISH_MAX_SORT_STRING_LEN   100
#define SWISH_TRUE                  1
#define SWISH_FALSE                 0

#define SWISH_DATE_FORMAT_STRING    "%Y-%m-%d %H:%M:%S %Z"
#define SWISH_URL_LENGTH            255

/* default config hash key names */
#define SWISH_HEADER_ROOT           "swish"
#define SWISH_INCLUDE_FILE          "IncludeConfigFile"
#define SWISH_CLASS_ATTRIBUTES      "XMLClassAttributes"
#define SWISH_PROP                  "PropertyNames"
#define SWISH_META                  "MetaNames"
#define SWISH_MIME                  "MIME"
#define SWISH_PARSERS               "Parsers"
#define SWISH_INDEX                 "Index"
#define SWISH_ALIAS                 "TagAlias"
#define SWISH_WORDS                 "Words"
#define SWISH_DEFAULT_PARSER        "default"
#define SWISH_PARSER_TXT            "TXT"
#define SWISH_PARSER_XML            "XML"
#define SWISH_PARSER_HTML           "HTML"
#define SWISH_DEFAULT_PARSER_TYPE   "HTML"
#define SWISH_INDEX_FORMAT          "Format"
#define SWISH_INDEX_NAME            "Name"
#define SWISH_INDEX_LOCALE          "Locale"
#define SWISH_INDEX_STEMMER_LANG    "Stemmer"
#define SWISH_DEFAULT_VALUE         "1"
#define SWISH_TOKENIZE              "Tokenize"
#define SWISH_CASCADE_META_CONTEXT  "CascadeMetaContext"
#define SWISH_IGNORE_XMLNS          "IgnoreXMLNameSpaces"

/* tags */
#define SWISH_DEFAULT_METANAME    "swishdefault"
#define SWISH_TITLE_METANAME      "swishtitle"
#define SWISH_TITLE_TAG           "title"
#define SWISH_BODY_TAG            "body"

/* mimes */
#define SWISH_DEFAULT_MIME        "text/html"

/* indexes */
#define SWISH_INDEX_FILENAME      "index.swish"
#define SWISH_XAPIAN_FORMAT       "Xapian"
#define SWISH_SWISH_FORMAT        "Native"
#define SWISH_ESTRAIER_FORMAT     "Estraier"
#define SWISH_KINOSEARCH_FORMAT   "KinoSearch"
#define SWISH_INDEX_FILEFORMAT    "Native"
#define SWISH_HEADER_FILE         "swish.xml"

/* properties */
#define SWISH_PROP_STRING          1
#define SWISH_PROP_DATE            2
#define SWISH_PROP_INT             3

#define SWISH_PROP_RECCNT          "swishreccount"
#define SWISH_PROP_RANK            "swishrank"
#define SWISH_PROP_DOCID           "swishfilenum"
#define SWISH_PROP_DOCPATH         "swishdocpath"
#define SWISH_PROP_DBFILE          "swishdbfile"
#define SWISH_PROP_TITLE           "swishtitle"
#define SWISH_PROP_SIZE            "swishdocsize"
#define SWISH_PROP_MTIME           "swishlastmodified"
#define SWISH_PROP_DESCRIPTION     "swishdescription"
#define SWISH_PROP_MIME            "swishmime"
#define SWISH_PROP_PARSER          "swishparser"
#define SWISH_PROP_NWORDS          "swishwordnum"
#define SWISH_PROP_ENCODING        "swishencoding"
#define SWISH_TOKENPOS_BUMPER      "\3"
#define SWISH_DOT                  '.'
#define SWISH_SPACE                ' '
#define SWISH_DOM_CHAR             '/'
#define SWISH_DOM_STR              "/"
#define SWISH_XMLNS_CHAR           ':'

/* built-in id values */
typedef enum {
    SWISH_META_DEFAULT_ID = 0,
    SWISH_META_TITLE_ID,
    SWISH_META_THIS_MUST_COME_LAST_ID
} SWISH_META_ID;

/* special since not stored */
#define SWISH_PROP_RANK_ID  -1
typedef enum {
    SWISH_PROP_DOCID_ID = 0,
    SWISH_PROP_DOCPATH_ID,
    SWISH_PROP_DBFILE_ID,
    SWISH_PROP_TITLE_ID,
    SWISH_PROP_SIZE_ID,
    SWISH_PROP_MTIME_ID,
    SWISH_PROP_DESCRIPTION_ID,
    SWISH_PROP_NWORDS_ID,
    SWISH_PROP_MIME_ID,
    SWISH_PROP_PARSER_ID,
    SWISH_PROP_ENCODING_ID,
    SWISH_PROP_THIS_MUST_COME_LAST_ID
} SWISH_PROP_ID;

/* xapian (maybe others) need string prefixes for metanames */
#define SWISH_PREFIX_URL            "U"
#define SWISH_PREFIX_MTIME          "T"


/* utils */
#define SWISH_MAX_WORD_LEN        256
#define SWISH_MAX_FILE_LEN        102400000 /* ~100 mb */

#if defined(WIN32) && !defined (__CYGWIN__)
#define SWISH_PATH_SEP             '\\'
#define SWISH_EXT_SEP              "\\."
#else
#define SWISH_PATH_SEP             '/'
#define SWISH_EXT_SEP              "/."
#endif

#define SWISH_EXT_CH               '.'

/* encodings */
#define SWISH_DEFAULT_ENCODING    "UTF-8"
#define SWISH_LATIN1_ENCODING     "ISO8859-1"
#define SWISH_LOCALE              "en_US.UTF-8"
#define SWISH_ENCODING_ERROR      100

/* debugging levels */
typedef enum {
    SWISH_DEBUG_DOCINFO     = 1,
    SWISH_DEBUG_TOKENIZER   = 2,
    SWISH_DEBUG_TOKENLIST   = 4,
    SWISH_DEBUG_PARSER      = 8,
    SWISH_DEBUG_CONFIG      = 16,
    SWISH_DEBUG_MEMORY      = 32,
    SWISH_DEBUG_NAMEDBUFFER = 64,
    SWISH_DEBUG_IO          = 128
} SWISH_DEBUG_LEVELS;

/* the FUNCTION__ logic below first appeared in Perl 5.8.8
 * mostly it is for Win32 compat
 */
#ifndef FUNCTION__
#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) || (defined(__SUNPRO_C))
/* C99 or close enough. */
#  define FUNCTION__ __func__
#else
#  if (defined(_MSC_VER) && _MSC_VER < 1300) || /* Pre-MSVC 7.0 has neither __func__ nor
 __FUNCTION and no good workarounds, either. */ \
      (defined(__DECC_VER)) /* Tru64 or VMS, and strict C89 being used, but not modern e
nough cc (in Tur64, -c99 not known, only -std1). */
#    define FUNCTION__ ""
#  else
#    define FUNCTION__ __FUNCTION__ /* Common extension. */
#  endif
#endif
#endif

#define SWISH_DEBUG_MSG(args...)                                    \
    swish_debug(__FILE__, __LINE__, FUNCTION__, args)

#define SWISH_CROAK(args...)                                        \
    swish_croak(__FILE__, __LINE__, FUNCTION__, args)

#define SWISH_WARN(args...)                                         \
    swish_warn(__FILE__, __LINE__, FUNCTION__, args)

#ifdef __cplusplus
extern "C" {
#endif

typedef char   boolean;
typedef struct swish_3                  swish_3;
typedef struct swish_StringList         swish_StringList;
typedef struct swish_Config             swish_Config;
typedef struct swish_ConfigFlags        swish_ConfigFlags;
typedef struct swish_ConfigValue        swish_ConfigValue;
typedef struct swish_DocInfo            swish_DocInfo;
typedef struct swish_MetaStackElement   swish_MetaStackElement;
typedef struct swish_MetaStackElement  *swish_MetaStackElementPtr;
typedef struct swish_MetaStack          swish_MetaStack;
typedef struct swish_MetaName           swish_MetaName;
typedef struct swish_Property           swish_Property;
typedef struct swish_Token              swish_Token;
typedef struct swish_TokenList          swish_TokenList;
typedef struct swish_TokenIterator      swish_TokenIterator;
typedef struct swish_ParserData         swish_ParserData;
typedef struct swish_Tag                swish_Tag;
typedef struct swish_TagStack           swish_TagStack;
typedef struct swish_Analyzer           swish_Analyzer;
typedef struct swish_Parser             swish_Parser;
typedef struct swish_NamedBuffer        swish_NamedBuffer;

/*
=head2 Data Structures
*/

struct swish_3
{
    int             ref_cnt;
    void           *stash;
    swish_Config   *config;
    swish_Analyzer *analyzer;
    swish_Parser   *parser;
};

struct swish_StringList
{
    unsigned int    n;
    unsigned int    max;
    xmlChar**       word;
};


struct swish_Config
{
    int                          ref_cnt;
    void                        *stash;      /* for bindings */
    xmlHashTablePtr              misc;
    xmlHashTablePtr              properties;
    xmlHashTablePtr              metanames;
    xmlHashTablePtr              tag_aliases;
    xmlHashTablePtr              parsers;
    xmlHashTablePtr              mimes;
    xmlHashTablePtr              index;
    xmlHashTablePtr              stringlists;
    struct swish_ConfigFlags    *flags;      /* shortcuts for parsing */
};

struct swish_ConfigFlags
{
    boolean         tokenize;
    boolean         cascade_meta_context;
    boolean         ignore_xmlns;
    xmlHashTablePtr meta_ids;
    xmlHashTablePtr prop_ids;
    //xmlHashTablePtr contexts;
};

struct swish_NamedBuffer
{
    int             ref_cnt;    /* for bindings */
    void           *stash;      /* for bindings */
    xmlHashTablePtr hash;       /* the meat */
};

struct swish_DocInfo
{
    time_t              mtime;
    off_t               size;
    xmlChar *           mime;
    xmlChar *           encoding;
    xmlChar *           uri;
    unsigned int        nwords;
    xmlChar *           ext;
    xmlChar *           parser;
    xmlChar *           action;
    boolean             is_gzipped;
    int                 ref_cnt;
};

struct swish_MetaName
{
    int                 ref_cnt;
    int                 id;
    xmlChar            *name;
    int                 bias;
    xmlChar            *alias_for;
};

struct swish_Property
{
    int                 ref_cnt;
    int                 id;
    xmlChar            *name;
    boolean             ignore_case;
    int                 type;
    boolean             verbatim;
    xmlChar            *alias_for;
    unsigned int        max;
    boolean             sort;
    boolean             presort;
    unsigned int        sort_length;
};

struct swish_Token
{
    unsigned int        pos;            // this token's position in document
    swish_MetaName     *meta;
    xmlChar            *value;
    xmlChar            *context;
    unsigned int        offset;
    unsigned int        len;
    int                 ref_cnt;
};

struct swish_TokenList
{
    unsigned int        n;
    unsigned int        pos;            // track position in document
    xmlHashTablePtr     contexts;       // cache contexts
    xmlBufferPtr        buf;
    swish_Token**       tokens;
    int                 ref_cnt;
};

struct swish_TokenIterator
{
    swish_TokenList     *tl;
    swish_Analyzer      *a;
    unsigned int         pos;           // position in iteration
    int                  ref_cnt;
};

struct swish_Tag
{
    xmlChar            *raw;            // tag as libxml2 sees it
    xmlChar            *baked;          // tag as libswish3 sees it
    xmlChar            *context;
    struct swish_Tag   *next;
    unsigned int        n;
};

struct swish_TagStack
{
    swish_Tag         *head;
    swish_Tag         *temp;
    unsigned int       count;
    char              *name;       // debugging aid -- name of the stack
};

struct swish_Analyzer
{
    unsigned int           maxwordlen;         // max word length
    unsigned int           minwordlen;         // min word length
    boolean                tokenize;           // should we parse into TokenList
    int                  (*tokenizer) (swish_TokenIterator*, xmlChar*, swish_MetaName*, xmlChar*);
    xmlChar*             (*stemmer)   (xmlChar*);
    boolean                lc;                 // should tokens be lowercased
    void                  *stash;              // for script bindings
    void                  *regex;              // optional regex
    int                    ref_cnt;            // for script bindings
};

struct swish_Parser
{
    int                    ref_cnt;             // for script bindings
    void                 (*handler)(swish_ParserData*); // handler reference
    void                  *stash;               // for script bindings
    int                    verbosity;           
};

struct swish_ParserData
{
    swish_3               *s3;                 // main object
    xmlBufferPtr           meta_buf;           // tmp MetaName buffer
    xmlBufferPtr           prop_buf;           // tmp Property buffer
    xmlChar               *tag;                // current tag name
    swish_DocInfo         *docinfo;            // document-specific properties
    boolean                no_index;           // toggle flag. should buffer be indexed.
    boolean                is_html;            // shortcut flag for html parser
    boolean                bump_word;          // boolean for moving word position/adding space
    unsigned int           offset;             // current offset position
    swish_TagStack        *metastack;          // stacks for tracking the tag => metaname
    swish_TagStack        *propstack;          // stacks for tracking the tag => property
    swish_TagStack        *domstack;           // stacks for tracking xml/html dom tree
    xmlParserCtxtPtr       ctxt;               // so we can free at end
    swish_TokenIterator   *token_iterator;     // token container
    swish_NamedBuffer     *properties;         // buffer all properties
    swish_NamedBuffer     *metanames;          // buffer all metanames
};

/*
=cut
*/

/*
=head2 Global Functions
*/
void            swish_setup();
const char *    swish_lib_version();
const char *    swish_libxml2_version();
/*
=cut
*/

/*
=head2 Top-Level Functions
*/
swish_3 *       swish_3_init( void (*handler) (swish_ParserData *), void *stash );
void            swish_3_free( swish_3 *s3 );
int             swish_parse_file( swish_3 * s3, xmlChar *filename);
unsigned int    swish_parse_fh( swish_3 * s3, FILE * fh);
int             swish_parse_buffer( swish_3 * s3, xmlChar * buf);
unsigned int    swish_parse_directory( swish_3 *s3, xmlChar *dir, boolean follow_symlinks );
/*
=cut
*/

/*
=head2 I/O Functions
*/
xmlChar *   swish_io_slurp_fh( FILE * fh, unsigned long flen, boolean binmode );
xmlChar *   swish_io_slurp_file_len( xmlChar *filename, off_t flen, boolean binmode );
xmlChar *   swish_io_slurp_gzfile_len( xmlChar *filename, off_t *flen, boolean binmode );
xmlChar *   swish_io_slurp_file( xmlChar *filename, off_t flen, boolean is_gzipped, boolean binmode );
long int    swish_io_count_operable_file_lines( xmlChar *filename );
boolean     swish_io_is_skippable_line( xmlChar *str );
/*
=cut
*/

/*
=head2 Filesystem Functions
*/
boolean     swish_fs_file_exists( xmlChar *filename );
boolean     swish_fs_is_dir( xmlChar *path );
boolean     swish_fs_is_file( xmlChar *path );
boolean     swish_fs_is_link( xmlChar *path );
off_t       swish_fs_get_file_size( xmlChar *path );
time_t      swish_fs_get_file_mtime( xmlChar *path );
xmlChar *   swish_fs_get_file_ext( xmlChar *url );
boolean     swish_fs_looks_like_gz( xmlChar *file );
/*
=cut
*/


/*
=head2 Hash Functions
*/
int         swish_hash_add( xmlHashTablePtr hash, xmlChar *key, void * value );
int         swish_hash_replace( xmlHashTablePtr hash, xmlChar *key, void *value );
int         swish_hash_delete( xmlHashTablePtr hash, xmlChar *key );
boolean     swish_hash_exists( xmlHashTablePtr hash, xmlChar *key );
int         swish_hash_exists_or_add( xmlHashTablePtr hash, xmlChar *key, xmlChar *value );
void        swish_hash_merge( xmlHashTablePtr hash1, xmlHashTablePtr hash2 );
void *      swish_hash_fetch( xmlHashTablePtr hash, xmlChar *key );
void        swish_hash_dump( xmlHashTablePtr hash, const char *label );
xmlHashTablePtr swish_hash_init(int size);
void        swish_hash_free( xmlHashTablePtr hash );
/*
=cut
*/

/*
=head2 Memory Functions
*/
void        swish_mem_init();
void *      swish_xrealloc(void *ptr, size_t size);
void *      swish_xmalloc( size_t size );
void        swish_xfree( void *ptr );
void        swish_mem_debug();
long int    swish_memcount_get();
void        swish_memcount_dec();
xmlChar *   swish_xstrdup( const xmlChar * ptr );
xmlChar *   swish_xstrndup( const xmlChar * ptr, int len );
/*
=cut
*/

/*
=head2 Time Functions
*/
double      swish_time_elapsed(void);
double      swish_time_cpu(void);
char *      swish_time_print(double time);
char *      swish_time_print_fine(double time);
char *      swish_time_format(time_t epoch);
/*
=cut
*/

/*
=head2 Error Functions
*/
void        swish_set_error_handle( FILE *where );
void        swish_croak(const char *file, int line, const char *func, const char *msg,...);
void        swish_warn(const char *file, int line, const char *func, const char *msg,...);
void        swish_debug(const char *file, int line, const char *func, const char *msg,...);
/*
=cut
*/

/*
=head2 String Functions
*/
char *              swish_get_locale();
void                swish_verify_utf8_locale();
boolean             swish_is_ascii( xmlChar *str );
int                 swish_bytes_in_wchar( int wchar );
int                 swish_utf8_chr_len( xmlChar *utf8 );
uint32_t            swish_utf8_codepoint( xmlChar *utf8 );
int                 swish_utf8_num_chrs( xmlChar *utf8 );
void                swish_utf8_next_chr( xmlChar *s, int *i );
void                swish_utf8_prev_chr( xmlChar *s, int *i );
xmlChar *           swish_str_escape_utf8( xmlChar *utf8 );
xmlChar *           swish_str_unescape_utf8( xmlChar *ascii );
wchar_t *           swish_locale_to_wchar(xmlChar * str);
xmlChar *           swish_wchar_to_locale(wchar_t * str);
wchar_t *           swish_wstr_tolower(wchar_t *s);
xmlChar *           swish_str_tolower(xmlChar *s );
xmlChar *           swish_utf8_str_tolower(xmlChar *s);
xmlChar *           swish_ascii_str_tolower(xmlChar *s);
xmlChar *           swish_str_skip_ws(xmlChar *s);
void                swish_str_trim_ws(xmlChar *string);
void                swish_str_ctrl_to_ws(xmlChar *s);
boolean             swish_str_all_ws(xmlChar * s);
boolean             swish_str_all_ws_len(xmlChar * s, int len);
void                swish_debug_wchars( const wchar_t * widechars );
int                 swish_wchar_t_comp(const void *s1, const void *s2);
int                 swish_sort_wchar(wchar_t *s);
swish_StringList *  swish_stringlist_build(xmlChar *line);
swish_StringList *  swish_stringlist_init();
void                swish_stringlist_free(swish_StringList *sl);
unsigned int        swish_stringlist_add_string(swish_StringList *sl, xmlChar *str);
void                swish_stringlist_merge(swish_StringList *sl1, swish_StringList *sl2);
swish_StringList *  swish_stringlist_copy(swish_StringList *sl);
swish_StringList *  swish_stringlist_parse_sort_string(xmlChar *sort_string, swish_Config *cfg);
void                swish_stringlist_debug(swish_StringList *sl);
int                 swish_string_to_int( char *buf );
boolean             swish_string_to_boolean( char *buf );
xmlChar *           swish_int_to_string( int val );
xmlChar *           swish_long_to_string( long val );
xmlChar *           swish_double_to_string( double val );
xmlChar *           swish_date_to_string( int y, int m, int d );
char                swish_get_C_escaped_char(xmlChar *s, xmlChar **se);
/*
=cut
*/


/*
=head2 Configuration Functions
*/
swish_Config *      swish_config_init();
void                swish_config_set_default( swish_Config *config );
void                swish_config_merge( swish_Config *config1, swish_Config *config2 );
swish_Config *      swish_config_add( swish_Config * config, xmlChar * conf );
swish_Config *      swish_config_parse( swish_Config * config, xmlChar * conf );
void                swish_config_debug( swish_Config * config );
void                swish_config_free( swish_Config * config);
xmlHashTablePtr     swish_mime_defaults();
xmlChar *           swish_mime_get_type( swish_Config * config, xmlChar * fileext );
xmlChar *           swish_mime_get_parser( swish_Config * config, xmlChar *mime );
void                swish_config_test_alias_fors( swish_Config *c );
swish_ConfigFlags * swish_config_flags_init();
void                swish_config_flags_free( swish_ConfigFlags *flags );
void                swish_config_test_alias_fors( swish_Config *config );
void                swish_config_test_unique_ids( swish_Config *config );

/*
=cut
*/

/*
=head2 Parser Functions
*/
swish_Parser *  swish_parser_init( void (*handler) (swish_ParserData *) );
void            swish_parser_free( swish_Parser * parser );
/*
=cut
*/

/*
=head2 Token Functions 
*/
swish_TokenList *   swish_token_list_init();
void                swish_token_list_free( swish_TokenList *tl );
int                 swish_token_list_add_token(    
                                        swish_TokenList *tl, 
                                        xmlChar *token,
                                        int token_len,
                                        swish_MetaName *meta,
                                        xmlChar *context );
int                 swish_token_list_set_token(
                                        swish_TokenList *tl,
                                        xmlChar *token,
                                        int len );
swish_Token *       swish_token_init();
void                swish_token_free( swish_Token *t );
swish_TokenIterator *swish_token_iterator_init( swish_Analyzer *a );
void                swish_token_iterator_free( swish_TokenIterator *ti );
swish_Token *       swish_token_iterator_next_token( swish_TokenIterator *it );
int                 swish_tokenize(     swish_TokenIterator *ti, 
                                        xmlChar *buf, 
                                        swish_MetaName *meta,
                                        xmlChar *context );
int                 swish_tokenize_ascii(    
                                        swish_TokenIterator *ti, 
                                        xmlChar *buf, 
                                        swish_MetaName *meta,
                                        xmlChar *context );
int                 swish_tokenize_utf8(    
                                        swish_TokenIterator *ti, 
                                        xmlChar *buf, 
                                        swish_MetaName *meta,
                                        xmlChar *context );
void                swish_token_list_debug( swish_TokenIterator *it );
xmlChar *           swish_token_list_get_token_value( swish_TokenList *tl, swish_Token *t );
void                swish_token_debug( swish_Token *t );

/*
=cut
*/

/*
=head2 Analyzer Functions
*/
swish_Analyzer *    swish_analyzer_init( swish_Config * config );
void                swish_analyzer_free( swish_Analyzer * analyzer );
/*
=cut
*/

/*
=head2 DocInfo Functions
*/
swish_DocInfo *     swish_docinfo_init();
void                swish_docinfo_free( swish_DocInfo * ptr );
int                 swish_docinfo_check(swish_DocInfo * docinfo, swish_Config * config);
int                 swish_docinfo_from_filesystem(  xmlChar *filename, 
                                                    swish_DocInfo * i, 
                                                    swish_ParserData *parser_data );
void                swish_docinfo_debug( swish_DocInfo * docinfo );
/*
=cut
*/

/*
=head2 Buffer Functions
*/
swish_NamedBuffer * swish_nb_init( xmlHashTablePtr confhash );
void                swish_nb_free( swish_NamedBuffer * nb );
void                swish_nb_debug( swish_NamedBuffer * nb, xmlChar * label );
void                swish_nb_add_buf( swish_NamedBuffer *nb, 
                                      xmlChar * name,
                                      xmlBufferPtr buf, 
                                      xmlChar * joiner,
                                      boolean cleanwsp,
                                      boolean autovivify);
void                swish_nb_add_str(   swish_NamedBuffer * nb, 
                                        xmlChar * name, 
                                        xmlChar * str,
                                        unsigned int len,
                                        xmlChar * joiner,
                                        boolean cleanwsp,
                                        boolean autovivify);
void                swish_buffer_append( xmlBufferPtr buf, xmlChar * txt, int len );
xmlChar*            swish_nb_get_value( swish_NamedBuffer* nb, xmlChar* key );
/*
=cut
*/

/*
=head2 Property Functions
*/
swish_Property *    swish_property_init( xmlChar *propname );
void                swish_property_free( swish_Property *prop );
void                swish_property_debug( swish_Property *prop );
int                 swish_property_get_id( xmlChar *propname, xmlHashTablePtr properties );
/*
=cut
*/

/*
=head2 MetaName Functions
*/
swish_MetaName *    swish_metaname_init( xmlChar *name);
void                swish_metaname_free( swish_MetaName *m );
void                swish_metaname_debug( swish_MetaName *m );
/*
=cut
*/

/*
=head2 Header Functions
*/
boolean             swish_header_validate(char *filename);
boolean             swish_header_merge(char *filename, swish_Config *c);
swish_Config *      swish_header_read(char *filename);
void                swish_header_write(char* filename, swish_Config* config);
/*
=cut
*/


#ifdef __cplusplus
}
#endif
#endif /* ! __LIBSWISH3_H__ */


/*************** end libswish3.h ************/


/*************** start getruntime.h ************/
/*
**
$Id: getruntime.h,v 1.6 2005/05/12 15:41:05 karman Exp $

    This file is part of Swish-e.

    Swish-e is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 2 of the License, or
    (at your option) any later version.

    Swish-e is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along  with Swish-e; if not, write to the Free Software
    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
    
    See the COPYING file that accompanies the Swish-e distribution for details
    of the GNU GPL and the special exception available for linking against
    the Swish-e library.
    
** Mon May  9 18:19:34 CDT 2005
** added GPL


**-------------------------------------------------------
**
**
*/


#ifndef GETRUNTIME_H
#define GETRUNTIME_H 1

#ifdef __cplusplus
extern "C" {
#endif

typedef double cpu_seconds;
cpu_seconds get_cpu_secs ();

#ifdef __cplusplus
}
#endif /* __cplusplus */

#endif /* GETRUNTIME_H */



/*************** end getruntime.h ************/


/*************** start getruntime.c ************/
/* Return time used so far, in microseconds.
   Copyright (C) 1994, 1999 Free Software Foundation, Inc.

This file is part of the libiberty library.
Libiberty is free software; you can redistribute it and/or
modify it under the terms of the GNU Library General Public
License as published by the Free Software Foundation; either
version 2 of the License, or (at your option) any later version.

Libiberty is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
Library General Public License for more details.

You should have received a copy of the GNU Library General Public
License along with libiberty; see the file COPYING.LIB.  If
not, write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
Boston, MA 02111-1307, USA.
*/

#ifndef LIBSWISH3_SINGLE_FILE
#include "acconfig.h"

/* For testing */
//
#undef HAVE_GETRUSAGE
//
#undef HAVE_SYS_RESOURCE_H
//
#undef HAVE_TIMES

/* There are several ways to get elapsed execution time; unfortunately no
   single way is available for all host systems, nor are there reliable
   ways to find out which way is correct for a given host. */

#include "getruntime.h"
#include <time.h>

#if defined (HAVE_GETRUSAGE) && defined (HAVE_SYS_RESOURCE_H)
#include <sys/time.h>
#include <sys/resource.h>
#endif

#ifdef HAVE_TIMES
#ifdef HAVE_SYS_PARAM_H
#include <sys/param.h>
#endif
#include <sys/times.h>
#endif

#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif

/* end #ifndef LIBSWISH3_SINGLE_FILE */
#endif

/* This is a fallback; if wrong, it will likely make obviously wrong
   results. */

#ifndef CLOCKS_PER_SEC
#define CLOCKS_PER_SEC 1
#endif

#ifdef _SC_CLK_TCK
#define GNU_HZ  sysconf(_SC_CLK_TCK)
#else
#ifdef HZ
#define GNU_HZ  HZ
#else
#ifdef CLOCKS_PER_SEC
#define GNU_HZ  CLOCKS_PER_SEC
#endif
#endif
#endif

cpu_seconds
get_cpu_secs(
)
{
#if defined (HAVE_GETRUSAGE) && defined (HAVE_SYS_RESOURCE_H)
    struct rusage rusage;
    cpu_seconds secs;

    getrusage(0, &rusage);
    secs = (cpu_seconds) (rusage.ru_utime.tv_sec + rusage.ru_stime.tv_sec);

    if (rusage.ru_utime.tv_usec > 500000)
        secs++;
    if (rusage.ru_stime.tv_usec > 500000)
        secs++;

    return secs;

#else /* ! HAVE_GETRUSAGE */
#ifdef HAVE_TIMES

/* This returns number of clock "ticks" since: */
/* In linux since boot, in BSD since 1/1/1970 */
/* Again, these are clock_t, which may overflow, but under linux it's 1/100 second so about 6000 hours */

    struct tms tms;

    times(&tms);

    return (cpu_seconds) ((tms.tms_utime + tms.tms_stime) / GNU_HZ);

#else /* ! HAVE_TIMES */
/* Fall back on clock and hope it's correctly implemented. */
/* clock() returns clock_t, which seems to be a long.  On Linux CLOCKS_PER_SEC is 10^6 */
/* so expect an overflow at about 35 minutes. */

    clock_t t = clock();
    if (t < 0)
        t = 0;

    return (cpu_seconds) (t / CLOCKS_PER_SEC);

#endif /* HAVE_TIMES */
#endif /* HAVE_GETRUSAGE */
}


/*************** end getruntime.c ************/


/*************** start utf8.c ************/
/* see http://cprogramming.com/tutorial/unicode.html 

this file is a simple UTF-8 string handling library based on the url above.
the .h and .c file have been combined and all functions labeled 'static'
so you must include utf8.c to get the library.
We include in string.c.
*/

/* http://cprogramming.com/tutorial/utf8.h */
#ifndef LIBSWISH3_SINGLE_FILE
#include <stdarg.h>
#endif

/* is c the start of a utf8 sequence? */
#define isutf(c) (((c)&0xC0)!=0x80)

/* convert UTF-8 data to wide character */
static int u8_toucs(
    uint32_t * dest,
    int sz,
    char *src,
    int srcsz
);

/* the opposite conversion */
static int u8_toutf8(
    char *dest,
    int sz,
    uint32_t * src,
    int srcsz
);

/* single character to UTF-8 */
static int u8_wc_toutf8(
    char *dest,
    uint32_t ch
);

/* character number to byte offset */
static int u8_offset(
    char *str,
    int charnum
);

/* byte offset to character number */
static int u8_charnum(
    char *s,
    int offset
);

/* return next character, updating an index variable */
static uint32_t u8_nextchar(
    char *s,
    int *i
);

/* move to next character */
static void u8_inc(
    char *s,
    int *i
);

/* move to previous character */
static void u8_dec(
    char *s,
    int *i
);

/* returns length of next utf-8 sequence */
static int u8_seqlen(
    char *s
);

/* assuming src points to the character after a backslash, read an
   escape sequence, storing the result in dest and returning the number of
   input characters processed */
static int u8_read_escape_sequence(
    char *src,
    uint32_t * dest
);

/* given a wide character, convert it to an ASCII escape sequence stored in
   buf, where buf is "sz" bytes. returns the number of characters output. */
static int u8_escape_wchar(
    char *buf,
    int sz,
    uint32_t ch
);

/* convert a string "src" containing escape sequences to UTF-8 */
static int u8_unescape(
    char *buf,
    int sz,
    char *src
);

/* convert UTF-8 "src" to ASCII with escape sequences.
   if escape_quotes is nonzero, quote characters will be preceded by
   backslashes as well. */
static int u8_escape(
    char *buf,
    int sz,
    char *src,
    int escape_quotes
);

/* utility predicates used by the above */
static int octal_digit(
    char c
);
static int hex_digit(
    char c
);

/* return a pointer to the first occurrence of ch in s, or NULL if not
   found. character index of found character returned in *charn. */
static char *u8_strchr(
    char *s,
    uint32_t ch,
    int *charn
);

/* same as the above, but searches a buffer of a given size instead of
   a NUL-terminated string. */
static char *u8_memchr(
    char *s,
    uint32_t ch,
    size_t sz,
    int *charn
);

/* count the number of characters in a UTF-8 string */
static int u8_strlen(
    char *s
);

static int u8_is_locale_utf8(
    char *locale
);

/* printf where the format string and arguments may be in UTF-8.
   you can avoid this function and just use ordinary printf() if the current
   locale is UTF-8. */
static int u8_vprintf(
    char *fmt,
    va_list ap
);
static int u8_printf(
    char *fmt,
    ...
);

/* http://cprogramming.com/tutorial/utf8.c */

/*
  Basic UTF-8 manipulation routines
  by Jeff Bezanson
  placed in the public domain Fall 2005

  This code is designed to provide the utilities you need to manipulate
  UTF-8 as an internal string encoding. These functions do not perform the
  error checking normally needed when handling UTF-8 data, so if you happen
  to be from the Unicode Consortium you will want to flay me alive.
  I do this because error checking can be performed at the boundaries (I/O),
  with these routines reserved for higher performance on data known to be
  valid.
*/
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <stdarg.h>
/*
#ifdef WIN32
#include <malloc.h>
#else
#include <alloca.h>
#endif
*/

static const uint32_t offsetsFromUTF8[6] = {
    0x00000000UL, 0x00003080UL, 0x000E2080UL,
    0x03C82080UL, 0xFA082080UL, 0x82082080UL
};

static const char trailingBytesForUTF8[256] = {
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
        0, 0, 0, 0, 0, 0, 0,
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
        0, 0, 0, 0, 0, 0, 0,
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
        0, 0, 0, 0, 0, 0, 0,
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
        0, 0, 0, 0, 0, 0, 0,
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
        0, 0, 0, 0, 0, 0, 0,
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
        0, 0, 0, 0, 0, 0, 0,
    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
        1, 1, 1, 1, 1, 1, 1,
    2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 4,
        4, 4, 4, 5, 5, 5, 5
};

/* returns length of next utf-8 sequence */
static int
u8_seqlen(
    char *s
)
{
    return trailingBytesForUTF8[(unsigned int)(unsigned char)s[0]] + 1;
}

/* conversions without error checking
   only works for valid UTF-8, i.e. no 5- or 6-byte sequences
   srcsz = source size in bytes, or -1 if 0-terminated
   sz = dest size in # of wide characters

   returns # characters converted
   dest will always be L'\0'-terminated, even if there isn't enough room
   for all the characters.
   if sz = srcsz+1 (i.e. 4*srcsz+4 bytes), there will always be enough space.
*/
static int
u8_toucs(
    uint32_t * dest,
    int sz,
    char *src,
    int srcsz
)
{
    uint32_t ch;
    char *src_end = src + srcsz;
    int nb;
    int i = 0;

    while (i < sz - 1) {
        nb = trailingBytesForUTF8[(unsigned char)*src];
        if (srcsz == -1) {
            if (*src == 0)
                goto done_toucs;
        }
        else {
            if (src + nb >= src_end)
                goto done_toucs;
        }
        ch = 0;
        switch (nb) {
            /*
               these fall through deliberately 
             */
        case 3:
            ch += (unsigned char)*src++;
            ch <<= 6;
        case 2:
            ch += (unsigned char)*src++;
            ch <<= 6;
        case 1:
            ch += (unsigned char)*src++;
            ch <<= 6;
        case 0:
            ch += (unsigned char)*src++;
        }
        ch -= offsetsFromUTF8[nb];
        dest[i++] = ch;
    }
  done_toucs:
    dest[i] = 0;
    return i;
}

/* srcsz = number of source characters, or -1 if 0-terminated
   sz = size of dest buffer in bytes

   returns # characters converted
   dest will only be '\0'-terminated if there is enough space. this is
   for consistency; imagine there are 2 bytes of space left, but the next
   character requires 3 bytes. in this case we could NUL-terminate, but in
   general we can't when there's insufficient space. therefore this function
   only NUL-terminates if all the characters fit, and there's space for
   the NUL as well.
   the destination string will never be bigger than the source string.
*/
static int
u8_toutf8(
    char *dest,
    int sz,
    uint32_t * src,
    int srcsz
)
{
    uint32_t ch;
    int i = 0;
    char *dest_end = dest + sz;

    while (srcsz < 0 ? src[i] != 0 : i < srcsz) {
        ch = src[i];
        if (ch < 0x80) {
            if (dest >= dest_end)
                return i;
            *dest++ = (char)ch;
        }
        else if (ch < 0x800) {
            if (dest >= dest_end - 1)
                return i;
            *dest++ = (ch >> 6) | 0xC0;
            *dest++ = (ch & 0x3F) | 0x80;
        }
        else if (ch < 0x10000) {
            if (dest >= dest_end - 2)
                return i;
            *dest++ = (ch >> 12) | 0xE0;
            *dest++ = ((ch >> 6) & 0x3F) | 0x80;
            *dest++ = (ch & 0x3F) | 0x80;
        }
        else if (ch < 0x110000) {
            if (dest >= dest_end - 3)
                return i;
            *dest++ = (ch >> 18) | 0xF0;
            *dest++ = ((ch >> 12) & 0x3F) | 0x80;
            *dest++ = ((ch >> 6) & 0x3F) | 0x80;
            *dest++ = (ch & 0x3F) | 0x80;
        }
        i++;
    }
    if (dest < dest_end)
        *dest = '\0';
    return i;
}

static int
u8_wc_toutf8(
    char *dest,
    uint32_t ch
)
{
    if (ch < 0x80) {
        dest[0] = (char)ch;
        return 1;
    }
    if (ch < 0x800) {
        dest[0] = (ch >> 6) | 0xC0;
        dest[1] = (ch & 0x3F) | 0x80;
        return 2;
    }
    if (ch < 0x10000) {
        dest[0] = (ch >> 12) | 0xE0;
        dest[1] = ((ch >> 6) & 0x3F) | 0x80;
        dest[2] = (ch & 0x3F) | 0x80;
        return 3;
    }
    if (ch < 0x110000) {
        dest[0] = (ch >> 18) | 0xF0;
        dest[1] = ((ch >> 12) & 0x3F) | 0x80;
        dest[2] = ((ch >> 6) & 0x3F) | 0x80;
        dest[3] = (ch & 0x3F) | 0x80;
        return 4;
    }
    return 0;
}

/* charnum => byte offset */
static int
u8_offset(
    char *str,
    int charnum
)
{
    int offs = 0;

    while (charnum > 0 && str[offs]) {
        (void)(isutf(str[++offs]) || isutf(str[++offs]) || isutf(str[++offs])
               || ++offs);
        charnum--;
    }
    return offs;
}

/* byte offset => charnum */
static int
u8_charnum(
    char *s,
    int offset
)
{
    int charnum = 0, offs = 0;

    while (offs < offset && s[offs]) {
        (void)(isutf(s[++offs]) || isutf(s[++offs]) || isutf(s[++offs])
               || ++offs);
        charnum++;
    }
    return charnum;
}

/* number of characters */
static int
u8_strlen(
    char *s
)
{
    int count = 0;
    int i = 0;

    while (u8_nextchar(s, &i) != 0)
        count++;

    return count;
}

/* reads the next utf-8 sequence out of a string, updating an index */
static uint32_t
u8_nextchar(
    char *s,
    int *i
)
{
    uint32_t ch = 0;
    int sz = 0;

    do {
        ch <<= 6;
        ch += (unsigned char)s[(*i)++];
        sz++;
    } while (s[*i] && !isutf(s[*i]));
    ch -= offsetsFromUTF8[sz - 1];

    return ch;
}

static void
u8_inc(
    char *s,
    int *i
)
{
    (void)(isutf(s[++(*i)]) || isutf(s[++(*i)]) || isutf(s[++(*i)]) || ++(*i));
}

static void
u8_dec(
    char *s,
    int *i
)
{
    (void)(isutf(s[--(*i)]) || isutf(s[--(*i)]) || isutf(s[--(*i)]) || --(*i));
}

static int
octal_digit(
    char c
)
{
    return (c >= '0' && c <= '7');
}

static int
hex_digit(
    char c
)
{
    return ((c >= '0' && c <= '9') || (c >= 'A' && c <= 'F')
            || (c >= 'a' && c <= 'f'));
}

/* assumes that src points to the character after a backslash
   returns number of input characters processed */
static int
u8_read_escape_sequence(
    char *str,
    uint32_t * dest
)
{
    uint32_t ch;
    char digs[9] = "\0\0\0\0\0\0\0\0\0";
    int dno = 0, i = 1;

    ch = (uint32_t) str[0];    /* take literal character */
    if (str[0] == 'n')
        ch = L'\n';
    else if (str[0] == 't')
        ch = L'\t';
    else if (str[0] == 'r')
        ch = L'\r';
    else if (str[0] == 'b')
        ch = L'\b';
    else if (str[0] == 'f')
        ch = L'\f';
    else if (str[0] == 'v')
        ch = L'\v';
    else if (str[0] == 'a')
        ch = L'\a';
    else if (octal_digit(str[0])) {
        i = 0;
        do {
            digs[dno++] = str[i++];
        } while (octal_digit(str[i]) && dno < 3);
        ch = strtol(digs, NULL, 8);
    }
    else if (str[0] == 'x') {
        while (hex_digit(str[i]) && dno < 2) {
            digs[dno++] = str[i++];
        }
        if (dno > 0)
            ch = strtol(digs, NULL, 16);
    }
    else if (str[0] == 'u') {
        while (hex_digit(str[i]) && dno < 4) {
            digs[dno++] = str[i++];
        }
        if (dno > 0)
            ch = strtol(digs, NULL, 16);
    }
    else if (str[0] == 'U') {
        while (hex_digit(str[i]) && dno < 8) {
            digs[dno++] = str[i++];
        }
        if (dno > 0)
            ch = strtol(digs, NULL, 16);
    }
    *dest = ch;

    return i;
}

/* convert a string with literal \uxxxx or \Uxxxxxxxx characters to UTF-8
   example: u8_unescape(mybuf, 256, "hello\\u220e")
   note the double backslash is needed if called on a C string literal */
static int
u8_unescape(
    char *buf,
    int sz,
    char *src
)
{
    int c = 0, amt;
    uint32_t ch;
    char temp[4];

    while (*src && c < sz) {
        if (*src == '\\') {
            src++;
            amt = u8_read_escape_sequence(src, &ch);
        }
        else {
            ch = (uint32_t) * src;
            amt = 1;
        }
        src += amt;
        amt = u8_wc_toutf8(temp, ch);
        if (amt > sz - c)
            break;
        memcpy(&buf[c], temp, amt);
        c += amt;
    }
    if (c < sz)
        buf[c] = '\0';
    return c;
}

static int
u8_escape_wchar(
    char *buf,
    int sz,
    uint32_t ch
)
{
    if (ch == L'\n')
        return snprintf(buf, sz, "\\n");
    else if (ch == L'\t')
        return snprintf(buf, sz, "\\t");
    else if (ch == L'\r')
        return snprintf(buf, sz, "\\r");
    else if (ch == L'\b')
        return snprintf(buf, sz, "\\b");
    else if (ch == L'\f')
        return snprintf(buf, sz, "\\f");
    else if (ch == L'\v')
        return snprintf(buf, sz, "\\v");
    else if (ch == L'\a')
        return snprintf(buf, sz, "\\a");
    else if (ch == L'\\')
        return snprintf(buf, sz, "\\\\");
    else if (ch < 32 || ch == 0x7f)
        return snprintf(buf, sz, "\\x%hhX", (unsigned char)ch);
    else if (ch > 0xFFFF)
        return snprintf(buf, sz, "\\U%.8X", (uint32_t) ch);
    else if (ch >= 0x80 && ch <= 0xFFFF)
        return snprintf(buf, sz, "\\u%.4hX", (unsigned short)ch);

    return snprintf(buf, sz, "%c", (char)ch);
}

static int
u8_escape(
    char *buf,
    int sz,
    char *src,
    int escape_quotes
)
{
    int c = 0, i = 0, amt;

    while (src[i] && c < sz) {
        if (escape_quotes && src[i] == '"') {
            amt = snprintf(buf, sz - c, "\\\"");
            i++;
        }
        else {
            amt = u8_escape_wchar(buf, sz - c, u8_nextchar(src, &i));
        }
        c += amt;
        buf += amt;
    }
    if (c < sz)
        *buf = '\0';
    return c;
}

static char *
u8_strchr(
    char *s,
    uint32_t ch,
    int *charn
)
{
    int i = 0, lasti = 0;
    uint32_t c;

    *charn = 0;
    while (s[i]) {
        c = u8_nextchar(s, &i);
        if (c == ch) {
            return &s[lasti];
        }
        lasti = i;
        (*charn)++;
    }
    return NULL;
}

static char *
u8_memchr(
    char *s,
    uint32_t ch,
    size_t sz,
    int *charn
)
{
    int i = 0, lasti = 0;
    uint32_t c;
    int csz;

    *charn = 0;
    while (i < sz) {
        c = csz = 0;
        do {
            c <<= 6;
            c += (unsigned char)s[i++];
            csz++;
        } while (i < sz && !isutf(s[i]));
        c -= offsetsFromUTF8[csz - 1];

        if (c == ch) {
            return &s[lasti];
        }
        lasti = i;
        (*charn)++;
    }
    return NULL;
}

static int
u8_is_locale_utf8(
    char *locale
)
{
    /*
       this code based on libutf8 
     */
    const char *cp = locale;

    for (; *cp != '\0' && *cp != '@' && *cp != '+' && *cp != ','; cp++) {
        if (*cp == '.') {
            const char *encoding = ++cp;
            for (; *cp != '\0' && *cp != '@' && *cp != '+' && *cp != ','; cp++);
            if ((cp - encoding == 5 && !strncmp(encoding, "UTF-8", 5))
                || (cp - encoding == 4 && !strncmp(encoding, "utf8", 4)))
                return 1;       /* it's UTF-8 */
            break;
        }
    }
    return 0;
}

static int
u8_vprintf(
    char *fmt,
    va_list ap
)
{
    int cnt, sz = 0;
    char *buf;
    uint32_t *wcs;

    sz = 512;
    buf = (char *)alloca(sz);
  try_print:
    cnt = vsnprintf(buf, sz, fmt, ap);
    if (cnt >= sz) {
        buf = (char *)alloca(cnt - sz + 1);
        sz = cnt + 1;
        goto try_print;
    }
    wcs = (uint32_t *) alloca((cnt + 1) * sizeof(uint32_t));
    cnt = u8_toucs(wcs, cnt + 1, buf, cnt);
    printf("%ls", (wchar_t *) wcs);
    return cnt;
}

static int
u8_printf(
    char *fmt,
    ...
)
{
    int cnt;
    va_list args;

    va_start(args, fmt);

    cnt = u8_vprintf(fmt, args);

    va_end(args);
    return cnt;
}


/*************** end utf8.c ************/


/*************** start config.c ************/
/*
 * This file is part of libswish3
 * Copyright (C) 2007 Peter Karman
 *
 *  libswish3 is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  libswish3 is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with libswish3; if not, write to the Free Software
 *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
*/

#ifndef LIBSWISH3_SINGLE_FILE
#include <sys/param.h>
#include <stdio.h>
#include <stdlib.h>
#include <stdarg.h>
#include <string.h>
#include <locale.h>
#include <err.h>

#include "libswish3.h"
#endif

extern int SWISH_DEBUG;

static void free_string(
    xmlChar *payload,
    xmlChar *key
);
static void free_props(
    swish_Property *prop,
    xmlChar *propname
);
static void free_metas(
    swish_MetaName *meta,
    xmlChar *metaname
);
static void config_printer(
    xmlChar *val,
    xmlChar *str,
    xmlChar *key
);
static void stringlist_printer(
    swish_StringList *strlist,
    xmlChar *str,
    xmlChar *key
);
static void property_printer(
    swish_Property *prop,
    xmlChar *str,
    xmlChar *propname
);
static void metaname_printer(
    swish_MetaName *meta,
    xmlChar *str,
    xmlChar *metaname
);
static void copy_property(
    swish_Property *prop2,
    xmlHashTablePtr props1,
    xmlChar *prop2name
);
static void merge_properties(
    xmlHashTablePtr props1,
    xmlHashTablePtr props2
);
static void copy_metaname(
    swish_MetaName *meta2,
    xmlHashTablePtr metas1,
    xmlChar *meta2name
);
static void merge_metanames(
    xmlHashTablePtr metas1,
    xmlHashTablePtr metas2
);
static void
free_stringlist(
    swish_StringList *strlist,
    xmlChar *key
);

static void
free_string(
    xmlChar *payload,
    xmlChar *key
)
{
    if (SWISH_DEBUG & SWISH_DEBUG_CONFIG)
        SWISH_DEBUG_MSG("   freeing config %s => %s", key, payload);

    swish_xfree(payload);
}

static void
free_stringlist(
    swish_StringList *strlist,
    xmlChar *key
)
{
    int i;
    if (SWISH_DEBUG & SWISH_DEBUG_CONFIG) {
        SWISH_DEBUG_MSG("   freeing config->stringlists %s [%d strings]", key, strlist->n);
        for(i=0; i<strlist->n; i++) {
            SWISH_DEBUG_MSG("     string: %s", strlist->word[i]);
        }
    }

    swish_stringlist_free(strlist);
}

static void
free_props(
    swish_Property *prop,
    xmlChar *propname
)
{
    if (SWISH_DEBUG & SWISH_DEBUG_CONFIG) {
        SWISH_DEBUG_MSG("   freeing config->prop %s", propname);
        swish_property_debug((swish_Property *)prop);
    }
    prop->ref_cnt--;
    if (prop->ref_cnt < 1) {
        swish_property_free(prop);
    }
}

static void
free_metas(
    swish_MetaName *meta,
    xmlChar *metaname
)
{
    if (SWISH_DEBUG & SWISH_DEBUG_CONFIG) {
        SWISH_DEBUG_MSG("   freeing config->meta %s", metaname);
        swish_metaname_debug((swish_MetaName *)meta);
    }
    meta->ref_cnt--;
    if (meta->ref_cnt < 1) {
        swish_metaname_free(meta);
    }
}

void
swish_config_free(
    swish_Config *config
)
{
    if (SWISH_DEBUG & SWISH_DEBUG_MEMORY) {
        SWISH_DEBUG_MSG("freeing config");
        SWISH_DEBUG_MSG("ptr addr: 0x%x  %d", (long int)config, (long int)config);
        swish_mem_debug();
    }

    xmlHashFree(config->misc, (xmlHashDeallocator)free_string);
    xmlHashFree(config->properties, (xmlHashDeallocator)free_props);
    xmlHashFree(config->metanames, (xmlHashDeallocator)free_metas);
    xmlHashFree(config->tag_aliases, (xmlHashDeallocator)free_string);
    xmlHashFree(config->parsers, (xmlHashDeallocator)free_string);
    xmlHashFree(config->mimes, (xmlHashDeallocator)free_string);
    xmlHashFree(config->index, (xmlHashDeallocator)free_string);
    xmlHashFree(config->stringlists, (xmlHashDeallocator)free_stringlist);
    swish_config_flags_free(config->flags);

    if (config->ref_cnt != 0) {
        SWISH_WARN("config ref_cnt != 0: %d", config->ref_cnt);
    }

    if (config->stash != NULL) {
        SWISH_WARN("possible memory leak: config->stash was not freed");
    }

    swish_xfree(config);
}

swish_ConfigFlags *
swish_config_init_flags(
)
{
    swish_ConfigFlags *flags;
    flags = swish_xmalloc(sizeof(swish_ConfigFlags));
    flags->tokenize = SWISH_TRUE;
    flags->cascade_meta_context = SWISH_FALSE;  /* add tokens to every metaname in the stack */
    flags->ignore_xmlns = SWISH_TRUE;
    flags->meta_ids = swish_hash_init(8);
    flags->prop_ids = swish_hash_init(8);
    //flags->contexts = swish_hash_init(8); // TODO cache these to save malloc/frees

    return flags;
}

void
swish_config_flags_free(
    swish_ConfigFlags * flags
)
{
    /*
       these hashes are for convenience and are really freed in swish_config_free() 
     */
    xmlHashFree(flags->meta_ids, NULL);
    xmlHashFree(flags->prop_ids, NULL);
    if (SWISH_DEBUG) {
        SWISH_DEBUG_MSG("config->tokenize was %d", flags->tokenize);
        SWISH_DEBUG_MSG("config->cascade_meta_context was %d", flags->cascade_meta_context);
        SWISH_DEBUG_MSG("config->ignore_xmlns was %d", flags->ignore_xmlns);
    }
    swish_xfree(flags);
}

/* init config object */
swish_Config *
swish_config_init(
)
{
    swish_Config *config;

    if (SWISH_DEBUG & SWISH_DEBUG_MEMORY) {
        SWISH_DEBUG_MSG("init config");
    }

/* the hashes will automatically grow as needed so we init with sane starting size */
    config = swish_xmalloc(sizeof(swish_Config));
    config->flags = swish_config_init_flags();
    config->misc = swish_hash_init(8);
    config->metanames = swish_hash_init(8);
    config->properties = swish_hash_init(8);
    config->parsers = swish_hash_init(8);
    config->index = swish_hash_init(8);
    config->tag_aliases = swish_hash_init(8);   /* alias => real */
    config->stringlists = swish_hash_init(8);
    config->mimes = NULL;
    config->ref_cnt = 0;
    config->stash = NULL;

    if (SWISH_DEBUG & SWISH_DEBUG_MEMORY) {
        SWISH_DEBUG_MSG("config ptr 0x%x", (long int)config);
    }

    return config;

}

void
swish_config_set_default(
    swish_Config *config
)
{
    swish_Property *tmpprop;
    swish_MetaName *tmpmeta;
    xmlChar *tmpbuf;

    if (SWISH_DEBUG & SWISH_DEBUG_CONFIG)
        SWISH_DEBUG_MSG("setting default config");

/* we xstrdup a lot in order to consistently free in swish_config_free() */

/* MIME types */
    config->mimes = swish_mime_defaults();

    if (SWISH_DEBUG & SWISH_DEBUG_CONFIG)
        SWISH_DEBUG_MSG("mime hash set");

/* metanames */
    // default
    tmpmeta = swish_metaname_init(swish_xstrdup((xmlChar *)SWISH_DEFAULT_METANAME));
    tmpmeta->ref_cnt++;
    tmpmeta->id = SWISH_META_DEFAULT_ID;
    tmpbuf = swish_int_to_string(SWISH_META_DEFAULT_ID);
    swish_hash_add(config->flags->meta_ids, tmpbuf, tmpmeta);
    swish_hash_add(config->metanames, (xmlChar *)SWISH_DEFAULT_METANAME, tmpmeta);
    swish_xfree(tmpbuf);
    if (SWISH_DEBUG & SWISH_DEBUG_CONFIG)
        SWISH_DEBUG_MSG("swishdefault metaname set");

    // title
    tmpmeta = swish_metaname_init(swish_xstrdup((xmlChar *)SWISH_TITLE_METANAME));
    tmpmeta->ref_cnt++;
    tmpmeta->id = SWISH_META_TITLE_ID;
    tmpbuf = swish_int_to_string(SWISH_META_TITLE_ID);
    swish_hash_add(config->flags->meta_ids, tmpbuf, tmpmeta);
    swish_hash_add(config->metanames, (xmlChar *)SWISH_TITLE_METANAME, tmpmeta);
    swish_xfree(tmpbuf);
    if (SWISH_DEBUG & SWISH_DEBUG_CONFIG)
        SWISH_DEBUG_MSG("swishtitle metaname set");

/* properties */
    // description
    tmpprop = swish_property_init(swish_xstrdup((xmlChar *)SWISH_PROP_DESCRIPTION));
    tmpprop->ref_cnt++;
    tmpprop->id = SWISH_PROP_DESCRIPTION_ID;
    tmpprop->sort = SWISH_FALSE;
    swish_hash_add(config->properties, (xmlChar *)SWISH_PROP_DESCRIPTION, tmpprop);
    tmpbuf = swish_int_to_string(SWISH_PROP_DESCRIPTION_ID);
    swish_hash_add(config->flags->prop_ids, tmpbuf, tmpprop);
    swish_xfree(tmpbuf);

    // title
    tmpprop = swish_property_init(swish_xstrdup((xmlChar *)SWISH_PROP_TITLE));
    tmpprop->ref_cnt++;
    tmpprop->id = SWISH_PROP_TITLE_ID;
    swish_hash_add(config->properties, (xmlChar *)SWISH_PROP_TITLE, tmpprop);
    tmpbuf = swish_int_to_string(SWISH_PROP_TITLE_ID);
    swish_hash_add(config->flags->prop_ids, tmpbuf, tmpprop);
    swish_xfree(tmpbuf);

/* parsers */
    swish_hash_add(config->parsers, (xmlChar *)"text/plain",
                   swish_xstrdup((xmlChar *)SWISH_PARSER_TXT));
    swish_hash_add(config->parsers, (xmlChar *)"application/xml",
                   swish_xstrdup((xmlChar *)SWISH_PARSER_XML));
    swish_hash_add(config->parsers, (xmlChar *)"text/html",
                   swish_xstrdup((xmlChar *)SWISH_PARSER_HTML));
    swish_hash_add(config->parsers, (xmlChar *)SWISH_DEFAULT_PARSER,
                   swish_xstrdup((xmlChar *)SWISH_DEFAULT_PARSER_TYPE));

/* index */
    swish_hash_add(config->index, (xmlChar *)SWISH_INDEX_FORMAT,
                   swish_xstrdup((xmlChar *)SWISH_INDEX_FILEFORMAT));
    swish_hash_add(config->index, (xmlChar *)SWISH_INDEX_NAME,
                   swish_xstrdup((xmlChar *)SWISH_INDEX_FILENAME));
    swish_hash_add(config->index, (xmlChar *)SWISH_INDEX_LOCALE,
                   swish_xstrdup((xmlChar *)setlocale(LC_CTYPE, NULL)));

/* aliases: other names a tag might be known as, for matching properties and
     * metanames */
    swish_hash_add(config->tag_aliases, (xmlChar *)SWISH_TITLE_TAG,
                   swish_xstrdup((xmlChar *)SWISH_TITLE_METANAME));
    swish_hash_add(config->tag_aliases, (xmlChar *)SWISH_BODY_TAG,
                   swish_xstrdup((xmlChar *)SWISH_PROP_DESCRIPTION));

    if (SWISH_DEBUG & SWISH_DEBUG_CONFIG) {
        SWISH_DEBUG_MSG("config_set_default done");
        swish_config_debug(config);
    }

}

swish_Config *
swish_config_add(
    swish_Config *config,
    xmlChar *conf
)
{
    config = swish_config_parse(config, conf);
    if (SWISH_DEBUG & SWISH_DEBUG_CONFIG)
        swish_config_debug(config);

    return config;
}

swish_Config *
swish_config_parse(
    swish_Config *config,
    xmlChar *conf
)
{
    swish_header_merge((char *)conf, config);
    return config;
}

static void
config_printer(
    xmlChar *val,
    xmlChar *str,
    xmlChar *key
)
{
    SWISH_DEBUG_MSG(" %s:  %s => %s", str, key, val);
}

static void 
stringlist_printer(
    swish_StringList *strlist,
    xmlChar *str,
    xmlChar *key
)
{
    int i;
    for(i=0; i<strlist->n; i++) {
        SWISH_DEBUG_MSG(" %s: %s => %s", str, key, strlist->word[i]);
    }
}

static void
property_printer(
    swish_Property *prop,
    xmlChar *str,
    xmlChar *propname
)
{
    SWISH_DEBUG_MSG(" %s:  %s =>", str, propname);
    swish_property_debug(prop);
}

static void
metaname_printer(
    swish_MetaName *meta,
    xmlChar *str,
    xmlChar *metaname
)
{
    SWISH_DEBUG_MSG(" %s:  %s =>", str, metaname);
    swish_metaname_debug(meta);
}

/* PUBLIC */
void
swish_config_debug(
    swish_Config *config
)
{
    SWISH_DEBUG_MSG("config->ref_cnt = %d", config->ref_cnt);
    SWISH_DEBUG_MSG("config->stash address = 0x%x  %d", (long int)config->stash,
                    (long int)config->stash);
    SWISH_DEBUG_MSG("ptr addr: 0x%x  %d", (long int)config, (long int)config);

    xmlHashScan(config->misc, (xmlHashScanner)config_printer, "misc conf");
    xmlHashScan(config->stringlists, (xmlHashScanner)stringlist_printer, "stringlists");
    xmlHashScan(config->properties, (xmlHashScanner)property_printer, "properties");
    xmlHashScan(config->metanames, (xmlHashScanner)metaname_printer, "metanames");
    xmlHashScan(config->parsers, (xmlHashScanner)config_printer, "parsers");
    xmlHashScan(config->mimes, (xmlHashScanner)config_printer, "mimes");
    xmlHashScan(config->index, (xmlHashScanner)config_printer, "index");
    xmlHashScan(config->tag_aliases, (xmlHashScanner)config_printer, "tag_aliases");
}

static void
copy_property(
    swish_Property *prop2,
    xmlHashTablePtr props1,
    xmlChar *prop2name
)
{
    swish_Property *prop1;

    if (swish_hash_exists(props1, prop2name)) {
        prop1 = swish_hash_fetch(props1, prop2name);
        if (prop1->name != NULL) {
            swish_xfree(prop1->name);
            prop1->name = swish_xstrdup(prop2->name);
        }
    }
    else {
        prop1 = swish_property_init(swish_xstrdup(prop2name));
        prop1->ref_cnt++;
        swish_hash_add(props1, prop1->name, prop1);
    }
/* 
    SWISH_DEBUG_MSG("%s prop1->id = %d    %s prop2->id = %d",
                    prop1->name, prop1->id, prop2->name, prop2->id);
 */
    prop1->id = prop2->id;        
    prop1->ignore_case = prop2->ignore_case;
    prop1->type = prop2->type;
    prop1->verbatim = prop2->verbatim;
    if (prop1->alias_for != NULL) {
        swish_xfree(prop1->alias_for);
    }
    if (prop2->alias_for != NULL) {
        prop1->alias_for = swish_xstrdup(prop2->alias_for);
    }
    prop1->max = prop2->max;
    prop1->sort = prop2->sort;

}

static void
merge_properties(
    xmlHashTablePtr props1,
    xmlHashTablePtr props2
)
{
    xmlHashScan(props2, (xmlHashScanner)copy_property, props1);
}

static void
copy_metaname(
    swish_MetaName *meta2,
    xmlHashTablePtr metas1,
    xmlChar *meta2name
)
{
    swish_MetaName *meta1;
    
    if (swish_hash_exists(metas1, meta2name)) {
        meta1 = swish_hash_fetch(metas1, meta2name);
        if (meta1->name != NULL) {
            swish_xfree(meta1->name);
            meta1->name = swish_xstrdup(meta2->name);
        }    
    }
    else {
        meta1 = swish_metaname_init(swish_xstrdup(meta2name));
        meta1->ref_cnt++;
        swish_hash_add(metas1, meta1->name, meta1);
    }
/*     
    SWISH_DEBUG_MSG("%s meta1->id = %d    %s meta2->id = %d",
                    meta1->name, meta1->id, meta2->name, meta2->id);
 */
    // only change id if meta2->id is not already spoken for.
    meta1->id = meta2->id;
    meta1->bias = meta2->bias;
    if (meta1->alias_for != NULL) {
        swish_xfree(meta1->alias_for);
    }
    if (meta2->alias_for != NULL) {
        meta1->alias_for = swish_xstrdup(meta2->alias_for);
    }

}

static void
merge_metanames(
    xmlHashTablePtr metas1,
    xmlHashTablePtr metas2
)
{
    xmlHashScan(metas2, (xmlHashScanner)copy_metaname, metas1);
}

static void
copy_strlist(
    swish_StringList *strlist2,
    xmlHashTablePtr strlists1,
    xmlChar *key
)
{
    swish_StringList *strlist1;
    if (swish_hash_exists(strlists1, key)) {
        strlist1 = swish_hash_fetch(strlists1, key);
        swish_stringlist_merge(strlist2, strlist1);
    }
    else {
        strlist1 = swish_stringlist_copy(strlist2);
        swish_hash_add(strlists1, key, strlist1);
    }
}

static void
merge_stringlists(
    xmlHashTablePtr strlists1,
    xmlHashTablePtr strlists2
)
{
    xmlHashScan(strlists2, (xmlHashScanner)copy_strlist, strlists1);
}

void
swish_config_merge(
    swish_Config *config1,
    swish_Config *config2
)
{

/* values in config2 override and are set in config1 */

    if (SWISH_DEBUG & SWISH_DEBUG_CONFIG) {
        SWISH_DEBUG_MSG("Merging config2 0x%lx into config1 0x%lx",
            config2, config1);
        swish_config_debug(config2);
        swish_config_debug(config1);
    }


    if (SWISH_DEBUG & SWISH_DEBUG_CONFIG) {
        SWISH_DEBUG_MSG("merge properties");
    }
    merge_properties(config1->properties, config2->properties);

    if (SWISH_DEBUG & SWISH_DEBUG_CONFIG) {
        SWISH_DEBUG_MSG("merge metanames");
    }
    merge_metanames(config1->metanames, config2->metanames);

    if (SWISH_DEBUG & SWISH_DEBUG_CONFIG) {
        SWISH_DEBUG_MSG("merge parsers");
    }
    swish_hash_merge(config1->parsers, config2->parsers);

    if (SWISH_DEBUG & SWISH_DEBUG_CONFIG) {
        SWISH_DEBUG_MSG("merge mimes");
    }
    swish_hash_merge(config1->mimes, config2->mimes);

    if (SWISH_DEBUG & SWISH_DEBUG_CONFIG) {
        SWISH_DEBUG_MSG("merge index");
    }
    swish_hash_merge(config1->index, config2->index);

    if (SWISH_DEBUG & SWISH_DEBUG_CONFIG) {
        SWISH_DEBUG_MSG("merge tag_aliases");
    }
    swish_hash_merge(config1->tag_aliases, config2->tag_aliases);

    if (SWISH_DEBUG & SWISH_DEBUG_CONFIG) {
        SWISH_DEBUG_MSG("merge misc");
    }
    swish_hash_merge(config1->misc, config2->misc);
    
    if (SWISH_DEBUG & SWISH_DEBUG_CONFIG) {
        SWISH_DEBUG_MSG("merge stringlists");
    }
    merge_stringlists(config1->stringlists, config2->stringlists);

    if (SWISH_DEBUG & SWISH_DEBUG_CONFIG) {
        SWISH_DEBUG_MSG("merge complete");
    }

/* set flags */
    if (swish_hash_exists(config2->misc, BAD_CAST SWISH_TOKENIZE)) {
        config2->flags->tokenize = (boolean)swish_string_to_int(swish_hash_fetch(config2->misc, BAD_CAST SWISH_TOKENIZE));
    }
    config1->flags->tokenize = config2->flags->tokenize;
    if (swish_hash_exists(config2->misc, BAD_CAST SWISH_CASCADE_META_CONTEXT)) {
        config2->flags->cascade_meta_context = 
            (boolean)swish_string_to_int(swish_hash_fetch(config2->misc, BAD_CAST SWISH_CASCADE_META_CONTEXT));
    }
    config1->flags->cascade_meta_context = config2->flags->cascade_meta_context;
    if (swish_hash_exists(config2->misc, BAD_CAST SWISH_IGNORE_XMLNS)) {
        config2->flags->ignore_xmlns = 
            (boolean)swish_string_to_int(swish_hash_fetch(config2->misc, BAD_CAST SWISH_IGNORE_XMLNS));
    }
    config1->flags->ignore_xmlns = config2->flags->ignore_xmlns;


    if (SWISH_DEBUG & SWISH_DEBUG_CONFIG) {
        SWISH_DEBUG_MSG("flags set");
    }

}


/*************** end config.c ************/


/*************** start docinfo.c ************/
/*
 * This file is part of libswish3
 * Copyright (C) 2007 Peter Karman
 *
 *  libswish3 is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  libswish3 is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with libswish3; if not, write to the Free Software
 *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
*/

/* docinfo.c -- stat and time of files */

#ifndef LIBSWISH3_SINGLE_FILE
#include <sys/param.h>
#include <stdio.h>
#include <string.h>
#include <err.h>
#include <errno.h>
#include <limits.h>
#include <stdlib.h>

#include "libswish3.h"
#endif

extern int errno;
extern int SWISH_DEBUG;

/* PUBLIC */
swish_DocInfo *
swish_docinfo_init(
)
{

    if (SWISH_DEBUG & SWISH_DEBUG_DOCINFO)
        SWISH_DEBUG_MSG("init'ing docinfo");

    swish_DocInfo *docinfo = swish_xmalloc(sizeof(swish_DocInfo));
    docinfo->ref_cnt = 0;
    docinfo->nwords = 0;
    docinfo->mtime = 0;
    docinfo->size = 0;
    docinfo->encoding = swish_xstrdup((xmlChar *)SWISH_DEFAULT_ENCODING);
    docinfo->uri = NULL;
    docinfo->mime = NULL;
    docinfo->parser = NULL;
    docinfo->ext = NULL;
    docinfo->action = NULL;
    docinfo->is_gzipped = SWISH_FALSE;

    /*
    if (SWISH_DEBUG & SWISH_DEBUG_DOCINFO) {
        SWISH_DEBUG_MSG("docinfo all ready");
        swish_docinfo_debug(docinfo);
    }
    */

    return docinfo;
}

/* PUBLIC */
void
swish_docinfo_free(
    swish_DocInfo *ptr
)
{
    if (SWISH_DEBUG & SWISH_DEBUG_DOCINFO)
        SWISH_DEBUG_MSG("freeing swish_DocInfo");

    if (SWISH_DEBUG & SWISH_DEBUG_DOCINFO)
        swish_docinfo_debug(ptr);
    
    if (ptr->ref_cnt != 0) {
        SWISH_WARN("docinfo ref_cnt != 0: %d", ptr->ref_cnt);
    }

    ptr->nwords = 0;            /* why is this required? */
    ptr->mtime = 0;
    ptr->size = 0;
    ptr->is_gzipped = SWISH_FALSE;

/* encoding and mime are malloced via xmlstrdup elsewhere */
    if (SWISH_DEBUG & SWISH_DEBUG_DOCINFO)
        SWISH_DEBUG_MSG("freeing docinfo->encoding");
    swish_xfree(ptr->encoding);

    if (SWISH_DEBUG & SWISH_DEBUG_DOCINFO)
        SWISH_DEBUG_MSG("freeing docinfo->mime");
    if (ptr->mime != NULL)
        swish_xfree(ptr->mime);

    if (SWISH_DEBUG & SWISH_DEBUG_DOCINFO)
        SWISH_DEBUG_MSG("freeing docinfo->uri");
    if (ptr->uri != NULL)
        swish_xfree(ptr->uri);

    if (SWISH_DEBUG & SWISH_DEBUG_DOCINFO)
        SWISH_DEBUG_MSG("freeing docinfo->ext");
    if (ptr->ext != NULL)
        swish_xfree(ptr->ext);

    if (SWISH_DEBUG & SWISH_DEBUG_DOCINFO)
        SWISH_DEBUG_MSG("freeing docinfo->parser");
    if (ptr->parser != NULL)
        swish_xfree(ptr->parser);

    if (SWISH_DEBUG & SWISH_DEBUG_DOCINFO)
        SWISH_DEBUG_MSG("freeing docinfo ptr");
    swish_xfree(ptr);

    if (SWISH_DEBUG & SWISH_DEBUG_DOCINFO)
        SWISH_DEBUG_MSG("swish_DocInfo all freed");
}

int
swish_docinfo_check(
    swish_DocInfo *docinfo,
    swish_Config *config
)
{
    int ok;
    xmlChar *ext;

    ok = 1;

    if (SWISH_DEBUG & SWISH_DEBUG_DOCINFO)
        swish_docinfo_debug(docinfo);

    if (!docinfo->uri)
        SWISH_CROAK("Failed to return required header Content-Location:");

    if (docinfo->size == -1)
        SWISH_CROAK
            ("Failed to return required header Content-Length: for doc '%s'",
             docinfo->uri);

/* might make this conditional on verbose level */
    if (docinfo->size == 0)
        SWISH_CROAK("Found zero Content-Length for doc '%s'", docinfo->uri);

    ext = swish_fs_get_file_ext(docinfo->uri);
    
/* this fails with non-filenames like db ids, etc. */

    if (docinfo->ext == NULL) {
        if (ext != NULL)
            docinfo->ext = swish_xstrdup(ext);
        else
            docinfo->ext = swish_xstrdup((xmlChar *)"none");

    }

    if (ext != NULL)
        swish_xfree(ext);

    if (!docinfo->mime) {
        if (SWISH_DEBUG & SWISH_DEBUG_DOCINFO)
            SWISH_DEBUG_MSG
                ("no MIME known. guessing based on uri extension '%s'",
                 docinfo->ext);
        docinfo->mime = swish_mime_get_type(config, docinfo->ext);
    }
    else {
        if (SWISH_DEBUG & SWISH_DEBUG_DOCINFO)
            SWISH_DEBUG_MSG("found MIME type in headers: '%s'", docinfo->mime);

    }

    if (!docinfo->parser) {
        if (SWISH_DEBUG & SWISH_DEBUG_DOCINFO)
            SWISH_DEBUG_MSG
                ("no parser defined in headers -- deducing from content type '%s'",
                 docinfo->mime);

        docinfo->parser = swish_mime_get_parser(config, docinfo->mime);
    }
    else {
        if (SWISH_DEBUG & SWISH_DEBUG_DOCINFO)
            SWISH_DEBUG_MSG("found parser in headers: '%s'", docinfo->parser);

    }

    if (SWISH_DEBUG & SWISH_DEBUG_DOCINFO)
        swish_docinfo_debug(docinfo);

    return ok;

}

/* PUBLIC */
int
swish_docinfo_from_filesystem(
    xmlChar *filename,
    swish_DocInfo *i,
    swish_ParserData *parser_data
)
{
    if (i->ext != NULL)
        swish_xfree(i->ext);

    i->ext = swish_fs_get_file_ext(filename);
    if (xmlStrEqual(i->ext, BAD_CAST "gz")) {
        i->is_gzipped = SWISH_TRUE;
        /* get new ext */
        xmlChar *copy = swish_xstrdup(filename);
        unsigned int len = xmlStrlen(filename);
        copy[len-3] = '\0';
        swish_xfree(i->ext);
        i->ext = swish_fs_get_file_ext(copy);
        swish_xfree(copy);
    }
    
    if (!swish_fs_file_exists(filename)) {
        SWISH_WARN("Can't stat '%s': %s", filename, strerror(errno));
        return 0;
    }

    if (SWISH_DEBUG & SWISH_DEBUG_DOCINFO)
        SWISH_DEBUG_MSG("handling url %s", filename);

    if (i->uri != NULL)
        swish_xfree(i->uri);

    i->uri = swish_xstrdup(filename);
    i->mtime = swish_fs_get_file_mtime(filename);
    i->size = swish_fs_get_file_size(filename);

    if (SWISH_DEBUG & SWISH_DEBUG_DOCINFO)
        SWISH_DEBUG_MSG("handling mime");

    if (i->mime != NULL)
        swish_xfree(i->mime);

    i->mime = swish_mime_get_type(parser_data->s3->config, i->ext);

    if (SWISH_DEBUG & SWISH_DEBUG_DOCINFO)
        SWISH_DEBUG_MSG("handling parser");

    if (i->parser != NULL)
        swish_xfree(i->parser);

    i->parser = swish_mime_get_parser(parser_data->s3->config, i->mime);

    return 1;

}

/* PUBLIC */
void
swish_docinfo_debug(
    swish_DocInfo *docinfo
)
{
    char *ts;
    ts = swish_time_format(docinfo->mtime);
    
    SWISH_DEBUG_MSG("DocInfo");
    SWISH_DEBUG_MSG("  docinfo ptr: %lu", (unsigned long)docinfo);
/* SWISH_DEBUG_MSG("  size of swish_DocInfo struct: %d", (int)sizeof(swish_DocInfo)); */
/* SWISH_DEBUG_MSG("  size of docinfo ptr: %d",           (int)sizeof(*docinfo)); */
    SWISH_DEBUG_MSG("  uri: %s (%d)", docinfo->uri, (int)sizeof(docinfo->uri));
    SWISH_DEBUG_MSG("  doc size: %lu bytes (%d)", (unsigned long)docinfo->size,
                    (int)sizeof(docinfo->size));
    SWISH_DEBUG_MSG("  doc mtime: %lu (%d)", (unsigned long)docinfo->mtime,
                    (int)sizeof(docinfo->mtime));
/* SWISH_DEBUG_MSG("  size of mime: %d",                  (int)sizeof(docinfo->mime)); */
/* SWISH_DEBUG_MSG("  size of encoding: %d",              (int)sizeof(docinfo->encoding)); */
    SWISH_DEBUG_MSG("  mtime str: %s", ts);
    SWISH_DEBUG_MSG("  mime type: %s", docinfo->mime);
    SWISH_DEBUG_MSG("  encoding: %s", docinfo->encoding);       /* only known after parsing has
                                                                   started ... */
    SWISH_DEBUG_MSG("  file ext: %s", docinfo->ext);
    SWISH_DEBUG_MSG("  parser: %s", docinfo->parser);
    SWISH_DEBUG_MSG("  nwords: %d", docinfo->nwords);
    SWISH_DEBUG_MSG("  is_gzipped: %d", docinfo->is_gzipped);
    
    swish_xfree(ts);
}


/*************** end docinfo.c ************/


/*************** start error.c ************/
/*
 * This file is part of libswish3
 * Copyright (C) 2007 Peter Karman
 *
 *  libswish3 is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  libswish3 is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with libswish3; if not, write to the Free Software
 *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
*/

/* error handling based on Swish-e ver2 error.c */

#ifndef LIBSWISH3_SINGLE_FILE
#include <stdio.h>
#include <stdlib.h>
#include <stdarg.h>
#include <errno.h>
#include <err.h>
#include <string.h>
#include <libxml/globals.h>

#include "libswish3.h"
#endif

extern int SWISH_DEBUG;
extern int SWISH_WARNINGS;

static FILE *error_handle = NULL;

void
swish_set_error_handle(
    FILE * where
)
{
    error_handle = where;
}

void
swish_croak(
    const char *file,
    int line,
    const char *func,
    const char *msgfmt,
    ...
)
{
    va_list args;

    if (!error_handle)
        error_handle = stderr;

    va_start(args, msgfmt);
    fprintf(error_handle, "Swish ERROR %s:%d %s: ", file, line, func);
    vfprintf(error_handle, msgfmt, args);
    fprintf(error_handle, "\n");
    va_end(args);

    if (!errno)
        errno = 1;

    exit(errno);
}

void
swish_warn(
    const char *file,
    int line,
    const char *func,
    const char *msgfmt,
    ...
)
{
    va_list args;

    if (!error_handle)
        error_handle = stderr;
        
    if (!SWISH_WARNINGS)
        return;

    va_start(args, msgfmt);
    fprintf(error_handle, "Swish WARNING %s:%d %s: ", file, line, func);
    vfprintf(error_handle, msgfmt, args);
    fprintf(error_handle, "\n");
    va_end(args);
}

void
swish_debug(
    const char *file,
    int line,
    const char *func,
    const char *msgfmt,
    ...
)
{
    va_list args;

    if (!error_handle)
        error_handle = stderr;

    va_start(args, msgfmt);
    fprintf(error_handle, "Swish DEBUG %s:%d %s: ", file, line, func);
    vfprintf(error_handle, msgfmt, args);
    fprintf(error_handle, "\n");
    va_end(args);
}


/*************** end error.c ************/


/*************** start hash.c ************/
/*
 * This file is part of libswish3
 * Copyright (C) 2007 Peter Karman
 *
 *  libswish3 is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  libswish3 is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with libswish3; if not, write to the Free Software
 *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
*/

/* wrappers to common functions in libxml2 hash */

#ifndef LIBSWISH3_SINGLE_FILE
#include <stdlib.h>

#include "libswish3.h"
#endif

extern int SWISH_DEBUG;

static void free_hashval(
    void *val,
    xmlChar *key
);
static void merge_hashes(
    xmlChar *value,
    xmlHashTablePtr hash1,
    xmlChar *key
);

static void
free_hashval(
    void *val,
    xmlChar *key
)
{
    swish_xfree(val);
}

int
swish_hash_add(
    xmlHashTablePtr hash,
    xmlChar *key,
    void *value
)
{
    int ret;
    ret = xmlHashAddEntry(hash, key, value);
    if (ret == -1) {
        SWISH_CROAK("xmlHashAddEntry for %s failed", key);
    }
    /*
    else {
        
        SWISH_DEBUG_MSG("key %s added to hash with ret value %d", key, ret);
        
    } 
    */   
    return ret;
}

int
swish_hash_exists_or_add(
    xmlHashTablePtr hash,
    xmlChar *key,
    xmlChar *value
)
{
    if (!swish_hash_exists(hash, key)) {
        return swish_hash_add(hash, key, swish_xstrdup( value ));
    }
    else {
        return 1;
    }
}

void
swish_hash_free(
    xmlHashTablePtr hash
)
{
    xmlHashFree(hash, (xmlHashDeallocator)free_hashval);
}

int
swish_hash_replace(
    xmlHashTablePtr hash,
    xmlChar *key,
    void *value
)
{
    int ret;
    ret =
        xmlHashUpdateEntry(hash, key, value, (xmlHashDeallocator) free_hashval);
    if (ret == -1)
        SWISH_CROAK("xmlHashUpdateEntry for %s failed", key);

    return ret;
}

int
swish_hash_delete(
    xmlHashTablePtr hash,
    xmlChar *key
)
{
    int ret;
    ret = xmlHashRemoveEntry(hash, key, (xmlHashDeallocator) free_hashval);
    if (ret == -1)
        SWISH_CROAK("xmlHashRemoveEntry for %s failed", key);

    return ret;
}

boolean
swish_hash_exists(
    xmlHashTablePtr hash,
    xmlChar *key
)
{
    return xmlHashLookup(hash, key) ? 1 : 0;
}

void *
swish_hash_fetch(
    xmlHashTablePtr hash,
    xmlChar *key
)
{
    return xmlHashLookup(hash, key);
}

xmlHashTablePtr
swish_hash_init(
    int size
)
{
    xmlHashTablePtr h;

    h = xmlHashCreate(size);
    if (h == NULL) {
        SWISH_CROAK("error creating hash of size %d", size);
        return NULL; // never get here
    }

    return h;
}

static void
merge_hashes(
    xmlChar *value,
    xmlHashTablePtr hash1,
    xmlChar *key
)
{
    if (swish_hash_exists(hash1, key)) {
        swish_hash_replace(hash1, key, swish_xstrdup(value));
    }
    else {
        swish_hash_add(hash1, key, swish_xstrdup(value));
    }
}

void
swish_hash_merge(
    xmlHashTablePtr hash1,
    xmlHashTablePtr hash2
)
{
    xmlHashScan(hash2, (xmlHashScanner) merge_hashes, hash1);
}

static void
dump_hash_value(
    xmlChar *val,
    xmlChar *label,
    xmlChar *key
)
{
    SWISH_DEBUG_MSG(" %s:  [0x%x] => [0x%x]", label, key, val);
    SWISH_DEBUG_MSG(" %s:  %s [0x%x] => %s [0x%x]", label, key, key, val, val);
}

void
swish_hash_dump(
    xmlHashTablePtr hash,
    const char *label
)
{
    SWISH_DEBUG_MSG("start hash_dump for %s [0x%x]", label, hash);
    xmlHashScan(hash, (xmlHashScanner) dump_hash_value, (xmlChar*)label);
    SWISH_DEBUG_MSG("end hash_dump for %s [0x%x]", label, hash);
}


/*************** end hash.c ************/


/*************** start fs.c ************/
/*
 * This file is part of libswish3
 * Copyright (C) 2009 Peter Karman
 *
 *  libswish3 is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  libswish3 is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with libswish3; if not, write to the Free Software
 *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
*/

#ifndef LIBSWISH3_SINGLE_FILE
#ifdef HAVE_STDLIB_H
#include <stdlib.h>
#endif
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#include <dirent.h>
#include <stdio.h>
#include <errno.h>
#include <err.h>
#include <string.h>
#include <sys/param.h>
#include <limits.h>
#include <assert.h>

#include "libswish3.h"

#endif

extern int errno;
extern int SWISH_DEBUG;

static xmlChar *findlast(
    xmlChar *str,
    xmlChar *set
);
static xmlChar *lastptr(
    xmlChar *str
);

boolean
swish_fs_file_exists(
    xmlChar *path
)
{
    struct stat info;
    if (stat((char *)path, &info)) {
        return 0;
    }
    return 1;
}

boolean
swish_fs_is_dir(
    xmlChar *path
)
{
    struct stat stbuf;

    if (stat((char *)path, &stbuf))
        return 0;
    return ((stbuf.st_mode & S_IFMT) == S_IFDIR) ? 1 : 0;
}

boolean
swish_fs_is_file(
    xmlChar *path
)
{
    struct stat stbuf;

    if (stat((char *)path, &stbuf))
        return 0;
    return ((stbuf.st_mode & S_IFMT) == S_IFREG) ? 1 : 0;
}

boolean
swish_fs_is_link(
    xmlChar *path
)
{
#ifdef HAVE_LSTAT
    struct stat stbuf;

    if (lstat((char *)path, &stbuf))
        return 0;
    return ((stbuf.st_mode & S_IFLNK) == S_IFLNK) ? 1 : 0;
#else
    return 0;
#endif
}

off_t
swish_fs_get_file_size(
    xmlChar *path
)
{
    struct stat stbuf;

    if (stat((char *)path, &stbuf))
        return -1;
    return stbuf.st_size;
}

time_t
swish_fs_get_file_mtime(
    xmlChar *path
)
{
    struct stat stbuf;

    if (stat((char *)path, &stbuf))
        return -1;
    return stbuf.st_mtime;
}

/* parse a URL to determine file ext */
/* inspired by http://www.tug.org/tex-archive/tools/zoo/ by Rahul Dhesi */
xmlChar *
swish_fs_get_file_ext(
    xmlChar *url
)
{
    xmlChar *p;

    if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER)
        SWISH_DEBUG_MSG("parsing url %s for extension", url);

    p = findlast(url, (xmlChar *)SWISH_EXT_SEP);        /* look for . or /         */

    if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER)
        SWISH_DEBUG_MSG("p = %s", p);

    if (p == NULL)
        return p;

    if (p != NULL && *p != SWISH_EXT_CH)        /* found .?                     */
        return NULL;            /* ... if not, ignore / */

    if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER)
        SWISH_DEBUG_MSG("p = %s", p);

    if (*p == SWISH_EXT_CH)
        p++;                    /* skip to next char after . */

    if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER)
        SWISH_DEBUG_MSG("ext is %s", p);

    return swish_str_tolower(p);
}

boolean
swish_fs_looks_like_gz(
    xmlChar *file
)
{
    xmlChar *ext;
    boolean is_eq;
    ext = swish_fs_get_file_ext(file);
    is_eq = xmlStrEqual(ext, BAD_CAST "gz");
    //SWISH_DEBUG_MSG("looks like gz? %d", is_eq);
    swish_xfree(ext);
    return is_eq;
}

/*******************/
/*
findlast() finds last occurrence in provided string of any of the characters
except the null character in the provided set.

If found, return value is pointer to character found, else it is NULL.
*/

static xmlChar *
findlast(
    xmlChar *str,
    xmlChar *set
)
{
    xmlChar *p;

    if (str == NULL || set == NULL || *str == '\0' || *set == '\0')
        return (NULL);

    p = lastptr(str);           /* pointer to last char of string */
    assert(p != NULL);

    while (p != str && xmlStrchr(set, *p) == NULL) {
        --p;
    }

/* either p == str or we found a character or both */
    if (xmlStrchr(set, *p) == NULL)
        return (NULL);
    else
        return (p);
}

/*
lastptr() returns a pointer to the last non-null character in the string, if
any.  If the string is null it returns NULL
*/

static xmlChar *
lastptr(
    xmlChar *str
)
{
    xmlChar *p;
    if (str == NULL)
        SWISH_CROAK("received null pointer while looking for last NULL");
    if (*str == '\0')
        return (NULL);
    p = str;
    while (*p != '\0')          /* find trailing null char */
        ++p;
    --p;                        /* point to just before it */
    return (p);
}


/*************** end fs.c ************/


/*************** start io.c ************/
/*
 * This file is part of libswish3
 * Copyright (C) 2007 Peter Karman
 *
 *  libswish3 is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  libswish3 is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with libswish3; if not, write to the Free Software
 *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
*/

/* simple I/O functions */

#ifndef LIBSWISH3_SINGLE_FILE
#include <stdio.h>
#include <errno.h>
#include <err.h>
#include <string.h>
#include <zlib.h>

#include "libswish3.h"
#endif

extern int SWISH_DEBUG;
extern int errno;

static void no_nulls(
    xmlChar *filename,
    xmlChar *buffer,
    long bytes_read
);

/* substitute embedded null chars with a newline so we can treat the buffer as a whole
 * string based on similar code in swish-e ver2 file.c */
static void
no_nulls(
    xmlChar *filename,
    xmlChar *buffer,
    long bytes_read
)
{
    if (xmlStrlen(buffer) < bytes_read) {
        long i;
        int j = 0;
        long i_bytes_read = bytes_read;

        for (i = 0; i < i_bytes_read; ++i) {
            if (buffer[i] == '\0') {
                buffer[i] = '\n';
                j++;
            }
            if (buffer[i] == SWISH_TOKENPOS_BUMPER[0]) {
                buffer[i] = '\n';
                j++;
            }
        }

        if (j) {
            SWISH_WARN
                ("Substituted %d embedded null or connector character(s) in file '%s' with newline(s)",
                 j, filename);
        }
    }

}

xmlChar *
swish_io_slurp_fh(
    FILE * fh,
    unsigned long flen,
    boolean binmode
)
{
    size_t bytes_read;
    xmlChar *buffer;

/* printf("slurping %d bytes\n", flen); */

    buffer = swish_xmalloc(flen + 1);
    *buffer = '\0';

    bytes_read = fread(buffer, sizeof(xmlChar), flen, fh);

    if (bytes_read != flen) {
        SWISH_CROAK("did not read expected bytes: %ld expected, %d read", flen,
                    bytes_read);
    }
    buffer[bytes_read] = '\0';  /* terminate the string */

/* printf("read %d bytes from stdin\n", bytes_read); */

    if (!binmode) {
        no_nulls((xmlChar *)"filehandle", buffer, (int)bytes_read);
    }
    
    return buffer;
}

xmlChar *
swish_io_slurp_file_len(
    xmlChar *filename,
    off_t flen,
    boolean binmode
)
{
    size_t bytes_read;
    FILE *fp;
    xmlChar *buffer;

    if (flen > SWISH_MAX_FILE_LEN) {
        flen = SWISH_MAX_FILE_LEN;
        SWISH_WARN("max file len %ld exceeded - cannot read %ld bytes from %s",
                   SWISH_MAX_FILE_LEN, flen, filename);

    }
    
    if (SWISH_DEBUG & SWISH_DEBUG_IO)
        SWISH_DEBUG_MSG("slurp file '%s' [%ld bytes]", filename, flen);

    buffer = swish_xmalloc(flen + 1);

    if ((fp = fopen((char *)filename, "r")) == 0) {
        SWISH_CROAK("Error reading file %s: %s", filename, strerror(errno));
    }

    bytes_read = fread(buffer, sizeof(xmlChar), flen, fp);

    if (bytes_read != flen) {
        SWISH_CROAK("did not read expected bytes: %ld expected, %d read (%s)",
                    flen, bytes_read, strerror(errno));
    }
    buffer[bytes_read] = '\0';  /* terminate the string */

/* close the stream */
    if (fclose(fp))
        SWISH_CROAK("error closing filehandle for %s: %s", 
            filename, strerror(errno));

    if (!binmode) {
        no_nulls(filename, buffer, (long)bytes_read);
    }
    
    return buffer;
}

xmlChar *
swish_io_slurp_gzfile_len(
    xmlChar *filename,
    off_t *flen,
    boolean binmode
)
{
    off_t bytes_read, buffer_len;
    int ret;
    gzFile fh;
    xmlChar *buffer;
    unsigned int buf_size;
    int compression_rate = 3;   /* seems about right */
    
    buf_size = sizeof(xmlChar)*(*flen)*compression_rate;
    buffer = swish_xmalloc(buf_size);
    buffer_len = 0;
    fh = gzopen((char*)filename, "r");
    if (fh == NULL) {
        SWISH_CROAK("Failed to open file '%s' for read: %s",
            filename, strerror(errno));
    }
    while ((bytes_read = gzread(fh, buffer, buf_size)) != 0) {
        if (bytes_read == -1) {
            SWISH_CROAK("Error reading gzipped file '%s': %s",
                filename, strerror(errno));
        }
        if (SWISH_DEBUG & SWISH_DEBUG_IO) {
            SWISH_DEBUG_MSG("Read %d bytes from %s", bytes_read, filename);
        }
        if (bytes_read < buf_size) {
            if (SWISH_DEBUG & SWISH_DEBUG_IO) {
                SWISH_DEBUG_MSG("Read to end of file");
            }
            buffer_len = bytes_read;
            break;
        }
        buf_size *= compression_rate;
        buffer = swish_xrealloc(buffer, buf_size);
        if (SWISH_DEBUG & SWISH_DEBUG_IO) {
            SWISH_DEBUG_MSG("grew buffer to %d", buf_size);
        }
        buffer_len = bytes_read;
        ret = gzrewind(fh);
        if (SWISH_DEBUG & SWISH_DEBUG_IO) {
            SWISH_DEBUG_MSG("gzrewind ret = %d", ret);
        }
    }
    ret = gzclose(fh);    // TODO check for err?
        
    buffer[buffer_len] = '\0';
    
    if (!binmode) {
        no_nulls(filename, buffer, (long)buffer_len);
    }
   
    if (SWISH_DEBUG & SWISH_DEBUG_IO) { 
        SWISH_DEBUG_MSG("slurped gzipped file '%s' buffer_len=%d buf_size=%d orig flen=%d", 
            filename, buffer_len, buf_size, *flen);
    }

    /* set the flen pointer to the actual length */
    *flen = buffer_len;
      
    return buffer;
}

xmlChar *
swish_io_slurp_file(
    xmlChar *filename,
    off_t file_len,
    boolean is_gzipped,
    boolean binmode
)
{
    if (!file_len) {
        file_len = swish_fs_get_file_size(filename);
    }
    if (!file_len || file_len == -1) {
        SWISH_CROAK("Can't stat %s: %s\n", filename, strerror(errno));
    }
    if (is_gzipped) {
        return swish_io_slurp_gzfile_len(filename, &file_len, binmode);
    }
    else {
        return swish_io_slurp_file_len(filename, file_len, binmode);
    }
}

long int
swish_io_count_operable_file_lines(
    xmlChar *filename
)
{
    long int count;
    FILE *fp;
    xmlChar line_in_file[SWISH_MAXSTRLEN];
    
    count = 0;
    
    fp = fopen((const char*)filename, "r");
    if (fp == NULL) {
        SWISH_CROAK("failed to open file: %s", filename);
    }
    while (fgets((char*)line_in_file, SWISH_MAXSTRLEN, fp) != NULL) {
        if (swish_io_is_skippable_line(line_in_file))  
            continue;
        
        count++;
        //SWISH_DEBUG_MSG("count %d for '%s'", count, line_in_file);

    }
    
    if (fclose(fp)) {
        SWISH_CROAK("error closing filelist");
    }

    return count;
}


boolean
swish_io_is_skippable_line(
    xmlChar *str
)
{
    xmlChar *line;

    /* skip leading white space */
    line = swish_str_skip_ws(str);
    
    //SWISH_DEBUG_MSG("line: '%s'", line);
        
    if (xmlStrlen(line) == 0 || (xmlStrlen(line) == 1 && line[0] == '\n')) {
        /* blank line */
        return SWISH_TRUE;
    }
    if (line[0] == '#') {
        /* skip comments */
        return SWISH_TRUE;
    }
    
    return SWISH_FALSE;
}


/*************** end io.c ************/


/*************** start mem.c ************/
/*
 * This file is part of libswish3
 * Copyright (C) 2007 Peter Karman
 *
 *  libswish3 is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  libswish3 is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with libswish3; if not, write to the Free Software
 *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
*/

/* mem.c -- graceful memory handling */

#ifndef LIBSWISH3_SINGLE_FILE
#include <sys/types.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <libxml/xmlstring.h>
#include <err.h>

#include "libswish3.h"
#endif

extern int SWISH_DEBUG;

static long int memcount = 0;

void
swish_mem_init(
)
{
    memcount = 0;
}

long int
swish_memcount_get(
)
{
    return memcount;
}

void
swish_memcount_dec(
)
{
    memcount--;
}

/* PUBLIC */
/* realloc a block of memory */
void *
swish_xrealloc(
    void *ptr,
    size_t size
)
{
    void *new_ptr = realloc(ptr, size);

    if (new_ptr == NULL)
        SWISH_CROAK("Out of memory (could not reallocate %lu more bytes)!",
                    (unsigned long)size);

    return new_ptr;
}

/* PUBLIC */
void *
swish_xmalloc(
    size_t size
)
{
    void *ptr;
    
    if (SWISH_DEBUG & SWISH_DEBUG_MEMORY) {
        SWISH_DEBUG_MSG("malloc %ld bytes", (long)size);
    }

    ptr = malloc(size);

    if (ptr == NULL)
        SWISH_CROAK("Out of memory! Can't malloc %lu bytes",
                    (unsigned long)size);

    memcount++;
    if (SWISH_DEBUG & SWISH_DEBUG_MEMORY) {
        SWISH_DEBUG_MSG("memcount = %ld", memcount);
        SWISH_DEBUG_MSG("xmalloc address: 0x%lx", ptr);
    }

    return ptr;
}

xmlChar *
swish_xstrdup(
    const xmlChar *ptr
)
{
    xmlChar *copy;
    memcount++;
    if (SWISH_DEBUG & SWISH_DEBUG_MEMORY)
        SWISH_DEBUG_MSG("memcount = %ld", memcount);
    copy = xmlStrdup(ptr);
    if (copy == NULL) 
        SWISH_CROAK("strdup returned NULL for %s", ptr);

    return copy;
}

xmlChar *
swish_xstrndup(
    const xmlChar *ptr,
    int len
)
{
    memcount++;
    if (SWISH_DEBUG & SWISH_DEBUG_MEMORY)
        SWISH_DEBUG_MSG("memcount = %ld", memcount);
    return (xmlStrndup(ptr, len));
}

void
swish_xfree(
    void *ptr
)
{
    if (ptr == NULL) {
        SWISH_WARN
            (" >>>>>>>>>>>>>> attempt to free NULL pointer <<<<<<<<<<<<<<");
        return;
    }
    
    if (SWISH_DEBUG & SWISH_DEBUG_MEMORY)
        SWISH_DEBUG_MSG("freeing %s 0x%lx", (char*)ptr, ptr);

    xmlFree(ptr);

    memcount--;

    if (SWISH_DEBUG & SWISH_DEBUG_MEMORY)
        SWISH_DEBUG_MSG("memcount = %ld", memcount);
}

void
swish_mem_debug(
)
{
/* SWISH_DEBUG_MSG("memcount = %ld", memcount); */
    if (memcount > 0)
        SWISH_WARN
            ("%ld more swish_xmalloc()s or swish_xstrdup()s than swish_xfree()s",
             memcount);

    if (memcount < 0)
        SWISH_WARN("too many swish_xfree()s %ld", memcount);
}


/*************** end mem.c ************/


/*************** start mime_types.c ************/
/*
 * This file is part of libswish3
 * Copyright (C) 2007 Peter Karman
 *
 *  libswish3 is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  libswish3 is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with libswish3; if not, write to the Free Software
 *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
*/

/* MIME types hash based on Apache 2.0 mime.types listing
see <http://www.iana.org/assignments/media-types/> for official registry.
*/

#ifndef LIBSWISH3_SINGLE_FILE
#include <err.h>
#include <stdlib.h>
#include <string.h>

#include "libswish3.h"
#endif

extern int SWISH_DEBUG;

//should be total number of strings(NOT pairs !) below
#define SWISH_MIME_TABLE_COUNT  304

static const char *SWISH_MIME_TABLE[] = {
    "ai", "application/postscript",
    "aif", "audio/x-aiff",
    "aifc", "audio/x-aiff",
    "aiff", "audio/x-aiff",
    "asc", "text/plain",
    "au", "audio/basic",
    "avi", "video/x-msvideo",
    "bcpio", "application/x-bcpio",
    "bin", "application/octet-stream",
    "bmp", "image/bmp",
    "cdf", "application/x-netcdf",
    "cgm", "image/cgm",
    "class", "application/octet-stream",
    "cpio", "application/x-cpio",
    "cpt", "application/mac-compactpro",
    "csh", "application/x-csh",
    "css", "text/css",
    "dcr", "application/x-director",
    "dir", "application/x-director",
    "djv", "image/vnd.djvu",
    "djvu", "image/vnd.djvu",
    "dll", "application/octet-stream",
    "dmg", "application/octet-stream",
    "dms", "application/octet-stream",
    "doc", "application/msword",
    "dtd", "application/xml-dtd",
    "dvi", "application/x-dvi",
    "dxr", "application/x-director",
    "eps", "application/postscript",
    "etx", "text/x-setext",
    "exe", "application/octet-stream",
    "ez", "application/andrew-inset",
    "gif", "image/gif",
    "gram", "application/srgs",
    "grxml", "application/srgs+xml",
    "gtar", "application/x-gtar",
    "gz", "application/x-gzip",
    "hdf", "application/x-hdf",
    "hqx", "application/mac-binhex40",
    "htm", "text/html",
    "html", "text/html",
    "ice", "x-conference/x-cooltalk",
    "ico", "image/x-icon",
    "ics", "text/calendar",
    "ief", "image/ief",
    "ifb", "text/calendar",
    "iges", "model/iges",
    "igs", "model/iges",
    "jpe", "image/jpeg",
    "jpeg", "image/jpeg",
    "jpg", "image/jpeg",
    "js", "application/x-javascript",
    "kar", "audio/midi",
    "latex", "application/x-latex",
    "lha", "application/octet-stream",
    "lzh", "application/octet-stream",
    "m3u", "audio/x-mpegurl",
    "m4u", "video/vnd.mpegurl",
    "man", "application/x-troff-man",
    "mathml", "application/mathml+xml",
    "me", "application/x-troff-me",
    "mesh", "model/mesh",
    "mid", "audio/midi",
    "midi", "audio/midi",
    "mif", "application/vnd.mif",
    "mov", "video/quicktime",
    "movie", "video/x-sgi-movie",
    "mp2", "audio/mpeg",
    "mp3", "audio/mpeg",
    "mpe", "video/mpeg",
    "mpeg", "video/mpeg",
    "mpg", "video/mpeg",
    "mpga", "audio/mpeg",
    "ms", "application/x-troff-ms",
    "msh", "model/mesh",
    "mxu", "video/vnd.mpegurl",
    "nc", "application/x-netcdf",
    "oda", "application/oda",
    "ogg", "application/ogg",
    "pbm", "image/x-portable-bitmap",
    "pdb", "chemical/x-pdb",
    "pdf", "application/pdf",
    "pgm", "image/x-portable-graymap",
    "pgn", "application/x-chess-pgn",
    "png", "image/png",
    "pnm", "image/x-portable-anymap",
    "ppm", "image/x-portable-pixmap",
    "ppt", "application/vnd.ms-powerpoint",
    "ps", "application/postscript",
    "qt", "video/quicktime",
    "ra", "audio/x-pn-realaudio",
    "ram", "audio/x-pn-realaudio",
    "ras", "image/x-cmu-raster",
    "rdf", "application/rdf+xml",
    "rgb", "image/x-rgb",
    "rm", "application/vnd.rn-realmedia",
    "roff", "application/x-troff",
    "rtf", "text/rtf",
    "rtx", "text/richtext",
    "sgm", "text/sgml",
    "sgml", "text/sgml",
    "sh", "application/x-sh",
    "shar", "application/x-shar",
    "silo", "model/mesh",
    "sit", "application/x-stuffit",
    "skd", "application/x-koan",
    "skm", "application/x-koan",
    "skp", "application/x-koan",
    "skt", "application/x-koan",
    "smi", "application/smil",
    "smil", "application/smil",
    "snd", "audio/basic",
    "so", "application/octet-stream",
    "spl", "application/x-futuresplash",
    "src", "application/x-wais-source",
    "sv4cpio", "application/x-sv4cpio",
    "sv4crc", "application/x-sv4crc",
    "svg", "image/svg+xml",
    "swf", "application/x-shockwave-flash",
    "t", "application/x-troff",
    "tar", "application/x-tar",
    "tcl", "application/x-tcl",
    "tex", "application/x-tex",
    "texi", "application/x-texinfo",
    "texinfo", "application/x-texinfo",
    "tif", "image/tiff",
    "tiff", "image/tiff",
    "tr", "application/x-troff",
    "tsv", "text/tab-separated-values",
    "txt", "text/plain",
    "ustar", "application/x-ustar",
    "vcd", "application/x-cdlink",
    "vrml", "model/vrml",
    "vxml", "application/voicexml+xml",
    "wav", "audio/x-wav",
    "wbmp", "image/vnd.wap.wbmp",
    "wbxml", "application/vnd.wap.wbxml",
    "wml", "text/vnd.wap.wml",
    "wmlc", "application/vnd.wap.wmlc",
    "wmls", "text/vnd.wap.wmlscript",
    "wmlsc", "application/vnd.wap.wmlscriptc",
    "wrl", "model/vrml",
    "xbm", "image/x-xbitmap",
    "xht", "application/xhtml+xml",
    "xhtml", "application/xhtml+xml",
    "xls", "application/vnd.ms-excel",
    "xml",       "application/xml", 
    /*"xml", "text/xml", */ /* w3 standard is application/xml now */
    "xpm", "image/x-xpixmap",
    "xsl", "application/xml",
    "xslt", "application/xslt+xml",
    "xul", "application/vnd.mozilla.xul+xml",
    "xwd", "image/x-xwindowdump",
    "xyz", "chemical/x-xyz",
    "zip", "application/zip"
};

/* create hash of file ext => mime type */
xmlHashTablePtr
swish_mime_defaults(
)
{
    int i;
    xmlChar *ext, *type;
    xmlHashTablePtr mimes;
    mimes = swish_hash_init(SWISH_MIME_TABLE_COUNT / 2);

    for (i = 0; i <= SWISH_MIME_TABLE_COUNT; i += 2) {
        ext = (xmlChar *)SWISH_MIME_TABLE[i];
        type = swish_xstrdup((xmlChar *)SWISH_MIME_TABLE[i + 1]);
        /*
        SWISH_DEBUG_MSG("%s: copy of %s [0x%x] at %s [0x%x]", 
            ext, SWISH_MIME_TABLE[i+1],SWISH_MIME_TABLE[i+1],type,type);
        */
        swish_hash_add(mimes, ext, type);
    }
    
    /*
    SWISH_DEBUG_MSG("mime_hash == 0x%x", mimes);

    for (i = 0; i <= SWISH_MIME_TABLE_COUNT; i += 2) {
        ext = (xmlChar *)SWISH_MIME_TABLE[i];
        if (!swish_hash_exists(mimes, ext)) {
            SWISH_CROAK("%s no in mimes hash", ext);
        }
        else {
            SWISH_DEBUG_MSG("fetched %s for %s", swish_hash_fetch(mimes, ext), ext);
        }
    }

    SWISH_DEBUG_MSG("mime_hash == 0x%x", mimes);
    */
    
    return mimes;
}

/* retrieve mime type from hash */
xmlChar *
swish_mime_get_type(
    swish_Config *config,
    xmlChar *fileext
)
{
    xmlChar *mime;
    mime = swish_hash_fetch(config->mimes, fileext);
    if (mime == NULL) {
        SWISH_WARN("No MIME type known for '%s' -- using '%s'", fileext,
                   SWISH_DEFAULT_MIME);
        mime = swish_xstrdup((xmlChar *)SWISH_DEFAULT_MIME);
    }
    return swish_xstrdup(mime);
}

/* returns parser type (TXT, HTML, XML) based on mime type */
xmlChar *
swish_mime_get_parser(
    swish_Config *config,
    xmlChar *mime
)
{
    xmlChar *parser;
    xmlChar *deftype;

    parser = swish_hash_fetch(config->parsers, mime);

    if (SWISH_DEBUG & SWISH_DEBUG_DOCINFO)
        SWISH_DEBUG_MSG("using parser '%s' based on MIME '%s'", parser, mime);

    deftype = swish_hash_fetch(config->parsers, (xmlChar *)SWISH_DEFAULT_PARSER);       /* error check?? */

    if (parser == NULL) {
        SWISH_WARN("No parser for MIME '%s' -- using '%s'", mime, deftype);
        parser = deftype;
    }

    return swish_xstrdup(parser);       /* so we don't change orig value -- MUST free */
}


/*************** end mime_types.c ************/


/*************** start parser.c ************/
/* 
* This file is part of libswish3
* Copyright (C) 2007 Peter Karman
*
*  libswish3 is free software; you can redistribute it and/or modify
*  it under the terms of the GNU General Public License as published by
*  the Free Software Foundation; either version 2 of the License, or
*  (at your option) any later version.
*
*  libswish3 is distributed in the hope that it will be useful,
*  but WITHOUT ANY WARRANTY; without even the implied warranty of
*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
*  GNU General Public License for more details.
*
*  You should have received a copy of the GNU General Public License
*  along with libswish3; if not, write to the Free Software
*  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
*/

/* 
* parse XML doc from memory using libxml2 SAX2 based on tutorial at
* http://www.jamesh.id.au/articles/libxml-sax/libxml-sax.html
*
* save all character() data to buffer, flushing on new metanames
* flush should split buffer into words, skipping nonwordchars/space, and
* lowercase all
*
* see iswlower(3) man page, etc.
*
* all the mb*() functions rely on locale to recognize multi-byte strings
*
*/

#ifndef LIBSWISH3_SINGLE_FILE

#ifdef HAVE_STDLIB_H
#include <stdlib.h>
#endif
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#include <stdio.h>
#include <locale.h>
#include <stdarg.h>
#include <err.h>
#include <errno.h>
#include <string.h>
#include <ctype.h>
#include <wctype.h>
#include <dirent.h>

#include <libxml/parserInternals.h>
#include <libxml/parser.h>
#include <libxml/HTMLparser.h>
#include <libxml/globals.h>
#include <libxml/xmlerror.h>
#include <libxml/tree.h>
#include <libxml/debugXML.h>
#include <libxml/xmlmemory.h>

#include "libswish3.h"
#endif

extern int errno;
extern int SWISH_DEBUG;

/* should we pass on libxml2 via SWISH_WARN() */
/* default is "on" per consistency with version 2.4.x */
int SWISH_PARSER_WARNINGS = 1;

static void get_env_vars(
);

static void flush_buffer(
    swish_ParserData *parser_data,
    xmlChar *metaname,
    xmlChar *context
);

static void tokenize(
    swish_ParserData *parser_data,
    xmlChar *string,
    int len,
    xmlChar *metaname,
    xmlChar *content
);

static void mystartDocument(
    void *parser_data
);
static void myendDocument(
    void *parser_data
);
static void mystartElement(
    void *parser_data,
    const xmlChar *name,
    const xmlChar **atts
);
static void myendElement(
    void *parser_data,
    const xmlChar *name
);

/* 
* SAX2 support 
*/
static void mystartElementNs(
    void *parser_data,
    const xmlChar *localname,
    const xmlChar *prefix,
    const xmlChar *URI,
    int nb_namespaces,
    const xmlChar **namespaces,
    int nb_attributes,
    int nb_defaulted,
    const xmlChar **attributes
);

static void myendElementNs(
    void *ctx ATTRIBUTE_UNUSED,
    const xmlChar *localname,
    const xmlChar *prefix,
    const xmlChar *URI
);

static void buffer_characters(
    swish_ParserData *parser_data,
    const xmlChar *ch,
    int len
);
static void mycharacters(
    void *parser_data,
    const xmlChar *ch,
    int len
);
static void mycomments(
    void *parser_data,
    const xmlChar *ch
);
static void myerr(
    void *user_data,
    xmlChar *msg,
    ...
);

static void open_tag(
    void *data,
    const xmlChar *tag,
    xmlChar **atts,
    const xmlChar *xmlns_prefix
);
static void close_tag(
    void *data,
    const xmlChar *tag,
    const xmlChar *xmlns_prefix
);
static xmlChar *bake_tag(
    swish_ParserData *parser_data,
    xmlChar *tag,
    xmlChar **atts,
    xmlChar *xmlns_prefix
);

static int docparser(
    swish_ParserData *parser_data,
    xmlChar *filename,
    xmlChar *buffer,
    int size
);
static int xml_parser(
    xmlSAXHandlerPtr sax,
    void *user_data,
    xmlChar *buffer,
    int size
);
static int html_parser(
    xmlSAXHandlerPtr sax,
    void *user_data,
    xmlChar *buffer,
    int size
);
static int txt_parser(
    swish_ParserData *parser_data,
    xmlChar *buffer,
    int size
);

static swish_ParserData *init_parser_data(
    swish_3 *s3
);
static void free_parser_data(
    swish_ParserData *parser_data
);

/* 
* parsing fh/buffer headers 
*/
typedef struct
{
    xmlChar **lines;
    int body_start;
    int nlines;
} HEAD;

static HEAD *buf_to_head(
    xmlChar *buf
);
static void free_head(
    HEAD * h
);
static swish_DocInfo *head_to_docinfo(
    HEAD * h
);

static xmlChar *document_encoding(
    xmlParserCtxtPtr ctxt
);

static void set_encoding(
    swish_ParserData *parser_data,
    xmlChar *buffer
);

/* tag tracker */
static xmlChar *flatten_tag_stack(
    xmlChar *baked,
    swish_TagStack *stack,
    char flatten_join
);
static void add_stack_to_prop_buf(
    xmlChar *baked,
    swish_ParserData *parser_data
);
static void push_tag_stack(
    swish_TagStack *stack,
    xmlChar *raw,
    xmlChar *baked,
    char flatten_join
);
static swish_Tag *pop_tag_stack(
    swish_TagStack *stack
);
static swish_Tag *pop_tag_stack_on_match(
    swish_TagStack *stack,
    xmlChar *raw
);
static void free_swishTag(
    swish_Tag * st
);
static void
free_swishTagStack(
    swish_TagStack *stack
);

/***********************************************************************
*                end prototypes
***********************************************************************/

swish_Parser *
swish_parser_init(
    void (*handler) (swish_ParserData *)
)
{
    swish_Parser *p = (swish_Parser *)swish_xmalloc(sizeof(swish_Parser));

    p->handler = handler;
    p->verbosity = 0;
    p->ref_cnt = 0;

/*
* libxml2 stuff 
*/
    xmlInitParser();
    xmlSubstituteEntitiesDefault(1);    /* resolve text entities */

/*
* debugging help 
*/
    get_env_vars();

    if (SWISH_DEBUG & SWISH_DEBUG_MEMORY) {
        SWISH_DEBUG_MSG("parser ptr 0x%x", (long int)p);
    }

    return p;
}

void
swish_parser_free(
    swish_Parser *p
)
{
    if (SWISH_DEBUG & SWISH_DEBUG_MEMORY) {
        SWISH_DEBUG_MSG("freeing parser");
        swish_mem_debug();
    }
    if (p->ref_cnt != 0) {
        SWISH_WARN("parser ref_cnt != 0: %d\n", p->ref_cnt);
    }
    xmlCleanupParser();
    xmlMemoryDump();
    swish_xfree(p);
}

/* 
* turn the literal xml/html tag into a swish tag for matching against
* metanames and properties 
*/
static xmlChar *
bake_tag(
    swish_ParserData *parser_data,
    xmlChar *tag,
    xmlChar **atts,
    xmlChar *xmlns_prefix
)
{
    int i, j, is_html_tag, size;
    xmlChar *swishtag,
            *tmpstr,
            *xmlns,
            *attr_lower, 
            *attr_val_lower, 
            *alias, 
            *metaname, 
            *metacontent, 
            *metaname_from_attr;
    swish_StringList *strlist;

    if (SWISH_DEBUG & SWISH_DEBUG_PARSER) {
        SWISH_DEBUG_MSG(" tag: %s   parser->tag: %s ", tag, parser_data->tag);
        if (atts != NULL) {
            SWISH_DEBUG_MSG(" has attributes [%d]", xmlStrlen((xmlChar *)atts));
            for (i = 0; (atts[i] != NULL); i += 2) {
                SWISH_DEBUG_MSG(" att: %s=", atts[i]);
                if (atts[i + 1] != NULL) {
                    SWISH_DEBUG_MSG(" '%s'", atts[i + 1]);
                }
            }
        }
    }

    metaname = NULL;
    metacontent = NULL;

/*
* normalize all tags 
*/
    swishtag = swish_str_tolower(tag);
    
    /* XML namespace support optional */
    if (xmlns_prefix != NULL && !parser_data->s3->config->flags->ignore_xmlns) {
        xmlns = swish_str_tolower(xmlns_prefix);
        if (SWISH_DEBUG & SWISH_DEBUG_PARSER) {
            SWISH_DEBUG_MSG("xmlns_prefix: '%s' (%s)", xmlns, xmlns_prefix);
        }
        size = xmlStrlen(swishtag) + xmlStrlen(xmlns) + 2;     /*  : + NUL */
        tmpstr = swish_xmalloc(size + 1);
        snprintf((char *)tmpstr, size, "%s%c%s", (char *)xmlns, 
            SWISH_XMLNS_CHAR, (char *)swishtag);
        swish_xfree(swishtag);
        swish_xfree(xmlns);
        swishtag = tmpstr;
    }

/*
* html tags 
*/
    if (parser_data->is_html) {

/*
           TODO config features about img tags and a/href tags 
*/
        if (xmlStrEqual(swishtag, (xmlChar *)"br")
            || xmlStrEqual(swishtag, (xmlChar *)"img")) {
            
            if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
                SWISH_DEBUG_MSG("found html tag '%s' ... bump_word = %d", swishtag, SWISH_TRUE);
            parser_data->bump_word = SWISH_TRUE;
        }
        else {
            const htmlElemDesc *element = htmlTagLookup(swishtag);

            if (!element)
                is_html_tag = 0;        /* flag that this might be a meta * name */

            else if (!element->isinline) {

/*
* need to bump token position so we don't match across block *
* elements 
*/
                if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
                    SWISH_DEBUG_MSG("found html !inline tag '%s' ... bump_word = %d", swishtag, SWISH_TRUE);
                parser_data->bump_word = SWISH_TRUE;

            }
            else {
            
                if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
                    SWISH_DEBUG_MSG("found html inline tag '%s' ... bump_word = %d", swishtag, SWISH_FALSE);
                parser_data->bump_word = SWISH_FALSE;
            
            }
        }

/*
* is this an HTML <meta> tag? treat 'name' attribute as a tag *
* and 'content' attribute as the tag content * we assume 'name'
* and 'content' are always in english. 
*/

        if (atts != NULL) {
            for (i = 0; (atts[i] != 0); i++) {

                if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
                    SWISH_DEBUG_MSG("%d HTML attr: %s", i, atts[i]);

                if (xmlStrEqual(atts[i], (xmlChar *)"name")) {

/*
* SWISH_DEBUG_MSG("found name: %s", atts[i+1]); 
*/
                    metaname = (xmlChar *)atts[i + 1];
                }

                else if (xmlStrEqual(atts[i], (xmlChar *)"content")) {

/*
* SWISH_DEBUG_MSG("found content: %s", atts[i+1]); 
*/
                    metacontent = (xmlChar *)atts[i + 1];
                }

            }
        }

        if (metaname != NULL) {
            if (metacontent != NULL) {
                if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
                    SWISH_DEBUG_MSG("found HTML meta: %s => %s", metaname, metacontent);

/*
* do not match across metas 
*/
                if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
                    SWISH_DEBUG_MSG("found HTML meta tag '%s' ... bump_word = %d", metaname, SWISH_TRUE);

                parser_data->bump_word = SWISH_TRUE;
                open_tag(parser_data, metaname, NULL, xmlns_prefix);
                buffer_characters(parser_data, metacontent, xmlStrlen(metacontent));
                close_tag(parser_data, metaname, xmlns_prefix);
                
                if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
                    SWISH_DEBUG_MSG("close_tag done. swishtag = '%s', parser->tag = '%s'", 
                        swishtag, parser_data->tag);
                        
                swish_xfree(parser_data->tag);  // metaname set recursively, so must free
                swish_xfree(swishtag);          // 'meta'
                
                return NULL;

            }
            else {
                SWISH_WARN("No content for meta tag '%s'", metaname);
            }
        }

    }

/*
* xml tags 
*/
    else {

        if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
            SWISH_DEBUG_MSG("found xml tag '%s' ... bump_word = %d", swishtag, SWISH_TRUE);
            
        parser_data->bump_word = SWISH_TRUE;

/*
    XML attributes are parsed in 2 ways:
    (1) <foo class="bar">text</foo>
        becomes
        <foo.bar>text</foo>
    
    (2) <foo class="bar">text</foo>
        becomes
        <foo.class>bar</foo.class><foo>text</foo>
        
    the (2)-style is similar to HTML parser for meta name/content

*/

        if (atts != NULL) {
            strlist = NULL;
            if (swish_hash_exists(parser_data->s3->config->stringlists, 
                    (xmlChar *)SWISH_CLASS_ATTRIBUTES)
            ) {
                strlist = swish_hash_fetch(parser_data->s3->config->stringlists,
                                 (xmlChar *)SWISH_CLASS_ATTRIBUTES);
            }

            for (i = 0; (atts[i] != NULL); i += 2) {

                if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
                    SWISH_DEBUG_MSG(" %d XML attr: %s=%s [%d]", i, atts[i], atts[i + 1],
                                    xmlStrlen(atts[i + 1]));

                attr_lower = swish_str_tolower(atts[i]);
                attr_val_lower = swish_str_tolower(atts[i + 1]);

/* is this attribute a metaname? */
                if (strlist != NULL) {
                    for (j = 0; j < strlist->n; j++) {
                        if (xmlStrEqual(strlist->word[j], attr_lower)) {
                            if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
                                SWISH_DEBUG_MSG("found %s: %s", attr_lower, attr_val_lower);
    
/* eligible attribute name, attribute value part of baked tag */
                            size = xmlStrlen(swishtag) + xmlStrlen(attr_val_lower) + 2;     /*  dot + NUL */
                            metaname = swish_xmalloc(size + 1);
                            snprintf((char *)metaname, size, "%s%c%s", (char *)swishtag, SWISH_DOT,
                                     (char *)attr_val_lower);
    
                            swish_xfree(swishtag);
                            swishtag = metaname;
                        }
                    }
                }
                
/* 
    explicit metaname with dotted notation. 
    attribute value considered document content, similar to how HTML parser works. 
*/
                size = xmlStrlen(swishtag) + xmlStrlen(attr_lower) + 2;     /*  dot + NUL */
                metaname_from_attr = swish_xmalloc(size + 1);
                snprintf((char *)metaname_from_attr, size, "%s%c%s", (char *)swishtag, SWISH_DOT, 
                    (char *)attr_lower);
                if (swish_hash_exists(parser_data->s3->config->metanames, metaname_from_attr)) {
                    if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
                        SWISH_DEBUG_MSG("found XML meta tag '%s' with content '%s'", 
                            metaname_from_attr, attr_val_lower);

                    parser_data->bump_word = SWISH_TRUE;
                    open_tag(parser_data, metaname_from_attr, NULL, xmlns_prefix);
                    buffer_characters(parser_data, attr_val_lower, xmlStrlen(attr_val_lower));
                    close_tag(parser_data, metaname_from_attr, xmlns_prefix);
                    swish_xfree(parser_data->tag);  // metaname set recursively, so must free
                
                    if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
                        SWISH_DEBUG_MSG("close_tag done. swishtag = '%s', parser->tag = '%s'", 
                            metaname_from_attr, parser_data->tag);
                
                }

                swish_xfree(metaname_from_attr);
                swish_xfree(attr_lower);
                swish_xfree(attr_val_lower);
            
            }
        }
    }

/*
* change our internal name for this tag if it is aliased in config 
*/
    alias = swish_hash_fetch(parser_data->s3->config->tag_aliases, swishtag);
    if (alias) {
        if (SWISH_DEBUG & SWISH_DEBUG_PARSER) {
            SWISH_DEBUG_MSG("%s alias -> %s", swishtag, alias); 
        }
        swish_xfree(swishtag);
        swishtag = swish_xstrdup(alias);
    }

    if (SWISH_DEBUG & SWISH_DEBUG_PARSER) {
        SWISH_DEBUG_MSG(" swishtag = %s", swishtag);
    }

    return swishtag;
}

static void
flush_buffer(
    swish_ParserData *parser_data,
    xmlChar *metaname,
    xmlChar *context
)
{
    swish_MetaName *meta;
    xmlChar *metaname_stored_as;
    swish_TagStack *s = parser_data->metastack;

    if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
        SWISH_DEBUG_MSG("buffer is >>%s<< before flush",
                        xmlBufferContent(parser_data->meta_buf));

/*
* add meta_buf as-is to metanames buffer under current tag. this
* gives us both tokens and raw text de-tagged but organized by
* metaname. If the metaname is an alias_for, use the target of the alias.
*/
    meta = swish_hash_fetch(parser_data->s3->config->metanames, metaname);
    if (meta->alias_for != NULL) {
        metaname_stored_as = meta->alias_for;
    }
    else {
        metaname_stored_as = metaname;
    }
    swish_nb_add_buf(parser_data->metanames, metaname_stored_as, parser_data->meta_buf,
                        (xmlChar *)SWISH_TOKENPOS_BUMPER, 0, 1);

/*
*  if cascade_meta_context is true, add tokens (buffer) to every metaname on the stack.
*/

    if (parser_data->s3->config->flags->cascade_meta_context) {
        for (s->temp = s->head; s->temp != NULL; s->temp = s->temp->next) {
            if (xmlStrEqual(s->temp->baked, metaname_stored_as))  /*  already added */
                continue;

            swish_nb_add_buf(parser_data->metanames, s->temp->baked,
                                parser_data->meta_buf, (xmlChar *)SWISH_TOKENPOS_BUMPER,
                                0, 1);
        }
    }

    if (parser_data->s3->analyzer->tokenize) {
        tokenize(parser_data, (xmlChar *)xmlBufferContent(parser_data->meta_buf),
                 xmlBufferLength(parser_data->meta_buf), metaname_stored_as, context);
    }

    xmlBufferEmpty(parser_data->meta_buf);

}

/* 
* SAX2 callback 
*/
static void
mystartDocument(
    void *data
)
{

/*
* swish_ParserData *parser_data = (swish_ParserData *) data; 
*/

    if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
        SWISH_DEBUG_MSG("startDocument()");

}

/* 
* SAX2 callback 
*/
static void
myendDocument(
    void *parser_data
)
{

    if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
        SWISH_DEBUG_MSG("endDocument()");

/*
* whatever's left 
*/
    flush_buffer(parser_data, (xmlChar *)SWISH_DEFAULT_METANAME,
                 (xmlChar *)SWISH_DEFAULT_METANAME);

}

/* 
* SAX1 callback 
*/
static void
mystartElement(
    void *data,
    const xmlChar *name,
    const xmlChar **atts
)
{
    open_tag(data, name, (xmlChar **)atts, NULL);
}

/* 
* SAX1 callback 
*/
static void
myendElement(
    void *data,
    const xmlChar *name
)
{
    close_tag(data, name, NULL);
}

/* 
* SAX2 handler 
*/
static void
mystartElementNs(
    void *data,
    const xmlChar *localname,
    const xmlChar *xmlns_prefix,
    const xmlChar *URI,
    int nb_namespaces,
    const xmlChar **namespaces,
    int nb_attributes,
    int nb_defaulted,
    const xmlChar **attributes
)
{
    int i, j, len;
    xmlChar **atts;
    atts = NULL;

    if (nb_attributes > 0) {
        atts = swish_xmalloc(((nb_attributes * 2) + 1) * sizeof(xmlChar *));
        j = 0;
        for (i = 0; i < nb_attributes * 5; i += 5) {
            atts[j] = (xmlChar *)attributes[i];
            len = (int)(attributes[i + 4] - attributes[i + 3]);
            if (len > 0) {
                atts[j + 1] = xmlStrsub(attributes[i + 3], 0, len);
            }
            else {
                atts[j] = NULL;
            }
            j += 2;
        }
        atts[j] = '\0';
    }

    if (SWISH_DEBUG & SWISH_DEBUG_PARSER) {
        SWISH_DEBUG_MSG(" tag: %s nb_attributes %d", localname, nb_attributes);
        if (atts != NULL) {
            for (i = 0; (atts[i] != NULL); i += 2) {
                SWISH_DEBUG_MSG(" att: %s=%s", atts[i], atts[i + 1]);
/* SWISH_DEBUG_MSG(" att: %s=", atts[i++], atts[i] || ""); */
            }
        }
    }

    open_tag(data, localname, atts, xmlns_prefix);

    if (atts != NULL) {
        for (i = 0; (atts[i] != NULL); i += 2) {
            xmlFree(atts[i+1]); /* do not use swish_xfree since we did not malloc it */
        }
        swish_xfree(atts);
    }
}

/* 
* SAX2 handler 
*/
static void
myendElementNs(
    void *data,
    const xmlChar *localname,
    const xmlChar *xmlns_prefix,
    const xmlChar *URI
)
{
    close_tag(data, localname, xmlns_prefix);
}

static void
open_tag(
    void *data,
    const xmlChar *tag,
    xmlChar **atts,
    const xmlChar *xmlns_prefix
)
{
    swish_ParserData *parser_data;
    xmlChar *baked;
    
    parser_data = (swish_ParserData *)data;
    
    if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
        SWISH_DEBUG_MSG("<%s>", tag);

    if (parser_data->tag != NULL) {
        if (SWISH_DEBUG & SWISH_DEBUG_PARSER) {
            SWISH_DEBUG_MSG("Freeing swishtag (parser_data->tag): '%s'", parser_data->tag);
        }
        swish_xfree(parser_data->tag);
        parser_data->tag = NULL;
    }

    parser_data->tag = bake_tag(
                parser_data, 
                (xmlChar *)tag, 
                (xmlChar **)atts, 
                (xmlChar *)xmlns_prefix);
        
    if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
        SWISH_DEBUG_MSG("checking config for '%s' in watched tags", parser_data->tag);

/* all tags on domstack */

    if (parser_data->tag == NULL) {
        push_tag_stack(parser_data->domstack, (xmlChar *)tag, (xmlChar *)tag, SWISH_DOT);
    }
    else {
        push_tag_stack(parser_data->domstack, (xmlChar *)tag, parser_data->tag, SWISH_DOT);
    }
    
/*
* set property if this tag is configured for it 
*/
    if (swish_hash_exists(parser_data->s3->config->properties, parser_data->tag)
        ||
        swish_hash_exists(parser_data->s3->config->properties, parser_data->domstack->head->context)
    ) {
        if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
            SWISH_DEBUG_MSG(" %s = new property", parser_data->tag);

        add_stack_to_prop_buf(NULL, parser_data);       /* NULL means all properties in the stack are added */
        xmlBufferEmpty(parser_data->prop_buf);
        
        if (swish_hash_exists(parser_data->s3->config->properties, parser_data->domstack->head->context))
            baked = parser_data->domstack->head->context;
        else
            baked = parser_data->tag;

        push_tag_stack(parser_data->propstack, (xmlChar *)tag, baked, SWISH_DOM_CHAR);

        if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
            SWISH_DEBUG_MSG("%s pushed ok unto propstack", baked);
    }

/*
* likewise for metastack 
*/
    if (swish_hash_exists(parser_data->s3->config->metanames, parser_data->tag)
        ||
        swish_hash_exists(parser_data->s3->config->metanames, parser_data->domstack->head->context)
    ) {
        if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
            SWISH_DEBUG_MSG(" %s = new metaname", parser_data->tag);

        flush_buffer(parser_data, parser_data->metastack->head->baked,
                     parser_data->metastack->head->context);
                     
        if (swish_hash_exists(parser_data->s3->config->properties, parser_data->domstack->head->context))
            baked = parser_data->domstack->head->context;
        else
            baked = parser_data->tag;

        push_tag_stack(parser_data->metastack, (xmlChar *)tag, baked, SWISH_DOM_CHAR);
    }

    if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
        SWISH_DEBUG_MSG("config check for '%s' done", parser_data->tag);

}

static void
close_tag(
    void *data,
    const xmlChar *tag,
    const xmlChar *xmlns_prefix
)
{
    swish_ParserData *parser_data;
    swish_Tag *st;

    parser_data = (swish_ParserData *)data;

    if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
        SWISH_DEBUG_MSG("</%s>", tag);
        
/*
* lowercase all names for comparison against metanames (which are
* also * lowercased) 
*/
    if (parser_data->tag != NULL) {
        if (SWISH_DEBUG & SWISH_DEBUG_PARSER) {
            SWISH_DEBUG_MSG("freeing parser_data->tag '%s'", parser_data->tag);
        }
        swish_xfree(parser_data->tag);
    }
    
    parser_data->tag = bake_tag(parser_data, (xmlChar *)tag, NULL, (xmlChar *)xmlns_prefix);

    if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
        SWISH_DEBUG_MSG(" endElement(%s) (%s)", (xmlChar *)tag, parser_data->tag);
        
    if (parser_data->tag == NULL)
        return;

    if ((st = pop_tag_stack_on_match(parser_data->propstack, (xmlChar *)tag)) != NULL) {

        add_stack_to_prop_buf(st->baked, parser_data);
        xmlBufferEmpty(parser_data->prop_buf);
        free_swishTag(st);
    }

    if ((st = pop_tag_stack_on_match(parser_data->metastack, (xmlChar *)tag)) != NULL) {

        //SWISH_DEBUG_MSG("flush_buffer before free_swishTag");
        flush_buffer(parser_data, st->baked, st->context);
        //SWISH_DEBUG_MSG("metastack pop_tag_stack_on_match free_swishTag");
        free_swishTag(st);
    }
    
    // always pop the raw domstack
    st = pop_tag_stack(parser_data->domstack);
    free_swishTag(st);
    //SWISH_DEBUG_MSG("free_swishTag raw DOMstack done");

}

/* 
* handle all characters in doc 
*/
static void
buffer_characters(
    swish_ParserData *parser_data,
    const xmlChar *ch,
    int len
)
{
    int i;
    xmlChar output[len];
    xmlBufferPtr buf = parser_data->meta_buf;

/*
* why not wchar_t ? len is number of bytes, not number of
* characters, so xmlChar (i.e., char) works
*/

/*
* SWISH_DEBUG_MSG( "sizeof output buf is %d; len was %d\n", sizeof(output),
* len );
*/

    if (SWISH_DEBUG & SWISH_DEBUG_PARSER) {
        SWISH_DEBUG_MSG("appending %d bytes to buffer (bump_word=%d)", 
            len, parser_data->bump_word);
    }

    for (i = 0; i < len; i++) {
        output[i] = ch[i];
    }
    output[i] = '\0';

    if (parser_data->bump_word && xmlBufferLength(buf)) {
        //SWISH_DEBUG_MSG("bump_word is true; appending TOKENPOS_BUMPER to buffer");
        swish_buffer_append(buf, (xmlChar *)SWISH_TOKENPOS_BUMPER, 1);
    }
    
    swish_buffer_append(buf, output, len);

    if (parser_data->bump_word && xmlBufferLength(parser_data->prop_buf)) {
        //SWISH_DEBUG_MSG("bump_word is true; appending TOKENPOS_BUMPER to buffer");
        swish_buffer_append(parser_data->prop_buf, (xmlChar *)SWISH_TOKENPOS_BUMPER, 1);
    }
    else if (xmlBufferLength(parser_data->prop_buf)) {
        swish_buffer_append(parser_data->prop_buf, (xmlChar*)" ", 1);
    }

    swish_buffer_append(parser_data->prop_buf, output, len);
    
    // reset
    parser_data->bump_word = SWISH_FALSE;
}

/* 
* SAX2 callback 
*/
static void
mycharacters(
    void *parser_data,
    const xmlChar *ch,
    int len
)
{
    if (SWISH_DEBUG & SWISH_DEBUG_PARSER) {
        int i;
        for (i=0; i<len; i++) {
            SWISH_DEBUG_MSG("%c [%d]", ch[i], i);
        }
    }
    
    buffer_characters(parser_data, ch, len);
}

/* 
* SAX2 callback 
*/
static void
mycomments(
    void *parser_data,
    const xmlChar *ch
)
{
    int len = strlen((char *)(char *)ch);

/*
* TODO: make comments indexing optional 
*/

/*
* TODO: enable noindex option 
*/
    return;

    buffer_characters(parser_data, ch, len);
}

/* 
* SAX2 callback 
*/
static void
myerr(
    void *data,
    xmlChar *msg,
    ...
)
{
    swish_ParserData *parser_data;
    va_list args;
    char str[1000];

    if (!SWISH_PARSER_WARNINGS)
        return;

    parser_data = (swish_ParserData *)data;

    SWISH_WARN("libxml2 error for %s:", parser_data->docinfo->uri);

    va_start(args, msg);
    vsnprintf((char *)str, 1000, (char *)msg, args);
    /* passing args as last param is ignored but quiets a gcc warning */
    xmlParserError(parser_data->ctxt, (char *)str, args);
    va_end(args);
}

/* 
* SAX2 callback 
*/
static void
mywarn(
    void *user_data,
    xmlChar *msg,
    ...
)
{
    swish_ParserData *parser_data;
    va_list args;
    char str[1000];

    if (!SWISH_PARSER_WARNINGS)
        return;

    parser_data = (swish_ParserData *)user_data;

    SWISH_WARN("libxml2 warning for %s:", parser_data->docinfo->uri);
    if (parser_data->ctxt == NULL) {
        SWISH_WARN("ctxt is null");
    }

    va_start(args, msg);
    vsnprintf((char *)str, 1000, (char *)msg, args);
    /* passing args as last param is ignored but quiets a gcc warning */
    xmlParserWarning(parser_data->ctxt, (char *)str, args);
    va_end(args);
}

/* 
* SAX2 handler struct for html and xml parsing 
*/

xmlSAXHandler my_parser = {
    NULL,                       /* internalSubset */
    NULL,                       /* isStandalone */
    NULL,                       /* hasInternalSubset */
    NULL,                       /* hasExternalSubset */
    NULL,                       /* resolveEntity */
    NULL,                       /* getEntity */
    NULL,                       /* entityDecl */
    NULL,                       /* notationDecl */
    NULL,                       /* attributeDecl */
    NULL,                       /* elementDecl */
    NULL,                       /* unparsedEntityDecl */
    NULL,                       /* setDocumentLocator */
    mystartDocument,            /* startDocument */
    myendDocument,              /* endDocument */
    mystartElement,             /* startElement */
    myendElement,               /* endElement */
    NULL,                       /* reference */
    mycharacters,               /* characters */
    NULL,                       /* ignorableWhitespace */
    NULL,                       /* processingInstruction */
    mycomments,                 /* comment */
    (warningSAXFunc) & mywarn,  /* xmlParserWarning */
    (errorSAXFunc) & mywarn,     /* xmlParserError */
    (fatalErrorSAXFunc) & myerr,        /* xmlfatalParserError */
    NULL,                       /* getParameterEntity */
    NULL,                       /* cdataBlock -- should we handle this too *
                                 * ?? */
    NULL,                       /* externalSubset; */
    XML_SAX2_MAGIC,
    NULL,
    mystartElementNs,           /* startElementNs */
    myendElementNs,             /* endElementNs */
    NULL                        /* xmlStructuredErrorFunc */
};

xmlSAXHandlerPtr my_parser_ptr = &my_parser;

static int
docparser(
    swish_ParserData *parser_data,
    xmlChar *filename,
    xmlChar *buffer,
    int size
)
{

    int ret;
    ret = 0;
    xmlChar *mime = (xmlChar *)parser_data->docinfo->mime;
    xmlChar *parser = (xmlChar *)parser_data->docinfo->parser;

    if (!size && !xmlStrlen(buffer) && !parser_data->docinfo->size) {
        SWISH_WARN("%s appears to be empty -- can't parse it", parser_data->docinfo->uri);

        return 1;
    }

    if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
        SWISH_DEBUG_MSG("%s -- using %s parser [%c]", parser_data->docinfo->uri, parser, parser[0]);

/*
* slurp file if not already in memory 
*/
    if (filename && !buffer) {
        if (parser_data->docinfo->is_gzipped) {
            buffer = swish_io_slurp_gzfile_len(
                filename, 
                &(parser_data->docinfo->size), 
                SWISH_FALSE
            );
            parser_data->docinfo->size = xmlStrlen(buffer);
        }
        else {
            buffer = swish_io_slurp_file_len(
                filename, 
                (off_t)parser_data->docinfo->size,
                SWISH_FALSE
            );
        }
        size = parser_data->docinfo->size;
    }

    if (parser[0] == 'H' || parser[0] == 'h') {
        parser_data->is_html = SWISH_TRUE;
        ret = html_parser(my_parser_ptr, parser_data, buffer, size);
    }

    else if (parser[0] == 'X' || parser[0] == 'x')
        ret = xml_parser(my_parser_ptr, parser_data, buffer, size);

    else if (parser[0] == 'T' || parser[0] == 't')
        ret = txt_parser(parser_data, (xmlChar *)buffer, size);

    else
        SWISH_CROAK("no parser known for MIME '%s'", mime);

    if (filename) {

/*
* SWISH_DEBUG_MSG( "freeing buffer"); 
*/
        swish_xfree(buffer);
    }

    return ret;

}

static swish_ParserData *
init_parser_data(
    swish_3 *s3
)
{

    if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
        SWISH_DEBUG_MSG("init parser_data");

    swish_ParserData *ptr = (swish_ParserData *)swish_xmalloc(sizeof(swish_ParserData));

    ptr->s3 = s3;
    ptr->s3->ref_cnt++;

    ptr->meta_buf = xmlBufferCreateSize(SWISH_BUFFER_CHUNK_SIZE);
    ptr->prop_buf = xmlBufferCreateSize(SWISH_BUFFER_CHUNK_SIZE);

    ptr->tag = NULL;
    ptr->token_iterator = swish_token_iterator_init(s3->analyzer);
    ptr->token_iterator->ref_cnt++;
    ptr->properties = swish_nb_init(s3->config->properties);
    ptr->properties->ref_cnt++;
    ptr->metanames = swish_nb_init(s3->config->metanames);
    ptr->metanames->ref_cnt++;

/*
*   set tokenizer if one has not been explicitly set
*/
    if (s3->analyzer->tokenizer == NULL) {
        s3->analyzer->tokenizer = (&swish_tokenize);
    }

/*
* prime the stacks 
*/
    ptr->metastack = (swish_TagStack *)swish_xmalloc(sizeof(swish_TagStack));
    ptr->metastack->name = "MetaStack";
    ptr->metastack->head = NULL;
    ptr->metastack->temp = NULL;
    ptr->metastack->count = 0;
    push_tag_stack(ptr->metastack, (xmlChar *)SWISH_DEFAULT_METANAME,
                   (xmlChar *)SWISH_DEFAULT_METANAME, SWISH_DOM_CHAR);

    ptr->propstack = (swish_TagStack *)swish_xmalloc(sizeof(swish_TagStack));
    ptr->propstack->name = "PropStack";
    ptr->propstack->head = NULL;
    ptr->propstack->temp = NULL;
    ptr->propstack->count = 0;
    push_tag_stack(ptr->propstack, (xmlChar *)SWISH_DOM_STR, (xmlChar *)SWISH_DOM_STR, SWISH_DOM_CHAR);
    
    ptr->domstack  = (swish_TagStack *)swish_xmalloc(sizeof(swish_TagStack));
    ptr->domstack->name  = "DOMStack";
    ptr->domstack->head  = NULL;
    ptr->domstack->temp  = NULL;
    ptr->domstack->count = 0;

/*
* gets toggled per-tag 
*/
    ptr->bump_word = SWISH_TRUE;

/*
* toggle 
*/
    ptr->no_index = SWISH_FALSE;

/*
* shortcut rather than looking parser up in hash for each tag event 
*/
    ptr->is_html = SWISH_FALSE;

/*
* always start at first byte 
*/
    ptr->offset = 0;

/*
* pointer to the xmlParserCtxt since we want to free it only after
* we're completely done with it. NOTE this is a change per libxml2
* vers > 2.6.16 
*/
    ptr->ctxt = NULL;

    if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
        SWISH_DEBUG_MSG("init done for parser_data");

    return ptr;

}

static void
free_swishTagStack(
    swish_TagStack *stack
)
{
    swish_Tag *st;
    
    while ((st = pop_tag_stack(stack)) != NULL) {
        if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
            SWISH_DEBUG_MSG("%s %d POP %s [%s] [%s]", stack->name,
                            stack->count, st->raw, st->baked, st->context);

        free_swishTag(st);
    }

    if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
        SWISH_DEBUG_MSG("freeing stack %s", stack->name);

    swish_xfree(stack);
}

static void
free_parser_data(
    swish_ParserData *ptr
)
{

    if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
        SWISH_DEBUG_MSG("freeing swish_ParserData");

/*
* dec ref count for shared ptr 
*/
    ptr->s3->ref_cnt--;

/*
* Pop the stacks 
*/
    free_swishTagStack(ptr->metastack);
    free_swishTagStack(ptr->propstack);
    free_swishTagStack(ptr->domstack);
    
/* free named buffers */

    if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
        SWISH_DEBUG_MSG("freeing swish_ParserData properties");

    ptr->properties->ref_cnt--;
    swish_nb_free(ptr->properties);

    if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
        SWISH_DEBUG_MSG("freeing swish_ParserData metanames");

    ptr->metanames->ref_cnt--;
    swish_nb_free(ptr->metanames);

    if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
        SWISH_DEBUG_MSG("freeing swish_ParserData xmlBuffer");

    xmlBufferFree(ptr->meta_buf);

    if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
        SWISH_DEBUG_MSG("freeing swish_ParserData prop xmlBuffer");

    xmlBufferFree(ptr->prop_buf);

    if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
        SWISH_DEBUG_MSG("freeing swish_ParserData tag");

    if (ptr->tag != NULL)
        swish_xfree(ptr->tag);

    if (ptr->ctxt != NULL) {

        if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
            SWISH_DEBUG_MSG("freeing swish_ParserData libxml2 parser ctxt");

        if (xmlStrEqual(ptr->docinfo->parser, (xmlChar *)SWISH_PARSER_XML))
            xmlFreeParserCtxt(ptr->ctxt);

        if (xmlStrEqual(ptr->docinfo->parser, (xmlChar *)SWISH_PARSER_HTML))
            htmlFreeParserCtxt(ptr->ctxt);
    }
    else {
        if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
            SWISH_DEBUG_MSG("swish_ParserData libxml2 parser ctxt already freed");

    }

    if (ptr->token_iterator != NULL) {

        if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
            SWISH_DEBUG_MSG("free swish_ParserData TokenIterator");

        ptr->token_iterator->ref_cnt--;
        swish_token_iterator_free(ptr->token_iterator);
    }

    if (ptr->docinfo != NULL) {

        if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
            SWISH_DEBUG_MSG("free swish_ParserData docinfo");

        ptr->docinfo->ref_cnt--;
        swish_docinfo_free(ptr->docinfo);

    }

    if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
        SWISH_DEBUG_MSG("freeing swish_ParserData ptr");

    swish_xfree(ptr);

    if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
        SWISH_DEBUG_MSG("swish_ParserData all freed");
}

static HEAD *
buf_to_head(
    xmlChar *buf
)
{
    int i, j, k;
    xmlChar *line;
    const xmlChar *newlines;
    HEAD *h;

    if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
        SWISH_DEBUG_MSG("parsing head from buffer: %s", buf);

    h = swish_xmalloc(sizeof(HEAD));
    h->lines = swish_xmalloc(SWISH_MAX_HEADERS * sizeof(line));
    h->nlines = 0;
    h->body_start = 0;
    line = swish_xmalloc(SWISH_MAXSTRLEN + 1);
    i = 0;
    j = 0;
    k = 0;

    while (j < SWISH_MAX_HEADERS && i <= SWISH_MAXSTRLEN) {

/*
* SWISH_DEBUG_MSG( "i = %d j = %d k = %d", i, j, k); 
*/

        if (buf[k] == '\n') {
            SWISH_CROAK("illegal newline to start doc header");
        }
        line[i] = buf[k];

/*
* fprintf(stderr, "%c", line[i]); 
*/
        i++;
        k++;

        if (buf[k] == '\n') {

            line[i] = '\0';
            h->lines[j++] = swish_xstrdup(line);
            h->nlines++;

/*
* get to the next char no matter what, then check if == '\n' 
*/
            k++;

            if (buf[k] == '\n' || buf[k] == '\0') {

                if (SWISH_DEBUG & SWISH_DEBUG_DOCINFO) {
                    SWISH_DEBUG_MSG("found blank header line at byte %d\n", k);
                }
                
                h->body_start = k + 1;
                break;
            }
            i = 0;

            continue;
        }
    }
    
    swish_xfree(line);

    /* sanity check */
    if (SWISH_DEBUG & SWISH_DEBUG_DOCINFO) {
        SWISH_DEBUG_MSG("finished parsing head from buffer");
        newlines = xmlStrstr((const xmlChar*)buf, (const xmlChar*)"\n\n");
        if (newlines != NULL) {
            SWISH_DEBUG_MSG("strstr found body start at %d; loop at %d",
            (int)(buf - newlines), h->body_start);
        }
    }


    return h;
}

static swish_DocInfo *
head_to_docinfo(
    HEAD * h
)
{
    int i;
    xmlChar *val, *line;

    swish_DocInfo *info = swish_docinfo_init();

    info->ref_cnt++;

    if (SWISH_DEBUG & SWISH_DEBUG_DOCINFO)
        SWISH_DEBUG_MSG("preparing to parse %d header lines", h->nlines);

    for (i = 0; i < h->nlines; i++) {

        line = h->lines[i];

        if (SWISH_DEBUG & SWISH_DEBUG_DOCINFO)
            SWISH_DEBUG_MSG("parsing header line: >%s<", line);

        val = (xmlChar *)xmlStrchr(line, ':');
        if(!val) {
            SWISH_CROAK("bad header line: %s", line);
        }
        val = swish_str_skip_ws(++val);

        if (SWISH_DEBUG & SWISH_DEBUG_DOCINFO) {
            SWISH_DEBUG_MSG("%d parsing header line: %s", i, line);

        }

        if (!xmlStrncasecmp(line, (const xmlChar *)"Content-Length", 14)) {
            if (!val)
                SWISH_WARN("Failed to parse Content-Length header '%s'", line);

            info->size = swish_string_to_int((char *)val);
            continue;
        }
        if (!xmlStrncasecmp(line, (const xmlChar *)"Last-Modified", 13)) {

            if (!val)
                SWISH_WARN("Failed to parse Last-Modified header '%s'", line);

            info->mtime = swish_string_to_int((char *)val);
            continue;
        }
        if (!xmlStrncasecmp(line, (const xmlChar *)"Last-Mtime", 10)) {

            SWISH_WARN("%s: Last-Mtime is deprecated in favor of Last-Modified", val);

            if (!val)
                SWISH_WARN("Failed to parse Last-Mtime header '%s'", line);

            info->mtime = swish_string_to_int((char *)val);
            continue;
        }
        if (!xmlStrncasecmp(line, (const xmlChar *)"Content-Location", 16)) {

            if (!val)
                SWISH_WARN("Failed to parse Content-Location header '%s'", line);

            if (!*val)
                SWISH_WARN("Failed to find path name in Content-Location header '%s'",
                           line);

            if (info->uri != NULL)
                swish_xfree(info->uri);

            info->uri = swish_xstrdup(val);
            continue;
        }
        if (!xmlStrncasecmp(line, (const xmlChar *)"Path-Name", 9)) {

            SWISH_WARN("%s: Path-Name is deprecated in favor of Content-Location", val);

            if (!val)
                SWISH_WARN("Failed to parse Path-Name header '%s'", line);

            if (!*val)
                SWISH_WARN("Failed to find path name in Path-Name header '%s'", line);

            if (info->uri != NULL)
                swish_xfree(info->uri);

            info->uri = swish_xstrdup(val);
            continue;
        }
        if (!xmlStrncasecmp(line, (const xmlChar *)"Document-Type", 13)) {

            SWISH_WARN("%s: Document-Type is deprecated in favor of Parser-Type", val);

            if (!val)
                SWISH_WARN("Failed to parse Document-Type header '%s'", line);

            if (!*val)
                SWISH_WARN("Failed to find path name in Document-Type header '%s'", line);

            if (info->parser != NULL)
                swish_xfree(info->parser);

            info->parser = swish_xstrdup(val);
            continue;
        }
        if (!xmlStrncasecmp(line, (const xmlChar *)"Parser-Type", 11)) {

            if (!val)
                SWISH_WARN("Failed to parse Parser-Type header '%s'", line);

            if (!*val)
                SWISH_WARN("Failed to find path name in Parser-Type header '%s'", line);

            if (info->parser != NULL)
                swish_xfree(info->parser);

            info->parser = swish_xstrdup(val);
            continue;
        }
        if (!xmlStrncasecmp(line, (const xmlChar *)"Content-Type", 12)) {

            if (!val)
                SWISH_WARN("Failed to parse Content-Type header '%s'", line);

            if (!*val)
                SWISH_WARN("Failed to find path name in Content-Type header '%s'", line);

/*
* TODO: get encoding out of this line too if
* present. example:   text/xml; charset=ISO-8859-1
*/

            if (info->mime != NULL)
                swish_xfree(info->mime);

            info->mime = swish_xstrdup(val);
            continue;
        }
        if (!xmlStrncasecmp(line, (const xmlChar *)"Encoding", 8)
            || !xmlStrncasecmp(line, (const xmlChar *)"Charset", 7)) {

            if (!val)
                SWISH_WARN("Failed to parse Encoding or Charset header '%s'", line);

            if (!*val)
                SWISH_WARN("Failed to find value in Encoding or Charset header '%s'",
                           line);

            if (info->encoding != NULL)
                swish_xfree(info->encoding);

            info->encoding = swish_xstrdup(val);
            continue;
        }
        if (!xmlStrncasecmp(line, (const xmlChar *)"Action", 11)) {

            if (!val)
                SWISH_WARN("Failed to parse Action header '%s'", line);

            if (!*val)
                SWISH_WARN("Failed to find value in Action header '%s'", line);

            if (info->action != NULL)
                swish_xfree(info->action);

            info->action = swish_xstrdup(val);
            continue;
        }

/*
* if we get here, unrecognized header line 
*/
        SWISH_WARN("Unknown header line: '%s'\n", line);

    }

    if (SWISH_DEBUG & SWISH_DEBUG_DOCINFO) {
        SWISH_DEBUG_MSG("returning %d header lines", h->nlines);
        swish_docinfo_debug(info);
    }

    return info;
}

static void
get_env_vars(
)
{

/*
* init the global env vars, but don't override if already set 
*/

    setenv("SWISH_PARSER_WARNINGS", "1", 0);
    SWISH_PARSER_WARNINGS = swish_string_to_int(getenv("SWISH_PARSER_WARNINGS"));

    if (SWISH_DEBUG) {
        SWISH_PARSER_WARNINGS = SWISH_DEBUG;
    }
    
}

unsigned int
swish_parse_fh(
    swish_3 *s3,
    FILE * fh
)
{
    xmlChar *ln;
    HEAD *head;
    int i;
    xmlChar *read_buffer;
    xmlBufferPtr head_buf;
    swish_ParserData *parser_data;
    int xmlErr;
    int min_headers, nheaders;
    double curTime;
    char *etime;
    unsigned int file_cnt;

    i = 0;
    file_cnt = 0;
    nheaders = 0;
    min_headers = 2;

    if (fh == NULL)
        fh = stdin;

    ln = swish_xmalloc(SWISH_MAXSTRLEN + 1);
    head_buf =
        xmlBufferCreateSize((SWISH_MAX_HEADERS * SWISH_MAXSTRLEN) + SWISH_MAX_HEADERS);

/*
* based on extprog.c 
*/
    while (fgets((char *)ln, SWISH_MAXSTRLEN, fh) != NULL) {

/*
* we don't use fgetws() because we don't care about * indiv
* characters yet 
*/

        xmlChar *end;
        xmlChar *line;

        line = swish_str_skip_ws(ln);   /* skip leading white space */
        end = (xmlChar *)strrchr((char *)line, '\n');

/*
* trim any white space at end of doc, including \n 
*/
        if (end) {
            while (end > line && isspace((int)*(end - 1)))
                end--;

            *end = '\0';
        }

        if (nheaders >= min_headers && xmlStrlen(line) == 0) {

/*
* blank line indicates body 
*/
            curTime = swish_time_elapsed();
            parser_data = init_parser_data(s3);
            head = buf_to_head((xmlChar *)xmlBufferContent(head_buf));
            parser_data->docinfo = head_to_docinfo(head);
            swish_docinfo_check(parser_data->docinfo, s3->config);

            if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
                SWISH_DEBUG_MSG("reading %ld bytes from filehandle",
                                (long int)parser_data->docinfo->size);

            read_buffer = swish_io_slurp_fh(fh, parser_data->docinfo->size, SWISH_FALSE);

/*
* parse 
*/
            xmlErr =
                docparser(parser_data, NULL, read_buffer, parser_data->docinfo->size);

            if (xmlErr)
                SWISH_WARN("parser returned error %d", xmlErr);

            if (SWISH_DEBUG & SWISH_DEBUG_PARSER) {
                SWISH_DEBUG_MSG
                    ("\n===============================================================\n");
                swish_docinfo_debug(parser_data->docinfo);
                SWISH_DEBUG_MSG("  word buffer length: %d bytes",
                                xmlBufferLength(parser_data->meta_buf));
                SWISH_DEBUG_MSG(" (%d words)", parser_data->docinfo->nwords);
            }
            if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
                SWISH_DEBUG_MSG("passing to handler");

/*
* pass to callback function 
*/
            (*s3->parser->handler) (parser_data);

            if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
                SWISH_DEBUG_MSG("handler done");

/*
* reset everything for next time 
*/

            swish_xfree(read_buffer);
            free_parser_data(parser_data);
            free_head(head);
            xmlBufferEmpty(head_buf);
            nheaders = 0;

/*
* count the file 
*/
            file_cnt++;

            if (SWISH_DEBUG) {
                etime = swish_time_print_fine(swish_time_elapsed() - curTime);
                SWISH_DEBUG_MSG("%s elapsed time", etime);
                swish_xfree(etime);
            }

/*
* timer 
*/
            curTime = swish_time_elapsed();

            if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
                SWISH_DEBUG_MSG
                    ("\n================ filehandle - done with file ===================\n");

        }
        else if (xmlStrlen(line) == 0) {

            SWISH_CROAK("Not enough header lines reading from filehandle");

        }
        else {

/*
* we are reading headers 
*/
            if (xmlBufferAdd(head_buf, line, -1))
                SWISH_CROAK("error adding header to buffer");

            if ((xmlBufferCCat(head_buf, "\n")) != 0)
                SWISH_CROAK("can't add newline to end of header buffer");

            nheaders++;
            
            if (SWISH_DEBUG & SWISH_DEBUG_DOCINFO) {
                SWISH_DEBUG_MSG("nheaders = %d for buffer >%s<", 
                    nheaders, xmlBufferContent(head_buf));
            }
        }

    }

    if (xmlBufferLength(head_buf)) {
        SWISH_CROAK("Some unparsed header lines remaining");
    }

    swish_xfree(ln);
    xmlBufferFree(head_buf);

    return file_cnt;
}

static void
free_head(
    HEAD * h
)
{
    int i;

    for (i = 0; i < h->nlines; i++) {
        swish_xfree(h->lines[i]);
    }
    swish_xfree(h->lines);
    swish_xfree(h);
}

/* 
* PUBLIC 
*/

/* 
* pass in a string including headers. like parsing fh, but only for one
* doc
*/
int
swish_parse_buffer(
    swish_3 *s3,
    xmlChar *buf
)
{

    int res;
    double curTime = swish_time_elapsed();
    HEAD *head;
    char *etime;

    head = buf_to_head(buf);

    if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
        SWISH_DEBUG_MSG("number of headlines: %d", head->nlines);

    swish_ParserData *parser_data = init_parser_data(s3);

    parser_data->docinfo = head_to_docinfo(head);
    swish_docinfo_check(parser_data->docinfo, s3->config);

/*
* reposition buf pointer at start of body (just past head) 
*/

    buf += head->body_start;

    res = docparser(parser_data, 0, buf, xmlStrlen(buf));

/*
* pass to callback function 
*/
    (*s3->parser->handler) (parser_data);

    if (SWISH_DEBUG & SWISH_DEBUG_PARSER) {
        swish_docinfo_debug(parser_data->docinfo);
        SWISH_DEBUG_MSG("  word buffer length: %d bytes",
                        xmlBufferLength(parser_data->meta_buf));
        SWISH_DEBUG_MSG(" (%d words)", parser_data->docinfo->nwords);
    }

/*
* free buffers 
*/
    free_head(head);
    free_parser_data(parser_data);

    if (SWISH_DEBUG) {
        etime = swish_time_print_fine(swish_time_elapsed() - curTime);
        SWISH_DEBUG_MSG("%s elapsed time", etime);
        swish_xfree(etime);
    }
    curTime = swish_time_elapsed();

    return res;

}

/* 
* PUBLIC 
*/
int
swish_parse_file(
    swish_3 *s3,
    xmlChar *filename
)
{
    int res;
    double curTime = swish_time_elapsed();
    char *etime;

    swish_ParserData *parser_data = init_parser_data(s3);

    parser_data->docinfo = swish_docinfo_init();
    parser_data->docinfo->ref_cnt++;

    if (!swish_docinfo_from_filesystem(filename, parser_data->docinfo, parser_data)) {
        SWISH_WARN("Skipping %s", filename);
        free_parser_data(parser_data);
        return 1;
    }

    res = docparser(parser_data, filename, 0, 0);

/*
* pass to callback function 
*/
    (*s3->parser->handler) (parser_data);

    if (SWISH_DEBUG & SWISH_DEBUG_PARSER) {
        swish_docinfo_debug(parser_data->docinfo);
        SWISH_DEBUG_MSG("  word buffer length: %d bytes",
                        xmlBufferLength(parser_data->meta_buf));
        SWISH_DEBUG_MSG(" (%d words)", parser_data->docinfo->nwords);
    }

/*
* free buffers 
*/
    free_parser_data(parser_data);

    if (SWISH_DEBUG & SWISH_DEBUG_PARSER) {
        etime = swish_time_print_fine(swish_time_elapsed() - curTime);
        SWISH_DEBUG_MSG("%s elapsed time", etime);
        swish_xfree(etime);
    }

    return res;

}

/*
 * based on swish-e 2.4 indexadir() in fs.c
 */
unsigned int
swish_parse_directory(
    swish_3 *s3,
    xmlChar *dir,
    boolean follow_symlinks
)
{
    DIR *dir_handle;
#ifdef NEXTSTEP
    struct direct   *dir_ptr;
#else
    struct dirent   *dir_ptr;
#endif
    xmlChar *pathbuf;
    unsigned int pathbuflen;
    unsigned int dir_len;
    unsigned int files_parsed;
    
    files_parsed = 0;
    
    if ((dir_handle = opendir((char*)dir)) == NULL) {
        SWISH_WARN("Failed to open directory '%s' : %s", dir, strerror(errno));
        return files_parsed;
    }

    pathbuflen = SWISH_MAXSTRLEN;
    pathbuf = (xmlChar*)swish_xmalloc(pathbuflen + 1);
    dir_len = xmlStrlen(dir);
    
    /* case of root dir */
    if ( dir_len == 1 && dir[0] == SWISH_PATH_SEP ) 
        dir_len = 0;
        
    while ((dir_ptr = readdir(dir_handle)) != NULL) {
        int file_len = strlen( dir_ptr->d_name );

        /* For security reasons, don't index dot files */
        /* TODO Check for hidden under Windows? */
        if ((dir_ptr->d_name)[0] == '.')
            continue;


        /* Build full path to file */

        /* reallocate filename buffer, if needed (dir + path + SWISH_PATH_SEP ) */
        if ( (dir_len + file_len + 1) > pathbuflen ) {
            pathbuflen = dir_len + file_len + 256;
            pathbuf = (xmlChar *)swish_xrealloc(pathbuf, pathbuflen + 1);
        }

        if ( dir_len )
            memcpy(pathbuf, dir, dir_len);

        pathbuf[dir_len] = SWISH_PATH_SEP;  // Add path separator
        memcpy(pathbuf + dir_len + 1, dir_ptr->d_name, file_len);
        pathbuf[dir_len + file_len + 1] = '\0';

        /* Check if the path is a symlink */
        if ( !follow_symlinks && swish_fs_is_link( pathbuf ) )
            continue;


        if ( swish_fs_is_dir(pathbuf) ) {
            /* recurse immediately. this is a depth-first algorithm */
            if (s3->parser->verbosity) {
                printf("Found directory: %s\n", pathbuf);
            }
            files_parsed += swish_parse_directory(s3, pathbuf, follow_symlinks);
        }
        else if (swish_fs_is_link(pathbuf) && follow_symlinks) {
            if (s3->parser->verbosity) {
                printf("Found symlink: %s\n", pathbuf);
            }
            // TODO?
        
        }
        else if (swish_fs_is_file(pathbuf)) {
            if (s3->parser->verbosity) {
                printf("Found file: %s\n", pathbuf);
            }
            if (!swish_parse_file(s3, pathbuf)) {
                files_parsed++;
            }
        }
        else {
            SWISH_CROAK("Unknown file in directory: %s", pathbuf);
        }
    }
    closedir(dir_handle);
    swish_xfree(pathbuf);

    return files_parsed;
}


/**
* based on libxml2 xmlSAXUserParseMemory in parser.c
* which we don't use directly so that we can get encoding
*/
static int
xml_parser(
    xmlSAXHandlerPtr sax,
    void *user_data,
    xmlChar *buffer,
    int size
)
{
    int ret = 0;
    xmlParserCtxtPtr ctxt;
    swish_ParserData *parser_data = (swish_ParserData *)user_data;
    xmlSAXHandlerPtr oldsax = NULL;

    if (sax == NULL)
        return -1;
    ctxt = xmlCreateMemoryParserCtxt((const char *)buffer, size);
    if (ctxt == NULL)
        return -1;
    oldsax = ctxt->sax;
    ctxt->sax = sax;
    ctxt->sax2 = 1;

/*
* always use sax2 -- this pulled from xmlDetextSAX2() 
*/
    ctxt->str_xml = xmlDictLookup(ctxt->dict, BAD_CAST "xml", 3);
    ctxt->str_xmlns = xmlDictLookup(ctxt->dict, BAD_CAST "xmlns", 5);
    ctxt->str_xml_ns = xmlDictLookup(ctxt->dict, XML_XML_NAMESPACE, 36);
    if ((ctxt->str_xml == NULL) || (ctxt->str_xmlns == NULL)
        || (ctxt->str_xml_ns == NULL)) {

/*
* xmlErrMemory is/was not a public func but is in
* parserInternals.h * basically, this is a bad, fatal error, so
* we'll just die 
*/

/*
* xmlErrMemory(ctxt, NULL); 
*/
        SWISH_CROAK("Fatal libxml2 memory error");
    }

    if (user_data != NULL)
        ctxt->userData = user_data;

/* track ctxt in parser_data during the actual parsing
   so that warnings/errors show context. But set to NULL
   afterwards so we don't try and free it when parser_data
   gets freed.
*/
    parser_data->ctxt = ctxt;
    xmlParseDocument(ctxt);
    parser_data->ctxt = NULL;

    if (ctxt->wellFormed)
        ret = 0;
    else {
        if (ctxt->errNo != 0)
            ret = ctxt->errNo;
        else
            ret = -1;
    }
    ctxt->sax = oldsax;
    if (ctxt->myDoc != NULL) {
        xmlFreeDoc(ctxt->myDoc);
        ctxt->myDoc = NULL;
    }

    if (parser_data->docinfo->encoding != NULL)
        swish_xfree(parser_data->docinfo->encoding);

    parser_data->docinfo->encoding = document_encoding(ctxt);

    xmlFreeParserCtxt(ctxt);

    return ret;
}

static int
html_parser(
    xmlSAXHandlerPtr sax,
    void *user_data,
    xmlChar *buffer,
    int size
)
{
    int ret;
    htmlParserCtxtPtr ctxt;
    xmlChar *default_encoding;
    htmlSAXHandlerPtr oldsax = 0;
    swish_ParserData *parser_data = (swish_ParserData *)user_data;
    default_encoding = (xmlChar *)getenv("SWISH_ENCODING");

    xmlInitParser();

    ctxt = htmlCreateMemoryParserCtxt((const char *)buffer, xmlStrlen(buffer));
    
    parser_data->ctxt = ctxt;

    if (parser_data->docinfo->encoding != NULL) {
        swish_xfree(parser_data->docinfo->encoding);
    }
        
    parser_data->docinfo->encoding = document_encoding(ctxt);
        
    if (parser_data->docinfo->encoding == NULL) {
        set_encoding(parser_data, (xmlChar *)buffer);
    }
    
    /*
     * HTML parser defaults to ISO-8859-1 and that's what Swish-e 2.x does.
     * Leave that default alone, since we assume any HTML docs that actually
     * care about encoding will explicitly specify it in the <meta> header,
     * which libxml2 should respect.
    if (ctxt->encoding == NULL) {
        xmlCharEncoding enc = xmlParseCharEncoding((char*)parser_data->docinfo->encoding);
        xmlSwitchEncoding(ctxt, enc);
    }
    */

    if (ctxt == 0) {
        return (0);
    }
    
    if (sax != 0) {
        oldsax = ctxt->sax;
        ctxt->sax = (htmlSAXHandlerPtr) sax;
        ctxt->userData = parser_data;
    }
    
    ret = htmlParseDocument(ctxt);

    if (sax != 0) {
        ctxt->sax = oldsax;
        ctxt->userData = 0;
    }

    return ret;
}

static int
txt_parser(
    swish_ParserData *parser_data,
    xmlChar *buffer,
    int size
)
{
    int err = 0;
    xmlChar *out, *enc;
    int outlen;

    out = NULL;
    enc = (xmlChar *)getenv("SWISH_ENCODING");
    outlen = 0;

/*
* TODO better encoding detection. for now we assume unknown text
* files are latin1 
*/
    set_encoding(parser_data, buffer);

    if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
        SWISH_DEBUG_MSG("txt parser encoding: %s", parser_data->docinfo->encoding);

    if (!xmlStrEqual(parser_data->docinfo->encoding, (xmlChar *)SWISH_DEFAULT_ENCODING)) {
        SWISH_WARN("%s docinfo->encoding %s != %s", 
            parser_data->docinfo->uri, parser_data->docinfo->encoding, SWISH_DEFAULT_ENCODING);

        if (!xmlStrncasecmp(parser_data->docinfo->encoding, (xmlChar *)SWISH_LATIN1_ENCODING, 9)) {
            outlen = size * 2;
            out = swish_xmalloc(outlen);

            if (!isolat1ToUTF8(out, &outlen, buffer, &size)) {
                SWISH_WARN("could not convert buf from %s (outlen: %d)", SWISH_LATIN1_ENCODING, outlen);
            }
            else {
                SWISH_WARN("converted %s from %s to %s", 
                    parser_data->docinfo->uri, SWISH_LATIN1_ENCODING, SWISH_DEFAULT_ENCODING);
            }
            
            size = outlen;
            buffer = out;
        }

        else if (xmlStrEqual(parser_data->docinfo->encoding, enc)) {
            if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
                SWISH_DEBUG_MSG("default env encoding -> %s", enc);

            if (xmlStrncasecmp(enc, (xmlChar *)SWISH_LATIN1_ENCODING, 9)) {
                SWISH_WARN
                    ("%s encoding is unknown (not %s) but LC_CTYPE is %s -- assuming file is %s",
                     parser_data->docinfo->uri, SWISH_DEFAULT_ENCODING, enc, SWISH_LATIN1_ENCODING);

            }

            outlen = size * 2;
            out = swish_xmalloc(outlen);

            if (!isolat1ToUTF8(out, &outlen, buffer, &size)) {
                SWISH_WARN("could not convert buf from %s (outlen: %d): %s", 
                    SWISH_LATIN1_ENCODING, outlen, buffer);
                swish_xfree(out);
                return SWISH_ENCODING_ERROR;
            }
            else {
                SWISH_WARN("converted %s from %s to %s", 
                    parser_data->docinfo->uri, SWISH_LATIN1_ENCODING, SWISH_DEFAULT_ENCODING);
            }

            size = outlen;
            buffer = out;

        }
    }

/*
* we obviously haven't any tags on which to trigger our metanames,
* so set default
* TODO get title somehow?
* TODO check config to determine if we should buffer swish_prop_description etc
*/

    push_tag_stack(parser_data->metastack, (xmlChar *)SWISH_DEFAULT_METANAME,
                   (xmlChar *)SWISH_DEFAULT_METANAME, SWISH_DOM_CHAR);

    if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
        SWISH_DEBUG_MSG("%s stack PUSH %s", parser_data->metastack->head->context);

    buffer_characters(parser_data, buffer, size);
    flush_buffer(parser_data, (xmlChar *)SWISH_DEFAULT_METANAME,
                 (xmlChar *)SWISH_DEFAULT_METANAME);

    if (out != NULL) {
        if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
            SWISH_DEBUG_MSG("tmp text buffer being freed");

        swish_xfree(out);
    }

    return err;
}

static void
set_encoding(
    swish_ParserData *parser_data,
    xmlChar *buffer
)
{

/*
* this feels like it doesn't work ... would iconv() be better ? 
*/

    swish_xfree(parser_data->docinfo->encoding);

    if (xmlCheckUTF8((const xmlChar *)buffer) != 0) {
        parser_data->docinfo->encoding = swish_xstrdup((xmlChar *)SWISH_DEFAULT_ENCODING);
    }
    else {
        parser_data->docinfo->encoding = swish_xstrdup((xmlChar *)getenv("SWISH_ENCODING"));
    }
}

static xmlChar *
document_encoding(
    xmlParserCtxtPtr ctxt
)
{
    xmlChar *enc;

    if (ctxt->encoding != NULL) {
        enc = swish_xstrdup(ctxt->encoding);
        //SWISH_DEBUG_MSG("ctxt->encoding == %s", enc);
    }
    else if (ctxt->inputTab[0]->encoding != NULL) {
        enc = swish_xstrdup(ctxt->inputTab[0]->encoding);
        //SWISH_DEBUG_MSG("ctxt->inputTab->encoding == %s", enc);
    }
    else {

/*
* if we get here, we didn't error with bad encoding via SAX,
* so assume the current locale encoding.
*/
        enc = swish_xstrdup((xmlChar *)getenv("SWISH_ENCODING"));
        //SWISH_DEBUG_MSG("using default SWISH_ENCODING == %s", enc);
    }

    return enc;
}

static void
tokenize(
    swish_ParserData *parser_data,
    xmlChar *string,
    int len,
    xmlChar *metaname,
    xmlChar *context
)
{
    swish_MetaName *meta;

    meta = swish_hash_fetch(parser_data->s3->config->metanames, metaname);

    if (len == 0)
        return;

    if (metaname == NULL)
        metaname = parser_data->metastack->head->baked;

    if (context == NULL)
        context = parser_data->metastack->head->context;

    parser_data->docinfo->nwords +=
            (*parser_data->s3->analyzer->tokenizer) (parser_data->token_iterator, 
                                                    string, meta, context);
    return;

}

static void
_debug_stack(
    swish_TagStack *stack
)
{
    int i = 0;

    SWISH_DEBUG_MSG("%s stack->count: %d", stack->name, stack->count);

    for (stack->temp = stack->head; stack->temp != NULL; stack->temp = stack->temp->next) {
        SWISH_DEBUG_MSG("  %d: count %d  tagstack: %s", i++, stack->temp->n,
                        stack->temp->raw);

    }

    if (i != stack->count) {
        SWISH_WARN("stack count appears wrong (%d items, but count=%d)", i, stack->count);

    }
    else {
        SWISH_DEBUG_MSG("tagstack looks ok");
    }
}

/* 
* return stack as single string of space-separated names 
*/
static xmlChar *
flatten_tag_stack(
    xmlChar *baked,
    swish_TagStack *stack,
    char flatten_join
)
{
    xmlChar *tmp;
    xmlChar *flat;
    int size;
    int i;

    i = 0;
    stack->temp = stack->head;

    if (baked != NULL) {
        flat = swish_xstrdup(baked);
    }
    else {
        flat = swish_xstrdup(stack->head->baked);
        stack->temp = stack->temp->next;
    }

    for (; stack->temp != NULL; stack->temp = stack->temp->next) {
        size =
            (
            (xmlStrlen(flat)*sizeof(xmlChar)) + 
            (xmlStrlen(stack->temp->baked)*sizeof(xmlChar)) +
            3   // flatten_join + nulls
            );
            
        tmp = swish_xmalloc(size);
                
        if (snprintf((char *)tmp, size, "%s%c%s",
             (char *)stack->temp->baked, flatten_join, (char *)flat)
            > 0
        ) {
            if (flat != NULL)
                swish_xfree(flat);

            flat = tmp;
        }
        else {
            SWISH_CROAK("sprintf failed to concat %s -> %s", stack->temp->baked, flat);
        }

    }

    return flat;

}
static void
add_stack_to_prop_buf(
    xmlChar *baked,
    swish_ParserData *parser_data
)
{
    swish_TagStack *stack;
    boolean cleanwsp;
    swish_Property *prop;
    xmlChar *prop_to_store;

    stack = parser_data->propstack;
    cleanwsp = 1;
    
    if (baked != NULL) {
        /* If the propertyname is an alias_for, use the target of the alias. */
        prop = swish_hash_fetch(parser_data->s3->config->properties, baked);
        if (prop->alias_for != NULL) {
            prop_to_store = prop->alias_for;
        }
        else {
            prop_to_store = baked;
        }
    
        /* override per-property */
        if (prop->verbatim) {
            cleanwsp = 0;
        }
    
        if (SWISH_DEBUG & SWISH_DEBUG_PARSER) {
            SWISH_DEBUG_MSG("adding property %s to buffer", prop_to_store);
        }

        swish_nb_add_buf(parser_data->properties, prop_to_store, parser_data->prop_buf,
                            (xmlChar *)SWISH_TOKENPOS_BUMPER, cleanwsp, 0);
    }

    /* Swish-e 2.x behavior is to add for each member in the stack */
    for (stack->temp = stack->head; stack->temp != NULL; stack->temp = stack->temp->next) {
        if (xmlStrEqual(stack->temp->baked, (xmlChar *)SWISH_DOM_STR))
            continue;

        swish_nb_add_buf(parser_data->properties, stack->temp->baked,
                         parser_data->prop_buf, (xmlChar *)SWISH_TOKENPOS_BUMPER,
                         cleanwsp, 0);
    }
    

}

static void
free_swishTag(
    swish_Tag * st
)
{
    if (SWISH_DEBUG & SWISH_DEBUG_PARSER) {
        SWISH_DEBUG_MSG(" freeing swishTag: (raw)%s (baked)%s (context)%s", 
            st->raw, st->baked, st->context);
    }

    //SWISH_DEBUG_MSG("free raw: %s", st->raw);
    swish_xfree(st->raw);
    //SWISH_DEBUG_MSG("free baked: %s", st->baked);
    swish_xfree(st->baked);
    //SWISH_DEBUG_MSG("free context: %s", st->context);
    swish_xfree(st->context);
    //SWISH_DEBUG_MSG("free swishTag");
    swish_xfree(st);
    //SWISH_DEBUG_MSG("free swishTag done");
}

static void
push_tag_stack(
    swish_TagStack *stack,
    xmlChar *raw,
    xmlChar *baked,
    char flatten_join
)
{

    swish_Tag *thistag = swish_xmalloc(sizeof(swish_Tag));

    if (SWISH_DEBUG & SWISH_DEBUG_PARSER) {
        SWISH_DEBUG_MSG("%s PUSH: tag = '%s'", stack->name, raw);
        _debug_stack(stack);
    }

/* assign this tag to the struct  */
    thistag->raw = swish_xstrdup(raw);

/*  the normalized tag */
    thistag->baked = swish_xstrdup(baked);

/* increment counter  */
    thistag->n = stack->count++;

/*  push */
    thistag->next = stack->head;
    stack->head = thistag;

/*  create context */
    thistag->context = flatten_tag_stack(NULL, stack, flatten_join);

    if (SWISH_DEBUG & SWISH_DEBUG_PARSER) {
        SWISH_DEBUG_MSG("%s size: %d  thistag count: %d  current head tag = '%s'",
                        stack->name, stack->count, thistag->n, stack->head->context);

        _debug_stack(stack);

    }

}

static swish_Tag *
pop_tag_stack(
    swish_TagStack *stack
)
{
/*  stack is completely empty */
    if (stack->head == NULL)
        return NULL;

    if (SWISH_DEBUG & SWISH_DEBUG_PARSER) {
        SWISH_DEBUG_MSG("%s POP: %s", stack->name, stack->head->raw);
        _debug_stack(stack);

    }

    if (stack->count > 1) {
        if (SWISH_DEBUG & SWISH_DEBUG_PARSER) {
            SWISH_DEBUG_MSG("%s %d: popping '%s'", stack->name, stack->head->n,
                            stack->head->raw);

        }

        stack->temp = stack->head;
        stack->head = stack->head->next;
        stack->count--;

    }
    else {

/*  the stack has only one member */

        if (SWISH_DEBUG & SWISH_DEBUG_PARSER) {
            SWISH_DEBUG_MSG("%s %d: popping '%s' will leave stack empty [%s]",
                            stack->name, stack->head->n, stack->head->raw,
                            stack->head->context);

        }

        stack->temp = stack->head;
        stack->head = NULL;
        stack->count--;

    }

    if (SWISH_DEBUG & SWISH_DEBUG_PARSER) {
        SWISH_DEBUG_MSG("%s stack count = %d", stack->name, stack->count);
    }

    return stack->temp;

}

/* 
* returns top of the stack if the current tag matches.
*/
static swish_Tag *
pop_tag_stack_on_match(
    swish_TagStack *stack,
    xmlChar *tag
)
{

    swish_Tag *st;

    st = NULL;

    if (SWISH_DEBUG & SWISH_DEBUG_PARSER) {
        SWISH_DEBUG_MSG("%s: POP if %s matches %s", stack->name, tag, stack->head->raw);
        _debug_stack(stack);
    }

    if (xmlStrEqual(stack->head->raw, tag)) {

        if (SWISH_DEBUG & SWISH_DEBUG_PARSER) {
            SWISH_DEBUG_MSG("%s POP '%s' == head", stack->name, tag);

        }

/*
* more than default meta 
*/
        if ((st = pop_tag_stack(stack)) != NULL) {

            if (SWISH_DEBUG & SWISH_DEBUG_PARSER) {
                SWISH_DEBUG_MSG("%s POPPED.  tag = %s  st->raw = %s", stack->name, tag,
                                st->raw);

                _debug_stack(stack);
            }

        }

/*
* only tag on stack. TODO do we ever get here? 
*/
        else if (stack->count) {
            if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
                SWISH_DEBUG_MSG("%s head %s", stack->name, stack->head->raw);

        }
        else {
            SWISH_CROAK("%s stack was empty", stack->name);
        }

    }
    else {

        if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
            SWISH_DEBUG_MSG("%s: no match for '%s'", stack->name, tag);

    }

    if (SWISH_DEBUG & SWISH_DEBUG_PARSER) {
        if (st != NULL)
            SWISH_DEBUG_MSG("POP on match returning: %s", st->raw);
        else
            SWISH_DEBUG_MSG("POP on match returning null");
    }

    return st;
}


/*************** end parser.c ************/


/*************** start namedbuffer.c ************/
/*
 * This file is part of libswish3
 * Copyright (C) 2007 Peter Karman
 *
 *  libswish3 is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  libswish3 is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with libswish3; if not, write to the Free Software
 *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
*/

/* named buffers are just a hash where each key is a text buffer
*/

#ifndef LIBSWISH3_SINGLE_FILE
#include <stdio.h>
#include <stdlib.h>
#include <locale.h>
#include <err.h>

#include "libswish3.h"
#endif

extern int SWISH_DEBUG;

static void free_name_from_hash(
    void *buffer,
    xmlChar *name
);
static void add_name_to_hash(
    void *ignored,
    xmlHashTablePtr nbhash,
    xmlChar *name
);
static void print_buffer(
    xmlBufferPtr buffer,
    xmlChar *label,
    xmlChar *name
);

static void
add_name_to_hash(
    void *ignored,
    xmlHashTablePtr nbhash,
    xmlChar *name
)
{
/* make sure we don't already have it */
    if (swish_hash_exists(nbhash, name)) {
        SWISH_WARN("%s is already in NamedBuffer hash -- ignoring", name);
        return;
    }

    if (SWISH_DEBUG == SWISH_DEBUG_NAMEDBUFFER)
        SWISH_DEBUG_MSG("  adding %s to NamedBuffer\n", name);

    swish_hash_add(nbhash, name, xmlBufferCreateSize((size_t) SWISH_BUFFER_CHUNK_SIZE));
}

static void
free_name_from_hash(
    void *buffer,
    xmlChar *name
)
{
    if (SWISH_DEBUG & SWISH_DEBUG_MEMORY)
        SWISH_DEBUG_MSG(" freeing NamedBuffer %s\n", name);

    xmlBufferFree(buffer);
}

swish_NamedBuffer *
swish_nb_init(
    xmlHashTablePtr confhash
)
{
    swish_NamedBuffer *nb = swish_xmalloc(sizeof(swish_NamedBuffer));
    nb->stash = NULL;
    nb->ref_cnt = 0;
    nb->hash = xmlHashCreate(8);        /* will grow as needed */

/* init a buffer for each key in confhash. Note that this inits hashes for alias_for
   PropertyNames and MetaNames, which we will never use. But it's easier this way.
*/
    xmlHashScan(confhash, (xmlHashScanner)add_name_to_hash, nb->hash);

    return nb;
}

void
swish_nb_free(
    swish_NamedBuffer * nb
)
{
    xmlHashFree(nb->hash, (xmlHashDeallocator)free_name_from_hash);

    if (nb->ref_cnt != 0) {
        SWISH_WARN("freeing NamedBuffer with ref_cnt != 0 (%d)", nb->ref_cnt);
    }

    if (nb->stash != NULL)
        SWISH_WARN("freeing NamedBuffer with non-null stash");

    swish_xfree(nb);
}

static void
print_buffer(
    xmlBufferPtr buffer,
    xmlChar *label,
    xmlChar *name
)
{
    const xmlChar *substr;
    const xmlChar *buf;
    int sub_len;

    SWISH_DEBUG_MSG("len=%d %s:<%s>%s</%s>", xmlBufferLength(buffer), 
                        label, name, xmlBufferContent(buffer), name);
    
    buf = xmlBufferContent(buffer);
    while ((substr = xmlStrstr(buf, (const xmlChar *)SWISH_TOKENPOS_BUMPER)) != NULL) {
        sub_len = substr - buf;
        SWISH_DEBUG_MSG("  len=%d <%s>%s</%s>", sub_len, name, xmlStrsub(buf, 0, sub_len), name );
        buf = substr + 1;
    }
    if (buf != NULL) {
        SWISH_DEBUG_MSG("  len=%d <%s>%s</%s>", xmlStrlen(buf), name, buf, name );
    }
}

void
swish_nb_debug(
    swish_NamedBuffer * nb,
    xmlChar *label
)
{
    xmlHashScan(nb->hash, (xmlHashScanner)print_buffer, label);
}

void
swish_nb_add_buf(
    swish_NamedBuffer * nb,
    xmlChar *name,
    xmlBufferPtr buf,
    xmlChar *joiner,
    boolean cleanwsp,
    boolean autovivify
)
{
    swish_nb_add_str(nb, name, (xmlChar *)xmlBufferContent(buf), xmlBufferLength(buf),
                        joiner, cleanwsp, autovivify);
}

void
swish_nb_add_str(
    swish_NamedBuffer * nb,
    xmlChar *name,
    xmlChar *str,
    unsigned int len,
    xmlChar *joiner,
    boolean cleanwsp,
    boolean autovivify
)
{
    xmlChar *nowhitesp;
    xmlBufferPtr buf = swish_hash_fetch(nb->hash, name);

/* if the str is nothing but whitespace, skip it */
    if (swish_str_all_ws(str)) {
        if (SWISH_DEBUG & SWISH_DEBUG_NAMEDBUFFER)
            SWISH_DEBUG_MSG("skipping all whitespace string '%s'", str);

        return;
    }

    if (!buf) {
        if (autovivify) {
/* spring to life */
            add_name_to_hash(NULL, nb->hash, name);
            buf = swish_hash_fetch(nb->hash, name);
        }

        if (!buf)
            SWISH_CROAK("%s is not a named buffer", name);

    }

/* if the buf already exists and we're about to add more, append the joiner */
    if (xmlBufferLength(buf)) {
        swish_buffer_append(buf, joiner, xmlStrlen(joiner));
    }

    if (cleanwsp) {
        //SWISH_DEBUG_MSG("before cleanwsp: '%s'", str);
        swish_str_ctrl_to_ws(str);
        nowhitesp = swish_str_skip_ws(str);
        swish_str_trim_ws(nowhitesp);
        //SWISH_DEBUG_MSG("after  cleanwsp: adding '%s' to buffer '%s'", nowhitesp, name);
        swish_buffer_append(buf, nowhitesp, xmlStrlen(nowhitesp));
    }
    else {
        if (SWISH_DEBUG & SWISH_DEBUG_NAMEDBUFFER) 
            SWISH_DEBUG_MSG("adding '%s' to buffer '%s'", str, name); 
        swish_buffer_append(buf, str, len);
    }

}

void
swish_buffer_append(
    xmlBufferPtr buf,
    xmlChar *txt,
    int txtlen
)
{
    int ret;

/* shouldn't happen */
    if (txtlen == 0) {
        return;
    }
    
    if (buf == NULL) {
        SWISH_CROAK("Can't append NULL pointer to buffer.");
    }

    ret = xmlBufferAdd(buf, (const xmlChar *)txt, txtlen);
    if (ret) {
        SWISH_CROAK("Problem adding \n>>%s<<\n length %d to buffer. Err: %d", txt, txtlen,
                    ret);
    }
}

xmlChar *
swish_nb_get_value(
    swish_NamedBuffer * nb,
    xmlChar *key
)
{
    xmlBufferPtr buf;
    buf = swish_hash_fetch(nb->hash, key);
    return (xmlChar *)xmlBufferContent(buf);
}


/*************** end namedbuffer.c ************/


/*************** start string.c ************/
/*
 * This file is part of libswish3
 * Copyright (C) 2007 Peter Karman
 *
 *  libswish3 is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  libswish3 is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with libswish3; if not, write to the Free Software
 *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
*/

/* string.c -- handle xmlChar and wchar_t strings
 * much of this module based on swstring.c in swish-e vers 2
 * but re-written for UTF-8 support
*/

#ifndef LIBSWISH3_SINGLE_FILE
#include <assert.h>
#include <wchar.h>
#include <wctype.h>
#include <ctype.h>
#include <string.h>
#include <stdlib.h>
#include <locale.h>
#include <err.h>
#include <limits.h>
#include <errno.h>

#include "libswish3.h"
#endif

extern int SWISH_DEBUG;

static xmlChar *getword(
    xmlChar **in_buf
);

#ifndef LIBSWISH3_SINGLE_FILE
#include "utf8.c"
#endif

/* these string conversion functions based on code from xapian-omega */
#define BUFSIZE 100
#define DATE_BUFSIZE 8
#define DATE_FMT "%04d%02d%02d"

#define CONVERT_TO_STRING(FMT) \
    xmlChar *str;\
    int ret;\
    str = swish_xmalloc(BUFSIZE);\
    ret = snprintf((char*)str, BUFSIZE, (FMT), val);\
    if (ret<0) SWISH_CROAK("snprintf failed with %d", ret);\
    return str;

int
swish_string_to_int(
    char *buf
)
{
    long i;
    errno = 0;
    i = strtol(buf, (char **)NULL, 10);
    /*
       Check for various possible errors 
     */
    if ((errno == ERANGE && (i == LONG_MAX || i == LONG_MIN))
        || (errno != 0 && i == 0)) {
        perror("strtol");
        exit(EXIT_FAILURE);
    }
    return (int)i;
}

boolean
swish_string_to_boolean(
    char *buf
)
{
    if (buf == '\0' || buf == NULL) {
        return SWISH_FALSE;
    }
    if (    buf[0] == 'Y' 
        ||  buf[0] == 'y'
        ||  buf[0] == '1'    
    ) {
        return SWISH_TRUE;
    }
    if (    buf[0] == 'N'
        ||  buf[0] == 'n'
        ||  buf[0] == '0'
    ) {
        return SWISH_FALSE;
    }
    
    return SWISH_FALSE; /* default */
}

xmlChar *
swish_int_to_string(
    int val
)
{
    CONVERT_TO_STRING("%d")
}

xmlChar *
swish_long_to_string(
    long val
)
{
    CONVERT_TO_STRING("%ld")
}

xmlChar *
swish_double_to_string(
    double val
)
{
    CONVERT_TO_STRING("%f")
}

xmlChar *
swish_date_to_string(
    int y,
    int m,
    int d
)
{
    char buf[DATE_BUFSIZE + 1];
    if (y < 0)
        y = 0;
    else if (y > 9999)
        y = 9999;
    if (m < 1)
        m = 1;
    else if (m > 12)
        m = 12;
    if (d < 1)
        d = 1;
    else if (d > 31)
        d = 31;
#ifdef SNPRINTF
    int len = SNPRINTF(buf, sizeof(buf), DATE_FMT, y, m, d);
    if (len == -1 || len >= DATE_BUFSIZE)
        buf[DATE_BUFSIZE] = '\0';
#else
    buf[DATE_BUFSIZE] = '\0';
    sprintf(buf, DATE_FMT, y, m, d);
    if (buf[DATE_BUFSIZE])
        abort();                /* Uh-oh, buffer overrun */
#endif
    return swish_xstrdup((xmlChar *)buf);
}

/* returns the UCS32 value for a UTF8 string -- the character's Unicode value.
   see http://scripts.sil.org/cms/scripts/page.php?site_id=nrsi&item_id=IWS-AppendixA
*/

uint32_t
swish_utf8_codepoint(
    xmlChar *utf8
)
{
    uint32_t len;
    len = swish_utf8_chr_len(utf8);

    switch (len) {

    case 1:
        return utf8[0];

    case 2:
        return (utf8[0] - 192) * 64 + utf8[1] - 128;

    case 3:
        return (utf8[0] - 224) * 4096 + (utf8[1] - 128) * 64 + utf8[2] - 128;

    case 4:
    default:
        return (utf8[0] - 240) * 262144 + (utf8[1] - 128) * 4096 + (utf8[2] - 128) * 64 +
            utf8[3] - 128;

    }
}

void
swish_utf8_next_chr(
    xmlChar *s,
    int *i
)
{
    u8_inc((char *)s, i);
}

void
swish_utf8_prev_chr(
    xmlChar *s, 
    int *i
)
{
    u8_dec((char *)s, i);
}


/* returns length of a UTF8 character, based on first byte (see below) */
int
swish_utf8_chr_len(
    xmlChar *utf8
)
{
    int n;
    n = xmlUTF8Size(utf8);
    if (n == -1)
        SWISH_CROAK("Bad UTF8 string: %s", utf8);
        
    return n;
}

/* returns the number of UCS32 codepoints (characters) in a UTF8 string */
int
swish_utf8_num_chrs(
    xmlChar *utf8
)
{
    int n;
    n = xmlUTF8Strlen(utf8);
    if (n == -1)
        SWISH_CROAK("Bad UTF8 string: %s", utf8);
        
    return n;
}

/* returns true if all bytes in the *str are in the ascii range.
 * this helps speed up string handling when we don't need to worry
 * about multi-byte chars.
*/

/* from the libxml2 xmlstring.c file:
     * utf is a string of 1, 2, 3 or 4 bytes.  The valid strings
     * are as follows (in "bit format"):
     *    0xxxxxxx                                      valid 1-byte
     *    110xxxxx 10xxxxxx                             valid 2-byte
     *    1110xxxx 10xxxxxx 10xxxxxx                    valid 3-byte
     *    11110xxx 10xxxxxx 10xxxxxx 10xxxxxx           valid 4-byte
*/

boolean
swish_is_ascii(
    xmlChar *str
)
{
    int i;
    int len = xmlStrlen(str);

    if (!len || str == NULL)
        return 0;

    for (i = 0; i < len; i++) {
        if (str[i] >= 0x80)
            return 0;

    }
    return 1;
}

char*
swish_get_locale(
)
{
    char *locale;
    
    /* initialize with LC_ALL sets all the relevant env vars */
    setlocale(LC_ALL, "");
    locale = setlocale(LC_ALL, "");
    if (locale == NULL || !strlen(locale)) {
        //SWISH_DEBUG_MSG("locale for LC_ALL was null");

/* use LC_CTYPE specifically: 
 * http://mail.nl.linux.org/linux-utf8/2001-09/msg00030.html 
 */
        locale = setlocale(LC_CTYPE, "");
        if (locale == NULL || !strlen(locale)) {
            //SWISH_DEBUG_MSG("locale for LC_CTYPE was null");
            locale = getenv("LANG");
            if (locale == NULL || !strlen(locale)) {
                //SWISH_DEBUG_MSG("getenv for LANG was null");
                locale = SWISH_LOCALE;
            }
        }
    }
    return locale;
}

void
swish_verify_utf8_locale(
)
{
    char *loc;
    const xmlChar *enc;

/* a bit about encodings: libxml2 takes whatever encoding the input XML is
 * (latin1, ascii, utf8, etc) and standardizes it using iconv (or other) in xmlChar as
 * UTF-8. However, we must ensure we have UTF-8 locale because all the mb* and wc*
 * routines rely on the locale to correctly interpret chars.
 *
 * See also
 * http://www.cl.cam.ac.uk/~mgk25/unicode.html#c
 * and
 * http://www.cl.cam.ac.uk/~mgk25/unicode.html#activate
 */

    loc = swish_get_locale(); 
    enc = xmlStrchr((xmlChar *)loc, (xmlChar)'.');

    if (enc != NULL) {
        enc++;
        if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER)
            SWISH_DEBUG_MSG("encoding = %s", enc);
    }
    else {
        if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER)
            SWISH_DEBUG_MSG("no encoding in %s, using %s", loc, SWISH_DEFAULT_ENCODING);

        enc = (xmlChar *)SWISH_DEFAULT_ENCODING;
    }

    setenv("SWISH_ENCODING", (char *)enc, 0);   /* remember in env var, if not already set */

    if (!loc) {
        SWISH_WARN("can't get locale via setlocale()");
    }
    else if (SWISH_DEBUG) {
        SWISH_DEBUG_MSG("current locale and encoding: %s %s", loc, enc);
    }

    if (u8_is_locale_utf8(loc)) {
/* a-ok */

        if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER)
            SWISH_DEBUG_MSG("locale looks like UTF-8");

    }
    else {
/* must be UTF-8 charset since libxml2 converts everything to UTF-8 */
        if (SWISH_DEBUG)
            SWISH_DEBUG_MSG
                ("Your locale (%s) was not UTF-8 so internally we are using %s", loc,
                 SWISH_LOCALE);

        if (!setlocale(LC_CTYPE, SWISH_LOCALE)) {
            SWISH_WARN("failed to set locale to %s from %s", SWISH_LOCALE, loc);
        }

    }

    if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER) 
        SWISH_DEBUG_MSG("active locale is %s", setlocale(LC_CTYPE, NULL));

}

xmlChar *
swish_str_escape_utf8(
    xmlChar *u8str
)
{
    xmlChar *escaped;
    int u8chrs, n_escaped, esc_len;
    
    u8chrs = swish_utf8_num_chrs(u8str);
    
    /* 10 is the max number of ascii chars needed to represent a utf8 chr:
     * \Uxxxxxxxx
     * 1234567890
     */
    esc_len = (10*u8chrs)+1;    /* +1 == nul */
    escaped = swish_xmalloc(esc_len);
    
    /*
    SWISH_DEBUG_MSG("escaping %s len %d for '%s'",
        escaped, esc_len, u8str);
    */    
    n_escaped = u8_escape((char*)escaped, esc_len, (char*)u8str, 0); // TODO quotes?
    
    return escaped;
}

xmlChar *
swish_str_unescape_utf8(
    xmlChar *ascii
)
{
    xmlChar *unescaped;
    int n_unescaped, ascii_len;
    
    ascii_len = xmlStrlen(ascii);        
    unescaped = swish_xmalloc(ascii_len+1);
    n_unescaped = u8_unescape((char*)unescaped, ascii_len+1, (char*)ascii);
    
    return unescaped;
}


/* based on swstring.c  */

int
swish_wchar_t_comp(
    const void *s1,
    const void *s2
)
{
    return (*(wchar_t *) s1 - *(wchar_t *) s2);
}

/* Sort a string */
int
swish_sort_wchar(
    wchar_t * s
)
{
    int i, j, len;
    i = 0;
    j = 0;
    len = wcslen(s);
    qsort(s, len, sizeof(wchar_t), &swish_wchar_t_comp);

/* printf("sorted array s is %d long\n", len); */

    for (i = 0; s[i] != 0; i++)
/* printf("%d = %lc (%d)\n", i, s[i], s[i]); */

        for (i = 1, j = 1; i < (len - 1); i++) {
            if (s[i] != s[j - 1]) {
                s[j++] = s[i];
/* printf("%d item is %lc (%d)\n", j, s[j], s[j]); */
            }
        }

    return s[j];

}

/* based on swstring.c in Swish-e but handles wide char strings instead */

wchar_t *
swish_wstr_tolower(
    wchar_t * s
)
{
    wchar_t *p = (wchar_t *) s;
    while (*p) {
        *p = (wchar_t) towlower(*p);
        p++;
    }
    return s;
}

/* convert a string to lowercase.
 * returns a new malloc'd string, so should be freed eventually
*/
xmlChar *
swish_str_tolower(
    xmlChar *s
)
{

    if (swish_is_ascii(s))
        return swish_ascii_str_tolower(s);
    else
        return swish_utf8_str_tolower(s);

}

/* convert utf8 to wchar,
   lowercase the wchar,
   then convert back to utf8
   and free the wchar
*/
xmlChar *
swish_utf8_str_tolower(
    xmlChar *s
)
{
    xmlChar *str;
    wchar_t *wstr;

/* convert mb to wide -- must free */
    wstr = swish_locale_to_wchar(s);

/* convert wide tolower */
    swish_wstr_tolower(wstr);

/* convert wide back to mb */
    str = swish_wchar_to_locale(wstr);

    swish_xfree(wstr);

    return str;
}

/* based on swstring.c in Swish-e */
xmlChar *
swish_ascii_str_tolower(
    xmlChar *s
)
{
    xmlChar *copy = swish_xstrdup(s);
    xmlChar *p = copy;
    while (*p) {
        *p = tolower(*p);
        p++;
    }
    return copy;
}

/*
  -- Skip white spaces...
  -- position to non space character
  -- return: ptr. to non space char or \0
  -- 2001-01-30  rasc

  TODO make utf8 safe. 
*/

xmlChar *
swish_str_skip_ws(
    xmlChar *s
)
{
    while (*s && isspace((int)(xmlChar)*s))
        s++;
    return s;
}

/*************************************
* Trim trailing white space
* Returns void
**************************************/

// TODO make utf8 safe
void
swish_str_trim_ws(
    xmlChar *s
)
{
    int i = xmlStrlen(s);

    while (i && isspace((int)s[i - 1]))
        s[--i] = '\0';
}

boolean
swish_str_all_ws(
    xmlChar *s
)
{
    return swish_str_all_ws_len(s, xmlStrlen(s));
}

boolean
swish_str_all_ws_len(
    xmlChar * s, 
    int len
)
{
    int i;
    for (i = 0; i < len; i++) {
        if (!isspace((int)s[i])) {
            return 0;
        }
    }
    return 1;
}

/* change all ascii controll chars < 32 to space */
void
swish_str_ctrl_to_ws(
    xmlChar *s
)
{
    int i, k;
    if (!swish_is_ascii(s)) // TODO utf8-safe
        return;
        
    i = xmlStrlen(s);
    for(k=0; k<i; k++) {
        if ((int)s[k] < 32)
            s[k] = SWISH_SPACE;
    }
}

void
swish_debug_wchars(
    const wchar_t * widechars
)
{
    int i;
    for (i = 0; widechars[i] != 0; i++) {
        printf(" >%lc< %ld %#lx \n", (wint_t) widechars[i], (long int)widechars[i],
               (long unsigned int)widechars[i]);
    }
}

/* returns the number of UTF-8 char* needed to hold the codepoint
   represented by 'ch'.
   similar to swish_utf8_chr_len() except that the arg is already
   a 4-byte container and we want to know how many of the 4 bytes
   we really need.
*/
int
swish_bytes_in_wchar(
    int ch
)
{
    int len = 0;

    if (ch < 0x80) {
        len = 1;
    }
    if (ch < 0x800) {
        len = 2;
    }
    if (ch < 0x10000) {
        len = 3;
    }
    if (ch < 0x110000) {
        len = 4;
    }

    if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER)
        SWISH_DEBUG_MSG(" %lc is %d bytes long", ch, len);

    return len;
}


/* from http://www.triptico.com/software/unicode.html */
wchar_t *
swish_locale_to_wchar(
    xmlChar *str
)
{
    wchar_t *ptr;
    size_t s;
    int len;
    char *loc;

/* first arg == 0 means 'calculate needed space' */
    s = mbstowcs(0, (const char *)str, 0);

    len = mblen((const char *)str, 4);

/* a size of -1 is triggered by an error in encoding; 
 * never happen in ISO-8859-* locales, but possible in UTF-8 
 */
    if (s == -1) {
        loc = swish_get_locale();
        SWISH_CROAK("error converting mbs to wide str under locale %s : %s", 
            loc, str);
    }


/* malloc the necessary space */
    ptr = swish_xmalloc((s + 1) * sizeof(wchar_t));

/* really do it */
    s = mbstowcs(ptr, (const char *)str, s);

/* ensure NUL termination */
    ptr[s] = '\0';

/* remember to free() ptr when done */
    return (ptr);
}

/* from http://www.triptico.com/software/unicode.html */
xmlChar *
swish_wchar_to_locale(
    wchar_t * str
)
{
    xmlChar *ptr;
    size_t s;

/* first arg == 0 means 'calculate needed space' */
    s = wcstombs(0, str, 0);

/* a size of -1 means there are characters that could not be converted to current
     * locale */
    if (s == -1)
        SWISH_CROAK("error converting wide chars to mbs: %ls", str);

/* malloc the necessary space */
    ptr = (xmlChar *)swish_xmalloc(s + 1);

/* really do it */
    s = wcstombs((char *)ptr, (const wchar_t *)str, s);

/* ensure NUL termination */
    ptr[s] = '\0';

/* remember to free() ptr when done */
    return (ptr);
}

/* StringList functions derived from swish-e vers 2 */
swish_StringList *
swish_stringlist_init(
)
{
    swish_StringList *sl = swish_xmalloc(sizeof(swish_StringList));
    sl->n = 0;
    sl->max = 2; /* 2 to allow for NUL-terminate */
    sl->word = swish_xmalloc(sl->max * sizeof(xmlChar *));
    return sl;
}

void
swish_stringlist_free(
    swish_StringList * sl
)
{
    while (sl->n)
        swish_xfree(sl->word[--sl->n]);

    swish_xfree(sl->word);
    swish_xfree(sl);
}

void
swish_stringlist_merge(
    swish_StringList * sl1,
    swish_StringList * sl2
)
{
    int i;
    // add sl1 -> sl2
    sl2->word =
        (xmlChar **)swish_xrealloc(sl2->word, (sl1->n + sl2->n) * sizeof(xmlChar *) + 1);
    for (i = 0; i < sl1->n; i++) {
        // copy is a little overhead, but keeps mem count simple
        sl2->word[sl2->n++] = swish_xstrdup(sl1->word[i]);
    }
    swish_stringlist_free(sl1);
}

swish_StringList *
swish_stringlist_copy(
    swish_StringList * sl
)
{
    swish_StringList *s2;
    int i;
    s2 = swish_stringlist_init();
    s2->word = (xmlChar **)swish_xrealloc(s2->word, sl->n * sizeof(xmlChar *) + 1);
    for (i = 0; i < sl->n; i++) {
        s2->word[i] = swish_xstrdup(sl->word[i]);
    }
    s2->n = sl->n;
    return s2;
}

void
swish_stringlist_debug(
    swish_StringList *sl
)
{
    int i;
    for (i=0; i<sl->n; i++) {
        SWISH_DEBUG_MSG("[%d] %s", i, sl->word[i]);
    }
}

swish_StringList *  
swish_stringlist_parse_sort_string(
    xmlChar *sort_string,
    swish_Config *cfg
)
{
    xmlChar *sort_string_lc, *prop, *dir, *normalized;
    swish_StringList *sl;
    int i, nlen;
    
    /*  normalize so we know we are comparing ASC vs asc 
     *  and since propertynames are always lowercased.
     */
    sort_string_lc = swish_str_tolower(sort_string);
    sl = swish_stringlist_build(sort_string);
    swish_xfree(sort_string_lc);
    
    /* 2x longer should be ample */
    nlen = 2*xmlStrlen(sort_string);
    normalized = swish_xmalloc(nlen);
    normalized[0] = '\0';
    
    /* create the normalized string */
    for (i=0; i < sl->n; i++) {
        prop = sl->word[i]; /* just for code clarity */
        if (cfg) {
            swish_property_get_id(prop, cfg->properties); /* will croak if invalid */
        }
        if (i < sl->n) {
            dir = sl->word[i+1];
        }
        else {
            dir = NULL;
        }
        normalized = xmlStrncat(normalized, BAD_CAST " ", 1);
        normalized = xmlStrncat(normalized, prop, xmlStrlen(prop));
        normalized = xmlStrncat(normalized, BAD_CAST " ", 1);
        if (xmlStrEqual(dir, BAD_CAST "asc")
            ||
            xmlStrEqual(dir, BAD_CAST "desc")
        ) {
            normalized = xmlStrncat(normalized, dir, xmlStrlen(dir));
            i++;    /* bump to next prop */
        }
        else {
            normalized = xmlStrncat(normalized, BAD_CAST "asc", 3);
        }
    }
    swish_stringlist_free(sl);
    sl = swish_stringlist_build(normalized);
    swish_xfree(normalized);
    return sl;
}

swish_StringList *
swish_stringlist_build(
    xmlChar *line
)
{
    swish_StringList *sl;
    xmlChar *p;

    if (!line)
        return (NULL);

    sl = swish_stringlist_init();
    p = (xmlChar *)strchr((const char *)line, '\n');
    if (p != NULL)
        *p = '\0';

    p = line;

    while ((p = getword(&line))) {
    
/* getword returns "" when not null, 
 * so need to free it if we are not using it 
 */
        if (!*p) {
            swish_xfree(p);
            break;
        }
        
        swish_stringlist_add_string(sl, (xmlChar*)p);
    }

/* Add an extra NUL */
    if (sl->n == sl->max) {
        sl->word =
            (xmlChar **)swish_xrealloc(sl->word, (sl->max += 1) * sizeof(xmlChar *));
    }

    sl->word[sl->n] = NULL;

    return sl;
}

unsigned int
swish_stringlist_add_string(
    swish_StringList *sl,
    xmlChar *str
)
{
    if (sl->n == sl->max) {
        sl->word = (xmlChar **)swish_xrealloc(sl->word, (sl->max *= 2) * sizeof(xmlChar *));
    }

    sl->word[sl->n++] = str;
    return sl->n;
}

/* Gets the next word in a line. If the word's in quotes,
 * include blank spaces in the word or phrase.
 * should be utf-8 compatible; only pitfall would be if a continuation byte
 * returns true for isspace().
*/

static xmlChar *
getword(
    xmlChar **in_buf
)
{
    xmlChar quotechar;
    xmlChar uc;
    xmlChar *s = *in_buf;
    xmlChar *start = *in_buf;
    xmlChar buf[SWISH_MAX_WORD_LEN + 1];
    xmlChar *cur_char = buf;
    int backslash = 0;

    quotechar = '\0';

    s = swish_str_skip_ws(s);

/* anything to read? */
    if (!*s) {
        *in_buf = s;
        return swish_xstrdup((xmlChar *)"\0");
    }

    if (*s == '\"' || *s == '\'')
        quotechar = *s++;

/* find end of "more words" or word */

    while (*s) {
        uc = (xmlChar)*s;

        if (uc == '\\' && !backslash && quotechar)
/* only enable backslash
         * inside of quotes */
        {
            s++;
            backslash++;
            continue;
        }

/* Can't see why we would need to escape these, can you? - always fed a
         * single line */
        if (uc == '\n' || uc == '\r') {
            s++;
            break;
        }

        if (!backslash) {
/* break on ending quote or unquoted space */

            if (uc == quotechar || (!quotechar && isspace((int)uc))) {
                s++;            /* past quote or space char. */
                break;
            }

        }
        else {
            backslash = 0;
        }

        *cur_char++ = *s++;

        if (cur_char - buf > SWISH_MAX_WORD_LEN) {
            SWISH_WARN("Parsed word '%s' exceeded max length of %d", start,
                       SWISH_MAX_WORD_LEN);
        }

    }

    if (backslash)
        *cur_char++ = '\\';

    *cur_char = '\0';

    *in_buf = s;

    return swish_xstrdup(buf);

}

/*
 * based on charDecode_C_Escape() in swstring.c in swish-e
 */

char    
swish_get_C_escaped_char(xmlChar *s, xmlChar **se)
{
    char    c,
           *se2;

    if (*s != '\\') {
        /* no escape   */
        c = *s;                 /* return same char */

    }
    else {

        switch (*(++s))
        {                       /* can be optimized ... */
        case 'a':
            c = '\a';
            break;
        case 'b':
            c = '\b';
            break;
        case 'f':
            c = '\f';
            break;
        case 'n':
            c = '\n';
            break;
        case 'r':
            c = '\r';
            break;
        case 't':
            c = '\t';
            break;
        case 'v':
            c = '\v';
            break;

        // TODO support full UTF-8
        case 'x':              /* Hex  \xff  */
            c = (char) strtoul((char*)++s, &se2, 16);
            s = (xmlChar*)--se2;
            break;

        case '0':              /* Oct  \0,  \012 */
            c = (char) strtoul((char*)s, &se2, 8);
            s = (xmlChar*)--se2;
            break;

        case '\0':             /* outch!! null after \ */
            s--;               /* it's a "\"    */
            
        default:
            c = *s;            /* the escaped character */
            break;
        }

    }

    if (se)
        *se = s + 1;
    return c;
}


/*************** end string.c ************/


/*************** start times.c ************/
/*
 * This file is part of libswish3
 * Copyright (C) 2007 Peter Karman
 *
 *  libswish3 is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  libswish3 is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with libswish3; if not, write to the Free Software
 *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
*/
/* based on Swish-e version 2 */

#ifndef LIBSWISH3_SINGLE_FILE
#include <stdio.h>
#include <time.h>
#include "getruntime.c"
#include "libswish3.h"
#endif

/*
  -- TimeHiRes returns a ClockTick value (double)
  -- in seconds.fractions
*/

#ifdef HAVE_BSDGETTIMEOFDAY
#define gettimeofday BSDgettimeofday
#endif

#ifdef NO_GETTOD

double
swish_time_elapsed(
    void
)
{
#ifdef HAVE_SYS_TIMEB_H
#include <sys/timeb.h>

    struct timeb ftimebuf;

    ftime(&ftimebuf);
    return (double)ftimebuf.time + (double)ftimebuf.millitm / 1000.0;

#else

    return ((double)clock()) / CLOCKS_PER_SEC;

#endif
}

#else

#include <sys/time.h>

double
swish_time_elapsed(
    void
)
{
    struct timeval t;
    int i;

    i = gettimeofday(&t, NULL);
    if (i)
        return 0;

    return (double)(t.tv_sec + t.tv_usec / 1000000.0);
}
#endif

/* return CPU time used */
double
swish_time_cpu(
    void
)
{
    return (double)get_cpu_secs();
}

char *
swish_time_print(
    double time
)
{
    int hh, mm, ss;
    int delta;
    char *str;

    if (time < 0)
        time = 0;

    delta = (int)(time + 0.5);
    ss = delta % 60;
    delta /= 60;
    hh = delta / 60;
    mm = delta % 60;

    str = swish_xmalloc(9);
    if (sprintf(str, "%02d:%02d:%02d", hh, mm, ss) > 0) {
        return str;
    }
    else {
        swish_xfree(str);
        return (char *)swish_xstrdup((xmlChar *)"unknown time");
    }
}

char *
swish_time_print_fine(
    double time
)
{
    char *str;

    if (time >= 10)
        time = 9.99999;

    str = swish_xmalloc(8);
    if (sprintf(str, "%1.5f", time) > 0)
        return str;

    else {
        swish_xfree(str);
        return (char *)swish_xstrdup((xmlChar *)"unknown fine time");
    }

}

char *
swish_time_format(
    time_t epoch
)
{
    char *h_mtime;
    h_mtime = (char*)swish_xmalloc(30);
    strftime(h_mtime, (unsigned long)30, SWISH_DATE_FORMAT_STRING,
             (struct tm *)localtime((time_t *) & (epoch)));
    return h_mtime;
}


/*************** end times.c ************/


/*************** start swish.c ************/
/*
 * This file is part of libswish3
 * Copyright (C) 2007 Peter Karman
 *
 *  libswish3 is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  libswish3 is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with libswish3; if not, write to the Free Software
 *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
*/

#ifndef LIBSWISH3_SINGLE_FILE
#include <stdlib.h>
#include "acconfig.h"
#include "libswish3.h"
#endif

int SWISH_DEBUG = 0;            /* global var */
int SWISH_WARNINGS = 1;         /* global var */

const char *
swish_lib_version(
)
{
    return (const char*)SWISH_LIB_VERSION;
}

const char *
swish_libxml2_version(
)
{
    return (const char*)LIBXML_DOTTED_VERSION;
}

swish_3 *
swish_3_init(
    void (*handler) (swish_ParserData *),
    void *stash
)
{
    swish_3 *s3;
    s3 = swish_xmalloc(sizeof(swish_3));
    s3->ref_cnt = 0;
    s3->config = swish_config_init();
    s3->config->ref_cnt++;
    swish_config_set_default(s3->config);
    s3->analyzer = swish_analyzer_init(s3->config);
    s3->analyzer->ref_cnt++;
    s3->parser = swish_parser_init(handler);
    s3->parser->ref_cnt++;
    s3->stash = stash;
    
    if (SWISH_DEBUG & SWISH_DEBUG_MEMORY) {
        SWISH_DEBUG_MSG("s3 ptr 0x%lx", s3);
    }
    
    return s3;
}

void
swish_3_free(
    swish_3 *s3
)
{    
    s3->parser->ref_cnt--;
    if (s3->parser->ref_cnt < 1) {
        swish_parser_free(s3->parser);
    }

    s3->analyzer->ref_cnt--;
    if (s3->analyzer->ref_cnt < 1) {
        swish_analyzer_free(s3->analyzer);
    }

    s3->config->ref_cnt--;
    if (s3->config->ref_cnt < 1) {
        swish_config_free(s3->config);
    }

    if (s3->ref_cnt != 0) {
        SWISH_WARN("s3 ref_cnt != 0: %d\n", s3->ref_cnt);
    }
    swish_xfree(s3);
}

/* MUST call this before instantiating any swish_3 objects */
void
swish_setup(
)
{

/* global var that scripts can check to determine what version of Swish they are
 * using. the second 0 indicates that it will not override it if already set */
    setenv("SWISH3", "1", 0);

/* global debug flag */
    setenv("SWISH_DEBUG", "0", 0);
    setenv("SWISH_DEBUG_MEMORY", "0", 0);
    setenv("SWISH_DEBUG_CONFIG", "0", 0);
    setenv("SWISH_DEBUG_DOCINFO", "0", 0);
    setenv("SWISH_DEBUG_IO", "0", 0);
    setenv("SWISH_DEBUG_TOKENLIST", "0", 0);
    setenv("SWISH_DEBUG_TOKENIZER", "0", 0);
    setenv("SWISH_DEBUG_PARSER", "0", 0);
    setenv("SWISH_DEBUG_NAMEDBUFFER", "0", 0);
    setenv("SWISH_WARNINGS", "1", 0);
    if (!SWISH_DEBUG) {

        SWISH_DEBUG += swish_string_to_int(getenv("SWISH_DEBUG"));

/* additional env vars just increase the global var value */

        if (swish_string_to_int(getenv("SWISH_DEBUG_MEMORY"))) {
            SWISH_DEBUG += SWISH_DEBUG_MEMORY;
        }
        if (swish_string_to_int(getenv("SWISH_DEBUG_CONFIG"))) {
            SWISH_DEBUG += SWISH_DEBUG_CONFIG;
        }
        if (swish_string_to_int(getenv("SWISH_DEBUG_DOCINFO"))) {
            SWISH_DEBUG += SWISH_DEBUG_DOCINFO;
        }
        if (swish_string_to_int(getenv("SWISH_DEBUG_TOKENLIST"))) {
            SWISH_DEBUG += SWISH_DEBUG_TOKENLIST;
        }
        if (swish_string_to_int(getenv("SWISH_DEBUG_TOKENIZER"))) {
            SWISH_DEBUG += SWISH_DEBUG_TOKENIZER;
        }
        if (swish_string_to_int(getenv("SWISH_DEBUG_PARSER"))) {
            SWISH_DEBUG += SWISH_DEBUG_PARSER;
        }
        if (swish_string_to_int(getenv("SWISH_DEBUG_NAMEDBUFFER"))) {
            SWISH_DEBUG += SWISH_DEBUG_NAMEDBUFFER;
        }
        if (swish_string_to_int(getenv("SWISH_DEBUG_IO"))) {
            SWISH_DEBUG += SWISH_DEBUG_IO;
        }   

        if (SWISH_DEBUG) {
            SWISH_DEBUG_MSG("SWISH_DEBUG set to %d", SWISH_DEBUG);
        }
    }
    
    SWISH_WARNINGS = swish_string_to_int(getenv("SWISH_WARNINGS"));

/*
     * initialize the library and check potential API mismatches
     * between the version it was compiled for and the actual shared
     * library used.
*/
    LIBXML_TEST_VERSION swish_mem_init();
    swish_verify_utf8_locale();

}


/*************** end swish.c ************/


/*************** start analyzer.c ************/
/*
 * This file is part of libswish3
 * Copyright (C) 2007 Peter Karman
 *
 *  libswish3 is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  libswish3 is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with libswish3; if not, write to the Free Software
 *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
*/

/* text analyzer
   tokenize strings, stemming

*/

#ifndef LIBSWISH3_SINGLE_FILE
#include "libswish3.h"
#endif

extern int SWISH_DEBUG;

swish_Analyzer *
swish_analyzer_init(
    swish_Config *config
)
{
    swish_Analyzer *a;
    a = swish_xmalloc(sizeof(swish_Analyzer));

/* TODO get these all from config */
    a->maxwordlen = SWISH_MAX_WORD_LEN;
    a->minwordlen = SWISH_MIN_WORD_LEN;
    a->lc = SWISH_TRUE;
    a->ref_cnt = 0;
    a->tokenize = config->flags->tokenize;

    if (!a->tokenize && SWISH_DEBUG)
        SWISH_DEBUG_MSG("skipping tokenizer");

/* tokenizer set in the parse* function */
    a->tokenizer = NULL;

/* TODO get stemmer via config */
    a->stemmer = NULL;

/* TODO standalone regex lib */
    a->regex = NULL;

    a->stash = NULL;
    
    if (SWISH_DEBUG & SWISH_DEBUG_MEMORY) {
        SWISH_DEBUG_MSG("analyzer ptr 0x%x", (long int)a);
    }

    return a;
}

/* 
   IMPORTANT -- any struct members that require unique free()s should
   do that prior to calling this function.
   stemmer, for example, or regex
*/

void
swish_analyzer_free(
    swish_Analyzer *a
)
{
    if (a->ref_cnt != 0) {
        SWISH_WARN("analyzer ref_cnt != 0: %d\n", a->ref_cnt);
    }
    if (SWISH_DEBUG & SWISH_DEBUG_MEMORY) {
        SWISH_DEBUG_MSG("free analyzer");
        swish_mem_debug();
    }
    
    if (a->stash != NULL)
        SWISH_WARN("Analyzer->stash not freed 0x%x", (long int)a->stash);
        
    if (a->regex != NULL)
        SWISH_WARN("Analyzer->regex not freed 0x%x", (long int)a->regex);
        
    if (a->stemmer != NULL)
        SWISH_WARN("Analyzer->stemmer not freed");
        
    
    swish_xfree(a);
}


/*************** end analyzer.c ************/


/*************** start property.c ************/
/*
 * This file is part of libswish3
 * Copyright (C) 2008 Peter Karman
 *
 *  libswish3 is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  libswish3 is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with libswish3; if not, write to the Free Software
 *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
*/

#ifndef LIBSWISH3_SINGLE_FILE
#include "libswish3.h"
#endif

extern int SWISH_DEBUG;

swish_Property *
swish_property_init(
    xmlChar *name
)
{
    swish_Property *p;
    p = swish_xmalloc(sizeof(swish_Property));
    p->ref_cnt      = 0;
    p->id           = -1;
    p->name         = name;
    p->ignore_case  = SWISH_TRUE;
    p->type         = SWISH_PROP_STRING;
    p->verbatim     = SWISH_FALSE;
    p->alias_for    = NULL;
    p->max          = 0;
    p->sort         = SWISH_TRUE;
    p->presort      = SWISH_TRUE;
    p->sort_length  = 0;
    return p;
}

void
swish_property_debug(
    swish_Property *p
)
{
    SWISH_DEBUG_MSG("\n\
    p->ref_cnt       = %d\n\
    p->id            = %d\n\
    p->name          = %s\n\
    p->ignore_case   = %d\n\
    p->type          = %d\n\
    p->verbatim      = %d\n\
    p->alias_for     = %s\n\
    p->max           = %d\n\
    p->sort          = %d\n\
    p->presort       = %d\n\
    p->sort_length   = %d\n\
    ", p->ref_cnt, p->id, p->name, p->ignore_case, p->type, p->verbatim, 
       p->alias_for, p->max, p->sort, p->presort, p->sort_length);
}

void
swish_property_free(
    swish_Property *p
)
{
    if (p->ref_cnt != 0) {
        SWISH_WARN("Property ref_cnt != 0: %d", p->ref_cnt);
    }

    if (p->name != NULL) {
        swish_xfree(p->name);
    }
    if (p->alias_for != NULL) {
        swish_xfree(p->alias_for);
    }

    swish_xfree(p);
}

int
swish_property_get_id(
    xmlChar *propname, 
    xmlHashTablePtr properties
)
{
    int prop_id = -2;
    swish_Property *prop;
    
    // special cases
    if (xmlStrEqual(propname, BAD_CAST SWISH_PROP_RANK)) {
        prop_id = SWISH_PROP_RANK_ID;
    }
    else if (xmlStrEqual(propname, BAD_CAST SWISH_PROP_DOCPATH)) {
        prop_id = SWISH_PROP_DOCPATH_ID;
    }
    else if (xmlStrEqual(propname, BAD_CAST SWISH_PROP_MTIME)) {
        prop_id = SWISH_PROP_MTIME_ID;
    }
    else if (xmlStrEqual(propname, BAD_CAST SWISH_PROP_SIZE)) {
        prop_id = SWISH_PROP_SIZE_ID;
    }
    else if (xmlStrEqual(propname, BAD_CAST SWISH_PROP_MIME)) {
        prop_id = SWISH_PROP_MIME_ID;
    }
    else if (xmlStrEqual(propname, BAD_CAST SWISH_PROP_PARSER)) {
        prop_id = SWISH_PROP_PARSER_ID;
    }
    else if (xmlStrEqual(propname, BAD_CAST SWISH_PROP_NWORDS)) {
        prop_id = SWISH_PROP_NWORDS_ID;
    }
    
    // look up the propname in the config
    else if (swish_hash_exists( properties, propname )) {
        prop = (swish_Property*)swish_hash_fetch( properties, propname );
        prop_id = prop->id;
    }
    else {
        SWISH_CROAK("No such PropertyName: %s", propname);
    }

    return prop_id;
}


/*************** end property.c ************/


/*************** start metaname.c ************/
/*
 * This file is part of libswish3
 * Copyright (C) 2008 Peter Karman
 *
 *  libswish3 is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  libswish3 is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with libswish3; if not, write to the Free Software
 *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
*/

#ifndef LIBSWISH3_SINGLE_FILE
#include "libswish3.h"
#endif

extern int SWISH_DEBUG;

swish_MetaName *
swish_metaname_init(
    xmlChar *name
)
{
    swish_MetaName *m;
    m = swish_xmalloc(sizeof(swish_MetaName));
    m->ref_cnt = 0;
    m->id = -1;
    m->name = name;
    m->bias = 0;
    m->alias_for = NULL;
    return m;
}

void
swish_metaname_debug(
    swish_MetaName *m
)
{
    SWISH_DEBUG_MSG("\n\
    m->ref_cnt      = %d\n\
    m->id           = %d\n\
    m->name         = %s\n\
    m->bias         = %d\n\
    m->alias_for    = %s\n\
    ", m->ref_cnt, m->id, m->name, m->bias, m->alias_for);
}

void
swish_metaname_free(
    swish_MetaName *m
)
{
    if (m->ref_cnt != 0) {
        SWISH_WARN("MetaName ref_cnt != 0: %d", m->ref_cnt);
    }

    if (m->name != NULL) {
        swish_xfree(m->name);
    }
    if (m->alias_for != NULL) {
        swish_xfree(m->alias_for);
    }

    swish_xfree(m);
}


/*************** end metaname.c ************/


/*************** start header.c ************/
/*
 * This file is part of libswish3
 * Copyright (C) 2008 Peter Karman
 *
 *  libswish3 is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  libswish3 is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with libswish3; if not, write to the Free Software
 *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
*/

/* read/write the swish.xml header file */

#ifndef LIBSWISH3_SINGLE_FILE
#include <libxml/xmlreader.h>
#include <libxml/xmlwriter.h>
#include <libxml/encoding.h>
#include <ctype.h>
#include "libswish3.h"
#endif

extern int SWISH_DEBUG;

/* local struct to ease passing around flags/state */
typedef struct
{
    boolean isprops;
    boolean ismetas;
    boolean isindex;
    boolean isparser;
    boolean isalias;
    boolean ismime;
    const xmlChar *parent_name;
    swish_Config *config;
    boolean is_valid;
    unsigned int prop_id;
    unsigned int meta_id;
} headmaker;

typedef struct
{
    void *thing1;
    void *thing2;
    void *thing3;
} temp_things;

static void read_metaname_aliases(
    xmlChar *str,
    headmaker * h,
    swish_MetaName *meta
);
static void read_metaname_attr(
    const xmlChar *attr,
    const xmlChar *attr_val,
    swish_MetaName *meta,
    headmaker * h
);
static void read_metaname(
    xmlTextReaderPtr reader,
    headmaker * h
);
static void read_property_aliases(
    xmlChar *str,
    headmaker * h,
    swish_Property *prop
);
static void read_property_attr(
    const xmlChar *attr,
    const xmlChar *attr_val,
    swish_Property *prop,
    headmaker * h
);
static void read_property(
    xmlTextReaderPtr reader,
    headmaker * h
);
static void process_node(
    xmlTextReaderPtr reader,
    headmaker * h
);
static void read_key_values_pair(
    xmlTextReaderPtr reader,
    xmlHashTablePtr hash,
    xmlChar *name
);
static void read_key_value_pair(
    xmlTextReaderPtr reader,
    xmlHashTablePtr hash,
    xmlChar *name
);
static void
read_key_value_stringlist(
    xmlTextReaderPtr reader,
    xmlHashTablePtr hash,
    xmlChar *name
);
static void read_header(
    char *filename,
    headmaker * h
);
static void test_meta_alias_for(
    swish_MetaName *meta,
    swish_Config *c,
    xmlChar *name
);
static void test_prop_alias_for(
    swish_Property *prop,
    swish_Config *c,
    xmlChar *name
);
static headmaker *init_headmaker(
);
static void
reset_headmaker(
    headmaker *h
);
static void write_open_tag(
    xmlTextWriterPtr writer,
    xmlChar *tag
);
static void write_close_tag(
    xmlTextWriterPtr writer
);
static void write_element_with_content(
    xmlTextWriterPtr writer,
    xmlChar *tag,
    xmlChar *content
);
static void write_metaname(
    swish_MetaName *meta,
    xmlTextWriterPtr writer,
    xmlChar *name
);
static void write_metanames(
    xmlTextWriterPtr writer,
    xmlHashTablePtr metanames
);
static void write_hash_entry(
    xmlChar *value,
    xmlTextWriterPtr writer,
    xmlChar *key
);
static void write_property(
    swish_Property *prop,
    xmlTextWriterPtr writer,
    xmlChar *name
);
static void write_properties(
    xmlTextWriterPtr writer,
    xmlHashTablePtr properties
);
static void write_parser(
    xmlChar *val,
    xmlTextWriterPtr writer,
    xmlChar *key
);
static void write_parsers(
    xmlTextWriterPtr writer,
    xmlHashTablePtr parsers
);
static void write_mime(
    xmlChar *type,
    temp_things *things,
    xmlChar *ext
);
static void write_mimes(
    xmlTextWriterPtr writer,
    xmlHashTablePtr mimes
);
static void write_index(
    xmlTextWriterPtr writer,
    xmlHashTablePtr index
);
static void write_tag_aliases(
    xmlTextWriterPtr writer,
    xmlHashTablePtr tag_aliases
);
static void write_misc(
    xmlTextWriterPtr writer,
    xmlHashTablePtr hash
);
static void handle_special_misc_flags(
    headmaker *h
);

static void
handle_special_misc_flags(
    headmaker *h
)
{
    if (swish_hash_exists(h->config->misc, BAD_CAST SWISH_TOKENIZE)) {
        /*
        SWISH_DEBUG_MSG("tokenize in config == %s", 
            swish_hash_fetch(h->config->misc, BAD_CAST SWISH_TOKENIZE));
        */
        h->config->flags->tokenize = 
            swish_string_to_boolean(swish_hash_fetch(h->config->misc, BAD_CAST SWISH_TOKENIZE));
    }
    if (swish_hash_exists(h->config->misc, BAD_CAST SWISH_CASCADE_META_CONTEXT)) {
        /*
        SWISH_DEBUG_MSG("cascade_meta_context in config == %s", 
            swish_hash_fetch(h->config->misc, BAD_CAST SWISH_CASCADE_META_CONTEXT));
        */
        h->config->flags->cascade_meta_context = 
            swish_string_to_boolean(swish_hash_fetch(h->config->misc, BAD_CAST SWISH_CASCADE_META_CONTEXT));
    }
    if (swish_hash_exists(h->config->misc, BAD_CAST SWISH_IGNORE_XMLNS)) {
        /*
        SWISH_DEBUG_MSG("ignore_xmlns in config == %s", 
            swish_hash_fetch(h->config->misc, BAD_CAST SWISH_IGNORE_XMLNS));
        */
        h->config->flags->ignore_xmlns = 
            swish_string_to_boolean(swish_hash_fetch(h->config->misc, BAD_CAST SWISH_IGNORE_XMLNS));
    }

}

static void
read_metaname_aliases(
    xmlChar *str,
    headmaker * h,
    swish_MetaName *meta
)
{
    swish_StringList *strlist;
    int i;

    strlist = swish_stringlist_build(str);

/* loop over each alias and create a MetaName for each,
       setting alias_for to meta->name
*/
    for (i = 0; i < strlist->n; i++) {

        if (!swish_hash_exists(h->config->metanames, strlist->word[i])) {
            swish_MetaName *newmeta;
            xmlChar *newname;
            newname = swish_str_tolower(strlist->word[i]);

/* is this an existing metaname? pull it from hash and update */
            if (swish_hash_exists(h->config->metanames, newname)) {
                newmeta = swish_hash_fetch(h->config->metanames, newname);
            }
/* else new metaname */
            else {
                newmeta = swish_metaname_init(newname);
                newmeta->ref_cnt++;
                newmeta->id = h->meta_id++;
                newmeta->bias = meta->bias;
                swish_hash_add(h->config->metanames, newmeta->name, newmeta);
            }

            newmeta->alias_for = swish_xstrdup(meta->name);

/* swish_metaname_debug(newmeta); */
        }
        else {
            SWISH_CROAK
                ("Cannot alias MetaName %s to %s because %s is already a real MetaName",
                 strlist->word[i], meta->name, strlist->word[i]);
        }

    }

    swish_stringlist_free(strlist);
}

static void
read_metaname_attr(
    const xmlChar *attr,
    const xmlChar *attr_val,
    swish_MetaName *meta,
    headmaker * h
)
{
    swish_MetaName *dupe;
        
    if (xmlStrEqual(attr, (xmlChar *)"bias")) {
        meta->bias = swish_string_to_int((char*)attr_val);
    }
    else if (xmlStrEqual(attr, (xmlChar *)"id")) {
        // make sure id is not already assigned
        if (swish_hash_exists(h->config->flags->meta_ids, (xmlChar*)attr_val)) {
            dupe = swish_hash_fetch(h->config->flags->meta_ids, (xmlChar*)attr_val);
            SWISH_CROAK("duplicate id %s on MetaName %s (already assigned to %s)",
                attr_val, meta->name, dupe->name);
        }
        meta->id = swish_string_to_int((char*)attr_val);
        // cache for id lookup
        swish_hash_add(h->config->flags->meta_ids, (xmlChar*)attr_val, meta);
    }
    else if (xmlStrEqual(attr, (xmlChar *)"alias_for")) {
        meta->alias_for = swish_str_tolower(BAD_CAST attr_val);
    }
    else {
        SWISH_CROAK("Unknown MetaName attribute: %s", attr);
    }
}

static void
read_metaname(
    xmlTextReaderPtr reader,
    headmaker * h
)
{
    const xmlChar *nodename;
    swish_MetaName *meta;
    
    nodename = xmlTextReaderConstName(reader);

    meta = swish_metaname_init(swish_str_tolower((xmlChar *)nodename));
    meta->ref_cnt++;

    if (xmlTextReaderHasValue(reader)
        && xmlTextReaderNodeType(reader) == XML_READER_TYPE_TEXT) 
    {
        if (!h->parent_name) {
            SWISH_CROAK("Illegal text in MetaNames section: '%s'",
                xmlTextReaderValue(reader));
        }   
        
        swish_xfree(meta->name);
        meta->name = swish_str_tolower((xmlChar *)h->parent_name);
        read_metaname_aliases(xmlTextReaderValue(reader), h, meta);
        meta->ref_cnt--;
        swish_metaname_free(meta);
        return;
    }

    if (xmlTextReaderHasAttributes(reader)) {

        xmlTextReaderMoveToFirstAttribute(reader);
        if (xmlStrEqual(xmlTextReaderConstPrefix(reader),(xmlChar*)"xmlns")) {
            if (xmlTextReaderMoveToNextAttribute(reader) == 1) {
                read_metaname_attr(xmlTextReaderConstName(reader),
                           xmlTextReaderConstValue(reader), meta, h);
            }
        }
        else {
            read_metaname_attr(xmlTextReaderConstName(reader),
                           xmlTextReaderConstValue(reader), meta, h);
        }
        
        while (
            xmlTextReaderMoveToNextAttribute(reader) == 1
            &&
            !xmlStrEqual(xmlTextReaderConstPrefix(reader),(xmlChar*)"xmlns")
        ) {
            read_metaname_attr(xmlTextReaderConstName(reader),
                               xmlTextReaderConstValue(reader), meta, h);
        }

    }

/*  must have an id */
    if (meta->id == -1) {
        meta->id = h->meta_id++;
    }

    if (!swish_hash_exists(h->config->metanames, meta->name)) {
        swish_hash_add(h->config->metanames, meta->name, meta);
    }
    else {
        SWISH_WARN("MetaName %s is already defined", meta->name);
/*  TODO could be alias. how to check? */
    }

/* swish_metaname_debug(meta); */

    h->parent_name = nodename;

}

static void
read_property_aliases(
    xmlChar *str,
    headmaker * h,
    swish_Property *prop
)
{
    swish_StringList *strlist;
    int i;

    strlist = swish_stringlist_build(str);

/* loop over each alias and create a Property for each,
   setting alias_for to prop->name
*/
    for (i = 0; i < strlist->n; i++) {

        if (!swish_hash_exists(h->config->properties, strlist->word[i])) {
            swish_Property *newprop =
                swish_property_init(swish_str_tolower(strlist->word[i]));
            newprop->ref_cnt++;
            newprop->alias_for = swish_xstrdup(prop->name);
            newprop->id = h->prop_id++;
            newprop->ignore_case = prop->ignore_case;
            newprop->type = prop->type;
            newprop->verbatim = prop->verbatim;
            newprop->max = prop->max;
            newprop->sort = prop->sort;
            swish_hash_add(h->config->properties, newprop->name, newprop);
            /* swish_property_debug(newprop); */
        }
        else {
            SWISH_CROAK
                ("Cannot alias Property %s to %s because %s is already a real Property",
                 strlist->word[i], prop->name, strlist->word[i]);
        }

    }

    swish_stringlist_free(strlist);
}

static void
read_property_attr(
    const xmlChar *attr,
    const xmlChar *attr_val,
    swish_Property *prop,
    headmaker * h
)
{
    swish_Property *dupe;
    
    if (xmlStrEqual(attr, (xmlChar *)"ignore_case")) {
        prop->ignore_case = swish_string_to_boolean((char *)attr_val);
    }
    else if (xmlStrEqual(attr, (xmlChar *)"max")) {
        prop->max = swish_string_to_int((char *)attr_val);
    }
    else if (xmlStrEqual(attr, (xmlChar *)"verbatim")) {
        prop->verbatim = swish_string_to_boolean((char *)attr_val);
    }
    else if (xmlStrEqual(attr, (xmlChar *)"sort")) {
        prop->sort = swish_string_to_boolean((char *)attr_val);
    }
    else if (xmlStrEqual(attr, (xmlChar *)"presort")) {
        prop->presort = swish_string_to_boolean((char *)attr_val);
    }
    else if (xmlStrEqual(attr, (xmlChar *)"sort_length")) {
        prop->sort_length = swish_string_to_int((char *)attr_val);
    }
    else if (xmlStrEqual(attr, (xmlChar *)"id")) {
        // make sure id is not already assigned
        if (swish_hash_exists(h->config->flags->prop_ids, (xmlChar*)attr_val)) {
            dupe = swish_hash_fetch(h->config->flags->prop_ids, (xmlChar*)attr_val);
            SWISH_CROAK("duplicate id %s on MetaName %s (already assigned to %s)",
                attr_val, prop->name, dupe->name);
        }
        prop->id = swish_string_to_int((char*)attr_val);
        // cache for id lookup
        swish_hash_add(h->config->flags->prop_ids, (xmlChar*)attr_val, prop);
    }
    else if (xmlStrEqual(attr, (xmlChar *)"type")) {
        if (xmlStrEqual(attr_val, (xmlChar *)"int")) {
            prop->type = SWISH_PROP_INT;
        }
        else if (xmlStrEqual(attr_val, (xmlChar *)"date")) {
            prop->type = SWISH_PROP_DATE;
        }
        else if (xmlStrEqual(attr_val, (xmlChar*)"string")
                ||
                 xmlStrEqual(attr_val, (xmlChar*)"text")
        ) {
            prop->type = SWISH_PROP_STRING;
        }
        else if (isdigit(attr_val[0])) {
            prop->type = swish_string_to_int((char*)attr_val);
        }
        else {
            SWISH_CROAK("Invalid value for PropertyName '%s' type: %s",
                prop->name, attr_val);
        }
    }
    else if (xmlStrEqual(attr, (xmlChar *)"alias_for")) {
        prop->alias_for = swish_str_tolower(BAD_CAST attr_val);
    }
    else {
        SWISH_CROAK("unknown Property attribute: %s", attr);
    }

}

static void
read_property(
    xmlTextReaderPtr reader,
    headmaker * h
)
{
    const xmlChar *nodename;
    swish_Property *prop;

    nodename = xmlTextReaderConstName(reader);
    prop = swish_property_init(swish_str_tolower((xmlChar *)nodename));
    prop->ref_cnt++;

    if (xmlTextReaderHasValue(reader)
        && xmlTextReaderNodeType(reader) == XML_READER_TYPE_TEXT) {
        
        if (!h->parent_name) {
            SWISH_CROAK("Illegal text in PropertyNames section: '%s'",
                xmlTextReaderValue(reader));
        }
        
        swish_xfree(prop->name);
        prop->name = swish_str_tolower((xmlChar *)h->parent_name);
        read_property_aliases(xmlTextReaderValue(reader), h, prop);
        prop->ref_cnt--;
        swish_property_free(prop);
        return;
    }

    if (xmlTextReaderHasAttributes(reader)) {

        xmlTextReaderMoveToFirstAttribute(reader);
        if (xmlStrEqual(xmlTextReaderConstPrefix(reader),(xmlChar*)"xmlns")) {
            if (xmlTextReaderMoveToNextAttribute(reader) == 1) {
                read_property_attr(xmlTextReaderConstName(reader),
                           xmlTextReaderConstValue(reader), prop, h);
            }
        }
        else {
            read_property_attr(xmlTextReaderConstName(reader),
                           xmlTextReaderConstValue(reader), prop, h);
        }

        while (
            xmlTextReaderMoveToNextAttribute(reader) == 1
            &&
            !xmlStrEqual(xmlTextReaderConstPrefix(reader),(xmlChar*)"xmlns")
        ) {
            read_property_attr(xmlTextReaderConstName(reader),
                               xmlTextReaderConstValue(reader), prop, h);
        }

    }

    if (prop->id == -1) {
        prop->id = h->prop_id++;
    }

    if (!swish_hash_exists(h->config->properties, prop->name)) {
        swish_hash_add(h->config->properties, prop->name, prop);
    }
    else {
/* swish_config_debug( h->config ); */
        SWISH_CROAK("Property %s is already defined", prop->name);
    }

/* swish_property_debug(prop); */

    h->parent_name = nodename;

}

static void
process_node(
    xmlTextReaderPtr reader,
    headmaker * h
)
{
    const xmlChar *name, *value;
    int type;

    type = xmlTextReaderNodeType(reader);
    name = xmlTextReaderConstLocalName(reader);
    value = xmlTextReaderConstValue(reader);

    if (SWISH_DEBUG & SWISH_DEBUG_CONFIG)
        SWISH_DEBUG_MSG("name %s  type %d  value %s", name, type, value);

    if (name == NULL)
        name = BAD_CAST "--";

    if (type == XML_READER_TYPE_COMMENT)
        return;

    if (xmlStrEqual(name, (const xmlChar *)SWISH_HEADER_ROOT)) {
        h->is_valid = 1;
        return;
    }
    if (!h->is_valid) {
        SWISH_CROAK("invalid header file");
    }

    if (swish_str_all_ws((xmlChar *)value)
        && xmlStrEqual(name, (xmlChar *)"#text")) {
        return;
    }            

    if (type == XML_READER_TYPE_END_ELEMENT) {
        if (xmlStrEqual(name, (const xmlChar *)SWISH_PROP)) {
            reset_headmaker(h);
            return;
        }
        else if (xmlStrEqual(name, (const xmlChar *)SWISH_META)) {
            reset_headmaker(h);
            return;
        }
        else if (xmlStrEqual(name, (const xmlChar *)SWISH_INDEX)) {
            reset_headmaker(h);
            return;
        }
        else if (xmlStrEqual(name, (const xmlChar *)SWISH_PARSERS)) {
            reset_headmaker(h);
            return;
        }
        else if (xmlStrEqual(name, (const xmlChar *)SWISH_MIME)) {
            reset_headmaker(h);
            return;
        }
        else if (xmlStrEqual(name, (const xmlChar *)SWISH_ALIAS)) {
            reset_headmaker(h);
            return;
        }

        //SWISH_DEBUG_MSG("END ELEMENT name %s  type %d  value %s", name, type, value);

        return;

    }
    else {
    
              
    /* the special include directive means we stop and process
     * that config file immediately instead of storing the value
     * in the hash.
     */
        if (xmlStrEqual(name, (xmlChar *)SWISH_INCLUDE_FILE)) {
            swish_header_merge((char*)value, h->config);
            return;
        }
        else if (xmlStrEqual(name, (const xmlChar *)SWISH_PROP)) {
            reset_headmaker(h);
            h->isprops = 1;
            return;
        }
        else if (xmlStrEqual(name, (const xmlChar *)SWISH_META)) {
            reset_headmaker(h);
            h->ismetas = 1;
            return;
        }
        else if (xmlStrEqual(name, (const xmlChar *)SWISH_INDEX)) {
            reset_headmaker(h);
            h->isindex = 1;
            return;
        }
        else if (xmlStrEqual(name, (const xmlChar *)SWISH_PARSERS)) {
            reset_headmaker(h);
            h->isparser = 1;
            return;
        }
        else if (xmlStrEqual(name, (const xmlChar *)SWISH_MIME)) {
            reset_headmaker(h);
            h->ismime = 1;
            return;
        }
        else if (xmlStrEqual(name, (const xmlChar *)SWISH_ALIAS)) {
            reset_headmaker(h);
            h->isalias = 1;
            return;
        }

        //SWISH_DEBUG_MSG("NOT END ELEMENT name %s  type %d  value %s", name, type, value);
    }

    if (type != XML_READER_TYPE_END_ELEMENT) {

        if (h->isprops) {
            read_property(reader, h);
            return;
        }
        else if (h->ismetas) {
            read_metaname(reader, h);
            return;
        }
        else if (h->isindex) {
            read_key_value_pair(reader, h->config->index, (xmlChar *)name);
            return;
        }
        else if (h->isparser) {
            read_key_values_pair(reader, h->config->parsers, (xmlChar *)name);
            return;
        }
        else if (h->ismime) {
            read_key_value_pair(reader, h->config->mimes, (xmlChar *)name);
            return;
        }
        else if (h->isalias) {
            read_key_values_pair(reader, h->config->tag_aliases, (xmlChar *)name);
            return;
        }
        else if (xmlStrEqual((xmlChar *)SWISH_CLASS_ATTRIBUTES, (xmlChar *)name)) {
            read_key_value_stringlist(reader, h->config->stringlists, (xmlChar *)name);
            return;
        }
        else if (type == XML_READER_TYPE_ELEMENT) {
            read_key_value_pair(reader, h->config->misc, (xmlChar *)name);
            handle_special_misc_flags(h);
            return;
        }

        /*
           SWISH_DEBUG_MSG("STILL NOT END ELEMENT name %s  type %d  value %s", name, type, value); 
         */

    }

}

static void
read_key_value_stringlist(
    xmlTextReaderPtr reader,
    xmlHashTablePtr hash,
    xmlChar *name
)
{
    swish_StringList *strlist;
    xmlChar *str;
    const xmlChar *value;

/* element. get text and add to misc */
    if (xmlTextReaderRead(reader) == 1) {
        if (xmlTextReaderNodeType(reader) == XML_READER_TYPE_TEXT) {

            value = xmlTextReaderConstValue(reader);
            str = swish_str_tolower((xmlChar *)value);
            strlist = swish_stringlist_build(str);
            if (swish_hash_exists(hash, name)) {
                swish_stringlist_merge(strlist, swish_hash_fetch(hash, name));
            }
            else {
                swish_hash_add(hash, name, strlist);
            }
            swish_xfree(str);
        }
        else {
            SWISH_CROAK("Top-level XML element missing value: %s", name);
        }
    }
    else {
        SWISH_CROAK("Error reading value for top-level XML element %s", name);
    }
}

static void
read_key_values_pair(
    xmlTextReaderPtr reader,
    xmlHashTablePtr hash,
    xmlChar *name
)
{
    swish_StringList *strlist;
    xmlChar *str;
    const xmlChar *value;
    int i;

/* element. get text and add to misc */
    if (xmlTextReaderRead(reader) == 1) {
        if (xmlTextReaderNodeType(reader) == XML_READER_TYPE_TEXT) {

            value = xmlTextReaderConstValue(reader);
            str = swish_str_tolower((xmlChar *)value);
            strlist = swish_stringlist_build(str);

            for (i = 0; i < strlist->n; i++) {
/*  SWISH_DEBUG_MSG("key_values pair: %s -> %s", strlist->word[i], name);  */
                if (swish_hash_exists(hash, strlist->word[i])) {
                    swish_hash_replace(hash, strlist->word[i], swish_xstrdup(name));
                }
                else {
                    swish_hash_add(hash, strlist->word[i], swish_xstrdup(name));
                }
            }

            swish_stringlist_free(strlist);
            swish_xfree(str);

        }
        else {
            SWISH_CROAK("Top-level XML element missing value: %s", name);
        }
    }
    else {
        SWISH_CROAK("Error reading value for top-level XML element %s", name);
    }

}

static void
read_key_value_pair(
    xmlTextReaderPtr reader,
    xmlHashTablePtr hash,
    xmlChar *name
)
{
    const xmlChar *value;

/* element. get text and add to misc */
    if (xmlTextReaderRead(reader) == 1) {
        if (xmlTextReaderNodeType(reader) == XML_READER_TYPE_TEXT) {
            value = xmlTextReaderConstValue(reader);            
            
            if (SWISH_DEBUG & SWISH_DEBUG_CONFIG) {
                SWISH_DEBUG_MSG("read key %s for value %s", name, value);
            }
            
            if (swish_hash_exists(hash, name)) {
            
                if (SWISH_DEBUG & SWISH_DEBUG_CONFIG) {
                    SWISH_DEBUG_MSG("replacing %s => %s in hash", name, value);
                }
                swish_hash_replace(hash, name, swish_xstrdup(value));
            }
            else {
            
                if (SWISH_DEBUG & SWISH_DEBUG_CONFIG) {
                    SWISH_DEBUG_MSG("adding %s => %s to hash", name, value);
                }
                swish_hash_add(hash, name, swish_xstrdup(value));
            }
        }
        else {
            SWISH_CROAK("Top-level XML element missing value: %s", name);
        }
    }
    else {
        SWISH_CROAK("Error reading value for top-level XML element %s", name);
    }

}

static void
read_header(
    char *filename,
    headmaker * h
)
{
    xmlTextReaderPtr reader;
    int ret;

/* parse either a filename, or, if we can't stat it,
 * assume conf is a XML string.
 */
    if (!swish_fs_file_exists((xmlChar*)filename)) {
        reader =
            xmlReaderForMemory((const char *)filename, xmlStrlen((xmlChar *)filename),
                               "[ swish.xml ]", NULL, 0);

        if (SWISH_DEBUG & SWISH_DEBUG_CONFIG) {
            SWISH_DEBUG_MSG("header parsed in-memory");
        }
    }
    else {
        reader = xmlReaderForFile(filename, NULL, 0);

        if (SWISH_DEBUG & SWISH_DEBUG_CONFIG) {
            SWISH_DEBUG_MSG("header parsed from file");
        }
    }

    if (reader != NULL) {
        ret = xmlTextReaderRead(reader);
        while (ret == 1) {
            process_node(reader, h);
            ret = xmlTextReaderRead(reader);
        }
        xmlFreeTextReader(reader);
        if (ret != 0) {
            SWISH_CROAK("%s : failed to parse\n", filename);
        }
    }
    else {
        SWISH_CROAK("Unable to open %s\n", filename);
    }

/*
 * Cleanup function for the XML library.
 */
    xmlCleanupParser();
}

static void
test_meta_alias_for(
    swish_MetaName *meta,
    swish_Config *c,
    xmlChar *name
)
{
    if (meta->alias_for != NULL && !swish_hash_exists(c->metanames, meta->alias_for)
        ) {
        SWISH_CROAK
            ("MetaName '%s' has alias_for value of '%s' but no such MetaName defined",
             name, meta->alias_for);
    }
}

static void
test_prop_alias_for(
    swish_Property *prop,
    swish_Config *c,
    xmlChar *name
)
{
    if (prop->alias_for != NULL && !swish_hash_exists(c->properties, prop->alias_for)) {
        SWISH_CROAK("Property '%s' has alias_for value of '%s' but no such Property defined",
             name, prop->alias_for);
    }
}

void
swish_config_test_alias_fors(
    swish_Config *c
)
{
    xmlHashScan(c->metanames, (xmlHashScanner)test_meta_alias_for, c);
    xmlHashScan(c->properties, (xmlHashScanner)test_prop_alias_for, c);
}


static headmaker *
init_headmaker(
)
{
    headmaker *h;
    h = swish_xmalloc(sizeof(headmaker));
    h->config = swish_config_init();
/*  mimes is set to NULL in default config but we need it to be a hash here. */
    h->config->mimes = swish_hash_init(8);
    reset_headmaker(h);
    h->prop_id = SWISH_PROP_THIS_MUST_COME_LAST_ID;
    h->meta_id = SWISH_META_THIS_MUST_COME_LAST_ID;
    return h;
}

static void
reset_headmaker(
    headmaker *h
)
{
    h->isprops = 0;
    h->ismetas = 0;
    h->isindex = 0;
    h->isalias = 0;
    h->isparser = 0;
    h->ismime = 0;
    h->parent_name = NULL;
}

boolean
swish_header_validate(
    char *filename
)
{
    headmaker *h;
    h = init_headmaker();
    read_header(filename, h);

/*  test that all the alias_for links resolve ok */
    swish_config_test_alias_fors(h->config);

    swish_config_debug(h->config);
    swish_config_free(h->config);
    swish_xfree(h);
    return 1;                   /* how to test ? */
}

boolean
swish_header_merge(
    char *filename,
    swish_Config *c
)
{
    headmaker *h;
    h = init_headmaker();
    read_header(filename, h);
    if (SWISH_DEBUG & SWISH_DEBUG_CONFIG) {
        SWISH_DEBUG_MSG("read_header complete");
    }
    swish_config_merge(c, h->config);
    if (SWISH_DEBUG & SWISH_DEBUG_CONFIG) {
        SWISH_DEBUG_MSG("config_merge complete");
    }
    swish_config_free(h->config);
    swish_xfree(h);
    if (SWISH_DEBUG & SWISH_DEBUG_CONFIG) {
        SWISH_DEBUG_MSG("temp head struct freed");
    }

/*  test that all the alias_for links resolve ok */
    swish_config_test_alias_fors(c);

    return 1;
}

swish_Config *
swish_header_read(
    char *filename
)
{
    headmaker *h;
    swish_Config *c;
    h = init_headmaker();
    read_header(filename, h);
    c = h->config;
    swish_xfree(h);
    return c;
}

static void
write_open_tag(
    xmlTextWriterPtr writer,
    xmlChar *tag
)
{
    int rc;
    if (SWISH_DEBUG & SWISH_DEBUG_CONFIG) {
        SWISH_DEBUG_MSG("writing open tag <%s>", tag);
    }
    rc = xmlTextWriterStartElement(writer, tag);
    if (SWISH_DEBUG & SWISH_DEBUG_CONFIG) {
        SWISH_DEBUG_MSG("wrote open tag <%s>", tag);
    }

    if (rc < 0) {
        SWISH_CROAK("Error writing element %s", tag);
    }
}

static void
write_close_tag(
    xmlTextWriterPtr writer
)
{
    int rc;
    if (SWISH_DEBUG & SWISH_DEBUG_CONFIG) {
        SWISH_DEBUG_MSG("writing close tag");
    }
    rc = xmlTextWriterEndElement(writer);
    if (SWISH_DEBUG & SWISH_DEBUG_CONFIG) {
        SWISH_DEBUG_MSG("wrote close tag");
    }
    if (rc < 0) {
        SWISH_CROAK("Error at xmlTextWriterEndElement");
    }
}

static void
write_element_with_content(
    xmlTextWriterPtr writer,
    xmlChar *tag,
    xmlChar *content
)
{
    int rc;
    rc = xmlTextWriterWriteElement(writer, tag, content);
    if (rc < 0) {
        SWISH_CROAK("Error writing element %s with content %s", tag, content);
    }
}

static void
write_metaname(
    swish_MetaName *meta,
    xmlTextWriterPtr writer,
    xmlChar *name
)
{
    int rc;
    write_open_tag(writer, name);
    rc = xmlTextWriterWriteFormatAttribute(writer, BAD_CAST "id", "%d", meta->id);
    if (rc < 0) {
        SWISH_CROAK("Error writing metaname id attribute for %s", name);
    }

    if (meta->alias_for != NULL) {
        rc = xmlTextWriterWriteAttribute(writer, BAD_CAST "alias_for", meta->alias_for);
        if (rc < 0) {
            SWISH_CROAK("Error writing metaname alias_for attribute for %s", name);
        }

    }
    else {
        rc = xmlTextWriterWriteFormatAttribute(writer, BAD_CAST "bias", "%d", meta->bias);
        if (rc < 0) {
            SWISH_CROAK("Error writing metaname bias attribute for %s", name);
        }
    }

    write_close_tag(writer);
}

static void
write_metanames(
    xmlTextWriterPtr writer,
    xmlHashTablePtr metanames
)
{
    xmlHashScan(metanames, (xmlHashScanner)write_metaname, writer);
}

static void
write_hash_entry(
    xmlChar *value,
    xmlTextWriterPtr writer,
    xmlChar *key
)
{
    write_element_with_content(writer, key, value);
}

static void
write_reverse_hash_entry(
    xmlChar *value,
    xmlTextWriterPtr writer,
    xmlChar *key
)
{
    write_element_with_content(writer, value, key);
}

static void
write_property(
    swish_Property *prop,
    xmlTextWriterPtr writer,
    xmlChar *name
)
{
    int rc;
    write_open_tag(writer, name);
    rc = xmlTextWriterWriteFormatAttribute(writer, BAD_CAST "id", "%d", prop->id);
    if (rc < 0) {
        SWISH_CROAK("Error writing property id attribute for %s", name);
    }

    if (prop->alias_for != NULL) {
        rc = xmlTextWriterWriteAttribute(writer, BAD_CAST "alias_for", prop->alias_for);
        if (rc < 0) {
            SWISH_CROAK("Error writing property alias_for attribute for %s", name);
        }
    }
    else {

/* all other attrs are irrelevant if this is an alias */
        rc = xmlTextWriterWriteFormatAttribute(writer, BAD_CAST "ignore_case", "%d",
                                               prop->ignore_case);
        if (rc < 0) {
            SWISH_CROAK("Error writing property ignore_case attribute for %s", name);
        }
        rc = xmlTextWriterWriteFormatAttribute(writer, BAD_CAST "verbatim", "%d",
                                               prop->verbatim);
        if (rc < 0) {
            SWISH_CROAK("Error writing property verbatim attribute for %s", name);
        }
        rc = xmlTextWriterWriteFormatAttribute(writer, BAD_CAST "type", "%d", prop->type);
        if (rc < 0) {
            SWISH_CROAK("Error writing property type attribute for %s", name);
        }
        rc = xmlTextWriterWriteFormatAttribute(writer, BAD_CAST "max", "%d", prop->max);
        if (rc < 0) {
            SWISH_CROAK("Error writing property max attribute for %s", name);
        }
        rc = xmlTextWriterWriteFormatAttribute(writer, BAD_CAST "sort", "%d", prop->sort);
        if (rc < 0) {
            SWISH_CROAK("Error writing property sort attribute for %s", name);
        }
        rc = xmlTextWriterWriteFormatAttribute(writer, BAD_CAST "sort_length", "%d", prop->sort_length);
        if (rc < 0) {
            SWISH_CROAK("Error writing property sort attribute for %s", name);
        }
        rc = xmlTextWriterWriteFormatAttribute(writer, BAD_CAST "presort", "%d", prop->presort);
        if (rc < 0) {
            SWISH_CROAK("Error writing property sort attribute for %s", name);
        }
    }
    write_close_tag(writer);
}

static void
write_properties(
    xmlTextWriterPtr writer,
    xmlHashTablePtr properties
)
{
    xmlHashScan(properties, (xmlHashScanner)write_property, writer);
}

static void
write_parser(
    xmlChar *val,
    xmlTextWriterPtr writer,
    xmlChar *key
)
{
    write_element_with_content(writer, val, key);
}

static void
write_parsers(
    xmlTextWriterPtr writer,
    xmlHashTablePtr parsers
)
{
    xmlHashScan(parsers, (xmlHashScanner)write_parser, writer);
}

static void
write_mime(
    xmlChar *type,
    temp_things  *things,
    xmlChar *ext
)
{
    if (   !swish_hash_exists((xmlHashTablePtr) things->thing1, ext)
        || !xmlStrEqual(swish_hash_fetch((xmlHashTablePtr) things->thing1, ext), type)
    ) {

/*
        if (!swish_hash_exists((xmlHashTablePtr) things->thing1, ext)) {
            SWISH_DEBUG_MSG("%s not in hash", ext);
        }
*/
        if (SWISH_DEBUG & SWISH_DEBUG_CONFIG) {
            SWISH_DEBUG_MSG("writing unique MIME %s => %s", ext, type);
        }
        write_element_with_content((xmlTextWriterPtr) things->thing3, ext, type);
        if (SWISH_DEBUG & SWISH_DEBUG_CONFIG) {
            SWISH_DEBUG_MSG("wrote unique MIME %s => %s", ext, type);
        }
    }
}

static void
write_mimes(
    xmlTextWriterPtr writer,
    xmlHashTablePtr mimes
)
{
/*  only write what differs from the default */
    temp_things *t;
    t = swish_xmalloc(sizeof(temp_things));
    t->thing1 = swish_mime_defaults();
    t->thing2 = mimes;
    t->thing3 = writer;
    xmlHashScan(mimes, (xmlHashScanner)write_mime, t);
    if (SWISH_DEBUG & SWISH_DEBUG_CONFIG) {
        SWISH_DEBUG_MSG("done writing MIMEs");
    }
    swish_hash_free(t->thing1);
    if (SWISH_DEBUG & SWISH_DEBUG_CONFIG) {
        SWISH_DEBUG_MSG("freed thing1 hash");
    }
    swish_xfree(t);
}

static void
write_index(
    xmlTextWriterPtr writer,
    xmlHashTablePtr index
)
{
    xmlHashScan(index, (xmlHashScanner)write_hash_entry, writer);
}

static void
write_tag_aliases(
    xmlTextWriterPtr writer,
    xmlHashTablePtr tag_aliases
)
{
    xmlHashScan(tag_aliases, (xmlHashScanner)write_reverse_hash_entry, writer);
}

static void
write_misc(
    xmlTextWriterPtr writer,
    xmlHashTablePtr hash
)
{
    xmlHashScan(hash, (xmlHashScanner)write_hash_entry, writer);
}

void
swish_header_write(
    char *uri,
    swish_Config *config
)
{
#if !defined(LIBXML_WRITER_ENABLED) || !defined(LIBXML_OUTPUT_ENABLED)
    SWISH_CROAK("libxml2 writer not compiled in this version of libxml2");
#else
    int rc;
    xmlTextWriterPtr writer;

    if (SWISH_DEBUG & SWISH_DEBUG_CONFIG) {
        swish_config_debug(config);
    }

/* Create a new XmlWriter for uri, with no compression. */
    writer = xmlNewTextWriterFilename((const char *)uri, 0);
    if (writer == NULL) {
        SWISH_CROAK("Error creating the xml writer\n");
    }

/* set some basic formatting rules. these make it easier to debug headers */
    rc = xmlTextWriterSetIndent(writer, 1);
    if (rc < 0) {
        SWISH_CROAK("failed to set indent on XML writer");
    }

/* Start the document with the xml default for the version,
     * encoding UTF-8 (default) and the default for the standalone
     * declaration. */
    rc = xmlTextWriterStartDocument(writer, NULL, NULL, NULL);
    if (rc < 0) {
        SWISH_CROAK("Error at xmlTextWriterStartDocument\n");
    }

/* root element
    NOTE the BAD_CAST macro is xml2 shortcut for (xmlChar*)
*/
    write_open_tag(writer, BAD_CAST SWISH_HEADER_ROOT);

/* Write a comment indicating a computer wrote this file */
    rc = xmlTextWriterWriteComment(writer, BAD_CAST "written by libswish3 - DO NOT EDIT");
    if (rc < 0) {
        SWISH_CROAK("Error at xmlTextWriterWriteComment\n");
    }

    // TODO check for these in reader and croak if mismatch
    if (!swish_hash_exists(config->misc, BAD_CAST "swish_version")) {
        write_element_with_content(writer, BAD_CAST "swish_version",
                                   BAD_CAST SWISH_VERSION);
    }
    if (!swish_hash_exists(config->misc, BAD_CAST "swish_lib_version")) {
        write_element_with_content(writer, BAD_CAST "swish_lib_version",
                                   BAD_CAST swish_lib_version());
    }

/* write MetaNames */
    write_open_tag(writer, BAD_CAST SWISH_META);
    write_metanames(writer, config->metanames);
    write_close_tag(writer);

/* write PropertyNames */
    write_open_tag(writer, BAD_CAST SWISH_PROP);
    write_properties(writer, config->properties);
    write_close_tag(writer);

/* write Parsers */
    write_open_tag(writer, BAD_CAST SWISH_PARSERS);
    write_parsers(writer, config->parsers);
    write_close_tag(writer);

/* write MIMEs */

    write_open_tag(writer, BAD_CAST SWISH_MIME);
    write_mimes(writer, config->mimes);
    write_close_tag(writer);

/* write index */
    write_open_tag(writer, BAD_CAST SWISH_INDEX);
    write_index(writer, config->index);
    write_close_tag(writer);

    write_open_tag(writer, BAD_CAST SWISH_ALIAS);
    write_tag_aliases(writer, config->tag_aliases);
    write_close_tag(writer);

/* misc tags have no parent */
    write_misc(writer, config->misc);

/* this function will close any open tags */
    rc = xmlTextWriterEndDocument(writer);
    if (rc < 0) {
        SWISH_CROAK("Error at xmlTextWriterEndDocument\n");
    }

    xmlFreeTextWriter(writer);
#endif
}


/*************** end header.c ************/


/*************** start tokenizer.c ************/
/*
 * This file is part of libswish3
 * Copyright (C) 2008 Peter Karman
 *
 *  libswish3 is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  libswish3 is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with libswish3; if not, write to the Free Software
 *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
*/

/* utf8 tokenizer */
#ifndef LIBSWISH3_SINGLE_FILE
#include <wchar.h>
#include <string.h>
#include <ctype.h>
#include <wctype.h>
#include <err.h>
#include <stdarg.h>

#include "libswish3.h"
#endif

extern int SWISH_DEBUG;

static int is_ignore_start_utf8(
    uint32_t c
);
static int is_ignore_end_utf8(
    uint32_t c
);
static int is_ignore_word_utf8(
    uint32_t c
);
static int is_ignore_start_ascii(
    char c
);
static int is_ignore_end_ascii(
    char c
);
static int is_ignore_word_ascii(
    char c
);
static void make_ascii_tables(
);
static int strip_utf8_chrs(
    xmlChar *token,
    int len
);
static int strip_ascii_chrs(
    xmlChar *word,
    int len
);

static int
is_ignore_start_utf8(
    uint32_t c
)
{
    return (!c || iswspace(c) || iswcntrl(c) || iswpunct(c)
        )
        ? 1 : 0;
}

static int
is_ignore_end_utf8(
    uint32_t c
)
{
    return (!c || iswspace(c) || iswcntrl(c) || iswpunct(c)
        )
        ? 1 : 0;
}

static int
is_ignore_word_utf8(
    uint32_t c
)
{
    if (c == '\'')              /*  contractions allowed */
        return 0;

    if (!c || iswspace(c) || iswcntrl(c) || iswpunct(c)
        )
        return 1;

    return 0;
}

static int
is_ignore_start_ascii(
    char c
)
{
    return (!c || isspace(c) || iscntrl(c) || ispunct(c)
        )
        ? 1 : 0;

}

static int
is_ignore_end_ascii(
    char c
)
{
    return (!c || isspace(c) || iscntrl(c) || ispunct(c)
        )
        ? 1 : 0;
}

static int
is_ignore_word_ascii(
    char c
)
{
    if (c == '\'')              /*  contractions allowed */
        return 0;

    return (!c || isspace(c) || iscntrl(c) || ispunct(c)
        )
        ? 1 : 0;
}

/************************************************
*   mimic the Swish-e WordCharacters lookup tables
*   using the default is*() functions.
*************************************************/

static boolean ascii_init = 0;
static char ascii_word_table[128];
static char ascii_start_table[128];
static char ascii_end_table[128];

static void
make_ascii_tables(
)
{
    int i;
    for (i = 0; i < 127; i++) {
        if (is_ignore_word_ascii(i))
            ascii_word_table[i] = 0;
        else
            ascii_word_table[i] = 1;

        if (is_ignore_end_ascii(i))
            ascii_end_table[i] = 0;
        else
            ascii_end_table[i] = 1;

        if (is_ignore_start_ascii(i))
            ascii_start_table[i] = 0;
        else
            ascii_start_table[i] = 1;

    }
    ascii_init = 1;
}

/*************************************************
* remove all ignorable start/end chars
* and return new length of token
* based on code in swish-e vers2 swish_words.c
*************************************************/

static int
strip_utf8_chrs(
    xmlChar *token,
    int len
)
{
    int i, j, k, chr_len, start, end, token_len, cp;
    xmlChar chr[5];

    start = 0;
    end = 0;

    if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER)
        SWISH_DEBUG_MSG("Before: %s", token);

// end chrs -- must do before start chars
    j = len;
    for (i = len; i >= 0; swish_utf8_prev_chr(token, &i)) {
        chr_len = j - i;
        if (!chr_len) {
            j = i;
            continue;
        }
        for (k = 0; k < chr_len; k++) {
            chr[k] = token[i + k];
        }
        chr[k] = '\0';
        if (is_ignore_end_utf8(swish_utf8_codepoint(chr))) {
            token[i] = '\0';
            end++;
        }
        else {
            break;
        }
    }

    chr[0] = '\0';
    j = 0;

// start chrs 
    for (i = 0; token[j] != '\0'; swish_utf8_next_chr(token, &i)) {
        chr_len = i - j;
        if (!chr_len) {
            j = i;
            continue;
        }
        for (k = 0; k < chr_len; k++) {
            chr[k] = token[j + k];
        }
        chr[k] = '\0';
        cp = swish_utf8_codepoint(chr);

        if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER)
            SWISH_DEBUG_MSG("start chr_len %d chr: %s  [%d]", chr_len, chr, cp);

        if (!is_ignore_start_utf8(cp)) {
            break;
        }
        else {
            if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER)
                SWISH_DEBUG_MSG("ignore_start %s", chr);

            token += i;
            start++;
        }
    }

    token_len = xmlStrlen(token) + 1;

    if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER)
        SWISH_DEBUG_MSG("After: %s (stripped %d start chars, %d end chars, len=%d)",
                        token, start, end, token_len);

    return token_len;
}

static int
strip_ascii_chrs(
    xmlChar *word,
    int len
)
{
    int i, j, k, wlen, start, end;
    start = 0;
    end = 0;

    if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER)
        SWISH_DEBUG_MSG("Before: %s", word);

/* end chars -- must do before start chars */

    i = len;

/* Iteratively strip off the last character if it's an ignore character */
    while (i-- > 0) {

        if (!ascii_end_table[word[i]]) {
            word[i] = '\0';
            end++;
        }
        else {
            break;
        }
    }

/* start chars */
    i = 0;

    while (word[i]) {
        k = i;
        if (ascii_start_table[word[k]]) {
            break;
        }
        else {
            i = k + 1;
            start++;
        }
    }

/* If all the chars are valid, just leave word alone */
    if (i != 0) {
        for (k = i, j = 0; word[k] != '\0'; j++, k++) {
            word[j] = word[k];
        }

/* Add the NUL */
        word[j] = '\0';
    }

    wlen = xmlStrlen(word) + 1;

    if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER)
        SWISH_DEBUG_MSG("After: %s (stripped %d start chars, %d end chars, wlen=%d)",
                        word, start, end, wlen);

    return wlen;
}

swish_TokenList *
swish_token_list_init(
)
{
    swish_TokenList *tl;
    tl = swish_xmalloc(sizeof(swish_TokenList));
    tl->buf = xmlBufferCreateSize((size_t) SWISH_BUFFER_CHUNK_SIZE);
    tl->n = 0;
    tl->pos = 0;
    tl->ref_cnt = 0;
    tl->tokens = swish_xmalloc(sizeof(swish_Token *) * SWISH_TOKEN_LIST_SIZE);
    tl->contexts = swish_hash_init(8);

    if (!ascii_init)
        make_ascii_tables();

    if (SWISH_DEBUG & SWISH_DEBUG_MEMORY) {
        SWISH_DEBUG_MSG("TokenList ptr 0x%x", (long int)tl);
        SWISH_DEBUG_MSG("TokenList->tokens ptr 0x%x", (long int)tl->tokens);
    }

    return tl;
}

void
swish_token_list_free(
    swish_TokenList *tl
)
{
    if (tl->ref_cnt != 0) {
        SWISH_WARN("freeing TokenList with ref_cnt != 0 (%d)", tl->ref_cnt);
    }

    while (tl->n) {
        tl->n--;
        tl->tokens[tl->n]->ref_cnt--;
        if (tl->tokens[tl->n]->ref_cnt < 1)
            swish_token_free(tl->tokens[tl->n]);
    }

    swish_xfree(tl->tokens);
    xmlBufferFree(tl->buf);
    swish_hash_free(tl->contexts); // MUST free **after** tokens
    swish_xfree(tl);
}

int
swish_token_list_add_token(
    swish_TokenList *tl,
    xmlChar *token,
    int token_len,
    swish_MetaName *meta,
    xmlChar *context
)
{
    int num_of_allocs;
    swish_Token *stoken;

    if (!token_len || !xmlStrlen(token)) {
        SWISH_CROAK("can't add empty token to token list");
    }

    if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER)
        SWISH_DEBUG_MSG("adding token: %s  meta=%s", token, meta->name);

    stoken = swish_token_init();
    stoken->offset  = xmlBufferLength(tl->buf);
    stoken->len     = token_len - 1;    // -1 to exclude the NUL
    stoken->pos     = ++tl->pos;
    stoken->meta    = meta;
    stoken->meta->ref_cnt++;
        
    /* add the token str to the token_list buffer */
    swish_token_list_set_token(tl, token, token_len);

    /* cache the context string and point at the cached value */
    swish_hash_exists_or_add( tl->contexts, context, context );
    stoken->context = swish_hash_fetch( tl->contexts, context );
    stoken->value   = swish_token_list_get_token_value( tl, stoken );
    stoken->ref_cnt++;

    num_of_allocs = tl->n / SWISH_TOKEN_LIST_SIZE;

    if (SWISH_DEBUG & SWISH_DEBUG_TOKENLIST) {
        SWISH_DEBUG_MSG("TokenList size: %d  num_allocs = %d  modulus %d", tl->n,
                        num_of_allocs, tl->n % SWISH_TOKEN_LIST_SIZE);
        swish_token_debug(stoken);
    }

    if (num_of_allocs && !(tl->n % SWISH_TOKEN_LIST_SIZE)) {
        if (SWISH_DEBUG & SWISH_DEBUG_TOKENLIST) {
            SWISH_DEBUG_MSG("realloc for tokens: 0x%x", (long int)tl->tokens);
        }

        tl->tokens =
            (swish_Token **)swish_xrealloc(tl->tokens,
                                           sizeof(swish_Token*) * (SWISH_TOKEN_LIST_SIZE *
                                                                  ++num_of_allocs));

    }
    tl->tokens[tl->n++] = stoken;
    return tl->n;
}

int
swish_token_list_set_token(
    swish_TokenList *tl,
    xmlChar *token,
    int len
)
{
    int ret;
    /* include the NUL so token->value can be treated like substr */
    ret = xmlBufferAdd(tl->buf, token, len);    
    if (ret != 0) {
        SWISH_CROAK("error appending token to buffer: %d", ret);
    }
    return ret;
}

swish_Token *
swish_token_init(
)
{
    swish_Token *t;
    t = swish_xmalloc(sizeof(swish_Token));
    t->pos = 0;
    t->offset = 0;
    t->meta = NULL;
    t->context = NULL;
    t->value = NULL;
    t->len = 0;
    t->ref_cnt = 0;
    return t;
}

void
swish_token_free(
    swish_Token *t
)
{
    if (t->ref_cnt != 0) {
        SWISH_WARN("freeing Token with ref_cnt != 0 (%d)", t->ref_cnt);
    }
    
    if (SWISH_DEBUG & SWISH_DEBUG_MEMORY) {
        SWISH_DEBUG_MSG("freeing Token 0x%x with MetaName ref_cnt %d", 
            (long int)t, t->meta->ref_cnt);
    }
    
    t->meta->ref_cnt--;
    if (t->meta->ref_cnt == 0) {
        if (SWISH_DEBUG & SWISH_DEBUG_MEMORY) {
            SWISH_DEBUG_MSG("Token's MetaName ref_cnt == 0 ... freeing MetaName");
        }
        swish_metaname_free(t->meta);
    }
    
    swish_xfree(t);
}

void
swish_token_debug(
    swish_Token *t
)
{
    SWISH_DEBUG_MSG("\n\
    t->ref_cnt      = %d\n\
    t->pos          = %d\n\
    t->context      = %s\n\
    t->meta         = %d [%s]\n\
    t->offset       = %d\n\
    t->len          = %d\n\
    t->value        = %s\n\
    ", t->ref_cnt, t->pos, t->context, t->meta->id, t->meta->name, t->offset, t->len, t->value);

}

void
swish_token_list_debug(
    swish_TokenIterator *it
)
{
    swish_Token *t;

    SWISH_DEBUG_MSG("Token buf:\n%s", xmlBufferContent(it->tl->buf));
    SWISH_DEBUG_MSG("Token buf length: %d\n", xmlBufferLength(it->tl->buf));
    SWISH_DEBUG_MSG("Number of tokens: %d", it->tl->n);

    while ((t = swish_token_iterator_next_token(it)) != NULL) {
        swish_token_debug(t);
    }
}

swish_TokenIterator *
swish_token_iterator_init(
    swish_Analyzer *a
)
{
    swish_TokenIterator *it;
    it = swish_xmalloc(sizeof(swish_TokenIterator));
    it->a = a;
    it->a->ref_cnt++;
    it->pos = 0;
    it->tl = swish_token_list_init();
    it->tl->ref_cnt++;
    it->ref_cnt = 0;
    return it;
}

void
swish_token_iterator_free(
    swish_TokenIterator *it
)
{
    if (it->ref_cnt != 0) {
        SWISH_WARN("freeing TokenIterator with ref_cnt != 0 (%d)", it->ref_cnt);
    }
    
    if (SWISH_DEBUG & SWISH_DEBUG_MEMORY) {
        SWISH_DEBUG_MSG(
        "freeing TokenIterator %d with TokenList ref_cnt %d and Analyzer ref_cnt %d", 
        it, it->tl->ref_cnt, it->a->ref_cnt);
    }
    
    it->a->ref_cnt--;
    if (SWISH_DEBUG & SWISH_DEBUG_MEMORY) {
        SWISH_DEBUG_MSG("freeing TokenIterator with Analyzer ref_cnt = %d",
            it->a->ref_cnt);
    }
    if (it->a->ref_cnt == 0)
        swish_analyzer_free(it->a);
        
    it->tl->ref_cnt--;
    if (it->tl->ref_cnt == 0)
        swish_token_list_free(it->tl);
    
    swish_xfree(it);
}

xmlChar *
swish_token_list_get_token_value(
    swish_TokenList *tl,
    swish_Token *t
)
{
    const xmlChar *buf;
    buf = xmlBufferContent(tl->buf);
    buf += t->offset;
    return (xmlChar*)buf;
}

swish_Token *
swish_token_iterator_next_token(
    swish_TokenIterator *it
)
{
    swish_Token *t;
    t = NULL;
    
/* SWISH_DEBUG_MSG("next_token: %d %d", it->pos, it->tl->n); */
    if (it->pos >= it->tl->n)
        return NULL;

    
    t = it->tl->tokens[it->pos++];
    t->value = swish_token_list_get_token_value(it->tl, t);
    return t;
}

/* returns number of tokens added to TokenList */
int
swish_tokenize(
    swish_TokenIterator *ti, 
    xmlChar *buf, 
    swish_MetaName *meta,
    xmlChar *context
)
{
    if (swish_is_ascii(buf)) {
        return swish_tokenize_ascii(ti, buf, meta, context);
    }
    else {
        return swish_tokenize_utf8(ti, buf, meta, context);
    }
}

int
swish_tokenize_utf8(
    swish_TokenIterator *ti, 
    xmlChar *buf, 
    swish_MetaName *meta,
    xmlChar *context
)
{
    uint32_t cp;
    int nstart, byte_pos, prev_pos, i, chr_len, token_len, maxwordlen, minwordlen;
    swish_TokenList *tl;
    boolean inside_token;
    xmlChar chr[5];             /*  max len of UCS32 plus NUL */
    xmlChar *token, *copy, *buf_lower;
    
    tl          = ti->tl;
    maxwordlen  = ti->a->maxwordlen;
    minwordlen  = ti->a->minwordlen;
    token       = swish_xmalloc(sizeof(xmlChar) * maxwordlen);
    buf_lower   = swish_utf8_str_tolower(buf);
    nstart      = tl->n;
    inside_token = 0;
    byte_pos    = 0;
    prev_pos    = 0;
    token_len   = 0;

    if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER)
        SWISH_DEBUG_MSG("starting tokenize3 for meta=%s", meta->name);

/*
       iterate over each utf8 character, evaluating its Unicode value,
       and creating tokens
*/

    for (   byte_pos = 0; 
            buf_lower[prev_pos] != '\0';
            swish_utf8_next_chr(buf_lower, &byte_pos)
    ) {
        chr_len = byte_pos - prev_pos;
        if (!chr_len) {
            prev_pos = byte_pos;
            continue;
        }

        for (i = 0; i < chr_len; i++) {
            chr[i] = buf_lower[prev_pos + i];
        }
        chr[i] = '\0';

        cp = swish_utf8_codepoint(chr);

        if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER) {
            SWISH_DEBUG_MSG("%d %d: ut8 chr '%s' unicode %d  len %d next byte: %d",
                            byte_pos, prev_pos, chr, cp, chr_len, buf_lower[prev_pos + 1]);

        }
        
        prev_pos = byte_pos;

        if (is_ignore_word_utf8(cp)) {

            if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER)
                SWISH_DEBUG_MSG("%s is ignore_word", chr);

            if (inside_token) {
                if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER)
                    SWISH_DEBUG_MSG("found end of token: '%s'", chr);

                inside_token = 0;       /*  turn off flag */

                token[++token_len] = '\0';
                copy = token;
                token_len = strip_utf8_chrs(token, token_len);

                if (token[0] != '\0' && token_len >= minwordlen) {

                    swish_token_list_add_token(tl, token, token_len, meta, context);

                }
                else {
                    if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER)
                        SWISH_DEBUG_MSG("skipping token '%s' -- too short: %d", token,
                                        token_len);
                }

                token = copy;   /*  restore to top of array so we do not leak */

                if (cp == SWISH_TOKENPOS_BUMPER[0]) {
                    if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER)
                        SWISH_DEBUG_MSG("found tokenpos bumper byte at pos %d", tl->pos);
                    tl->pos++;
                }

                continue;

            }
            else {
                if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER)
                    SWISH_DEBUG_MSG("ignoring chr '%s'", chr);

                if (cp == SWISH_TOKENPOS_BUMPER[0]) {
                    if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER)
                        SWISH_DEBUG_MSG("found tokenpos bumper byte at pos %d", tl->pos);
                    tl->pos++;
                }

                continue;
            }

        }
        else {

            if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER)
                SWISH_DEBUG_MSG("%s is NOT ignore_word", chr);

            if (inside_token) {

                /* edge case */
                if ((chr_len + token_len) > maxwordlen) {
                    if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER)
                        SWISH_DEBUG_MSG("token_len = %d  forcing end of token: '%s'",
                                        token_len, chr);
                    continue;
                }

                if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER)
                    SWISH_DEBUG_MSG("adding to token: '%s'", chr);

                memcpy(&token[token_len], chr, chr_len * sizeof(xmlChar));
                token[token_len + chr_len] = '\0';
                token_len += chr_len;

                if (token_len >= maxwordlen || buf_lower[byte_pos] == '\0') {

                    if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER)
                        SWISH_DEBUG_MSG("token_len = %d  forcing end of token: '%s'",
                                        token_len, chr);

                    inside_token = 0;   /*  turn off flag */

                    token[++token_len] = '\0';
                    copy = token;
                    token_len = strip_utf8_chrs(token, token_len);

                    if (token[0] != '\0' && token_len >= minwordlen) {

                        swish_token_list_add_token(tl, token, token_len, meta, context);

                    }
                    else {
                        if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER)
                            SWISH_DEBUG_MSG("skipping token '%s' -- too short: %d", token,
                                            token_len);
                    }

                    token = copy;       /*  restore to top of array */

                }

                if (cp == SWISH_TOKENPOS_BUMPER[0]) {
                    if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER)
                        SWISH_DEBUG_MSG("found tokenpos bumper byte at pos %d", tl->pos);
                    tl->pos++;
                }

                continue;

            }
            else {

                if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER)
                    SWISH_DEBUG_MSG("start a token with '%s'", chr);

                token[0] = '\0';
                token_len = 0;
                inside_token = 1;       /*  turn on flag */
                /* edge case */
                if (chr_len > maxwordlen)
                    continue;

                memcpy(&token[0], chr, chr_len * sizeof(xmlChar));
                token[chr_len] = '\0';
                token_len += chr_len;
                
                /* special case for one-character tokens */
                if (buf_lower[prev_pos] == '\0' && minwordlen == 1) {
                    inside_token        = 0;
                    token[token_len++]  = '\0';
                    swish_token_list_add_token(tl, token, token_len, meta, context);
                }

                if (cp == SWISH_TOKENPOS_BUMPER[0]) {
                    if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER)
                        SWISH_DEBUG_MSG("found tokenpos bumper byte at pos %d", tl->pos);
                    tl->pos++;
                }

                continue;

            }

        }

    }

    swish_xfree(token);
    swish_xfree(buf_lower);
    return tl->n - nstart;
}

int
swish_tokenize_ascii(
    swish_TokenIterator *ti, 
    xmlChar *buf, 
    swish_MetaName *meta,
    xmlChar *context
)
{
    char c, nextc;
    boolean inside_token;
    int i, token_len, nstart, maxwordlen, minwordlen;
    xmlChar *token, *copy;
    swish_TokenList *tl;
    
    tl              = ti->tl;
    maxwordlen      = ti->a->maxwordlen;
    minwordlen      = ti->a->minwordlen;
    token           = swish_xmalloc(sizeof(xmlChar) * maxwordlen);
    nstart          = tl->n;
    token_len       = 0;
    token[0]        = '\0';
    inside_token    = 0;

    if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER)
        SWISH_DEBUG_MSG("tokenizing string: '%s'", buf);

    for (i = 0; buf[i] != '\0'; i++) {
        c = (char)tolower(buf[i]);
        nextc = (char)tolower(buf[i + 1]);

        if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER)
            SWISH_DEBUG_MSG(" char: %c lower: %c  int: %d %#x (next is %c)", buf[i], c,
                            (int)c, (unsigned int)c, nextc);
                            
        if (!ascii_word_table[(int)c]) {

            if (inside_token) {
                if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER)
                    SWISH_DEBUG_MSG("found end of token: '%c' at %d", c, i);

                inside_token = 0;
                token[token_len++] = '\0';
                copy = token;
                token_len = strip_ascii_chrs(token, token_len);

                if (token[0] != '\0' && token_len >= minwordlen) {
                    swish_token_list_add_token(tl, token, token_len, meta, context);
                }
                else {
                    if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER)
                        SWISH_DEBUG_MSG("skipping token '%s' -- too short: %d", token,
                                        token_len);
                }

                token = copy;
                
                if (c == SWISH_TOKENPOS_BUMPER[0]) {
                    if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER)
                        SWISH_DEBUG_MSG("found tokenpos bumper byte at pos %d", tl->pos);
                    tl->pos++;
                }


                continue;

            }
            else {
                if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER)
                    SWISH_DEBUG_MSG("ignoring char '%c'", c);

                if (c == SWISH_TOKENPOS_BUMPER[0]) {
                    if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER)
                        SWISH_DEBUG_MSG("found tokenpos bumper byte at pos %d", tl->pos);
                    tl->pos++;
                }

                continue;
            }

        }
        else {

            if (inside_token) {

                if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER)
                    SWISH_DEBUG_MSG("adding to token: '%c' %d", c, i);

                token[token_len++] = c;

                if (token_len >= maxwordlen || nextc == '\0') {

                    if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER)
                        SWISH_DEBUG_MSG("forcing end of token: '%c' %d", c, i);

                    inside_token = 0;

                    token[token_len++] = '\0';
                    copy = token;
                    token_len = strip_ascii_chrs(token, token_len);

                    if (token[0] != '\0' && token_len >= minwordlen) {
                        swish_token_list_add_token(tl, token, token_len, meta, context);
                    }
                    else {
                        if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER)
                            SWISH_DEBUG_MSG("skipping token '%s' -- too short: %d", token,
                                            token_len);
                    }

                    token = copy;

                }

                if (c == SWISH_TOKENPOS_BUMPER[0]) {
                    if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER)
                        SWISH_DEBUG_MSG("found tokenpos bumper byte at pos %d", tl->pos);
                    tl->pos++;
                }

                continue;

            }
            else {

                if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER)
                    SWISH_DEBUG_MSG("start a token with '%c' %d", c, i);

                token_len = 0;
                inside_token = 1;
                token[token_len++] = c;
                
                /* special case for one-character tokens */
                if (nextc == '\0' && minwordlen == 1) {
                    inside_token        = 0;
                    token[token_len++]  = '\0';
                    swish_token_list_add_token(tl, token, token_len, meta, context);
                }
                
                if (c == SWISH_TOKENPOS_BUMPER[0]) {
                    if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER)
                        SWISH_DEBUG_MSG("found tokenpos bumper byte at pos %d", tl->pos);
                    tl->pos++;
                }

                continue;

            }

        }

    }

    swish_xfree(token);
    return tl->n - nstart;
}


/*************** end tokenizer.c ************/
