root/libswish3/trunk/src/libswish3/parser.c

/* 
* This file is part of libswish3
* Copyright (C) 2007 Peter Karman
*
*  libswish3 is free software; you can redistribute it and/or modify
*  it under the terms of the GNU General Public License as published by
*  the Free Software Foundation; either version 2 of the License, or
*  (at your option) any later version.
*
*  libswish3 is distributed in the hope that it will be useful,
*  but WITHOUT ANY WARRANTY; without even the implied warranty of
*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
*  GNU General Public License for more details.
*
*  You should have received a copy of the GNU General Public License
*  along with libswish3; if not, write to the Free Software
*  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
*/

/* 
* parse XML doc from memory using libxml2 SAX2 based on tutorial at
* http://www.jamesh.id.au/articles/libxml-sax/libxml-sax.html
*
* save all character() data to buffer, flushing on new metanames
* flush should split buffer into words, skipping nonwordchars/space, and
* lowercase all
*
* see iswlower(3) man page, etc.
*
* all the mb*() functions rely on locale to recognize multi-byte strings
*
*/

#include <stdio.h>
#include <stdlib.h>
#include <locale.h>
#include <stdarg.h>
#include <err.h>
#include <string.h>
#include <ctype.h>
#include <wctype.h>
#include <errno.h>
#include <libxml/parserInternals.h>
#include <libxml/parser.h>
#include <libxml/HTMLparser.h>
#include <libxml/globals.h>
#include <libxml/xmlerror.h>
#include <libxml/tree.h>
#include <libxml/debugXML.h>
#include <libxml/xmlmemory.h>

#include "libswish3.h"

extern int errno;
extern int SWISH_DEBUG;

// should we pass on libxml2 via SWISH_WARN()
int SWISH_PARSER_WARNINGS = 0;

static void get_env_vars(
);

static void flush_buffer(
    swish_ParserData *parser_data,
    xmlChar *metaname,
    xmlChar *context
);

static void tokenize(
    swish_ParserData *parser_data,
    xmlChar *string,
    int len,
    xmlChar *metaname,
    xmlChar *content
);

static void mystartDocument(
    void *parser_data
);
static void myendDocument(
    void *parser_data
);
static void mystartElement(
    void *parser_data,
    const xmlChar *name,
    const xmlChar **atts
);
static void myendElement(
    void *parser_data,
    const xmlChar *name
);

/* 
* SAX2 support 
*/
static void mystartElementNs(
    void *parser_data,
    const xmlChar *localname,
    const xmlChar *prefix,
    const xmlChar *URI,
    int nb_namespaces,
    const xmlChar **namespaces,
    int nb_attributes,
    int nb_defaulted,
    const xmlChar **attributes
);

static void myendElementNs(
    void *ctx ATTRIBUTE_UNUSED,
    const xmlChar *localname,
    const xmlChar *prefix,
    const xmlChar *URI
);

static void buffer_characters(
    swish_ParserData *parser_data,
    const xmlChar *ch,
    int len
);
static void mycharacters(
    void *parser_data,
    const xmlChar *ch,
    int len
);
static void mycomments(
    void *parser_data,
    const xmlChar *ch
);
static void myerr(
    void *user_data,
    xmlChar *msg,
    ...
);

static void open_tag(
    void *data,
    const xmlChar *tag,
    xmlChar **atts
);
static void close_tag(
    void *data,
    const xmlChar *tag
);
static xmlChar *bake_tag(
    swish_ParserData *parser_data,
    xmlChar *tag,
    xmlChar **atts
);

static int docparser(
    swish_ParserData *parser_data,
    xmlChar *filename,
    xmlChar *buffer,
    int size
);
static int xml_parser(
    xmlSAXHandlerPtr sax,
    void *user_data,
    xmlChar *buffer,
    int size
);
static int html_parser(
    xmlSAXHandlerPtr sax,
    void *user_data,
    xmlChar *buffer,
    int size
);
static int txt_parser(
    swish_ParserData *parser_data,
    xmlChar *buffer,
    int size
);

static swish_ParserData *init_parser_data(
    swish_3 *s3
);
static void free_parser_data(
    swish_ParserData *parser_data
);

/* 
* parsing fh/buffer headers 
*/
typedef struct
{
    xmlChar **lines;
    int body_start;
    int nlines;
} HEAD;

static HEAD *buf_to_head(
    xmlChar *buf
);
static void free_head(
    HEAD * h
);
static swish_DocInfo *head_to_docinfo(
    HEAD * h
);

static xmlChar *document_encoding(
    xmlParserCtxtPtr ctxt
);

static void set_encoding(
    swish_ParserData *parser_data,
    xmlChar *buffer
);

/* tag tracker */
static xmlChar *flatten_tag_stack(
    xmlChar *baked,
    swish_TagStack *stack,
    char flatten_join
);
static void add_stack_to_prop_buf(
    xmlChar *baked,
    swish_ParserData *parser_data
);
static void push_tag_stack(
    swish_TagStack *stack,
    xmlChar *raw,
    xmlChar *baked,
    char flatten_join
);
static swish_Tag *pop_tag_stack(
    swish_TagStack *stack
);
static swish_Tag *pop_tag_stack_on_match(
    swish_TagStack *stack,
    xmlChar *raw
);
static void free_swishTag(
    swish_Tag * st
);
static void
free_swishTagStack(
    swish_TagStack *stack
);

/***********************************************************************
*                end prototypes
***********************************************************************/

swish_Parser *
swish_init_parser(
    void (*handler) (swish_ParserData *)
)
{
    swish_Parser *p = (swish_Parser *)swish_xmalloc(sizeof(swish_Parser));

    p->handler = handler;
    p->ref_cnt = 0;

/*
* libxml2 stuff 
*/
    xmlInitParser();
    xmlSubstituteEntitiesDefault(1);    /* resolve text entities */

/*
* debugging help 
*/
    get_env_vars();

    if (SWISH_DEBUG & SWISH_DEBUG_MEMORY) {
        SWISH_DEBUG_MSG("parser ptr 0x%x", (long int)p);
    }

    return p;
}

void
swish_free_parser(
    swish_Parser *p
)
{
    if (SWISH_DEBUG & SWISH_DEBUG_MEMORY) {
        SWISH_DEBUG_MSG("freeing parser");
        swish_mem_debug();
    }
    if (p->ref_cnt != 0) {
        SWISH_WARN("parser ref_cnt != 0: %d\n", p->ref_cnt);
    }
    xmlCleanupParser();
    xmlMemoryDump();
    swish_xfree(p);
}

/* 
* turn the literal xml/html tag into a swish tag for matching against
* metanames and properties 
*/
static xmlChar *
bake_tag(
    swish_ParserData *parser_data,
    xmlChar *tag,
    xmlChar **atts
)
{
    int i, j, is_html_tag, size;
    xmlChar *swishtag, *attr_lower, *attr_val_lower, *alias, *metaname, *metacontent;
    swish_StringList *strlist;

    if (SWISH_DEBUG & SWISH_DEBUG_PARSER) {
        SWISH_DEBUG_MSG(" tag: %s   parser->tag: %s ", tag, parser_data->tag);
        if (atts != NULL) {
            SWISH_DEBUG_MSG(" has attributes [%d]", xmlStrlen((xmlChar *)atts));
            for (i = 0; (atts[i] != NULL); i += 2) {
                SWISH_DEBUG_MSG(" att: %s=", atts[i]);
                if (atts[i + 1] != NULL) {
                    SWISH_DEBUG_MSG(" '%s'", atts[i + 1]);
                }
            }
        }
    }

    metaname = NULL;
    metacontent = NULL;

/*
* normalize all tags 
*/
    swishtag = swish_str_tolower(tag);

/*
* html tags 
*/
    if (parser_data->is_html) {

/*
           TODO config features about img tags and a/href tags 
*/
        if (xmlStrEqual(swishtag, (xmlChar *)"br")
            || xmlStrEqual(swishtag, (xmlChar *)"img")) {
            
            if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
                SWISH_DEBUG_MSG("found html tag '%s' ... bump_word = 1", swishtag);
            parser_data->bump_word = 1;
        }
        else {
            const htmlElemDesc *element = htmlTagLookup(swishtag);

            if (!element)
                is_html_tag = 0;        /* flag that this might be a meta * name */

            else if (!element->isinline) {

/*
* need to bump token position so we don't match across block *
* elements 
*/
                if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
                    SWISH_DEBUG_MSG("found html !inline tag '%s' ... bump_word = 1", swishtag);
                parser_data->bump_word = 1;

            }
            else {
            
                if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
                    SWISH_DEBUG_MSG("found html inline tag '%s' ... bump_word = 0", swishtag);
                parser_data->bump_word = 0;
            
            }
        }

/*
* is this an HTML <meta> tag? treat 'name' attribute as a tag *
* and 'content' attribute as the tag content * we assume 'name'
* and 'content' are always in english. 
*/

        if (atts != 0) {
            for (i = 0; (atts[i] != 0); i++) {

                if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
                    SWISH_DEBUG_MSG("%d HTML attr: %s", i, atts[i]);

                if (xmlStrEqual(atts[i], (xmlChar *)"name")) {

/*
* SWISH_DEBUG_MSG("found name: %s", atts[i+1]); 
*/
                    metaname = (xmlChar *)atts[i + 1];
                }

                else if (xmlStrEqual(atts[i], (xmlChar *)"content")) {

/*
* SWISH_DEBUG_MSG("found content: %s", atts[i+1]); 
*/
                    metacontent = (xmlChar *)atts[i + 1];
                }

            }
        }

        if (metaname != NULL) {
            if (metacontent != NULL) {
                if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
                    SWISH_DEBUG_MSG("found HTML meta: %s => %s", metaname, metacontent);

/*
* do not match across metas 
*/
                if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
                    SWISH_DEBUG_MSG("found html meta tag '%s' ... bump_word = 1", metaname);
                parser_data->bump_word = 1;
                open_tag(parser_data, metaname, NULL);
                buffer_characters(parser_data, metacontent, xmlStrlen(metacontent));
                close_tag(parser_data, metaname);
                swish_xfree(swishtag);
                return NULL;

            }
            else {
                SWISH_WARN("No content for meta tag '%s'", metaname);
            }
        }

    }

/*
* xml tags 
*/
    else {

/*
* TODO make this configurable ala swish2 
*/

        if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
            SWISH_DEBUG_MSG("found xml tag '%s' ... bump_word = 1", swishtag);
        parser_data->bump_word = 1;

        if (atts != NULL
            && swish_hash_exists(parser_data->s3->config->stringlists,
                                 (xmlChar *)SWISH_CLASS_ATTRIBUTES)) {
            strlist =
                swish_hash_fetch(parser_data->s3->config->stringlists,
                                 (xmlChar *)SWISH_CLASS_ATTRIBUTES);

            for (i = 0; (atts[i] != NULL); i += 2) {

                if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
                    SWISH_DEBUG_MSG(" %d XML attr: %s=%s [%d]", i, atts[i], atts[i + 1],
                                    xmlStrlen(atts[i + 1]));

                attr_lower = swish_str_tolower(atts[i]);
                attr_val_lower = swish_str_tolower(atts[i + 1]);

/*
                   is it one of ours? 
*/
                for (j = 0; j < strlist->n; j++) {
                    if (xmlStrEqual(strlist->word[j], attr_lower)) {
                        if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
                            SWISH_DEBUG_MSG("found %s: %s", attr_lower, attr_val_lower);

/*  eligible attribute name */
                        size = xmlStrlen(swishtag) + xmlStrlen(attr_val_lower) + 2;     /*  dot + NULL */
                        metaname = swish_xmalloc(size + 1);
                        snprintf((char *)metaname, size, "%s.%s", (char *)swishtag,
                                 (char *)attr_val_lower);

                        swish_xfree(swishtag);
                        swishtag = metaname;
                    }
                }

                swish_xfree(attr_lower);
                swish_xfree(attr_val_lower);

            }
        }

    }

/*
* change our internal name for this tag if it is aliased in config 
*/
    alias = swish_hash_fetch(parser_data->s3->config->tag_aliases, swishtag);
    if (alias) {

/*
* SWISH_DEBUG_MSG("%s alias -> %s", swishtag, alias); 
*/
        swish_xfree(swishtag);
        swishtag = swish_xstrdup(alias);
    }

    if (SWISH_DEBUG & SWISH_DEBUG_PARSER) {
        SWISH_DEBUG_MSG(" swishtag = %s", swishtag);
    }

    return swishtag;
}

static void
flush_buffer(
    swish_ParserData *parser_data,
    xmlChar *metaname,
    xmlChar *context
)
{
    swish_TagStack *s = parser_data->metastack;

    if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
        SWISH_DEBUG_MSG("buffer is >>%s<< before flush",
                        xmlBufferContent(parser_data->meta_buf));

/*
* add meta_buf as-is to metanames buffer under current tag. this
* gives us both tokens and raw text de-tagged but organized by
* metaname. 
*/
    swish_add_buf_to_nb(parser_data->metanames, metaname, parser_data->meta_buf,
                        (xmlChar *)SWISH_TOKENPOS_BUMPER, 0, 1);

/*
*  add to every metaname on the stack.
*  Disabling this for now, as it ought to be up the handler() to decide
*  to index a token under multiple metanames, and we associate context
*  with the TokenList
*/

    if (parser_data->s3->config->flags->context_as_meta) {
        for (s->temp = s->head; s->temp != NULL; s->temp = s->temp->next) {
            if (xmlStrEqual(s->temp->baked, metaname))  /*  already added */
                continue;

            swish_add_buf_to_nb(parser_data->metanames, s->temp->baked,
                                parser_data->meta_buf, (xmlChar *)SWISH_TOKENPOS_BUMPER,
                                0, 1);
        }
    }

    if (parser_data->s3->analyzer->tokenize) {
        tokenize(parser_data, (xmlChar *)xmlBufferContent(parser_data->meta_buf),
                 xmlBufferLength(parser_data->meta_buf), metaname, context);
    }

    xmlBufferEmpty(parser_data->meta_buf);

}

/* 
* SAX2 callback 
*/
static void
mystartDocument(
    void *data
)
{

/*
* swish_ParserData *parser_data = (swish_ParserData *) data; 
*/

    if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
        SWISH_DEBUG_MSG("startDocument()");

}

/* 
* SAX2 callback 
*/
static void
myendDocument(
    void *parser_data
)
{

    if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
        SWISH_DEBUG_MSG("endDocument()");

/*
* whatever's left 
*/
    flush_buffer(parser_data, (xmlChar *)SWISH_DEFAULT_METANAME,
                 (xmlChar *)SWISH_DEFAULT_METANAME);

}

/* 
* SAX1 callback 
*/
static void
mystartElement(
    void *data,
    const xmlChar *name,
    const xmlChar **atts
)
{
    open_tag(data, name, (xmlChar **)atts);
}

/* 
* SAX1 callback 
*/
static void
myendElement(
    void *data,
    const xmlChar *name
)
{
    close_tag(data, name);
}

/* 
* SAX2 handler 
*/
static void
mystartElementNs(
    void *data,
    const xmlChar *localname,
    const xmlChar *prefix,
    const xmlChar *URI,
    int nb_namespaces,
    const xmlChar **namespaces,
    int nb_attributes,
    int nb_defaulted,
    const xmlChar **attributes
)
{
    int i, j, len;
    xmlChar **atts;
    atts = NULL;

    if (nb_attributes > 0) {
        atts = swish_xmalloc(((nb_attributes * 2) + 1) * sizeof(xmlChar *));
        j = 0;
        for (i = 0; i < nb_attributes * 5; i += 5) {
            atts[j] = (xmlChar *)attributes[i];
            len = (int)(attributes[i + 4] - attributes[i + 3]);
            if (len > 0) {
                atts[j + 1] = xmlStrsub(attributes[i + 3], 0, len);
            }
            else {
                atts[j] = NULL;
            }
            j += 2;
        }
        atts[j] = NULL;
    }

    if (SWISH_DEBUG & SWISH_DEBUG_PARSER) {
        SWISH_DEBUG_MSG(" tag: %s nb_attributes %d", localname, nb_attributes);
        if (atts != NULL) {
            for (i = 0; (atts[i] != NULL); i += 2) {
                SWISH_DEBUG_MSG(" att: %s=%s", atts[i], atts[i + 1]);
/* SWISH_DEBUG_MSG(" att: %s=", atts[i++], atts[i] || ""); */
            }
        }
    }

    open_tag(data, localname, atts);

    if (atts != NULL) {
        swish_xfree(atts);
    }
}

/* 
* SAX2 handler 
*/
static void
myendElementNs(
    void *data,
    const xmlChar *localname,
    const xmlChar *prefix,
    const xmlChar *URI
)
{
    close_tag(data, localname);
}

static void
open_tag(
    void *data,
    const xmlChar *tag,
    xmlChar **atts
)
{
    swish_ParserData *parser_data;
    xmlChar *baked;
    
    parser_data = (swish_ParserData *)data;
    
    if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
        SWISH_DEBUG_MSG("<%s>", tag);

    if (parser_data->tag != NULL)
        swish_xfree(parser_data->tag);

    parser_data->tag = bake_tag(parser_data, (xmlChar *)tag, (xmlChar **)atts);
        
    if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
        SWISH_DEBUG_MSG("checking config for '%s' in watched tags", parser_data->tag);

/* all tags on domstack */

    if (parser_data->tag == NULL) {
        push_tag_stack(parser_data->domstack, (xmlChar *)tag, (xmlChar *)tag, SWISH_DOT);
    }
    else {
        push_tag_stack(parser_data->domstack, (xmlChar *)tag, parser_data->tag, SWISH_DOT);
    }
    
/*
* set property if this tag is configured for it 
*/
    if (swish_hash_exists(parser_data->s3->config->properties, parser_data->tag)
        ||
        swish_hash_exists(parser_data->s3->config->properties, parser_data->domstack->head->context)
    ) {
        if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
            SWISH_DEBUG_MSG(" %s = new property", parser_data->tag);

        add_stack_to_prop_buf(NULL, parser_data);       /* NULL means all properties in the stack are added */
        xmlBufferEmpty(parser_data->prop_buf);
        
        if (swish_hash_exists(parser_data->s3->config->properties, parser_data->domstack->head->context))
            baked = parser_data->domstack->head->context;
        else
            baked = parser_data->tag;

        push_tag_stack(parser_data->propstack, (xmlChar *)tag, baked, SWISH_SPACE);

        if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
            SWISH_DEBUG_MSG("%s pushed ok unto propstack", parser_data->tag);
    }

/*
* likewise for metastack 
*/
    if (swish_hash_exists(parser_data->s3->config->metanames, parser_data->tag)
        ||
        swish_hash_exists(parser_data->s3->config->metanames, parser_data->domstack->head->context)
    ) {
        if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
            SWISH_DEBUG_MSG(" %s = new metaname", parser_data->tag);

        flush_buffer(parser_data, parser_data->metastack->head->baked,
                     parser_data->metastack->head->context);
                     
        if (swish_hash_exists(parser_data->s3->config->properties, parser_data->domstack->head->context))
            baked = parser_data->domstack->head->context;
        else
            baked = parser_data->tag;

        push_tag_stack(parser_data->metastack, (xmlChar *)tag, baked, SWISH_SPACE);
    }

    if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
        SWISH_DEBUG_MSG("config check for '%s' done", parser_data->tag);

}

static void
close_tag(
    void *data,
    const xmlChar *tag
)
{
    swish_ParserData *parser_data;
    swish_Tag *st;

    parser_data = (swish_ParserData *)data;

    if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
        SWISH_DEBUG_MSG("</%s>", tag);
        
/*
* lowercase all names for comparison against metanames (which are
* also * lowercased) 
*/
    if (parser_data->tag != NULL)
        swish_xfree(parser_data->tag);

    parser_data->tag = bake_tag(parser_data, (xmlChar *)tag, NULL);

    if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
        SWISH_DEBUG_MSG(" endElement(%s) (%s)", (xmlChar *)tag, parser_data->tag);
        
    if (parser_data->tag == NULL)
        return;

    if ((st = pop_tag_stack_on_match(parser_data->propstack, (xmlChar *)tag)) != NULL) {

        add_stack_to_prop_buf(st->baked, parser_data);
        xmlBufferEmpty(parser_data->prop_buf);
        free_swishTag(st);
    }

    if ((st = pop_tag_stack_on_match(parser_data->metastack, (xmlChar *)tag)) != NULL) {

        flush_buffer(parser_data, st->baked, st->context);
        free_swishTag(st);
    }
    
    // always pop the raw domstack
    st = pop_tag_stack(parser_data->domstack);
    free_swishTag(st);

}

/* 
* handle all characters in doc 
*/
static void
buffer_characters(
    swish_ParserData *parser_data,
    const xmlChar *ch,
    int len
)
{
    int i;
    xmlChar output[len];
    xmlBufferPtr buf = parser_data->meta_buf;

/*
* why not wchar_t ? len is number of bytes, not number of
* characters, so xmlChar (i.e., char) works
*/

/*
* SWISH_DEBUG_MSG( "sizeof output buf is %d; len was %d\n", sizeof(output),
* len );
*/

/*
* SWISH_DEBUG_MSG( "characters"); 
*/

    for (i = 0; i < len; i++) {
        output[i] = ch[i];
    }
    output[i] = (xmlChar)NULL;

    if (parser_data->bump_word && xmlBufferLength(buf)) {
        swish_append_buffer(buf, (xmlChar *)SWISH_TOKENPOS_BUMPER, 1);
    }
    
    swish_append_buffer(buf, output, len);

    if (parser_data->bump_word && xmlBufferLength(parser_data->prop_buf)) {
        swish_append_buffer(parser_data->prop_buf, (xmlChar *)SWISH_TOKENPOS_BUMPER, 1);
    }
    else if (xmlBufferLength(parser_data->prop_buf)) {
        swish_append_buffer(parser_data->prop_buf, (xmlChar*)" ", 1);
    }

    swish_append_buffer(parser_data->prop_buf, output, len);
}

/* 
* SAX2 callback 
*/
static void
mycharacters(
    void *parser_data,
    const xmlChar *ch,
    int len
)
{
    if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
        SWISH_DEBUG_MSG(" >> mycharacters()");

    buffer_characters(parser_data, ch, len);
}

/* 
* SAX2 callback 
*/
static void
mycomments(
    void *parser_data,
    const xmlChar *ch
)
{
    int len = strlen((char *)(char *)ch);

/*
* TODO: make comments indexing optional 
*/

/*
* TODO: enable noindex option 
*/
    return;

    buffer_characters(parser_data, ch, len);
}

/* 
* SAX2 callback 
*/
static void
myerr(
    void *data,
    xmlChar *msg,
    ...
)
{
    swish_ParserData *parser_data;
    va_list args;
    char str[1000];

    if (!SWISH_PARSER_WARNINGS)
        return;

    parser_data = (swish_ParserData *)data;

    SWISH_WARN("libxml2 error for %s:", parser_data->docinfo->uri);

    va_start(args, msg);
    vsnprintf((char *)str, 1000, (char *)msg, args);
    xmlParserError(parser_data->ctxt, (char *)str);
    va_end(args);
}

/* 
* SAX2 callback 
*/
static void
mywarn(
    void *user_data,
    xmlChar *msg,
    ...
)
{
    swish_ParserData *parser_data;
    va_list args;
    char str[1000];

    if (!SWISH_PARSER_WARNINGS)
        return;

    parser_data = (swish_ParserData *)user_data;

    SWISH_WARN("libxml2 warning for %s:", parser_data->docinfo->uri);

    va_start(args, msg);
    vsnprintf((char *)str, 1000, (char *)msg, args);
    xmlParserWarning(parser_data->ctxt, (char *)str);
    va_end(args);
}

/* 
* SAX2 handler struct for html and xml parsing 
*/

xmlSAXHandler my_parser = {
    NULL,                       /* internalSubset */
    NULL,                       /* isStandalone */
    NULL,                       /* hasInternalSubset */
    NULL,                       /* hasExternalSubset */
    NULL,                       /* resolveEntity */
    NULL,                       /* getEntity */
    NULL,                       /* entityDecl */
    NULL,                       /* notationDecl */
    NULL,                       /* attributeDecl */
    NULL,                       /* elementDecl */
    NULL,                       /* unparsedEntityDecl */
    NULL,                       /* setDocumentLocator */
    mystartDocument,            /* startDocument */
    myendDocument,              /* endDocument */
    mystartElement,             /* startElement */
    myendElement,               /* endElement */
    NULL,                       /* reference */
    mycharacters,               /* characters */
    NULL,                       /* ignorableWhitespace */
    NULL,                       /* processingInstruction */
    mycomments,                 /* comment */
    (warningSAXFunc) & mywarn,  /* xmlParserWarning */
    (errorSAXFunc) & myerr,     /* xmlParserError */
    (fatalErrorSAXFunc) & myerr,        /* xmlfatalParserError */
    NULL,                       /* getParameterEntity */
    NULL,                       /* cdataBlock -- should we handle this too *
                                 * ?? */
    NULL,                       /* externalSubset; */
    XML_SAX2_MAGIC,
    NULL,
    mystartElementNs,           /* startElementNs */
    myendElementNs,             /* endElementNs */
    NULL                        /* xmlStructuredErrorFunc */
};

xmlSAXHandlerPtr my_parser_ptr = &my_parser;

static int
docparser(
    swish_ParserData *parser_data,
    xmlChar *filename,
    xmlChar *buffer,
    int size
)
{

    int ret;
    xmlChar *mime = (xmlChar *)parser_data->docinfo->mime;
    xmlChar *parser = (xmlChar *)parser_data->docinfo->parser;

    if (!size && !xmlStrlen(buffer) && !parser_data->docinfo->size) {
        SWISH_WARN("%s appears to be empty -- can't parse it", parser_data->docinfo->uri);

        return 1;
    }

    if (SWISH_DEBUG & SWISH_DEBUG_PARSER)
        SWISH_DEBUG_MSG("%s -- using %s parser", parser_data->docinfo->uri, parser);

/*
* slurp file if not already in memory 
*/
    if (filename && !buffer) {
        buffer = swish_slurp_file_len(filename, (long)parser_data->docinfo->size);
        size = parser_data->docinfo->size;
    }

    if (parser[0] == 'H') {
        parser_data->is_html = 1;
        ret = html_parser(my_parser_ptr, parser_data, buffer, size);
    }

    else if (parser[0] == 'X')
        ret = xml_parser(my_parser_ptr, parser_data, buffer, size);

    else if (parser[0] == 'T')
        ret = txt_parser(parser_data, (xmlChar *)buffer, size);