Changeset 1930

Timestamp:

04/30/07 23:08:43 (1 year ago)

Author:

karpet

Message:

refactor to buffer all MetaNames? as well as PropertyNames? in NamedBuffer?

Files:

• libswish3/trunk/bindings/perl/3.xs (modified) (12 diffs)
• libswish3/trunk/bindings/perl/typemap (modified) (1 diff)
• libswish3/trunk/src/libswish3/Makefile.am (modified) (1 diff)
• libswish3/trunk/src/libswish3/libswish3.h (modified) (12 diffs)
• libswish3/trunk/src/libswish3/mem.c (modified) (1 diff)
• libswish3/trunk/src/libswish3/namedbuffer.c (added)
• libswish3/trunk/src/libswish3/parser.c (modified) (23 diffs)
• libswish3/trunk/src/libswish3/properties.c (deleted)
• libswish3/trunk/src/libswish3/words.c (modified) (8 diffs)
• libswish3/trunk/src/swish_lint.c (modified) (1 diff)
• libswish3/trunk/src/swish_words.c (modified) (2 diffs)

libswish3/trunk/bindings/perl/3.xs

@@ -72 +72 @@
- */
-
-private package vars
+all XS stuff is prefixed with 'sp_' for Swish Perl
-
-#define DEFAULT_BASE_CLASS  "SWISH::3::Parser"
-#define CONFIG_CLASS        "SWISH::3::Config"
-#define ANALYZER_CLASS      "SWISH::3::Analyzer"
-
-
-
+sp_
+sp_
+sp_
-
-
@@ -340 +340 @@
-    swish_debug_docinfo( parse_data->docinfo );
-    swish_debug_wordlist( parse_data->wordlist );
-
+
+
-    warn("\n");
-}
@@ -377 +378 @@
-    SV *token_re;
-    swish_WordList *list;
+    swish_WordList *(*token_handler)(xmlChar * start_ptr, int tok_bytes, int start, int end, ...);
-    va_list args;
-    va_start(args, str);
@@ -392 +394 @@
-    xmlChar    *str_end         = str_start + str_len;
-    
-
+
+
-    
-    /* extract regexp struct from qr// entity */
@@ -408 +411 @@
-    SvREADONLY_on(wrapper);
-    SvLEN(wrapper) = 0;
-TODO conditional on swish_is_ascii() ??
+we trust str is already utf-8 encoded.
-    
-    /* wrap the string in an SV to please the regex engine */
@@ -415 +418 @@
-    SvPOK_on(wrapper);
-
-WordL
+wordl
-    num_code_points = 0;
-    
@@ -423 +426 @@
-        xmlChar * end_ptr   = str + rx->endp[0];
-        int start, end, tok_bytes, tok_pts;
-        xmlChar * token;
-
-        /* get start and end offsets in Unicode code points */
@@ -446 +448 @@
-        tok_pts   = end - start;
-        tok_bytes = end_ptr - start_ptr;
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-            
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-    return list;
-}
-
-
-/*******************************************************************************
@@ -980 +1017 @@
-        if (self->ref_cnt < 1)
-        {
-WordL
+wordl
-        }
-        
@@ -1030 +1067 @@
-        
-    CODE:
-Hash,
+erties->hash, 
-        RETVAL = newSVpvn((char*)xmlBufferContent(buf), xmlBufferLength(buf));
-        
@@ -1210 +1247 @@
-        RETVAL->regex = (void*)SvREFCNT_inc( regex );
-        RETVAL->tokenizer = &sp_tokenize;
+        RETVAL->stash = &sp_token_handler;
-        
-    OUTPUT:
@@ -1296 +1334 @@
-# TODO: get/set methods, including way to set tokenizer func ref
-
+
+# tokenize_isw() uses native libswish3 tokenizer
+swish_WordList *
+tokenize_isw(self, str, ...)
+    SV * self;
+    SV * str;
+
+    PREINIT:
+        char * CLASS;
+        xmlChar * metaname = SWISH_DEFAULT_METANAME;   
+        xmlChar * context  = SWISH_DEFAULT_METANAME;
+        unsigned int word_pos    = 0;
+        unsigned int offset      = 0;
+        xmlChar * buf = SvPV(str, PL_na);
+        
+    CODE:
+        CLASS = sp_which_class("WordList");
+        
+        if (!SvUTF8(str))
+        {
+            if (swish_is_ascii(buf))
+                SvUTF8_on(str);     /* flags original SV ?? */
+            else
+                croak("%s is not flagged as a UTF-8 string and is not ASCII", buf);
+        }
+        
+        if ( items > 2 )
+        {
+            word_pos = (int)SvIV(ST(2));
+            
+            if ( items > 3 )
+                offset = (int)SvIV(ST(3));
+                
+            if ( items > 4 )
+                metaname = SvPV(ST(4), PL_na);
+                
+            if ( items > 5 )
+                context = SvPV(ST(5), PL_na);
+                
+        }
+                
+        swish_init_words(); /* in case it wasn't initialized elsewhere... */
+        RETVAL = swish_tokenize(
+                        (swish_Analyzer*)sp_ptr_from_object(self),
+                        buf,
+                        word_pos,
+                        offset,
+                        metaname,
+                        context
+                        );
+        
+        RETVAL->ref_cnt++;
+        
+        /* TODO do we need to worry about free()ing metaname and context ?? */
+                        
+    OUTPUT:
+        RETVAL
+        

libswish3/trunk/bindings/perl/typemap

@@ -11 +11 @@
-swish_Analyzer *           O_OBJECT
-swish_Parser *             O_OBJECT
+swish_NamedBuffer *        O_OBJECT
-
-INPUT

libswish3/trunk/src/libswish3/Makefile.am

@@ -20 +20 @@
-                        mime_types.c \
-                        parser.c \
-properties
+namedbuffer
-                        string.c \
-                        times.c \

libswish3/trunk/src/libswish3/libswish3.h

@@ -52 +52 @@
-#define SWISH_INCLUDE_FILE   "IncludeConfigFile"
-#define SWISH_PROP           "PropertyNames"
+#define SWISH_PROP_ASIS      "nostripchars"
-#define SWISH_PROP_MAX       "PropertyNamesMaxLength"
-#define SWISH_PROP_SORT      "PropertyNamesSortKeyLength"
@@ -129 +130 @@
-#define SWISH_DEBUG_TOKENIZER   5
-#define SWISH_DEBUG_PARSER      9
+#define SWISH_DEBUG_NAMEDBUFFER 15
-
-#ifdef __cplusplus
@@ -170 +172 @@
-void        swish_mem_debug();
-xmlChar *   swish_xstrdup( const xmlChar * ptr );
+xmlChar *   swish_xstrndup( const xmlChar * ptr, int len );
-
-/* time functions */
@@ -205 +208 @@
-struct swish_Config
-{
-
-*          stash;      /* also
+         
+          *stash;      /*
-    xmlHashTablePtr conf;       /* the meat */
-};
@@ -259 +262 @@
-typedef struct swish_Analyzer          swish_Analyzer;
-typedef struct swish_Parser            swish_Parser;
+typedef struct swish_NamedBuffer            swish_NamedBuffer;
+
+struct swish_NamedBuffer
+{
+    int             ref_cnt;    /* for scripting languages */
+    void           *stash;      /* for scripting languages */
+    xmlHashTablePtr hash;       /* the meat */
+};
-
-struct swish_DocInfo
@@ -299 +310 @@
-    xmlChar            *name;
-    struct swish_Tag   *next;
-
+ 
-};
-
@@ -321 +332 @@
-    void                  *stash;              // for script bindings
-    void                  *regex;              // optional regex
-
+         
-};
-
@@ -336 +347 @@
-struct swish_ParseData
-{
-ext
-
+mp text (MetaName)
+tmp 
-    xmlChar               *tag;                // current tag name
-    swish_DocInfo         *docinfo;            // document-specific properties
-    swish_Config          *config;             // global config
+    unsigned int           context_as_meta;    // index tokens under all applicable MetaNames
-    unsigned int           no_index;           // toggle flag for special comments
-    unsigned int           is_html;            // shortcut flag for html parser
@@ -348 +360 @@
-    swish_TagStack        *metastack;          // stacks for tracking the tag => metaname
-    swish_TagStack        *propstack;          // stacks for tracking the tag => property
-
+               // so we can free at end
-    swish_WordList        *wordlist;           // linked list of words
-
+
+
-    swish_Analyzer        *analyzer;           // Analyzer struct
-
+    
-};
-
@@ -374 +387 @@
-
-
-/* utility buffers */
-void                swish_append_buffer(xmlBufferPtr buf, const xmlChar * txt, int txtlen);
-
-
-
-/* word functions */
-void                swish_init_words();
-WordL
-WordL
+wordl
+wordl
-swish_WordList *    swish_tokenize( swish_Analyzer * analyzer, xmlChar * str, ... );
-
@@ -418 +426 @@
-                                            int word_pos, 
-                                            int offset );
+
+int                 swish_add_to_wordlist_len(  
+                                            swish_WordList * list, 
+                                            xmlChar * str,
+                                            int len,
+                                            xmlChar * metaname,
+                                            xmlChar * context,
+                                            int word_pos, 
+                                            int offset );
-                                            
-void                swish_debug_wordlist( swish_WordList * list );
@@ -429 +446 @@
-void                swish_free_docinfo( swish_DocInfo * ptr );
-int                 swish_check_docinfo(swish_DocInfo * docinfo, swish_Config * config);
-
+
+
+
-void                swish_debug_docinfo( swish_DocInfo * docinfo );
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-

libswish3/trunk/src/libswish3/mem.c

@@ -76 +76 @@
-}
-
+xmlChar * swish_xstrndup( const xmlChar * ptr, int len )
+{
+    memcount++;
+    if ( SWISH_DEBUG > 20 )
+        swish_debug_msg( "memcount = %ld", memcount);
+    return( xmlStrndup( ptr, len ) );
+}
+
-void swish_xfree( void *ptr )
-{    

libswish3/trunk/src/libswish3/parser.c

@@ -64 +64 @@
-static void     get_env_vars();
-
-swish_ParseData * parse_data, xmlChar * metaname);
-
-xmlHashTablePtr propHash, 
-
+   swish_ParseData * parse_data, 
+
+);
+
-static void     tokenize(       swish_ParseData * parse_data, 
-                                xmlChar * string, 
@@ -186 +186 @@
-build_tag(swish_ParseData * parse_data, xmlChar * tag, xmlChar ** atts)
-{
-       
-      
-
-
+
+
+
+   
-    metacontent = NULL;
-    
-    /* normalize all tags */
-    swishtag = swish_str_tolower(tag);
-
-
-    /* html tags */
@@ -278 +277 @@
-
-
-> 2
-
-startElement
+== SWISH_DEBUG_PARSER
+
+build_tag 
-        if (atts != 0)
-        {
@@ -293 +292 @@
-        }
-        fprintf(stderr, ")\n");
-
-    }
-
-
-    /* change our internal name for this tag if it is aliased in config */
-parse_data->
+swish
-    if (alias)
-    {
+        //swish_debug_msg("%s alias -> %s", swishtag, alias);
-        swish_xfree(swishtag);
-        swishtag = swish_xstrdup(alias);
@@ -309 +308 @@
-}
-
-void
-swish_append_buffer(xmlBufferPtr buf, const xmlChar * txt, int txtlen)
-{
-    int ret;
-    
-    if (txtlen == 0)
-        /* shouldn't happen */
-        return;
-        
-    if (buf == NULL)
-    {
-        swish_fatal_err("bad news. buf ptr is NULL");
-        
-    }
-
-    ret = xmlBufferAdd( buf, txt, txtlen );
-    if (ret)
-    {
-        swish_fatal_err("problem adding \n>>%s<<\n length %d to buffer. Err: %d", 
-                        txt, txtlen, ret);
-    }
-    
-}
-
-static void
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
-    if (SWISH_DEBUG > 10)
-        swish_debug_msg("buffer is >>%s<< before flush, word_pos = %d", 
@@ -375 +324 @@
-    if (parse_data->word_pos)
-        parse_data->word_pos++;
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-                );
+    }
-
-    xmlBufferEmpty(parse_data->buf_ptr);
@@ -409 +381 @@
-        swish_debug_msg("endDocument()");
-
-
+
+
-
-}
@@ -466 +439 @@
-
-
-
-
+
-        swish_debug_msg("checking config for '%s' in watched tags", parse_data->tag);
-
-
-    /* set property if this tag is configured for it */
- *) 
-
-> 8
+*)
+
+== SWISH_DEBUG_PARSER
-            swish_debug_msg(" %s = new property", parse_data->tag);
-
@@ -486 +458 @@
-
-    /* likewise for metastack */
- *) 
-
-> 8
+*)
+
+== SWISH_DEBUG_PARSER
-            swish_debug_msg(" %s = new metaname", parse_data->tag);
-
-parse_data, NULL
+
+ parse_data, parse_data->metastack->head->name, parse_data->metastack->flat 
-
-        parse_data->metastack = push_tag_stack(parse_data->metastack, parse_data->tag);
-    }
-    
-> 8
+== SWISH_DEBUG_PARSER
-        swish_debug_msg("config check for '%s' done", parse_data->tag);
-
@@ -505 +477 @@
-close_tag(void *data, const xmlChar * tag)
-{
-*metaname
+ *context
-    swish_ParseData *parse_data;
-    parse_data = (swish_ParseData *) data;
@@ -519 +491 @@
-        swish_debug_msg(" endElement(%s) (%s)", (xmlChar *) tag, parse_data->tag);
-
-metaname
-
-parse_data->tag
+context
+
+context
-        add_stack_to_prop_buf(parse_data->tag, parse_data);
-        xmlBufferEmpty(parse_data->prop_buf);
-metaname
-
-
-metaname
+context
+
+
+context
-    {
-        /* swish_debug_msg("popped %s from metastack", parse_data->tag); */
-metaname
-metaname
+parse_data->tag, context
+context
-    }
-
@@ -572 +544 @@
-
-    if (parse_data->bump_word && xmlBufferLength(parse_data->prop_buf))
+    {
+        //swish_debug_msg("   appending ' ' to prop_buf");
-        swish_append_buffer(parse_data->prop_buf, (xmlChar *) " ", 1);
-
+
+
+
-    swish_append_buffer(parse_data->prop_buf, output, len);
-
@@ -771 +747 @@
-    
-    ptr->tag = NULL;
-
-
+
+
+
-
-    /* prime the stacks */
@@ -781 +758 @@
-    ptr->metastack->flat = NULL;
-    ptr->metastack->count = 0;
- *) 
+*)
-
-    ptr->propstack = (swish_TagStack *) swish_xmalloc(sizeof(swish_TagStack));
@@ -789 +766 @@
-    ptr->propstack->flat = NULL;
-    ptr->propstack->count = 0;
- *) "_");    /* no such property --
-                                     *
+*)"_");    
+/* no such property
-
-    /* gets toggled per-tag */
@@ -807 +784 @@
-    ptr->offset = 0;
-
+    /* TODO make this configurable */
+    ptr->context_as_meta = 1;
-
-    /* pointer to the xmlParserCtxt since we want to free it only after we're
@@ -857 +836 @@
-
-    if (SWISH_DEBUG > 9)
-
-
-
+
+
+
+
+
+
+
+
+
-
-    if (SWISH_DEBUG > 9)
@@ -905 +890 @@
-            swish_debug_msg("free swish_ParseData wordList");
-
-WordL
+wordl
-    }
-
@@ -1672 +1657 @@
-
-    parse_data->metastack = push_tag_stack( parse_data->metastack, 
- *) 
+*)
-
-    if (SWISH_DEBUG > 2)
@@ -1678 +1663 @@
-
-    chars_to_words(parse_data, buffer, size);
-NULL
+(xmlChar*)SWISH_DEFAULT_METANAME, (xmlChar*)SWISH_DEFAULT_METANAME
-    
-    if (out != NULL)
@@ -1745 +1730 @@
-    )
-{
-
-    if (parse_data->analyzer->tokenize == 0)
-        return;
-
-    if (len == 0)
@@ -1796 +1778 @@
-    if (tmplist->nwords == 0)
-    {
-WordL
+wordl
-        return;
-    }
@@ -1904 +1886 @@
-add_stack_to_prop_buf(xmlChar * tag, swish_ParseData * parse_data)
-{
-
+
+
+
+
+
+
+
+
+
-
-    if (tag != NULL)
-
+
+
+
+
+
-
-    for (s->temp = s->head; s->temp != NULL; s->temp = s->temp->next)
-    {
-
+
+
+
+
+
+
+
+
-    }
-

libswish3/trunk/src/libswish3/words.c

@@ -42 +42 @@
-static int      bytes_in_chr(wint_t c);
-static void     make_ascii_tables();
-
+
+
+
+
+
+
+
+
+
-
-static int initialized = 0;
@@ -60 +68 @@
-
-swish_WordList *
-WordL
+wordl
-{
-    swish_WordList *wl = (swish_WordList *) swish_xmalloc(sizeof(swish_WordList));
@@ -72 +80 @@
-
-void
-WordL
+wordl
-{
-    swish_Word    *t;
@@ -133 +141 @@
-{
-
-WordL
+wordl
-
-
@@ -270 +278 @@
-
-    int byte_count = 0;
-WordL
+wordl
-    xmlChar * utf8_str;
-
@@ -492 +500 @@
-    char            c, nextc, in_word;
-    int             i, w, wl, byte_count;
-WordL
+wordl
-    xmlChar        * word = swish_xmalloc(sizeof(xmlChar*) * analyzer->maxwordlen);
-
@@ -824 +832 @@
-*
-***********************************************/
+static int
+add_to_wordlist(
+        swish_WordList * list,
+        xmlChar * word,
+        int len,
+        xmlChar * metaname,
+        xmlChar * context,
+        int word_pos,
+        int offset
+)
+{
+    swish_Word  *thisword = (swish_Word *) swish_xmalloc(sizeof(swish_Word));
+    
+    if (SWISH_DEBUG == SWISH_DEBUG_TOKENIZER)
+    {
+        swish_debug_msg(" >>>>>>>>swish_Word<<<<<<<<:  %s", word);
+        swish_debug_msg("     --METANAME--:  %s", metaname);
+        swish_debug_msg("     --CONTEXT---:  %s", context);
+        swish_debug_msg("     --POSITION--:  %d", word_pos);
+        swish_debug_msg("     --OFFSET----:  %d", offset);
+        swish_debug_msg("     --WORD LEN--:  %d", len);
+    }
+
+    /* add to wordlist */
+
+    thisword->word     = word;
+    thisword->position = word_pos;
+    
+    if (metaname != NULL)
+        thisword->metaname = swish_xstrdup(metaname);
+    else
+        thisword->metaname = swish_xstrdup((xmlChar*)SWISH_DEFAULT_METANAME);
+        
+    if (context != NULL)
+        thisword->context  = swish_xstrdup(context);
+    else
+        thisword->context  = swish_xstrdup((xmlChar*)SWISH_DEFAULT_METANAME);
+        
+    thisword->end_offset   = offset;
+    thisword->start_offset = offset - len + 1;  /* +1 because want the first byte */
+
+    /* add thisword to list */
+    if (list->head == 0)
+    {
+        list->head = thisword;
+        thisword->prev = 0;
+    }
+    else
+    {
+        list->tail->next = thisword;
+        thisword->prev = list->tail;
+    }
+
+    list->tail = thisword;
+    thisword->next = 0;
+
+    /* increment total count */
+    list->nwords++;
+
+    return len;
+}
+
-size_t
-swish_add_to_wordlist(
@@ -835 +905 @@
-{
-
-
-
-
-
-
-
-
-
-
-
-