Changeset 1923

Timestamp:

03/19/07 11:58:02 (1 year ago)

Author:

karpet

Message:

expose the tokenizer into Perl space for benchmarking

Files:

• libswish3/trunk/CONTRIB (added)
• libswish3/trunk/README (modified) (1 diff)
• libswish3/trunk/bindings/perl/3.xs (modified) (6 diffs)
• libswish3/trunk/bindings/perl/t/02xml2_version.t (modified) (2 diffs)
• libswish3/trunk/bindings/perl/t/10tokenize.t (added)
• libswish3/trunk/bindings/perl/typemap (modified) (1 diff)
• libswish3/trunk/libswish3-config.in (modified) (1 diff)
• libswish3/trunk/libswish3.build (modified) (2 diffs)
• libswish3/trunk/src/libswish3/config.c (modified) (1 diff)
• libswish3/trunk/src/libswish3/libswish3.h (modified) (3 diffs)
• libswish3/trunk/src/libswish3/mem.c (modified) (1 diff)
• libswish3/trunk/src/libswish3/words.c (modified) (33 diffs)
• libswish3/trunk/src/swish_lint.c (modified) (1 diff)

libswish3/trunk/README

@@ -29 +29 @@
-# ReplaceRules
-# SpiderDirectory
+
+
+
+Getting Started 
+-----------------
+
+See the INSTALL doc.
+
+But basically:
+
+ ./bootstrap (only necessary if you are doing development)
+ ./configure
+ make && make test
+ sudo make install
+
+

libswish3/trunk/bindings/perl/3.xs

@@ -217 +217 @@
-MODULE = SWISH::3       PACKAGE = SWISH::3
-
-dis
+en
-
-SV*
@@ -240 +240 @@
-        RETVAL    
-      
-
+
+
+
+
+
+
+
+
+
+
+
-# *********************************************************************************
-MODULE = SWISH::3       PACKAGE = SWISH::3::Constants
@@ -391 +401 @@
-        RETVAL
-        
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-# *******************************************************************************
@@ -581 +644 @@
-
-
+void
+DESTROY(self)
+    swish_WordList * self
+    
+    CODE:
+        self->ref_cnt--;
+        if (!self->ref_cnt)
+        {
+            swish_free_WordList(self);
+        }
+        
+
-
-# *******************************************************************************
@@ -657 +732 @@
-    CODE:
-        CLASS = _which_class("WordList");
+        
+# MUST increment refcnt 2x so that SWISH::3::Parser::WordList::DESTROY
+# does not free it.
+        self->wordlist->ref_cnt += 2;
-        RETVAL = self->wordlist;
-        
@@ -774 +853 @@
-        {
-            swish_free_config(self);
-            swish_mem_debug();
-        }
-        

libswish3/trunk/bindings/perl/t/02xml2_version.t

@@ -1 @@
-3
+4
-
-use Devel::Peek;
@@ -11 +11 @@
-diag($s);
-
+ok( my $l = SWISH::3->libswish3_version, "libswish3 version");
+diag($l);
+
-#Dump $v;

libswish3/trunk/bindings/perl/typemap

@@ -1 +1 @@
-TYPEMAP
- 
- 
+
+
-swish_Config *             O_OBJECT
- 
+
-swish_ParseData *          O_OBJECT
- 
+
-swish_WordList *           O_OBJECT
-swish_DocInfo *            O_OBJECT

libswish3/trunk/libswish3-config.in

@@ -76 +76 @@
-
-    --libs)
-p
+3
-        ;;
-

libswish3/trunk/libswish3.build

@@ -7 +7 @@
-use File::Path;
-
-parser
-parser
+3
+3
-my $doc      = $htdocs . '/doc';
-my $dl       = $htdocs . '/download';
@@ -38 +38 @@
-print P "<ul>";
-for my $pod (
-parser.7 swishp_lint.1 swishp_words.1 swishp_isw.1 libswishp
+_lint.1 swish_words.1 swish_isw.1 libswish3
-{
-    shell(

libswish3/trunk/src/libswish3/config.c

@@ -160 +160 @@
-
-    swish_xfree(config);
+    
+    swish_mem_debug();
-}
-

libswish3/trunk/src/libswish3/libswish3.h

@@ -29 +29 @@
-
-
+#define SWISH_LIB_VERSION         "0.1.0"
-#define SWISH_VERSION             "3.0.0"
-#define SWISH_BUFFER_CHUNK_SIZE   10000
@@ -41 +42 @@
-
-#define SWISH_CONTRACTIONS         1
-#define SWISH_NO_LOWER_UTF         1
-/* chars > 127 will NOT be lowercased by default */
-
-#define SWISH_SPECIAL_ARG         1
@@ -279 +278 @@
-    swish_Word    *current;        // for iterating
-    unsigned int   nwords;
+    unsigned int   ref_cnt;         // for scripting languages
-};
-

libswish3/trunk/src/libswish3/mem.c

@@ -37 +37 @@
-{
-    memcount = 0;
+    
+    if (SWISH_DEBUG)
+        return;
+        
-    /* init the global env vars, but don't override if already set */
-    setenv("SWISH_DEBUG", "0", 0);

libswish3/trunk/src/libswish3/words.c

@@ -30 +30 @@
-#include "libswish3.h"
-
-static int      WORD
+extern int      SWISH
-static int      strip_ascii_chars(xmlChar * word, int len);
-static int      strip_wide_chars(wchar_t * word, int len);
@@ -44 +44 @@
-/**********************************************************************************************/
-
+/* we have our own set_debug here because we might be calling these tokenizing
+   functions without ever calling swish_init_config()
+ */
+ 
-static void set_debug()
-{
+    if (SWISH_DEBUG)
+        return;
+        
-    setenv("SWISH_DEBUG", "0", 0);
-    /* init the global env var, but don't override if already set */
-WORD
+SWISH
-}
-
@@ -58 +65 @@
-{
-    swish_WordList *wl = (swish_WordList *) swish_xmalloc(sizeof(swish_WordList));
-
-
+   
+   
-    wl->current = NULL;
-
-
-
-
+
+
-    return wl;
-}
@@ -73 +78 @@
-    swish_Word    *t;
-
-WORD
+SWISH
-        swish_debug_msg("freeing swish_WordList");
-
@@ -80 +85 @@
-    while (list->current != NULL)
-    {
-WORD
+SWISH
-            swish_debug_msg("free metaname: %s", list->current->metaname);
-            
-        swish_xfree(list->current->metaname);
-        
-WORD
+SWISH
-            swish_debug_msg("free context: %s", list->current->context);
-            
-        swish_xfree(list->current->context);
-        
-WORD
+SWISH
-            swish_debug_msg("free word: %s", list->current->word);
-            
-        swish_xfree(list->current->word);
-                
-WORD
+SWISH
-            swish_debug_msg("free Word struct");
-            
@@ -103 +108 @@
-    }
-    
-WORD
+SWISH
-        swish_debug_msg("reset nwords");
-        
-    list->nwords = 0;
-    
-WORD
+SWISH
-        swish_debug_msg("free list");
-        
@@ -220 +225 @@
-    }
-    
-WORD
+SWISH
-        swish_debug_msg(" %lc is %d bytes long", ch, len);
-        
@@ -239 +244 @@
-)
-{
+
-    int byte_count = 0;
-    swish_WordList *list = swish_init_WordList();
@@ -261 +267 @@
-    in_word = 0;
-
-WORD
+SWISH
-        swish_debug_msg("parsing string: '%ls' into words", wide);
-
@@ -270 +276 @@
-        byte_count += bytes_in_char((wint_t)c);
-
-WORD
+SWISH
-            swish_debug_msg(" wchar: %lc lower: %lc  int: %d %#x\n    orig: %lc %ld %#lx (next is %lc)",
-                   (wint_t) wide[i],
@@ -294 +300 @@
-            if (in_word)
-            {
-WORD
+SWISH
-                    swish_debug_msg("found end of word: >%lc<", (wint_t)c);
-
@@ -316 +322 @@
-                else
-                {
-WORD
+SWISH
-                        swish_debug_msg("skipping word >%s< -- too short: %d", utf8_str, wl);
-                }
@@ -327 +333 @@
-            else
-            {
-WORD
+SWISH
-                    swish_debug_msg("ignoring char >%lc<", (wint_t)c);
-                    
@@ -340 +346 @@
-            {
-            
-WORD
+SWISH
-                    swish_debug_msg("adding to word: >%lc<", (wint_t)c);
-
@@ -349 +355 @@
-                {
-
-WORD
+SWISH
-                        swish_debug_msg("forcing end of word: >%lc<", (wint_t)c);
-
@@ -371 +377 @@
-                    else
-                    {
-WORD
+SWISH
-                            swish_debug_msg("skipping word >%ls< -- too short: %d", word, wl);
-                    }
@@ -385 +391 @@
-            {
-
-WORD
+SWISH
-                    swish_debug_msg("start a word with >%lc<", (wint_t)c);
-
@@ -473 +479 @@
-    in_word = 0;
-
-WORD
+SWISH
-        swish_debug_msg("parsing string: '%s' into words", str);
-
@@ -488 +494 @@
-        byte_count++;
-
-WORD
+SWISH
-            swish_debug_msg(" char: %c lower: %c  int: %d %#x (next is %c)",
-                   str[i],
@@ -509 +515 @@
-            if (in_word)
-            {
-WORD
+SWISH
-                    swish_debug_msg("found end of word: >%c<", c);
-
@@ -530 +536 @@
-                else
-                {
-WORD
+SWISH
-                        swish_debug_msg("skipping word >%s< -- too short: %d", word, wl);
-                }
@@ -539 +545 @@
-            else
-            {
-WORD
+SWISH
-                    swish_debug_msg("ignoring char >%c<", c);
-                    
@@ -552 +558 @@
-            {
-            
-WORD
+SWISH
-                    swish_debug_msg("adding to word: >%c<", c);
-
@@ -561 +567 @@
-                {
-
-WORD
+SWISH
-                        swish_debug_msg("forcing end of word: >%c<", c);
-
@@ -582 +588 @@
-                    else
-                    {
-WORD
+SWISH
-                            swish_debug_msg("skipping word >%s< -- too short: %d", word, wl);
-                    }
@@ -594 +600 @@
-            {
-
-WORD
+SWISH
-                    swish_debug_msg("start a word with >%c<", c);
-
@@ -631 +637 @@
-)
-{
-
+
+
+
-    if (swish_is_ascii( str ))
-    {
@@ -670 +678 @@
-    end = 0;
-
-WORD
+SWISH
-        swish_debug_msg("Before: %ls", word);
-
@@ -721 +729 @@
-    }
-
-WORD
+SWISH
-        swish_debug_msg("After: %ls (stripped %d start chars, %d end chars)", word, start, end);
-
@@ -735 +743 @@
-    end = 0;
-
-WORD
+SWISH
-        swish_debug_msg("Before: %s", word);
-
@@ -786 +794 @@
-    }
-
-WORD
+SWISH
-        swish_debug_msg("After: %s (stripped %d start chars, %d end chars)", word, start, end);
-
@@ -811 +819 @@
-    size_t          len = xmlStrlen(word);
-
-WORD
+SWISH
-    {
-        swish_debug_msg(" >>>>>>>>swish_Word<<<<<<<<:  %s", word);
@@ -823 +831 @@
-    thisword->word     = swish_xstrdup(word);
-    thisword->position = word_pos;
-
-
+
+
+
+
+
+
+
+
+
+
+
-    thisword->end_offset   = offset - 1;
-    thisword->start_offset = offset - len;

libswish3/trunk/src/swish_lint.c

@@ -59 +59 @@
-void swish_version()
-{
-p
+3
-}
-