Changeset 2141 for libswish3/trunk

Timestamp:

04/30/08 00:03:02 (2 months ago)

Author:

karpet

Message:

port the ascii optimizations in words.c to tokenizer.c and expose some previously private functions to the public API

Files:

• libswish3/trunk/src/libswish3/config.c (modified) (2 diffs)
• libswish3/trunk/src/libswish3/libswish3.h (modified) (4 diffs)
• libswish3/trunk/src/libswish3/string.c (modified) (11 diffs)
• libswish3/trunk/src/libswish3/words.c (modified) (3 diffs)

libswish3/trunk/src/libswish3/config.c

@@ -19 +19 @@
-*/
-
-/* parse XML-style config files
- *
- * based on http://www.yolinux.com/TUTORIALS/GnomeLibXml2.html
- *
-*/
-
-#include <sys/param.h>
-#include <stdio.h>
@@ -196 +190 @@
-    flags->meta_ids = swish_init_hash(8);
-    flags->prop_ids = swish_init_hash(8);
-
+//
-
-    return flags;

libswish3/trunk/src/libswish3/libswish3.h

@@ -236 +236 @@
-    xmlHashTablePtr meta_ids;
-    xmlHashTablePtr prop_ids;
-
+//
-};
-
@@ -477 +477 @@
-*/
-void                swish_verify_utf8_locale();
-
+
+
-int                 swish_utf8_chr_len( xmlChar *utf8 );
-int                 swish_utf8_codepoint( xmlChar *utf8 );
@@ -487 +488 @@
-wchar_t *           swish_wstr_tolower(wchar_t *s);
-xmlChar *           swish_str_tolower(xmlChar *s );
+xmlChar *           swish_utf8_str_tolower(xmlChar *s);
+xmlChar *           swish_ascii_str_tolower(xmlChar *s);
-xmlChar *           swish_str_skip_ws(xmlChar *s);
-void                swish_str_trim_ws(xmlChar *string);
@@ -609 +612 @@
-swish_Token *       swish_next_token( swish_TokenIterator *it );
-int                 swish_tokenize3(    swish_3 *s3, 
+                                        swish_TokenList * tl, 
+                                        xmlChar *buf, 
+                                        swish_MetaName *meta,
+                                        xmlChar *context );
+int                 swish_tokenize3_ascii(    
+                                        swish_3 *s3, 
+                                        swish_TokenList * tl, 
+                                        xmlChar *buf, 
+                                        swish_MetaName *meta,
+                                        xmlChar *context );
+int                 swish_tokenize3_utf8(    
+                                        swish_3 *s3, 
-                                        swish_TokenList * tl, 
-                                        xmlChar *buf, 

libswish3/trunk/src/libswish3/string.c

@@ -40 +40 @@
-static xmlChar *getword(
-    xmlChar **in_buf
-);
-static xmlChar *utf8_str_tolower(
-    xmlChar *s
-);
-static xmlChar *ascii_str_tolower(
-    xmlChar *s
-);
-static xmlChar *findlast(
@@ -302 +296 @@
-*/
-
-
+
-swish_is_ascii(
-    xmlChar *str
@@ -351 +345 @@
-
-/* a bit about encodings: libxml2 takes whatever encoding the input XML is
-
-
-
+
+
+
+
-
-/* use LC_CTYPE specifically: http://mail.nl.linux.org/linux-utf8/2001-09/msg00030.html */
@@ -462 +457 @@
-
-    if (swish_is_ascii(s))
-
+swish_
-    else
-
+swish_
-
-}
@@ -473 +468 @@
-   and free the wchar
-*/
-static 
-
+
+swish_
-    xmlChar *s
-)
@@ -496 +491 @@
-
-/* based on swstring.c in Swish-e */
-static 
-
+
+swish_
-    xmlChar *s
-)
@@ -516 +511 @@
-  -- 2001-01-30  rasc
-
-should be utf8 safe, unless a continuation byte evals true to isspace()
+TODO make utf8 safe. 
-*/
-
@@ -534 +529 @@
-**************************************/
-
+// TODO make utf8 safe
-void
-swish_str_trim_ws(
@@ -572 +568 @@
-}
-
+/* returns the number of UTF-8 char* needed to hold the codepoint
+   represented by 'ch'.
+   similar to swish_utf8_chr_len() except that the arg is already
+   a 4-byte container and we want to know how many of the 4 bytes
+   we really need.
+*/
+int
+swish_bytes_in_wchar(
+    int ch
+)
+{
+    int len = 0;
+
+    if (ch < 0x80) {
+        len = 1;
+    }
+    if (ch < 0x800) {
+        len = 2;
+    }
+    if (ch < 0x10000) {
+        len = 3;
+    }
+    if (ch < 0x110000) {
+        len = 4;
+    }
+
+    if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER)
+        SWISH_DEBUG_MSG(" %lc is %d bytes long", ch, len);
+
+    return len;
+}
+
+
-/* from http://www.triptico.com/software/unicode.html */
-wchar_t *
@@ -587 +616 @@
-    len = mblen((const char *)str, 4);
-
-never happen in ISO-8859-*
-    * locales, but possible in UTF-8 */
-   if (s == -1) {
-    SWISH_WARN("error converting mbs to wide str: %s", str);
-return (0
-
+
+* never happen in ISO-8859-* locales, but possible in UTF-8 
+*/
+if (s == -1)
+SWISH_CROAK("error converting mbs to wide str: %s", str
+
-
-/* malloc the necessary space */
@@ -621 +650 @@
-/* a size of -1 means there are characters that could not be converted to current
-     * locale */
-
-
-
-
+
+
-
-/* malloc the necessary space */

libswish3/trunk/src/libswish3/words.c

@@ -49 +49 @@
-    int c
-);
-static int bytes_in_chr(
-    int ch
-);
-static void make_ascii_tables(
-);
@@ -248 +245 @@
-}
-
-/* returns the number of UTF-8 char* needed to hold the codepoint
-   represented by 'ch'.
-   similar to swish_utf8_chr_len() except that the arg is already
-   a 4-byte container and we want to know how many of the 4 bytes
-   we really need.
-*/
-static int
-bytes_in_chr(
-    int ch
-)
-{
-    int len = 0;
-
-    if (ch < 0x80) {
-        len = 1;
-    }
-    if (ch < 0x800) {
-        len = 2;
-    }
-    if (ch < 0x10000) {
-        len = 3;
-    }
-    if (ch < 0x110000) {
-        len = 4;
-    }
-
-    if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER)
-        SWISH_DEBUG_MSG(" %lc is %d bytes long", ch, len);
-
-    return len;
-}
-
-swish_WordList *
@@ -319 +285 @@
-
-    for (i = 0; wide[i] != '\0'; i++) {
-in
-in
-bytes_in_chr((wint_t) 
+wchar_
+wchar_
+swish_bytes_in_wchar(
-
-        if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER)