Changeset 2155 for libswish3/trunk/src/libswish3

Timestamp:

09/18/08 23:51:41 (4 months ago)

Author:

karpet

Message:

avoid xmlStrncat because (a) it fails under linux and (b) the realloc is unnecessary

Files:

• libswish3/trunk/src/libswish3/libswish3.h (modified) (1 diff)
• libswish3/trunk/src/libswish3/parser.c (modified) (2 diffs)
• libswish3/trunk/src/libswish3/tokenizer.c (modified) (2 diffs)

libswish3/trunk/src/libswish3/libswish3.h

@@ -144 +144 @@
-#define SWISH_DEFAULT_ENCODING    "UTF-8"
-#define SWISH_LOCALE              "en_US.UTF-8"
-
+
-
-/* debugging levels */

libswish3/trunk/src/libswish3/parser.c

@@ -1944 +1944 @@
-
-    if (parser_data->docinfo->encoding != (xmlChar *)SWISH_DEFAULT_ENCODING) {
+        SWISH_WARN("%s docinfo->encoding %s != %s", 
+            parser_data->docinfo->uri, parser_data->docinfo->encoding, SWISH_DEFAULT_ENCODING);
+
-        if (!xmlStrncasecmp(parser_data->docinfo->encoding, (xmlChar *)"iso-8859-1", 10)) {
-            out = swish_xmalloc(size * 2);
@@ -1970 +1973 @@
-
-            if (!isolat1ToUTF8(out, &outlen, buffer, &size)) {
-
+
+
+
+
+
+
+
-            }
-

libswish3/trunk/src/libswish3/tokenizer.c

@@ -677 +677 @@
-            if (inside_token) {
-
+                /* edge case */
+                if ((chr_len + token_len) > s3->analyzer->maxwordlen) {
+                    if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER)
+                        SWISH_DEBUG_MSG("token_len = %d  forcing end of token: '%s'",
+                                        token_len, chr);
+                    continue;
+                }
+
-                if (SWISH_DEBUG & SWISH_DEBUG_TOKENIZER)
-                    SWISH_DEBUG_MSG("adding to token: '%s'", chr);
-
-
+
+
-                token_len += chr_len;
-
@@ -727 +736 @@
-                token_len = 0;
-                inside_token = 1;       /*  turn on flag */
-
+
+
+
+
+
+
-                token_len += chr_len;
-