Changeset 2107

Timestamp:

03/31/08 14:39:58 (2 months ago)

Author:

jmruiz

Message:

Getting rid of remove_worddata_longs routine in compress.c

Files:

• swish-e/branches/2.6/src/compress.c (modified) (1 diff)
• swish-e/branches/2.6/src/compress.h (modified) (1 diff)
• swish-e/branches/2.6/src/db_write.c (modified) (1 diff)
• swish-e/branches/2.6/src/dump.c (modified) (5 diffs)
• swish-e/branches/2.6/src/merge.c (modified) (3 diffs)
• swish-e/branches/2.6/src/search.c (modified) (2 diffs)

swish-e/branches/2.6/src/compress.c

@@ -936 +936 @@
-}
-
-
-/* 2002/09 jmruiz
-** This routine changes longs in worddata by shorter compressed
-** numbers.
-**
-** Here are two reasons for using compressed numbers in worddata
-** instead of longs:
-**   - Compressed numbers are more portable: longs are usually 4 bytes
-**     long in a 32 bit machine but in a 64 bit alpha they are 8 bytes
-**     long (this a waste of space).
-**   - The obvious one is that compressed numbers use less disk space
-**
-** BTW, Any change in worddata will also affect to dump.c, merge.c and search.c
-** (getfileinfo routine).
-**
-**  worddata has the following format before entering the routine
-**  <tfreq><metaID><nextposmetaID><data><metaID><nextposmetaID><data>...
-**
-**  Entering this routine nextposmetaID is the offset to next metaid
-**  in bytes starting to count them from the begining of worddata.
-**  It is a packed long number (sizeof(long) bytes).
-**
-**  Exiting this routine, nextposmetaID has changed to be the size of
-**  the data block and is stored as a compressed number.
-**
-**  In other words, worddata has the following format:
-**  <tfreq><metaID><data_len><data><metaID><data_len><data>...
-**
-*/
-void    remove_worddata_longs(unsigned char *worddata,int *sz_worddata)
-{
-    unsigned char *src,*dst;   //source and dest pointers for worddata
-    unsigned int metaID, tfrequency, data_len;
-    unsigned long nextposmetaID;
-
-    src = worddata;
-
-    /* Jump over tfrequency and get first metaID */
-    tfrequency = uncompress2(&src);     /* tfrequency */
-    metaID = uncompress2(&src);     /* metaID */
-    dst = src;
-
-    while(1)
-    {
-        /* Get offset to next one */
-        nextposmetaID = UNPACKLONG2(src);
-        src += sizeof(long);
-
-        /* Compute data length for this metaID */
-        data_len = (int)nextposmetaID - (src - worddata);
-
-        /* Store data_len as a compressed number */
-        dst = compress3(data_len,dst);
-
-        /* This must not happen. Anyway check it */
-        if(dst > src)
-            progerr("Internal error in remove_worddata_longs");
-
-        /* dst may be smaller than src. So move the data */
-        /* valgrind complains that dst and src overlap, */
-        /* which results in undefined behavior with memcpy, so use memmove. */
-        /* Q: Is it correct for dst and src to overlap here? */
-        memmove(dst,src,data_len);
-
-        /* Increase pointers */
-        src += data_len;
-        dst += data_len;
-
-        /* Check if we are at the end of the buffer */
-        if ((src - worddata) == *sz_worddata)
-            break;   /* End of worddata */
-
-        /* Get next metaID */
-        metaID = uncompress2(&src);
-        dst = compress3(metaID,dst);
-    }
-    /* Adjust to new size */
-    *sz_worddata = dst - worddata;
-}

swish-e/branches/2.6/src/compress.h

@@ -63 +63 @@
-int compress_worddata(unsigned char *, int, int );
-void uncompress_worddata(unsigned char **,int *, int);
-void    remove_worddata_longs(unsigned char *,int *);
-
-/* Here is the worst case size for a compressed number 

swish-e/branches/2.6/src/db_write.c

@@ -306 +306 @@
-/* 04/2002 jmruiz
-** New simpler routine to write worddata
-**
-** 10/2002 jmruiz
-** Add extra compression for worddata. Call to remove_worddata_longs
-*/
-void write_worddata(SWISH * sw, ENTRY * ep, IndexFILE * indexf )
-{
-    int zlib_size;
-
-    /* Get some extra compression */
-    remove_worddata_longs(sw->Index->worddata_buffer,&sw->Index->sz_worddata_buffer);
-
-    if(sw->compressPositions)

swish-e/branches/2.6/src/dump.c

@@ -190 +190 @@
-    unsigned int       *posdata;
-    int     metadata_length;
+    long    r_nextposmeta;
-    char    word[2];
-    char   *resultword;
@@ -248 +249 @@
-            tmpval = uncompress2(&s);     /* tfrequency */
-            metaID = uncompress2(&s);     /* metaID */
-
+
+
+
+
-
-            filenum = 0;
@@ -274 +278 @@
-                    filenum = 0;
-                    metaID = uncompress2(&s);
-
+
+
+
+
+
-                    start = s;
-                }
@@ -331 +339 @@
-            tmpval = uncompress2(&s);     /* tfrequency */
-            metaID = uncompress2(&s);     /* metaID */
-
+
+
+
+
+
-
-            filenum = 0;
@@ -428 +440 @@
-                    filenum = 0;
-                    metaID = uncompress2(&s);
-
+
+
+
+
+
-                    start = s;
-                }

swish-e/branches/2.6/src/merge.c

@@ -93 +93 @@
-    DB_WORDID   *wordID, *tmp;
-    int          metadata_length = 0;
+    long         r_nextposmeta;
-    unsigned char   *worddata;
-    unsigned char   *s, *start;
@@ -230 +231 @@
-                            if (metaID)
-                            {
-
+
+
+
-                            }
-
@@ -279 +282 @@
-                                    filenum = 0;
-                                    metaID = uncompress2(&s);
-
+
+
+
+
+
-                                    start = s;
-                                }

swish-e/branches/2.6/src/search.c

@@ -1509 +1509 @@
-    DB_WORDID    *wordID, *tmp;
-    int     metadata_length;
+    long    r_nextposmeta;
-    char   *p;
-    int     tfrequency = 0;
@@ -1728 +1729 @@
-           while (curmetaID)
-           {
-
+
+
+
+
-            
-               if (curmetaID >= metaID)