Changeset 1487

Timestamp:

07/25/04 23:29:20 (4 years ago)

Author:

karman

Message:

Added IDF word weighting in getrank() to weight words more heavily that appear less often.

Files:

• trunk/swish-e/pod/CHANGES.pod (modified) (1 diff)
• trunk/swish-e/src/rank.c (modified) (7 diffs)

trunk/swish-e/pod/CHANGES.pod

@@ -10 +10 @@
-
-=over 4
+
+=item Added IDF word weighting in getrank()
+
+Added Inverse Document Frequency calculation to the getrank() routine.
+This will allow the relative frequency of a word in relationship to other
+words in the query to impact the ranking of documents.
+
+Example: if 'foo' is present twice as often as 'bar' in the collection as a whole,
+a search for 'foo bar' will weight documents with 'bar' more heavily (i.e., higher
+rank) than those with 'foo'. 
+
+The impact is greatest when OR'ing words in a query rather than
+AND'ing them (which is the default).
+
-
-=item Updates to the example scripts

trunk/swish-e/src/rank.c

@@ -289 +289 @@
-    int         metaID;
-    int         freq;
+    int         total_files;
+    int         idf;
+    int         tfreq;
+    int         mystruct;
+    
-#ifdef DEBUG_RANK
-    int        struct_tally[256];
@@ -294 +299 @@
-        struct_tally[i] = 0;
-#endif
+
+
+    
-
-    /* has rank already been calculated? */
@@ -303 +311 @@
-    sw      = indexf->sw;
-    posdata = r->posdata;
-
+
-
-    /* Get bias for the current metaID - metaID is stored in the rank for ease here */
@@ -328 +336 @@
-    /*  this word.  If the word is not found in many files then it should be ranked higher */
-
+
-    rank = 1;
-    freq = r->frequency;
@@ -336 +345 @@
-    {
-        /* GET_STRUCTURE must return value in range! */
-
+
+
+
-#ifdef DEBUG_RANK
-        // fprintf(stderr, "Word entry %d at position %d has struct %d\n", i,  GET_POSITION(posdata[i]),  GET_STRUCTURE(posdata[i]) );
@@ -355 +366 @@
-    if ( rank < 1 )
-        rank = 1;
+
+
+    /* weight rank by word's idf */
+
+    /* 
+    IDF is the Inverse Document Frequency, or, the weight of the word in relationship to the 
+    collection of documents as a whole.
+    Multiply the weight against the rank to give greater weight to words that appear less often
+    in the collection.
+    
+    The biggest impact should be seen when OR'ing words together instead of AND'ing them.
+    
+    karman - Sun Jul 25 22:18:15 CDT 2004
+    */
+    
+    total_files = indexf->header.totalfiles;
+    tfreq       = r->tfrequency;
+    idf         = (int) log( total_files / tfreq );
+
+#ifdef DEBUG_RANK
+        fprintf(stderr, "Total files: %d   Total word freq: %d   IDF: %d\n", total_files, tfreq, idf );
+        fprintf(stderr, "Rank before IDF weighting: %d\n", rank );
+#endif
+        rank = rank * idf;
+        
+#ifdef DEBUG_RANK
+        fprintf(stderr, "Rank after IDF weighting: %d\n", rank );
+#endif
-
-
@@ -385 +424 @@
-
-
-
-    /* Return if IgnoreTotalWordCountWhenRanking is true (the default) */
-    if ( indexf->header.ignoreTotalWordCountWhenRanking )