X-Git-Url: http://git.equinoxoli.org/?p=migration-tools.git;a=blobdiff_plain;f=match_fingerprints;h=dd93d9577bd547a32c7715c30a5da41c43ee71df;hp=4300429df383a55416d9cbd59479b92aab83c387;hb=ebb7ad9f114865db720271deacb5d2bec173aba7;hpb=3ec4fd758b018abda104bbeffa5ebf02b832030b

diff --git a/match_fingerprints b/match_fingerprints
index 4300429..dd93d95 100755
--- a/match_fingerprints
+++ b/match_fingerprints
@@ -14,6 +14,8 @@ my @recs = (); # fingerprints belonging to each record
 my %seen = (); # records we've already seen
 my $lastscore = 0; # previous fingerprint's score
 
+my %subs  = (); # error-checking hashe
+
 open FP, '<', $ARGV[0] or die "Can't open input file: $!\n";
 
 print "Loading and ranking fingerprints\n";
@@ -85,8 +87,19 @@ sub dump_records {
     open OUT, '>', $conf->{output}
       or die "Can't open ", $conf->{output}, "$!\n";
     for my $rec (@recs) {
-        print OUT $rec->{id}, "\t$_\n"
-          for ( @{ $fps{ $rec->{sha1} } } );
+        for ( @{ $fps{ $rec->{sha1} } } ) {
+            # check for dupes and die if they exist
+            die "Collision: dupe sub record $_\n" if $subs{$_};
+            $subs{$_} = 1;
+            die "Collision: lead in sub list ", $rec->{id}, "\n"
+              if $subs{ $rec->{id} };
+
+            # we don't want subs below threshold
+            next if ($_ < $conf->{threshold});
+
+            # still here? output.
+            print OUT $rec->{id}, "\t$_\n"
+        }
     }
 }
 
@@ -99,6 +112,7 @@ sub initialize {
 
     my $rc = GetOptions( $c,
                          'output|o=s',
+                         'threshold|t=i',
                          'help|h',
                        );
     show_help() unless $rc;
@@ -112,11 +126,13 @@ sub initialize {
         print "Required option: ", join(', ', @missing), " missing!\n";
         show_help();
     }
+
+    $c->{threshold} = 0 unless $c->{threshold};
 }
 
 sub show_help {
     print <<HELP;
-Usage is: compress_fingerprints -o OUTPUTFILE INPUTFILE
+Usage is: compress_fingerprints [-t THRESHOLD] -o OUTPUTFILE INPUTFILE
 HELP
 exit;
 }