X-Git-Url: http://git.equinoxoli.org/?p=migration-tools.git;a=blobdiff_plain;f=fingerprinter;h=681ca4640fc92d094c050bbb875d66d021ad21ed;hp=231a8b2957d71569b3acb795be09ead1828924b2;hb=a69691879e272240ac27709c1945bfc2b119234e;hpb=64acc6180f3164f4a4b6132bf4334e1148f0cf44

diff --git a/fingerprinter b/fingerprinter
index 231a8b2..681ca46 100755
--- a/fingerprinter
+++ b/fingerprinter
@@ -9,7 +9,7 @@ use Unicode::Normalize;
 use MARC::File::XML ( BinaryEncoding => 'utf-8' );
 
 my $conf  = {}; # configuration hashref
-my $count = 0;
+my $count = 0; my $scount = 0;
 my $start = time;
 $| = 1;
 
@@ -26,7 +26,7 @@ for my $file (@ARGV) {
     my $batch = undef; my $record = undef;
 
     #$batch = MARC::Batch->new('XML', $records); # The other part of the hack
-    $batch = MARC::Batch->new('XML', $file);
+    $batch = MARC::Batch->new($conf->{marctype}, $file);
     $batch->strict_off();
     $batch->warnings_off();
 
@@ -48,6 +48,7 @@ for my $file (@ARGV) {
         unless (marc_isvalid($marc))
           { dump_exception($marc); next; }
         dump_fingerprints($marc);
+        $scount++; progress_ticker();
     }
 }
 
@@ -74,10 +75,22 @@ sub populate_marc {
     # date1, date2
     my $my_008 = $record->field('008');
     $my_008 = $my_008->as_string() if ($my_008);
-    unless (defined $my_008 and length $my_008 == 40)
-      { print XF ">> Bad 008 length in rec ",$marc{id},"\n"; return \%marc }
-    $marc{date1} = substr($my_008,7,4) if ($my_008);
-    $marc{date2} = substr($my_008,11,4) if ($my_008); # UNUSED
+    if (defined $my_008) {
+        unless (length $my_008 == 40) {
+            $my_008 = $my_008 . (' ' x (40 - length($my_008)));
+            print XF ">> Short 008 padded to ",length($my_008)," at rec $count\n";
+        }
+        $marc{date1} = substr($my_008,7,4) if ($my_008);
+        $marc{date2} = substr($my_008,11,4) if ($my_008); # UNUSED
+    }
+    unless ($marc{date1} and $marc{date1} =~ /\d{4}/) {
+        my $my_260 = $record->field('260');
+        my $date1 = $my_260->subfield('c') if $my_260;
+        if (defined $date1 and $date1 =~ /\d{4}/) {
+            $marc{date1} = $date1;
+            print XF ">> using 260c as date1 at rec $count\n";
+        }
+    }
 
     # item_form
     if ( $marc{record_type} =~ /[gkroef]/ ) { # MAP, VIS
@@ -102,6 +115,17 @@ sub populate_marc {
         }
     }
 
+    # oclc
+    $marc{oclc} = [];
+    push @{ $marc{oclc} }, $record->field('001')->as_string()
+      if ($record->field('001') and $record->field('003') and
+          $record->field('003')->as_string() =~ /OCo{0,1}LC/);
+    for ($record->field('035')) {
+        my $oclc = $_->subfield('a');
+        push @{ $marc{oclc} }, $oclc
+          if (defined $oclc and $oclc =~ /\(OCoLC\)/ and $oclc =~/([0-9]+)/);
+    }
+
     # "Accompanying material" (300e)
     $marc{accomp} = $record->field('300')->subfield('e')
       if $record->field('300');
@@ -192,7 +216,7 @@ sub dump_fingerprints {
                           $marc->{date1}, $marc->{record_type},
                           $marc->{bib_lvl}, $marc->{title}), "\n";
     } else {
-        if ((scalar @{ $marc->{isbns} } > 0) && $marc->{pages}) {
+        if ((scalar @{ $marc->{isbns} } > 0) and $marc->{pages}) {
             # case a : isbn and pages
             foreach my $isbn ( @{ $marc->{isbns}} ) {
                 print OF join("\t", $marc->{id}, "case a",
@@ -203,6 +227,13 @@ sub dump_fingerprints {
             }
         }
 
+
+=pod
+
+Looks like case b isn't a terribly good idea most of the time -- it
+will generate many spurious matches in series books in particular. So
+for now, simply turning it off.
+
         if ($marc->{edition}) { # case b : edition
             print OF join("\t", $marc->{id}, "case b",
                               $marc->{item_form}, $marc->{date1},
@@ -210,6 +241,8 @@ sub dump_fingerprints {
                               $marc->{title}, $marc->{edition}), "\n";
         }
 
+=cut
+
         if ($marc->{issn}) { # case c : issn
             print OF join("\t", $marc->{id}, "case c",
                               $marc->{item_form}, $marc->{date1},
@@ -225,16 +258,26 @@ sub dump_fingerprints {
         }
 
         if ($marc->{accomp}) { # case e : accomp
-            print OF join("\t", $marc->{id}, "case d",
+            print OF join("\t", $marc->{id}, "case e",
                               $marc->{item_form}, $marc->{date1},
                               $marc->{record_type}, $marc->{bib_lvl},
                               $marc->{title}, $marc->{accomp}) ,"\n";
         }
 
+        # case o: oclc
+        if (scalar @{$marc->{oclc} }) {
+            for (@{$marc->{oclc} }) {
+                print OF join("\t", $marc->{id}, "case o",
+                              $marc->{item_form}, $marc->{date1},
+                              $marc->{record_type}, $marc->{bib_lvl},
+                              $marc->{title}, $_, "\n");
+            }
+        }
+
         # case z : author, publisher, pubyear, pages
         if ($marc->{author} and $marc->{publisher} and $marc->{pubyear}
             and $marc->{pages}) {
-            print OF join("\t", $marc->{id}, "case e",
+            print OF join("\t", $marc->{id}, "case z",
                               $marc->{item_form}, $marc->{date1},
                               $marc->{record_type}, $marc->{bib_lvl},
                               $marc->{title}, $marc->{author},
@@ -290,6 +333,7 @@ sub initialize {
                          'incoming',
                          'incumbent',
                          'exception|x=s',
+                         'marctype|m=s',
                          'output|o=s',
                          'runtype|r=s',
                          'subfield|s=s',
@@ -304,13 +348,17 @@ sub initialize {
     if ($c->{incoming}) {
         $c->{tag} = 903 unless defined $c->{tag};
         $c->{subfield} = 'a' unless defined $c->{subfield};
+        $c->{marctype} = 'XML' unless defined $c->{marctype};
         $c->{output} = 'incoming.fp' unless defined $c->{output};
         $c->{exception} = 'incoming.ex' unless defined $c->{exception};
+        $c->{runtype} = 'full' unless defined $c->{runtype};
     } elsif ($c->{incumbent}) {
         $c->{tag} = 901 unless defined $c->{tag};
         $c->{subfield} = 'c' unless defined $c->{subfield};
+        $c->{marctype} = 'XML' unless defined $c->{marctype};
         $c->{output} = 'incumbent.fp' unless defined $c->{output};
         $c->{exception} = 'incumbent.ex' unless defined $c->{exception};
+        $c->{runtype} = 'full' unless defined $c->{runtype};
     }
 
     my @keys = keys %{$c};
@@ -330,8 +378,9 @@ sub initialize {
 
 sub progress_ticker {
     return if $conf->{quiet};
-    printf("> %d (%d/s)\r", $count, ($count / (time - $start + 1)))
-      if ($count % 100 == 0);
+    printf("\r> %d recs seen; %d processed", $count, $scount);
+    printf(" (%d/s)", ($count / (time - $start + 1)))
+      if ($count % 500 == 0);
 }
 
 =head2 show_help