X-Git-Url: http://git.equinoxoli.org/?p=migration-tools.git;a=blobdiff_plain;f=fingerprinter;h=7b20e3e43297d93d06594acdf3b4dbffa0b8cc2a;hp=7cadd539dcaea1ccf95a6af8532cc2626e669d97;hb=d478b47d3998baa905f245e8e05faf83bc314069;hpb=a0b6d8c7307149d5f236d5c01f94610ab764d336

diff --git a/fingerprinter b/fingerprinter
index 7cadd53..7b20e3e 100755
--- a/fingerprinter
+++ b/fingerprinter
@@ -5,33 +5,30 @@ use open ':utf8';
 
 use Getopt::Long;
 use MARC::Batch;
-use MARC::File::XML ( BinaryEncoding => 'utf-8' );
-use MARC::Field;
 use Unicode::Normalize;
+use MARC::File::XML ( BinaryEncoding => 'utf-8' );
 
 my $conf  = {}; # configuration hashref
-my $count = 0;
+my $count = 0; my $scount = 0;
+my $start = time;
 $| = 1;
 
 initialize($conf);
 
 open OF, '>', $conf->{output};
-binmode(OF, ':utf8');
 open XF, '>', $conf->{exception};
-binmode(XF, ':utf8');
 
 for my $file (@ARGV) {
     print XF "Processing $file\n";
-    open my $records, '<:utf8', $file;
     my $batch = undef; my $record = undef;
 
-    $batch = MARC::Batch->new('XML', $records);
+    $batch = MARC::Batch->new($conf->{marctype}, $file);
     $batch->strict_off();
     $batch->warnings_off();
 
-    while ( eval { $record = $batch->next } ) {
-        my $marc = undef;
+    while ( $record = $batch->next ) {
         $count++; progress_ticker();
+        my $marc = undef;
         unless ( defined $record )
           { dump_exception($marc); next; }
 
@@ -42,16 +39,20 @@ for my $file (@ARGV) {
             next;
         }
 
+        # populate and normalize marc
         $marc = populate_marc($record, $id);
-        $marc = normalize_marc($marc);
+        normalize_marc($marc);
         unless (marc_isvalid($marc))
           { dump_exception($marc); next; }
+
+        # if everything looks good, score it and dump fingerprints
+        score_marc($marc, $record);
         dump_fingerprints($marc);
+        $scount++; progress_ticker();
     }
 }
-print "\nProcessed $count records\n" unless $conf->{quiet};
-
 
+print "\nSuccessfully processed:\t$count\n" unless $conf->{quiet};
 
 =head2 populate_marc
 
@@ -74,10 +75,22 @@ sub populate_marc {
     # date1, date2
     my $my_008 = $record->field('008');
     $my_008 = $my_008->as_string() if ($my_008);
-    unless (length $my_008 == 40)
-      { print XF ">> Bad 008 field length in rec. $id\n"; return \%marc }
-    $marc{date1} = substr($my_008,7,4) if ($my_008);
-    $marc{date2} = substr($my_008,11,4) if ($my_008); # UNUSED
+    if (defined $my_008) {
+        unless (length $my_008 == 40) {
+            $my_008 = $my_008 . (' ' x (40 - length($my_008)));
+            print XF ">> Short 008 padded to ",length($my_008)," at rec $count\n";
+        }
+        $marc{date1} = substr($my_008,7,4) if ($my_008);
+        $marc{date2} = substr($my_008,11,4) if ($my_008); # UNUSED
+    }
+    unless ($marc{date1} and $marc{date1} =~ /\d{4}/) {
+        my $my_260 = $record->field('260');
+        my $date1 = $my_260->subfield('c') if $my_260;
+        if (defined $date1 and $date1 =~ /\d{4}/) {
+            $marc{date1} = $date1;
+            print XF ">> using 260c as date1 at rec $count\n";
+        }
+    }
 
     # item_form
     if ( $marc{record_type} =~ /[gkroef]/ ) { # MAP, VIS
@@ -102,7 +115,20 @@ sub populate_marc {
         }
     }
 
-    
+    # oclc
+    $marc{oclc} = [];
+    push @{ $marc{oclc} }, $record->field('001')->as_string()
+      if ($record->field('001') and $record->field('003') and
+          $record->field('003')->as_string() =~ /OCo{0,1}LC/);
+    for ($record->field('035')) {
+        my $oclc = $_->subfield('a');
+        push @{ $marc{oclc} }, $oclc
+          if (defined $oclc and $oclc =~ /\(OCoLC\)/ and $oclc =~/([0-9]+)/);
+    }
+
+    # "Accompanying material" (300e)
+    $marc{accomp} = $record->field('300')->subfield('e')
+      if $record->field('300');
 
     # issn, lccn, title, desc, pages, pub, pubyear, edition
     $marc{lccn} = $record->field('010')->subfield('a') if $record->field('010');
@@ -110,7 +136,7 @@ sub populate_marc {
     $marc{desc} = $record->field('300')->subfield('a') if $record->field('300');
     $marc{pages} = $1 if (defined $marc{desc} and $marc{desc} =~ /(\d+)/);
     $marc{title} = $record->field('245')->subfield('a')
-      if defined $record->field('245');
+      if $record->field('245');
     $marc{edition} = $record->field('250')->subfield('a')
       if $record->field('250');
     if ($record->field('260')) {
@@ -178,6 +204,75 @@ sub marc_isvalid {
 }
 
 
+=head2 score_marc
+
+Assign a score to the record based on various criteria.
+
+Score is constructed by pushing elements onto a list. At the end of
+the routine, the list is flattened into a string via join();
+
+=cut
+
+sub score_marc {
+    my ($marc, $record) = @_;
+    my @score = ();
+    my $chunk;
+
+    # Is this an OCLC record?
+    if ($conf->{scores}{oclc}) {
+        if ($record->field('008')) {
+            $chunk = $record->field('008')->as_string();
+            push @score, ( $chunk =~ /^o/i ? 1 : 0 );
+        } else {
+            push @score, 0;
+        }
+    }
+
+    # does 040a contain "dlc"?
+    if ($conf->{scores}{dlc}) {
+        if ($record->field('040') and $record->field('040')->subfield('a')) {
+            $chunk = $record->field('040')->subfield('a');
+            push @score, ( $chunk =~ /dlc/i ? 1 : 0 );
+        } else {
+            push @score, 0;
+        }
+    }
+
+    # number of 650 datafields
+    # zero-padded to 4 digits with printf
+    if ($conf->{scores}{num_650}) {
+        if ($record->field('650')) {
+            my @tags = $record->field('650');
+            push @score, scalar @tags;
+        } else {
+            push @score, 0;
+        }
+    }
+
+    # number of tags in total
+    # zero-padded to 4 digits with printf
+    if ($conf->{scores}{num_tags}) {
+        my @tags = $record->fields;
+        push @score, scalar @tags;
+    }
+
+    # encoding level
+    if ($conf->{scores}{enc_lvl}) {
+        my $enc = substr($record->leader, 17, 1);
+        my %levels = ( ' ' => 9, 1 => 8, 2 => 7,  3  => 6,  4  => 5, 5 => 4,
+                       6   => 3, 7 => 2, 8 => 1, 'u' => 0, 'z' => 0 );
+        if (defined $enc and $levels{$enc})
+          { push @score, $levels{$enc} }
+        else
+          { push @score, 0 }
+    }
+
+    # put score in marc hash
+    $marc->{score} = join('', '{oclc:', $score[0], ',dlc:', $score[1],
+                          ',num_650:', $score[2], ',num_tags:', $score[3],
+                          ',enc_lvl:', $score[4], '}');
+}
+
 =head2 dump_fingerprints
 
 =cut
@@ -185,53 +280,69 @@ sub marc_isvalid {
 sub dump_fingerprints {
     my ($marc) = @_;
 
-    if ($conf->{runtype} eq "primary") {
-        print OF join("\t",$marc->{id}, $marc->{item_form},
-                          $marc->{date1}, $marc->{record_type},
-                          $marc->{bib_lvl}, $marc->{title}), "\n";
-    } else {
-        if ((scalar @{ $marc->{isbns} } > 0) && $marc->{pages}) {
-            # case a : isbn and pages
-            foreach my $isbn ( @{ $marc->{isbns}} ) {
-                print OF join("\t", $marc->{id}, "case a",
-                                  $marc->{item_form}, $marc->{date1},
-                                  $marc->{record_type},
-                                  $marc->{bib_lvl}, $marc->{title},
-                                  $isbn, $marc->{pages}), "\n";
-            }
-        }
+    if ($conf->{fingerprints}{baseline}) {
+        print OF join("\t", $marc->{score}, $marc->{id}, 'baseline',
+                      $marc->{item_form}, $marc->{date1}, $marc->{record_type},
+                      $marc->{bib_lvl}, $marc->{title}), "\n";
+    }
 
-        if ($marc->{edition}) { # case b : edition
-            print OF join("\t", $marc->{id}, "case b",
-                              $marc->{item_form}, $marc->{date1},
-                              $marc->{record_type}, $marc->{bib_lvl},
-                              $marc->{title}, $marc->{edition}), "\n";
+    if ($conf->{fingerprints}{oclc} and scalar @{$marc->{oclc} }) {
+        for (@{$marc->{oclc} }) {
+            print OF join("\t", $marc->{score}, $marc->{id}, "oclc",
+                          $marc->{item_form}, $marc->{date1},
+                          $marc->{record_type}, $marc->{bib_lvl},
+                          $marc->{title}, $_, "\n");
         }
+    }
 
-        if ($marc->{issn}) { # case c : issn
-            print OF join("\t", $marc->{id}, "case c",
+    if ($conf->{fingerprints}{isbn}) {
+        if ((scalar @{ $marc->{isbns} } > 0) and $marc->{pages}) {
+            foreach my $isbn ( @{ $marc->{isbns}} ) {
+                print OF join("\t", $marc->{score}, $marc->{id}, "isbn",
                               $marc->{item_form}, $marc->{date1},
-                              $marc->{record_type}, $marc->{bib_lvl},
-                              $marc->{title}, $marc->{issn}), "\n";
+                              $marc->{record_type},
+                              $marc->{bib_lvl}, $marc->{title},
+                              $isbn, $marc->{pages}), "\n";
+            }
         }
+    }
 
-        if ($marc->{lccn}) { # case d : lccn
-            print OF join("\t", $marc->{id}, "case d",
-                              $marc->{item_form}, $marc->{date1},
-                              $marc->{record_type}, $marc->{bib_lvl},
-                              $marc->{title}, $marc->{lccn}) ,"\n";
-        }
+    if ($conf->{fingerprints}{edition} and $marc->{edition}) {
+        print OF join("\t", $marc->{score}, $marc->{id}, "edition",
+                      $marc->{item_form}, $marc->{date1},
+                      $marc->{record_type}, $marc->{bib_lvl},
+                      $marc->{title}, $marc->{edition}), "\n";
+    }
 
-        # case e : author, publisher, pubyear, pages
-        if ($marc->{author} and $marc->{publisher} and $marc->{pubyear}
-            and $marc->{pages}) {
-            print OF join("\t", $marc->{id}, "case e",
-                              $marc->{item_form}, $marc->{date1},
-                              $marc->{record_type}, $marc->{bib_lvl},
-                              $marc->{title}, $marc->{author},
-                              $marc->{publisher}, $marc->{pubyear},
-                              $marc->{pages}), "\n";
-        }
+    if ($conf->{fingerprints}{issn} and $marc->{issn}) {
+        print OF join("\t", $marc->{score}, $marc->{id}, "issn",
+                      $marc->{item_form}, $marc->{date1},
+                      $marc->{record_type}, $marc->{bib_lvl},
+                      $marc->{title}, $marc->{issn}), "\n";
+    }
+
+    if ($conf->{fingerprints}{lccn} and $marc->{lccn}) {
+        print OF join("\t", $marc->{score}, $marc->{id}, "lccn",
+                      $marc->{item_form}, $marc->{date1},
+                      $marc->{record_type}, $marc->{bib_lvl},
+                      $marc->{title}, $marc->{lccn}) ,"\n";
+    }
+
+    if ($conf->{fingerprints}{accomp} and $marc->{accomp}) {
+        print OF join("\t", $marc->{score}, $marc->{id}, "accomp",
+                      $marc->{item_form}, $marc->{date1},
+                      $marc->{record_type}, $marc->{bib_lvl},
+                      $marc->{title}, $marc->{accomp}) ,"\n";
+    }
+
+    if ($conf->{fingerprints}{authpub} and $marc->{author} and
+        $marc->{publisher} and $marc->{pubyear} and $marc->{pages}) {
+        print OF join("\t", $marc->{score}, $marc->{id}, "authpub",
+                      $marc->{item_form}, $marc->{date1},
+                      $marc->{record_type}, $marc->{bib_lvl},
+                      $marc->{title}, $marc->{author},
+                      $marc->{publisher}, $marc->{pubyear},
+                      $marc->{pages}), "\n";
     }
 }
 
@@ -251,9 +362,11 @@ sub dump_exception {
     }
     print XF "Record ", $marc->{id}, " did not make the cut: ";
     print XF "Missing item_form. " unless ($marc->{item_form});
-    print XF "Missing date1. " unless (defined $marc->{date1});
-    print XF "Invalid date1: ", $marc->{date1}
-      unless ($marc->{date1} =~ /\d{4}/);
+    unless (defined $marc->{date1})
+      { print XF "Missing date1. " }
+    else
+      { print XF "Invalid date1: ", $marc->{date1}, " "
+          unless ($marc->{date1} =~ /\d{4}/); }
     print XF "Missing record_type. " unless ($marc->{record_type});
     print XF "Missing bib_lvl. " unless ($marc->{bib_lvl});
     print XF "Missing title. " unless ($marc->{title});
@@ -276,27 +389,77 @@ sub initialize {
     binmode(STDIN, ':utf8');
 
     my $rc = GetOptions( $c,
+                         'incoming',
+                         'incumbent',
                          'exception|x=s',
+                         'marctype|m=s',
                          'output|o=s',
                          'runtype|r=s',
                          'subfield|s=s',
                          'tag|t=s',
+                         'fingerprints=s',
+                         'scores=s',
                          'quiet|q',
                          'help|h',
                        );
     show_help() unless $rc;
+    show_help() if ($c->{help});
+
+    # check fingerprints list for validity
+    if ($c->{fingerprints}) {
+        my %fps = ();
+        my %valid_fps = ( oclc => 1, isbn => 1, issn => 1, lccn => 1,
+                          edition => 1, accomp => 1, authpub => 1,
+                          baseline => 1, crap => 1,
+                        );
+        for (split /,/, $c->{fingerprints}) {
+            die "Invalid fingerprint '$_'\n" unless $valid_fps{$_};
+            $fps{$_} = 1;
+        }
+        $c->{fingerprints} = \%fps
+    } else {
+        $c->{fingerprints} = {oclc => 1, isbn => 1, edition => 1, issn => 1,
+                              lccn => 1, accomp => 1, authpub => 1};
+    }
+    # check scores list for validity
+    if ($c->{scores}) {
+        my %scores = ();
+        my %valid_scores = ( oclc => 1, dlc => 1, num_650 => 1,
+                             num_tags => 1, enc_lvl => 1,
+                           );
+        for (split /,/, $c->{scores}) {
+            die "Invalid score mode '$_'\n" unless $valid_scores{$_};
+            $scores{$_} = 1;
+        }
+        $c->{scores} = \%scores;
+    } else {
+        $c->{scores} = {oclc => 1, dlc => 1, num_650 => 1,
+                        num_tags => 1, enc_lvl => 1};
+    }
+
+    # set defaults if told to do so
+    if ($c->{incoming}) {
+        $c->{tag} = 903 unless defined $c->{tag};
+        $c->{subfield} = 'a' unless defined $c->{subfield};
+        $c->{marctype} = 'XML' unless defined $c->{marctype};
+        $c->{output} = 'incoming.fp' unless defined $c->{output};
+        $c->{exception} = 'incoming.ex' unless defined $c->{exception};
+    } elsif ($c->{incumbent}) {
+        $c->{tag} = 901 unless defined $c->{tag};
+        $c->{subfield} = 'c' unless defined $c->{subfield};
+        $c->{marctype} = 'XML' unless defined $c->{marctype};
+        $c->{output} = 'incumbent.fp' unless defined $c->{output};
+        $c->{exception} = 'incumbent.ex' unless defined $c->{exception};
+    }
 
     my @keys = keys %{$c};
     show_help() unless (@ARGV and @keys);
-    for my $key ('runtype', 'tag', 'subfield', 'output', 'exception') {
-        push @missing, $key unless $c->{$key}
-    }
+    for my $key ('tag', 'subfield', 'output', 'exception')
+      { push @missing, $key unless $c->{$key} }
     if (@missing) {
         print "Required option: ", join(', ', @missing), " missing!\n";
         show_help();
     }
-
-    show_help() if ($c->{help});
 }
 
 
@@ -306,13 +469,9 @@ sub initialize {
 
 sub progress_ticker {
     return if $conf->{quiet};
-
-    if ($count % 100 == 0) {
-        print '|';
-        print " $count \n" unless ($count % 1400);
-    } elsif ($count % 20 == 0) {
-        print '.';
-    }
+    printf("\r> %d recs seen; %d processed", $count, $scount);
+    printf(" (%d/s)", ($count / (time - $start + 1)))
+      if ($count % 500 == 0);
 }
 
 =head2 show_help
@@ -323,15 +482,22 @@ Display usage message when things go wrong
 
 sub show_help {
 print <<HELP;
-Usage is: fingerprinter [REQUIRED ARGS] [OPTIONS] <filelist>
+Usage is: $0 [REQUIRED ARGS] [OPTIONS] <filelist>
 Req'd Arguments
-  --runtype=(primary|full) -r  Do 'primary' or 'full' fingerprinting
   --tag=N                  -t  Which tag to use
   --subfield=X             -s  Which subfield to use
   --output=<file>          -o  Output filename
   --exceptions=<file>      -x  Exception report filename
 Options
-  --quiet  -q  Don't write status messages to STDOUT
+  --incoming   '-t 903 -s a -o incoming.fp -x incoming.ex'
+  --incumbent  '-t 901 -s c -o incumbent.fp -x incumbent.ex'
+
+  --fingerprints=LIST  Fingerprints to generate, comma separated
+                       Default: oclc,isbn,edition,issn,lccn,accomp,authpub
+                       Others:  baseline
+  --scores=LIST  Scores to calculate, comma separated
+                 Default: oclc,dlc,num_650,num_tags,enc_level
+  --quiet    -q  Don't write status messages to STDOUT
 HELP
 exit 1;
 }