X-Git-Url: http://git.equinoxoli.org/?p=migration-tools.git;a=blobdiff_plain;f=fingerprinter;h=06e1718861d9257986154d4ae0282737d6755b35;hp=246252d1bd4c14a10eae3402489a854ecad57889;hb=51e88210a486e30304b5226820f191e19f1e8501;hpb=ba2286c9f842fde94f970bfef8b2c49e8ab41f33

diff --git a/fingerprinter b/fingerprinter
index 246252d..06e1718 100755
--- a/fingerprinter
+++ b/fingerprinter
@@ -16,16 +16,12 @@ $| = 1;
 initialize($conf);
 
 open OF, '>', $conf->{output};
-binmode(OF, ':utf8');
 open XF, '>', $conf->{exception};
-binmode(XF, ':utf8');
 
 for my $file (@ARGV) {
     print XF "Processing $file\n";
-    #open my $records, '<:utf8', $file; # This hack dosn't play nice with the use MARC::File::XML ( BinaryEncoding => 'utf-8' ); hack
     my $batch = undef; my $record = undef;
 
-    #$batch = MARC::Batch->new('XML', $records); # The other part of the hack
     $batch = MARC::Batch->new($conf->{marctype}, $file);
     $batch->strict_off();
     $batch->warnings_off();
@@ -43,10 +39,14 @@ for my $file (@ARGV) {
             next;
         }
 
+        # populate and normalize marc
         $marc = populate_marc($record, $id);
-        $marc = normalize_marc($marc);
+        normalize_marc($marc);
         unless (marc_isvalid($marc))
           { dump_exception($marc); next; }
+
+        # if everything looks good, score it and dump fingerprints
+        score_marc($marc, $record);
         dump_fingerprints($marc);
         $scount++; progress_ticker();
     }
@@ -75,10 +75,22 @@ sub populate_marc {
     # date1, date2
     my $my_008 = $record->field('008');
     $my_008 = $my_008->as_string() if ($my_008);
-    unless (defined $my_008 and length $my_008 == 40)
-      { print XF ">> Bad 008 length in rec ",$marc{id},"\n"; return \%marc }
-    $marc{date1} = substr($my_008,7,4) if ($my_008);
-    $marc{date2} = substr($my_008,11,4) if ($my_008); # UNUSED
+    if (defined $my_008) {
+        unless (length $my_008 == 40) {
+            $my_008 = $my_008 . (' ' x (40 - length($my_008)));
+            print XF ">> Short 008 padded to ",length($my_008)," at rec $count\n";
+        }
+        $marc{date1} = substr($my_008,7,4) if ($my_008);
+        $marc{date2} = substr($my_008,11,4) if ($my_008); # UNUSED
+    }
+    unless ($marc{date1} and $marc{date1} =~ /\d{4}/) {
+        my $my_260 = $record->field('260');
+        my $date1 = $my_260->subfield('c') if $my_260;
+        if (defined $date1 and $date1 =~ /\d{4}/) {
+            $marc{date1} = $date1;
+            print XF ">> using 260c as date1 at rec $count\n";
+        }
+    }
 
     # item_form
     if ( $marc{record_type} =~ /[gkroef]/ ) { # MAP, VIS
@@ -103,6 +115,17 @@ sub populate_marc {
         }
     }
 
+    # oclc
+    $marc{oclc} = [];
+    push @{ $marc{oclc} }, $record->field('001')->as_string()
+      if ($record->field('001') and $record->field('003') and
+          $record->field('003')->as_string() =~ /OCo{0,1}LC/);
+    for ($record->field('035')) {
+        my $oclc = $_->subfield('a');
+        push @{ $marc{oclc} }, $oclc
+          if (defined $oclc and $oclc =~ /\(OCoLC\)/ and $oclc =~/([0-9]+)/);
+    }
+
     # "Accompanying material" (300e)
     $marc{accomp} = $record->field('300')->subfield('e')
       if $record->field('300');
@@ -181,6 +204,60 @@ sub marc_isvalid {
 }
 
 
+=head2 score_marc
+
+Assign a score to the record based on various criteria.
+
+Score is constructed by pushing elements onto a list. At the end of
+the routine, the list is flattened into a string via join();
+
+=cut
+
+sub score_marc {
+    my ($marc, $record) = @_;
+    my @score = [];
+    my $chunk;
+
+    # Is this an OCLC record?
+    if $record->field('008') {
+        $chunk = $record->field('008')->as_string();
+        push @score, ($chunk =~ /^o/i);
+    } else {
+        push @score, 0;
+    }
+
+    # does 040a contain "dlc"?
+    if $record->field('040') {
+        $chunk = $record->field('040')->subfield('a');
+        push @score, ($chunk =~ /dlc/i);
+    } else {
+        push @score, 0;
+    }
+
+    # number of 650 datafields
+    # zero-padded to 4 digits with printf
+    if $record->field('650') {
+        my @tags = $record->field('650');
+        push @score, printf("04%d", $#tags);
+    } else {
+        push @score, '0000';
+    }
+
+    # number of tags in total
+    # zero-padded to 4 digits with printf
+    my @tags = $record->fields;
+    push @score, printf("04%d", $#tags);
+
+    # encoding level
+    my $enc = substr($record->leader, 17, 1);
+    my %levels = ( ' ' => 9, 1 => 8, 2 => 7,  3  => 6,  4  => 5, 5 => 4,
+                   6   => 3, 7 => 2, 8 => 1, 'u' => 0, 'z' => 0 );
+    push @score, $levels{$enc};
+
+    # put score in marc hash
+    $marc->{score} = join('', @score);
+}
+
 =head2 dump_fingerprints
 
 =cut
@@ -188,60 +265,83 @@ sub marc_isvalid {
 sub dump_fingerprints {
     my ($marc) = @_;
 
-    if ($conf->{runtype} eq "primary") {
+    if ($conf->{fingerprints}{baseline}) {
         print OF join("\t",$marc->{id}, $marc->{item_form},
                           $marc->{date1}, $marc->{record_type},
                           $marc->{bib_lvl}, $marc->{title}), "\n";
-    } else {
-        if ((scalar @{ $marc->{isbns} } > 0) && $marc->{pages}) {
-            # case a : isbn and pages
+    }
+
+    if ($conf->{fingerprints}{oclc} and scalar @{$marc->{oclc} }) {
+        for (@{$marc->{oclc} }) {
+            print OF join("\t", $marc->{id}, "case o",
+                          $marc->{item_form}, $marc->{date1},
+                          $marc->{record_type}, $marc->{bib_lvl},
+                          $marc->{title}, $_, "\n");
+        }
+        $good_fp = 1;
+    }
+
+    if ($conf->{fingerprints}{isbn}) {
+        if ((scalar @{ $marc->{isbns} } > 0) and $marc->{pages}) {
             foreach my $isbn ( @{ $marc->{isbns}} ) {
                 print OF join("\t", $marc->{id}, "case a",
-                                  $marc->{item_form}, $marc->{date1},
-                                  $marc->{record_type},
-                                  $marc->{bib_lvl}, $marc->{title},
-                                  $isbn, $marc->{pages}), "\n";
+                              $marc->{item_form}, $marc->{date1},
+                              $marc->{record_type},
+                              $marc->{bib_lvl}, $marc->{title},
+                              $isbn, $marc->{pages}), "\n";
             }
+            $good_fp = 1;
         }
+    }
 
-        if ($marc->{edition}) { # case b : edition
-            print OF join("\t", $marc->{id}, "case b",
-                              $marc->{item_form}, $marc->{date1},
-                              $marc->{record_type}, $marc->{bib_lvl},
-                              $marc->{title}, $marc->{edition}), "\n";
-        }
+    if ($conf->{fingerprints}{edition} and $marc->{edition}) {
+        print OF join("\t", $marc->{id}, "case b",
+                      $marc->{item_form}, $marc->{date1},
+                      $marc->{record_type}, $marc->{bib_lvl},
+                      $marc->{title}, $marc->{edition}), "\n";
+        $good_fp = 1;
+    }
 
-        if ($marc->{issn}) { # case c : issn
-            print OF join("\t", $marc->{id}, "case c",
-                              $marc->{item_form}, $marc->{date1},
-                              $marc->{record_type}, $marc->{bib_lvl},
-                              $marc->{title}, $marc->{issn}), "\n";
-        }
+    if ($conf->{fingerprints}{issn} and $marc->{issn}) {
+        print OF join("\t", $marc->{id}, "case c",
+                      $marc->{item_form}, $marc->{date1},
+                      $marc->{record_type}, $marc->{bib_lvl},
+                      $marc->{title}, $marc->{issn}), "\n";
+        $good_fp = 1;
+    }
 
-        if ($marc->{lccn}) { # case d : lccn
-            print OF join("\t", $marc->{id}, "case d",
-                              $marc->{item_form}, $marc->{date1},
-                              $marc->{record_type}, $marc->{bib_lvl},
-                              $marc->{title}, $marc->{lccn}) ,"\n";
-        }
+    if ($conf->{fingerprints}{lccn} and $marc->{lccn}) {
+        print OF join("\t", $marc->{id}, "case d",
+                      $marc->{item_form}, $marc->{date1},
+                      $marc->{record_type}, $marc->{bib_lvl},
+                      $marc->{title}, $marc->{lccn}) ,"\n";
+        $good_fp = 1;
+    }
 
-        if ($marc->{accomp}) { # case e : accomp
-            print OF join("\t", $marc->{id}, "case d",
-                              $marc->{item_form}, $marc->{date1},
-                              $marc->{record_type}, $marc->{bib_lvl},
-                              $marc->{title}, $marc->{accomp}) ,"\n";
-        }
+    if ($conf->{fingerprints}{accomp} and $marc->{accomp}) {
+        print OF join("\t", $marc->{id}, "case e",
+                      $marc->{item_form}, $marc->{date1},
+                      $marc->{record_type}, $marc->{bib_lvl},
+                      $marc->{title}, $marc->{accomp}) ,"\n";
+        $good_fp = 1;
+    }
 
-        # case z : author, publisher, pubyear, pages
-        if ($marc->{author} and $marc->{publisher} and $marc->{pubyear}
-            and $marc->{pages}) {
-            print OF join("\t", $marc->{id}, "case e",
-                              $marc->{item_form}, $marc->{date1},
-                              $marc->{record_type}, $marc->{bib_lvl},
-                              $marc->{title}, $marc->{author},
-                              $marc->{publisher}, $marc->{pubyear},
-                              $marc->{pages}), "\n";
-        }
+    if ($conf->{fingerprints}{authpub} and $marc->{author} and
+        $marc->{publisher} and $marc->{pubyear} and $marc->{pages}) {
+        print OF join("\t", $marc->{id}, "case z",
+                      $marc->{item_form}, $marc->{date1},
+                      $marc->{record_type}, $marc->{bib_lvl},
+                      $marc->{title}, $marc->{author},
+                      $marc->{publisher}, $marc->{pubyear},
+                      $marc->{pages}), "\n";
+        $good_fp = 1;
+    }
+
+        # case poo : nothing good; dump a primary and move on
+    if ($conf->{fingerprints}{crap}) {
+        print OF join("\t", $marc->{id}, "case poo", $marc->{item_form},
+                      $marc->{date1}, $marc->{record_type},
+                      $marc->{bib_lvl}, $marc->{title}), "\n";
     }
 }
 
@@ -296,12 +396,32 @@ sub initialize {
                          'runtype|r=s',
                          'subfield|s=s',
                          'tag|t=s',
+                         'fingerprints=s',
+                         'scores=s',
                          'quiet|q',
                          'help|h',
                        );
     show_help() unless $rc;
     show_help() if ($c->{help});
 
+    # check fingerprints list for validity
+    if ($c->{fingerprints}) {
+        my %valid_fps = ( oclc => 1, isbn => 1, issn => 1, lccn => 1,
+                          edition => 1, accomp => 1, authpub => 1,
+                          baseline => 1, crap => 1,
+                        );
+        for (split /,/, $c->{fingerprints})
+          { die "Invalid fingerprint '$_'\n" unless $valid_fps{$_} }
+    }
+    # check scores list for validity
+    if ($c->{scores}) {
+        my %valid_scores = ( oclc => 1, dlc => 1, num_650 => 1,
+                             num_tags => 1, enc_lvl => 1,
+                           );
+        for (split /,/, $c->{scores})
+          { die "Invalid score mode '$_'\n" unless $valid_scores{$_} }
+    }
+
     # set defaults if told to do so
     if ($c->{incoming}) {
         $c->{tag} = 903 unless defined $c->{tag};
@@ -359,10 +479,14 @@ Req'd Arguments
 Options
   --incoming     Set -r to 'full'; -t, -s, -o, -x to incoming defaults
   --incumbent    Set -r to 'full'; -t, -s, -o, -x to incumbent defaults
-
-  Example: '$0 --incoming' is equivalent to
-           '$0 -r full -t 903 -s a -o incoming.fp -x incoming.ex'
-
+     Example: '$0 --incoming' is equivalent to
+              '$0 -r full -t 903 -s a -o incoming.fp -x incoming.ex'
+
+  --fingerprints=LIST  Fingerprints to generate, comma separated
+                       Default: oclc,isbn,edition,issn,lccn,accomp,authpub
+                       Others:  baseline,crap
+  --scores=LIST  Scores to calculate, comma separated
+                 Default: oclc,dlc,num_650,num_tags,enc_level
   --quiet    -q  Don't write status messages to STDOUT
 HELP
 exit 1;