X-Git-Url: http://git.equinoxoli.org/?p=migration-tools.git;a=blobdiff_plain;f=fingerprinter;h=681ca4640fc92d094c050bbb875d66d021ad21ed;hp=2f27fbaf1ee8e70ddbcc8d075130c388a3ab797c;hb=a69691879e272240ac27709c1945bfc2b119234e;hpb=a0fa1a0cf079d8bf9e1f72784d2cd00cbe8e0074 diff --git a/fingerprinter b/fingerprinter index 2f27fba..681ca46 100755 --- a/fingerprinter +++ b/fingerprinter @@ -5,12 +5,11 @@ use open ':utf8'; use Getopt::Long; use MARC::Batch; -use MARC::File::XML ( BinaryEncoding => 'utf-8' ); -use MARC::Field; use Unicode::Normalize; +use MARC::File::XML ( BinaryEncoding => 'utf-8' ); my $conf = {}; # configuration hashref -my $count = 0; +my $count = 0; my $scount = 0; my $start = time; $| = 1; @@ -23,16 +22,17 @@ binmode(XF, ':utf8'); for my $file (@ARGV) { print XF "Processing $file\n"; - open my $records, '<:utf8', $file; + #open my $records, '<:utf8', $file; # This hack dosn't play nice with the use MARC::File::XML ( BinaryEncoding => 'utf-8' ); hack my $batch = undef; my $record = undef; - $batch = MARC::Batch->new('XML', $records); + #$batch = MARC::Batch->new('XML', $records); # The other part of the hack + $batch = MARC::Batch->new($conf->{marctype}, $file); $batch->strict_off(); $batch->warnings_off(); - while ( eval { $record = $batch->next } ) { - my $marc = undef; + while ( $record = $batch->next ) { $count++; progress_ticker(); + my $marc = undef; unless ( defined $record ) { dump_exception($marc); next; } @@ -48,11 +48,11 @@ for my $file (@ARGV) { unless (marc_isvalid($marc)) { dump_exception($marc); next; } dump_fingerprints($marc); + $scount++; progress_ticker(); } } -print "\nProcessed $count records\n" unless $conf->{quiet}; - +print "\nSuccessfully processed:\t$count\n" unless $conf->{quiet}; =head2 populate_marc @@ -75,10 +75,22 @@ sub populate_marc { # date1, date2 my $my_008 = $record->field('008'); $my_008 = $my_008->as_string() if ($my_008); - unless (length $my_008 == 40) - { print XF ">> Bad 008 field length in rec. $id\n"; return \%marc } - $marc{date1} = substr($my_008,7,4) if ($my_008); - $marc{date2} = substr($my_008,11,4) if ($my_008); # UNUSED + if (defined $my_008) { + unless (length $my_008 == 40) { + $my_008 = $my_008 . (' ' x (40 - length($my_008))); + print XF ">> Short 008 padded to ",length($my_008)," at rec $count\n"; + } + $marc{date1} = substr($my_008,7,4) if ($my_008); + $marc{date2} = substr($my_008,11,4) if ($my_008); # UNUSED + } + unless ($marc{date1} and $marc{date1} =~ /\d{4}/) { + my $my_260 = $record->field('260'); + my $date1 = $my_260->subfield('c') if $my_260; + if (defined $date1 and $date1 =~ /\d{4}/) { + $marc{date1} = $date1; + print XF ">> using 260c as date1 at rec $count\n"; + } + } # item_form if ( $marc{record_type} =~ /[gkroef]/ ) { # MAP, VIS @@ -103,9 +115,20 @@ sub populate_marc { } } + # oclc + $marc{oclc} = []; + push @{ $marc{oclc} }, $record->field('001')->as_string() + if ($record->field('001') and $record->field('003') and + $record->field('003')->as_string() =~ /OCo{0,1}LC/); + for ($record->field('035')) { + my $oclc = $_->subfield('a'); + push @{ $marc{oclc} }, $oclc + if (defined $oclc and $oclc =~ /\(OCoLC\)/ and $oclc =~/([0-9]+)/); + } + # "Accompanying material" (300e) $marc{accomp} = $record->field('300')->subfield('e') - if (defined $record->field('300')->subfield('e')); + if $record->field('300'); # issn, lccn, title, desc, pages, pub, pubyear, edition $marc{lccn} = $record->field('010')->subfield('a') if $record->field('010'); @@ -113,7 +136,7 @@ sub populate_marc { $marc{desc} = $record->field('300')->subfield('a') if $record->field('300'); $marc{pages} = $1 if (defined $marc{desc} and $marc{desc} =~ /(\d+)/); $marc{title} = $record->field('245')->subfield('a') - if defined $record->field('245'); + if $record->field('245'); $marc{edition} = $record->field('250')->subfield('a') if $record->field('250'); if ($record->field('260')) { @@ -193,7 +216,7 @@ sub dump_fingerprints { $marc->{date1}, $marc->{record_type}, $marc->{bib_lvl}, $marc->{title}), "\n"; } else { - if ((scalar @{ $marc->{isbns} } > 0) && $marc->{pages}) { + if ((scalar @{ $marc->{isbns} } > 0) and $marc->{pages}) { # case a : isbn and pages foreach my $isbn ( @{ $marc->{isbns}} ) { print OF join("\t", $marc->{id}, "case a", @@ -204,6 +227,13 @@ sub dump_fingerprints { } } + +=pod + +Looks like case b isn't a terribly good idea most of the time -- it +will generate many spurious matches in series books in particular. So +for now, simply turning it off. + if ($marc->{edition}) { # case b : edition print OF join("\t", $marc->{id}, "case b", $marc->{item_form}, $marc->{date1}, @@ -211,6 +241,8 @@ sub dump_fingerprints { $marc->{title}, $marc->{edition}), "\n"; } +=cut + if ($marc->{issn}) { # case c : issn print OF join("\t", $marc->{id}, "case c", $marc->{item_form}, $marc->{date1}, @@ -226,16 +258,26 @@ sub dump_fingerprints { } if ($marc->{accomp}) { # case e : accomp - print OF join("\t", $marc->{id}, "case d", + print OF join("\t", $marc->{id}, "case e", $marc->{item_form}, $marc->{date1}, $marc->{record_type}, $marc->{bib_lvl}, $marc->{title}, $marc->{accomp}) ,"\n"; } + # case o: oclc + if (scalar @{$marc->{oclc} }) { + for (@{$marc->{oclc} }) { + print OF join("\t", $marc->{id}, "case o", + $marc->{item_form}, $marc->{date1}, + $marc->{record_type}, $marc->{bib_lvl}, + $marc->{title}, $_, "\n"); + } + } + # case z : author, publisher, pubyear, pages if ($marc->{author} and $marc->{publisher} and $marc->{pubyear} and $marc->{pages}) { - print OF join("\t", $marc->{id}, "case e", + print OF join("\t", $marc->{id}, "case z", $marc->{item_form}, $marc->{date1}, $marc->{record_type}, $marc->{bib_lvl}, $marc->{title}, $marc->{author}, @@ -261,9 +303,11 @@ sub dump_exception { } print XF "Record ", $marc->{id}, " did not make the cut: "; print XF "Missing item_form. " unless ($marc->{item_form}); - print XF "Missing date1. " unless (defined $marc->{date1}); - print XF "Invalid date1: ", $marc->{date1} - unless ($marc->{date1} =~ /\d{4}/); + unless (defined $marc->{date1}) + { print XF "Missing date1. " } + else + { print XF "Invalid date1: ", $marc->{date1}, " " + unless ($marc->{date1} =~ /\d{4}/); } print XF "Missing record_type. " unless ($marc->{record_type}); print XF "Missing bib_lvl. " unless ($marc->{bib_lvl}); print XF "Missing title. " unless ($marc->{title}); @@ -286,7 +330,10 @@ sub initialize { binmode(STDIN, ':utf8'); my $rc = GetOptions( $c, + 'incoming', + 'incumbent', 'exception|x=s', + 'marctype|m=s', 'output|o=s', 'runtype|r=s', 'subfield|s=s', @@ -295,18 +342,33 @@ sub initialize { 'help|h', ); show_help() unless $rc; + show_help() if ($c->{help}); + + # set defaults if told to do so + if ($c->{incoming}) { + $c->{tag} = 903 unless defined $c->{tag}; + $c->{subfield} = 'a' unless defined $c->{subfield}; + $c->{marctype} = 'XML' unless defined $c->{marctype}; + $c->{output} = 'incoming.fp' unless defined $c->{output}; + $c->{exception} = 'incoming.ex' unless defined $c->{exception}; + $c->{runtype} = 'full' unless defined $c->{runtype}; + } elsif ($c->{incumbent}) { + $c->{tag} = 901 unless defined $c->{tag}; + $c->{subfield} = 'c' unless defined $c->{subfield}; + $c->{marctype} = 'XML' unless defined $c->{marctype}; + $c->{output} = 'incumbent.fp' unless defined $c->{output}; + $c->{exception} = 'incumbent.ex' unless defined $c->{exception}; + $c->{runtype} = 'full' unless defined $c->{runtype}; + } my @keys = keys %{$c}; show_help() unless (@ARGV and @keys); - for my $key ('runtype', 'tag', 'subfield', 'output', 'exception') { - push @missing, $key unless $c->{$key} - } + for my $key ('runtype', 'tag', 'subfield', 'output', 'exception') + { push @missing, $key unless $c->{$key} } if (@missing) { print "Required option: ", join(', ', @missing), " missing!\n"; show_help(); } - - show_help() if ($c->{help}); } @@ -316,8 +378,9 @@ sub initialize { sub progress_ticker { return if $conf->{quiet}; - printf("> %d (%d/s)\r", $count, ((time - $start) / $count)) - if ($count % 100 == 0); + printf("\r> %d recs seen; %d processed", $count, $scount); + printf(" (%d/s)", ($count / (time - $start + 1))) + if ($count % 500 == 0); } =head2 show_help @@ -328,7 +391,7 @@ Display usage message when things go wrong sub show_help { print < +Usage is: $0 [REQUIRED ARGS] [OPTIONS] Req'd Arguments --runtype=(primary|full) -r Do 'primary' or 'full' fingerprinting --tag=N -t Which tag to use @@ -336,7 +399,13 @@ Req'd Arguments --output= -o Output filename --exceptions= -x Exception report filename Options - --quiet -q Don't write status messages to STDOUT + --incoming Set -r to 'full'; -t, -s, -o, -x to incoming defaults + --incumbent Set -r to 'full'; -t, -s, -o, -x to incumbent defaults + + Example: '$0 --incoming' is equivalent to + '$0 -r full -t 903 -s a -o incoming.fp -x incoming.ex' + + --quiet -q Don't write status messages to STDOUT HELP exit 1; }