X-Git-Url: http://git.equinoxoli.org/?p=migration-tools.git;a=blobdiff_plain;f=fingerprinter;h=681ca4640fc92d094c050bbb875d66d021ad21ed;hp=06af0ff1829bb1fe5ec4bec23fd209a671559f7a;hb=a69691879e272240ac27709c1945bfc2b119234e;hpb=547523838f0099913b8c0a42638eb59471a17fc8 diff --git a/fingerprinter b/fingerprinter index 06af0ff..681ca46 100755 --- a/fingerprinter +++ b/fingerprinter @@ -6,9 +6,10 @@ use open ':utf8'; use Getopt::Long; use MARC::Batch; use Unicode::Normalize; +use MARC::File::XML ( BinaryEncoding => 'utf-8' ); my $conf = {}; # configuration hashref -my $count = 0; +my $count = 0; my $scount = 0; my $start = time; $| = 1; @@ -21,16 +22,17 @@ binmode(XF, ':utf8'); for my $file (@ARGV) { print XF "Processing $file\n"; - open my $records, '<:utf8', $file; + #open my $records, '<:utf8', $file; # This hack dosn't play nice with the use MARC::File::XML ( BinaryEncoding => 'utf-8' ); hack my $batch = undef; my $record = undef; - $batch = MARC::Batch->new('XML', $records); + #$batch = MARC::Batch->new('XML', $records); # The other part of the hack + $batch = MARC::Batch->new($conf->{marctype}, $file); $batch->strict_off(); $batch->warnings_off(); - while ( eval { $record = $batch->next } ) { - my $marc = undef; + while ( $record = $batch->next ) { $count++; progress_ticker(); + my $marc = undef; unless ( defined $record ) { dump_exception($marc); next; } @@ -46,11 +48,11 @@ for my $file (@ARGV) { unless (marc_isvalid($marc)) { dump_exception($marc); next; } dump_fingerprints($marc); + $scount++; progress_ticker(); } } -print "\nProcessed $count records\n" unless $conf->{quiet}; - +print "\nSuccessfully processed:\t$count\n" unless $conf->{quiet}; =head2 populate_marc @@ -73,10 +75,22 @@ sub populate_marc { # date1, date2 my $my_008 = $record->field('008'); $my_008 = $my_008->as_string() if ($my_008); - unless (defined $my_008 and length $my_008 == 40) - { print XF ">> Bad 008 length in rec ",$marc{id},"\n"; return \%marc } - $marc{date1} = substr($my_008,7,4) if ($my_008); - $marc{date2} = substr($my_008,11,4) if ($my_008); # UNUSED + if (defined $my_008) { + unless (length $my_008 == 40) { + $my_008 = $my_008 . (' ' x (40 - length($my_008))); + print XF ">> Short 008 padded to ",length($my_008)," at rec $count\n"; + } + $marc{date1} = substr($my_008,7,4) if ($my_008); + $marc{date2} = substr($my_008,11,4) if ($my_008); # UNUSED + } + unless ($marc{date1} and $marc{date1} =~ /\d{4}/) { + my $my_260 = $record->field('260'); + my $date1 = $my_260->subfield('c') if $my_260; + if (defined $date1 and $date1 =~ /\d{4}/) { + $marc{date1} = $date1; + print XF ">> using 260c as date1 at rec $count\n"; + } + } # item_form if ( $marc{record_type} =~ /[gkroef]/ ) { # MAP, VIS @@ -101,6 +115,17 @@ sub populate_marc { } } + # oclc + $marc{oclc} = []; + push @{ $marc{oclc} }, $record->field('001')->as_string() + if ($record->field('001') and $record->field('003') and + $record->field('003')->as_string() =~ /OCo{0,1}LC/); + for ($record->field('035')) { + my $oclc = $_->subfield('a'); + push @{ $marc{oclc} }, $oclc + if (defined $oclc and $oclc =~ /\(OCoLC\)/ and $oclc =~/([0-9]+)/); + } + # "Accompanying material" (300e) $marc{accomp} = $record->field('300')->subfield('e') if $record->field('300'); @@ -191,7 +216,7 @@ sub dump_fingerprints { $marc->{date1}, $marc->{record_type}, $marc->{bib_lvl}, $marc->{title}), "\n"; } else { - if ((scalar @{ $marc->{isbns} } > 0) && $marc->{pages}) { + if ((scalar @{ $marc->{isbns} } > 0) and $marc->{pages}) { # case a : isbn and pages foreach my $isbn ( @{ $marc->{isbns}} ) { print OF join("\t", $marc->{id}, "case a", @@ -202,6 +227,13 @@ sub dump_fingerprints { } } + +=pod + +Looks like case b isn't a terribly good idea most of the time -- it +will generate many spurious matches in series books in particular. So +for now, simply turning it off. + if ($marc->{edition}) { # case b : edition print OF join("\t", $marc->{id}, "case b", $marc->{item_form}, $marc->{date1}, @@ -209,6 +241,8 @@ sub dump_fingerprints { $marc->{title}, $marc->{edition}), "\n"; } +=cut + if ($marc->{issn}) { # case c : issn print OF join("\t", $marc->{id}, "case c", $marc->{item_form}, $marc->{date1}, @@ -224,16 +258,26 @@ sub dump_fingerprints { } if ($marc->{accomp}) { # case e : accomp - print OF join("\t", $marc->{id}, "case d", + print OF join("\t", $marc->{id}, "case e", $marc->{item_form}, $marc->{date1}, $marc->{record_type}, $marc->{bib_lvl}, $marc->{title}, $marc->{accomp}) ,"\n"; } + # case o: oclc + if (scalar @{$marc->{oclc} }) { + for (@{$marc->{oclc} }) { + print OF join("\t", $marc->{id}, "case o", + $marc->{item_form}, $marc->{date1}, + $marc->{record_type}, $marc->{bib_lvl}, + $marc->{title}, $_, "\n"); + } + } + # case z : author, publisher, pubyear, pages if ($marc->{author} and $marc->{publisher} and $marc->{pubyear} and $marc->{pages}) { - print OF join("\t", $marc->{id}, "case e", + print OF join("\t", $marc->{id}, "case z", $marc->{item_form}, $marc->{date1}, $marc->{record_type}, $marc->{bib_lvl}, $marc->{title}, $marc->{author}, @@ -286,7 +330,10 @@ sub initialize { binmode(STDIN, ':utf8'); my $rc = GetOptions( $c, + 'incoming', + 'incumbent', 'exception|x=s', + 'marctype|m=s', 'output|o=s', 'runtype|r=s', 'subfield|s=s', @@ -295,6 +342,24 @@ sub initialize { 'help|h', ); show_help() unless $rc; + show_help() if ($c->{help}); + + # set defaults if told to do so + if ($c->{incoming}) { + $c->{tag} = 903 unless defined $c->{tag}; + $c->{subfield} = 'a' unless defined $c->{subfield}; + $c->{marctype} = 'XML' unless defined $c->{marctype}; + $c->{output} = 'incoming.fp' unless defined $c->{output}; + $c->{exception} = 'incoming.ex' unless defined $c->{exception}; + $c->{runtype} = 'full' unless defined $c->{runtype}; + } elsif ($c->{incumbent}) { + $c->{tag} = 901 unless defined $c->{tag}; + $c->{subfield} = 'c' unless defined $c->{subfield}; + $c->{marctype} = 'XML' unless defined $c->{marctype}; + $c->{output} = 'incumbent.fp' unless defined $c->{output}; + $c->{exception} = 'incumbent.ex' unless defined $c->{exception}; + $c->{runtype} = 'full' unless defined $c->{runtype}; + } my @keys = keys %{$c}; show_help() unless (@ARGV and @keys); @@ -304,8 +369,6 @@ sub initialize { print "Required option: ", join(', ', @missing), " missing!\n"; show_help(); } - - show_help() if ($c->{help}); } @@ -315,8 +378,9 @@ sub initialize { sub progress_ticker { return if $conf->{quiet}; - printf("> %d (%d/s)\r", $count, ($count / (time - $start + 1))) - if ($count % 100 == 0); + printf("\r> %d recs seen; %d processed", $count, $scount); + printf(" (%d/s)", ($count / (time - $start + 1))) + if ($count % 500 == 0); } =head2 show_help @@ -327,7 +391,7 @@ Display usage message when things go wrong sub show_help { print < +Usage is: $0 [REQUIRED ARGS] [OPTIONS] Req'd Arguments --runtype=(primary|full) -r Do 'primary' or 'full' fingerprinting --tag=N -t Which tag to use @@ -335,7 +399,13 @@ Req'd Arguments --output= -o Output filename --exceptions= -x Exception report filename Options - --quiet -q Don't write status messages to STDOUT + --incoming Set -r to 'full'; -t, -s, -o, -x to incoming defaults + --incumbent Set -r to 'full'; -t, -s, -o, -x to incumbent defaults + + Example: '$0 --incoming' is equivalent to + '$0 -r full -t 903 -s a -o incoming.fp -x incoming.ex' + + --quiet -q Don't write status messages to STDOUT HELP exit 1; }