X-Git-Url: http://git.equinoxoli.org/?p=migration-tools.git;a=blobdiff_plain;f=fingerprinter;h=681ca4640fc92d094c050bbb875d66d021ad21ed;hp=7bad37e623fcc3782e3659bca2465652f334ecaa;hb=a69691879e272240ac27709c1945bfc2b119234e;hpb=c11696a5cb129c79a3d9abe2534b5e061c6d942e diff --git a/fingerprinter b/fingerprinter index 7bad37e..681ca46 100755 --- a/fingerprinter +++ b/fingerprinter @@ -5,15 +5,15 @@ use open ':utf8'; use Getopt::Long; use MARC::Batch; -use MARC::File::XML ( BinaryEncoding => 'utf-8' ); -use MARC::Field; use Unicode::Normalize; +use MARC::File::XML ( BinaryEncoding => 'utf-8' ); my $conf = {}; # configuration hashref -my $count = 0; +my $count = 0; my $scount = 0; +my $start = time; $| = 1; -initialyze($conf); +initialize($conf); open OF, '>', $conf->{output}; binmode(OF, ':utf8'); @@ -22,32 +22,37 @@ binmode(XF, ':utf8'); for my $file (@ARGV) { print XF "Processing $file\n"; - open my $records, '<:utf8', $file; + #open my $records, '<:utf8', $file; # This hack dosn't play nice with the use MARC::File::XML ( BinaryEncoding => 'utf-8' ); hack + my $batch = undef; my $record = undef; - my $batch = MARC::Batch->new('XML', $records); + #$batch = MARC::Batch->new('XML', $records); # The other part of the hack + $batch = MARC::Batch->new($conf->{marctype}, $file); $batch->strict_off(); $batch->warnings_off(); - while ( my $record = $batch->next ) { + while ( $record = $batch->next ) { $count++; progress_ticker(); + my $marc = undef; + unless ( defined $record ) + { dump_exception($marc); next; } my $id = $record->field($conf->{tag}); unless ($id) { - print XF "ERROR: Record $count in $file is missing a", - $conf->{tag}, "field.\n", $record->as_formatted(), "\n=====\n"; + print XF "ERROR: Record $count in $file is missing a ", + $conf->{tag}, " field.\n", $record->as_formatted(), "\n=====\n"; next; } - my $marc = populate_marc($record, $id); - $marc = normalize_marc($marc); + $marc = populate_marc($record, $id); + $marc = normalize_marc($marc); unless (marc_isvalid($marc)) { dump_exception($marc); next; } dump_fingerprints($marc); + $scount++; progress_ticker(); } } -print "\nProcessed $count records\n" unless $conf->{quiet}; - +print "\nSuccessfully processed:\t$count\n" unless $conf->{quiet}; =head2 populate_marc @@ -70,8 +75,22 @@ sub populate_marc { # date1, date2 my $my_008 = $record->field('008'); $my_008 = $my_008->as_string() if ($my_008); - $marc{date1} = substr($my_008,7,4) if ($my_008); - $marc{date2} = substr($my_008,11,4) if ($my_008); # UNUSED + if (defined $my_008) { + unless (length $my_008 == 40) { + $my_008 = $my_008 . (' ' x (40 - length($my_008))); + print XF ">> Short 008 padded to ",length($my_008)," at rec $count\n"; + } + $marc{date1} = substr($my_008,7,4) if ($my_008); + $marc{date2} = substr($my_008,11,4) if ($my_008); # UNUSED + } + unless ($marc{date1} and $marc{date1} =~ /\d{4}/) { + my $my_260 = $record->field('260'); + my $date1 = $my_260->subfield('c') if $my_260; + if (defined $date1 and $date1 =~ /\d{4}/) { + $marc{date1} = $date1; + print XF ">> using 260c as date1 at rec $count\n"; + } + } # item_form if ( $marc{record_type} =~ /[gkroef]/ ) { # MAP, VIS @@ -96,13 +115,28 @@ sub populate_marc { } } + # oclc + $marc{oclc} = []; + push @{ $marc{oclc} }, $record->field('001')->as_string() + if ($record->field('001') and $record->field('003') and + $record->field('003')->as_string() =~ /OCo{0,1}LC/); + for ($record->field('035')) { + my $oclc = $_->subfield('a'); + push @{ $marc{oclc} }, $oclc + if (defined $oclc and $oclc =~ /\(OCoLC\)/ and $oclc =~/([0-9]+)/); + } + + # "Accompanying material" (300e) + $marc{accomp} = $record->field('300')->subfield('e') + if $record->field('300'); + # issn, lccn, title, desc, pages, pub, pubyear, edition $marc{lccn} = $record->field('010')->subfield('a') if $record->field('010'); $marc{issn} = $record->field('022')->subfield('a') if $record->field('022'); $marc{desc} = $record->field('300')->subfield('a') if $record->field('300'); $marc{pages} = $1 if (defined $marc{desc} and $marc{desc} =~ /(\d+)/); $marc{title} = $record->field('245')->subfield('a') - if defined $record->field('245'); + if $record->field('245'); $marc{edition} = $record->field('250')->subfield('a') if $record->field('250'); if ($record->field('260')) { @@ -182,7 +216,7 @@ sub dump_fingerprints { $marc->{date1}, $marc->{record_type}, $marc->{bib_lvl}, $marc->{title}), "\n"; } else { - if ((scalar @{ $marc->{isbns} } > 0) && $marc->{pages}) { + if ((scalar @{ $marc->{isbns} } > 0) and $marc->{pages}) { # case a : isbn and pages foreach my $isbn ( @{ $marc->{isbns}} ) { print OF join("\t", $marc->{id}, "case a", @@ -193,6 +227,13 @@ sub dump_fingerprints { } } + +=pod + +Looks like case b isn't a terribly good idea most of the time -- it +will generate many spurious matches in series books in particular. So +for now, simply turning it off. + if ($marc->{edition}) { # case b : edition print OF join("\t", $marc->{id}, "case b", $marc->{item_form}, $marc->{date1}, @@ -200,6 +241,8 @@ sub dump_fingerprints { $marc->{title}, $marc->{edition}), "\n"; } +=cut + if ($marc->{issn}) { # case c : issn print OF join("\t", $marc->{id}, "case c", $marc->{item_form}, $marc->{date1}, @@ -214,10 +257,27 @@ sub dump_fingerprints { $marc->{title}, $marc->{lccn}) ,"\n"; } - # case e : author, publisher, pubyear, pages + if ($marc->{accomp}) { # case e : accomp + print OF join("\t", $marc->{id}, "case e", + $marc->{item_form}, $marc->{date1}, + $marc->{record_type}, $marc->{bib_lvl}, + $marc->{title}, $marc->{accomp}) ,"\n"; + } + + # case o: oclc + if (scalar @{$marc->{oclc} }) { + for (@{$marc->{oclc} }) { + print OF join("\t", $marc->{id}, "case o", + $marc->{item_form}, $marc->{date1}, + $marc->{record_type}, $marc->{bib_lvl}, + $marc->{title}, $_, "\n"); + } + } + + # case z : author, publisher, pubyear, pages if ($marc->{author} and $marc->{publisher} and $marc->{pubyear} and $marc->{pages}) { - print OF join("\t", $marc->{id}, "case e", + print OF join("\t", $marc->{id}, "case z", $marc->{item_form}, $marc->{date1}, $marc->{record_type}, $marc->{bib_lvl}, $marc->{title}, $marc->{author}, @@ -237,10 +297,17 @@ Write line of exception report sub dump_exception { my ($marc) = @_; + unless (defined $marc) { + print XF "Undefined record at line $count; likely bad XML\n"; + return; + } print XF "Record ", $marc->{id}, " did not make the cut: "; print XF "Missing item_form. " unless ($marc->{item_form}); - print XF "Missing valid date1. " - unless (defined $marc->{date1} and $marc->{date1} =~ /\d{4}/); + unless (defined $marc->{date1}) + { print XF "Missing date1. " } + else + { print XF "Invalid date1: ", $marc->{date1}, " " + unless ($marc->{date1} =~ /\d{4}/); } print XF "Missing record_type. " unless ($marc->{record_type}); print XF "Missing bib_lvl. " unless ($marc->{bib_lvl}); print XF "Missing title. " unless ($marc->{title}); @@ -248,14 +315,14 @@ sub dump_exception { } -=head2 initialyze +=head2 initialize Performs boring script initialization. Handles argument parsing, mostly. =cut -sub initialyze { +sub initialize { my ($c) = @_; my @missing = (); @@ -263,7 +330,10 @@ sub initialyze { binmode(STDIN, ':utf8'); my $rc = GetOptions( $c, + 'incoming', + 'incumbent', 'exception|x=s', + 'marctype|m=s', 'output|o=s', 'runtype|r=s', 'subfield|s=s', @@ -272,18 +342,33 @@ sub initialyze { 'help|h', ); show_help() unless $rc; + show_help() if ($c->{help}); + + # set defaults if told to do so + if ($c->{incoming}) { + $c->{tag} = 903 unless defined $c->{tag}; + $c->{subfield} = 'a' unless defined $c->{subfield}; + $c->{marctype} = 'XML' unless defined $c->{marctype}; + $c->{output} = 'incoming.fp' unless defined $c->{output}; + $c->{exception} = 'incoming.ex' unless defined $c->{exception}; + $c->{runtype} = 'full' unless defined $c->{runtype}; + } elsif ($c->{incumbent}) { + $c->{tag} = 901 unless defined $c->{tag}; + $c->{subfield} = 'c' unless defined $c->{subfield}; + $c->{marctype} = 'XML' unless defined $c->{marctype}; + $c->{output} = 'incumbent.fp' unless defined $c->{output}; + $c->{exception} = 'incumbent.ex' unless defined $c->{exception}; + $c->{runtype} = 'full' unless defined $c->{runtype}; + } my @keys = keys %{$c}; show_help() unless (@ARGV and @keys); - for my $key ('runtype', 'tag', 'subfield', 'output', 'exception') { - push @missing, $key unless $c->{$key} - } + for my $key ('runtype', 'tag', 'subfield', 'output', 'exception') + { push @missing, $key unless $c->{$key} } if (@missing) { print "Required option: ", join(', ', @missing), " missing!\n"; show_help(); } - - show_help() if ($c->{help}); } @@ -293,13 +378,9 @@ sub initialyze { sub progress_ticker { return if $conf->{quiet}; - - if ($count % 100 == 0) { - print '|'; - print " $count \n" unless ($count % 1400); - } elsif ($count % 20 == 0) { - print '.'; - } + printf("\r> %d recs seen; %d processed", $count, $scount); + printf(" (%d/s)", ($count / (time - $start + 1))) + if ($count % 500 == 0); } =head2 show_help @@ -310,7 +391,7 @@ Display usage message when things go wrong sub show_help { print < +Usage is: $0 [REQUIRED ARGS] [OPTIONS] Req'd Arguments --runtype=(primary|full) -r Do 'primary' or 'full' fingerprinting --tag=N -t Which tag to use @@ -318,7 +399,13 @@ Req'd Arguments --output= -o Output filename --exceptions= -x Exception report filename Options - --quiet -q Don't write status messages to STDOUT + --incoming Set -r to 'full'; -t, -s, -o, -x to incoming defaults + --incumbent Set -r to 'full'; -t, -s, -o, -x to incumbent defaults + + Example: '$0 --incoming' is equivalent to + '$0 -r full -t 903 -s a -o incoming.fp -x incoming.ex' + + --quiet -q Don't write status messages to STDOUT HELP exit 1; }