X-Git-Url: http://git.equinoxoli.org/?p=migration-tools.git;a=blobdiff_plain;f=fingerprinter;h=71376b9d0940872b65a8e81e6e14d816c25df67c;hp=919744cf9831225f6cff65b89eab2e2cd38caf44;hb=679aecdc2528f2d980470a35905ee3ac92a502e2;hpb=f813038b3c19ab6e5cf457f6cbac0afad20dd0ab diff --git a/fingerprinter b/fingerprinter index 919744c..71376b9 100755 --- a/fingerprinter +++ b/fingerprinter @@ -5,15 +5,15 @@ use open ':utf8'; use Getopt::Long; use MARC::Batch; -use MARC::File::XML ( BinaryEncoding => 'utf-8' ); -use MARC::Field; use Unicode::Normalize; +use MARC::File::XML ( BinaryEncoding => 'utf-8' ); my $conf = {}; # configuration hashref -my $count = 0; +my $count = 0; my $scount = 0; +my $start = time; $| = 1; -initialyze($conf); +initialize($conf); open OF, '>', $conf->{output}; binmode(OF, ':utf8'); @@ -22,35 +22,37 @@ binmode(XF, ':utf8'); for my $file (@ARGV) { print XF "Processing $file\n"; - open my $records, '<:utf8', $file; + #open my $records, '<:utf8', $file; # This hack dosn't play nice with the use MARC::File::XML ( BinaryEncoding => 'utf-8' ); hack + my $batch = undef; my $record = undef; - my $batch = MARC::Batch->new('XML', $records); + #$batch = MARC::Batch->new('XML', $records); # The other part of the hack + $batch = MARC::Batch->new('XML', $file); $batch->strict_off(); $batch->warnings_off(); - while ( my $record = $batch->next ) { + while ( $record = $batch->next ) { $count++; progress_ticker(); + my $marc = undef; + unless ( defined $record ) + { dump_exception($marc); next; } my $id = $record->field($conf->{tag}); unless ($id) { - print XF "ERROR: Record $count in $file is missing a", - $conf->{tag}, "field.\n", $record->as_formatted(), "\n=====\n"; + print XF "ERROR: Record $count in $file is missing a ", + $conf->{tag}, " field.\n", $record->as_formatted(), "\n=====\n"; next; } - my $marc = populate_marc($record, $id); - $marc = normalize_marc($marc); - unless ($marc->{item_form} and ($marc->{date1} =~ /\d{4}/) and - $marc->{record_type} and $marc->{bib_lvl} and $marc->{title}) { - dump_exception($marc); - next; - } + $marc = populate_marc($record, $id); + $marc = normalize_marc($marc); + unless (marc_isvalid($marc)) + { dump_exception($marc); next; } dump_fingerprints($marc); + $scount++; progress_ticker(); } } -print "\nProcessed $count records\n" unless $conf->{quiet}; - +print "\nSuccessfully processed:\t$count\n" unless $conf->{quiet}; =head2 populate_marc @@ -73,6 +75,8 @@ sub populate_marc { # date1, date2 my $my_008 = $record->field('008'); $my_008 = $my_008->as_string() if ($my_008); + unless (defined $my_008 and length $my_008 == 40) + { print XF ">> Bad 008 length in rec ",$marc{id},"\n"; return \%marc } $marc{date1} = substr($my_008,7,4) if ($my_008); $marc{date2} = substr($my_008,11,4) if ($my_008); # UNUSED @@ -99,13 +103,17 @@ sub populate_marc { } } + # "Accompanying material" (300e) + $marc{accomp} = $record->field('300')->subfield('e') + if $record->field('300'); + # issn, lccn, title, desc, pages, pub, pubyear, edition $marc{lccn} = $record->field('010')->subfield('a') if $record->field('010'); $marc{issn} = $record->field('022')->subfield('a') if $record->field('022'); $marc{desc} = $record->field('300')->subfield('a') if $record->field('300'); $marc{pages} = $1 if (defined $marc{desc} and $marc{desc} =~ /(\d+)/); $marc{title} = $record->field('245')->subfield('a') - if defined $record->field('245'); + if $record->field('245'); $marc{edition} = $record->field('250')->subfield('a') if $record->field('250'); if ($record->field('260')) { @@ -158,6 +166,21 @@ sub normalize_marc { +=head2 marc_isvalid + +Checks MARC record to see if neccessary fingerprinting data is +available + +=cut + +sub marc_isvalid { + my ($marc) = @_; + return 1 if ($marc->{item_form} and ($marc->{date1} =~ /\d{4}/) and + $marc->{record_type} and $marc->{bib_lvl} and $marc->{title}); + return 0; +} + + =head2 dump_fingerprints =cut @@ -202,7 +225,14 @@ sub dump_fingerprints { $marc->{title}, $marc->{lccn}) ,"\n"; } - # case e : author, publisher, pubyear, pages + if ($marc->{accomp}) { # case e : accomp + print OF join("\t", $marc->{id}, "case d", + $marc->{item_form}, $marc->{date1}, + $marc->{record_type}, $marc->{bib_lvl}, + $marc->{title}, $marc->{accomp}) ,"\n"; + } + + # case z : author, publisher, pubyear, pages if ($marc->{author} and $marc->{publisher} and $marc->{pubyear} and $marc->{pages}) { print OF join("\t", $marc->{id}, "case e", @@ -225,10 +255,17 @@ Write line of exception report sub dump_exception { my ($marc) = @_; + unless (defined $marc) { + print XF "Undefined record at line $count; likely bad XML\n"; + return; + } print XF "Record ", $marc->{id}, " did not make the cut: "; print XF "Missing item_form. " unless ($marc->{item_form}); - print XF "Missing valid date1. " - unless (defined $marc->{date1} and $marc->{date1} =~ /\d{4}/); + unless (defined $marc->{date1}) + { print XF "Missing date1. " } + else + { print XF "Invalid date1: ", $marc->{date1}, " " + unless ($marc->{date1} =~ /\d{4}/); } print XF "Missing record_type. " unless ($marc->{record_type}); print XF "Missing bib_lvl. " unless ($marc->{bib_lvl}); print XF "Missing title. " unless ($marc->{title}); @@ -236,14 +273,14 @@ sub dump_exception { } -=head2 initialyze +=head2 initialize Performs boring script initialization. Handles argument parsing, mostly. =cut -sub initialyze { +sub initialize { my ($c) = @_; my @missing = (); @@ -251,7 +288,10 @@ sub initialyze { binmode(STDIN, ':utf8'); my $rc = GetOptions( $c, + 'incoming', + 'incumbent', 'exception|x=s', + 'marctype|m=s', 'output|o=s', 'runtype|r=s', 'subfield|s=s', @@ -260,18 +300,33 @@ sub initialyze { 'help|h', ); show_help() unless $rc; + show_help() if ($c->{help}); + + # set defaults if told to do so + if ($c->{incoming}) { + $c->{tag} = 903 unless defined $c->{tag}; + $c->{subfield} = 'a' unless defined $c->{subfield}; + $c->{marctype} = 'XML' unless defined $c->{marctype}; + $c->{output} = 'incoming.fp' unless defined $c->{output}; + $c->{exception} = 'incoming.ex' unless defined $c->{exception}; + $c->{runtype} = 'full' unless defined $c->{runtype}; + } elsif ($c->{incumbent}) { + $c->{tag} = 901 unless defined $c->{tag}; + $c->{subfield} = 'c' unless defined $c->{subfield}; + $c->{marctype} = 'XML' unless defined $c->{marctype}; + $c->{output} = 'incumbent.fp' unless defined $c->{output}; + $c->{exception} = 'incumbent.ex' unless defined $c->{exception}; + $c->{runtype} = 'full' unless defined $c->{runtype}; + } my @keys = keys %{$c}; show_help() unless (@ARGV and @keys); - for my $key ('runtype', 'tag', 'subfield', 'output', 'exception') { - push @missing, $key unless $c->{$key} - } + for my $key ('runtype', 'tag', 'subfield', 'output', 'exception') + { push @missing, $key unless $c->{$key} } if (@missing) { print "Required option: ", join(', ', @missing), " missing!\n"; show_help(); } - - show_help() if ($c->{help}); } @@ -281,13 +336,9 @@ sub initialyze { sub progress_ticker { return if $conf->{quiet}; - - if ($count % 100 == 0) { - print '|'; - print " $count \n" unless ($count % 1400); - } elsif ($count % 20 == 0) { - print '.'; - } + printf("\r> %d recs seen; %d processed", $count, $scount); + printf(" (%d/s)", ($count / (time - $start + 1))) + if ($count % 500 == 0); } =head2 show_help @@ -298,7 +349,7 @@ Display usage message when things go wrong sub show_help { print < +Usage is: $0 [REQUIRED ARGS] [OPTIONS] Req'd Arguments --runtype=(primary|full) -r Do 'primary' or 'full' fingerprinting --tag=N -t Which tag to use @@ -306,7 +357,13 @@ Req'd Arguments --output= -o Output filename --exceptions= -x Exception report filename Options - --quiet -q Don't write status messages to STDOUT + --incoming Set -r to 'full'; -t, -s, -o, -x to incoming defaults + --incumbent Set -r to 'full'; -t, -s, -o, -x to incumbent defaults + + Example: '$0 --incoming' is equivalent to + '$0 -r full -t 903 -s a -o incoming.fp -x incoming.ex' + + --quiet -q Don't write status messages to STDOUT HELP exit 1; }