X-Git-Url: http://git.equinoxoli.org/?p=migration-tools.git;a=blobdiff_plain;f=fingerprinter;h=06f62f7c94ded84974ddc965582e9fb3ced3d903;hp=65e98dc5d121aa0868169d68de49741dadcee78f;hb=f26caf3c919e7c65b48be7f268bee6ade13b2eac;hpb=e70990d9e21b666598569a2ebd21b67b88472779 diff --git a/fingerprinter b/fingerprinter index 65e98dc..06f62f7 100755 --- a/fingerprinter +++ b/fingerprinter @@ -5,12 +5,11 @@ use open ':utf8'; use Getopt::Long; use MARC::Batch; -use MARC::File::XML ( BinaryEncoding => 'utf-8' ); -use MARC::Field; use Unicode::Normalize; +use MARC::File::XML ( BinaryEncoding => 'utf-8' ); my $conf = {}; # configuration hashref -my $count = 0; +my $count = 0; my $scount = 0; my $start = time; $| = 1; @@ -23,16 +22,17 @@ binmode(XF, ':utf8'); for my $file (@ARGV) { print XF "Processing $file\n"; - open my $records, '<:utf8', $file; + #open my $records, '<:utf8', $file; # This hack dosn't play nice with the use MARC::File::XML ( BinaryEncoding => 'utf-8' ); hack my $batch = undef; my $record = undef; - $batch = MARC::Batch->new('XML', $records); + #$batch = MARC::Batch->new('XML', $records); # The other part of the hack + $batch = MARC::Batch->new($conf->{marctype}, $file); $batch->strict_off(); $batch->warnings_off(); - while ( eval { $record = $batch->next } ) { - my $marc = undef; + while ( $record = $batch->next ) { $count++; progress_ticker(); + my $marc = undef; unless ( defined $record ) { dump_exception($marc); next; } @@ -48,11 +48,11 @@ for my $file (@ARGV) { unless (marc_isvalid($marc)) { dump_exception($marc); next; } dump_fingerprints($marc); + $scount++; progress_ticker(); } } -print "\nProcessed $count records\n" unless $conf->{quiet}; - +print "\nSuccessfully processed:\t$count\n" unless $conf->{quiet}; =head2 populate_marc @@ -75,11 +75,20 @@ sub populate_marc { # date1, date2 my $my_008 = $record->field('008'); $my_008 = $my_008->as_string() if ($my_008); - unless (length $my_008 == 40) - { print XF ">> Bad 008 field length in rec. $id\n"; return \%marc } + unless (defined $my_008 and length $my_008 == 40) + { print XF ">> Bad 008 length in rec ",$marc{id},"\n"; return \%marc } $marc{date1} = substr($my_008,7,4) if ($my_008); $marc{date2} = substr($my_008,11,4) if ($my_008); # UNUSED + unless ($marc{date1} and $marc{date1} =~ /\d{4}/) { + my $my_260 = $record->field('260'); + my $date1 = $my_260->subfield('c') if $my_260; + if (defined $date1 and $date1 =~ /\d{4}/) { + $marc{date1} = $date1; + print XF ">> using 260c as date1 on $id\n"; + } + } + # item_form if ( $marc{record_type} =~ /[gkroef]/ ) { # MAP, VIS $marc{item_form} = substr($my_008,29,1) if ($my_008); @@ -113,7 +122,7 @@ sub populate_marc { $marc{desc} = $record->field('300')->subfield('a') if $record->field('300'); $marc{pages} = $1 if (defined $marc{desc} and $marc{desc} =~ /(\d+)/); $marc{title} = $record->field('245')->subfield('a') - if defined $record->field('245'); + if $record->field('245'); $marc{edition} = $record->field('250')->subfield('a') if $record->field('250'); if ($record->field('260')) { @@ -261,9 +270,11 @@ sub dump_exception { } print XF "Record ", $marc->{id}, " did not make the cut: "; print XF "Missing item_form. " unless ($marc->{item_form}); - print XF "Missing date1. " unless (defined $marc->{date1}); - print XF "Invalid date1: ", $marc->{date1} - unless ($marc->{date1} =~ /\d{4}/); + unless (defined $marc->{date1}) + { print XF "Missing date1. " } + else + { print XF "Invalid date1: ", $marc->{date1}, " " + unless ($marc->{date1} =~ /\d{4}/); } print XF "Missing record_type. " unless ($marc->{record_type}); print XF "Missing bib_lvl. " unless ($marc->{bib_lvl}); print XF "Missing title. " unless ($marc->{title}); @@ -286,7 +297,10 @@ sub initialize { binmode(STDIN, ':utf8'); my $rc = GetOptions( $c, + 'incoming', + 'incumbent', 'exception|x=s', + 'marctype|m=s', 'output|o=s', 'runtype|r=s', 'subfield|s=s', @@ -295,18 +309,33 @@ sub initialize { 'help|h', ); show_help() unless $rc; + show_help() if ($c->{help}); + + # set defaults if told to do so + if ($c->{incoming}) { + $c->{tag} = 903 unless defined $c->{tag}; + $c->{subfield} = 'a' unless defined $c->{subfield}; + $c->{marctype} = 'XML' unless defined $c->{marctype}; + $c->{output} = 'incoming.fp' unless defined $c->{output}; + $c->{exception} = 'incoming.ex' unless defined $c->{exception}; + $c->{runtype} = 'full' unless defined $c->{runtype}; + } elsif ($c->{incumbent}) { + $c->{tag} = 901 unless defined $c->{tag}; + $c->{subfield} = 'c' unless defined $c->{subfield}; + $c->{marctype} = 'XML' unless defined $c->{marctype}; + $c->{output} = 'incumbent.fp' unless defined $c->{output}; + $c->{exception} = 'incumbent.ex' unless defined $c->{exception}; + $c->{runtype} = 'full' unless defined $c->{runtype}; + } my @keys = keys %{$c}; show_help() unless (@ARGV and @keys); - for my $key ('runtype', 'tag', 'subfield', 'output', 'exception') { - push @missing, $key unless $c->{$key} - } + for my $key ('runtype', 'tag', 'subfield', 'output', 'exception') + { push @missing, $key unless $c->{$key} } if (@missing) { print "Required option: ", join(', ', @missing), " missing!\n"; show_help(); } - - show_help() if ($c->{help}); } @@ -316,8 +345,9 @@ sub initialize { sub progress_ticker { return if $conf->{quiet}; - printf("> %d (%d/s)\r", $count, ((time - $start) / $count)) - if ($count % 100 == 0); + printf("\r> %d recs seen; %d processed", $count, $scount); + printf(" (%d/s)", ($count / (time - $start + 1))) + if ($count % 500 == 0); } =head2 show_help @@ -328,7 +358,7 @@ Display usage message when things go wrong sub show_help { print < +Usage is: $0 [REQUIRED ARGS] [OPTIONS] Req'd Arguments --runtype=(primary|full) -r Do 'primary' or 'full' fingerprinting --tag=N -t Which tag to use @@ -336,7 +366,13 @@ Req'd Arguments --output= -o Output filename --exceptions= -x Exception report filename Options - --quiet -q Don't write status messages to STDOUT + --incoming Set -r to 'full'; -t, -s, -o, -x to incoming defaults + --incumbent Set -r to 'full'; -t, -s, -o, -x to incumbent defaults + + Example: '$0 --incoming' is equivalent to + '$0 -r full -t 903 -s a -o incoming.fp -x incoming.ex' + + --quiet -q Don't write status messages to STDOUT HELP exit 1; }