X-Git-Url: http://git.equinoxoli.org/?p=migration-tools.git;a=blobdiff_plain;f=extract_holdings;h=028378d4e063d7bc67087e551d20d150952ad6a9;hp=78fbde5c8a93ec13f27551da5e082523db9819d2;hb=59a641b21b06f721bb092b8a889a03638e0ad295;hpb=e62ff5b8e90d327bc36e4f347ed92a919cdcd539 diff --git a/extract_holdings b/extract_holdings index 78fbde5..028378d 100755 --- a/extract_holdings +++ b/extract_holdings @@ -1,9 +1,26 @@ #!/usr/bin/perl + +# Copyright 2009-2012, Equinox Software, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + use strict; use warnings; use Getopt::Long; -use Equinox::Migration::MapDrivenMARCXMLProc 1.003; +use Equinox::Migration::MapDrivenMARCXMLProc 1.005; use Equinox::Migration::MARCXMLSampler; my $VERSION = '1.001'; @@ -22,37 +39,43 @@ $| = 1; # run samples if we've been asked for them run_samples($c) if ($c->{sample} or $c->{samplemap} or $c->{samplestr}); extract_holdings($c) if ($c->{map}); +print "\n"; #-------------------------- sub extract_holdings { my ($c) = @_; - print "Parsing records for extraction... "; + print "Parsing records for extraction:\n"; my $m = Equinox::Migration::MapDrivenMARCXMLProc->new( marcfile => $c->{marcfile}, mapfile => $c->{map}, verbose => 1, ); + print "Writing holdings to output file(s)...\n"; # open main holdings file - open HOLDINGS, '>', ($c->{prefix} . "-HOLDINGS.pg"); + open HOLDINGS, '>', ($c->{prefix} . "-" . $c->{suffix} . ".pg"); # create multi files my $multis = $m->get_multis; my %MULTIFILE = (); for my $t ( keys %{$multis} ) { for my $s ( keys %{$multis->{$t}}) - { open my $fh, ">", ($c->{prefix} . "-HOLDINGS-MULT-$t$s.pg"); $MULTIFILE{"$t$s"} = $fh } + { open my $fh, ">", ($c->{prefix} . "-" . $c->{suffix} . "-MULT-$t$s.pg"); $MULTIFILE{"$t$s"} = $fh } } + my $parallel_fields = $m->get_parallel_fields; + my $i = 0; # record counter my $j = 0; # holdings counter while ( $m->{data}{recs}[$i] ) { - print HOLDINGS "BEGIN;\n\negid\thseq\t" unless $j; + print HOLDINGS "BEGIN;\n\negid, hseq, " unless $j; my $rec = $m->{data}{recs}[$i]; + my $k = 0; # holding-within-record pointer - # for each holdings tag in the record... - for my $holdidx ( @{$rec->{tmap}{ $c->{holdings} }} ) { + for my $holdidx ( @{ $rec->{tmap}{ $c->{holdings} } } ) { + # for each holdings tag in the record... my $tagid = $rec->{tags}[$holdidx]{tag}; + $k++; my @out = (); # clear the output buffer push @out, $rec->{egid}; # slug in the egid first thing @@ -61,14 +84,16 @@ sub extract_holdings { # grab the unary mappings and slug 'em in for my $sub ( sort keys %{$rec->{tags}[$holdidx]{uni}} ) { push @out, $rec->{tags}[$holdidx]{uni}{$sub}; - print HOLDINGS "l_", $m->name($tagid, $sub),"\t" unless $j; + print HOLDINGS "l_", $m->name($tagid, $sub),", " unless $j; } # handle holdings multis for my $sub ( sort keys %{$multis->{$tagid}} ) { for my $value ( @{$rec->{tags}[$holdidx]{multi}{$sub}} ) { my $fh = $MULTIFILE{"$tagid$sub"}; - print $fh join("\t", $rec->{egid}, $j, $value), "\n"; + my $clean_value = $value; + $clean_value =~ s/[\r\n\t]//g; + print $fh join("\t", $rec->{egid}, $j, $clean_value), "\n"; } } @@ -76,22 +101,63 @@ sub extract_holdings { # now get everything else in the mapping for my $othertag ( sort keys %{$rec->{tmap}} ) { next if $othertag eq $c->{holdings}; # ignoring the holdings, o'course - my $idx = $rec->{tmap}{$othertag}[0]; # get index into tags struct - for my $sub ( sort keys %{$rec->{tags}[$idx]{uni}} ) { - push @out, $rec->{tags}[$idx]{uni}{$sub}; - print "l_", $m->name($rec->{tags}[$idx]{tag}, $sub), "\t" - unless $j; + my $test_idx = $rec->{tmap}{$othertag}[0]; # get index into tags struct + unless (defined $test_idx) { + push @out, ''; + next; + } + + # handle parallel fields + if (exists($parallel_fields->{$othertag})) { + my $num_fields = $#{ $rec->{tmap}{$othertag} }; + my $tag_idx; + if ($holdidx > $num_fields) { + $tag_idx = -1; + } else { + $tag_idx = $rec->{tmap}{$othertag}[$holdidx]; + } + for my $sub ( sort keys %{ $parallel_fields->{$othertag } } ) { + push @out, $tag_idx > -1 ? $rec->{tags}[$tag_idx]{parallel}{$sub}->[0] : ''; + print HOLDINGS "l_", $m->name($rec->{tags}[$tag_idx]{tag}, $sub), ", " unless $j; + } + } + + # handle only first other tag unless it is known to be multi + my $limit = 0; + if (exists($multis->{$othertag})) { + $limit = $#{ $rec->{tmap}{$othertag} }; + } + foreach my $idx (0..$limit) { + my $tag_idx = $rec->{tmap}{$othertag}[$idx]; + for my $sub ( sort keys %{$rec->{tags}[$tag_idx]{uni}} ) { + if ($m->first_only($rec->{tags}[$tag_idx]{tag}, $sub)) { + push @out, ($k == 1) ? $rec->{tags}[$tag_idx]{uni}{$sub} : ''; + } else { + push @out, $rec->{tags}[$tag_idx]{uni}{$sub}; + } + print HOLDINGS "l_", $m->name($rec->{tags}[$tag_idx]{tag}, $sub), ", " unless $j; + } + next unless exists($multis->{$othertag}); + for my $sub ( sort keys %{$multis->{$othertag}} ) { + next if $m->first_only($rec->{tags}[$tag_idx]{tag}, $sub) and ($k > 1); + for my $value ( @{$rec->{tags}[$tag_idx]{multi}{$sub}} ) { + my $fh = $MULTIFILE{"$othertag$sub"}; + my $clean_value = $value; + $clean_value =~ s/[\r\n\t]//g; + print $fh normalize_output(join("\t", $rec->{egid}, $j, $clean_value)), "\n"; + } + } } } # and dump it print HOLDINGS "\n" unless $j; - print HOLDINGS join("\t", @out); + print HOLDINGS normalize_output(join("\t", @out)); print HOLDINGS "\n"; $j++; } $i++; - print "\r$i"; + print "\r$i $j"; } print "\n"; } @@ -122,19 +188,19 @@ sub dump_sample_detail { my $count = $s->{data}{rcnt}; my $scnt = $s->{data}{scnt}; - open DETAIL, '>', ($c->{prefix} . "-HOLDINGS-DETAIL.txt"); + open DETAIL, '>', ($c->{prefix} . "-" . $c->{suffix} . "-DETAIL.txt"); select DETAIL; for my $tag (sort keys %{ $tags }) { print ">>>>> TAG $tag\n\n"; for my $subkey (sort keys %{ $tags->{$tag} }) { my $sub = $tags->{$tag}{$subkey}; print "|| $subkey | ", $sub->{value}, " | ", - $sub->{count}, "/", $sub->{tcnt}, " | ||\n"; + $sub->{count}, "/", $sub->{tcnt}, " | ", ($sub->{count} > $sub->{tcnt}) ? "MULTI" : "", " ||\n"; } print "\n"; } close DETAIL; - open SCOUNT, '>', ($c->{prefix} . "-HOLDINGS-SUBCOUNTS.txt"); + open SCOUNT, '>', ($c->{prefix} . "-" . $c->{suffix} . "-SUBCOUNTS.txt"); select SCOUNT; for my $tag (sort keys %{ $scnt }) { print ">>>>> TAG $tag\n\n"; @@ -143,6 +209,8 @@ sub dump_sample_detail { print "\n"; } select STDOUT; + print "Saved results as ", ($c->{prefix} . "-" . $c->{suffix} . "-DETAIL.txt"), " and ", + ($c->{prefix} . "-" . $c->{suffix} . "-SUBCOUNTS.txt"), "\n"; } sub dump_sample_overview { @@ -153,7 +221,7 @@ sub dump_sample_overview { my @tagsbyname = sort keys %{$tags}; my @tagsbycount = reverse sort { $tags->{$a} <=> $tags->{$b} } keys %{$tags}; - open SAMPLE, '>', ($c->{prefix} . "-HOLDINGS-OVERVIEW.txt"); + open SAMPLE, '>', ($c->{prefix} . "-" . $c->{suffix} . "-OVERVIEW.txt"); select SAMPLE; print "SAMPLE REPORT FOR ", $c->{prefix},": $count records\n\n"; print "FOUND TAGS (BY TAG) FOUND TAGS (BY COUNT)\n"; @@ -168,7 +236,7 @@ sub dump_sample_overview { print " (", sprintf("%03d", int($tags->{ $tagsbycount[$i] } / $count * 100)), "%)\n"; } select STDOUT; - print "\n"; + print "Saved results as ", ($c->{prefix} . "-" . $c->{suffix} . "-OVERVIEW.txt"), "\n"; close SAMPLE; } @@ -185,11 +253,13 @@ sub initialize { 'sample|s', 'samplemap|sm=s', 'samplestr|ss=s', - 'marcfile|m=s', - 'map=s', + 'marcfile=s', + 'map|m=s', 'holdings|h=i', 'copyid|c=s', 'prefix|p=s', + 'suffix|o=s', + 'disable-pg-normalization', 'version|v', 'help', ); @@ -201,6 +271,13 @@ sub initialize { if ($c->{map} and !($c->{holdings} and $c->{copyid})); show_version() if $c->{version}; + if (! $c->{suffix}) { + $c->{suffix} = 'HOLDINGS'; + } + if ($c->{prefix} and !$c->{marcfile}) { + $c->{marcfile} = $c->{prefix} . ".clean.marc.xml"; + } + my @keys = keys %{$c}; for my $key ('prefix', 'marcfile') { push @missing, $key unless $c->{$key} } @@ -212,34 +289,46 @@ sub initialize { return $c; } +sub normalize_output { + my $str = shift; + $str =~ s!\\!\\\\!g unless $c->{'disable-pg-normalization'}; + return $str; +} + sub show_help { my ($msg) = @_; print "\nERROR - $msg\n" if $msg; print <