X-Git-Url: http://git.equinoxoli.org/?p=migration-tools.git;a=blobdiff_plain;f=extract_holdings;h=51ea2b84152f73e8c4320d4a93365f49b443b14e;hp=ee8bb93406e08e1e480dea7bb85b16f330ae3c5f;hb=0445ca1b7fb163fd20346fedaaa7e8612f8f93c3;hpb=e12bbe250b16834143d00d478d14c60ccfcd8075 diff --git a/extract_holdings b/extract_holdings index ee8bb93..51ea2b8 100755 --- a/extract_holdings +++ b/extract_holdings @@ -3,7 +3,7 @@ use strict; use warnings; use Getopt::Long; -use Equinox::Migration::MapDrivenMARCXMLProc; +use Equinox::Migration::MapDrivenMARCXMLProc 1.005; use Equinox::Migration::MARCXMLSampler; my $VERSION = '1.001'; @@ -22,70 +22,104 @@ $| = 1; # run samples if we've been asked for them run_samples($c) if ($c->{sample} or $c->{samplemap} or $c->{samplestr}); extract_holdings($c) if ($c->{map}); +print "\n"; #-------------------------- sub extract_holdings { my ($c) = @_; - print "Parsing records for extraction... "; + print "Parsing records for extraction:\n"; my $m = Equinox::Migration::MapDrivenMARCXMLProc->new( marcfile => $c->{marcfile}, - mapfile => $c->{map} ); + mapfile => $c->{map}, + verbose => 1, + ); + + print "Writing holdings to output file(s)...\n"; # open main holdings file open HOLDINGS, '>', ($c->{prefix} . "-HOLDINGS.pg"); - # open the files for multi mappings - # FIXME DO THIS - open X, '>', ($c->{prefix} . "-HOLDINGS-privnotes.pg"); - open Z, '>', ($c->{prefix} . "-HOLDINGS-pubnotes.pg"); - select HOLDINGS; - - while (my $rec = $m->parse_record) { - # for each holdings tag in the record... - for my $holdidx ( @{$rec->{tmap}{ $c->{holdings} }} ) { + # create multi files + my $multis = $m->get_multis; + my %MULTIFILE = (); + for my $t ( keys %{$multis} ) { + for my $s ( keys %{$multis->{$t}}) + { open my $fh, ">", ($c->{prefix} . "-HOLDINGS-MULT-$t$s.pg"); $MULTIFILE{"$t$s"} = $fh } + } + + my $i = 0; # record counter + my $j = 0; # holdings counter + + while ( $m->{data}{recs}[$i] ) { + print HOLDINGS "BEGIN;\n\negid, hseq, " unless $j; + my $rec = $m->{data}{recs}[$i]; + my $k = 0; # holding-within-record pointer + + for my $holdidx ( @{ $rec->{tmap}{ $c->{holdings} } } ) { + # for each holdings tag in the record... my $tagid = $rec->{tags}[$holdidx]{tag}; + $k++; - print STDOUT "\r", $m->recno; my @out = (); # clear the output buffer push @out, $rec->{egid}; # slug in the egid first thing - print "BEGIN;\n\negid\t" if ($m->recno == 1); + push @out, $j; # holding seq goes next # grab the unary mappings and slug 'em in for my $sub ( sort keys %{$rec->{tags}[$holdidx]{uni}} ) { push @out, $rec->{tags}[$holdidx]{uni}{$sub}; - print "l_", $m->name($tagid, $sub),"\t" - if ($m->recno == 1); + print HOLDINGS "l_", $m->name($tagid, $sub),", " unless $j; } - for my $x (@{$rec->{tags}[$holdidx]{multi}{x}} ) { - print X $rec->{egid}, "\t", - $rec->{tags}[$holdidx]{uni}{ $c->{copyid} }, - "\t$x\n"; - } - for my $z (@{$rec->{tags}[$holdidx]{multi}{z}} ) { - print Z $rec->{egid}, "\t", - $rec->{tags}[$holdidx]{uni}{ $c->{copyid} }, - "\t$z\n"; + # handle holdings multis + for my $sub ( sort keys %{$multis->{$tagid}} ) { + for my $value ( @{$rec->{tags}[$holdidx]{multi}{$sub}} ) { + my $fh = $MULTIFILE{"$tagid$sub"}; + print $fh join("\t", $rec->{egid}, $j, $value), "\n"; + } } # now get everything else in the mapping for my $othertag ( sort keys %{$rec->{tmap}} ) { next if $othertag eq $c->{holdings}; # ignoring the holdings, o'course - my $idx = $rec->{tmap}{$othertag}[0]; # get index into tags struct - for my $sub ( sort keys %{$rec->{tags}[$idx]{uni}} ) { - push @out, $rec->{tags}[$idx]{uni}{$sub}; - print "l_", $m->name($rec->{tags}[$idx]{tag}, $sub), "\t" - if ($m->recno == 1); + my $test_idx = $rec->{tmap}{$othertag}[0]; # get index into tags struct + unless (defined $test_idx) { + push @out, ''; + next; + } + # handle only first other tag unless it is known to be multi + my $limit = 0; + if (exists($multis->{$othertag})) { + $limit = $#{ $rec->{tmap}{$othertag} }; + } + foreach my $idx (0..$limit) { + my $tag_idx = $rec->{tmap}{$othertag}[$idx]; + for my $sub ( sort keys %{$rec->{tags}[$tag_idx]{uni}} ) { + if ($m->first_only($rec->{tags}[$tag_idx]{tag}, $sub)) { + push @out, ($k == 1) ? $rec->{tags}[$tag_idx]{uni}{$sub} : ''; + } else { + push @out, $rec->{tags}[$tag_idx]{uni}{$sub}; + } + print HOLDINGS "l_", $m->name($rec->{tags}[$tag_idx]{tag}, $sub), ", " unless $j; + } + next unless exists($multis->{$othertag}); + for my $sub ( sort keys %{$multis->{$othertag}} ) { + next if $m->first_only($rec->{tags}[$tag_idx]{tag}, $sub) and ($k > 1); + for my $value ( @{$rec->{tags}[$tag_idx]{multi}{$sub}} ) { + my $fh = $MULTIFILE{"$othertag$sub"}; + print $fh normalize_output(join("\t", $rec->{egid}, $j, $value)), "\n"; + } + } } } # and dump it - print "\n" if ($m->recno == 1); - print join("\t", @out); - print "\n"; + print HOLDINGS "\n" unless $j; + print HOLDINGS normalize_output(join("\t", @out)); + print HOLDINGS "\n"; + $j++; } - + $i++; + print "\r$i $j"; } - select STDOUT; print "\n"; } @@ -94,6 +128,7 @@ sub extract_holdings { sub run_samples { my ($c) = @_; my $s; + print "Parsing records for sampling... "; if ($c->{samplemap}) { $s = Equinox::Migration::MARCXMLSampler->new( marcfile => $c->{marcfile}, mapfile => $c->{samplemap}); @@ -103,8 +138,6 @@ sub run_samples { } else { $s = Equinox::Migration::MARCXMLSampler->new( marcfile => $c->{marcfile} ); } - print "Parsing records for sampling... "; - $s->parse_records; dump_sample_overview($c, $s) if $c->{sample}; dump_sample_detail($c, $s) if ($c->{samplemap} or $c->{samplestr}); @@ -179,11 +212,12 @@ sub initialize { 'sample|s', 'samplemap|sm=s', 'samplestr|ss=s', - 'marcfile|m=s', - 'map=s', + 'marcfile=s', + 'map|m=s', 'holdings|h=i', 'copyid|c=s', 'prefix|p=s', + 'disable-pg-normalization', 'version|v', 'help', ); @@ -192,9 +226,13 @@ sub initialize { show_help("Nothing to do!") unless ($c->{map} or $c->{sample} or $c->{samplemap} or $c->{samplestr}); show_help("map, holdings, and copyid must be specified together!") - if ($c->{map} and !$c->{holdings} and !$c->{copyid}); + if ($c->{map} and !($c->{holdings} and $c->{copyid})); show_version() if $c->{version}; + if ($c->{prefix} and !$c->{marcfile}) { + $c->{marcfile} = $c->{prefix} . ".clean.marc.xml"; + } + my @keys = keys %{$c}; for my $key ('prefix', 'marcfile') { push @missing, $key unless $c->{$key} } @@ -206,6 +244,12 @@ sub initialize { return $c; } +sub normalize_output { + my $str = shift; + $str =~ s!\\!\\\\!g unless $c->{'disable-pg-normalization'}; + return $str; +} + sub show_help { my ($msg) = @_; print "\nERROR - $msg\n" if $msg; @@ -215,25 +259,30 @@ Usage is: extract_holdings -p PREFIX -m MARCFILE [ARGUMENTS] REQUIRED ARGUMENTS --prefix -p Prefix string for output filenames - --marcfile -m MARCXML to use as source data + --marcfile MARCXML to use as source data + Defaults to 'PREFIX.clean.marc.xml' SAMPLING ARGUMENTS --sample -s Generate a report of all tags in the MARC data - --samplemap -sm Specify a E::M::STL map file which will be used to generate - subfield breakdown reports about specific tags in the MARC - data + --samplemap -sm Specify a Equinox::Migration::SimpleTagList map file which + will be used to generate subfield breakdown reports about + specific tags in the MARC data --samplestr -ss As above, but with a one-liner map specified on the command line as a string (e.g. '-ss "852 999"') If --samplemap and --samplestr are both specified, --samplemap wins. HOLDINGS EXTRACTION ARGUMENTS - --map E::M::SM map file which will be used to extract holdings data - from the input MARC file + --map -m Equinox::Migration::SubfieldMapper map file which will be + used to extract holdings data from the input MARC file --holdings -h Specifies actual holdings tag --copyid -c Specifies subfield of holdings with unique copy identifier + --disable-pg-normalization By default, output is normalized so that a Postgres + copy or \\copy can import the data without choking on + backslashes; use this command-line option if + output is not meant to be consumed by psql. - Both these must be given together. + All three of these must be given together. HELP exit; }