X-Git-Url: http://git.equinoxoli.org/?p=migration-tools.git;a=blobdiff_plain;f=extract_holdings;h=03e68351de7acbf50057cc29a297fbb5905d270c;hp=b4115409a7fcafcf32d832f1ad81e45309fa3cfd;hb=5d2e4c4541d612bafbb5a7bf1ed48a3b655b526c;hpb=07e09b509ccddb459137a98759781b6f8dcb396b diff --git a/extract_holdings b/extract_holdings index b411540..03e6835 100755 --- a/extract_holdings +++ b/extract_holdings @@ -2,99 +2,205 @@ use strict; use warnings; -use XML::Twig; -use YAML; -use JSON; +use Getopt::Long; +use Equinox::Migration::MapDrivenMARCXMLProc; +use Equinox::Migration::MARCXMLSampler; -my $marcxml = shift || help(); - -open HOLDINGS, '>', "holdings"; -open X, '>', "holdings.x"; -open Z, '>', "holdings.z"; -open ALL852, '>', "holdings.all852"; +my $VERSION = '1.000'; +my $c = initialize(); $| = 1; -my $holdings = {}; -my $copyid = 0; -my $count = 0; -my %all852 = ( x => {}, z => {} ); # hash of all subfields in all 852s - -my $t = XML::Twig->new( twig_handlers => { record => \&record } ); -$t->parsefile($marcxml); -#print ALL852 to_json(\%all852); -print ALL852 Dump(%all852); - -sub record { - my($t, $r)= @_; - $holdings = { copies => [] }; - - my @dfields = $r->children('datafield'); - for my $d (@dfields) { - process_datafields($d) - } - for my $copy (@{$holdings->{copies}}) - { print_reports($copy) } - $r->purge; - $count++; print "\r$count"; -} - -sub process_datafields { - my ($d) = @_; - # get 903 - if ($d->{'att'}->{'tag'} == 903) { - my $s = $d->first_child('subfield'); - $holdings->{id} = $s->text;; - } +# run samples if we've been asked for them +run_samples($c) if ($c->{sample} or $c->{samplemap} or $c->{samplestr}); +extract_holdings($c) if ($c->{map}); + +#-------------------------- + +sub extract_holdings { + my ($c) = @_; + print "Parsing records for extraction... "; + my $m = Equinox::Migration::MapDrivenMARCXMLProc->new( marcfile => $c->{marcfile}, + mapfile => $c->{map} ); + # open main holdings file + open HOLDINGS, '>', ($c->{prefix} . "-HOLDINGS.pg"); + # open the files for multi mappings + # FIXME DO THIS + select HOLDINGS; + + while (my $rec = $m->parse_record) { + # for each holdings tag in the record... + for my $holdidx ( @{$rec->{tmap}{ $c->{holdings} }} ) { + print STDOUT "\r", $m->recno; + my @out = (); # clear the output buffer + push @out, $rec->{egid}; # slug in the egid first thing + print "BEGIN;\n\negid\t" if ($m->recno == 1); + + # grab the unary mappings and slug 'em in + for my $sub ( sort keys %{$rec->{tags}[$holdidx]{uni}} ) { + push @out, $rec->{tags}[$holdidx]{uni}{$sub}; + print "l_", $m->name($rec->{tags}[$holdidx]{tag}, $sub),"\t" + if ($m->recno == 1); + } + + # now get everything else in the mapping + for my $othertag ( sort keys %{$rec->{tmap}} ) { + next if $othertag eq $c->{holdings}; # ignoring the holdings, o'course + my $idx = $rec->{tmap}{$othertag}[0]; # get index into tags struct + for my $sub ( sort keys %{$rec->{tags}[$idx]{uni}} ) { + push @out, $rec->{tags}[$idx]{uni}{$sub}; + print "l_", $m->name($rec->{tags}[$idx]{tag}, $sub), "\t" + if ($m->recno == 1); + } + } + + # and dump it + print "\n" if ($m->recno == 1); + print join("\t", @out); + print "\n"; + } - # and holdings data - if ($d->{'att'}->{'tag'} == 852) { - push @{$holdings->{copies}}, { x =>[], z => [] }; - $holdings->{copies}[-1]{copyid} = $copyid; - my @subs = $d->children('subfield'); - for my $s (@subs) - { process_subs($s) } - $copyid++; } + select STDOUT; + print "\n"; } -sub process_subs { - my ($s) = @_; - my $copy = $holdings->{copies}[-1]; +#-------------------------- + +sub run_samples { + my ($c) = @_; + my $s; + if ($c->{samplemap}) { + $s = Equinox::Migration::MARCXMLSampler->new( marcfile => $c->{marcfile}, + mapfile => $c->{samplemap}); + } elsif ($c->{samplestr}) { + $s = Equinox::Migration::MARCXMLSampler->new( marcfile => $c->{marcfile}, + mapstring => $c->{samplestr}); + } else { + $s = Equinox::Migration::MARCXMLSampler->new( marcfile => $c->{marcfile} ); + } + print "Parsing records for sampling... "; + $s->parse_records; - my $code = $s->{'att'}->{'code'}; - my $value = $s->text; + dump_sample_overview($c, $s) if $c->{sample}; + dump_sample_detail($c, $s) if ($c->{samplemap} or $c->{samplestr}); +} - if ($code eq 'x' or $code eq 'z') { - push @{$copy->{$code}}, $value; - my ($k,$v) = split /:/, $value; - $all852{$code}{$k} = $v; - } else { - $copy->{$code} = $value; - $all852{$code} = $value; +sub dump_sample_detail { + my ($c, $s) = @_; + my $tags = $s->{data}{samp}; + my $count = $s->{data}{rcnt}; + + open DETAIL, '>', ($c->{prefix} . "-HOLDINGS-DETAIL.txt"); + select DETAIL; + for my $tag (sort keys %{$tags}) { + print ">>>>> TAG $tag\n"; + for my $subkey (sort keys %{$tags->{$tag}}) { + my $sub = $tags->{$tag}{$subkey}; + print " Subfield: $subkey\n"; + print " Sample: '", $sub->{value}, "'\n"; + print " Count: ", $sub->{count}, " in ", $sub->{tcnt}, " tags\n\n"; + #print "(", int($sub->{count} / $sub->{rcnt}), "%)\n"; + } } + select STDOUT; + print "\n"; + close DETAIL; } -sub print_reports { - my ($copy) = @_; - my $note = 0; - for (@{$copy->{x}}) { - print X join("\t", $holdings->{id}, $copy->{copyid}, $note, $_), "\n"; - $note++; +sub dump_sample_overview { + my ($c, $s) = @_; + my $tags = $s->{data}{tags}; + my $count = $s->{data}{rcnt}; + + my @tagsbyname = sort keys %{$tags}; + my @tagsbycount = reverse sort { $tags->{$a} <=> $tags->{$b} } keys %{$tags}; + + open SAMPLE, '>', ($c->{prefix} . "-HOLDINGS-OVERVIEW.txt"); + select SAMPLE; + print "SAMPLE REPORT FOR ", $c->{prefix},": $count records\n\n"; + print "FOUND TAGS (BY TAG) FOUND TAGS (BY COUNT)\n"; + print "------------------------ --------------------------\n"; + for my $i (0 .. @tagsbyname - 1) { + print $tagsbyname[$i], (" " x (14 - length $tags->{ $tagsbyname[$i] })), + $tags->{ $tagsbyname[$i] }; + print " (", sprintf("%03d", int($tags->{ $tagsbyname[$i] } / $count * 100)), "%)"; + print " "; + print $tagsbycount[$i], (" " x (16 - length $tags->{ $tagsbycount[$i] })), + $tags->{ $tagsbycount[$i] }; + print " (", sprintf("%03d", int($tags->{ $tagsbycount[$i] } / $count * 100)), "%)\n"; } - $note = 0; - for (@{$copy->{z}}) { - print Z join("\t", $holdings->{id}, $copy->{copyid}, $note, $_), "\n"; - $note++; - } - print HOLDINGS join("\t", $holdings->{id}, $copy->{copyid}, - $copy->{b}, $copy->{p}, $copy->{h}, $copy->{9}), "\n"; + select STDOUT; + print "\n"; + close SAMPLE; } +#-------------------------- + +sub initialize { + my $c = {}; + my @missing = (); + + # set mode on existing filehandles + binmode(STDIN, ':utf8'); + + my $rc = GetOptions( $c, + 'sample|s', + 'samplemap|sm=s', + 'samplestr|ss=s', + 'marcfile|m=s', + 'map=s', + 'holdings|h=i', + 'prefix|p=s', + 'version|v', + 'help', + ); + show_help() unless $rc; + show_help() if ($c->{help}); + show_help("map and holdings must be specified together!") + if ($c->{map} and !$c->{holdings}); + show_version() if $c->{version}; + + my @keys = keys %{$c}; + for my $key ('prefix') + { push @missing, $key unless $c->{$key} } + if (@missing) { + print "Required option: ", join(', ', @missing), " missing!\n"; + show_help(); + } + + return $c; +} -sub help { +sub show_help { + my ($msg) = @_; + print "\nERROR - $msg\n" if $msg; print <