X-Git-Url: http://git.equinoxoli.org/?p=migration-tools.git;a=blobdiff_plain;f=extract_holdings;h=05574163e69da994b93c6d3774e1bfc6f5a429ad;hp=eea04652c9729e0dc18942ab1254a077e031460b;hb=31dfec235ae3eb0f0aada9a48fdf025449e3f290;hpb=6c25f0626b026e59b9051c31d3eddfb6105f3c56 diff --git a/extract_holdings b/extract_holdings index eea0465..0557416 100755 --- a/extract_holdings +++ b/extract_holdings @@ -2,111 +2,174 @@ use strict; use warnings; -use XML::Twig; -use YAML::Tiny; use Getopt::Long; +use Equinox::Migration::MapDrivenMARCXMLProc 1.003; +use Equinox::Migration::MARCXMLSampler; +my $VERSION = '1.001'; -my $conf = initialize(); -my $marcxml = shift; +=pod -open HOLDINGS, '>', $conf->{output}; -open X, '>', $conf->{pubnotesfile}; -open Z, '>', $conf->{privnotesfile}; -my $holdings = {}; -my %sample = ( x => {}, z => {} ); # hash of all subfields in all 852s -my $copyid = 0; +TODO + * Have detail mode report on number of subfields per datafield +=cut + +my $c = initialize(); $| = 1; -my $count = 0; -my $total = `grep -c 'new; -my $t = XML::Twig->new( twig_handlers => { record => \&record } ); -$t->parsefile($marcxml); -$yaml->[0] = \%sample; -$yaml->write('holdings.sample'); -print "\n\n"; - -sub record { - my($t, $r)= @_; - $holdings = { copies => [] }; - - my @dfields = $r->children('datafield'); - for my $d (@dfields) - { process_datafields($d) } - - for my $copy (@{$holdings->{copies}}) - { print_reports($copy) } - $r->purge; - - $count++; - $percent = int(($count / $total) * 100); - print "\r$percent% done ($count)" if ($percent != $prevper); - $prevper = $percent; -} -sub process_datafields { - my ($d) = @_; - # get 903 - if ($d->{'att'}->{'tag'} == 903) { - my $s = $d->first_child('subfield'); - $holdings->{id} = $s->text;; - } +# run samples if we've been asked for them +run_samples($c) if ($c->{sample} or $c->{samplemap} or $c->{samplestr}); +extract_holdings($c) if ($c->{map}); - # and holdings data - if ($d->{'att'}->{'tag'} == $conf->{tag}) { - push @{$holdings->{copies}}, { x =>[], z => [] }; - $holdings->{copies}[-1]{copyid} = $copyid; - my @subs = $d->children('subfield'); - for my $s (@subs) - { process_subs($s) } - $copyid++; +#-------------------------- + +sub extract_holdings { + my ($c) = @_; + print "Parsing records for extraction... "; + my $m = Equinox::Migration::MapDrivenMARCXMLProc->new( marcfile => $c->{marcfile}, + mapfile => $c->{map}, + verbose => 1, + ); + print "Writing holdings to output file(s)...\n"; + # open main holdings file + open HOLDINGS, '>', ($c->{prefix} . "-HOLDINGS.pg"); + # create multi files + my @multifiles = <*HOLDINGS-MULT*>; + unlink @multifiles; + + my $i = 0; # record counter + my $j = 0; # holdings counter + my $multis = $m->get_multis; + + while ( $m->{data}{recs}[$i] ) { + print HOLDINGS "BEGIN;\n\negid\thseq\t" unless $j; + my $rec = $m->{data}{recs}[$i]; + + # for each holdings tag in the record... + for my $holdidx ( @{$rec->{tmap}{ $c->{holdings} }} ) { + my $tagid = $rec->{tags}[$holdidx]{tag}; + + my @out = (); # clear the output buffer + push @out, $rec->{egid}; # slug in the egid first thing + push @out, $j; # holding seq goes next + + # grab the unary mappings and slug 'em in + for my $sub ( sort keys %{$rec->{tags}[$holdidx]{uni}} ) { + push @out, $rec->{tags}[$holdidx]{uni}{$sub}; + print HOLDINGS "l_", $m->name($tagid, $sub),"\t" unless $j; + } + + # handle holdings multis + for my $sub ( sort keys %{$multis->{$tagid}} ) { + open MULT, ">>", ($c->{prefix} . "HOLDINGS-MULT" . "$tagid$sub.pg"); + for my $value ( @{$rec->{tags}[$holdidx]{multi}{$sub}} ) + { print MULT join("\t", $rec->{egid}, $j, $value), "\n" } + close MULT; + } + + + # now get everything else in the mapping + for my $othertag ( sort keys %{$rec->{tmap}} ) { + next if $othertag eq $c->{holdings}; # ignoring the holdings, o'course + my $idx = $rec->{tmap}{$othertag}[0]; # get index into tags struct + for my $sub ( sort keys %{$rec->{tags}[$idx]{uni}} ) { + push @out, $rec->{tags}[$idx]{uni}{$sub}; + print "l_", $m->name($rec->{tags}[$idx]{tag}, $sub), "\t" + unless $j; + } + } + + # and dump it + print HOLDINGS "\n" unless $j; + print HOLDINGS join("\t", @out); + print HOLDINGS "\n"; + $j++; + } + $i++; + print "\r$i"; } + print "\n"; } -sub process_subs { - my ($s) = @_; - my $copy = $holdings->{copies}[-1]; - - my $code = $s->{'att'}->{'code'}; - my $value = $s->text; +#-------------------------- - if ($code eq $conf->{pubnotes} or $code eq $conf->{privnotes}) { - push @{$copy->{$code}}, $value; - my ($k,$v) = split /:/, $value; - $sample{$code}{$k} = $v; +sub run_samples { + my ($c) = @_; + my $s; + print "Parsing records for sampling... "; + if ($c->{samplemap}) { + $s = Equinox::Migration::MARCXMLSampler->new( marcfile => $c->{marcfile}, + mapfile => $c->{samplemap}); + } elsif ($c->{samplestr}) { + $s = Equinox::Migration::MARCXMLSampler->new( marcfile => $c->{marcfile}, + mapstring => $c->{samplestr}); } else { - $copy->{$code} = $value; - $sample{$code} = $value; + $s = Equinox::Migration::MARCXMLSampler->new( marcfile => $c->{marcfile} ); } + + dump_sample_overview($c, $s) if $c->{sample}; + dump_sample_detail($c, $s) if ($c->{samplemap} or $c->{samplestr}); } -sub print_reports { - return unless defined $holdings->{id}; - my ($copy) = @_; - my $note = 0; - for (@{$copy->{x}}) { - print X join("\t", $holdings->{id}, $copy->{copyid}, $note, $_), "\n"; - $note++; +sub dump_sample_detail { + my ($c, $s) = @_; + my $tags = $s->{data}{samp}; + my $count = $s->{data}{rcnt}; + my $scnt = $s->{data}{scnt}; + + open DETAIL, '>', ($c->{prefix} . "-HOLDINGS-DETAIL.txt"); + select DETAIL; + for my $tag (sort keys %{ $tags }) { + print ">>>>> TAG $tag\n\n"; + for my $subkey (sort keys %{ $tags->{$tag} }) { + my $sub = $tags->{$tag}{$subkey}; + print "|| $subkey | ", $sub->{value}, " | ", + $sub->{count}, "/", $sub->{tcnt}, " | ||\n"; + } + print "\n"; } - $note = 0; - for (@{$copy->{z}}) { - print Z join("\t", $holdings->{id}, $copy->{copyid}, $note, $_), "\n"; - $note++; + close DETAIL; + open SCOUNT, '>', ($c->{prefix} . "-HOLDINGS-SUBCOUNTS.txt"); + select SCOUNT; + for my $tag (sort keys %{ $scnt }) { + print ">>>>> TAG $tag\n\n"; + for my $len (sort keys %{ $scnt->{$tag} }) + { print "|| $len | ", $scnt->{$tag}{$len}, " ||\n" } + print "\n"; } - my @fields = (); - for ( @{$conf->{fields}} ) - { $copy->{$_} = '' unless defined $copy->{$_}; push @fields, $copy->{$_} } - print HOLDINGS join("\t", $holdings->{id}, $copy->{copyid}, @fields), "\n"; + select STDOUT; } +sub dump_sample_overview { + my ($c, $s) = @_; + my $tags = $s->{data}{tags}; + my $count = $s->{data}{rcnt}; + + my @tagsbyname = sort keys %{$tags}; + my @tagsbycount = reverse sort { $tags->{$a} <=> $tags->{$b} } keys %{$tags}; + + open SAMPLE, '>', ($c->{prefix} . "-HOLDINGS-OVERVIEW.txt"); + select SAMPLE; + print "SAMPLE REPORT FOR ", $c->{prefix},": $count records\n\n"; + print "FOUND TAGS (BY TAG) FOUND TAGS (BY COUNT)\n"; + print "------------------------ --------------------------\n"; + for my $i (0 .. @tagsbyname - 1) { + print $tagsbyname[$i], (" " x (14 - length $tags->{ $tagsbyname[$i] })), + $tags->{ $tagsbyname[$i] }; + print " (", sprintf("%03d", int($tags->{ $tagsbyname[$i] } / $count * 100)), "%)"; + print " "; + print $tagsbycount[$i], (" " x (16 - length $tags->{ $tagsbycount[$i] })), + $tags->{ $tagsbycount[$i] }; + print " (", sprintf("%03d", int($tags->{ $tagsbycount[$i] } / $count * 100)), "%)\n"; + } + select STDOUT; + print "\n"; + close SAMPLE; +} -#------------------------------------------------ - +#-------------------------- sub initialize { my $c = {}; @@ -116,59 +179,66 @@ sub initialize { binmode(STDIN, ':utf8'); my $rc = GetOptions( $c, - 'fields|f=s', - 'output|o=s', + 'sample|s', + 'samplemap|sm=s', + 'samplestr|ss=s', + 'marcfile|m=s', + 'map=s', + 'holdings|h=i', + 'copyid|c=s', 'prefix|p=s', - 'pubnotes|pub=i', - 'pubnotesfile=s', - 'privnotes|priv=s', - 'privnotesfile=s', - 'tag|t=i', - 'help|h', + 'version|v', + 'help', ); show_help() unless $rc; show_help() if ($c->{help}); - - # set defaults - $c->{prefix} = (defined $c->{prefix}) ? ($c->{prefix} . '.') : ''; - $c->{tag} = $c->{tag} || '852'; - $c->{output} = - $c->{output} || join('', $c->{prefix}, "holdings.pg"); - $c->{pubnotes} = $c->{pubnotes} || 'x'; - $c->{pubnotesfile} = - $c->{pubnotesfile} || join('', $c->{prefix}, "holdings.pubnote.pg"); - $c->{privnotes} = $c->{privnotes} || 'z'; - $c->{privnotesfile} = - $c->{privnotesfile} || join('', $c->{prefix}, "holdings.privnote.pg"); + show_help("Nothing to do!") + unless ($c->{map} or $c->{sample} or $c->{samplemap} or $c->{samplestr}); + show_help("map, holdings, and copyid must be specified together!") + if ($c->{map} and !($c->{holdings} and $c->{copyid})); + show_version() if $c->{version}; my @keys = keys %{$c}; - show_help() unless (@ARGV and @keys); - for my $key ('fields', 'tag') + for my $key ('prefix', 'marcfile') { push @missing, $key unless $c->{$key} } if (@missing) { print "Required option: ", join(', ', @missing), " missing!\n"; show_help(); } - # explode and validate fields string - process_fields($c); return $c; } -sub process_fields { - my ($c) = @_; - my @holdings_fields = split /,/, $c->{fields}; - for (@holdings_fields) { - die "Field names must be alphanumeric!\n" if /\W/; - die "Field names must be single characters!\n" - if /\w{2,}/; - } - $c->{fields} = \@holdings_fields; -} - sub show_help { + my ($msg) = @_; + print "\nERROR - $msg\n" if $msg; print <