X-Git-Url: http://git.equinoxoli.org/?p=migration-tools.git;a=blobdiff_plain;f=extract_holdings;h=028378d4e063d7bc67087e551d20d150952ad6a9;hp=eea04652c9729e0dc18942ab1254a077e031460b;hb=8f3c70aa5aaeb1c4742c4d15da7fc535035b3a2e;hpb=6c25f0626b026e59b9051c31d3eddfb6105f3c56

diff --git a/extract_holdings b/extract_holdings
index eea0465..028378d 100755
--- a/extract_holdings
+++ b/extract_holdings
@@ -1,112 +1,246 @@
 #!/usr/bin/perl
+
+# Copyright 2009-2012, Equinox Software, Inc.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
 use strict;
 use warnings;
 
-use XML::Twig;
-use YAML::Tiny;
 use Getopt::Long;
+use Equinox::Migration::MapDrivenMARCXMLProc 1.005;
+use Equinox::Migration::MARCXMLSampler;
+
+my $VERSION = '1.001';
 
+=pod
 
-my $conf = initialize();
-my $marcxml = shift;
+TODO
 
-open HOLDINGS, '>', $conf->{output};
-open X, '>', $conf->{pubnotesfile};
-open Z, '>', $conf->{privnotesfile};
-my $holdings = {};
-my %sample = ( x => {}, z => {} ); # hash of all subfields in all 852s
-my $copyid = 0;
+  * Have detail mode report on number of subfields per datafield
 
+=cut
 
+my $c = initialize();
 $| = 1;
-my $count = 0;
-my $total = `grep -c '<record' $marcxml`;
-my $percent = 0;
-my $prevper = -1;
-
-my $yaml = YAML::Tiny->new;
-my $t = XML::Twig->new( twig_handlers => { record => \&record } );
-$t->parsefile($marcxml);
-$yaml->[0] = \%sample;
-$yaml->write('holdings.sample');
-print "\n\n";
-
-sub record {
-    my($t, $r)= @_;
-    $holdings = { copies => [] };
-
-    my @dfields = $r->children('datafield');
-    for my $d (@dfields) 
-      { process_datafields($d) }
-
-    for my $copy (@{$holdings->{copies}})
-      { print_reports($copy) }
-    $r->purge;
-
-    $count++;
-    $percent = int(($count / $total) * 100);
-    print "\r$percent% done ($count)" if ($percent != $prevper);
-    $prevper = $percent;
-}
 
-sub process_datafields {
-    my ($d) = @_;
-    # get 903
-    if ($d->{'att'}->{'tag'} == 903) {
-        my $s = $d->first_child('subfield');
-        $holdings->{id} = $s->text;;
+# run samples if we've been asked for them
+run_samples($c) if ($c->{sample} or $c->{samplemap} or $c->{samplestr});
+extract_holdings($c) if ($c->{map});
+print "\n";
+
+#--------------------------
+
+sub extract_holdings {
+    my ($c) = @_;
+    print "Parsing records for extraction:\n";
+    my $m = Equinox::Migration::MapDrivenMARCXMLProc->new( marcfile => $c->{marcfile},
+                                                           mapfile  => $c->{map},
+                                                           verbose  => 1,
+                                                         );
+
+    print "Writing holdings to output file(s)...\n";
+    # open main holdings file
+    open HOLDINGS, '>', ($c->{prefix} . "-" . $c->{suffix} . ".pg");
+    # create multi files
+    my $multis = $m->get_multis;
+    my %MULTIFILE = ();
+    for my $t ( keys %{$multis} ) {
+        for my $s ( keys %{$multis->{$t}}) 
+          { open my $fh, ">", ($c->{prefix} . "-" . $c->{suffix} . "-MULT-$t$s.pg"); $MULTIFILE{"$t$s"} = $fh }
     }
 
-    # and holdings data
-    if ($d->{'att'}->{'tag'} == $conf->{tag}) {
-        push @{$holdings->{copies}}, { x =>[], z => [] };
-        $holdings->{copies}[-1]{copyid} = $copyid;
-        my @subs = $d->children('subfield');
-        for my $s (@subs)
-          { process_subs($s) }
-        $copyid++;
+    my $parallel_fields = $m->get_parallel_fields;
+
+    my $i = 0; # record counter
+    my $j = 0; # holdings counter
+
+    while (  $m->{data}{recs}[$i] ) {
+        print HOLDINGS "BEGIN;\n\negid, hseq, " unless $j;
+        my $rec = $m->{data}{recs}[$i];
+        my $k = 0; # holding-within-record pointer
+
+        for my $holdidx ( @{ $rec->{tmap}{ $c->{holdings} } } ) {
+            # for each holdings tag in the record...
+            my $tagid = $rec->{tags}[$holdidx]{tag};
+            $k++;
+
+            my @out = ();            # clear the output buffer
+            push @out, $rec->{egid}; # slug in the egid first thing
+            push @out, $j;           # holding seq goes next
+
+            # grab the unary mappings and slug 'em in
+            for my $sub ( sort keys %{$rec->{tags}[$holdidx]{uni}} ) {
+                push @out, $rec->{tags}[$holdidx]{uni}{$sub};
+                print HOLDINGS "l_", $m->name($tagid, $sub),", " unless $j;
+            }
+
+            # handle holdings multis
+            for my $sub ( sort keys %{$multis->{$tagid}} ) {
+                for my $value ( @{$rec->{tags}[$holdidx]{multi}{$sub}} ) {
+                  my $fh = $MULTIFILE{"$tagid$sub"};
+                  my $clean_value = $value;
+                  $clean_value =~ s/[\r\n\t]//g;
+                  print $fh join("\t", $rec->{egid}, $j, $clean_value), "\n";
+              }
+            }
+
+
+            # now get everything else in the mapping
+            for my $othertag ( sort keys %{$rec->{tmap}} ) {
+                next if $othertag eq $c->{holdings};  # ignoring the holdings, o'course
+                my $test_idx = $rec->{tmap}{$othertag}[0]; # get index into tags struct
+                unless (defined $test_idx) {
+                    push @out, '';
+                    next;
+                }
+
+                # handle parallel fields
+                if (exists($parallel_fields->{$othertag})) {
+                    my $num_fields = $#{ $rec->{tmap}{$othertag} };
+                    my $tag_idx;
+                    if ($holdidx > $num_fields) {
+                        $tag_idx = -1;
+                    } else {
+                        $tag_idx = $rec->{tmap}{$othertag}[$holdidx];
+                    }
+                    for my $sub ( sort keys %{ $parallel_fields->{$othertag } } ) {
+                        push @out, $tag_idx > -1 ? $rec->{tags}[$tag_idx]{parallel}{$sub}->[0] : '';
+                        print HOLDINGS "l_", $m->name($rec->{tags}[$tag_idx]{tag}, $sub), ", " unless $j;
+                    }
+                }
+
+                # handle only first other tag unless it is known to be multi
+                my $limit = 0;
+                if (exists($multis->{$othertag})) {
+                    $limit = $#{ $rec->{tmap}{$othertag} };
+                }
+                foreach my $idx (0..$limit) {
+                    my $tag_idx = $rec->{tmap}{$othertag}[$idx];
+                    for my $sub ( sort keys %{$rec->{tags}[$tag_idx]{uni}} ) {
+                        if ($m->first_only($rec->{tags}[$tag_idx]{tag}, $sub)) {
+                            push @out, ($k == 1) ? $rec->{tags}[$tag_idx]{uni}{$sub} : '';
+                        } else {
+                            push @out, $rec->{tags}[$tag_idx]{uni}{$sub};
+                        }
+                        print HOLDINGS "l_", $m->name($rec->{tags}[$tag_idx]{tag}, $sub), ", " unless $j;
+                    }
+                    next unless exists($multis->{$othertag});
+                    for my $sub ( sort keys %{$multis->{$othertag}} ) {
+                        next if $m->first_only($rec->{tags}[$tag_idx]{tag}, $sub) and ($k > 1);
+                        for my $value ( @{$rec->{tags}[$tag_idx]{multi}{$sub}} ) {
+                            my $fh = $MULTIFILE{"$othertag$sub"};
+                            my $clean_value = $value;
+                            $clean_value =~ s/[\r\n\t]//g;
+                            print $fh normalize_output(join("\t", $rec->{egid}, $j, $clean_value)), "\n";
+                        }
+                    }
+                }
+            }
+
+            # and dump it
+            print HOLDINGS "\n" unless $j;
+            print HOLDINGS normalize_output(join("\t", @out));
+            print HOLDINGS "\n";
+            $j++;
+        }
+        $i++;
+        print "\r$i $j";
     }
+    print "\n";
 }
 
-sub process_subs {
-    my ($s) = @_;
-    my $copy = $holdings->{copies}[-1];
+#--------------------------
 
-    my $code = $s->{'att'}->{'code'};
-    my $value = $s->text;
-
-    if ($code eq $conf->{pubnotes} or $code eq $conf->{privnotes}) {
-        push @{$copy->{$code}}, $value;
-        my ($k,$v) = split /:/, $value;
-        $sample{$code}{$k} = $v;
+sub run_samples {
+    my ($c) = @_;
+    my $s;
+    print "Parsing records for sampling... ";
+    if ($c->{samplemap}) {
+        $s = Equinox::Migration::MARCXMLSampler->new( marcfile => $c->{marcfile},
+                                                      mapfile  => $c->{samplemap});
+    } elsif ($c->{samplestr}) {
+        $s = Equinox::Migration::MARCXMLSampler->new( marcfile  => $c->{marcfile},
+                                                      mapstring => $c->{samplestr});
     } else {
-        $copy->{$code} = $value;
-        $sample{$code} = $value;
+        $s = Equinox::Migration::MARCXMLSampler->new( marcfile => $c->{marcfile} );
     }
+
+    dump_sample_overview($c, $s) if $c->{sample};
+    dump_sample_detail($c, $s) if ($c->{samplemap} or $c->{samplestr});
 }
 
-sub print_reports {
-    return unless defined $holdings->{id};
-    my ($copy) = @_;
-    my $note = 0;
-    for (@{$copy->{x}}) {
-        print X join("\t", $holdings->{id}, $copy->{copyid}, $note, $_), "\n";
-        $note++;
+sub dump_sample_detail {
+    my ($c, $s) = @_;
+    my $tags = $s->{data}{samp};
+    my $count = $s->{data}{rcnt};
+    my $scnt  = $s->{data}{scnt};
+
+    open DETAIL, '>', ($c->{prefix} . "-" . $c->{suffix} . "-DETAIL.txt");
+    select DETAIL;
+    for my $tag (sort keys %{ $tags }) {
+        print ">>>>> TAG $tag\n\n";
+        for my $subkey (sort keys %{ $tags->{$tag} }) {
+            my $sub = $tags->{$tag}{$subkey};
+            print "|| $subkey | ", $sub->{value}, " | ", 
+              $sub->{count}, "/", $sub->{tcnt}, " | ", ($sub->{count} > $sub->{tcnt}) ? "MULTI" : "", " ||\n";
+        }
+        print "\n";
     }
-    $note = 0;
-    for (@{$copy->{z}}) {
-        print Z join("\t", $holdings->{id}, $copy->{copyid}, $note, $_), "\n";
-        $note++;
+    close DETAIL;
+    open SCOUNT, '>', ($c->{prefix} . "-" . $c->{suffix} . "-SUBCOUNTS.txt");
+    select SCOUNT;
+    for my $tag (sort keys %{ $scnt }) {
+        print ">>>>> TAG $tag\n\n";
+        for my $len (sort keys %{ $scnt->{$tag} }) 
+          { print "|| $len | ", $scnt->{$tag}{$len}, " ||\n" }
+        print "\n";
     }
-    my @fields = ();
-    for ( @{$conf->{fields}} )
-      { $copy->{$_} = '' unless defined $copy->{$_}; push @fields, $copy->{$_} }
-    print HOLDINGS join("\t", $holdings->{id}, $copy->{copyid}, @fields), "\n";
+    select STDOUT;
+    print "Saved results as ", ($c->{prefix} . "-" . $c->{suffix} . "-DETAIL.txt"), " and ",
+        ($c->{prefix} . "-" . $c->{suffix} . "-SUBCOUNTS.txt"), "\n";
 }
 
+sub dump_sample_overview {
+    my ($c, $s) = @_;
+    my $tags = $s->{data}{tags};
+    my $count = $s->{data}{rcnt};
 
-#------------------------------------------------
+    my @tagsbyname  = sort keys %{$tags};
+    my @tagsbycount = reverse sort { $tags->{$a} <=> $tags->{$b} } keys %{$tags};
 
+    open SAMPLE, '>', ($c->{prefix} . "-" . $c->{suffix} . "-OVERVIEW.txt");
+    select SAMPLE;
+    print "SAMPLE REPORT FOR ", $c->{prefix},": $count records\n\n";
+    print "FOUND TAGS (BY TAG)           FOUND TAGS (BY COUNT)\n";
+    print "------------------------      --------------------------\n";
+    for my $i (0 .. @tagsbyname - 1) {
+        print $tagsbyname[$i], (" " x (14 - length $tags->{ $tagsbyname[$i] })),
+          $tags->{ $tagsbyname[$i] };
+        print " (", sprintf("%03d", int($tags->{ $tagsbyname[$i] } / $count * 100)), "%)";
+        print "      ";
+        print $tagsbycount[$i], (" " x (16 - length $tags->{ $tagsbycount[$i] })),
+          $tags->{ $tagsbycount[$i] };
+        print " (", sprintf("%03d", int($tags->{ $tagsbycount[$i] } / $count * 100)), "%)\n";
+    }
+    select STDOUT;
+    print "Saved results as ", ($c->{prefix} . "-" . $c->{suffix} . "-OVERVIEW.txt"), "\n";
+    close SAMPLE;
+}
+
+#--------------------------
 
 sub initialize {
     my $c = {};
@@ -116,59 +250,87 @@ sub initialize {
     binmode(STDIN, ':utf8');
 
     my $rc = GetOptions( $c,
-                         'fields|f=s',
-                         'output|o=s',
+                         'sample|s',
+                         'samplemap|sm=s',
+                         'samplestr|ss=s',
+                         'marcfile=s',
+                         'map|m=s',
+                         'holdings|h=i',
+                         'copyid|c=s',
                          'prefix|p=s',
-                         'pubnotes|pub=i',
-                         'pubnotesfile=s',
-                         'privnotes|priv=s',
-                         'privnotesfile=s',
-                         'tag|t=i',
-                         'help|h',
+                         'suffix|o=s',
+                         'disable-pg-normalization',
+                         'version|v',
+                         'help',
                        );
     show_help() unless $rc;
     show_help() if ($c->{help});
+    show_help("Nothing to do!")
+      unless ($c->{map} or $c->{sample} or $c->{samplemap} or $c->{samplestr});
+    show_help("map, holdings, and copyid must be specified together!")
+      if ($c->{map} and !($c->{holdings} and $c->{copyid}));
+    show_version() if $c->{version};
 
-    # set defaults
-    $c->{prefix} = (defined $c->{prefix}) ? ($c->{prefix} . '.') : '';
-    $c->{tag} = $c->{tag} || '852';
-    $c->{output} =
-      $c->{output} || join('', $c->{prefix}, "holdings.pg");
-    $c->{pubnotes} = $c->{pubnotes} || 'x';
-    $c->{pubnotesfile} =
-      $c->{pubnotesfile} || join('', $c->{prefix}, "holdings.pubnote.pg");
-    $c->{privnotes} = $c->{privnotes} || 'z';
-    $c->{privnotesfile} =
-      $c->{privnotesfile} || join('', $c->{prefix}, "holdings.privnote.pg");
+    if (! $c->{suffix}) {
+        $c->{suffix} = 'HOLDINGS';
+    }
+    if ($c->{prefix} and !$c->{marcfile}) {
+        $c->{marcfile} = $c->{prefix} . ".clean.marc.xml";
+    }
 
     my @keys = keys %{$c};
-    show_help() unless (@ARGV and @keys);
-    for my $key ('fields', 'tag')
+    for my $key ('prefix', 'marcfile')
       { push @missing, $key unless $c->{$key} }
     if (@missing) {
         print "Required option: ", join(', ', @missing), " missing!\n";
         show_help();
     }
 
-    # explode and validate fields string
-    process_fields($c);
     return $c;
 }
 
-sub process_fields {
-    my ($c) = @_;
-    my @holdings_fields = split /,/, $c->{fields};
-    for (@holdings_fields) { 
-        die "Field names must be alphanumeric!\n" if /\W/;
-        die "Field names must be single characters!\n"
-          if /\w{2,}/;
-    }
-    $c->{fields} = \@holdings_fields;
+sub normalize_output {
+    my $str = shift;
+    $str =~ s!\\!\\\\!g unless $c->{'disable-pg-normalization'};
+    return $str;
 }
 
 sub show_help {
+    my ($msg) = @_;
+    print "\nERROR - $msg\n" if $msg;
     print <<HELP;
-Usage is: extract_holdings MARCXML_FILE HOLDINGS_TAG
+
+Usage is: extract_holdings --prefix PREFIX --marcfile MARCFILE [ARGUMENTS]
+
+REQUIRED ARGUMENTS
+  --prefix   -p  Prefix string for output filenames
+  --suffix   -o  Suffix string for output filenames (defaults to HOLDINGS)
+  --marcfile     MARCXML to use as source data
+                 Defaults to 'PREFIX.clean.marc.xml'
+
+SAMPLING ARGUMENTS
+  --sample    -s   Generate a report of all tags in the MARC data
+  --samplemap -sm  Specify a Equinox::Migration::SimpleTagList map file which
+                   will be used to generate subfield breakdown reports about
+                   specific tags in the MARC data
+  --samplestr -ss  As above, but with a one-liner map specified on the command
+                   line as a string (e.g. '-ss "852 999"')
+
+  If --samplemap and --samplestr are both specified, --samplemap wins.
+
+HOLDINGS EXTRACTION ARGUMENTS
+  --map      -m  Equinox::Migration::SubfieldMapper map file which will be
+                 used to extract holdings data from the input MARC file
+  --holdings -h  Specifies actual holdings tag
+  --copyid   -c  Specifies subfield of holdings with unique copy identifier
+  --disable-pg-normalization  By default, output is normalized so that a Postgres
+                              copy or \\copy can import the data without choking on
+                              backslashes; use this command-line option if
+                              output is not meant to be consumed by psql.
+
+  All three of these must be given together.
 HELP
     exit;
 }
+
+sub show_version { print "extract_holdings v$VERSION\n"; exit }