X-Git-Url: http://git.equinoxoli.org/?p=migration-tools.git;a=blobdiff_plain;f=extract_holdings;h=028378d4e063d7bc67087e551d20d150952ad6a9;hp=60a39a9cef2c41235f574b0359469d6208cb4303;hb=e2850e245100f5ec46bb4dfa950864e7a44c6731;hpb=ddcd8ddf3892dcc744c055aa5b8e89ee29a8a74a

diff --git a/extract_holdings b/extract_holdings
index 60a39a9..028378d 100755
--- a/extract_holdings
+++ b/extract_holdings
@@ -1,112 +1,246 @@
 #!/usr/bin/perl
+
+# Copyright 2009-2012, Equinox Software, Inc.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
 use strict;
 use warnings;
 
-use XML::Twig;
-use YAML::Tiny;
 use Getopt::Long;
-use Equinox::Migration::SubfieldMapper;
+use Equinox::Migration::MapDrivenMARCXMLProc 1.005;
+use Equinox::Migration::MARCXMLSampler;
 
-$| = 1;
+my $VERSION = '1.001';
+
+=pod
+
+TODO
+
+  * Have detail mode report on number of subfields per datafield
+
+=cut
 
 my $c = initialize();
-my $marcxml = shift;
-my $copyid = 0;
-my $holdings;
-
-my $count = 0;
-my $total = `grep -c '<record' $marcxml`;
-my $percent = 0;
-my $prevper = -1;
-
-my $yaml = YAML::Tiny->new;
-my $t = XML::Twig->new( twig_handlers => { record => \&record } );
-$t->parsefile($marcxml);
-write_sample_fieds();
-
-sub record {
-    my($t, $r)= @_;
-    $holdings = {};
-
-    my @dfields = $r->children('datafield');
-    for my $d (@dfields) {
-        process_datafields($d);
+$| = 1;
+
+# run samples if we've been asked for them
+run_samples($c) if ($c->{sample} or $c->{samplemap} or $c->{samplestr});
+extract_holdings($c) if ($c->{map});
+print "\n";
+
+#--------------------------
+
+sub extract_holdings {
+    my ($c) = @_;
+    print "Parsing records for extraction:\n";
+    my $m = Equinox::Migration::MapDrivenMARCXMLProc->new( marcfile => $c->{marcfile},
+                                                           mapfile  => $c->{map},
+                                                           verbose  => 1,
+                                                         );
+
+    print "Writing holdings to output file(s)...\n";
+    # open main holdings file
+    open HOLDINGS, '>', ($c->{prefix} . "-" . $c->{suffix} . ".pg");
+    # create multi files
+    my $multis = $m->get_multis;
+    my %MULTIFILE = ();
+    for my $t ( keys %{$multis} ) {
+        for my $s ( keys %{$multis->{$t}}) 
+          { open my $fh, ">", ($c->{prefix} . "-" . $c->{suffix} . "-MULT-$t$s.pg"); $MULTIFILE{"$t$s"} = $fh }
     }
-    write_data_out();
-    $r->purge;
 
-    $count++;
-    $percent = int(($count / $total) * 100);
-    print "\r$percent% done ($count)";# if ($percent != $prevper);
-    $prevper = $percent;
-}
+    my $parallel_fields = $m->get_parallel_fields;
 
-sub process_datafields {
-    my ($d) = @_;
-    my $tag = $d->{'att'}->{'tag'};
-
-    if ($tag == 903) {
-        my $s = $d->first_child('subfield');
-        $holdings->{id} = $s->text;;
-    } elsif ($c->{map}->has($tag)) {
-        push @{$holdings->{copies}}, { tag => $tag };
-        my @subs = $d->children('subfield');
-        for my $sub (@subs)
-          { process_subs($tag,$sub) }
-    }
-}
+    my $i = 0; # record counter
+    my $j = 0; # holdings counter
 
-sub process_subs {
-    my ($tag, $sub) = @_;
-    my $code = $sub->{'att'}->{'code'};
+    while (  $m->{data}{recs}[$i] ) {
+        print HOLDINGS "BEGIN;\n\negid, hseq, " unless $j;
+        my $rec = $m->{data}{recs}[$i];
+        my $k = 0; # holding-within-record pointer
 
-    unless ($c->{map}->has($tag, $code)) {
-        # this is a subfield code we don't have mapped. report on it if this is a sample tag
-        push @{$c->{sample}{$tag}}, $code if defined $c->{sample}{tag};
-        return;
-    }
+        for my $holdidx ( @{ $rec->{tmap}{ $c->{holdings} } } ) {
+            # for each holdings tag in the record...
+            my $tagid = $rec->{tags}[$holdidx]{tag};
+            $k++;
 
-    my $copy = $holdings->{copies}[-1];
-    my $field = $c->{map}->field($tag, $code);
-    if ($c->{map}->mod($field) eq 'multi') {
-        my $name = $tag . $code;
-        push @{$copy->{multi}{$name}}, $sub->text;
-    } else {
-        $copy->{uni}{$code} = $sub->text;
-    }
-}
+            my @out = ();            # clear the output buffer
+            push @out, $rec->{egid}; # slug in the egid first thing
+            push @out, $j;           # holding seq goes next
 
+            # grab the unary mappings and slug 'em in
+            for my $sub ( sort keys %{$rec->{tags}[$holdidx]{uni}} ) {
+                push @out, $rec->{tags}[$holdidx]{uni}{$sub};
+                print HOLDINGS "l_", $m->name($tagid, $sub),", " unless $j;
+            }
 
-#------------------------------------------------
+            # handle holdings multis
+            for my $sub ( sort keys %{$multis->{$tagid}} ) {
+                for my $value ( @{$rec->{tags}[$holdidx]{multi}{$sub}} ) {
+                  my $fh = $MULTIFILE{"$tagid$sub"};
+                  my $clean_value = $value;
+                  $clean_value =~ s/[\r\n\t]//g;
+                  print $fh join("\t", $rec->{egid}, $j, $clean_value), "\n";
+              }
+            }
 
-sub write_data_out {
-    my $i = 0;
 
-    for my $copy (@{$holdings->{copies}}) {
-        print HOLDINGS $holdings->{id}, "\t$i\t", $copy->{tag};
-        for ( sort keys %{ $c->{map}{fields} } ) {
-            next if ($c->{map}->mod($_) =~ /^bib/);
-            if (defined $copy->{uni}{ $c->{map}{fields}{$_}{sub} }) {
-                print HOLDINGS "\t", $copy->{uni}{ $c->{map}{fields}{$_}{sub} };
-            } else {
-                print HOLDINGS "\t";
+            # now get everything else in the mapping
+            for my $othertag ( sort keys %{$rec->{tmap}} ) {
+                next if $othertag eq $c->{holdings};  # ignoring the holdings, o'course
+                my $test_idx = $rec->{tmap}{$othertag}[0]; # get index into tags struct
+                unless (defined $test_idx) {
+                    push @out, '';
+                    next;
+                }
+
+                # handle parallel fields
+                if (exists($parallel_fields->{$othertag})) {
+                    my $num_fields = $#{ $rec->{tmap}{$othertag} };
+                    my $tag_idx;
+                    if ($holdidx > $num_fields) {
+                        $tag_idx = -1;
+                    } else {
+                        $tag_idx = $rec->{tmap}{$othertag}[$holdidx];
+                    }
+                    for my $sub ( sort keys %{ $parallel_fields->{$othertag } } ) {
+                        push @out, $tag_idx > -1 ? $rec->{tags}[$tag_idx]{parallel}{$sub}->[0] : '';
+                        print HOLDINGS "l_", $m->name($rec->{tags}[$tag_idx]{tag}, $sub), ", " unless $j;
+                    }
+                }
+
+                # handle only first other tag unless it is known to be multi
+                my $limit = 0;
+                if (exists($multis->{$othertag})) {
+                    $limit = $#{ $rec->{tmap}{$othertag} };
+                }
+                foreach my $idx (0..$limit) {
+                    my $tag_idx = $rec->{tmap}{$othertag}[$idx];
+                    for my $sub ( sort keys %{$rec->{tags}[$tag_idx]{uni}} ) {
+                        if ($m->first_only($rec->{tags}[$tag_idx]{tag}, $sub)) {
+                            push @out, ($k == 1) ? $rec->{tags}[$tag_idx]{uni}{$sub} : '';
+                        } else {
+                            push @out, $rec->{tags}[$tag_idx]{uni}{$sub};
+                        }
+                        print HOLDINGS "l_", $m->name($rec->{tags}[$tag_idx]{tag}, $sub), ", " unless $j;
+                    }
+                    next unless exists($multis->{$othertag});
+                    for my $sub ( sort keys %{$multis->{$othertag}} ) {
+                        next if $m->first_only($rec->{tags}[$tag_idx]{tag}, $sub) and ($k > 1);
+                        for my $value ( @{$rec->{tags}[$tag_idx]{multi}{$sub}} ) {
+                            my $fh = $MULTIFILE{"$othertag$sub"};
+                            my $clean_value = $value;
+                            $clean_value =~ s/[\r\n\t]//g;
+                            print $fh normalize_output(join("\t", $rec->{egid}, $j, $clean_value)), "\n";
+                        }
+                    }
+                }
             }
-        }
-        print HOLDINGS "\n";
 
-        for my $m (sort keys %{$copy->{multi}}) {
-            my $fh = $c->{files}{multi}{$m};
-            print $fh join("\t", $holdings->{id}, $i, @{$copy->{multi}{$m}}), "\n";
+            # and dump it
+            print HOLDINGS "\n" unless $j;
+            print HOLDINGS normalize_output(join("\t", @out));
+            print HOLDINGS "\n";
+            $j++;
         }
         $i++;
+        print "\r$i $j";
+    }
+    print "\n";
+}
+
+#--------------------------
+
+sub run_samples {
+    my ($c) = @_;
+    my $s;
+    print "Parsing records for sampling... ";
+    if ($c->{samplemap}) {
+        $s = Equinox::Migration::MARCXMLSampler->new( marcfile => $c->{marcfile},
+                                                      mapfile  => $c->{samplemap});
+    } elsif ($c->{samplestr}) {
+        $s = Equinox::Migration::MARCXMLSampler->new( marcfile  => $c->{marcfile},
+                                                      mapstring => $c->{samplestr});
+    } else {
+        $s = Equinox::Migration::MARCXMLSampler->new( marcfile => $c->{marcfile} );
     }
+
+    dump_sample_overview($c, $s) if $c->{sample};
+    dump_sample_detail($c, $s) if ($c->{samplemap} or $c->{samplestr});
 }
 
-sub write_sample_fields {
+sub dump_sample_detail {
+    my ($c, $s) = @_;
+    my $tags = $s->{data}{samp};
+    my $count = $s->{data}{rcnt};
+    my $scnt  = $s->{data}{scnt};
+
+    open DETAIL, '>', ($c->{prefix} . "-" . $c->{suffix} . "-DETAIL.txt");
+    select DETAIL;
+    for my $tag (sort keys %{ $tags }) {
+        print ">>>>> TAG $tag\n\n";
+        for my $subkey (sort keys %{ $tags->{$tag} }) {
+            my $sub = $tags->{$tag}{$subkey};
+            print "|| $subkey | ", $sub->{value}, " | ", 
+              $sub->{count}, "/", $sub->{tcnt}, " | ", ($sub->{count} > $sub->{tcnt}) ? "MULTI" : "", " ||\n";
+        }
+        print "\n";
+    }
+    close DETAIL;
+    open SCOUNT, '>', ($c->{prefix} . "-" . $c->{suffix} . "-SUBCOUNTS.txt");
+    select SCOUNT;
+    for my $tag (sort keys %{ $scnt }) {
+        print ">>>>> TAG $tag\n\n";
+        for my $len (sort keys %{ $scnt->{$tag} }) 
+          { print "|| $len | ", $scnt->{$tag}{$len}, " ||\n" }
+        print "\n";
+    }
+    select STDOUT;
+    print "Saved results as ", ($c->{prefix} . "-" . $c->{suffix} . "-DETAIL.txt"), " and ",
+        ($c->{prefix} . "-" . $c->{suffix} . "-SUBCOUNTS.txt"), "\n";
 }
 
-#------------------------------------------------
+sub dump_sample_overview {
+    my ($c, $s) = @_;
+    my $tags = $s->{data}{tags};
+    my $count = $s->{data}{rcnt};
+
+    my @tagsbyname  = sort keys %{$tags};
+    my @tagsbycount = reverse sort { $tags->{$a} <=> $tags->{$b} } keys %{$tags};
+
+    open SAMPLE, '>', ($c->{prefix} . "-" . $c->{suffix} . "-OVERVIEW.txt");
+    select SAMPLE;
+    print "SAMPLE REPORT FOR ", $c->{prefix},": $count records\n\n";
+    print "FOUND TAGS (BY TAG)           FOUND TAGS (BY COUNT)\n";
+    print "------------------------      --------------------------\n";
+    for my $i (0 .. @tagsbyname - 1) {
+        print $tagsbyname[$i], (" " x (14 - length $tags->{ $tagsbyname[$i] })),
+          $tags->{ $tagsbyname[$i] };
+        print " (", sprintf("%03d", int($tags->{ $tagsbyname[$i] } / $count * 100)), "%)";
+        print "      ";
+        print $tagsbycount[$i], (" " x (16 - length $tags->{ $tagsbycount[$i] })),
+          $tags->{ $tagsbycount[$i] };
+        print " (", sprintf("%03d", int($tags->{ $tagsbycount[$i] } / $count * 100)), "%)\n";
+    }
+    select STDOUT;
+    print "Saved results as ", ($c->{prefix} . "-" . $c->{suffix} . "-OVERVIEW.txt"), "\n";
+    close SAMPLE;
+}
 
+#--------------------------
 
 sub initialize {
     my $c = {};
@@ -116,75 +250,87 @@ sub initialize {
     binmode(STDIN, ':utf8');
 
     my $rc = GetOptions( $c,
-                         'sample|s=s',
+                         'sample|s',
+                         'samplemap|sm=s',
+                         'samplestr|ss=s',
+                         'marcfile=s',
                          'map|m=s',
-                         'ils=s',
-                         'library=s',
+                         'holdings|h=i',
+                         'copyid|c=s',
                          'prefix|p=s',
-                         'help|h',
+                         'suffix|o=s',
+                         'disable-pg-normalization',
+                         'version|v',
+                         'help',
                        );
     show_help() unless $rc;
     show_help() if ($c->{help});
+    show_help("Nothing to do!")
+      unless ($c->{map} or $c->{sample} or $c->{samplemap} or $c->{samplestr});
+    show_help("map, holdings, and copyid must be specified together!")
+      if ($c->{map} and !($c->{holdings} and $c->{copyid}));
+    show_version() if $c->{version};
+
+    if (! $c->{suffix}) {
+        $c->{suffix} = 'HOLDINGS';
+    }
+    if ($c->{prefix} and !$c->{marcfile}) {
+        $c->{marcfile} = $c->{prefix} . ".clean.marc.xml";
+    }
 
     my @keys = keys %{$c};
-    show_help() unless (@ARGV and @keys);
-    for my $key ('prefix', 'map', 'ils')
+    for my $key ('prefix', 'marcfile')
       { push @missing, $key unless $c->{$key} }
     if (@missing) {
         print "Required option: ", join(', ', @missing), " missing!\n";
         show_help();
     }
 
-    # generate subfield map
-    $c->{map} = Equinox::Migration::SubfieldMapper->new( file => $c->{map} );
-
-    # explode sample tags string
-    if (defined $c->{sample}) {
-        my $sample = $c->{sample};
-        $c->{sample} = {};
-        for (split /,/, $c->{sample}) {
-            $c->{sample}{$_} = [];
-        }
-    }
-
-    # open required files
-    open HOLDINGS, '>', ($c->{prefix} . ".holdings.pg");
-    for my $f (keys %{$c->{map}{fields}}) {
-        if ($c->{map}->mod($f)) {
-            next if ($c->{map}->mod($f) =~ /bib/);
-            open my $mfh, '>', join('.', $c->{prefix}, "holdings",
-                                    $c->{map}{fields}{$f}{tag},
-                                    $c->{map}{fields}{$f}{sub}, "pg");
-            $c->{files}{multi}{ ($c->{map}{fields}{$f}{tag} . $c->{map}{fields}{$f}{sub}) }
-              = $mfh;
-        }
-    }
-
-    # print file headers
-    print HOLDINGS "BEGIN;\n";
-    print HOLDINGS "CREATE TABLE ", $c->{prefix}, ".asset_copy_", $c->{ils};
-    print HOLDINGS $c->{library} if (defined $c->{library});
-    print HOLDINGS " (eg_bib_id INTEGER, eg_copy_id INTEGER, l_tag INTEGER";
-    for ( sort keys %{ $c->{map}{fields} } ) {
-        next if ($c->{map}->mod($_) =~ /bib/);
-        print HOLDINGS ", l_", $_, " TEXT";
-    }
-    print HOLDINGS ") INHERITS (", $c->{prefix}, ".asset_copy);\n";
-    print HOLDINGS "COPY ", $c->{prefix}, ".asset_copy_", $c->{ils};
-    print HOLDINGS $c->{library} if (defined $c->{library});
-    print HOLDINGS " (eg_bib_id, eg_copy_id, l_tag";
-    for ( sort keys %{ $c->{map}{fields} } ) {
-        print HOLDINGS ", l_", $_;
-    }
-    print HOLDINGS ") FROM STDIN;\n";
-
-
     return $c;
 }
 
+sub normalize_output {
+    my $str = shift;
+    $str =~ s!\\!\\\\!g unless $c->{'disable-pg-normalization'};
+    return $str;
+}
+
 sub show_help {
+    my ($msg) = @_;
+    print "\nERROR - $msg\n" if $msg;
     print <<HELP;
-Usage is: ...
+
+Usage is: extract_holdings --prefix PREFIX --marcfile MARCFILE [ARGUMENTS]
+
+REQUIRED ARGUMENTS
+  --prefix   -p  Prefix string for output filenames
+  --suffix   -o  Suffix string for output filenames (defaults to HOLDINGS)
+  --marcfile     MARCXML to use as source data
+                 Defaults to 'PREFIX.clean.marc.xml'
+
+SAMPLING ARGUMENTS
+  --sample    -s   Generate a report of all tags in the MARC data
+  --samplemap -sm  Specify a Equinox::Migration::SimpleTagList map file which
+                   will be used to generate subfield breakdown reports about
+                   specific tags in the MARC data
+  --samplestr -ss  As above, but with a one-liner map specified on the command
+                   line as a string (e.g. '-ss "852 999"')
+
+  If --samplemap and --samplestr are both specified, --samplemap wins.
+
+HOLDINGS EXTRACTION ARGUMENTS
+  --map      -m  Equinox::Migration::SubfieldMapper map file which will be
+                 used to extract holdings data from the input MARC file
+  --holdings -h  Specifies actual holdings tag
+  --copyid   -c  Specifies subfield of holdings with unique copy identifier
+  --disable-pg-normalization  By default, output is normalized so that a Postgres
+                              copy or \\copy can import the data without choking on
+                              backslashes; use this command-line option if
+                              output is not meant to be consumed by psql.
+
+  All three of these must be given together.
 HELP
     exit;
 }
+
+sub show_version { print "extract_holdings v$VERSION\n"; exit }