X-Git-Url: http://git.equinoxoli.org/?p=migration-tools.git;a=blobdiff_plain;f=extract_holdings;h=03e68351de7acbf50057cc29a297fbb5905d270c;hp=eea04652c9729e0dc18942ab1254a077e031460b;hb=5d2e4c4541d612bafbb5a7bf1ed48a3b655b526c;hpb=6c25f0626b026e59b9051c31d3eddfb6105f3c56

diff --git a/extract_holdings b/extract_holdings
index eea0465..03e6835 100755
--- a/extract_holdings
+++ b/extract_holdings
@@ -2,111 +2,140 @@
 use strict;
 use warnings;
 
-use XML::Twig;
-use YAML::Tiny;
 use Getopt::Long;
+use Equinox::Migration::MapDrivenMARCXMLProc;
+use Equinox::Migration::MARCXMLSampler;
 
+my $VERSION = '1.000';
 
-my $conf = initialize();
-my $marcxml = shift;
-
-open HOLDINGS, '>', $conf->{output};
-open X, '>', $conf->{pubnotesfile};
-open Z, '>', $conf->{privnotesfile};
-my $holdings = {};
-my %sample = ( x => {}, z => {} ); # hash of all subfields in all 852s
-my $copyid = 0;
+my $c = initialize();
+$| = 1;
 
+# run samples if we've been asked for them
+run_samples($c) if ($c->{sample} or $c->{samplemap} or $c->{samplestr});
+extract_holdings($c) if ($c->{map});
 
-$| = 1;
-my $count = 0;
-my $total = `grep -c '<record' $marcxml`;
-my $percent = 0;
-my $prevper = -1;
-
-my $yaml = YAML::Tiny->new;
-my $t = XML::Twig->new( twig_handlers => { record => \&record } );
-$t->parsefile($marcxml);
-$yaml->[0] = \%sample;
-$yaml->write('holdings.sample');
-print "\n\n";
-
-sub record {
-    my($t, $r)= @_;
-    $holdings = { copies => [] };
-
-    my @dfields = $r->children('datafield');
-    for my $d (@dfields) 
-      { process_datafields($d) }
-
-    for my $copy (@{$holdings->{copies}})
-      { print_reports($copy) }
-    $r->purge;
-
-    $count++;
-    $percent = int(($count / $total) * 100);
-    print "\r$percent% done ($count)" if ($percent != $prevper);
-    $prevper = $percent;
-}
+#--------------------------
 
-sub process_datafields {
-    my ($d) = @_;
-    # get 903
-    if ($d->{'att'}->{'tag'} == 903) {
-        my $s = $d->first_child('subfield');
-        $holdings->{id} = $s->text;;
-    }
+sub extract_holdings {
+    my ($c) = @_;
+    print "Parsing records for extraction... ";
+    my $m = Equinox::Migration::MapDrivenMARCXMLProc->new( marcfile => $c->{marcfile},
+                                                           mapfile  => $c->{map} );
+    # open main holdings file
+    open HOLDINGS, '>', ($c->{prefix} . "-HOLDINGS.pg");
+    # open the files for multi mappings
+    # FIXME DO THIS
+    select HOLDINGS;
+
+    while (my $rec = $m->parse_record) {
+        # for each holdings tag in the record...
+        for my $holdidx ( @{$rec->{tmap}{ $c->{holdings} }} ) {
+            print STDOUT "\r", $m->recno;
+            my @out = ();            # clear the output buffer
+            push @out, $rec->{egid}; # slug in the egid first thing
+            print "BEGIN;\n\negid\t" if ($m->recno == 1);
+
+            # grab the unary mappings and slug 'em in
+            for my $sub ( sort keys %{$rec->{tags}[$holdidx]{uni}} ) {
+                push @out, $rec->{tags}[$holdidx]{uni}{$sub};
+                print "l_", $m->name($rec->{tags}[$holdidx]{tag}, $sub),"\t"
+                  if ($m->recno == 1);
+            }
+
+            # now get everything else in the mapping
+            for my $othertag ( sort keys %{$rec->{tmap}} ) {
+                next if $othertag eq $c->{holdings};  # ignoring the holdings, o'course
+                my $idx = $rec->{tmap}{$othertag}[0]; # get index into tags struct
+                for my $sub ( sort keys %{$rec->{tags}[$idx]{uni}} ) {
+                    push @out, $rec->{tags}[$idx]{uni}{$sub};
+                    print "l_", $m->name($rec->{tags}[$idx]{tag}, $sub), "\t"
+                      if ($m->recno == 1);
+                }
+            }
+
+            # and dump it
+            print "\n" if ($m->recno == 1);
+            print join("\t", @out);
+            print "\n";
+        }
 
-    # and holdings data
-    if ($d->{'att'}->{'tag'} == $conf->{tag}) {
-        push @{$holdings->{copies}}, { x =>[], z => [] };
-        $holdings->{copies}[-1]{copyid} = $copyid;
-        my @subs = $d->children('subfield');
-        for my $s (@subs)
-          { process_subs($s) }
-        $copyid++;
     }
+    select STDOUT;
+    print "\n";
 }
 
-sub process_subs {
-    my ($s) = @_;
-    my $copy = $holdings->{copies}[-1];
-
-    my $code = $s->{'att'}->{'code'};
-    my $value = $s->text;
+#--------------------------
 
-    if ($code eq $conf->{pubnotes} or $code eq $conf->{privnotes}) {
-        push @{$copy->{$code}}, $value;
-        my ($k,$v) = split /:/, $value;
-        $sample{$code}{$k} = $v;
+sub run_samples {
+    my ($c) = @_;
+    my $s;
+    if ($c->{samplemap}) {
+        $s = Equinox::Migration::MARCXMLSampler->new( marcfile => $c->{marcfile},
+                                                      mapfile  => $c->{samplemap});
+    } elsif ($c->{samplestr}) {
+        $s = Equinox::Migration::MARCXMLSampler->new( marcfile  => $c->{marcfile},
+                                                      mapstring => $c->{samplestr});
     } else {
-        $copy->{$code} = $value;
-        $sample{$code} = $value;
+        $s = Equinox::Migration::MARCXMLSampler->new( marcfile => $c->{marcfile} );
     }
+    print "Parsing records for sampling... ";
+    $s->parse_records;
+
+    dump_sample_overview($c, $s) if $c->{sample};
+    dump_sample_detail($c, $s) if ($c->{samplemap} or $c->{samplestr});
 }
 
-sub print_reports {
-    return unless defined $holdings->{id};
-    my ($copy) = @_;
-    my $note = 0;
-    for (@{$copy->{x}}) {
-        print X join("\t", $holdings->{id}, $copy->{copyid}, $note, $_), "\n";
-        $note++;
-    }
-    $note = 0;
-    for (@{$copy->{z}}) {
-        print Z join("\t", $holdings->{id}, $copy->{copyid}, $note, $_), "\n";
-        $note++;
+sub dump_sample_detail {
+    my ($c, $s) = @_;
+    my $tags = $s->{data}{samp};
+    my $count = $s->{data}{rcnt};
+
+    open DETAIL, '>', ($c->{prefix} . "-HOLDINGS-DETAIL.txt");
+    select DETAIL;
+    for my $tag (sort keys %{$tags}) {
+        print ">>>>> TAG $tag\n";
+        for my $subkey (sort keys %{$tags->{$tag}}) {
+            my $sub = $tags->{$tag}{$subkey};
+            print "  Subfield: $subkey\n";
+            print "  Sample:   '", $sub->{value}, "'\n";
+            print "  Count:    ", $sub->{count}, " in ", $sub->{tcnt}, " tags\n\n";
+            #print "(", int($sub->{count} / $sub->{rcnt}), "%)\n";
+        }
     }
-    my @fields = ();
-    for ( @{$conf->{fields}} )
-      { $copy->{$_} = '' unless defined $copy->{$_}; push @fields, $copy->{$_} }
-    print HOLDINGS join("\t", $holdings->{id}, $copy->{copyid}, @fields), "\n";
+    select STDOUT;
+    print "\n";
+    close DETAIL;
 }
 
+sub dump_sample_overview {
+    my ($c, $s) = @_;
+    my $tags = $s->{data}{tags};
+    my $count = $s->{data}{rcnt};
+
+    my @tagsbyname  = sort keys %{$tags};
+    my @tagsbycount = reverse sort { $tags->{$a} <=> $tags->{$b} } keys %{$tags};
+
+    open SAMPLE, '>', ($c->{prefix} . "-HOLDINGS-OVERVIEW.txt");
+    select SAMPLE;
+    print "SAMPLE REPORT FOR ", $c->{prefix},": $count records\n\n";
+    print "FOUND TAGS (BY TAG)           FOUND TAGS (BY COUNT)\n";
+    print "------------------------      --------------------------\n";
+    for my $i (0 .. @tagsbyname - 1) {
+        print $tagsbyname[$i], (" " x (14 - length $tags->{ $tagsbyname[$i] })),
+          $tags->{ $tagsbyname[$i] };
+        print " (", sprintf("%03d", int($tags->{ $tagsbyname[$i] } / $count * 100)), "%)";
+        print "      ";
+        print $tagsbycount[$i], (" " x (16 - length $tags->{ $tagsbycount[$i] })),
+          $tags->{ $tagsbycount[$i] };
+        print " (", sprintf("%03d", int($tags->{ $tagsbycount[$i] } / $count * 100)), "%)\n";
+    }
+    select STDOUT;
+    print "\n";
+    close SAMPLE;
+}
 
-#------------------------------------------------
-
+#--------------------------
 
 sub initialize {
     my $c = {};
@@ -116,59 +145,62 @@ sub initialize {
     binmode(STDIN, ':utf8');
 
     my $rc = GetOptions( $c,
-                         'fields|f=s',
-                         'output|o=s',
+                         'sample|s',
+                         'samplemap|sm=s',
+                         'samplestr|ss=s',
+                         'marcfile|m=s',
+                         'map=s',
+                         'holdings|h=i',
                          'prefix|p=s',
-                         'pubnotes|pub=i',
-                         'pubnotesfile=s',
-                         'privnotes|priv=s',
-                         'privnotesfile=s',
-                         'tag|t=i',
-                         'help|h',
+                         'version|v',
+                         'help',
                        );
     show_help() unless $rc;
     show_help() if ($c->{help});
-
-    # set defaults
-    $c->{prefix} = (defined $c->{prefix}) ? ($c->{prefix} . '.') : '';
-    $c->{tag} = $c->{tag} || '852';
-    $c->{output} =
-      $c->{output} || join('', $c->{prefix}, "holdings.pg");
-    $c->{pubnotes} = $c->{pubnotes} || 'x';
-    $c->{pubnotesfile} =
-      $c->{pubnotesfile} || join('', $c->{prefix}, "holdings.pubnote.pg");
-    $c->{privnotes} = $c->{privnotes} || 'z';
-    $c->{privnotesfile} =
-      $c->{privnotesfile} || join('', $c->{prefix}, "holdings.privnote.pg");
+    show_help("map and holdings must be specified together!")
+      if ($c->{map} and !$c->{holdings});
+    show_version() if $c->{version};
 
     my @keys = keys %{$c};
-    show_help() unless (@ARGV and @keys);
-    for my $key ('fields', 'tag')
+    for my $key ('prefix')
       { push @missing, $key unless $c->{$key} }
     if (@missing) {
         print "Required option: ", join(', ', @missing), " missing!\n";
         show_help();
     }
 
-    # explode and validate fields string
-    process_fields($c);
     return $c;
 }
 
-sub process_fields {
-    my ($c) = @_;
-    my @holdings_fields = split /,/, $c->{fields};
-    for (@holdings_fields) { 
-        die "Field names must be alphanumeric!\n" if /\W/;
-        die "Field names must be single characters!\n"
-          if /\w{2,}/;
-    }
-    $c->{fields} = \@holdings_fields;
-}
-
 sub show_help {
+    my ($msg) = @_;
+    print "\nERROR - $msg\n" if $msg;
     print <<HELP;
-Usage is: extract_holdings MARCXML_FILE HOLDINGS_TAG
+
+Usage is: extract_holdings -p PREFIX -m MARCFILE [ARGUMENTS]
+
+REQUIRED ARGUMENTS
+  --prefix   -p  Prefix string for output filenames
+  --marcfile -m  MARCXML to use as source data
+
+SAMPLING ARGUMENTS
+  --sample    -s   Generate a report of all tags in the MARC data
+  --samplemap -sm  Specify a E::M::STL map file which will be used to generate
+                   subfield breakdown reports about specific tags in the MARC
+                   data
+  --samplestr -ss  As above, but with a one-liner map specified on the command
+                   line as a string (e.g. '-ss "852 999"')
+
+  If --samplemap and --samplestr are both specified, --samplemap wins.
+
+HOLDINGS EXTRACTION ARGUMENTS
+  --map          E::M::SM map file which will be used to extract holdings data
+                 from the input MARC file
+  --holdings -h  Specifies actual holdings tag
+
+  Both these must be given together.
 HELP
     exit;
 }
+
+sub show_version { print "extract_holdings v$VERSION\n"; exit }