X-Git-Url: http://git.equinoxoli.org/?p=migration-tools.git;a=blobdiff_plain;f=extract_holdings;h=29bd56d093554b584ed9c75fd10d0302e714a98e;hp=eea04652c9729e0dc18942ab1254a077e031460b;hb=5920a6497c3eeb26fd66cdeae6cb3386686a561a;hpb=6c25f0626b026e59b9051c31d3eddfb6105f3c56

diff --git a/extract_holdings b/extract_holdings
index eea0465..29bd56d 100755
--- a/extract_holdings
+++ b/extract_holdings
@@ -2,111 +2,79 @@
 use strict;
 use warnings;
 
-use XML::Twig;
-use YAML::Tiny;
 use Getopt::Long;
+use Equinox::Migration::MapDrivenMARCXMLProc;
+use Equinox::Migration::MARCXMLSampler;
 
 
-my $conf = initialize();
-my $marcxml = shift;
+my $c = initialize();
 
-open HOLDINGS, '>', $conf->{output};
-open X, '>', $conf->{pubnotesfile};
-open Z, '>', $conf->{privnotesfile};
-my $holdings = {};
-my %sample = ( x => {}, z => {} ); # hash of all subfields in all 852s
-my $copyid = 0;
+# run samples if we've been asked for them
+run_samples($c) if ($c->{sample} or $c->{samplemap} or $c->{samplestr});
 
 
-$| = 1;
-my $count = 0;
-my $total = `grep -c '<record' $marcxml`;
-my $percent = 0;
-my $prevper = -1;
-
-my $yaml = YAML::Tiny->new;
-my $t = XML::Twig->new( twig_handlers => { record => \&record } );
-$t->parsefile($marcxml);
-$yaml->[0] = \%sample;
-$yaml->write('holdings.sample');
-print "\n\n";
-
-sub record {
-    my($t, $r)= @_;
-    $holdings = { copies => [] };
-
-    my @dfields = $r->children('datafield');
-    for my $d (@dfields) 
-      { process_datafields($d) }
-
-    for my $copy (@{$holdings->{copies}})
-      { print_reports($copy) }
-    $r->purge;
-
-    $count++;
-    $percent = int(($count / $total) * 100);
-    print "\r$percent% done ($count)" if ($percent != $prevper);
-    $prevper = $percent;
-}
-
-sub process_datafields {
-    my ($d) = @_;
-    # get 903
-    if ($d->{'att'}->{'tag'} == 903) {
-        my $s = $d->first_child('subfield');
-        $holdings->{id} = $s->text;;
+sub run_samples {
+    my ($c) = @_;
+    my $s;
+    if ($c->{samplemap}) {
+        $s = Equinox::Migration::MARCXMLSampler->new( marcfile => $c->{marcfile},
+                                                      mapfile  => $c->{samplemap});
+    } elsif ($c->{samplestr}) {
+        $s = Equinox::Migration::MARCXMLSampler->new( marcfile  => $c->{marcfile},
+                                                      mapstring => $c->{samplestr});
+    } else {
+        $s = Equinox::Migration::MARCXMLSampler->new( marcfile => $c->{marcfile} );
     }
+    $s->parse_records;
 
-    # and holdings data
-    if ($d->{'att'}->{'tag'} == $conf->{tag}) {
-        push @{$holdings->{copies}}, { x =>[], z => [] };
-        $holdings->{copies}[-1]{copyid} = $copyid;
-        my @subs = $d->children('subfield');
-        for my $s (@subs)
-          { process_subs($s) }
-        $copyid++;
-    }
+    dump_sample_overview($c, $s) if $c->{sample};
+    dump_sample_detail($c, $s) if ($c->{samplemap} or $c->{samplestr});
 }
 
-sub process_subs {
-    my ($s) = @_;
-    my $copy = $holdings->{copies}[-1];
-
-    my $code = $s->{'att'}->{'code'};
-    my $value = $s->text;
-
-    if ($code eq $conf->{pubnotes} or $code eq $conf->{privnotes}) {
-        push @{$copy->{$code}}, $value;
-        my ($k,$v) = split /:/, $value;
-        $sample{$code}{$k} = $v;
-    } else {
-        $copy->{$code} = $value;
-        $sample{$code} = $value;
+sub dump_sample_detail {
+    my ($c, $s) = @_;
+    my $tags = $s->{data}{samp};
+    my $count = $s->{data}{rcnt};
+
+    open DETAIL, '>', ($c->{prefix} . "-HOLDINGS-DETAIL.txt");
+    select DETAIL;
+    for my $tag (sort keys %{$tags}) {
+        print ">>>>> TAG $tag\n";
+        for my $subkey (sort keys %{$tags->{$tag}}) {
+            my $sub = $tags->{$tag}{$subkey};
+            print "  Subfield: $subkey\n";
+            print "  Sample:   '", $sub->{value}, "'\n";
+            print "  Count:    ", $sub->{count}, " in ", $sub->{tcnt}, " tags\n\n";
+            #print "(", int($sub->{count} / $sub->{rcnt}), "%)\n";
+        }
     }
+    select STDOUT;
+    close DETAIL;
 }
 
-sub print_reports {
-    return unless defined $holdings->{id};
-    my ($copy) = @_;
-    my $note = 0;
-    for (@{$copy->{x}}) {
-        print X join("\t", $holdings->{id}, $copy->{copyid}, $note, $_), "\n";
-        $note++;
-    }
-    $note = 0;
-    for (@{$copy->{z}}) {
-        print Z join("\t", $holdings->{id}, $copy->{copyid}, $note, $_), "\n";
-        $note++;
+sub dump_sample_overview {
+    my ($c, $s) = @_;
+    my $tags = $s->{data}{tags};
+    my $count = $s->{data}{rcnt};
+
+    my @tagsbyname  = sort keys %{$tags};
+    my @tagsbycount = reverse sort { $tags->{$a} <=> $tags->{$b} } keys %{$tags};
+
+    open SAMPLE, '>', ($c->{prefix} . "-HOLDINGS-OVERVIEW.txt");
+    select SAMPLE;
+    print "FOUND TAGS (BY TAG)    FOUND TAGS (BY COUNT)\n";
+    print "-------------------    ---------------------\n";
+    for my $i (0 .. @tagsbyname - 1) {
+        print $tagsbyname[$i], (" " x (14 - length $tags->{ $tagsbyname[$i] })),
+          $tags->{ $tagsbyname[$i] }, "      ";
+        print $tagsbycount[$i], (" " x (16 - length $tags->{ $tagsbycount[$i] })),
+          $tags->{ $tagsbycount[$i] }, "\n";
     }
-    my @fields = ();
-    for ( @{$conf->{fields}} )
-      { $copy->{$_} = '' unless defined $copy->{$_}; push @fields, $copy->{$_} }
-    print HOLDINGS join("\t", $holdings->{id}, $copy->{copyid}, @fields), "\n";
+    select STDOUT;
+    close SAMPLE;
 }
 
-
-#------------------------------------------------
-
+#--------------------------
 
 sub initialize {
     my $c = {};
@@ -116,59 +84,38 @@ sub initialize {
     binmode(STDIN, ':utf8');
 
     my $rc = GetOptions( $c,
-                         'fields|f=s',
-                         'output|o=s',
+                         'sample|s',
+                         'samplemap|sm=s',
+                         'samplestr|ss=s',
+                         'marcfile|m=s',
+                         'ils|i=s',
+                         'library|l=s',
                          'prefix|p=s',
-                         'pubnotes|pub=i',
-                         'pubnotesfile=s',
-                         'privnotes|priv=s',
-                         'privnotesfile=s',
-                         'tag|t=i',
                          'help|h',
                        );
     show_help() unless $rc;
     show_help() if ($c->{help});
 
-    # set defaults
-    $c->{prefix} = (defined $c->{prefix}) ? ($c->{prefix} . '.') : '';
-    $c->{tag} = $c->{tag} || '852';
-    $c->{output} =
-      $c->{output} || join('', $c->{prefix}, "holdings.pg");
-    $c->{pubnotes} = $c->{pubnotes} || 'x';
-    $c->{pubnotesfile} =
-      $c->{pubnotesfile} || join('', $c->{prefix}, "holdings.pubnote.pg");
-    $c->{privnotes} = $c->{privnotes} || 'z';
-    $c->{privnotesfile} =
-      $c->{privnotesfile} || join('', $c->{prefix}, "holdings.privnote.pg");
-
     my @keys = keys %{$c};
-    show_help() unless (@ARGV and @keys);
-    for my $key ('fields', 'tag')
+    for my $key ('prefix', 'ils')
       { push @missing, $key unless $c->{$key} }
     if (@missing) {
         print "Required option: ", join(', ', @missing), " missing!\n";
         show_help();
     }
 
-    # explode and validate fields string
-    process_fields($c);
     return $c;
 }
 
-sub process_fields {
-    my ($c) = @_;
-    my @holdings_fields = split /,/, $c->{fields};
-    for (@holdings_fields) { 
-        die "Field names must be alphanumeric!\n" if /\W/;
-        die "Field names must be single characters!\n"
-          if /\w{2,}/;
-    }
-    $c->{fields} = \@holdings_fields;
-}
-
 sub show_help {
     print <<HELP;
-Usage is: extract_holdings MARCXML_FILE HOLDINGS_TAG
+Usage is: extract_holdings -p PREFIX -m MAPFILE -i ILS [OPTIONS] <MARCXML>
+  prefix  p  Prefix string for output filenames
+  map     m  Subfield map file to use
+  ils     i  Legacy ILS name (affects SQL generation)
+
+  sample  s  Comma-delineated list of datafield tags to be sampled
+  library l  Legacy library name (affects SQL generation)
 HELP
     exit;
 }