X-Git-Url: http://git.equinoxoli.org/?p=migration-tools.git;a=blobdiff_plain;f=mig-bin%2Fmig-loadbibs;h=a6db91c5f4a0614aa1e52d09452699ca408d035c;hp=96b12e89da3c084b06ca09a349e33589ea56e64b;hb=56f5223cb936c8f62a928eece4b6f40ea5bc9b06;hpb=e3f85c63930a230a6e7a5c64dbfc003e76db363c

diff --git a/mig-bin/mig-loadbibs b/mig-bin/mig-loadbibs
index 96b12e8..a6db91c 100755
--- a/mig-bin/mig-loadbibs
+++ b/mig-bin/mig-loadbibs
@@ -34,12 +34,6 @@ use strict;
 use warnings;
 
 use DBI;
-use Data::Dumper;
-use MARC::Record;
-use MARC::Batch;
-use MARC::File;
-use MARC::File::XML;
-use MARC::Charset 'marc8_to_utf8';
 #binmode STDIN, ':bytes';
 use Env qw(
     HOME PGHOST PGPORT PGUSER PGDATABASE MIGSCHEMA
@@ -50,7 +44,6 @@ use Switch;
 use Cwd 'abs_path';
 use FindBin;
 use UNIVERSAL;
-use Unicode::Normalize;
 my $mig_bin = "$FindBin::Bin/";
 use lib "$FindBin::Bin/";
 use Mig;
@@ -61,6 +54,8 @@ pod2usage(-verbose => 1) if ! $ARGV[1];
 my $next_arg_is_file = 0;
 my $append = 0;
 my $next_arg_is_source = 0;
+my $next_arg_is_stage = 0;
+my $stage_table = 'biblio_record_entry';
 my $source = 'default';
 my $file_is_xml = 0;
 my $dbh = Mig::db_connect();
@@ -91,75 +86,45 @@ foreach my $arg (@ARGV) {
         $next_arg_is_source = 0;
         next;
     }
-	if ($arg eq '--append') {
-		$append = 1;
-		next;
-	}
-    if ($arg eq '--xml') {
-        $file_is_xml = 1;
-        next;
-    }
-}
-
-create_child_table($dbh); #and test to see if it exists 
-
-# normal stage table creation
-if ($append == 0) { 
-    drop_stage_table($dbh); 
-    create_stage_table($dbh);
-    }
-if ($file_is_xml) {
-	$batch = MARC::Batch->new('XML',$infile);
-} else {
-	$batch = MARC::Batch->new('USMARC',$infile);
-} 
-$batch->strict_off();
-
-while ( my $record = $batch->next() ) {
-	my $xml = $record->as_xml_record();
-    $xml = marc8_to_utf8($xml);
-	$i++;
-	$xml = clean_marc($xml);
-	$xml = '$_$' . $xml . '$_$';
-	my @warnings = $batch->warnings();
-	my $warning_string;
-	if (@warnings) { $warning_string = "'" . join(':',@warnings) . "'"; } else { $warning_string = "'none'"; }
-	my $sql = "INSERT INTO $MIGSCHEMA.biblio_record_entry_legacy (marc,x_source,x_warnings) VALUES ($xml,'$source',$warning_string);";
-    my $sth = $dbh->prepare($sql);
-    $sth->execute();
-	report_progress("Records staged", $i) if 0 != $i % 100;
 }
 
-$dbh->do(qq/
-    CREATE INDEX ${MIGSCHEMA}_biblio_record_entry_legacy_idx ON
-        $MIGSCHEMA.biblio_record_entry_legacy (id);
-/);
-
-print "Finis.\n";
+my $bre_test = check_for_table($dbh,'biblio_record_entry');
+if ($bre_test == 0) { create_child_bre($dbh); }
+
+my $xmig_test = check_for_column($dbh,'biblio_record_entry','x_migrate');
+if ($xmig_test == 0) { add_column($dbh,'biblio_record_entry','x_migrate','BOOLEAN DEFAULT TRUE');
+
+my $xsource_test = check_for_column($dbh,'biblio_record_entry','x_source');
+if ($xsource_test == 0) { add_column($dbh,'biblio_record_entry','x_source','TEXT');
+
+my $last_xact;
+if ($source) { $last_xact = "'$MIGSCHEMA $source'" } else { $last_xact = "'$MIGSCHEMA'"; }
+
+#flatten out MARC XML FILE
+open my $xml, "<:encoding(utf8)", $infile or abort('could not open MARC XML file');
+$i = 0;
+my $record;
+while(my $line = <$xml>) {
+	 if ($line =~ /^<\/?collection/) { next; }
+	 chomp $line;
+	 $record = $record . $line;
+	 if ($line =~ /^<\/record/) {
+		stage_record($dbh,$record,$last_xact); 
+		$record = '';
+	}
 
+close $xml;
 
-sub drop_stage_table {
-    my $dbh = shift;
-    $dbh->do("DROP TABLE IF EXISTS $MIGSCHEMA.biblio_record_entry_legacy;");
-    return();
-}
 
-sub create_stage_table {
-	my $dbh = shift;
+#load the MARC XML FILE TO STAGING 
+report_progress("Records staged", $i) if 0 != $i % 100;
 
-    $dbh->do("CREATE UNLOGGED TABLE $MIGSCHEMA.biblio_record_entry_legacy (
-            l_bib_id    TEXT,
-            x_source    TEXT,
-            x_warnings  TEXT,
-            x_migrate   BOOLEAN DEFAULT TRUE
-        ) INHERITS ($MIGSCHEMA.biblio_record_entry);");
+print "Finis.\n";
 
-    return();
-}
+# beyond here be functions 
 
-sub create_child_table {
+sub create_child_bre {
     my $dbh = shift;
-
     $dbh->do("DO \$\$ 
         DECLARE
             t   BOOLEAN;
@@ -173,21 +138,6 @@ sub create_child_table {
     return ();
 }
 
-sub clean_marc {
-    my $xml = shift;
-    $xml = marc8_to_utf8($xml);
-    $xml =~ s/\n//sog;
-    $xml =~ s/^<\?xml.+\?\s*>//go;
-    $xml =~ s/>\s+</></go;
-    $xml =~ s/\p{Cc}//go;
-    $xml = NFC($xml);
-    $xml =~ s/&(?!\S+;)/&amp;/gso;
-    $xml =~ s/([\x{0080}-\x{fffd}])/sprintf('&#x%X;',ord($1))/sgoe;
-    $xml =~ s/[\x00-\x1f]//go;
-    return $xml;
-}
-
-
 sub abort {
     my $msg = shift;
     print STDERR "$0: $msg", "\n";
@@ -202,3 +152,48 @@ sub report_progress {
         print STDERR "$msg\n";
     }
 }
+
+sub stage_record {
+    my $dbh = shift;
+    my $record = shift;
+	my $last_xact = shift;
+	$record = '$_$' . $record . '$_$';
+    my $sql = "INSERT INTO $MIGSCHEMA.biblio_record_entry (last_xact_id,marc) VALUES ($last_xact,$record);";
+    my $sth = $dbh->prepare($sql);
+    $sth->execute();
+	return;
+}
+
+sub check_for_table {
+    my $dbh = shift;
+    my $table = shift;
+    my $sql = "SELECT 1 FROM information_schema.tables WHERE table_schema = '$MIGSCHEMA' AND table_name = '$table';";
+    my $sth = $dbh->prepare($sql);
+    $sth->execute();
+    my @sqlresult = $sth->fetchrow_array;
+    my $r = pop @sqlresult;
+    if ($r) { return $r; } else { return 0; }
+}
+
+sub check_for_column {
+    my $dbh = shift;
+    my $table = shift;
+	my $column = shift;
+    my $sql = "SELECT 1 FROM information_schema.columns WHERE table_schema = '$MIGSCHEMA' AND table_name = '$table' AND column_name = $column;";
+    my $sth = $dbh->prepare($sql);
+    $sth->execute();
+    my @sqlresult = $sth->fetchrow_array;
+    my $r = pop @sqlresult;
+    if ($r) { return $r; } else { return 0; }
+}
+
+sub add_column {
+    my $dbh = shift;
+    my $table = shift;
+    my $column = shift;
+	my $column_type = shift;
+    my $sql = "ALTER TABLE $MIGSCHEMA.$table ADD COLUMN $COLUMN $COLUMN_TYPE;";
+	my $r = check_for_column($dbh,$table,$column);
+	if ($r == 0) { abort('failed to create column'; } else { return $r; }
+}
+