#!/usr/bin/perl

###############################################################################
=pod

=item B<loadbibs> --stage_file foo.mrc 

Takes a load of bibs from a binary marc file and loads them into mig staging table 
of bibio_record_entry.

Takes these optional arguments:

--append 

When used it does not drop the staging table and instead adds onto it.  

--source

Sets an x_source value on the staging table to the one supplied instead of the 
default of none.

--xml 

By default the program assumes a USMARC file.  This flag will identify it as 
a MARCXML file instead.

=back

=cut

###############################################################################

use strict;
use warnings;

use DBI;
use Data::Dumper;
use MARC::Record;
use MARC::Batch;
use MARC::File;
use MARC::File::XML;
use MARC::Charset 'marc8_to_utf8';
#binmode STDIN, ':bytes';
use Env qw(
    HOME PGHOST PGPORT PGUSER PGDATABASE MIGSCHEMA
    MIGBASEWORKDIR MIGBASEGITDIR MIGGITDIR MIGWORKDIR
);
use Pod::Usage;
use Switch;
use Cwd 'abs_path';
use FindBin;
use UNIVERSAL;
use Unicode::Normalize;
my $mig_bin = "$FindBin::Bin/";
use lib "$FindBin::Bin/";
use Mig;

pod2usage(-verbose => 2) if defined $ARGV[0] && $ARGV[0] eq '--help';
pod2usage(-verbose => 1) if ! $ARGV[1];

my $next_arg_is_file = 0;
my $append = 0;
my $next_arg_is_source = 0;
my $next_arg_is_stage = 0;
my $stage_table = 'biblio_record_entry_legacy';
my $source = 'default';
my $file_is_xml = 0;
my $dbh = Mig::db_connect();
my $infile;
my $i = 0;
my $batch;
binmode STDIN, ':utf8';

#MARC::Charset->assume_unicode(1); 
MARC::Charset->ignore_errors(1);

foreach my $arg (@ARGV) {
    if ($arg eq '--stage_file') {
        $next_arg_is_file = 1;
        next;
    }
    if ($next_arg_is_file) {
        $infile = $arg;
        $next_arg_is_file = 0;
        next;
    }
    if ($arg eq '--source') {
        $next_arg_is_source = 1;
        next;
    }
    if ($next_arg_is_source) {
        $source = $arg;
        $next_arg_is_source = 0;
        next;
    }
    if ($arg eq '--stage_table') {
        $next_arg_is_stage = 1;
        next;
    }
    if ($next_arg_is_stage) {
        $stage_table = $arg;
        $next_arg_is_stage = 0;
        next;
    }
	if ($arg eq '--append') {
		$append = 1;
		next;
	}
    if ($arg eq '--xml') {
        $file_is_xml = 1;
        next;
    }
}

create_child_table($dbh); #and test to see if it exists 

# normal stage table creation
if ($append == 0) { 
    drop_stage_table($dbh,$stage_table); 
    create_stage_table($dbh,$stage_table);
    }
if ($file_is_xml) {
	$batch = MARC::Batch->new('XML',$infile);
} else {
	$batch = MARC::Batch->new('USMARC',$infile);
} 
$batch->strict_off();

my $record;
#while ( my $record = $batch->next() ) {
while ( eval {$record = $batch->next()} or do { if (!$record and !$@) { last; } else { next; }} ) {
	my $xml = $record->as_xml_record();
    $xml = marc8_to_utf8($xml);
	$i++;
	$xml = clean_marc($xml);
	$xml = '$_$' . $xml . '$_$';
	my @warnings = $batch->warnings();
	my $warning_string;
	if (@warnings) { $warning_string = "'" . join(':',@warnings) . "'"; } else { $warning_string = "'none'"; }
	my $sql = "INSERT INTO $MIGSCHEMA.$stage_table (marc,x_source,x_warnings) VALUES ($xml,'$source',$warning_string);";
    my $sth = $dbh->prepare($sql);
    eval { $sth->execute() };
	report_progress("Records staged", $i) if 0 != $i % 100;
}

$dbh->do(qq/
    CREATE INDEX ${MIGSCHEMA}_biblio_record_entry_legacy_idx ON
        $MIGSCHEMA.biblio_record_entry_legacy (id);
/);

print "Finis.\n";


sub drop_stage_table {
    my $dbh = shift;
    my $stage_table = shift;
    my $tablecheck = check_for_mig_table($dbh,$stage_table);
    my $answer = 'null';
    if ($tablecheck == 1) { $answer = prompt('Do you want to drop $MIGSCHEMA.$stage_table? This will not remove any bibs loaded to production. y/n'); }
    if ($tablecheck == 1 and $answer eq 'y') { $dbh->do("DROP TABLE IF EXISTS $MIGSCHEMA.$stage_table;"); } 
    if ($tablecheck == 1 and $answer ne 'y') { abort('Table not dropped, bib load aborted.'); }
    return();
}

sub create_stage_table {
	my $dbh = shift;
    my $stage_table = shift;

    $dbh->do("CREATE UNLOGGED TABLE $MIGSCHEMA.$stage_table (
            l_bib_id    TEXT,
            x_source    TEXT,
            x_warnings  TEXT,
            x_migrate   BOOLEAN DEFAULT TRUE
        ) INHERITS ($MIGSCHEMA.biblio_record_entry);");

    return();
}

sub create_child_table {
    my $dbh = shift;

    $dbh->do("DO \$\$ 
        DECLARE
            t   BOOLEAN;
        BEGIN
        SELECT EXISTS(SELECT 1 FROM information_schema.tables WHERE table_schema = '$MIGSCHEMA' AND table_name = 'biblio_record_entry') INTO t;
        IF t = FALSE THEN
            PERFORM migration_tools.build_specific_base_staging_table ('$MIGSCHEMA','biblio.record_entry');
        END IF;
        END \$\$;");

    return ();
}

sub clean_marc {
    my $xml = shift;
    $xml = marc8_to_utf8($xml);
    $xml =~ s/\n//sog;
    $xml =~ s/^<\?xml.+\?\s*>//go;
    $xml =~ s/>\s+</></go;
    $xml =~ s/\p{Cc}//go;
    $xml = NFC($xml);
    $xml =~ s/&(?!\S+;)/&amp;/gso;
    $xml =~ s/([\x{0080}-\x{fffd}])/sprintf('&#x%X;',ord($1))/sgoe;
    $xml =~ s/[\x00-\x1f]//go;
    return $xml;
}


sub abort {
    my $msg = shift;
    print STDERR "$0: $msg", "\n";
    exit 1;
}

sub report_progress {
    my ($msg, $counter) = @_;
    if (defined $counter) {
        print STDERR "$msg: $counter\n";
    } else {
        print STDERR "$msg\n";
    }
}

sub check_for_mig_table {
    my $dbh = shift;
    my $table = shift;
    my $sql = "SELECT 1 FROM information_schema.tables WHERE table_schema = '$MIGSCHEMA' AND table_name = '$table';";
    my $sth = $dbh->prepare($sql);
    $sth->execute();
    my @sqlresult = $sth->fetchrow_array;
    my $r = pop @sqlresult;
    if ($r) { return $r; } else { return 0; }
}

sub prompt {
  my ($query) = @_; 
  local $| = 1; 
  print $query;
  chomp(my $answer = <STDIN>);
  return $answer;
}