3 ###############################################################################
6 =item B<loadbibs> --stage_file foo.mrc
8 Takes a load of bibs from a binary marc file and loads them into mig staging table
11 Takes these optional arguments:
15 When used it does not drop the staging table and instead adds onto it.
19 Sets an x_source value on the staging table to the one supplied instead of the
24 By default the program assumes a USMARC file. This flag will identify it as
25 a MARCXML file instead.
31 ###############################################################################
42 use MARC::Charset 'marc8_to_utf8';
43 #binmode STDIN, ':bytes';
45 HOME PGHOST PGPORT PGUSER PGDATABASE MIGSCHEMA
46 MIGBASEWORKDIR MIGBASEGITDIR MIGGITDIR MIGWORKDIR
53 use Unicode::Normalize;
54 my $mig_bin = "$FindBin::Bin/";
55 use lib "$FindBin::Bin/";
58 pod2usage(-verbose => 2) if defined $ARGV[0] && $ARGV[0] eq '--help';
59 pod2usage(-verbose => 1) if ! $ARGV[1];
61 my $next_arg_is_file = 0;
63 my $next_arg_is_source = 0;
64 my $next_arg_is_stage = 0;
65 my $stage_table = 'biblio_record_entry_legacy';
66 my $source = 'default';
68 my $dbh = Mig::db_connect();
72 binmode STDIN, ':utf8';
74 #MARC::Charset->assume_unicode(1);
75 MARC::Charset->ignore_errors(1);
77 foreach my $arg (@ARGV) {
78 if ($arg eq '--stage_file') {
79 $next_arg_is_file = 1;
82 if ($next_arg_is_file) {
84 $next_arg_is_file = 0;
87 if ($arg eq '--source') {
88 $next_arg_is_source = 1;
91 if ($next_arg_is_source) {
93 $next_arg_is_source = 0;
96 if ($arg eq '--stage_table') {
97 $next_arg_is_stage = 1;
100 if ($next_arg_is_stage) {
102 $next_arg_is_stage = 0;
105 if ($arg eq '--append') {
109 if ($arg eq '--xml') {
115 create_child_table($dbh); #and test to see if it exists
117 # normal stage table creation
119 drop_stage_table($dbh,$stage_table);
120 create_stage_table($dbh,$stage_table);
123 $batch = MARC::Batch->new('XML',$infile);
125 $batch = MARC::Batch->new('USMARC',$infile);
127 $batch->strict_off();
130 #while ( my $record = $batch->next() ) {
131 while ( eval {$record = $batch->next()} or do { if (!$record and !$@) { last; } else { next; }} ) {
132 my $xml = $record->as_xml_record();
133 $xml = marc8_to_utf8($xml);
135 $xml = clean_marc($xml);
136 $xml = '$_$' . $xml . '$_$';
137 my @warnings = $batch->warnings();
139 if (@warnings) { $warning_string = "'" . join(':',@warnings) . "'"; } else { $warning_string = "'none'"; }
140 my $sql = "INSERT INTO $MIGSCHEMA.$stage_table (marc,x_source,x_warnings) VALUES ($xml,'$source',$warning_string);";
141 my $sth = $dbh->prepare($sql);
142 eval { $sth->execute() };
143 report_progress("Records staged", $i) if 0 != $i % 100;
147 CREATE INDEX ${MIGSCHEMA}_biblio_record_entry_legacy_idx ON
148 $MIGSCHEMA.biblio_record_entry_legacy (id);
154 sub drop_stage_table {
156 my $stage_table = shift;
157 my $tablecheck = check_for_mig_table($dbh,$stage_table);
159 if ($tablecheck == 1) { $answer = prompt('Do you want to drop $MIGSCHEMA.$stage_table? This will not remove any bibs loaded to production. y/n'); }
160 if ($tablecheck == 1 and $answer eq 'y') { $dbh->do("DROP TABLE IF EXISTS $MIGSCHEMA.$stage_table;"); }
161 if ($tablecheck == 1 and $answer ne 'y') { abort('Table not dropped, bib load aborted.'); }
165 sub create_stage_table {
167 my $stage_table = shift;
169 $dbh->do("CREATE UNLOGGED TABLE $MIGSCHEMA.$stage_table (
173 x_migrate BOOLEAN DEFAULT TRUE
174 ) INHERITS ($MIGSCHEMA.biblio_record_entry);");
179 sub create_child_table {
186 SELECT EXISTS(SELECT 1 FROM information_schema.tables WHERE table_schema = '$MIGSCHEMA' AND table_name = 'biblio_record_entry') INTO t;
188 PERFORM migration_tools.build_specific_base_staging_table ('$MIGSCHEMA','biblio.record_entry');
197 $xml = marc8_to_utf8($xml);
199 $xml =~ s/^<\?xml.+\?\s*>//go;
200 $xml =~ s/>\s+</></go;
201 $xml =~ s/\p{Cc}//go;
203 $xml =~ s/&(?!\S+;)/&/gso;
204 $xml =~ s/([\x{0080}-\x{fffd}])/sprintf('&#x%X;',ord($1))/sgoe;
205 $xml =~ s/[\x00-\x1f]//go;
212 print STDERR "$0: $msg", "\n";
216 sub report_progress {
217 my ($msg, $counter) = @_;
218 if (defined $counter) {
219 print STDERR "$msg: $counter\n";
221 print STDERR "$msg\n";
225 sub check_for_mig_table {
228 my $sql = "SELECT 1 FROM information_schema.tables WHERE table_schema = '$MIGSCHEMA' AND table_name = '$table';";
229 my $sth = $dbh->prepare($sql);
231 my @sqlresult = $sth->fetchrow_array;
232 my $r = pop @sqlresult;
233 if ($r) { return $r; } else { return 0; }
240 chomp(my $answer = <STDIN>);