initialize($conf);
populate_trash() if ($conf->{trash});
-my @record = (); # current record storage
-my %recmeta = (); # metadata about current record
-my $ptr = 0; # record index pointer
-
-my $input = shift || 'incoming.marc.xml';
-
-open MARC, '<:utf8', $input;
+# set up files, since everything appears to be in order
+open MARC, '<:utf8', (shift || 'incoming.marc.xml');
open my $NUMARC, '>:utf8', $conf->{output};
+open my $EXMARC, '>:utf8', $conf->{exception};
+open my $OLD2NEW, '>', 'old2new.incoming.map'
+ if ($conf->{'renumber-from'} and $conf->{'original-subfield'});
print $NUMARC '<?xml version="1.0" encoding="UTF-8"?>',"\n";
print $NUMARC '<collection xmlns="http://www.loc.gov/MARC21/slim">',"\n";
-open my $EXMARC, '>:utf8', $conf->{exception};
+my @record = (); # current record storage
+my %recmeta = (); # metadata about current record
+my $ptr = 0; # record index pointer
# this is the dispatch table which drives command selection in
# edit(), below
}
do_automated_cleanups();
+ stow_record_data();
$ptr = 0;
until ($ptr == $#record) {
$record[$ptr] =~ s/tag="-/tag="0/g;
$record[$ptr] =~ s/tag="(\d\d) /tag="0$1/g;
- # stow tag data if we're looking at it
- if ($record[$ptr] =~ m/<datafield tag="(.{3})" ind1="(.)" ind2="(.)">/) {
- $recmeta{tag} = $1;
- $recmeta{ind1} = $2;
- $recmeta{ind2} = $3;
- }
-
# automatable subfield maladies
$record[$ptr] =~ s/code=" ">c/code="c">/;
$record[$ptr] =~ s/code=" ">\$/code="c"$>/;
}
}
+sub stow_record_data {
+ # get tag data if we're looking at it
+ if ($record[$ptr] =~ m/<datafield tag="(.{3})" ind1="(.)" ind2="(.)">/) {
+ $recmeta{tag} = $1;
+ $recmeta{ind1} = $2;
+ $recmeta{ind2} = $3;
+
+ # and since we are looking at a tag, see if it's the original id
+ if ($conf->{'original-subfield'} and $1 == $conf->{'original-tag'}) {
+ my $line = $record[$ptr]; my $lptr = $ptr;
+ my $osub = $conf->{'original-subfield'};
+ $recmeta{oid} = 'NONE';
+
+ until ($line =~ m|</record>|) {
+ $lptr++;
+ $line = $record[$lptr];
+ $recmeta{oid} = $1
+ if ($line =~ /<subfield code="$osub">(.+?)</);
+ }
+ }
+ }
+}
+
#-----------------------------------------------------------------------------------
# driver routines
#-----------------------------------------------------------------------------------
-
=head2 edit
Handles the Term::ReadLine loop
print_fullcontext();
# stow original problem line
- $conf->{origline} = $record[$ptr];
+ $recmeta{origline} = $record[$ptr];
while (1) {
my $line = $term->readline('marc-cleanup>');
my $term = $commands{$chunks[0]}->(@chunks[1..$#chunks]);
last if $term;
} else {
+ $recmeta{prevline} = $record[$ptr];
$record[$ptr] = "$line\n";
print_context();
}
# add 903(?) with new record id
my $renumber = '';
if ($conf->{'renumber-from'}) {
+ $recmeta{nid} = $conf->{'renumber=from'};
$renumber = join('', ' <datafield tag="', $conf->{'renumber-tag'},
'" ind1=" " ind2=" "> <subfield code="',
- $conf->{'renumber-subfield'}, '">',
- $conf->{'renumber-from'}, "</subfield></datafield>\n");
+ $conf->{'renumber-subfield'},
+ '">', $recmeta{nid}, "</subfield></datafield>\n");
my @tmp = @record[0 .. $#record - 1];
my $last = $record[$#record];
@record = (@tmp, $renumber, $last);
unless ($conf->{nocollapse})
{ s/\n// for (@record) }
+ # write to old->new map file if needed
+ print $OLD2NEW $recmeta{oid}, "\t", $recmeta{nid}, "\n"
+ if ($conf->{'renumber-from'} and $conf->{'original-subfield'});
+
+ # and finally, actually write the record
print $FH @record;
}
$from = join('', $from, $char);
}
- $conf->{prevline} = $record[$ptr];
+ $recmeta{prevline} = $record[$ptr];
$record[$ptr] =~ s/$from/$to/;
print_context();
return 0;
}
sub merge_lines {
- $conf->{prevline} = $record[$ptr];
+ $recmeta{prevline} = $record[$ptr];
# remove <subfield stuff; extract (probably wrong) subfield code
$record[$ptr] =~ s/^\s*<subfield code="(.*?)">//;
# and move to front of line
}
sub flip_line {
- unless ($conf->{prevline})
+ unless ($recmeta{prevline})
{ print $OUT "No previously edited line to flip\n"; return }
my $temp = $record[$ptr];
- $record[$ptr] = $conf->{prevline};
- $conf->{prevline} = $temp;
+ $record[$ptr] = $recmeta{prevline};
+ $recmeta{prevline} = $temp;
print_context();
return 0;
}
sub kill_line {
- $conf->{killline} = $record[$ptr];
+ $recmeta{killline} = $record[$ptr];
my @a = @record[0 .. $ptr - 1];
my @b = @record[$ptr + 1 .. $#record];
@record = (@a, @b);
}
sub yank_line {
- unless ($conf->{killline})
+ unless ($recmeta{killline})
{ print $OUT "No killed line to yank\n"; return }
my @a = @record[0 .. $ptr - 1];
my @b = @record[$ptr .. $#record];
}
sub display_lines {
- print $OUT "\nOrig. edit line :", $conf->{origline};
- print $OUT "Current flip line:", $conf->{prevline} if $conf->{prevline};
- print $OUT "Last killed line :", $conf->{killline} if $conf->{killline};
+ print $OUT "\nOrig. edit line :", $recmeta{origline};
+ print $OUT "Current flip line:", $recmeta{prevline} if $recmeta{prevline};
+ print $OUT "Last killed line :", $recmeta{killline} if $recmeta{killline};
print $OUT "\n";
return 0;
}
}
# remove original id sequence tag from trash hash if we know it
- trash_add($conf->{'renumber-orig'}, 1)
- if ($conf->{'renumber-orig'} and $conf->{trash}{ $conf->{'renumber-orig'} });
+ trash_add($conf->{'original-tag'}, 1)
+ if ($conf->{'original-tag'} and $conf->{trash}{ $conf->{'original-tag'} });
}
sub trash_add_range {
'output|o=s',
'nocollapse|n',
'renumber-from|rf=i',
- 'renumber-orig|ro=i',
'renumber-tag|rt=i',
'renumber-subfield|rs=i',
+ 'original-tag|ot=i',
+ 'original-subfield|os=i',
+ 'script',
'strip-nines',
'trash|t=s',
'trashhelp',
show_trashhelp() if ($c->{trashhelp});
# defaults
- $c->{output} = 'incoming.cleaned.marc.xml' unless defined $c->{output};
- $c->{exception} = 'incoming.exception.marc.xml' unless defined $c->{exception};
+ $c->{output} = 'cleaned.incoming.marc.xml' unless defined $c->{output};
+ $c->{exception} = 'cleaned.exceptions.marc.xml' unless defined $c->{exception};
$c->{'renumber-tag'} = 903 unless defined $c->{'renumber-tag'};
$c->{'renumber-subfield'} = 'a' unless defined $c->{'renumber-subfield'};
$c->{window} = 5;
Usage is: marc-cleanup [OPTIONS] <filelist>
Options
--output -o Cleaned MARCXML output filename
- (default: incoming.cleaned.marc.xml)
+ (default: cleaned.incoming.marc.xml)
--exception -x Exception (dumped records) MARCXML filename
- (incoming.exception.marc.xml)
+ (cleaned.exceptions.marc.xml)
--trash File containing trash tag data (see --trashhelp)
--renumber-from=NUM -rf Begin renumbering id sequence with this number
--renumber-tag -rt Tag to use in renumbering (default: 903)
--renumber-subfield -rs Subfield code to use in renumbering (default: a)
- --renumber-orig -ro Original id tag; will be kept in output even if
+ --original-tag -ot Original id tag; will be kept in output even if
it appears in the trash file
+ --original-subfield -os Original id subfield code. If this is specified
+ and renumbering is in effect, an old-to-new mapping
+ file (old2new.incoming.map) will be generated.
--nocollapse -n Don't compress records to one line on output
--autoscrub -a Automatically remove non-numeric tags in data
--strip-nines Automatically remove any existing 901/903 tags in data
+
+ --script Store human-initiated ops in scriptfile (.mcscript)
+ Not yet implemented
+HELP
+exit;
+}
+
+sub show_trashhelp {
+ print <<HELP;
+The marc-cleanup trash tags file is a simple plaintext file. It is a
+line oriented format. There are three basic tokens:
+
+ * The tag
+ * The tag range
+ * The "except" clause
+
+Any number of tags and/or tag ranges can appear on a single line. A
+tag cannot appear twice in the file, either alone or as part of a
+range. This is to prevent errors in the trash tag listing. Items do
+not have to be sorted within a line. These following lines are valid:
+
+ 850 852 870..879 886 890 896..899
+ 214 696..699 012
+
+Ranges must be ordered internally. That is, "870..879" is valid while
+"879..870" is not.
+
+Finally, there can be only one "except" clause on a line. It is
+composed of the word "except" followed by one or more tags or
+ranges. Except clauses must follow a range, and all tags within the
+clause must be within the range which the clause follows.
+
+ 900..997 except 935 950..959 987 994
+
+is a valid example.
HELP
exit;
}
projects / migration-tools.git / blobdiff