X-Git-Url: http://git.equinoxoli.org/?p=migration-tools.git;a=blobdiff_plain;f=Equinox-Migration%2Flib%2FEquinox%2FMigration%2FMapDrivenMARCXMLProc.pm;h=ca82ca7e6328c6a17b48de26f1216a791a64cd30;hp=28545467cd636fe23847f434e4b4419888833fe4;hb=d1d70c501f6592ae84fc351e4325e5a5f9a67376;hpb=2663e385ad3d1bba9ecd0ff191ce92b11dce8a1b diff --git a/Equinox-Migration/lib/Equinox/Migration/MapDrivenMARCXMLProc.pm b/Equinox-Migration/lib/Equinox/Migration/MapDrivenMARCXMLProc.pm index 2854546..ca82ca7 100644 --- a/Equinox-Migration/lib/Equinox/Migration/MapDrivenMARCXMLProc.pm +++ b/Equinox-Migration/lib/Equinox/Migration/MapDrivenMARCXMLProc.pm @@ -4,16 +4,7 @@ use warnings; use strict; use XML::Twig; -use Equinox::Migration::SubfieldMapper 1.002; - -# FIXME -# -# sample functionality should be extracted into a new module which -# uses E::M::SM to drive sampling of individual datafields, and -# reports ALL datafields which occur -# -# --sample should give the list of all datafields -# --samplefile should take a SM map as teh argument and introspect the mapped datafields +use Equinox::Migration::SubfieldMapper 1.004; =head1 NAME @@ -22,11 +13,18 @@ Equinox::Migration::MapDrivenMARCXMLProc =head1 VERSION -Version 1.000 +Version 1.004 =cut -our $VERSION = '1.000'; +our $VERSION = '1.004'; + +my $dstore; +my $sfmap; +my @mods = qw( multi bib required ); +my $multis = {}; +my $reccount; +my $verbose = 0; =head1 SYNOPSIS @@ -48,101 +46,68 @@ and C (the MARC data to be processed). my $m = Equinox::Migration::MapDrivenMARCXMLProc->new( mapfile => FILE, marcfile => FILE ); -There is an optional third, argument, C, which specifies a -arrayref of datafields to "sample" by reporting on subfields which are -found in the data but not in the map. - - my $m = Equinox::Migration::MapDrivenMARCXMLProc->new( mapfile => FILE, - marcfile => FILE, - sample => \@TAGS - ); - -See L for more info. - =cut sub new { my ($class, %args) = @_; - my $self = bless { mods => { multi => {}, - once => {}, - required => {}, - }, - data => { recs => undef, # X::T record objects - rptr => 0, # next record pointer - crec => undef, # parsed record storage - stag => undef, # list of tags to sample - umap => undef, # unmapped data samples - }, + $verbose = 1 if $args{verbose}; + + my $self = bless { multis => \$multis, }, $class; # initialize map and taglist - die "Argument 'mapfile' must be specified\n" unless (defined $args{mapfile}); - my @mods = keys %{$self->{mods}}; - $self->{map} = Equinox::Migration::SubfieldMapper->new( file => $args{mapfile}, - mods => \@mods ); - $self->{data}{tags} = $self->{map}->tags; + die "Argument 'mapfile' must be specified\n" unless ($args{mapfile}); + $sfmap = Equinox::Migration::SubfieldMapper->new( file => $args{mapfile}, + mods => \@mods ); + + # initialize datastore + $dstore = {}; + $reccount = 0; # next record ptr + $dstore->{tags} = $sfmap->tags; # list of all tags + $self->{data} = $dstore; # initialize twig - die "Argument 'marcfile' must be specified\n" unless (defined $args{marcfile}); + die "Argument 'marcfile' must be specified\n" unless ($args{marcfile}); if (-r $args{marcfile}) { - $self->{twig} = XML::Twig->new; - $self->{twig}->parsefile($args{marcfile}); - my @records = $self->{twig}->root->children; - $self->{data}{recs} = \@records; + my $xmltwig = XML::Twig->new( twig_handlers => { record => \&parse_record } ); + $xmltwig->parsefile( $args{marcfile} ); } else { die "Can't open marc file: $!\n"; } - # if we have a sample arg, set up the sample set and umap hash - if (defined $args{sample}) { - for my $s ( @{$args{sample}}) - { $self->{data}{stag}{$s} = 1 } - $self->{data}{umap} = {}; - } - return $self; } - =head2 parse_record -Extracts data from the next record, per the mapping file. Returns a -normalized datastructure (see L for details) on -success; returns 0 otherwise. - - while (my $rec = $m->parse_record) { - # handle extracted record data - } +Extracts data from the next record, per the mapping file. =cut sub parse_record { - my ($self) = @_; - - # get the next record and wipe current parsed record - return 0 unless defined $self->{data}{recs}[ $self->{data}{rptr} ]; - my $record = $self->{data}{recs}[ $self->{data}{rptr} ]; - $self->{data}{crec} = { egid => undef, bib => undef, tags => undef }; + my ($twig, $record) = @_; + my $crec = {}; # current record my @fields = $record->children; for my $f (@fields) - { $self->process_field($f) } + { process_field($f, $crec) } # cleanup memory and increment pointer $record->purge; - $self->{data}{rptr}++; + $reccount++; - # FIXME check for required fields here + # check for required fields + check_required($crec); + push @{ $dstore->{recs} }, $crec; - return $self->{data}{crec}; + print STDERR "$reccount\n" + if ($verbose and !($reccount % 1000)); } sub process_field { - my ($self, $field) = @_; - my $map = $self->{map}; + my ($field, $crec) = @_; my $tag = $field->{'att'}->{'tag'}; - my $crec = $self->{data}{crec}; # leader unless (defined $tag) { @@ -156,56 +121,109 @@ sub process_field { $crec->{egid} = $sub->text; return; } - if ($map->has($tag)) { + if ($sfmap->has($tag)) { push @{$crec->{tags}}, { tag => $tag, uni => undef, multi => undef }; + push @{$crec->{tmap}{$tag}}, (@{$crec->{tags}} - 1); my @subs = $field->children('subfield'); for my $sub (@subs) - { $self->process_subs($tag, $sub) } - # check map to ensure all declared subs have a value - my $mods = $map->mods($field); - for my $mappedsub ( @{ $map->subfields($tag) } ) { + { process_subs($tag, $sub, $crec) } + + # check map to ensure all declared tags and subs have a value + for my $mappedsub ( @{ $sfmap->subfields($tag) } ) { + my $fieldname = $sfmap->field($tag, $mappedsub); + my $mods = $sfmap->mods($fieldname); next if $mods->{multi}; $crec->{tags}[-1]{uni}{$mappedsub} = '' unless defined $crec->{tags}[-1]{uni}{$mappedsub}; } + for my $mappedtag ( @{ $sfmap->tags }) { + $crec->{tmap}{$mappedtag} = undef + unless defined $crec->{tmap}{$mappedtag}; + } } } sub process_subs { - my ($self, $tag, $sub) = @_; - my $map = $self->{map}; + my ($tag, $sub, $crec) = @_; my $code = $sub->{'att'}->{'code'}; # handle unmapped tag/subs - unless ($map->has($tag, $code)) { - my $u = $self->{data}{umap}; - my $s = $self->{data}{stag}; - return unless (defined $s->{$tag}); - - # set a value, total-seen count and records-seen-in count - $u->{$tag}{$code}{value} = $sub->text unless defined $u->{$tag}{$code}; - $u->{$tag}{$code}{count}++; - $u->{$tag}{$code}{rcnt}++ unless ( defined $u->{$tag}{$code}{last} and - $u->{$tag}{$code}{last} == $self->{data}{rptr} ); - $u->{$tag}{$code}{last} = $self->{data}{rptr}; - return; - } + return unless ($sfmap->has($tag, $code)); - # fetch our datafield struct and fieldname - my $dataf = $self->{data}{crec}{tags}[-1]; - my $field = $map->field($tag, $code); + # fetch our datafield struct and fiel + my $dataf = $crec->{tags}[-1]; + my $field = $sfmap->field($tag, $code); - # handle modifiers, or slug data in normally - if (my $mods = $map->mods($field)) { + # test filters + for my $filter ( @{$sfmap->filters($field)} ) { + return if ($sub->text =~ /$filter/i); + } + + # handle multi modifier + if (my $mods = $sfmap->mods($field)) { if ($mods->{multi}) { - my $name = $tag . $code; - push @{$dataf->{multi}{$name}}, $sub->text; + push @{$dataf->{multi}{$code}}, $sub->text; + $multis->{$tag}{$code} = 1; + return; + } + } + + # if this were a multi field, it would be handled already. make sure its a singleton + die "Multiple occurances of a non-multi field: $tag$code at rec ", + ($reccount + 1),"\n" if (defined $dataf->{uni}{$code}); + + # everything seems okay + $dataf->{uni}{$code} = $sub->text; +} + + +sub check_required { + my ($crec) = @_; + my $mods = $sfmap->mods; + + for my $tag_id (keys %{$mods->{required}}) { + for my $code (@{$mods->{required}{$tag_id}}) { + my $found = 0; + + for my $tag (@{$crec->{tags}}) { + $found = 1 if ($tag->{multi}{($tag_id . $code)}); + $found = 1 if ($tag->{uni}{$code}); + } + + die "Required mapping $tag_id$code not found in rec ",$reccount,"\n" + unless ($found); } - } else { - die "Multiple occurances of a non-multi field: $tag$code at rec ",($self->{data}{rptr} + 1),"\n" - if (defined $dataf->{uni}{$code}); - $dataf->{uni}{$code} = $sub->text; } + +} + +=head2 recno + +Returns current record number (starting from zero) + +=cut + +sub recno { my ($self) = @_; return $self->{data}{rcnt} } + +=head2 name + +Returns mapped fieldname when passed a tag, and code + + my $name = $m->name(999,'a'); + +=cut + +sub name { my ($self, $t, $c) = @_; return $sfmap->field($t, $c) } + +=head2 get_multis + +Returns hashref of C<{tag}{code}> for all mapped multi fields + +=cut + +sub get_multis { + my ($self) = @_; + return $multis; } =head1 MODIFIERS @@ -226,19 +244,11 @@ Occurring zero or one time is legal for a C mapping. A mapping which is not flagged as C, but which occurs more than once per datafield will cause a fatal error. -=head2 bib - -The C modifier declares that a mapping is "bib-level", and should -be encountered once per B instead of once per B -- -which is another way of saying that it occurs in a non-repeating -datafield or in a controlfield. - =head2 required -By default, if a mapping does not occur in a datafield (or record, in -the case of C mappings), processing continues normally. if a -mapping has the C modifier, however, it must appear, or a -fatal error will occur. +By default, if a mapping does not occur in a datafield, processing +continues normally. if a mapping has the C modifier, +however, it must appear, or a fatal error will occur. =head1 PARSED RECORDS @@ -250,36 +260,21 @@ Given: Then C<$rec> will look like: { - egid => evergreen_record_id, - bib => { - (tag_id . sub_code)1 => value1, - (tag_id . sub_code)2 => value2, - ... - }, + egid => evergreen_record_id, tags => [ { tag => tag_id, - multi => { (tag_id . sub_code) => [ val1, val2, ... ] }, + multi => { code => [ val1, val2, ... ] }, uni => { code => value, code2 => value2, ... }, }, ... - ] + ], + tmap => { tag_id => [ INDEX_LIST ], tag_id2 => [ INDEX_LIST ], ... } } That is, there is an C key which points to the Evergreen ID of -that record, a C key which points to a hashref, and a C -key which points to an arrayref. - -=head3 C - -A reference to a hash which holds extracted data which occurs only -once per record (and is therefore "bib-level"; the default assumption -is that a tag/subfield pair can occur multiple times per record). The -keys are composed of tag id and subfield code, catenated -(e.g. 901c). The values are the contents of that subfield of that tag. - -If there are no tags defined as bib-level in the mapfile, C will -be C. +that record, a C key which points to an arrayref, and a C +key which points to a hashref. =head3 C @@ -290,9 +285,8 @@ Each tag hash holds its own id (e.g. C<998>), and two references to two more hashrefs, C and C. The C hash holds the extracted data for tag/sub mappings which -have the C modifier on them. The keys in C are -composed of the tag id and subfield code, catenated -(e.g. C<901c>). The values are arrayrefs containing the content of all +have the C modifier on them. The keys in C subfield +codes. The values are arrayrefs containing the content of all instances of that subfield in that instance of that tag. If no tags are defined as C, it will be C. @@ -307,31 +301,22 @@ datafield will be given a value of '' (the null string) in the current record struct. Oppose subfields which are not mapped, which will be C. -=head1 UNMAPPED TAGS - -If the C argument is passed to L, there will also be a -structure which holds data about unmapped subfields encountered in -mapped tags which are also in the declared sample set. This -information is collected over the life of the object and is not reset -for every record processed (as the current record data neccessarily -is). - - { tag_id => { - sub_code => { value => VALUE, - count => COUNT, - rcnt => RCOUNT - }, - ... - }, - ... - } +=head3 tmap + +A hashref, where each key (a tag id like "650") points to a listref +containing the index (or indices) of C where that tag has +extracted data. -For each mapped tag, for each unmapped subfield, there is a hash of -data about that subfield containing +The intended use of this is to simplify the processing of data from +tags which can appear more than once in a MARC record, like +holdings. If your holdings data is in 852, C{852}> will be a +listref with the indices of C which hold the data from the 852 +datafields. - * value - A sample of the subfield text - * count - Total number of times the subfield was seen - * rcnt - The number of records the subfield was seen in +Complimentarily, C prevents data from singular datafields from +having to be copied for every instance of a multiple datafield, as it +lets you get the data from that record's one instance of whichever +field you're looking for. =head1 AUTHOR