1 package Equinox::Migration::MapDrivenMARCXMLProc;
7 use Equinox::Migration::SubfieldMapper 1.002;
11 # sample functionality should be extracted into a new module which
12 # uses E::M::SM to drive sampling of individual datafields, and
13 # reports ALL datafields which occur
15 # --sample should give the list of all datafields
16 # --samplefile should take a SM map as teh argument and introspect the mapped datafields
21 Equinox::Migration::MapDrivenMARCXMLProc
29 our $VERSION = '1.000';
36 use Equinox::Migration::MapDrivenMARCXMLProc;
44 Takes two required arguments: C<mapfile> (which will be passed along
45 to L<Equinox::Migration::SubfieldMapper> as the basis for its map),
46 and C<marcfile> (the MARC data to be processed).
48 my $m = Equinox::Migration::MapDrivenMARCXMLProc->new( mapfile => FILE,
51 There is an optional third, argument, C<sample>, which specifies a
52 arrayref of datafields to "sample" by reporting on subfields which are
53 found in the data but not in the map.
55 my $m = Equinox::Migration::MapDrivenMARCXMLProc->new( mapfile => FILE,
60 See L</UNMAPPED TAGS> for more info.
65 my ($class, %args) = @_;
67 my $self = bless { mods => { multi => {},
71 data => { recs => undef, # X::T record objects
72 rptr => 0, # next record pointer
73 crec => undef, # parsed record storage
74 stag => undef, # list of tags to sample
75 umap => undef, # unmapped data samples
79 # initialize map and taglist
80 die "Argument 'mapfile' must be specified\n" unless (defined $args{mapfile});
81 my @mods = keys %{$self->{mods}};
82 $self->{map} = Equinox::Migration::SubfieldMapper->new( file => $args{mapfile},
84 $self->{data}{tags} = $self->{map}->tags;
87 die "Argument 'marcfile' must be specified\n" unless (defined $args{marcfile});
88 if (-r $args{marcfile}) {
89 $self->{twig} = XML::Twig->new;
90 $self->{twig}->parsefile($args{marcfile});
91 my @records = $self->{twig}->root->children;
92 $self->{data}{recs} = \@records;
94 die "Can't open marc file: $!\n";
97 # if we have a sample arg, set up the sample set and umap hash
98 if (defined $args{sample}) {
99 for my $s ( @{$args{sample}})
100 { $self->{data}{stag}{$s} = 1 }
101 $self->{data}{umap} = {};
110 Extracts data from the next record, per the mapping file. Returns a
111 normalized datastructure (see L</format_record> for details) on
112 success; returns 0 otherwise.
114 while (my $rec = $m->parse_record) {
115 # handle extracted record data
123 # get the next record and wipe current parsed record
124 return 0 unless defined $self->{data}{recs}[ $self->{data}{rptr} ];
125 my $record = $self->{data}{recs}[ $self->{data}{rptr} ];
126 $self->{data}{crec} = { egid => undef, bib => undef, tags => undef };
128 my @fields = $record->children;
130 { $self->process_field($f) }
132 # cleanup memory and increment pointer
134 $self->{data}{rptr}++;
136 # FIXME check for required fields here
138 return $self->{data}{crec};
142 my ($self, $field) = @_;
143 my $map = $self->{map};
144 my $tag = $field->{'att'}->{'tag'};
145 my $crec = $self->{data}{crec};
148 unless (defined $tag) {
155 my $sub = $field->first_child('subfield');
156 $crec->{egid} = $sub->text;
159 if ($map->has($tag)) {
160 push @{$crec->{tags}}, { tag => $tag, uni => undef, multi => undef };
161 my @subs = $field->children('subfield');
163 { $self->process_subs($tag, $sub) }
164 # check map to ensure all declared subs have a value
165 my $mods = $map->mods($field);
166 for my $mappedsub ( @{ $map->subfields($tag) } ) {
167 next if $mods->{multi};
168 $crec->{tags}[-1]{uni}{$mappedsub} = ''
169 unless defined $crec->{tags}[-1]{uni}{$mappedsub};
175 my ($self, $tag, $sub) = @_;
176 my $map = $self->{map};
177 my $code = $sub->{'att'}->{'code'};
179 # handle unmapped tag/subs
180 unless ($map->has($tag, $code)) {
181 my $u = $self->{data}{umap};
182 my $s = $self->{data}{stag};
183 return unless (defined $s->{$tag});
185 # set a value, total-seen count and records-seen-in count
186 $u->{$tag}{$code}{value} = $sub->text unless defined $u->{$tag}{$code};
187 $u->{$tag}{$code}{count}++;
188 $u->{$tag}{$code}{rcnt}++ unless ( defined $u->{$tag}{$code}{last} and
189 $u->{$tag}{$code}{last} == $self->{data}{rptr} );
190 $u->{$tag}{$code}{last} = $self->{data}{rptr};
194 # fetch our datafield struct and fieldname
195 my $dataf = $self->{data}{crec}{tags}[-1];
196 my $field = $map->field($tag, $code);
198 # handle modifiers, or slug data in normally
199 if (my $mods = $map->mods($field)) {
200 if ($mods->{multi}) {
201 my $name = $tag . $code;
202 push @{$dataf->{multi}{$name}}, $sub->text;
205 die "Multiple occurances of a non-multi field: $tag$code at rec ",($self->{data}{rptr} + 1),"\n"
206 if (defined $dataf->{uni}{$code});
207 $dataf->{uni}{$code} = $sub->text;
213 MapDrivenMARCXMLProc implements the following modifiers, and passes
214 them to L<Equinox::Migration::SubfieldMapper>, meaning that specifying
215 any other modifiers in a MDMP map file will cause a fatal error when
220 If a mapping is declared to be C<multi>, then MDMP expects to see more
221 than one instance of that subfield per datafield, and the data is
222 handled accordingly (see L</PARSED RECORDS> below).
224 Occurring zero or one time is legal for a C<multi> mapping.
226 A mapping which is not flagged as C<multi>, but which occurs more than
227 once per datafield will cause a fatal error.
231 The C<bib> modifier declares that a mapping is "bib-level", and should
232 be encountered once per B<record> instead of once per B<datafield> --
233 which is another way of saying that it occurs in a non-repeating
234 datafield or in a controlfield.
238 By default, if a mapping does not occur in a datafield (or record, in
239 the case of C<bib> mappings), processing continues normally. if a
240 mapping has the C<required> modifier, however, it must appear, or a
241 fatal error will occur.
243 =head1 PARSED RECORDS
247 my $m = Equinox::Migration::MapDrivenMARCXMLProc->new(ARGUMENTS);
248 $rec = $m->parse_record;
250 Then C<$rec> will look like:
253 egid => evergreen_record_id,
255 (tag_id . sub_code)1 => value1,
256 (tag_id . sub_code)2 => value2,
262 multi => { (tag_id . sub_code) => [ val1, val2, ... ] },
263 uni => { code => value, code2 => value2, ... },
269 That is, there is an C<egid> key which points to the Evergreen ID of
270 that record, a C<bib> key which points to a hashref, and a C<tags>
271 key which points to an arrayref.
275 A reference to a hash which holds extracted data which occurs only
276 once per record (and is therefore "bib-level"; the default assumption
277 is that a tag/subfield pair can occur multiple times per record). The
278 keys are composed of tag id and subfield code, catenated
279 (e.g. 901c). The values are the contents of that subfield of that tag.
281 If there are no tags defined as bib-level in the mapfile, C<bib> will
286 A reference to a list of anonymous hashes, one for each instance of
287 each tag which occurs in the map.
289 Each tag hash holds its own id (e.g. C<998>), and two references to
290 two more hashrefs, C<multi> and C<uni>.
292 The C<multi> hash holds the extracted data for tag/sub mappings which
293 have the C<multiple> modifier on them. The keys in C<multi> are
294 composed of the tag id and subfield code, catenated
295 (e.g. C<901c>). The values are arrayrefs containing the content of all
296 instances of that subfield in that instance of that tag. If no tags
297 are defined as C<multi>, it will be C<undef>.
299 The C<uni> hash holds data for tag/sub mappings which occur only once
300 per instance of a tag (but may occur multiple times in a record due to
301 there being multiple instances of that tag in a record). Keys are
302 subfield codes and values are subfield content.
304 All C<uni> subfields occuring in the map are guaranteed to be
305 defined. Sufields which are mapped but do not occur in a particular
306 datafield will be given a value of '' (the null string) in the current
307 record struct. Oppose subfields which are not mapped, which will be
312 If the C<sample> argument is passed to L</new>, there will also be a
313 structure which holds data about unmapped subfields encountered in
314 mapped tags which are also in the declared sample set. This
315 information is collected over the life of the object and is not reset
316 for every record processed (as the current record data neccessarily
320 sub_code => { value => VALUE,
329 For each mapped tag, for each unmapped subfield, there is a hash of
330 data about that subfield containing
332 * value - A sample of the subfield text
333 * count - Total number of times the subfield was seen
334 * rcnt - The number of records the subfield was seen in
338 Shawn Boyette, C<< <sboyette at esilibrary.com> >>
342 Please report any bugs or feature requests to the above email address.
346 You can find documentation for this module with the perldoc command.
348 perldoc Equinox::Migration::MapDrivenMARCXMLProc
351 =head1 COPYRIGHT & LICENSE
353 Copyright 2009 Equinox, all rights reserved.
355 This program is free software; you can redistribute it and/or modify it
356 under the same terms as Perl itself.
361 1; # End of Equinox::Migration::MapDrivenMARCXMLProc