1 CREATE OR REPLACE FUNCTION migration_tools.reingest_staged_record_attributes (rid BIGINT, pattr_list TEXT[] DEFAULT NULL, prmarc TEXT DEFAULT NULL, rdeleted BOOL DEFAULT TRUE) RETURNS INTEGER[] AS $func$
8 xfrm config.xml_transform%ROWTYPE;
9 attr_vector INT[] := '{}'::INT[];
10 attr_vector_tmp INT[];
11 attr_list TEXT[] := pattr_list;
13 norm_attr_value TEXT[];
16 attr_def config.record_attr_definition%ROWTYPE;
17 ccvm_row config.coded_value_map%ROWTYPE;
21 IF attr_list IS NULL OR rdeleted THEN -- need to do the full dance on INSERT or undelete
22 SELECT ARRAY_AGG(name) INTO attr_list FROM config.record_attr_definition
25 fixed_field IS NOT NULL OR
27 phys_char_sf IS NOT NULL OR
34 SELECT marc INTO rmarc FROM biblio_record_entry_legacy WHERE id = rid;
37 FOR attr_def IN SELECT * FROM config.record_attr_definition WHERE NOT composite AND name = ANY( attr_list ) ORDER BY format LOOP
39 jump_past := FALSE; -- This gets set when we are non-multi and have found something
40 attr_value := '{}'::TEXT[];
41 norm_attr_value := '{}'::TEXT[];
42 attr_vector_tmp := '{}'::INT[];
44 SELECT * INTO ccvm_row FROM config.coded_value_map c WHERE c.ctype = attr_def.name LIMIT 1;
46 IF attr_def.tag IS NOT NULL THEN -- tag (and optional subfield list) selection
47 SELECT ARRAY_AGG(value) INTO attr_value
48 FROM (SELECT * FROM metabib.full_rec ORDER BY tag, subfield) AS x
50 AND tag LIKE attr_def.tag
52 WHEN attr_def.sf_list IS NOT NULL
53 THEN POSITION(subfield IN attr_def.sf_list) > 0
59 IF NOT attr_def.multi THEN
60 attr_value := ARRAY[ARRAY_TO_STRING(attr_value, COALESCE(attr_def.joiner,' '))];
65 IF NOT jump_past AND attr_def.fixed_field IS NOT NULL THEN -- a named fixed field, see config.marc21_ff_pos_map.fixed_field
66 attr_value := attr_value || vandelay.marc21_extract_fixed_field_list(rmarc, attr_def.fixed_field);
68 IF NOT attr_def.multi THEN
69 attr_value := ARRAY[attr_value[1]];
74 IF NOT jump_past AND attr_def.xpath IS NOT NULL THEN -- and xpath expression
76 SELECT INTO xfrm * FROM config.xml_transform WHERE name = attr_def.format;
78 -- See if we can skip the XSLT ... it's expensive
79 IF prev_xfrm IS NULL OR prev_xfrm <> xfrm.name THEN
80 -- Can't skip the transform
81 IF xfrm.xslt <> '---' THEN
82 transformed_xml := oils_xslt_process(rmarc,xfrm.xslt);
84 transformed_xml := rmarc;
87 prev_xfrm := xfrm.name;
90 IF xfrm.name IS NULL THEN
91 -- just grab the marcxml (empty) transform
92 SELECT INTO xfrm * FROM config.xml_transform WHERE xslt = '---' LIMIT 1;
93 prev_xfrm := xfrm.name;
96 FOR tmp_xml IN SELECT UNNEST(oils_xpath(attr_def.xpath, transformed_xml, ARRAY[ARRAY[xfrm.prefix, xfrm.namespace_uri]])) LOOP
97 tmp_val := oils_xpath_string(
100 COALESCE(attr_def.joiner,' '),
101 ARRAY[ARRAY[xfrm.prefix, xfrm.namespace_uri]]
103 IF tmp_val IS NOT NULL AND BTRIM(tmp_val) <> '' THEN
104 attr_value := attr_value || tmp_val;
105 EXIT WHEN NOT attr_def.multi;
110 IF NOT jump_past AND attr_def.phys_char_sf IS NOT NULL THEN -- a named Physical Characteristic, see config.marc21_physical_characteristic_*_map
111 SELECT ARRAY_AGG(m.value) INTO tmp_array
112 FROM vandelay.marc21_physical_characteristics(rmarc) v
113 LEFT JOIN config.marc21_physical_characteristic_value_map m ON (m.id = v.value)
114 WHERE v.subfield = attr_def.phys_char_sf AND (m.value IS NOT NULL AND BTRIM(m.value) <> '')
115 AND ( ccvm_row.id IS NULL OR ( ccvm_row.id IS NOT NULL AND v.id IS NOT NULL) );
117 attr_value := attr_value || tmp_array;
119 IF NOT attr_def.multi THEN
120 attr_value := ARRAY[attr_value[1]];
125 -- apply index normalizers to attr_value
126 FOR tmp_val IN SELECT value FROM UNNEST(attr_value) x(value) LOOP
128 SELECT n.func AS func,
129 n.param_count AS param_count,
131 FROM config.index_normalizer n
132 JOIN config.record_attr_index_norm_map m ON (m.norm = n.id)
133 WHERE attr = attr_def.name
135 EXECUTE 'SELECT ' || normalizer.func || '(' ||
136 COALESCE( quote_literal( tmp_val ), 'NULL' ) ||
138 WHEN normalizer.param_count > 0
139 THEN ',' || REPLACE(REPLACE(BTRIM(normalizer.params,'[]'),E'\'',E'\\\''),E'"',E'\'')
145 IF tmp_val IS NOT NULL AND tmp_val <> '' THEN
146 -- note that a string that contains only blanks
147 -- is a valid value for some attributes
148 norm_attr_value := norm_attr_value || tmp_val;
152 IF attr_def.filter THEN
153 -- Create unknown uncontrolled values and find the IDs of the values
154 IF ccvm_row.id IS NULL THEN
155 FOR tmp_val IN SELECT value FROM UNNEST(norm_attr_value) x(value) LOOP
156 IF tmp_val IS NOT NULL AND BTRIM(tmp_val) <> '' THEN
157 BEGIN -- use subtransaction to isolate unique constraint violations
158 INSERT INTO metabib.uncontrolled_record_attr_value ( attr, value ) VALUES ( attr_def.name, tmp_val );
159 EXCEPTION WHEN unique_violation THEN END;
163 SELECT ARRAY_AGG(id) INTO attr_vector_tmp FROM metabib.uncontrolled_record_attr_value WHERE attr = attr_def.name AND value = ANY( norm_attr_value );
165 SELECT ARRAY_AGG(id) INTO attr_vector_tmp FROM config.coded_value_map WHERE ctype = attr_def.name AND code = ANY( norm_attr_value );
168 -- Add the new value to the vector
169 attr_vector := attr_vector || attr_vector_tmp;
174 IF ARRAY_LENGTH(pattr_list, 1) > 0 THEN
175 SELECT vlist INTO attr_vector_tmp FROM metabib.record_attr_vector_list WHERE source = rid;
176 SELECT attr_vector_tmp - ARRAY_AGG(id::INT) INTO attr_vector_tmp FROM metabib.full_attr_id_map WHERE attr = ANY (pattr_list);
177 attr_vector := attr_vector || attr_vector_tmp;
180 -- On to composite attributes, now that the record attrs have been pulled. Processed in name order, so later composite
181 -- attributes can depend on earlier ones.
182 PERFORM metabib.compile_composite_attr_cache_init();
183 FOR attr_def IN SELECT * FROM config.record_attr_definition WHERE composite AND name = ANY( attr_list ) ORDER BY name LOOP
185 FOR ccvm_row IN SELECT * FROM config.coded_value_map c WHERE c.ctype = attr_def.name ORDER BY value LOOP
187 tmp_val := metabib.compile_composite_attr( ccvm_row.id );
188 CONTINUE WHEN tmp_val IS NULL OR tmp_val = ''; -- nothing to do
190 IF attr_def.filter THEN
191 IF attr_vector @@ tmp_val::query_int THEN
192 attr_vector = attr_vector + intset(ccvm_row.id);
193 EXIT WHEN NOT attr_def.multi;
197 IF attr_def.sorter THEN
198 IF attr_vector @@ tmp_val THEN
199 DELETE FROM metabib.record_sorter WHERE source = rid AND attr = attr_def.name;
200 INSERT INTO metabib.record_sorter (source, attr, value) VALUES (rid, attr_def.name, ccvm_row.code);
210 $func$ LANGUAGE PLPGSQL;
213 CREATE OR REPLACE FUNCTION migration_tools.view_staged_vlist (rid BIGINT) RETURNS TABLE (r_ctype text, r_code text, r_value text) AS $func$
219 SELECT reingest_staged_record_attributes(rid) INTO vlist;
221 RETURN QUERY SELECT ctype, code, value FROM config.coded_value_map WHERE id IN (SELECT UNNEST(vlist));
223 $func$ LANGUAGE PLPGSQL;
225 CREATE OR REPLACE FUNCTION migration_tools.stage_vlist (rid BIGINT) RETURNS VOID AS $func$
230 SELECT reingest_staged_record_attributes(rid) INTO vlist;
232 SELECT ARRAY_AGG(code) FROM config.coded_value_map WHERE id IN (SELECT UNNEST(vlist))
233 AND ctype = 'search_format' INTO search;
235 UPDATE biblio_record_entry_legacy SET x_search_format = search WHERE id = rid;
237 $func$ LANGUAGE PLPGSQL;
239 CREATE OR REPLACE FUNCTION migration_tools.show_staged_vlist (rid BIGINT) RETURNS TEXT[] AS $func$
244 SELECT reingest_staged_record_attributes(rid) INTO vlist;
246 SELECT ARRAY_AGG(code) FROM config.coded_value_map WHERE id IN (SELECT UNNEST(vlist))
247 AND ctype = 'search_format' INTO search;
251 $func$ LANGUAGE PLPGSQL;
254 CREATE OR REPLACE FUNCTION migration_tools.postfix_vlist (rid BIGINT) RETURNS VOID AS $func$
259 SELECT reingest_staged_record_attributes(rid) INTO vlist;
261 SELECT ARRAY_AGG(code) FROM config.coded_value_map WHERE id IN (SELECT UNNEST(vlist))
262 AND ctype = 'search_format' INTO search;
264 UPDATE biblio_record_entry_legacy SET x_after_search_format = search WHERE id = rid;
266 $func$ LANGUAGE PLPGSQL;
269 CREATE OR REPLACE FUNCTION migration_tools.set_exp_sfs (rid BIGINT) RETURNS VOID AS $func$
275 SELECT circ_mods FROM biblio_record_entry_legacy WHERE id = rid INTO cms;
276 IF cms IS NOT NULL THEN
277 FOREACH y IN ARRAY cms LOOP
278 w := w || (SELECT sf1 FROM circ_mod_to_sf_map WHERE circ_mod = y);
279 w := w || (SELECT sf2 FROM circ_mod_to_sf_map WHERE circ_mod = y);
280 w := w || (SELECT sf3 FROM circ_mod_to_sf_map WHERE circ_mod = y);
282 UPDATE biblio_record_entry_legacy SET expected_sfs = w WHERE id = rid;
285 $func$ LANGUAGE PLPGSQL;
287 DROP AGGREGATE IF EXISTS anyarray_agg(anyarray);
288 CREATE AGGREGATE anyarray_agg(anyarray) (
289 SFUNC = anyarray_agg_statefunc,
292 COMMENT ON AGGREGATE anyarray_agg(anyarray) IS
293 'Concatenates arrays into a single array when aggregating.';
295 DROP FUNCTION IF EXISTS migration_tools.anyarray_agg_statefunc(anyarray, anyarray);
296 CREATE FUNCTION anyarray_agg_statefunc(state anyarray, value anyarray)
299 SELECT array_cat($1, $2)
301 LANGUAGE sql IMMUTABLE;
302 COMMENT ON FUNCTION anyarray_agg_statefunc(anyarray, anyarray) IS
303 'Used internally by aggregate anyarray_agg(anyarray).';
305 DROP FUNCTION IF EXISTS migration_tools.anyarray_sort(anyarray);
306 CREATE OR REPLACE FUNCTION migration_tools.anyarray_sort(with_array anyarray)
310 return_array with_array%TYPE := '{}';
312 SELECT ARRAY_AGG(sorted_vals.val) AS array_value
314 ( SELECT UNNEST(with_array) AS val
316 ) AS sorted_vals INTO return_array;
319 $BODY$ LANGUAGE plpgsql;
321 DROP FUNCTION IF EXISTS migration_tools.anyarray_uniq(anyarray);
322 CREATE OR REPLACE FUNCTION migration_tools.anyarray_uniq(with_array anyarray)
326 -- The variable used to track iteration over "with_array".
329 -- The array to be returned by this function.
330 return_array with_array%TYPE := '{}';
332 IF with_array IS NULL THEN
336 IF with_array = '{}' THEN
340 -- Iterate over each element in "concat_array".
341 FOR loop_offset IN ARRAY_LOWER(with_array, 1)..ARRAY_UPPER(with_array, 1) LOOP
342 IF with_array[loop_offset] IS NULL THEN
344 ( SELECT 1 FROM UNNEST(return_array) AS s(a)
346 THEN return_array = ARRAY_APPEND(return_array, with_array[loop_offset]);
348 -- When an array contains a NULL value, ANY() returns NULL instead of FALSE...
349 ELSEIF NOT(with_array[loop_offset] = ANY(return_array)) OR NOT(NULL IS DISTINCT FROM (with_array[loop_offset] = ANY(return_array))) THEN
350 return_array = ARRAY_APPEND(return_array, with_array[loop_offset]);
356 $BODY$ LANGUAGE plpgsql;
358 DROP FUNCTION IF EXISTS migration_tools.modify_staged_fixed_fields (BIGINT,TEXT);
359 CREATE OR REPLACE FUNCTION migration_tools.modify_staged_fixed_fields (bib_id BIGINT, xcode TEXT)
371 xiform_exclude CHAR(1)[];
372 xsrform_exclude CHAR(1)[];
374 ysrform_exclude TEXT;
376 SELECT itype, iform, phy, phyv, phyp, biblevel, iform_exclude, srform_exclude FROM search_format_map WHERE code = xcode
377 INTO xitype, xiform, xphy, xphyv, xphyp, xbiblevel, xiform_exclude, xsrform_exclude;
378 IF xiform_exclude IS NOT NULL THEN
379 yiform_exclude := ARRAY_TO_STRING(xiform_exclude,',');
381 yiform_exclude := '';
383 IF xsrform_exclude IS NOT NULL THEN
384 ysrform_exclude := ARRAY_TO_STRING(ysrform_exclude,',');
386 ysrform_exclude := '';
388 SELECT modify_fixed_fields(marc,xcode,xitype,xiform,xphy,xphyv,xphyp,xbiblevel,yiform_exclude,ysrform_exclude) FROM biblio_record_entry_legacy WHERE id = bib_id INTO r;
389 UPDATE biblio_record_entry_legacy SET marc = r WHERE id = bib_id;
394 DROP FUNCTION IF EXISTS migration_tools.modify_fixed_fields (TEXT, TEXT, CHAR(1), CHAR(1), CHAR(1), CHAR(1), SMALLINT, CHAR(1), TEXT, TEXT);
395 CREATE OR REPLACE FUNCTION migration_tools.modify_fixed_fields (TEXT, TEXT, CHAR(1), CHAR(1), CHAR(1), CHAR(1), SMALLINT, CHAR(1), TEXT, TEXT)
400 # assumption is that there should only be a single format per item
406 use MARC::File::XML (BinaryEncoding => 'utf8');
410 my ($marcxml, $code, $itype, $iform, $phy, $phyv, $phyp, $biblevel, $iform_exclude_temp, $srform_exclude_temp) = @_;
413 if ($iform_exclude_temp) { @iform_exclude = split /,/, $iform_exclude_temp; }
415 if ($srform_exclude_temp) { @srform_exclude = split /,/, $srform_exclude_temp; }
417 $marcxml =~ s/(<leader>.........)./${1}a/;
418 eval { $marc = MARC::Record->new_from_xml($marcxml, 'UTF-8'); };
420 import MARC::File::XML (BinaryEncoding => 'utf8');
421 return 'failed to parse marcxml';
424 my $ldr = $marc->leader();
425 if ($itype) { substr($ldr,6,1) = $itype; } else { substr($ldr,6,1) = '|'; }
426 if ($biblevel) { substr($ldr,7,1) = $biblevel; } else { substr($ldr,7,1) = '|'; }
430 my $zze_str = '0000000000000000000000000000000000000000';
433 $zedzedeight = $marc->field('008');
435 $zze_str = $zedzedeight->data();
437 if (length($zze_str) < 30) {
438 my $nneight = MARC::Field->new( 918, '1', '0', 'a' => $zze_str );
439 $marc->insert_fields_ordered($nneight);
440 $zze_str = '0000000000000000000000000000000000000000';
442 if ($itype eq 'e' or $itype eq 'g' or $itype eq 'k')
443 { $formchar = substr($zze_str,29,1); }
444 else { $formchar = substr($zze_str,23,1); }
445 if (@iform_exclude and $itype) {
446 if ($itype eq 'e' or $itype eq 'g' or $itype eq 'k') { #visual materials
447 if ($formchar ~~ @iform_exclude) { substr($zze_str,29,1) = '|'; }
448 } else { if ($formchar ~~ @iform_exclude) { substr($zze_str,23,1) = '|'; } }
451 if ($itype eq 'e' or $itype eq 'g' or $itype eq 'k') { #visual materials
452 substr($zze_str,29,1) = $iform;
454 substr($zze_str,23,1) = $iform;
457 if ($itype eq 'e' or $itype eq 'g' or $itype eq 'k') { #visual materials
458 substr($zze_str,29,1) = '|';
460 substr($zze_str,23,1) = '|';
464 $new_zze = MARC::Field->new('008',$zze_str);
465 if ($zedzedeight) { $zedzedeight->replace_with($new_zze); } else
466 { $marc->insert_fields_ordered($new_zze); }
468 my @todelzzsx = $marc->field('006');
469 #save the old 006s in 916 fields
470 foreach my $sx (@todelzzsx) {
471 my $nfield = MARC::Field->new( 916, '1', '0', 'a' => $sx->data() );
472 $marc->insert_fields_ordered($nfield);
474 $marc->delete_fields(@todelzzsx);
476 my $zzsx_str = '00000000000000000';
477 if ($iform) { substr($zzsx_str,6,1) = $iform; }
478 my $zedzedsix = MARC::Field->new('006', $zzsx_str);
479 $marc->insert_fields_ordered($zedzedsix);
481 my @todelzzsv = $marc->field('007');
482 #save the old 007s in 917 fields
483 foreach my $sv (@todelzzsv) {
484 my $nfield = MARC::Field->new( 917, '1', '0', 'a' => $sv->data() );
485 $marc->insert_fields_ordered($nfield);
487 $marc->delete_fields(@todelzzsv);
489 my $nn = MARC::Field->new( 919, '1', '0', 'a' => 'record modified by automated fixed field changes' );
490 $marc->insert_fields_ordered($nn);
495 if ($phy eq 'o' or $phy eq 'q' or $phy eq 'z' or $phy eq 't') { $zzs_str = '00'; }
496 if ($phy eq 's' or $phy eq 'c') { $zzs_str = '00000000000000'; }
497 if ($phy eq 'r') { $zzs_str = '00000000000'; }
498 if ($phy eq 'm') { $zzs_str = '00000000000000000000000'; }
499 if ($phy eq 'a') { $zzs_str = '00000000'; }
500 if ($phy eq 'd') { $zzs_str = '000000'; }
501 if ($phy eq 'f') { $zzs_str = '0000000000'; }
502 if ($phy eq 'g') { $zzs_str = '000000000'; }
503 if ($phy eq 'h') { $zzs_str = '0000000000000'; }
504 if ($phy eq 'k') { $zzs_str = '000000'; }
505 if ($phy eq 'v') { $zzs_str = '000000000'; }
506 substr($zzs_str,0,1) = $phy;
507 substr($zzs_str,$phyp,1) = $phyv;
508 $zedzedseven = MARC::Field->new('007', $zzs_str);
509 $marc->insert_fields_ordered($zedzedseven);
511 return $marc->as_xml_record;