X-Git-Url: http://git.equinoxoli.org/?p=migration-tools.git;a=blobdiff_plain;f=sql%2Fbase%2Fbase.sql;h=f5c120afc4101142df033b43e4688a4fa960064e;hp=4e715d5433085b968397e1a382402dd3e7187e6c;hb=1f7d2d142333cfb4edfa33c374ae2841116add6d;hpb=3b69a3f894fc8ecd477e0017e534659cf3b655c3

diff --git a/sql/base/base.sql b/sql/base/base.sql
index 4e715d5..f5c120a 100644
--- a/sql/base/base.sql
+++ b/sql/base/base.sql
@@ -23,7 +23,7 @@
 -- SELECT migration_tools.build('foo');
 -- SELECT * FROM foo.fields_requiring_mapping;
 -- \d foo.actor_usr
--- create some incoming ILS specific staging tables, like CREATE foo.legacy_items ( l_barcode TEXT, .. ) INHERITS foo.asset_copy;
+-- create some incoming ILS specific staging tables, like CREATE foo.legacy_items ( l_barcode TEXT, .. ) INHERITS (foo.asset_copy);
 -- Do some mapping, like UPDATE foo.legacy_items SET barcode = TRIM(BOTH ' ' FROM l_barcode);
 -- Then, to move into production, do: select migration_tools.insert_base_into_production('foo')
 
@@ -117,7 +117,7 @@ CREATE OR REPLACE FUNCTION migration_tools.init (TEXT) RETURNS VOID AS $$
         END;
         PERFORM migration_tools.exec( $1, 'DROP TABLE IF EXISTS ' || migration_schema || '.config;' );
         PERFORM migration_tools.exec( $1, 'CREATE TABLE ' || migration_schema || '.config ( key TEXT UNIQUE, value TEXT);' );
-        PERFORM migration_tools.exec( $1, 'INSERT INTO ' || migration_schema || '.config (key,value) VALUES ( ''production_tables'', ''asset.call_number,asset.call_number_prefix,asset.call_number_suffix,asset.copy_location,asset.copy,asset.stat_cat,asset.stat_cat_entry,asset.stat_cat_entry_copy_map,asset.copy_note,actor.usr,actor.card,actor.usr_address,actor.stat_cat,actor.stat_cat_entry,actor.stat_cat_entry_usr_map,actor.usr_note,actor.usr_standing_penalty,action.circulation,action.hold_request,action.hold_notification,action.hold_request_note,action.hold_transit_copy,action.transit_copy,money.grocery,money.billing,money.cash_payment,money.forgive_payment,acq.provider,acq.provider_address,acq.provider_note,acq.provider_contact,acq.provider_contact_address,acq.fund,acq.fund_allocation,acq.fund_tag,acq.fund_tag_map,acq.funding_source,acq.funding_source_credit,acq.lineitem,acq.purchase_order,acq.po_item,acq.invoice,acq.invoice_item,acq.invoice_entry,acq.lineitem_detail,acq.fund_debit,acq.fund_transfer,acq.po_note,config.circ_matrix_matchpoint,config.circ_matrix_limit_set_map,config.hold_matrix_matchpoint'' );' );
+        PERFORM migration_tools.exec( $1, 'INSERT INTO ' || migration_schema || '.config (key,value) VALUES ( ''production_tables'', ''asset.call_number,asset.call_number_prefix,asset.call_number_suffix,asset.copy_location,asset.copy,asset.copy_alert,asset.stat_cat,asset.stat_cat_entry,asset.stat_cat_entry_copy_map,asset.copy_note,actor.usr,actor.card,actor.usr_address,actor.stat_cat,actor.stat_cat_entry,actor.stat_cat_entry_usr_map,actor.usr_note,actor.usr_standing_penalty,actor.usr_setting,action.circulation,action.hold_request,action.hold_notification,action.hold_request_note,action.hold_transit_copy,action.transit_copy,money.grocery,money.billing,money.cash_payment,money.forgive_payment,acq.provider,acq.provider_address,acq.provider_note,acq.provider_contact,acq.provider_contact_address,acq.fund,acq.fund_allocation,acq.fund_tag,acq.fund_tag_map,acq.funding_source,acq.funding_source_credit,acq.lineitem,acq.purchase_order,acq.po_item,acq.invoice,acq.invoice_item,acq.invoice_entry,acq.lineitem_detail,acq.fund_debit,acq.fund_transfer,acq.po_note,config.circ_matrix_matchpoint,config.circ_matrix_limit_set_map,config.hold_matrix_matchpoint,asset.copy_tag,asset.copy_tag_copy_map,config.copy_tag_type,serial.item,serial.item_note,serial.record_entry,biblio.record_entry'' );' );
         PERFORM migration_tools.exec( $1, 'INSERT INTO ' || migration_schema || '.config (key,value) VALUES ( ''country_code'', ''USA'' );' );
         PERFORM migration_tools.exec( $1, 'DROP TABLE IF EXISTS ' || migration_schema || '.fields_requiring_mapping;' );
         PERFORM migration_tools.exec( $1, 'CREATE TABLE ' || migration_schema || '.fields_requiring_mapping( table_schema TEXT, table_name TEXT, column_name TEXT, data_type TEXT);' );
@@ -261,6 +261,45 @@ CREATE OR REPLACE FUNCTION migration_tools.build_specific_base_staging_table (TE
     END;
 $$ LANGUAGE PLPGSQL STRICT VOLATILE;
 
+CREATE OR REPLACE FUNCTION migration_tools.create_linked_legacy_table_from (TEXT,TEXT,TEXT) RETURNS VOID AS $$
+    DECLARE
+        migration_schema ALIAS FOR $1;
+        parent_table ALIAS FOR $2;
+        source_table ALIAS FOR $3;
+        columns RECORD;
+        create_sql TEXT;
+        insert_sql TEXT;
+        column_list TEXT := '';
+        column_count INTEGER := 0;
+    BEGIN
+        create_sql := 'CREATE TABLE ' || migration_schema || '.' || parent_table || '_legacy ( ';
+        FOR columns IN
+            SELECT table_schema, table_name, column_name, data_type
+            FROM information_schema.columns
+            WHERE table_schema = migration_schema AND table_name = source_table
+        LOOP
+            column_count := column_count + 1;
+            if column_count > 1 then
+                create_sql := create_sql || ', ';
+                column_list := column_list || ', ';
+            end if;
+            create_sql := create_sql || columns.column_name || ' ';
+            if columns.data_type = 'ARRAY' then
+                create_sql := create_sql || 'TEXT[]';
+            else
+                create_sql := create_sql || columns.data_type;
+            end if;
+            column_list := column_list || columns.column_name;
+        END LOOP;
+        create_sql := create_sql || ' ) INHERITS ( ' || migration_schema || '.' || parent_table || ' );';
+        --RAISE INFO 'create_sql = %', create_sql;
+        EXECUTE create_sql;
+        insert_sql := 'INSERT INTO ' || migration_schema || '.' || parent_table || '_legacy (' || column_list || ') SELECT ' || column_list || ' FROM ' || migration_schema || '.' || source_table || ';';
+        --RAISE INFO 'insert_sql = %', insert_sql;
+        EXECUTE insert_sql;
+    END;
+$$ LANGUAGE PLPGSQL STRICT VOLATILE;
+
 CREATE OR REPLACE FUNCTION migration_tools.insert_base_into_production (TEXT) RETURNS VOID AS $$
     DECLARE
         migration_schema ALIAS FOR $1;
@@ -287,6 +326,31 @@ CREATE OR REPLACE FUNCTION migration_tools.insert_into_production (TEXT,TEXT) RE
     END;
 $$ LANGUAGE PLPGSQL STRICT VOLATILE;
 
+CREATE OR REPLACE FUNCTION migration_tools.name_parse_out_first_middle_last_comma_suffix (TEXT) RETURNS TEXT[] AS $$
+    DECLARE
+        full_name TEXT := $1;
+        before_comma TEXT;
+        family_name TEXT := '';
+        first_given_name TEXT := '';
+        second_given_name TEXT := '';
+        suffix TEXT := '';
+        prefix TEXT := '';
+    BEGIN
+        before_comma := BTRIM( REGEXP_REPLACE(full_name,E'^(.+),.+$',E'\\1') );
+        suffix := CASE WHEN full_name ~ ',' THEN BTRIM( REGEXP_REPLACE(full_name,E'^.+,(.+)$',E'\\1') ) ELSE '' END;
+
+        IF suffix = before_comma THEN
+            suffix := '';
+        END IF;
+
+        family_name := BTRIM( REGEXP_REPLACE(before_comma,E'^.+\\s(.+)$',E'\\1') );
+        first_given_name := BTRIM( REGEXP_REPLACE(before_comma,E'^(.+?)\\s.+$',E'\\1') );
+        second_given_name := BTRIM( CASE WHEN before_comma ~ '^.+\s.+\s.+$' THEN REGEXP_REPLACE(before_comma,E'^.+\\s(.+)\\s.+$',E'\\1') ELSE '' END );
+
+        RETURN ARRAY[ family_name, prefix, first_given_name, second_given_name, suffix ];
+    END;
+$$ LANGUAGE PLPGSQL STRICT IMMUTABLE;
+
 CREATE OR REPLACE FUNCTION migration_tools.name_parse_out_last_comma_prefix_first_middle_suffix (TEXT) RETURNS TEXT[] AS $$
     DECLARE
         full_name TEXT := $1;
@@ -421,6 +485,185 @@ CREATE OR REPLACE FUNCTION migration_tools.name_parse_out_last_first_middle_and_
     END;
 $$ LANGUAGE PLPGSQL STRICT IMMUTABLE;
 
+CREATE OR REPLACE FUNCTION migration_tools.name_parse_out_fuller_last_first_middle_and_random_affix (TEXT) RETURNS TEXT[] AS $$
+    DECLARE
+        full_name TEXT := $1;
+        temp TEXT;
+        family_name TEXT := '';
+        first_given_name TEXT := '';
+        second_given_name TEXT := '';
+        suffix TEXT := '';
+        prefix TEXT := '';
+    BEGIN
+        temp := BTRIM(full_name);
+        -- Use values, not structure, for prefix/suffix, unless we come up with a better idea
+        --IF temp ~ '^\S{2,}\.' THEN
+        --    prefix := REGEXP_REPLACE(temp, '^(\S{2,}\.).*$','\1');
+        --    temp := BTRIM(REGEXP_REPLACE(temp, '^\S{2,}\.(.*)$','\1'));
+        --END IF;
+        --IF temp ~ '\S{2,}\.$' THEN
+        --    suffix := REGEXP_REPLACE(temp, '^.*(\S{2,}\.)$','\1');
+        --    temp := REGEXP_REPLACE(temp, '^(.*)\S{2,}\.$','\1');
+        --END IF;
+        IF temp ilike '%MR.%' THEN
+            prefix := 'Mr.';
+            temp := BTRIM(REGEXP_REPLACE( temp, E'MR\.\\s*', '', 'i' ));
+        END IF;
+        IF temp ilike '%MRS.%' THEN
+            prefix := 'Mrs.';
+            temp := BTRIM(REGEXP_REPLACE( temp, E'MRS\.\\s*', '', 'i' ));
+        END IF;
+        IF temp ilike '%MS.%' THEN
+            prefix := 'Ms.';
+            temp := BTRIM(REGEXP_REPLACE( temp, E'MS\.\\s*', '', 'i' ));
+        END IF;
+        IF temp ilike '%DR.%' THEN
+            prefix := 'Dr.';
+            temp := BTRIM(REGEXP_REPLACE( temp, E'DR\.\\s*', '', 'i' ));
+        END IF;
+        IF temp ilike '%JR.%' THEN
+            suffix := 'Jr.';
+            temp := BTRIM(REGEXP_REPLACE( temp, E'JR\.\\s*', '', 'i' ));
+        END IF;
+        IF temp ilike '%JR,%' THEN
+            suffix := 'Jr.';
+            temp := BTRIM(REGEXP_REPLACE( temp, E'JR,\\s*', ',', 'i' ));
+        END IF;
+        IF temp ilike '%SR.%' THEN
+            suffix := 'Sr.';
+            temp := BTRIM(REGEXP_REPLACE( temp, E'SR\.\\s*', '', 'i' ));
+        END IF;
+        IF temp ilike '%SR,%' THEN
+            suffix := 'Sr.';
+            temp := BTRIM(REGEXP_REPLACE( temp, E'SR,\\s*', ',', 'i' ));
+        END IF;
+        IF temp like '%III%' THEN
+            suffix := 'III';
+            temp := BTRIM(REGEXP_REPLACE( temp, E'III', '' ));
+        END IF;
+        IF temp like '%II%' THEN
+            suffix := 'II';
+            temp := BTRIM(REGEXP_REPLACE( temp, E'II', '' ));
+        END IF;
+
+        IF temp ~ ',' THEN
+            family_name = BTRIM(REGEXP_REPLACE(temp,'^(.*?,).*$','\1'));
+            temp := BTRIM(REPLACE( temp, family_name, '' ));
+            family_name := REPLACE( family_name, ',', '' );
+            IF temp ~ ' ' THEN
+                first_given_name := BTRIM( REGEXP_REPLACE(temp,'^(.+)\s(.+)$','\1') );
+                second_given_name := BTRIM( REGEXP_REPLACE(temp,'^(.+)\s(.+)$','\2') );
+            ELSE
+                first_given_name := temp;
+                second_given_name := '';
+            END IF;
+        ELSE
+            IF temp ~ '^\S+\s+\S+\s+\S+$' THEN
+                first_given_name := BTRIM( REGEXP_REPLACE(temp,'^(\S+)\s*(\S+)\s*(\S+)$','\1') );
+                second_given_name := BTRIM( REGEXP_REPLACE(temp,'^(\S+)\s*(\S+)\s*(\S+)$','\2') );
+                family_name := BTRIM( REGEXP_REPLACE(temp,'^(\S+)\s*(\S+)\s*(\S+)$','\3') );
+            ELSE
+                first_given_name := BTRIM( REGEXP_REPLACE(temp,'^(\S+)\s*(\S+)$','\1') );
+                second_given_name := temp;
+                family_name := BTRIM( REGEXP_REPLACE(temp,'^(\S+)\s*(\S+)$','\2') );
+            END IF;
+        END IF;
+
+        RETURN ARRAY[ family_name, prefix, first_given_name, second_given_name, suffix ];
+    END;
+$$ LANGUAGE PLPGSQL STRICT IMMUTABLE;
+
+CREATE OR REPLACE FUNCTION migration_tools.name_parse_out_fuller_last_first_middle_and_random_affix2 (TEXT) RETURNS TEXT[] AS $$
+    DECLARE
+        full_name TEXT := $1;
+        temp TEXT;
+        family_name TEXT := '';
+        first_given_name TEXT := '';
+        second_given_name TEXT := '';
+        suffix TEXT := '';
+        prefix TEXT := '';
+    BEGIN
+        temp := BTRIM(full_name);
+        -- Use values, not structure, for prefix/suffix, unless we come up with a better idea
+        --IF temp ~ '^\S{2,}\.' THEN
+        --    prefix := REGEXP_REPLACE(temp, '^(\S{2,}\.).*$','\1');
+        --    temp := BTRIM(REGEXP_REPLACE(temp, '^\S{2,}\.(.*)$','\1'));
+        --END IF;
+        --IF temp ~ '\S{2,}\.$' THEN
+        --    suffix := REGEXP_REPLACE(temp, '^.*(\S{2,}\.)$','\1');
+        --    temp := REGEXP_REPLACE(temp, '^(.*)\S{2,}\.$','\1');
+        --END IF;
+        IF temp ilike '%MR.%' THEN
+            prefix := 'Mr.';
+            temp := BTRIM(REGEXP_REPLACE( temp, E'MR\.\\s*', '', 'i' ));
+        END IF;
+        IF temp ilike '%MRS.%' THEN
+            prefix := 'Mrs.';
+            temp := BTRIM(REGEXP_REPLACE( temp, E'MRS\.\\s*', '', 'i' ));
+        END IF;
+        IF temp ilike '%MS.%' THEN
+            prefix := 'Ms.';
+            temp := BTRIM(REGEXP_REPLACE( temp, E'MS\.\\s*', '', 'i' ));
+        END IF;
+        IF temp ilike '%DR.%' THEN
+            prefix := 'Dr.';
+            temp := BTRIM(REGEXP_REPLACE( temp, E'DR\.\\s*', '', 'i' ));
+        END IF;
+        IF temp ilike '%JR.%' THEN
+            suffix := 'Jr.';
+            temp := BTRIM(REGEXP_REPLACE( temp, E'JR\.\\s*', '', 'i' ));
+        END IF;
+        IF temp ilike '%JR,%' THEN
+            suffix := 'Jr.';
+            temp := BTRIM(REGEXP_REPLACE( temp, E'JR,\\s*', ',', 'i' ));
+        END IF;
+        IF temp ilike '%SR.%' THEN
+            suffix := 'Sr.';
+            temp := BTRIM(REGEXP_REPLACE( temp, E'SR\.\\s*', '', 'i' ));
+        END IF;
+        IF temp ilike '%SR,%' THEN
+            suffix := 'Sr.';
+            temp := BTRIM(REGEXP_REPLACE( temp, E'SR,\\s*', ',', 'i' ));
+        END IF;
+        IF temp like '%III%' THEN
+            suffix := 'III';
+            temp := BTRIM(REGEXP_REPLACE( temp, E'III', '' ));
+        END IF;
+        IF temp like '%II%' THEN
+            suffix := 'II';
+            temp := BTRIM(REGEXP_REPLACE( temp, E'II', '' ));
+        END IF;
+
+        IF temp ~ ',' THEN
+            family_name = BTRIM(REGEXP_REPLACE(temp,'^(.*?,).*$','\1'));
+            temp := BTRIM(REPLACE( temp, family_name, '' ));
+            family_name := REPLACE( family_name, ',', '' );
+            IF temp ~ ' ' THEN
+                first_given_name := BTRIM( REGEXP_REPLACE(temp,'^(.+)\s(.+)$','\1') );
+                second_given_name := BTRIM( REGEXP_REPLACE(temp,'^(.+)\s(.+)$','\2') );
+            ELSE
+                first_given_name := temp;
+                second_given_name := '';
+            END IF;
+        ELSE
+            IF temp ~ '^\S+\s+\S+\s+\S+$' THEN
+                first_given_name := BTRIM( REGEXP_REPLACE(temp,'^(\S+)\s*(\S+)\s*(\S+)$','\1') );
+                second_given_name := BTRIM( REGEXP_REPLACE(temp,'^(\S+)\s*(\S+)\s*(\S+)$','\2') );
+                family_name := BTRIM( REGEXP_REPLACE(temp,'^(\S+)\s*(\S+)\s*(\S+)$','\3') );
+            ELSE
+                first_given_name := BTRIM( REGEXP_REPLACE(temp,'^(\S+)\s*(\S+)$','\1') );
+                second_given_name := temp;
+                family_name := BTRIM( REGEXP_REPLACE(temp,'^(\S+)\s*(\S+)$','\2') );
+            END IF;
+        END IF;
+
+        family_name := BTRIM(REPLACE(REPLACE(family_name,',',''),'"',''));
+        first_given_name := BTRIM(REPLACE(REPLACE(first_given_name,',',''),'"',''));
+        second_given_name := BTRIM(REPLACE(REPLACE(second_given_name,',',''),'"',''));
+
+        RETURN ARRAY[ family_name, prefix, first_given_name, second_given_name, suffix ];
+    END;
+$$ LANGUAGE PLPGSQL STRICT IMMUTABLE;
 
 CREATE OR REPLACE FUNCTION migration_tools.address_parse_out_citystatezip (TEXT) RETURNS TEXT[] AS $$
     DECLARE
@@ -452,6 +695,648 @@ CREATE OR REPLACE FUNCTION migration_tools.address_parse_out_citystatezip (TEXT)
     END;
 $$ LANGUAGE PLPGSQL STRICT VOLATILE;
 
+-- try to parse data like this: 100 MAIN ST$COVINGTON, GA 30016
+CREATE OR REPLACE FUNCTION migration_tools.parse_out_address (TEXT) RETURNS TEXT[] AS $$
+    DECLARE
+        fullstring TEXT := $1;
+        address1 TEXT := '';
+        address2 TEXT := '';
+        scratch1 TEXT := '';
+        scratch2 TEXT := '';
+        city TEXT := '';
+        state TEXT := '';
+        zip TEXT := '';
+    BEGIN
+        zip := CASE
+            WHEN fullstring ~ E'\\d\\d\\d\\d\\d'
+            THEN REGEXP_REPLACE( fullstring, E'^.*(\\d\\d\\d\\d\\d-?\\d*).*$', E'\\1' )
+            ELSE ''
+        END;
+        fullstring := REGEXP_REPLACE( fullstring, E'^(.*)\\d\\d\\d\\d\\d-?\\d*(.*)$', E'\\1\\2');
+
+        IF fullstring ~ ',' THEN
+            state := REGEXP_REPLACE( fullstring, E'^(.*),(.*)$', E'\\2');
+            scratch1 := REGEXP_REPLACE( fullstring, E'^(.*),(.*)$', E'\\1');
+        ELSE
+            IF fullstring ~ E'\\s+[A-Z][A-Z]\\s*$' THEN
+                state := REGEXP_REPLACE( fullstring, E'^.*,?\\s+([A-Z][A-Z])\\s*$', E'\\1' );
+                scratch1 := REGEXP_REPLACE( fullstring, E'^(.*?),?\\s+[A-Z][A-Z](\\s*)$', E'\\1\\2' );
+            ELSE
+                IF fullstring ~ E'^\\S+$'  THEN
+                    scratch1 := fullstring;
+                    state := 'N/A';
+                ELSE
+                    state := REGEXP_REPLACE( fullstring, E'^(.*?),?\\s*(\\S+)\\s*$', E'\\2');
+                    scratch1 := REGEXP_REPLACE( fullstring, E'^(.*?),?\\s*(\\S+)\\s*$', E'\\1');
+                END IF;
+            END IF;
+        END IF;
+
+        IF scratch1 ~ '[\$]' THEN
+            scratch2 := REGEXP_REPLACE( scratch1, E'^(.+)[\$](.+?)$', E'\\1');
+            city := REGEXP_REPLACE( scratch1, E'^(.+)[\$](.+?)$', E'\\2');
+        ELSE
+            IF scratch1 ~ '\s' THEN
+                scratch2 := REGEXP_REPLACE( scratch1, E'^(.+)\\s+(.+?)$', E'\\1');
+                city := REGEXP_REPLACE( scratch1, E'^(.+)\\s+(.+?)$', E'\\2');
+            ELSE
+                scratch2 := 'N/A';
+                city := scratch1;
+            END IF;
+        END IF;
+
+        IF scratch2 ~ '^\d' THEN
+            address1 := scratch2;
+            address2 := '';
+        ELSE
+            address1 := REGEXP_REPLACE( scratch2, E'^(.+?)(\\d.+)$', E'\\1');
+            address2 := REGEXP_REPLACE( scratch2, E'^(.+?)(\\d.+)$', E'\\2');
+        END IF;
+
+        RETURN ARRAY[
+             TRIM(BOTH ' ' FROM address1)
+            ,TRIM(BOTH ' ' FROM address2)
+            ,TRIM(BOTH ' ' FROM city)
+            ,TRIM(BOTH ' ' FROM state)
+            ,TRIM(BOTH ' ' FROM zip)
+        ];
+    END;
+$$ LANGUAGE PLPGSQL STRICT VOLATILE;
+
+CREATE OR REPLACE FUNCTION migration_tools.parse_out_address2 (TEXT) RETURNS TEXT AS $$
+    my ($address) = @_;
+
+    use Geo::StreetAddress::US;
+
+    my $a = Geo::StreetAddress::US->parse_location($address);
+
+    return [
+         "$a->{number} $a->{prefix} $a->{street} $a->{type} $a->{suffix}"
+        ,"$a->{sec_unit_type} $a->{sec_unit_num}"
+        ,$a->{city}
+        ,$a->{state}
+        ,$a->{zip}
+    ];
+$$ LANGUAGE PLPERLU STABLE;
+
+DROP TABLE IF EXISTS migration_tools.usps_suffixes;
+CREATE TABLE migration_tools.usps_suffixes ( suffix_from TEXT, suffix_to TEXT );
+INSERT INTO migration_tools.usps_suffixes VALUES
+    ('ALLEE','ALY'),
+    ('ALLEY','ALY'),
+    ('ALLY','ALY'),
+    ('ALY','ALY'),
+    ('ANEX','ANX'),
+    ('ANNEX','ANX'),
+    ('ANNX','ANX'),
+    ('ANX','ANX'),
+    ('ARCADE','ARC'),
+    ('ARC','ARC'),
+    ('AV','AVE'),
+    ('AVE','AVE'),
+    ('AVEN','AVE'),
+    ('AVENU','AVE'),
+    ('AVENUE','AVE'),
+    ('AVN','AVE'),
+    ('AVNUE','AVE'),
+    ('BAYOO','BYU'),
+    ('BAYOU','BYU'),
+    ('BCH','BCH'),
+    ('BEACH','BCH'),
+    ('BEND','BND'),
+    ('BLF','BLF'),
+    ('BLUF','BLF'),
+    ('BLUFF','BLF'),
+    ('BLUFFS','BLFS'),
+    ('BLVD','BLVD'),
+    ('BND','BND'),
+    ('BOT','BTM'),
+    ('BOTTM','BTM'),
+    ('BOTTOM','BTM'),
+    ('BOUL','BLVD'),
+    ('BOULEVARD','BLVD'),
+    ('BOULV','BLVD'),
+    ('BRANCH','BR'),
+    ('BR','BR'),
+    ('BRDGE','BRG'),
+    ('BRG','BRG'),
+    ('BRIDGE','BRG'),
+    ('BRK','BRK'),
+    ('BRNCH','BR'),
+    ('BROOK','BRK'),
+    ('BROOKS','BRKS'),
+    ('BTM','BTM'),
+    ('BURG','BG'),
+    ('BURGS','BGS'),
+    ('BYPA','BYP'),
+    ('BYPAS','BYP'),
+    ('BYPASS','BYP'),
+    ('BYP','BYP'),
+    ('BYPS','BYP'),
+    ('CAMP','CP'),
+    ('CANYN','CYN'),
+    ('CANYON','CYN'),
+    ('CAPE','CPE'),
+    ('CAUSEWAY','CSWY'),
+    ('CAUSWAY','CSWY'),
+    ('CEN','CTR'),
+    ('CENT','CTR'),
+    ('CENTER','CTR'),
+    ('CENTERS','CTRS'),
+    ('CENTR','CTR'),
+    ('CENTRE','CTR'),
+    ('CIRC','CIR'),
+    ('CIR','CIR'),
+    ('CIRCL','CIR'),
+    ('CIRCLE','CIR'),
+    ('CIRCLES','CIRS'),
+    ('CK','CRK'),
+    ('CLB','CLB'),
+    ('CLF','CLF'),
+    ('CLFS','CLFS'),
+    ('CLIFF','CLF'),
+    ('CLIFFS','CLFS'),
+    ('CLUB','CLB'),
+    ('CMP','CP'),
+    ('CNTER','CTR'),
+    ('CNTR','CTR'),
+    ('CNYN','CYN'),
+    ('COMMON','CMN'),
+    ('COR','COR'),
+    ('CORNER','COR'),
+    ('CORNERS','CORS'),
+    ('CORS','CORS'),
+    ('COURSE','CRSE'),
+    ('COURT','CT'),
+    ('COURTS','CTS'),
+    ('COVE','CV'),
+    ('COVES','CVS'),
+    ('CP','CP'),
+    ('CPE','CPE'),
+    ('CRCL','CIR'),
+    ('CRCLE','CIR'),
+    ('CR','CRK'),
+    ('CRECENT','CRES'),
+    ('CREEK','CRK'),
+    ('CRESCENT','CRES'),
+    ('CRES','CRES'),
+    ('CRESENT','CRES'),
+    ('CREST','CRST'),
+    ('CRK','CRK'),
+    ('CROSSING','XING'),
+    ('CROSSROAD','XRD'),
+    ('CRSCNT','CRES'),
+    ('CRSE','CRSE'),
+    ('CRSENT','CRES'),
+    ('CRSNT','CRES'),
+    ('CRSSING','XING'),
+    ('CRSSNG','XING'),
+    ('CRT','CT'),
+    ('CSWY','CSWY'),
+    ('CT','CT'),
+    ('CTR','CTR'),
+    ('CTS','CTS'),
+    ('CURVE','CURV'),
+    ('CV','CV'),
+    ('CYN','CYN'),
+    ('DALE','DL'),
+    ('DAM','DM'),
+    ('DIV','DV'),
+    ('DIVIDE','DV'),
+    ('DL','DL'),
+    ('DM','DM'),
+    ('DR','DR'),
+    ('DRIV','DR'),
+    ('DRIVE','DR'),
+    ('DRIVES','DRS'),
+    ('DRV','DR'),
+    ('DVD','DV'),
+    ('DV','DV'),
+    ('ESTATE','EST'),
+    ('ESTATES','ESTS'),
+    ('EST','EST'),
+    ('ESTS','ESTS'),
+    ('EXP','EXPY'),
+    ('EXPRESS','EXPY'),
+    ('EXPRESSWAY','EXPY'),
+    ('EXPR','EXPY'),
+    ('EXPW','EXPY'),
+    ('EXPY','EXPY'),
+    ('EXTENSION','EXT'),
+    ('EXTENSIONS','EXTS'),
+    ('EXT','EXT'),
+    ('EXTN','EXT'),
+    ('EXTNSN','EXT'),
+    ('EXTS','EXTS'),
+    ('FALL','FALL'),
+    ('FALLS','FLS'),
+    ('FERRY','FRY'),
+    ('FIELD','FLD'),
+    ('FIELDS','FLDS'),
+    ('FLAT','FLT'),
+    ('FLATS','FLTS'),
+    ('FLD','FLD'),
+    ('FLDS','FLDS'),
+    ('FLS','FLS'),
+    ('FLT','FLT'),
+    ('FLTS','FLTS'),
+    ('FORD','FRD'),
+    ('FORDS','FRDS'),
+    ('FOREST','FRST'),
+    ('FORESTS','FRST'),
+    ('FORGE','FRG'),
+    ('FORGES','FRGS'),
+    ('FORG','FRG'),
+    ('FORK','FRK'),
+    ('FORKS','FRKS'),
+    ('FORT','FT'),
+    ('FRD','FRD'),
+    ('FREEWAY','FWY'),
+    ('FREEWY','FWY'),
+    ('FRG','FRG'),
+    ('FRK','FRK'),
+    ('FRKS','FRKS'),
+    ('FRRY','FRY'),
+    ('FRST','FRST'),
+    ('FRT','FT'),
+    ('FRWAY','FWY'),
+    ('FRWY','FWY'),
+    ('FRY','FRY'),
+    ('FT','FT'),
+    ('FWY','FWY'),
+    ('GARDEN','GDN'),
+    ('GARDENS','GDNS'),
+    ('GARDN','GDN'),
+    ('GATEWAY','GTWY'),
+    ('GATEWY','GTWY'),
+    ('GATWAY','GTWY'),
+    ('GDN','GDN'),
+    ('GDNS','GDNS'),
+    ('GLEN','GLN'),
+    ('GLENS','GLNS'),
+    ('GLN','GLN'),
+    ('GRDEN','GDN'),
+    ('GRDN','GDN'),
+    ('GRDNS','GDNS'),
+    ('GREEN','GRN'),
+    ('GREENS','GRNS'),
+    ('GRN','GRN'),
+    ('GROVE','GRV'),
+    ('GROVES','GRVS'),
+    ('GROV','GRV'),
+    ('GRV','GRV'),
+    ('GTWAY','GTWY'),
+    ('GTWY','GTWY'),
+    ('HARB','HBR'),
+    ('HARBOR','HBR'),
+    ('HARBORS','HBRS'),
+    ('HARBR','HBR'),
+    ('HAVEN','HVN'),
+    ('HAVN','HVN'),
+    ('HBR','HBR'),
+    ('HEIGHT','HTS'),
+    ('HEIGHTS','HTS'),
+    ('HGTS','HTS'),
+    ('HIGHWAY','HWY'),
+    ('HIGHWY','HWY'),
+    ('HILL','HL'),
+    ('HILLS','HLS'),
+    ('HIWAY','HWY'),
+    ('HIWY','HWY'),
+    ('HL','HL'),
+    ('HLLW','HOLW'),
+    ('HLS','HLS'),
+    ('HOLLOW','HOLW'),
+    ('HOLLOWS','HOLW'),
+    ('HOLW','HOLW'),
+    ('HOLWS','HOLW'),
+    ('HRBOR','HBR'),
+    ('HT','HTS'),
+    ('HTS','HTS'),
+    ('HVN','HVN'),
+    ('HWAY','HWY'),
+    ('HWY','HWY'),
+    ('INLET','INLT'),
+    ('INLT','INLT'),
+    ('IS','IS'),
+    ('ISLAND','IS'),
+    ('ISLANDS','ISS'),
+    ('ISLANDS','SLNDS'),
+    ('ISLANDS','SS'),
+    ('ISLE','ISLE'),
+    ('ISLES','ISLE'),
+    ('ISLND','IS'),
+    ('I','SLNDS'),
+    ('ISS','ISS'),
+    ('JCTION','JCT'),
+    ('JCT','JCT'),
+    ('JCTN','JCT'),
+    ('JCTNS','JCTS'),
+    ('JCTS','JCTS'),
+    ('JUNCTION','JCT'),
+    ('JUNCTIONS','JCTS'),
+    ('JUNCTN','JCT'),
+    ('JUNCTON','JCT'),
+    ('KEY','KY'),
+    ('KEYS','KYS'),
+    ('KNL','KNL'),
+    ('KNLS','KNLS'),
+    ('KNOL','KNL'),
+    ('KNOLL','KNL'),
+    ('KNOLLS','KNLS'),
+    ('KY','KY'),
+    ('KYS','KYS'),
+    ('LAKE','LK'),
+    ('LAKES','LKS'),
+    ('LA','LN'),
+    ('LANDING','LNDG'),
+    ('LAND','LAND'),
+    ('LANE','LN'),
+    ('LANES','LN'),
+    ('LCK','LCK'),
+    ('LCKS','LCKS'),
+    ('LDGE','LDG'),
+    ('LDG','LDG'),
+    ('LF','LF'),
+    ('LGT','LGT'),
+    ('LIGHT','LGT'),
+    ('LIGHTS','LGTS'),
+    ('LK','LK'),
+    ('LKS','LKS'),
+    ('LNDG','LNDG'),
+    ('LNDNG','LNDG'),
+    ('LN','LN'),
+    ('LOAF','LF'),
+    ('LOCK','LCK'),
+    ('LOCKS','LCKS'),
+    ('LODGE','LDG'),
+    ('LODG','LDG'),
+    ('LOOP','LOOP'),
+    ('LOOPS','LOOP'),
+    ('MALL','MALL'),
+    ('MANOR','MNR'),
+    ('MANORS','MNRS'),
+    ('MDW','MDW'),
+    ('MDWS','MDWS'),
+    ('MEADOW','MDW'),
+    ('MEADOWS','MDWS'),
+    ('MEDOWS','MDWS'),
+    ('MEWS','MEWS'),
+    ('MILL','ML'),
+    ('MILLS','MLS'),
+    ('MISSION','MSN'),
+    ('MISSN','MSN'),
+    ('ML','ML'),
+    ('MLS','MLS'),
+    ('MNR','MNR'),
+    ('MNRS','MNRS'),
+    ('MNTAIN','MTN'),
+    ('MNT','MT'),
+    ('MNTN','MTN'),
+    ('MNTNS','MTNS'),
+    ('MOTORWAY','MTWY'),
+    ('MOUNTAIN','MTN'),
+    ('MOUNTAINS','MTNS'),
+    ('MOUNTIN','MTN'),
+    ('MOUNT','MT'),
+    ('MSN','MSN'),
+    ('MSSN','MSN'),
+    ('MTIN','MTN'),
+    ('MT','MT'),
+    ('MTN','MTN'),
+    ('NCK','NCK'),
+    ('NECK','NCK'),
+    ('ORCHARD','ORCH'),
+    ('ORCH','ORCH'),
+    ('ORCHRD','ORCH'),
+    ('OVAL','OVAL'),
+    ('OVERPASS','OPAS'),
+    ('OVL','OVAL'),
+    ('PARK','PARK'),
+    ('PARKS','PARK'),
+    ('PARKWAY','PKWY'),
+    ('PARKWAYS','PKWY'),
+    ('PARKWY','PKWY'),
+    ('PASSAGE','PSGE'),
+    ('PASS','PASS'),
+    ('PATH','PATH'),
+    ('PATHS','PATH'),
+    ('PIKE','PIKE'),
+    ('PIKES','PIKE'),
+    ('PINE','PNE'),
+    ('PINES','PNES'),
+    ('PK','PARK'),
+    ('PKWAY','PKWY'),
+    ('PKWY','PKWY'),
+    ('PKWYS','PKWY'),
+    ('PKY','PKWY'),
+    ('PLACE','PL'),
+    ('PLAINES','PLNS'),
+    ('PLAIN','PLN'),
+    ('PLAINS','PLNS'),
+    ('PLAZA','PLZ'),
+    ('PLN','PLN'),
+    ('PLNS','PLNS'),
+    ('PL','PL'),
+    ('PLZA','PLZ'),
+    ('PLZ','PLZ'),
+    ('PNES','PNES'),
+    ('POINT','PT'),
+    ('POINTS','PTS'),
+    ('PORT','PRT'),
+    ('PORTS','PRTS'),
+    ('PRAIRIE','PR'),
+    ('PRARIE','PR'),
+    ('PRK','PARK'),
+    ('PR','PR'),
+    ('PRR','PR'),
+    ('PRT','PRT'),
+    ('PRTS','PRTS'),
+    ('PT','PT'),
+    ('PTS','PTS'),
+    ('RADIAL','RADL'),
+    ('RADIEL','RADL'),
+    ('RADL','RADL'),
+    ('RAD','RADL'),
+    ('RAMP','RAMP'),
+    ('RANCHES','RNCH'),
+    ('RANCH','RNCH'),
+    ('RAPID','RPD'),
+    ('RAPIDS','RPDS'),
+    ('RDGE','RDG'),
+    ('RDG','RDG'),
+    ('RDGS','RDGS'),
+    ('RD','RD'),
+    ('RDS','RDS'),
+    ('REST','RST'),
+    ('RIDGE','RDG'),
+    ('RIDGES','RDGS'),
+    ('RIVER','RIV'),
+    ('RIV','RIV'),
+    ('RIVR','RIV'),
+    ('RNCH','RNCH'),
+    ('RNCHS','RNCH'),
+    ('ROAD','RD'),
+    ('ROADS','RDS'),
+    ('ROUTE','RTE'),
+    ('ROW','ROW'),
+    ('RPD','RPD'),
+    ('RPDS','RPDS'),
+    ('RST','RST'),
+    ('RUE','RUE'),
+    ('RUN','RUN'),
+    ('RVR','RIV'),
+    ('SHL','SHL'),
+    ('SHLS','SHLS'),
+    ('SHOAL','SHL'),
+    ('SHOALS','SHLS'),
+    ('SHOAR','SHR'),
+    ('SHOARS','SHRS'),
+    ('SHORE','SHR'),
+    ('SHORES','SHRS'),
+    ('SHR','SHR'),
+    ('SHRS','SHRS'),
+    ('SKYWAY','SKWY'),
+    ('SMT','SMT'),
+    ('SPG','SPG'),
+    ('SPGS','SPGS'),
+    ('SPNG','SPG'),
+    ('SPNGS','SPGS'),
+    ('SPRING','SPG'),
+    ('SPRINGS','SPGS'),
+    ('SPRNG','SPG'),
+    ('SPRNGS','SPGS'),
+    ('SPUR','SPUR'),
+    ('SPURS','SPUR'),
+    ('SQRE','SQ'),
+    ('SQR','SQ'),
+    ('SQRS','SQS'),
+    ('SQ','SQ'),
+    ('SQUARE','SQ'),
+    ('SQUARES','SQS'),
+    ('SQU','SQ'),
+    ('STA','STA'),
+    ('STATION','STA'),
+    ('STATN','STA'),
+    ('STN','STA'),
+    ('STRA','STRA'),
+    ('STRAVEN','STRA'),
+    ('STRAVENUE','STRA'),
+    ('STRAVE','STRA'),
+    ('STRAVN','STRA'),
+    ('STRAV','STRA'),
+    ('STREAM','STRM'),
+    ('STREETS','STS'),
+    ('STREET','ST'),
+    ('STREME','STRM'),
+    ('STRM','STRM'),
+    ('STR','ST'),
+    ('STRT','ST'),
+    ('STRVN','STRA'),
+    ('STRVNUE','STRA'),
+    ('ST','ST'),
+    ('SUMIT','SMT'),
+    ('SUMITT','SMT'),
+    ('SUMMIT','SMT'),
+    ('TERRACE','TER'),
+    ('TERR','TER'),
+    ('TER','TER'),
+    ('THROUGHWAY','TRWY'),
+    ('TPKE','TPKE'),
+    ('TPK','TPKE'),
+    ('TRACES','TRCE'),
+    ('TRACE','TRCE'),
+    ('TRACKS','TRAK'),
+    ('TRACK','TRAK'),
+    ('TRAFFICWAY','TRFY'),
+    ('TRAILS','TRL'),
+    ('TRAIL','TRL'),
+    ('TRAK','TRAK'),
+    ('TRCE','TRCE'),
+    ('TRFY','TRFY'),
+    ('TRKS','TRAK'),
+    ('TRK','TRAK'),
+    ('TRLS','TRL'),
+    ('TRL','TRL'),
+    ('TRNPK','TPKE'),
+    ('TRPK','TPKE'),
+    ('TR','TRL'),
+    ('TUNEL','TUNL'),
+    ('TUNLS','TUNL'),
+    ('TUNL','TUNL'),
+    ('TUNNELS','TUNL'),
+    ('TUNNEL','TUNL'),
+    ('TUNNL','TUNL'),
+    ('TURNPIKE','TPKE'),
+    ('TURNPK','TPKE'),
+    ('UNDERPASS','UPAS'),
+    ('UNIONS','UNS'),
+    ('UNION','UN'),
+    ('UN','UN'),
+    ('VALLEYS','VLYS'),
+    ('VALLEY','VLY'),
+    ('VALLY','VLY'),
+    ('VDCT','IA'),
+    ('VIADCT','VIA'),
+    ('VIADUCT','IA'),
+    ('VIADUCT','VIA'),
+    ('VIA','VIA'),
+    ('VIEWS','VWS'),
+    ('VIEW','VW'),
+    ('VILLAGES','VLGS'),
+    ('VILLAGE','VLG'),
+    ('VILLAG','VLG'),
+    ('VILLE','VL'),
+    ('VILLG','VLG'),
+    ('VILLIAGE','VLG'),
+    ('VILL','VLG'),
+    ('VISTA','VIS'),
+    ('VIST','VIS'),
+    ('VIS','VIS'),
+    ('VLGS','VLGS'),
+    ('VLG','VLG'),
+    ('VLLY','VLY'),
+    ('VL','VL'),
+    ('VLYS','VLYS'),
+    ('VLY','VLY'),
+    ('VSTA','VIS'),
+    ('VST','VIS'),
+    ('VWS','VWS'),
+    ('VW','VW'),
+    ('WALKS','WALK'),
+    ('WALK','WALK'),
+    ('WALL','WALL'),
+    ('WAYS','WAYS'),
+    ('WAY','WAY'),
+    ('WELLS','WLS'),
+    ('WELL','WL'),
+    ('WLS','WLS'),
+    ('WY','WAY'),
+    ('XING','XING');
+
+-- this function should get a smaller range of inputs and benefit more from STABLE, hopefully speeding things up
+CREATE OR REPLACE FUNCTION migration_tools._normalize_address_suffix (TEXT) RETURNS TEXT AS $$
+    DECLARE
+        suffix TEXT := $1;
+		_r RECORD;
+    BEGIN
+        --RAISE INFO 'suffix = %', suffix;
+		FOR _r IN (SELECT * FROM migration_tools.usps_suffixes) LOOP
+			suffix := REGEXP_REPLACE( suffix, _r.suffix_from, _r.suffix_to, 'i');
+		END LOOP;
+		RETURN suffix;
+    END;
+$$ LANGUAGE PLPGSQL STRICT STABLE;
+
+CREATE OR REPLACE FUNCTION migration_tools.normalize_address_suffix (TEXT) RETURNS TEXT AS $$
+    BEGIN
+		RETURN CASE
+            WHEN $1 ~ '\s\S+$' THEN REGEXP_REPLACE( $1, '^(.*\s)(\S+)$', '\1' ) || migration_tools._normalize_address_suffix( REGEXP_REPLACE( $1, '^(.*\s)(\S+)$', '\2' ) )
+            ELSE $1
+        END;
+    END;
+$$ LANGUAGE PLPGSQL STRICT STABLE;
+
 CREATE OR REPLACE FUNCTION migration_tools.rebarcode (o TEXT, t BIGINT) RETURNS TEXT AS $$
     DECLARE
         n TEXT := o;
@@ -772,6 +1657,24 @@ CREATE OR REPLACE FUNCTION migration_tools.attempt_hz_date (TEXT,TEXT) RETURNS D
     END;
 $$ LANGUAGE PLPGSQL STRICT STABLE;
 
+CREATE OR REPLACE FUNCTION migration_tools.attempt_sierra_timestamp (TEXT,TEXT) RETURNS TIMESTAMP AS $$
+    DECLARE
+        attempt_value ALIAS FOR $1;
+        fail_value ALIAS FOR $2;
+        output TIMESTAMP;
+    BEGIN
+            output := REGEXP_REPLACE(attempt_value,E'^(..)(..)(..)(..)(..)$',E'20\\1-\\2-\\3 \\4:\\5')::TIMESTAMP;
+            RETURN output;
+    EXCEPTION
+        WHEN OTHERS THEN
+            FOR output IN
+                EXECUTE 'SELECT ' || quote_literal(fail_value) || '::TIMESTAMP AS a;'
+            LOOP
+                RETURN output;
+            END LOOP;
+    END;
+$$ LANGUAGE PLPGSQL STRICT STABLE;
+
 CREATE OR REPLACE FUNCTION migration_tools.attempt_money (TEXT,TEXT) RETURNS NUMERIC(8,2) AS $$
     DECLARE
         attempt_value ALIAS FOR $1;
@@ -909,6 +1812,47 @@ CREATE OR REPLACE FUNCTION migration_tools.add_code39mod43_checkdigit (TEXT) RET
     return $barcode . $checkdigit;
 $$ LANGUAGE PLPERLU STRICT STABLE;
 
+-- add_mod16_checkdigit
+--   $barcode      source barcode
+--
+-- https://www.activebarcode.com/codes/checkdigit/modulo16.html
+
+CREATE OR REPLACE FUNCTION migration_tools.add_mod16_checkdigit (TEXT) RETURNS TEXT AS $$
+    my $barcode = shift;
+
+    my @digits = split //, $barcode;
+    my $total = 0;
+    foreach $digit (@digits) {
+        if ($digit =~ /[0-9]/) { $total += $digit;
+        } elsif ($digit eq '-') { $total += 10;
+        } elsif ($digit eq '$') { $total += 11;
+        } elsif ($digit eq ':') { $total += 12;
+        } elsif ($digit eq '/') { $total += 13;
+        } elsif ($digit eq '.') { $total += 14;
+        } elsif ($digit eq '+') { $total += 15;
+        } elsif ($digit eq 'A') { $total += 16;
+        } elsif ($digit eq 'B') { $total += 17;
+        } elsif ($digit eq 'C') { $total += 18;
+        } elsif ($digit eq 'D') { $total += 19;
+        } else { die "invalid digit <$digit>";
+        }
+    }
+    my $remainder = $total % 16;
+    my $difference = 16 - $remainder;
+    my $checkdigit;
+    if ($difference < 10) { $checkdigit = $difference;
+    } elsif ($difference == 10) { $checkdigit = '-';
+    } elsif ($difference == 11) { $checkdigit = '$';
+    } elsif ($difference == 12) { $checkdigit = ':';
+    } elsif ($difference == 13) { $checkdigit = '/';
+    } elsif ($difference == 14) { $checkdigit = '.';
+    } elsif ($difference == 15) { $checkdigit = '+';
+    } else { die "error calculating checkdigit";
+    }
+
+    return $barcode . $checkdigit;
+$$ LANGUAGE PLPERLU STRICT STABLE;
+
 CREATE OR REPLACE FUNCTION migration_tools.attempt_phone (TEXT,TEXT) RETURNS TEXT AS $$
   DECLARE
     phone TEXT := $1;
@@ -1344,6 +2288,56 @@ END;
 
 $$ LANGUAGE plpgsql;
 
+-- TODO: make another version of the procedure below that can work with specified copy staging tables
+-- The following should track the logic of OpenILS::Application::AppUtils::get_copy_price
+CREATE OR REPLACE FUNCTION migration_tools.get_copy_price( item BIGINT ) RETURNS NUMERIC AS $$
+DECLARE
+    context_lib             INT;
+    charge_lost_on_zero     BOOLEAN;
+    min_price               NUMERIC;
+    max_price               NUMERIC;
+    default_price           NUMERIC;
+    working_price           NUMERIC;
+
+BEGIN
+
+    SELECT INTO context_lib CASE WHEN call_number = -1 THEN circ_lib ELSE owning_lib END
+        FROM asset.copy ac, asset.call_number acn WHERE ac.call_number = acn.id AND ac.id = item;
+
+    SELECT INTO charge_lost_on_zero value
+        FROM actor.org_unit_ancestor_setting('circ.charge_lost_on_zero',context_lib);
+
+    SELECT INTO min_price value
+        FROM actor.org_unit_ancestor_setting('circ.min_item_price',context_lib);
+
+    SELECT INTO max_price value
+        FROM actor.org_unit_ancestor_setting('circ.max_item_price',context_lib);
+
+    SELECT INTO default_price value
+        FROM actor.org_unit_ancestor_setting('cat.default_item_price',context_lib);
+
+    SELECT INTO working_price price FROM asset.copy WHERE id = item;
+
+    IF (working_price IS NULL OR (working_price = 0 AND charge_lost_on_zero)) THEN
+        working_price := default_price;
+    END IF;
+
+    IF (max_price IS NOT NULL AND working_price > max_price) THEN
+        working_price := max_price;
+    END IF;
+
+    IF (min_price IS NOT NULL AND working_price < min_price) THEN
+        IF (working_price <> 0 OR charge_lost_on_zero IS NULL OR charge_lost_on_zero) THEN
+            working_price := min_price;
+        END IF;
+    END IF;
+
+    RETURN working_price;
+
+END;
+
+$$ LANGUAGE plpgsql;
+
 CREATE OR REPLACE FUNCTION migration_tools.apply_circ_matrix_to_specific_circ( tablename TEXT, circ BIGINT ) RETURNS VOID AS $$
 
 -- Usage:
@@ -1418,7 +2412,7 @@ BEGIN
       recurring_fine = rrf.normal,
       max_fine =
         CASE rmf.is_percent
-          WHEN TRUE THEN (rmf.amount / 100.0) * ac.price
+          WHEN TRUE THEN (rmf.amount / 100.0) * migration_tools.get_copy_price(ac.id)
           ELSE rmf.amount
         END,
       renewal_remaining = rcd.max_renewals,
@@ -2044,6 +3038,24 @@ BEGIN
 END;
 $FUNC$ LANGUAGE PLPGSQL;
 
+CREATE OR REPLACE FUNCTION migration_tools.split_rows_on_column_with_delimiter(schemaname TEXT, tablename TEXT, matchcol TEXT, delimiter TEXT) RETURNS VOID AS $FUNC$
+DECLARE
+    id BIGINT;
+    loopq TEXT;
+    cols TEXT[];
+    splitst TEXT;
+BEGIN
+    loopq := 'SELECT id FROM ' || schemaname || '.' || tablename || ' WHERE ' || matchcol || ' ~ $1 ORDER BY id';
+    SELECT ARRAY_AGG(column_name::TEXT) INTO cols FROM information_schema.columns WHERE table_schema = schemaname AND table_name = tablename AND column_name <> 'id' AND column_name <> matchcol;
+    FOR id IN EXECUTE loopq USING delimiter LOOP
+       RAISE NOTICE 'splitting row from %.% with id = %', schemaname, tablename, id;
+       splitst := 'INSERT INTO ' || schemaname || '.' || tablename || ' (' || ARRAY_TO_STRING(cols, ',') || ', ' || matchcol || ') SELECT ' || ARRAY_TO_STRING(cols, ',') || ', s.token ' ||
+                 ' FROM ' || schemaname || '.' || tablename || ' t, UNNEST(STRING_TO_ARRAY(t.' || matchcol || ', $2)) s(token) WHERE id = $1';
+       EXECUTE splitst USING id, delimiter;
+    END LOOP;
+END;
+$FUNC$ LANGUAGE PLPGSQL;
+
 CREATE OR REPLACE FUNCTION migration_tools.merge_marc_fields( TEXT, TEXT, TEXT[] ) RETURNS TEXT AS $func$
 
 use strict;
@@ -2381,32 +3393,148 @@ CREATE OR REPLACE FUNCTION migration_tools.duplicate_template_but_change_delay (
     END;
 $$ LANGUAGE PLPGSQL STRICT VOLATILE;
 
-CREATE OR REPLACE FUNCTION migration_tools.get_marc_tag (TEXT, TEXT, TEXT, TEXT) RETURNS TEXT AS $$
-    my ($marcxml, $tag, $subfield, $delimiter) = @_;
-
-    use MARC::Record;
-    use MARC::File::XML;
-    use MARC::Field;
-
-    my $field;
-    eval {
-        my $marc = MARC::Record->new_from_xml($marcxml, 'UTF-8');
-        $field = $marc->field($tag);
-    };
-    return $field->as_string($subfield,$delimiter);
-$$ LANGUAGE PLPERLU STABLE;
-
-CREATE OR REPLACE FUNCTION migration_tools.get_marc_tags (TEXT, TEXT, TEXT, TEXT) RETURNS TEXT[] AS $$
-    my ($marcxml, $tag, $subfield, $delimiter) = @_;
-
-    use MARC::Record;
-    use MARC::File::XML;
-    use MARC::Field;
-
-    my @fields;
-    eval {
-        my $marc = MARC::Record->new_from_xml($marcxml, 'UTF-8');
-        @fields = $marc->field($tag);
+-- example: SELECT * FROM migration_tools.duplicate_templates(3,'{5,6}');
+CREATE OR REPLACE FUNCTION migration_tools.duplicate_templates (INTEGER, INTEGER[]) RETURNS VOID AS $$
+    DECLARE
+        org ALIAS FOR $1;
+        target_event_defs ALIAS FOR $2;
+    BEGIN
+        DROP TABLE IF EXISTS new_atevdefs;
+        CREATE TEMP TABLE new_atevdefs (atevdef INTEGER);
+        FOR i IN array_lower(target_event_defs,1) .. array_upper(target_event_defs,1) LOOP
+            INSERT INTO action_trigger.event_definition (
+                active
+                ,owner
+                ,name
+                ,hook
+                ,validator
+                ,reactor
+                ,cleanup_success
+                ,cleanup_failure
+                ,delay
+                ,max_delay
+                ,usr_field
+                ,opt_in_setting
+                ,delay_field
+                ,group_field
+                ,template
+                ,granularity
+                ,repeat_delay
+            ) SELECT
+                'f'
+                ,org
+                ,name || ' (clone of '||target_event_defs[i]||')'
+                ,hook
+                ,validator
+                ,reactor
+                ,cleanup_success
+                ,cleanup_failure
+                ,delay
+                ,max_delay
+                ,usr_field
+                ,opt_in_setting
+                ,delay_field
+                ,group_field
+                ,template
+                ,granularity
+                ,repeat_delay
+            FROM
+                action_trigger.event_definition
+            WHERE
+                id = target_event_defs[i]
+            ;
+            RAISE INFO 'created atevdef with id = %', currval('action_trigger.event_definition_id_seq');
+            INSERT INTO new_atevdefs SELECT currval('action_trigger.event_definition_id_seq');
+            INSERT INTO action_trigger.environment (
+                event_def
+                ,path
+                ,collector
+                ,label
+            ) SELECT
+                currval('action_trigger.event_definition_id_seq')
+                ,path
+                ,collector
+                ,label
+            FROM
+                action_trigger.environment
+            WHERE
+                event_def = target_event_defs[i]
+            ;
+            INSERT INTO action_trigger.event_params (
+                event_def
+                ,param
+                ,value
+            ) SELECT
+                currval('action_trigger.event_definition_id_seq')
+                ,param
+                ,value
+            FROM
+                action_trigger.event_params
+            WHERE
+                event_def = target_event_defs[i]
+            ;
+        END LOOP;
+        RAISE INFO '-- UPDATE action_trigger.event_definition SET active = TRUE WHERE id in (%);', (SELECT array_to_string(array_agg(atevdef),',') from new_atevdefs);
+    END;
+$$ LANGUAGE PLPGSQL STRICT VOLATILE;
+
+CREATE OR REPLACE FUNCTION migration_tools.reset_event (BIGINT) RETURNS VOID AS $$
+    UPDATE
+        action_trigger.event
+    SET
+         start_time = NULL
+        ,update_time = NULL
+        ,complete_time = NULL
+        ,update_process = NULL
+        ,state = 'pending'
+        ,template_output = NULL
+        ,error_output = NULL
+        ,async_output = NULL
+    WHERE
+        id = $1;
+$$ LANGUAGE SQL;
+
+CREATE OR REPLACE FUNCTION migration_tools.get_marc_leader (TEXT) RETURNS TEXT AS $$
+    my ($marcxml) = @_;
+
+    use MARC::Record;
+    use MARC::File::XML;
+    use MARC::Field;
+
+    my $field;
+    eval {
+        my $marc = MARC::Record->new_from_xml($marcxml, 'UTF-8');
+        $field = $marc->leader();
+    };
+    return $field;
+$$ LANGUAGE PLPERLU STABLE;
+
+CREATE OR REPLACE FUNCTION migration_tools.get_marc_tag (TEXT, TEXT, TEXT, TEXT) RETURNS TEXT AS $$
+    my ($marcxml, $tag, $subfield, $delimiter) = @_;
+
+    use MARC::Record;
+    use MARC::File::XML;
+    use MARC::Field;
+
+    my $field;
+    eval {
+        my $marc = MARC::Record->new_from_xml($marcxml, 'UTF-8');
+        $field = $marc->field($tag);
+    };
+    return $field->as_string($subfield,$delimiter);
+$$ LANGUAGE PLPERLU STABLE;
+
+CREATE OR REPLACE FUNCTION migration_tools.get_marc_tags (TEXT, TEXT, TEXT, TEXT) RETURNS TEXT[] AS $$
+    my ($marcxml, $tag, $subfield, $delimiter) = @_;
+
+    use MARC::Record;
+    use MARC::File::XML;
+    use MARC::Field;
+
+    my @fields;
+    eval {
+        my $marc = MARC::Record->new_from_xml($marcxml, 'UTF-8');
+        @fields = $marc->field($tag);
     };
     my @texts;
     foreach my $field (@fields) {
@@ -2415,6 +3543,27 @@ CREATE OR REPLACE FUNCTION migration_tools.get_marc_tags (TEXT, TEXT, TEXT, TEXT
     return \@texts;
 $$ LANGUAGE PLPERLU STABLE;
 
+CREATE OR REPLACE FUNCTION migration_tools.get_marc_tags_filtered (TEXT, TEXT, TEXT, TEXT, TEXT) RETURNS TEXT[] AS $$
+    my ($marcxml, $tag, $subfield, $delimiter, $match) = @_;
+
+    use MARC::Record;
+    use MARC::File::XML;
+    use MARC::Field;
+
+    my @fields;
+    eval {
+        my $marc = MARC::Record->new_from_xml($marcxml, 'UTF-8');
+        @fields = $marc->field($tag);
+    };
+    my @texts;
+    foreach my $field (@fields) {
+        if ($field->as_string() =~ qr/$match/) {
+            push @texts, $field->as_string($subfield,$delimiter);
+        }
+    }
+    return \@texts;
+$$ LANGUAGE PLPERLU STABLE;
+
 CREATE OR REPLACE FUNCTION migration_tools.find_hold_matrix_matchpoint (INTEGER) RETURNS INTEGER AS $$
     SELECT action.find_hold_matrix_matchpoint(
         (SELECT pickup_lib FROM action.hold_request WHERE id = $1),
@@ -2449,3 +3598,1729 @@ CREATE OR REPLACE FUNCTION migration_tools.find_circ_matrix_matchpoint (INTEGER)
         )
     );
 $$ LANGUAGE SQL;
+
+CREATE OR REPLACE FUNCTION migration_tools.assert (BOOLEAN) RETURNS VOID AS $$
+    DECLARE
+        test ALIAS FOR $1;
+    BEGIN
+        IF NOT test THEN
+            RAISE EXCEPTION 'assertion';
+        END IF;
+    END;
+$$ LANGUAGE PLPGSQL STRICT VOLATILE;
+
+CREATE OR REPLACE FUNCTION migration_tools.assert (BOOLEAN,TEXT) RETURNS VOID AS $$
+    DECLARE
+        test ALIAS FOR $1;
+        msg ALIAS FOR $2;
+    BEGIN
+        IF NOT test THEN
+            RAISE EXCEPTION '%', msg;
+        END IF;
+    END;
+$$ LANGUAGE PLPGSQL STRICT VOLATILE;
+
+CREATE OR REPLACE FUNCTION migration_tools.assert (BOOLEAN,TEXT,TEXT) RETURNS TEXT AS $$
+    DECLARE
+        test ALIAS FOR $1;
+        fail_msg ALIAS FOR $2;
+        success_msg ALIAS FOR $3;
+    BEGIN
+        IF NOT test THEN
+            RAISE EXCEPTION '%', fail_msg;
+        END IF;
+        RETURN success_msg;
+    END;
+$$ LANGUAGE PLPGSQL STRICT VOLATILE;
+
+-- push bib sequence and return starting value for reserved range
+CREATE OR REPLACE FUNCTION migration_tools.push_bib_sequence(INTEGER) RETURNS BIGINT AS $$
+    DECLARE
+        bib_count ALIAS FOR $1;
+        output BIGINT;
+    BEGIN
+        PERFORM setval('biblio.record_entry_id_seq',(SELECT MAX(id) FROM biblio.record_entry) + bib_count + 2000);
+        FOR output IN
+            SELECT CEIL(MAX(id)/1000)*1000+1000 FROM biblio.record_entry WHERE id < (SELECT last_value FROM biblio.record_entry_id_seq)
+        LOOP
+            RETURN output;
+        END LOOP;
+    END;
+$$ LANGUAGE PLPGSQL STRICT VOLATILE;
+
+-- set a new salted password
+
+CREATE OR REPLACE FUNCTION migration_tools.set_salted_passwd(INTEGER,TEXT) RETURNS BOOLEAN AS $$
+    DECLARE
+        usr_id              ALIAS FOR $1;
+        plain_passwd        ALIAS FOR $2;
+        plain_salt          TEXT;
+        md5_passwd          TEXT;
+    BEGIN
+
+        SELECT actor.create_salt('main') INTO plain_salt;
+
+        SELECT MD5(plain_passwd) INTO md5_passwd;
+        
+        PERFORM actor.set_passwd(usr_id, 'main', MD5(plain_salt || md5_passwd), plain_salt);
+
+        RETURN TRUE;
+
+    END;
+$$ LANGUAGE PLPGSQL STRICT VOLATILE;
+
+
+-- convenience functions for handling copy_location maps
+
+CREATE OR REPLACE FUNCTION migration_tools.handle_shelf (TEXT,TEXT,TEXT,INTEGER) RETURNS VOID AS $$
+    DECLARE
+        table_schema ALIAS FOR $1;
+        table_name ALIAS FOR $2;
+        org_shortname ALIAS FOR $3;
+        org_range ALIAS FOR $4;
+        proceed BOOLEAN;
+        org INTEGER;
+        -- if x_org is on the mapping table, it'll take precedence over the passed org_shortname param
+        -- though we'll still use the passed org for the full path traversal when needed
+        x_org_found BOOLEAN;
+        x_org INTEGER;
+        org_list INTEGER[];
+        o INTEGER;
+    BEGIN
+        EXECUTE 'SELECT EXISTS (
+            SELECT 1
+            FROM information_schema.columns
+            WHERE table_schema = $1
+            AND table_name = $2
+            and column_name = ''desired_shelf''
+        )' INTO proceed USING table_schema, table_name;
+        IF NOT proceed THEN
+            RAISE EXCEPTION 'Missing column desired_shelf';
+        END IF;
+
+        EXECUTE 'SELECT EXISTS (
+            SELECT 1
+            FROM information_schema.columns
+            WHERE table_schema = $1
+            AND table_name = $2
+            and column_name = ''x_org''
+        )' INTO x_org_found USING table_schema, table_name;
+
+        SELECT id INTO org FROM actor.org_unit WHERE shortname = org_shortname;
+        IF org IS NULL THEN
+            RAISE EXCEPTION 'Cannot find org by shortname';
+        END IF;
+
+        SELECT INTO org_list ARRAY_ACCUM(id) FROM actor.org_unit_full_path( org );
+
+        EXECUTE 'ALTER TABLE '
+            || quote_ident(table_name)
+            || ' DROP COLUMN IF EXISTS x_shelf';
+        EXECUTE 'ALTER TABLE '
+            || quote_ident(table_name)
+            || ' ADD COLUMN x_shelf INTEGER';
+
+        IF x_org_found THEN
+            EXECUTE 'UPDATE ' || quote_ident(table_name) || ' a'
+                || ' SET x_shelf = b.id FROM asset_copy_location b'
+                || ' WHERE BTRIM(UPPER(a.desired_shelf)) = BTRIM(UPPER(b.name))'
+                || ' AND b.owning_lib = x_org'
+                || ' AND NOT b.deleted';
+            EXECUTE 'UPDATE ' || quote_ident(table_name) || ' a'
+                || ' SET x_shelf = b.id FROM asset.copy_location b'
+                || ' WHERE BTRIM(UPPER(a.desired_shelf)) = BTRIM(UPPER(b.name))'
+                || ' AND b.owning_lib = x_org'
+                || ' AND x_shelf IS NULL'
+                || ' AND NOT b.deleted';
+        ELSE
+            EXECUTE 'UPDATE ' || quote_ident(table_name) || ' a'
+                || ' SET x_shelf = b.id FROM asset_copy_location b'
+                || ' WHERE BTRIM(UPPER(a.desired_shelf)) = BTRIM(UPPER(b.name))'
+                || ' AND b.owning_lib = $1'
+                || ' AND NOT b.deleted'
+            USING org;
+            EXECUTE 'UPDATE ' || quote_ident(table_name) || ' a'
+                || ' SET x_shelf = b.id FROM asset_copy_location b'
+                || ' WHERE BTRIM(UPPER(a.desired_shelf)) = BTRIM(UPPER(b.name))'
+                || ' AND b.owning_lib = $1'
+                || ' AND x_shelf IS NULL'
+                || ' AND NOT b.deleted'
+            USING org;
+        END IF;
+
+        FOREACH o IN ARRAY org_list LOOP
+            EXECUTE 'UPDATE ' || quote_ident(table_name) || ' a'
+                || ' SET x_shelf = b.id FROM asset.copy_location b'
+                || ' WHERE BTRIM(UPPER(a.desired_shelf)) = BTRIM(UPPER(b.name))'
+                || ' AND b.owning_lib = $1 AND x_shelf IS NULL'
+                || ' AND NOT b.deleted'
+            USING o;
+        END LOOP;
+
+        EXECUTE 'SELECT migration_tools.assert(
+            NOT EXISTS (SELECT 1 FROM ' || quote_ident(table_name) || ' WHERE desired_shelf <> '''' AND x_shelf IS NULL),
+            ''Cannot find a desired location'',
+            ''Found all desired locations''
+        );';
+
+    END;
+$$ LANGUAGE PLPGSQL STRICT VOLATILE;
+
+-- convenience functions for handling circmod maps
+
+CREATE OR REPLACE FUNCTION migration_tools.handle_circmod (TEXT,TEXT) RETURNS VOID AS $$
+    DECLARE
+        table_schema ALIAS FOR $1;
+        table_name ALIAS FOR $2;
+        proceed BOOLEAN;
+    BEGIN
+        EXECUTE 'SELECT EXISTS (
+            SELECT 1
+            FROM information_schema.columns
+            WHERE table_schema = $1
+            AND table_name = $2
+            and column_name = ''desired_circmod''
+        )' INTO proceed USING table_schema, table_name;
+        IF NOT proceed THEN
+            RAISE EXCEPTION 'Missing column desired_circmod'; 
+        END IF;
+
+        EXECUTE 'ALTER TABLE '
+            || quote_ident(table_name)
+            || ' DROP COLUMN IF EXISTS x_circmod';
+        EXECUTE 'ALTER TABLE '
+            || quote_ident(table_name)
+            || ' ADD COLUMN x_circmod TEXT';
+
+        EXECUTE 'UPDATE ' || quote_ident(table_name) || ' a'
+            || ' SET x_circmod = code FROM config.circ_modifier b'
+            || ' WHERE BTRIM(UPPER(a.desired_circmod)) = BTRIM(UPPER(b.code))';
+
+        EXECUTE 'UPDATE ' || quote_ident(table_name) || ' a'
+            || ' SET x_circmod = code FROM config.circ_modifier b'
+            || ' WHERE BTRIM(UPPER(a.desired_circmod)) = BTRIM(UPPER(b.name))'
+            || ' AND x_circmod IS NULL';
+
+        EXECUTE 'UPDATE ' || quote_ident(table_name) || ' a'
+            || ' SET x_circmod = code FROM config.circ_modifier b'
+            || ' WHERE BTRIM(UPPER(a.desired_circmod)) = BTRIM(UPPER(b.description))'
+            || ' AND x_circmod IS NULL';
+
+        EXECUTE 'SELECT migration_tools.assert(
+            NOT EXISTS (SELECT 1 FROM ' || quote_ident(table_name) || ' WHERE desired_circmod <> '''' AND x_circmod IS NULL),
+            ''Cannot find a desired circulation modifier'',
+            ''Found all desired circulation modifiers''
+        );';
+
+    END;
+$$ LANGUAGE PLPGSQL STRICT VOLATILE;
+
+-- convenience functions for handling item status maps
+
+CREATE OR REPLACE FUNCTION migration_tools.handle_status (TEXT,TEXT) RETURNS VOID AS $$
+    DECLARE
+        table_schema ALIAS FOR $1;
+        table_name ALIAS FOR $2;
+        proceed BOOLEAN;
+    BEGIN
+        EXECUTE 'SELECT EXISTS (
+            SELECT 1
+            FROM information_schema.columns
+            WHERE table_schema = $1
+            AND table_name = $2
+            and column_name = ''desired_status''
+        )' INTO proceed USING table_schema, table_name;
+        IF NOT proceed THEN
+            RAISE EXCEPTION 'Missing column desired_status'; 
+        END IF;
+
+        EXECUTE 'ALTER TABLE '
+            || quote_ident(table_name)
+            || ' DROP COLUMN IF EXISTS x_status';
+        EXECUTE 'ALTER TABLE '
+            || quote_ident(table_name)
+            || ' ADD COLUMN x_status INTEGER';
+
+        EXECUTE 'UPDATE ' || quote_ident(table_name) || ' a'
+            || ' SET x_status = id FROM config.copy_status b'
+            || ' WHERE BTRIM(UPPER(a.desired_status)) = BTRIM(UPPER(b.name))';
+
+        EXECUTE 'SELECT migration_tools.assert(
+            NOT EXISTS (SELECT 1 FROM ' || quote_ident(table_name) || ' WHERE desired_status <> '''' AND x_status IS NULL),
+            ''Cannot find a desired copy status'',
+            ''Found all desired copy statuses''
+        );';
+
+    END;
+$$ LANGUAGE PLPGSQL STRICT VOLATILE;
+
+-- convenience functions for handling org maps
+
+CREATE OR REPLACE FUNCTION migration_tools.handle_org (TEXT,TEXT) RETURNS VOID AS $$
+    DECLARE
+        table_schema ALIAS FOR $1;
+        table_name ALIAS FOR $2;
+        proceed BOOLEAN;
+    BEGIN
+        EXECUTE 'SELECT EXISTS (
+            SELECT 1
+            FROM information_schema.columns
+            WHERE table_schema = $1
+            AND table_name = $2
+            and column_name = ''desired_org''
+        )' INTO proceed USING table_schema, table_name;
+        IF NOT proceed THEN
+            RAISE EXCEPTION 'Missing column desired_org'; 
+        END IF;
+
+        EXECUTE 'ALTER TABLE '
+            || quote_ident(table_name)
+            || ' DROP COLUMN IF EXISTS x_org';
+        EXECUTE 'ALTER TABLE '
+            || quote_ident(table_name)
+            || ' ADD COLUMN x_org INTEGER';
+
+        EXECUTE 'UPDATE ' || quote_ident(table_name) || ' a'
+            || ' SET x_org = b.id FROM actor.org_unit b'
+            || ' WHERE BTRIM(a.desired_org) = BTRIM(b.shortname)';
+
+        EXECUTE 'SELECT migration_tools.assert(
+            NOT EXISTS (SELECT 1 FROM ' || quote_ident(table_name) || ' WHERE desired_org <> '''' AND x_org IS NULL),
+            ''Cannot find a desired org unit'',
+            ''Found all desired org units''
+        );';
+
+    END;
+$$ LANGUAGE PLPGSQL STRICT VOLATILE;
+
+-- convenience function for handling desired_not_migrate
+
+CREATE OR REPLACE FUNCTION migration_tools.handle_not_migrate (TEXT,TEXT) RETURNS VOID AS $$
+    DECLARE
+        table_schema ALIAS FOR $1;
+        table_name ALIAS FOR $2;
+        proceed BOOLEAN;
+    BEGIN
+        EXECUTE 'SELECT EXISTS (
+            SELECT 1
+            FROM information_schema.columns
+            WHERE table_schema = $1
+            AND table_name = $2
+            and column_name = ''desired_not_migrate''
+        )' INTO proceed USING table_schema, table_name;
+        IF NOT proceed THEN
+            RAISE EXCEPTION 'Missing column desired_not_migrate'; 
+        END IF;
+
+        EXECUTE 'ALTER TABLE '
+            || quote_ident(table_name)
+            || ' DROP COLUMN IF EXISTS x_migrate';
+        EXECUTE 'ALTER TABLE '
+            || quote_ident(table_name)
+            || ' ADD COLUMN x_migrate BOOLEAN';
+
+        EXECUTE 'UPDATE ' || quote_ident(table_name) || ' a'
+            || ' SET x_migrate = CASE'
+            || ' WHEN BTRIM(desired_not_migrate) = ''TRUE'' THEN FALSE'
+            || ' WHEN BTRIM(desired_not_migrate) = ''DNM'' THEN FALSE'
+            || ' WHEN BTRIM(desired_not_migrate) = ''Do Not Migrate'' THEN FALSE'
+            || ' WHEN BTRIM(desired_not_migrate) = ''FALSE'' THEN TRUE'
+            || ' WHEN BTRIM(desired_not_migrate) = ''Migrate'' THEN TRUE'
+            || ' WHEN BTRIM(desired_not_migrate) = '''' THEN TRUE'
+            || ' END';
+
+        EXECUTE 'SELECT migration_tools.assert(
+            NOT EXISTS (SELECT 1 FROM ' || quote_ident(table_name) || ' WHERE x_migrate IS NULL),
+            ''Not all desired_not_migrate values understood'',
+            ''All desired_not_migrate values understood''
+        );';
+
+    END;
+$$ LANGUAGE PLPGSQL STRICT VOLATILE;
+
+-- convenience function for handling desired_not_migrate
+
+CREATE OR REPLACE FUNCTION migration_tools.handle_barred_or_blocked (TEXT,TEXT) RETURNS VOID AS $$
+    DECLARE
+        table_schema ALIAS FOR $1;
+        table_name ALIAS FOR $2;
+        proceed BOOLEAN;
+    BEGIN
+        EXECUTE 'SELECT EXISTS (
+            SELECT 1
+            FROM information_schema.columns
+            WHERE table_schema = $1
+            AND table_name = $2
+            and column_name = ''desired_barred_or_blocked''
+        )' INTO proceed USING table_schema, table_name;
+        IF NOT proceed THEN
+            RAISE EXCEPTION 'Missing column desired_barred_or_blocked'; 
+        END IF;
+
+        EXECUTE 'ALTER TABLE '
+            || quote_ident(table_name)
+            || ' DROP COLUMN IF EXISTS x_barred';
+        EXECUTE 'ALTER TABLE '
+            || quote_ident(table_name)
+            || ' ADD COLUMN x_barred BOOLEAN';
+
+        EXECUTE 'ALTER TABLE '
+            || quote_ident(table_name)
+            || ' DROP COLUMN IF EXISTS x_blocked';
+        EXECUTE 'ALTER TABLE '
+            || quote_ident(table_name)
+            || ' ADD COLUMN x_blocked BOOLEAN';
+
+        EXECUTE 'UPDATE ' || quote_ident(table_name) || ' a'
+            || ' SET x_barred = CASE'
+            || ' WHEN BTRIM(desired_barred_or_blocked) = ''Barred'' THEN TRUE'
+            || ' WHEN BTRIM(desired_barred_or_blocked) = ''Blocked'' THEN FALSE'
+            || ' WHEN BTRIM(desired_barred_or_blocked) = ''Neither'' THEN FALSE'
+            || ' WHEN BTRIM(desired_barred_or_blocked) = '''' THEN FALSE'
+            || ' END';
+
+        EXECUTE 'UPDATE ' || quote_ident(table_name) || ' a'
+            || ' SET x_blocked = CASE'
+            || ' WHEN BTRIM(desired_barred_or_blocked) = ''Blocked'' THEN TRUE'
+            || ' WHEN BTRIM(desired_barred_or_blocked) = ''Barred'' THEN FALSE'
+            || ' WHEN BTRIM(desired_barred_or_blocked) = ''Neither'' THEN FALSE'
+            || ' WHEN BTRIM(desired_barred_or_blocked) = '''' THEN FALSE'
+            || ' END';
+
+        EXECUTE 'SELECT migration_tools.assert(
+            NOT EXISTS (SELECT 1 FROM ' || quote_ident(table_name) || ' WHERE x_barred IS NULL or x_blocked IS NULL),
+            ''Not all desired_barred_or_blocked values understood'',
+            ''All desired_barred_or_blocked values understood''
+        );';
+
+    END;
+$$ LANGUAGE PLPGSQL STRICT VOLATILE;
+
+-- convenience function for handling desired_profile
+
+CREATE OR REPLACE FUNCTION migration_tools.handle_profile (TEXT,TEXT) RETURNS VOID AS $$
+    DECLARE
+        table_schema ALIAS FOR $1;
+        table_name ALIAS FOR $2;
+        proceed BOOLEAN;
+    BEGIN
+        EXECUTE 'SELECT EXISTS (
+            SELECT 1
+            FROM information_schema.columns
+            WHERE table_schema = $1
+            AND table_name = $2
+            and column_name = ''desired_profile''
+        )' INTO proceed USING table_schema, table_name;
+        IF NOT proceed THEN
+            RAISE EXCEPTION 'Missing column desired_profile'; 
+        END IF;
+
+        EXECUTE 'ALTER TABLE '
+            || quote_ident(table_name)
+            || ' DROP COLUMN IF EXISTS x_profile';
+        EXECUTE 'ALTER TABLE '
+            || quote_ident(table_name)
+            || ' ADD COLUMN x_profile INTEGER';
+
+        EXECUTE 'UPDATE ' || quote_ident(table_name) || ' a'
+            || ' SET x_profile = b.id FROM permission.grp_tree b'
+            || ' WHERE BTRIM(UPPER(a.desired_profile)) = BTRIM(UPPER(b.name))';
+
+        EXECUTE 'SELECT migration_tools.assert(
+            NOT EXISTS (SELECT 1 FROM ' || quote_ident(table_name) || ' WHERE desired_profile <> '''' AND x_profile IS NULL),
+            ''Cannot find a desired profile'',
+            ''Found all desired profiles''
+        );';
+
+    END;
+$$ LANGUAGE PLPGSQL STRICT VOLATILE;
+
+-- convenience function for handling desired actor stat cats
+
+CREATE OR REPLACE FUNCTION migration_tools.vivicate_actor_sc_and_sce (TEXT,TEXT,TEXT,TEXT) RETURNS VOID AS $$
+    DECLARE
+        table_schema ALIAS FOR $1;
+        table_name ALIAS FOR $2;
+        field_suffix ALIAS FOR $3; -- for distinguishing between desired_sce1, desired_sce2, etc.
+        org_shortname ALIAS FOR $4;
+        proceed BOOLEAN;
+        org INTEGER;
+        org_list INTEGER[];
+        sc TEXT;
+        sce TEXT;
+    BEGIN
+
+        SELECT 'desired_sc' || field_suffix INTO sc;
+        SELECT 'desired_sce' || field_suffix INTO sce;
+
+        EXECUTE 'SELECT EXISTS (
+            SELECT 1
+            FROM information_schema.columns
+            WHERE table_schema = $1
+            AND table_name = $2
+            and column_name = $3
+        )' INTO proceed USING table_schema, table_name, sc;
+        IF NOT proceed THEN
+            RAISE EXCEPTION 'Missing column %', sc; 
+        END IF;
+        EXECUTE 'SELECT EXISTS (
+            SELECT 1
+            FROM information_schema.columns
+            WHERE table_schema = $1
+            AND table_name = $2
+            and column_name = $3
+        )' INTO proceed USING table_schema, table_name, sce;
+        IF NOT proceed THEN
+            RAISE EXCEPTION 'Missing column %', sce; 
+        END IF;
+
+        SELECT id INTO org FROM actor.org_unit WHERE shortname = org_shortname;
+        IF org IS NULL THEN
+            RAISE EXCEPTION 'Cannot find org by shortname';
+        END IF;
+        SELECT INTO org_list ARRAY_ACCUM(id) FROM actor.org_unit_full_path( org );
+
+        -- caller responsible for their own truncates though we try to prevent duplicates
+        EXECUTE 'INSERT INTO actor_stat_cat (owner, name)
+            SELECT DISTINCT
+                 $1
+                ,BTRIM('||sc||')
+            FROM 
+                ' || quote_ident(table_name) || '
+            WHERE
+                NULLIF(BTRIM('||sc||'),'''') IS NOT NULL
+                AND NOT EXISTS (
+                    SELECT id
+                    FROM actor.stat_cat
+                    WHERE owner = ANY ($2)
+                    AND name = BTRIM('||sc||')
+                )
+                AND NOT EXISTS (
+                    SELECT id
+                    FROM actor_stat_cat
+                    WHERE owner = ANY ($2)
+                    AND name = BTRIM('||sc||')
+                )
+            ORDER BY 2;'
+        USING org, org_list;
+
+        EXECUTE 'INSERT INTO actor_stat_cat_entry (stat_cat, owner, value)
+            SELECT DISTINCT
+                COALESCE(
+                    (SELECT id
+                        FROM actor.stat_cat
+                        WHERE owner = ANY ($2)
+                        AND BTRIM('||sc||') = BTRIM(name))
+                   ,(SELECT id
+                        FROM actor_stat_cat
+                        WHERE owner = ANY ($2)
+                        AND BTRIM('||sc||') = BTRIM(name))
+                )
+                ,$1
+                ,BTRIM('||sce||')
+            FROM 
+                ' || quote_ident(table_name) || '
+            WHERE
+                    NULLIF(BTRIM('||sc||'),'''') IS NOT NULL
+                AND NULLIF(BTRIM('||sce||'),'''') IS NOT NULL
+                AND NOT EXISTS (
+                    SELECT id
+                    FROM actor.stat_cat_entry
+                    WHERE stat_cat = (
+                        SELECT id
+                        FROM actor.stat_cat
+                        WHERE owner = ANY ($2)
+                        AND BTRIM('||sc||') = BTRIM(name)
+                    ) AND value = BTRIM('||sce||')
+                    AND owner = ANY ($2)
+                )
+                AND NOT EXISTS (
+                    SELECT id
+                    FROM actor_stat_cat_entry
+                    WHERE stat_cat = (
+                        SELECT id
+                        FROM actor_stat_cat
+                        WHERE owner = ANY ($2)
+                        AND BTRIM('||sc||') = BTRIM(name)
+                    ) AND value = BTRIM('||sce||')
+                    AND owner = ANY ($2)
+                )
+            ORDER BY 1,3;'
+        USING org, org_list;
+    END;
+$$ LANGUAGE PLPGSQL STRICT VOLATILE;
+
+CREATE OR REPLACE FUNCTION migration_tools.handle_actor_sc_and_sce (TEXT,TEXT,TEXT,TEXT) RETURNS VOID AS $$
+    DECLARE
+        table_schema ALIAS FOR $1;
+        table_name ALIAS FOR $2;
+        field_suffix ALIAS FOR $3; -- for distinguishing between desired_sce1, desired_sce2, etc.
+        org_shortname ALIAS FOR $4;
+        proceed BOOLEAN;
+        org INTEGER;
+        org_list INTEGER[];
+        o INTEGER;
+        sc TEXT;
+        sce TEXT;
+    BEGIN
+        SELECT 'desired_sc' || field_suffix INTO sc;
+        SELECT 'desired_sce' || field_suffix INTO sce;
+        EXECUTE 'SELECT EXISTS (
+            SELECT 1
+            FROM information_schema.columns
+            WHERE table_schema = $1
+            AND table_name = $2
+            and column_name = $3
+        )' INTO proceed USING table_schema, table_name, sc;
+        IF NOT proceed THEN
+            RAISE EXCEPTION 'Missing column %', sc; 
+        END IF;
+        EXECUTE 'SELECT EXISTS (
+            SELECT 1
+            FROM information_schema.columns
+            WHERE table_schema = $1
+            AND table_name = $2
+            and column_name = $3
+        )' INTO proceed USING table_schema, table_name, sce;
+        IF NOT proceed THEN
+            RAISE EXCEPTION 'Missing column %', sce; 
+        END IF;
+
+        SELECT id INTO org FROM actor.org_unit WHERE shortname = org_shortname;
+        IF org IS NULL THEN
+            RAISE EXCEPTION 'Cannot find org by shortname';
+        END IF;
+
+        SELECT INTO org_list ARRAY_ACCUM(id) FROM actor.org_unit_full_path( org );
+
+        EXECUTE 'ALTER TABLE '
+            || quote_ident(table_name)
+            || ' DROP COLUMN IF EXISTS x_sc' || field_suffix;
+        EXECUTE 'ALTER TABLE '
+            || quote_ident(table_name)
+            || ' ADD COLUMN x_sc' || field_suffix || ' INTEGER';
+        EXECUTE 'ALTER TABLE '
+            || quote_ident(table_name)
+            || ' DROP COLUMN IF EXISTS x_sce' || field_suffix;
+        EXECUTE 'ALTER TABLE '
+            || quote_ident(table_name)
+            || ' ADD COLUMN x_sce' || field_suffix || ' INTEGER';
+
+
+        EXECUTE 'UPDATE ' || quote_ident(table_name) || '
+            SET
+                x_sc' || field_suffix || ' = id
+            FROM
+                (SELECT id, name, owner FROM actor_stat_cat
+                    UNION SELECT id, name, owner FROM actor.stat_cat) u
+            WHERE
+                    BTRIM(UPPER(u.name)) = BTRIM(UPPER(' || sc || '))
+                AND u.owner = ANY ($1);'
+        USING org_list;
+
+        EXECUTE 'UPDATE ' || quote_ident(table_name) || '
+            SET
+                x_sce' || field_suffix || ' = id
+            FROM
+                (SELECT id, stat_cat, owner, value FROM actor_stat_cat_entry
+                    UNION SELECT id, stat_cat, owner, value FROM actor.stat_cat_entry) u
+            WHERE
+                    u.stat_cat = x_sc' || field_suffix || '
+                AND BTRIM(UPPER(u.value)) = BTRIM(UPPER(' || sce || '))
+                AND u.owner = ANY ($1);'
+        USING org_list;
+
+        EXECUTE 'SELECT migration_tools.assert(
+            NOT EXISTS (SELECT 1 FROM ' || quote_ident(table_name) || ' WHERE desired_sc' || field_suffix || ' <> '''' AND x_sc' || field_suffix || ' IS NULL),
+            ''Cannot find a desired stat cat'',
+            ''Found all desired stat cats''
+        );';
+
+        EXECUTE 'SELECT migration_tools.assert(
+            NOT EXISTS (SELECT 1 FROM ' || quote_ident(table_name) || ' WHERE desired_sce' || field_suffix || ' <> '''' AND x_sce' || field_suffix || ' IS NULL),
+            ''Cannot find a desired stat cat entry'',
+            ''Found all desired stat cat entries''
+        );';
+
+    END;
+$$ LANGUAGE PLPGSQL STRICT VOLATILE;
+
+-- convenience functions for adding shelving locations
+DROP FUNCTION IF EXISTS migration_tools.find_shelf(INT,TEXT);
+CREATE OR REPLACE FUNCTION migration_tools.find_shelf(org_id INT, shelf_name TEXT) RETURNS INTEGER AS $$
+DECLARE
+    return_id   INT;
+    d           INT;
+    cur_id      INT;
+BEGIN
+    SELECT INTO d MAX(distance) FROM actor.org_unit_ancestors_distance(org_id);
+    WHILE d >= 0
+    LOOP
+        SELECT INTO cur_id id FROM actor.org_unit_ancestor_at_depth(org_id,d);
+        SELECT INTO return_id id FROM asset.copy_location WHERE owning_lib = cur_id AND name ILIKE shelf_name;
+        IF return_id IS NOT NULL THEN
+                RETURN return_id;
+        END IF;
+        d := d - 1;
+    END LOOP;
+
+    RETURN NULL;
+END
+$$ LANGUAGE plpgsql;
+
+-- may remove later but testing using this with new migration scripts and not loading acls until go live
+
+DROP FUNCTION IF EXISTS migration_tools.find_mig_shelf(INT,TEXT);
+CREATE OR REPLACE FUNCTION migration_tools.find_mig_shelf(org_id INT, shelf_name TEXT) RETURNS INTEGER AS $$
+DECLARE
+    return_id   INT;
+    d           INT;
+    cur_id      INT;
+BEGIN
+    SELECT INTO d MAX(distance) FROM actor.org_unit_ancestors_distance(org_id);
+    WHILE d >= 0
+    LOOP
+        SELECT INTO cur_id id FROM actor.org_unit_ancestor_at_depth(org_id,d);
+        
+        SELECT INTO return_id id FROM 
+            (SELECT * FROM asset.copy_location UNION ALL SELECT * FROM asset_copy_location) x
+            WHERE owning_lib = cur_id AND name ILIKE shelf_name;
+        IF return_id IS NOT NULL THEN
+                RETURN return_id;
+        END IF;
+        d := d - 1;
+    END LOOP;
+
+    RETURN NULL;
+END
+$$ LANGUAGE plpgsql;
+
+DROP FUNCTION IF EXISTS migration_tools.munge_sf9(INTEGER,TEXT,TEXT);
+CREATE OR REPLACE FUNCTION migration_tools.merge_group(bib_id INTEGER,new_sf9 TEXT,force TEXT DEFAULT 'false')
+    RETURNS BOOLEAN AS 
+$BODY$
+DECLARE
+	marc_xml	TEXT;
+	new_marc	TEXT;
+BEGIN
+	SELECT marc FROM biblio.record_entry WHERE id = bib_id INTO marc_xml;
+	
+	SELECT munge_sf9(marc_xml,new_sf9,force) INTO new_marc;
+	UPDATE biblio.record_entry SET marc = new_marc WHERE id = bib_id;
+	
+	RETURN true;
+END;
+$BODY$ LANGUAGE plpgsql;
+
+DROP FUNCTION IF EXISTS migration_tools.munge_sf9(TEXT,TEXT,TEXT);
+CREATE OR REPLACE FUNCTION migration_tools.munge_sf9(marc_xml TEXT, new_9_to_set TEXT, force TEXT)
+ RETURNS TEXT
+ LANGUAGE plperlu
+AS $function$
+use strict;
+use warnings;
+
+use MARC::Record;
+use MARC::File::XML (BinaryEncoding => 'utf8');
+
+binmode(STDERR, ':bytes');
+binmode(STDOUT, ':utf8');
+binmode(STDERR, ':utf8');
+
+my $marc_xml = shift;
+my $new_9_to_set = shift;
+my $force = shift;
+
+$marc_xml =~ s/(<leader>.........)./${1}a/;
+
+eval {
+    $marc_xml = MARC::Record->new_from_xml($marc_xml);
+};
+if ($@) {
+    #elog("could not parse $bibid: $@\n");
+    import MARC::File::XML (BinaryEncoding => 'utf8');
+    return $marc_xml;
+}
+
+my @uris = $marc_xml->field('856');
+return $marc_xml->as_xml_record() unless @uris;
+
+foreach my $field (@uris) {
+    my $ind1 = $field->indicator('1');
+    if (!defined $ind1) { next; }
+    if ($ind1 ne '1' && $ind1 ne '4' && $force eq 'false') { next; }
+	if ($ind1 ne '1' && $ind1 ne '4' && $force eq 'true') { $field->set_indicator(1,'4'); }
+    my $ind2 = $field->indicator('2');
+    if (!defined $ind2) { next; }
+    if ($ind2 ne '0' && $ind2 ne '1' && $force eq 'false') { next; }
+    if ($ind2 ne '0' && $ind2 ne '1' && $force eq 'true') { $field->set_indicator(2,'0'); }
+    $field->add_subfields( '9' => $new_9_to_set );
+}
+
+return $marc_xml->as_xml_record();
+
+$function$;
+
+DROP FUNCTION IF EXISTS migration_tools.owner_change_sf9_substring_match(TEXT,TEXT,TEXT);
+CREATE OR REPLACE FUNCTION migration_tools.owner_change_sf9_substring_match (marc_xml TEXT, substring_old_value TEXT, new_value TEXT, fix_indicators TEXT)
+ RETURNS TEXT
+ LANGUAGE plperlu
+AS $function$
+use strict;
+use warnings;
+
+use MARC::Record;
+use MARC::File::XML (BinaryEncoding => 'utf8');
+
+binmode(STDERR, ':bytes');
+binmode(STDOUT, ':utf8');
+binmode(STDERR, ':utf8');
+
+my $marc_xml = shift;
+my $substring_old_value = shift;
+my $new_value = shift;
+my $fix_indicators = shift;
+
+$marc_xml =~ s/(<leader>.........)./${1}a/;
+
+eval {
+    $marc_xml = MARC::Record->new_from_xml($marc_xml);
+};
+if ($@) {
+    #elog("could not parse $bibid: $@\n");
+    import MARC::File::XML (BinaryEncoding => 'utf8');
+    return $marc_xml;
+}
+
+my @uris = $marc_xml->field('856');
+return $marc_xml->as_xml_record() unless @uris;
+
+foreach my $field (@uris) {
+    my $ind1 = $field->indicator('1');
+    if (defined $ind1) {
+	    if ($ind1 ne '1' && $ind1 ne '4' && $fix_indicators eq 'true') {
+            $field->set_indicator(1,'4');
+        }
+    }
+    my $ind2 = $field->indicator('2');
+    if (defined $ind2) {
+        if ($ind2 ne '0' && $ind2 ne '1' && $fix_indicators eq 'true') {
+            $field->set_indicator(2,'0');
+        }
+    }
+    if ($field->as_string('9') =~ qr/$substring_old_value/) {
+        $field->delete_subfield('9');
+        $field->add_subfields( '9' => $new_value );
+    }
+    $marc_xml->delete_field($field); # -- we're going to dedup and add them back
+}
+
+my %hash = (map { ($_->as_usmarc => $_) } @uris); # -- courtesy of an old Mike Rylander post :-)
+$marc_xml->insert_fields_ordered( values( %hash ) );
+
+return $marc_xml->as_xml_record();
+
+$function$;
+-- strip marc tag
+DROP FUNCTION IF EXISTS migration_tools.strip_tag(TEXT,TEXT);
+CREATE OR REPLACE FUNCTION migration_tools.strip_tag(marc TEXT, tag TEXT)
+ RETURNS TEXT
+ LANGUAGE plperlu
+AS $function$
+use strict;
+use warnings;
+
+use MARC::Record;
+use MARC::File::XML (BinaryEncoding => 'utf8');
+
+binmode(STDERR, ':bytes');
+binmode(STDOUT, ':utf8');
+binmode(STDERR, ':utf8');
+
+my $marc_xml = shift;
+my $tag = shift;
+
+$marc_xml =~ s/(<leader>.........)./${1}a/;
+
+eval {
+    $marc_xml = MARC::Record->new_from_xml($marc_xml);
+};
+if ($@) {
+    #elog("could not parse $bibid: $@\n");
+    import MARC::File::XML (BinaryEncoding => 'utf8');
+    return $marc_xml;
+}
+
+my @fields = $marc_xml->field($tag);
+return $marc_xml->as_xml_record() unless @fields;
+
+$marc_xml->delete_fields(@fields);
+
+return $marc_xml->as_xml_record();
+
+$function$;
+
+-- convenience function for linking to the item staging table
+
+CREATE OR REPLACE FUNCTION migration_tools.handle_item_barcode (TEXT,TEXT,TEXT,TEXT,BOOLEAN) RETURNS VOID AS $$
+    DECLARE
+        table_schema ALIAS FOR $1;
+        table_name ALIAS FOR $2;
+        foreign_column_name ALIAS FOR $3;
+        main_column_name ALIAS FOR $4;
+        btrim_desired ALIAS FOR $5;
+        proceed BOOLEAN;
+    BEGIN
+        EXECUTE 'SELECT EXISTS (
+            SELECT 1
+            FROM information_schema.columns
+            WHERE table_schema = $1
+            AND table_name = $2
+            and column_name = $3
+        )' INTO proceed USING table_schema, table_name, foreign_column_name;
+        IF NOT proceed THEN
+            RAISE EXCEPTION '%.% missing column %', table_schema, table_name, foreign_column_name; 
+        END IF;
+
+        EXECUTE 'SELECT EXISTS (
+            SELECT 1
+            FROM information_schema.columns
+            WHERE table_schema = $1
+            AND table_name = ''asset_copy_legacy''
+            and column_name = $2
+        )' INTO proceed USING table_schema, main_column_name;
+        IF NOT proceed THEN
+            RAISE EXCEPTION 'No %.asset_copy_legacy with column %', table_schema, main_column_name; 
+        END IF;
+
+        EXECUTE 'ALTER TABLE '
+            || quote_ident(table_name)
+            || ' DROP COLUMN IF EXISTS x_item';
+        EXECUTE 'ALTER TABLE '
+            || quote_ident(table_name)
+            || ' ADD COLUMN x_item BIGINT';
+
+        IF btrim_desired THEN
+            EXECUTE 'UPDATE ' || quote_ident(table_name) || ' a'
+                || ' SET x_item = b.id FROM asset_copy_legacy b'
+                || ' WHERE BTRIM(a.' || quote_ident(foreign_column_name)
+                || ') = BTRIM(b.' || quote_ident(main_column_name) || ')';
+        ELSE
+            EXECUTE 'UPDATE ' || quote_ident(table_name) || ' a'
+                || ' SET x_item = b.id FROM asset_copy_legacy b'
+                || ' WHERE a.' || quote_ident(foreign_column_name)
+                || ' = b.' || quote_ident(main_column_name);
+        END IF;
+
+        --EXECUTE 'SELECT migration_tools.assert(
+        --    NOT EXISTS (SELECT 1 FROM ' || quote_ident(table_name) || ' WHERE ' || quote_ident(foreign_column_name) || ' <> '''' AND x_item IS NULL),
+        --    ''Cannot link every barcode'',
+        --    ''Every barcode linked''
+        --);';
+
+    END;
+$$ LANGUAGE PLPGSQL STRICT VOLATILE;
+
+-- convenience function for linking to the user staging table
+
+CREATE OR REPLACE FUNCTION migration_tools.handle_user_barcode (TEXT,TEXT,TEXT,TEXT,BOOLEAN) RETURNS VOID AS $$
+    DECLARE
+        table_schema ALIAS FOR $1;
+        table_name ALIAS FOR $2;
+        foreign_column_name ALIAS FOR $3;
+        main_column_name ALIAS FOR $4;
+        btrim_desired ALIAS FOR $5;
+        proceed BOOLEAN;
+    BEGIN
+        EXECUTE 'SELECT EXISTS (
+            SELECT 1
+            FROM information_schema.columns
+            WHERE table_schema = $1
+            AND table_name = $2
+            and column_name = $3
+        )' INTO proceed USING table_schema, table_name, foreign_column_name;
+        IF NOT proceed THEN
+            RAISE EXCEPTION '%.% missing column %', table_schema, table_name, foreign_column_name; 
+        END IF;
+
+        EXECUTE 'SELECT EXISTS (
+            SELECT 1
+            FROM information_schema.columns
+            WHERE table_schema = $1
+            AND table_name = ''actor_usr_legacy''
+            and column_name = $2
+        )' INTO proceed USING table_schema, main_column_name;
+        IF NOT proceed THEN
+            RAISE EXCEPTION 'No %.actor_usr_legacy with column %', table_schema, main_column_name; 
+        END IF;
+
+        EXECUTE 'ALTER TABLE '
+            || quote_ident(table_name)
+            || ' DROP COLUMN IF EXISTS x_user';
+        EXECUTE 'ALTER TABLE '
+            || quote_ident(table_name)
+            || ' ADD COLUMN x_user INTEGER';
+
+        IF btrim_desired THEN
+            EXECUTE 'UPDATE ' || quote_ident(table_name) || ' a'
+                || ' SET x_user = b.id FROM actor_usr_legacy b'
+                || ' WHERE BTRIM(a.' || quote_ident(foreign_column_name)
+                || ') = BTRIM(b.' || quote_ident(main_column_name) || ')';
+        ELSE
+            EXECUTE 'UPDATE ' || quote_ident(table_name) || ' a'
+                || ' SET x_user = b.id FROM actor_usr_legacy b'
+                || ' WHERE a.' || quote_ident(foreign_column_name)
+                || ' = b.' || quote_ident(main_column_name);
+        END IF;
+
+        --EXECUTE 'SELECT migration_tools.assert(
+        --    NOT EXISTS (SELECT 1 FROM ' || quote_ident(table_name) || ' WHERE ' || quote_ident(foreign_column_name) || ' <> '''' AND x_user IS NULL),
+        --    ''Cannot link every barcode'',
+        --    ''Every barcode linked''
+        --);';
+
+    END;
+$$ LANGUAGE PLPGSQL STRICT VOLATILE;
+
+-- convenience function for linking two tables
+-- e.g. select migration_tools.handle_link(:'migschema','asset_copy','barcode','test_foo','l_barcode','x_acp_id',false);
+CREATE OR REPLACE FUNCTION migration_tools.handle_link (TEXT,TEXT,TEXT,TEXT,TEXT,TEXT,BOOLEAN) RETURNS VOID AS $$
+    DECLARE
+        table_schema ALIAS FOR $1;
+        table_a ALIAS FOR $2;
+        column_a ALIAS FOR $3;
+        table_b ALIAS FOR $4;
+        column_b ALIAS FOR $5;
+        column_x ALIAS FOR $6;
+        btrim_desired ALIAS FOR $7;
+        proceed BOOLEAN;
+    BEGIN
+        EXECUTE 'SELECT EXISTS (
+            SELECT 1
+            FROM information_schema.columns
+            WHERE table_schema = $1
+            AND table_name = $2
+            and column_name = $3
+        )' INTO proceed USING table_schema, table_a, column_a;
+        IF NOT proceed THEN
+            RAISE EXCEPTION '%.% missing column %', table_schema, table_a, column_a; 
+        END IF;
+
+        EXECUTE 'SELECT EXISTS (
+            SELECT 1
+            FROM information_schema.columns
+            WHERE table_schema = $1
+            AND table_name = $2
+            and column_name = $3
+        )' INTO proceed USING table_schema, table_b, column_b;
+        IF NOT proceed THEN
+            RAISE EXCEPTION '%.% missing column %', table_schema, table_b, column_b; 
+        END IF;
+
+        EXECUTE 'ALTER TABLE '
+            || quote_ident(table_b)
+            || ' DROP COLUMN IF EXISTS ' || quote_ident(column_x);
+        EXECUTE 'ALTER TABLE '
+            || quote_ident(table_b)
+            || ' ADD COLUMN ' || quote_ident(column_x) || ' BIGINT';
+
+        IF btrim_desired THEN
+            EXECUTE 'UPDATE ' || quote_ident(table_b) || ' b'
+                || ' SET ' || quote_ident(column_x) || ' = a.id FROM ' || quote_ident(table_a) || ' a'
+                || ' WHERE BTRIM(a.' || quote_ident(column_a)
+                || ') = BTRIM(b.' || quote_ident(column_b) || ')';
+        ELSE
+            EXECUTE 'UPDATE ' || quote_ident(table_b) || ' b'
+                || ' SET ' || quote_ident(column_x) || ' = a.id FROM ' || quote_ident(table_a) || ' a'
+                || ' WHERE a.' || quote_ident(column_a)
+                || ' = b.' || quote_ident(column_b);
+        END IF;
+
+    END;
+$$ LANGUAGE PLPGSQL STRICT VOLATILE;
+
+-- convenience function for linking two tables, but copying column w into column x instead of "id"
+-- e.g. select migration_tools.handle_link2(:'migschema','asset_copy','barcode','test_foo','l_barcode','id','x_acp_id',false);
+CREATE OR REPLACE FUNCTION migration_tools.handle_link2 (TEXT,TEXT,TEXT,TEXT,TEXT,TEXT,TEXT,BOOLEAN) RETURNS VOID AS $$
+    DECLARE
+        table_schema ALIAS FOR $1;
+        table_a ALIAS FOR $2;
+        column_a ALIAS FOR $3;
+        table_b ALIAS FOR $4;
+        column_b ALIAS FOR $5;
+        column_w ALIAS FOR $6;
+        column_x ALIAS FOR $7;
+        btrim_desired ALIAS FOR $8;
+        proceed BOOLEAN;
+    BEGIN
+        EXECUTE 'SELECT EXISTS (
+            SELECT 1
+            FROM information_schema.columns
+            WHERE table_schema = $1
+            AND table_name = $2
+            and column_name = $3
+        )' INTO proceed USING table_schema, table_a, column_a;
+        IF NOT proceed THEN
+            RAISE EXCEPTION '%.% missing column %', table_schema, table_a, column_a; 
+        END IF;
+
+        EXECUTE 'SELECT EXISTS (
+            SELECT 1
+            FROM information_schema.columns
+            WHERE table_schema = $1
+            AND table_name = $2
+            and column_name = $3
+        )' INTO proceed USING table_schema, table_b, column_b;
+        IF NOT proceed THEN
+            RAISE EXCEPTION '%.% missing column %', table_schema, table_b, column_b; 
+        END IF;
+
+        EXECUTE 'ALTER TABLE '
+            || quote_ident(table_b)
+            || ' DROP COLUMN IF EXISTS ' || quote_ident(column_x);
+        EXECUTE 'ALTER TABLE '
+            || quote_ident(table_b)
+            || ' ADD COLUMN ' || quote_ident(column_x) || ' TEXT';
+
+        IF btrim_desired THEN
+            EXECUTE 'UPDATE ' || quote_ident(table_b) || ' b'
+                || ' SET ' || quote_ident(column_x) || ' = a.' || quote_ident(column_w) || ' FROM ' || quote_ident(table_a) || ' a'
+                || ' WHERE BTRIM(a.' || quote_ident(column_a)
+                || ') = BTRIM(b.' || quote_ident(column_b) || ')';
+        ELSE
+            EXECUTE 'UPDATE ' || quote_ident(table_b) || ' b'
+                || ' SET ' || quote_ident(column_x) || ' = a.' || quote_ident(column_w) || ' FROM ' || quote_ident(table_a) || ' a'
+                || ' WHERE a.' || quote_ident(column_a)
+                || ' = b.' || quote_ident(column_b);
+        END IF;
+
+    END;
+$$ LANGUAGE PLPGSQL STRICT VOLATILE;
+
+-- convenience function for linking two tables, but copying column w into column x instead of "id". Unlike handle_link2, this one won't drop the target column, and it also doesn't have a final boolean argument for btrim
+-- e.g. select migration_tools.handle_link3(:'migschema','asset_copy','barcode','test_foo','l_barcode','id','x_acp_id');
+CREATE OR REPLACE FUNCTION migration_tools.handle_link3 (TEXT,TEXT,TEXT,TEXT,TEXT,TEXT,TEXT) RETURNS VOID AS $$
+    DECLARE
+        table_schema ALIAS FOR $1;
+        table_a ALIAS FOR $2;
+        column_a ALIAS FOR $3;
+        table_b ALIAS FOR $4;
+        column_b ALIAS FOR $5;
+        column_w ALIAS FOR $6;
+        column_x ALIAS FOR $7;
+        proceed BOOLEAN;
+    BEGIN
+        EXECUTE 'SELECT EXISTS (
+            SELECT 1
+            FROM information_schema.columns
+            WHERE table_schema = $1
+            AND table_name = $2
+            and column_name = $3
+        )' INTO proceed USING table_schema, table_a, column_a;
+        IF NOT proceed THEN
+            RAISE EXCEPTION '%.% missing column %', table_schema, table_a, column_a; 
+        END IF;
+
+        EXECUTE 'SELECT EXISTS (
+            SELECT 1
+            FROM information_schema.columns
+            WHERE table_schema = $1
+            AND table_name = $2
+            and column_name = $3
+        )' INTO proceed USING table_schema, table_b, column_b;
+        IF NOT proceed THEN
+            RAISE EXCEPTION '%.% missing column %', table_schema, table_b, column_b; 
+        END IF;
+
+        EXECUTE 'UPDATE ' || quote_ident(table_b) || ' b'
+            || ' SET ' || quote_ident(column_x) || ' = a.' || quote_ident(column_w) || ' FROM ' || quote_ident(table_a) || ' a'
+            || ' WHERE a.' || quote_ident(column_a)
+            || ' = b.' || quote_ident(column_b);
+
+    END;
+$$ LANGUAGE PLPGSQL STRICT VOLATILE;
+
+CREATE OR REPLACE FUNCTION migration_tools.handle_link3_skip_null_or_empty_string (TEXT,TEXT,TEXT,TEXT,TEXT,TEXT,TEXT) RETURNS VOID AS $$
+    DECLARE
+        table_schema ALIAS FOR $1;
+        table_a ALIAS FOR $2;
+        column_a ALIAS FOR $3;
+        table_b ALIAS FOR $4;
+        column_b ALIAS FOR $5;
+        column_w ALIAS FOR $6;
+        column_x ALIAS FOR $7;
+        proceed BOOLEAN;
+    BEGIN
+        EXECUTE 'SELECT EXISTS (
+            SELECT 1
+            FROM information_schema.columns
+            WHERE table_schema = $1
+            AND table_name = $2
+            and column_name = $3
+        )' INTO proceed USING table_schema, table_a, column_a;
+        IF NOT proceed THEN
+            RAISE EXCEPTION '%.% missing column %', table_schema, table_a, column_a; 
+        END IF;
+
+        EXECUTE 'SELECT EXISTS (
+            SELECT 1
+            FROM information_schema.columns
+            WHERE table_schema = $1
+            AND table_name = $2
+            and column_name = $3
+        )' INTO proceed USING table_schema, table_b, column_b;
+        IF NOT proceed THEN
+            RAISE EXCEPTION '%.% missing column %', table_schema, table_b, column_b; 
+        END IF;
+
+        EXECUTE 'UPDATE ' || quote_ident(table_b) || ' b'
+            || ' SET ' || quote_ident(column_x) || ' = a.' || quote_ident(column_w) || ' FROM ' || quote_ident(table_a) || ' a'
+            || ' WHERE a.' || quote_ident(column_a)
+            || ' = b.' || quote_ident(column_b)
+            || ' AND NULLIF(a.' || quote_ident(column_w) || ','''') IS NOT NULL';
+
+    END;
+$$ LANGUAGE PLPGSQL STRICT VOLATILE;
+
+CREATE OR REPLACE FUNCTION migration_tools.handle_link3_skip_null (TEXT,TEXT,TEXT,TEXT,TEXT,TEXT,TEXT) RETURNS VOID AS $$
+    DECLARE
+        table_schema ALIAS FOR $1;
+        table_a ALIAS FOR $2;
+        column_a ALIAS FOR $3;
+        table_b ALIAS FOR $4;
+        column_b ALIAS FOR $5;
+        column_w ALIAS FOR $6;
+        column_x ALIAS FOR $7;
+        proceed BOOLEAN;
+    BEGIN
+        EXECUTE 'SELECT EXISTS (
+            SELECT 1
+            FROM information_schema.columns
+            WHERE table_schema = $1
+            AND table_name = $2
+            and column_name = $3
+        )' INTO proceed USING table_schema, table_a, column_a;
+        IF NOT proceed THEN
+            RAISE EXCEPTION '%.% missing column %', table_schema, table_a, column_a; 
+        END IF;
+
+        EXECUTE 'SELECT EXISTS (
+            SELECT 1
+            FROM information_schema.columns
+            WHERE table_schema = $1
+            AND table_name = $2
+            and column_name = $3
+        )' INTO proceed USING table_schema, table_b, column_b;
+        IF NOT proceed THEN
+            RAISE EXCEPTION '%.% missing column %', table_schema, table_b, column_b; 
+        END IF;
+
+        EXECUTE 'UPDATE ' || quote_ident(table_b) || ' b'
+            || ' SET ' || quote_ident(column_x) || ' = a.' || quote_ident(column_w) || ' FROM ' || quote_ident(table_a) || ' a'
+            || ' WHERE a.' || quote_ident(column_a)
+            || ' = b.' || quote_ident(column_b)
+            || ' AND a.' || quote_ident(column_w) || ' IS NOT NULL';
+
+    END;
+$$ LANGUAGE PLPGSQL STRICT VOLATILE;
+
+CREATE OR REPLACE FUNCTION migration_tools.handle_link3_skip_true (TEXT,TEXT,TEXT,TEXT,TEXT,TEXT,TEXT) RETURNS VOID AS $$
+    DECLARE
+        table_schema ALIAS FOR $1;
+        table_a ALIAS FOR $2;
+        column_a ALIAS FOR $3;
+        table_b ALIAS FOR $4;
+        column_b ALIAS FOR $5;
+        column_w ALIAS FOR $6;
+        column_x ALIAS FOR $7;
+        proceed BOOLEAN;
+    BEGIN
+        EXECUTE 'SELECT EXISTS (
+            SELECT 1
+            FROM information_schema.columns
+            WHERE table_schema = $1
+            AND table_name = $2
+            and column_name = $3
+        )' INTO proceed USING table_schema, table_a, column_a;
+        IF NOT proceed THEN
+            RAISE EXCEPTION '%.% missing column %', table_schema, table_a, column_a; 
+        END IF;
+
+        EXECUTE 'SELECT EXISTS (
+            SELECT 1
+            FROM information_schema.columns
+            WHERE table_schema = $1
+            AND table_name = $2
+            and column_name = $3
+        )' INTO proceed USING table_schema, table_b, column_b;
+        IF NOT proceed THEN
+            RAISE EXCEPTION '%.% missing column %', table_schema, table_b, column_b; 
+        END IF;
+
+        EXECUTE 'UPDATE ' || quote_ident(table_b) || ' b'
+            || ' SET ' || quote_ident(column_x) || ' = a.' || quote_ident(column_w) || ' FROM ' || quote_ident(table_a) || ' a'
+            || ' WHERE a.' || quote_ident(column_a)
+            || ' = b.' || quote_ident(column_b)
+            || ' AND a.' || quote_ident(column_w) || ' IS NOT TRUE';
+
+    END;
+$$ LANGUAGE PLPGSQL STRICT VOLATILE;
+
+CREATE OR REPLACE FUNCTION migration_tools.handle_link3_skip_false (TEXT,TEXT,TEXT,TEXT,TEXT,TEXT,TEXT) RETURNS VOID AS $$
+    DECLARE
+        table_schema ALIAS FOR $1;
+        table_a ALIAS FOR $2;
+        column_a ALIAS FOR $3;
+        table_b ALIAS FOR $4;
+        column_b ALIAS FOR $5;
+        column_w ALIAS FOR $6;
+        column_x ALIAS FOR $7;
+        proceed BOOLEAN;
+    BEGIN
+        EXECUTE 'SELECT EXISTS (
+            SELECT 1
+            FROM information_schema.columns
+            WHERE table_schema = $1
+            AND table_name = $2
+            and column_name = $3
+        )' INTO proceed USING table_schema, table_a, column_a;
+        IF NOT proceed THEN
+            RAISE EXCEPTION '%.% missing column %', table_schema, table_a, column_a; 
+        END IF;
+
+        EXECUTE 'SELECT EXISTS (
+            SELECT 1
+            FROM information_schema.columns
+            WHERE table_schema = $1
+            AND table_name = $2
+            and column_name = $3
+        )' INTO proceed USING table_schema, table_b, column_b;
+        IF NOT proceed THEN
+            RAISE EXCEPTION '%.% missing column %', table_schema, table_b, column_b; 
+        END IF;
+
+        EXECUTE 'UPDATE ' || quote_ident(table_b) || ' b'
+            || ' SET ' || quote_ident(column_x) || ' = a.' || quote_ident(column_w) || ' FROM ' || quote_ident(table_a) || ' a'
+            || ' WHERE a.' || quote_ident(column_a)
+            || ' = b.' || quote_ident(column_b)
+            || ' AND a.' || quote_ident(column_w) || ' IS NOT FALSE';
+
+    END;
+$$ LANGUAGE PLPGSQL STRICT VOLATILE;
+
+CREATE OR REPLACE FUNCTION migration_tools.handle_link3_concat_skip_null (TEXT,TEXT,TEXT,TEXT,TEXT,TEXT,TEXT) RETURNS VOID AS $$
+    DECLARE
+        table_schema ALIAS FOR $1;
+        table_a ALIAS FOR $2;
+        column_a ALIAS FOR $3;
+        table_b ALIAS FOR $4;
+        column_b ALIAS FOR $5;
+        column_w ALIAS FOR $6;
+        column_x ALIAS FOR $7;
+        proceed BOOLEAN;
+    BEGIN
+        EXECUTE 'SELECT EXISTS (
+            SELECT 1
+            FROM information_schema.columns
+            WHERE table_schema = $1
+            AND table_name = $2
+            and column_name = $3
+        )' INTO proceed USING table_schema, table_a, column_a;
+        IF NOT proceed THEN
+            RAISE EXCEPTION '%.% missing column %', table_schema, table_a, column_a; 
+        END IF;
+
+        EXECUTE 'SELECT EXISTS (
+            SELECT 1
+            FROM information_schema.columns
+            WHERE table_schema = $1
+            AND table_name = $2
+            and column_name = $3
+        )' INTO proceed USING table_schema, table_b, column_b;
+        IF NOT proceed THEN
+            RAISE EXCEPTION '%.% missing column %', table_schema, table_b, column_b; 
+        END IF;
+
+        EXECUTE 'UPDATE ' || quote_ident(table_b) || ' b'
+            || ' SET ' || quote_ident(column_x) || ' = CONCAT_WS('' ; '',b.' || quote_ident(column_x) || ',a.' || quote_ident(column_w) || ') FROM ' || quote_ident(table_a) || ' a'
+            || ' WHERE a.' || quote_ident(column_a)
+            || ' = b.' || quote_ident(column_b)
+            || ' AND NULLIF(a.' || quote_ident(column_w) || ','''') IS NOT NULL';
+
+    END;
+$$ LANGUAGE PLPGSQL STRICT VOLATILE;
+
+-- convenience function for handling desired asset stat cats
+
+CREATE OR REPLACE FUNCTION migration_tools.vivicate_asset_sc_and_sce (TEXT,TEXT,TEXT,TEXT) RETURNS VOID AS $$
+    DECLARE
+        table_schema ALIAS FOR $1;
+        table_name ALIAS FOR $2;
+        field_suffix ALIAS FOR $3; -- for distinguishing between desired_sce1, desired_sce2, etc.
+        org_shortname ALIAS FOR $4;
+        proceed BOOLEAN;
+        org INTEGER;
+        org_list INTEGER[];
+        sc TEXT;
+        sce TEXT;
+    BEGIN
+
+        SELECT 'desired_sc' || field_suffix INTO sc;
+        SELECT 'desired_sce' || field_suffix INTO sce;
+
+        EXECUTE 'SELECT EXISTS (
+            SELECT 1
+            FROM information_schema.columns
+            WHERE table_schema = $1
+            AND table_name = $2
+            and column_name = $3
+        )' INTO proceed USING table_schema, table_name, sc;
+        IF NOT proceed THEN
+            RAISE EXCEPTION 'Missing column %', sc; 
+        END IF;
+        EXECUTE 'SELECT EXISTS (
+            SELECT 1
+            FROM information_schema.columns
+            WHERE table_schema = $1
+            AND table_name = $2
+            and column_name = $3
+        )' INTO proceed USING table_schema, table_name, sce;
+        IF NOT proceed THEN
+            RAISE EXCEPTION 'Missing column %', sce; 
+        END IF;
+
+        SELECT id INTO org FROM actor.org_unit WHERE shortname = org_shortname;
+        IF org IS NULL THEN
+            RAISE EXCEPTION 'Cannot find org by shortname';
+        END IF;
+        SELECT INTO org_list ARRAY_ACCUM(id) FROM actor.org_unit_full_path( org );
+
+        -- caller responsible for their own truncates though we try to prevent duplicates
+        EXECUTE 'INSERT INTO asset_stat_cat (owner, name)
+            SELECT DISTINCT
+                 $1
+                ,BTRIM('||sc||')
+            FROM 
+                ' || quote_ident(table_name) || '
+            WHERE
+                NULLIF(BTRIM('||sc||'),'''') IS NOT NULL
+                AND NOT EXISTS (
+                    SELECT id
+                    FROM asset.stat_cat
+                    WHERE owner = ANY ($2)
+                    AND name = BTRIM('||sc||')
+                )
+                AND NOT EXISTS (
+                    SELECT id
+                    FROM asset_stat_cat
+                    WHERE owner = ANY ($2)
+                    AND name = BTRIM('||sc||')
+                )
+            ORDER BY 2;'
+        USING org, org_list;
+
+        EXECUTE 'INSERT INTO asset_stat_cat_entry (stat_cat, owner, value)
+            SELECT DISTINCT
+                COALESCE(
+                    (SELECT id
+                        FROM asset.stat_cat
+                        WHERE owner = ANY ($2)
+                        AND BTRIM('||sc||') = BTRIM(name))
+                   ,(SELECT id
+                        FROM asset_stat_cat
+                        WHERE owner = ANY ($2)
+                        AND BTRIM('||sc||') = BTRIM(name))
+                )
+                ,$1
+                ,BTRIM('||sce||')
+            FROM 
+                ' || quote_ident(table_name) || '
+            WHERE
+                    NULLIF(BTRIM('||sc||'),'''') IS NOT NULL
+                AND NULLIF(BTRIM('||sce||'),'''') IS NOT NULL
+                AND NOT EXISTS (
+                    SELECT id
+                    FROM asset.stat_cat_entry
+                    WHERE stat_cat = (
+                        SELECT id
+                        FROM asset.stat_cat
+                        WHERE owner = ANY ($2)
+                        AND BTRIM('||sc||') = BTRIM(name)
+                    ) AND value = BTRIM('||sce||')
+                    AND owner = ANY ($2)
+                )
+                AND NOT EXISTS (
+                    SELECT id
+                    FROM asset_stat_cat_entry
+                    WHERE stat_cat = (
+                        SELECT id
+                        FROM asset_stat_cat
+                        WHERE owner = ANY ($2)
+                        AND BTRIM('||sc||') = BTRIM(name)
+                    ) AND value = BTRIM('||sce||')
+                    AND owner = ANY ($2)
+                )
+            ORDER BY 1,3;'
+        USING org, org_list;
+    END;
+$$ LANGUAGE PLPGSQL STRICT VOLATILE;
+
+CREATE OR REPLACE FUNCTION migration_tools.handle_asset_sc_and_sce (TEXT,TEXT,TEXT,TEXT) RETURNS VOID AS $$
+    DECLARE
+        table_schema ALIAS FOR $1;
+        table_name ALIAS FOR $2;
+        field_suffix ALIAS FOR $3; -- for distinguishing between desired_sce1, desired_sce2, etc.
+        org_shortname ALIAS FOR $4;
+        proceed BOOLEAN;
+        org INTEGER;
+        org_list INTEGER[];
+        o INTEGER;
+        sc TEXT;
+        sce TEXT;
+    BEGIN
+        SELECT 'desired_sc' || field_suffix INTO sc;
+        SELECT 'desired_sce' || field_suffix INTO sce;
+        EXECUTE 'SELECT EXISTS (
+            SELECT 1
+            FROM information_schema.columns
+            WHERE table_schema = $1
+            AND table_name = $2
+            and column_name = $3
+        )' INTO proceed USING table_schema, table_name, sc;
+        IF NOT proceed THEN
+            RAISE EXCEPTION 'Missing column %', sc; 
+        END IF;
+        EXECUTE 'SELECT EXISTS (
+            SELECT 1
+            FROM information_schema.columns
+            WHERE table_schema = $1
+            AND table_name = $2
+            and column_name = $3
+        )' INTO proceed USING table_schema, table_name, sce;
+        IF NOT proceed THEN
+            RAISE EXCEPTION 'Missing column %', sce; 
+        END IF;
+
+        SELECT id INTO org FROM actor.org_unit WHERE shortname = org_shortname;
+        IF org IS NULL THEN
+            RAISE EXCEPTION 'Cannot find org by shortname';
+        END IF;
+
+        SELECT INTO org_list ARRAY_ACCUM(id) FROM actor.org_unit_full_path( org );
+
+        EXECUTE 'ALTER TABLE '
+            || quote_ident(table_name)
+            || ' DROP COLUMN IF EXISTS x_sc' || field_suffix;
+        EXECUTE 'ALTER TABLE '
+            || quote_ident(table_name)
+            || ' ADD COLUMN x_sc' || field_suffix || ' INTEGER';
+        EXECUTE 'ALTER TABLE '
+            || quote_ident(table_name)
+            || ' DROP COLUMN IF EXISTS x_sce' || field_suffix;
+        EXECUTE 'ALTER TABLE '
+            || quote_ident(table_name)
+            || ' ADD COLUMN x_sce' || field_suffix || ' INTEGER';
+
+
+        EXECUTE 'UPDATE ' || quote_ident(table_name) || '
+            SET
+                x_sc' || field_suffix || ' = id
+            FROM
+                (SELECT id, name, owner FROM asset_stat_cat
+                    UNION SELECT id, name, owner FROM asset.stat_cat) u
+            WHERE
+                    BTRIM(UPPER(u.name)) = BTRIM(UPPER(' || sc || '))
+                AND u.owner = ANY ($1);'
+        USING org_list;
+
+        EXECUTE 'UPDATE ' || quote_ident(table_name) || '
+            SET
+                x_sce' || field_suffix || ' = id
+            FROM
+                (SELECT id, stat_cat, owner, value FROM asset_stat_cat_entry
+                    UNION SELECT id, stat_cat, owner, value FROM asset.stat_cat_entry) u
+            WHERE
+                    u.stat_cat = x_sc' || field_suffix || '
+                AND BTRIM(UPPER(u.value)) = BTRIM(UPPER(' || sce || '))
+                AND u.owner = ANY ($1);'
+        USING org_list;
+
+        EXECUTE 'SELECT migration_tools.assert(
+            NOT EXISTS (SELECT 1 FROM ' || quote_ident(table_name) || ' WHERE desired_sc' || field_suffix || ' <> '''' AND x_sc' || field_suffix || ' IS NULL),
+            ''Cannot find a desired stat cat'',
+            ''Found all desired stat cats''
+        );';
+
+        EXECUTE 'SELECT migration_tools.assert(
+            NOT EXISTS (SELECT 1 FROM ' || quote_ident(table_name) || ' WHERE desired_sce' || field_suffix || ' <> '''' AND x_sce' || field_suffix || ' IS NULL),
+            ''Cannot find a desired stat cat entry'',
+            ''Found all desired stat cat entries''
+        );';
+
+    END;
+$$ LANGUAGE PLPGSQL STRICT VOLATILE;
+
+DROP FUNCTION IF EXISTS migration_tools.btrim_lcolumns(TEXT,TEXT);
+CREATE OR REPLACE FUNCTION migration_tools.btrim_lcolumns(s_name TEXT, t_name TEXT) RETURNS BOOLEAN
+ LANGUAGE plpgsql
+AS $function$
+DECLARE
+    c_name     TEXT;
+BEGIN
+
+    FOR c_name IN SELECT column_name FROM information_schema.columns WHERE 
+            table_name = t_name
+            AND table_schema = s_name
+            AND (data_type='text' OR data_type='character varying')
+            AND column_name like 'l_%'
+    LOOP
+       EXECUTE FORMAT('UPDATE ' || s_name || '.' || t_name || ' SET ' || c_name || ' = BTRIM(' || c_name || ')'); 
+    END LOOP;  
+
+    RETURN TRUE;
+END
+$function$;
+
+DROP FUNCTION IF EXISTS migration_tools.btrim_columns(TEXT,TEXT);
+CREATE OR REPLACE FUNCTION migration_tools.btrim_columns(s_name TEXT, t_name TEXT) RETURNS BOOLEAN
+ LANGUAGE plpgsql
+AS $function$
+DECLARE
+    c_name     TEXT;
+BEGIN
+
+    FOR c_name IN SELECT column_name FROM information_schema.columns WHERE 
+            table_name = t_name
+            AND table_schema = s_name
+            AND (data_type='text' OR data_type='character varying')
+    LOOP
+       EXECUTE FORMAT('UPDATE ' || s_name || '.' || t_name || ' SET ' || c_name || ' = BTRIM(' || c_name || ')'); 
+    END LOOP;  
+
+    RETURN TRUE;
+END
+$function$;
+
+DROP FUNCTION IF EXISTS migration_tools.null_empty_lcolumns(TEXT,TEXT);
+CREATE OR REPLACE FUNCTION migration_tools.null_empty_lcolumns(s_name TEXT, t_name TEXT) RETURNS BOOLEAN
+ LANGUAGE plpgsql
+AS $function$
+DECLARE
+    c_name     TEXT;
+BEGIN
+
+    FOR c_name IN SELECT column_name FROM information_schema.columns WHERE 
+            table_name = t_name
+            AND table_schema = s_name
+            AND (data_type='text' OR data_type='character varying')
+            AND column_name like 'l_%'
+    LOOP
+       EXECUTE FORMAT('UPDATE ' || s_name || '.' || t_name || ' SET ' || c_name || ' = NULL WHERE ' || c_name || ' = '''' '); 
+    END LOOP;  
+
+    RETURN TRUE;
+END
+$function$;
+
+DROP FUNCTION IF EXISTS migration_tools.null_empty_columns(TEXT,TEXT);
+CREATE OR REPLACE FUNCTION migration_tools.null_empty_columns(s_name TEXT, t_name TEXT) RETURNS BOOLEAN
+ LANGUAGE plpgsql
+AS $function$
+DECLARE
+    c_name     TEXT;
+BEGIN
+
+    FOR c_name IN SELECT column_name FROM information_schema.columns WHERE
+            table_name = t_name
+            AND table_schema = s_name
+            AND (data_type='text' OR data_type='character varying')
+    LOOP
+       EXECUTE FORMAT('UPDATE ' || s_name || '.' || t_name || ' SET ' || c_name || ' = NULL WHERE ' || c_name || ' = '''' ');
+    END LOOP;
+
+    RETURN TRUE;
+END
+$function$;
+
+
+-- convenience function for handling item barcode collisions in asset_copy_legacy
+
+CREATE OR REPLACE FUNCTION migration_tools.handle_asset_barcode_collisions(migration_schema TEXT) RETURNS VOID AS $function$
+DECLARE
+    x_barcode TEXT;
+    x_id BIGINT;
+    row_count NUMERIC;
+    internal_collision_count NUMERIC := 0;
+    incumbent_collision_count NUMERIC := 0;
+BEGIN
+    FOR x_barcode IN SELECT barcode FROM asset_copy_legacy WHERE x_migrate GROUP BY 1 HAVING COUNT(*) > 1
+    LOOP
+        FOR x_id IN SELECT id FROM asset_copy WHERE barcode = x_barcode
+        LOOP
+            UPDATE asset_copy SET barcode = migration_schema || '_internal_collision_' || id || '_' || barcode WHERE id = x_id;
+            GET DIAGNOSTICS row_count = ROW_COUNT;
+            internal_collision_count := internal_collision_count + row_count;
+        END LOOP;
+    END LOOP;
+    RAISE INFO '% internal collisions', internal_collision_count;
+    FOR x_barcode IN SELECT a.barcode FROM asset.copy a, asset_copy_legacy b WHERE x_migrate AND a.deleted IS FALSE AND a.barcode = b.barcode
+    LOOP
+        FOR x_id IN SELECT id FROM asset_copy_legacy WHERE barcode = x_barcode
+        LOOP
+            UPDATE asset_copy_legacy SET barcode = migration_schema || '_incumbent_collision_' || id || '_' || barcode WHERE id = x_id;
+            GET DIAGNOSTICS row_count = ROW_COUNT;
+            incumbent_collision_count := incumbent_collision_count + row_count;
+        END LOOP;
+    END LOOP;
+    RAISE INFO '% incumbent collisions', incumbent_collision_count;
+END
+$function$ LANGUAGE plpgsql;
+
+-- convenience function for handling patron barcode/usrname collisions in actor_usr_legacy
+-- this should be ran prior to populating actor_card
+
+CREATE OR REPLACE FUNCTION migration_tools.handle_actor_barcode_collisions(migration_schema TEXT) RETURNS VOID AS $function$
+DECLARE
+    x_barcode TEXT;
+    x_id BIGINT;
+    row_count NUMERIC;
+    internal_collision_count NUMERIC := 0;
+    incumbent_barcode_collision_count NUMERIC := 0;
+    incumbent_usrname_collision_count NUMERIC := 0;
+BEGIN
+    FOR x_barcode IN SELECT usrname FROM actor_usr_legacy WHERE x_migrate GROUP BY 1 HAVING COUNT(*) > 1
+    LOOP
+        FOR x_id IN SELECT id FROM actor_usr_legacy WHERE x_migrate AND usrname = x_barcode
+        LOOP
+            UPDATE actor_usr_legacy SET usrname = migration_schema || '_internal_collision_' || id || '_' || usrname WHERE id = x_id;
+            GET DIAGNOSTICS row_count = ROW_COUNT;
+            internal_collision_count := internal_collision_count + row_count;
+        END LOOP;
+    END LOOP;
+    RAISE INFO '% internal usrname/barcode collisions', internal_collision_count;
+
+    FOR x_barcode IN
+        SELECT a.barcode FROM actor.card a, actor_usr_legacy b WHERE x_migrate AND a.barcode = b.usrname
+    LOOP
+        FOR x_id IN SELECT DISTINCT id FROM actor_usr_legacy WHERE x_migrate AND usrname = x_barcode
+        LOOP
+            UPDATE actor_usr_legacy SET usrname = migration_schema || '_incumbent_barcode_collision_' || id || '_' || usrname WHERE id = x_id;
+            GET DIAGNOSTICS row_count = ROW_COUNT;
+            incumbent_barcode_collision_count := incumbent_barcode_collision_count + row_count;
+        END LOOP;
+    END LOOP;
+    RAISE INFO '% incumbent barcode collisions', incumbent_barcode_collision_count;
+
+    FOR x_barcode IN
+        SELECT a.usrname FROM actor.usr a, actor_usr_legacy b WHERE x_migrate AND a.deleted IS FALSE AND a.usrname = b.usrname
+    LOOP
+        FOR x_id IN SELECT DISTINCT id FROM actor_usr_legacy WHERE x_migrate AND usrname = x_barcode
+        LOOP
+            UPDATE actor_usr_legacy SET usrname = migration_schema || '_incumbent_usrname_collision_' || id || '_' || usrname WHERE id = x_id;
+            GET DIAGNOSTICS row_count = ROW_COUNT;
+            incumbent_usrname_collision_count := incumbent_usrname_collision_count + row_count;
+        END LOOP;
+    END LOOP;
+    RAISE INFO '% incumbent usrname collisions (post barcode collision munging)', incumbent_usrname_collision_count;
+END
+$function$ LANGUAGE plpgsql;