X-Git-Url: http://git.equinoxoli.org/?p=migration-tools.git;a=blobdiff_plain;f=sql%2Fbase%2Fbase.sql;h=84c35f326478583da2dde0019cd47b846e3b02e3;hp=f75ba2562ced28f611f61bfa9db6f8a876e54589;hb=8783c6c281d0aca08c1b88d1067d80a24a513696;hpb=b919be887d03ee11eef2c3193cea2ba6c1786d49

diff --git a/sql/base/base.sql b/sql/base/base.sql
index f75ba25..84c35f3 100644
--- a/sql/base/base.sql
+++ b/sql/base/base.sql
@@ -117,7 +117,7 @@ CREATE OR REPLACE FUNCTION migration_tools.init (TEXT) RETURNS VOID AS $$
         END;
         PERFORM migration_tools.exec( $1, 'DROP TABLE IF EXISTS ' || migration_schema || '.config;' );
         PERFORM migration_tools.exec( $1, 'CREATE TABLE ' || migration_schema || '.config ( key TEXT UNIQUE, value TEXT);' );
-        PERFORM migration_tools.exec( $1, 'INSERT INTO ' || migration_schema || '.config (key,value) VALUES ( ''production_tables'', ''asset.call_number,asset.call_number_prefix,asset.call_number_suffix,asset.copy_location,asset.copy,asset.stat_cat,asset.stat_cat_entry,asset.stat_cat_entry_copy_map,asset.copy_note,actor.usr,actor.card,actor.usr_address,actor.stat_cat,actor.stat_cat_entry,actor.stat_cat_entry_usr_map,actor.usr_note,actor.usr_standing_penalty,actor.usr_setting,action.circulation,action.hold_request,action.hold_notification,action.hold_request_note,action.hold_transit_copy,action.transit_copy,money.grocery,money.billing,money.cash_payment,money.forgive_payment,acq.provider,acq.provider_address,acq.provider_note,acq.provider_contact,acq.provider_contact_address,acq.fund,acq.fund_allocation,acq.fund_tag,acq.fund_tag_map,acq.funding_source,acq.funding_source_credit,acq.lineitem,acq.purchase_order,acq.po_item,acq.invoice,acq.invoice_item,acq.invoice_entry,acq.lineitem_detail,acq.fund_debit,acq.fund_transfer,acq.po_note,config.circ_matrix_matchpoint,config.circ_matrix_limit_set_map,config.hold_matrix_matchpoint,asset.copy_tag,asset.copy_tag_copy_map,config.copy_tag_type,serial.item,serial.item_note,serial.record_entry'' );' );
+        PERFORM migration_tools.exec( $1, 'INSERT INTO ' || migration_schema || '.config (key,value) VALUES ( ''production_tables'', ''asset.call_number,asset.call_number_prefix,asset.call_number_suffix,asset.copy_location,asset.copy,asset.stat_cat,asset.stat_cat_entry,asset.stat_cat_entry_copy_map,asset.copy_note,actor.usr,actor.card,actor.usr_address,actor.stat_cat,actor.stat_cat_entry,actor.stat_cat_entry_usr_map,actor.usr_note,actor.usr_standing_penalty,actor.usr_setting,action.circulation,action.hold_request,action.hold_notification,action.hold_request_note,action.hold_transit_copy,action.transit_copy,money.grocery,money.billing,money.cash_payment,money.forgive_payment,acq.provider,acq.provider_address,acq.provider_note,acq.provider_contact,acq.provider_contact_address,acq.fund,acq.fund_allocation,acq.fund_tag,acq.fund_tag_map,acq.funding_source,acq.funding_source_credit,acq.lineitem,acq.purchase_order,acq.po_item,acq.invoice,acq.invoice_item,acq.invoice_entry,acq.lineitem_detail,acq.fund_debit,acq.fund_transfer,acq.po_note,config.circ_matrix_matchpoint,config.circ_matrix_limit_set_map,config.hold_matrix_matchpoint,asset.copy_tag,asset.copy_tag_copy_map,config.copy_tag_type,serial.item,serial.item_note,serial.record_entry,biblio.record_entry'' );' );
         PERFORM migration_tools.exec( $1, 'INSERT INTO ' || migration_schema || '.config (key,value) VALUES ( ''country_code'', ''USA'' );' );
         PERFORM migration_tools.exec( $1, 'DROP TABLE IF EXISTS ' || migration_schema || '.fields_requiring_mapping;' );
         PERFORM migration_tools.exec( $1, 'CREATE TABLE ' || migration_schema || '.fields_requiring_mapping( table_schema TEXT, table_name TEXT, column_name TEXT, data_type TEXT);' );
@@ -607,6 +607,22 @@ CREATE OR REPLACE FUNCTION migration_tools.parse_out_address (TEXT) RETURNS TEXT
     END;
 $$ LANGUAGE PLPGSQL STRICT VOLATILE;
 
+CREATE OR REPLACE FUNCTION migration_tools.parse_out_address2 (TEXT) RETURNS TEXT AS $$
+    my ($address) = @_;
+
+    use Geo::StreetAddress::US;
+
+    my $a = Geo::StreetAddress::US->parse_location($address);
+
+    return [
+         "$a->{number} $a->{prefix} $a->{street} $a->{type} $a->{suffix}"
+        ,"$a->{sec_unit_type} $a->{sec_unit_num}"
+        ,$a->{city}
+        ,$a->{state}
+        ,$a->{zip}
+    ];
+$$ LANGUAGE PLPERLU STABLE;
+
 CREATE OR REPLACE FUNCTION migration_tools.rebarcode (o TEXT, t BIGINT) RETURNS TEXT AS $$
     DECLARE
         n TEXT := o;
@@ -1082,6 +1098,51 @@ CREATE OR REPLACE FUNCTION migration_tools.add_code39mod43_checkdigit (TEXT) RET
     return $barcode . $checkdigit;
 $$ LANGUAGE PLPERLU STRICT STABLE;
 
+-- add_mod16_checkdigit
+--   $barcode      source barcode
+--
+-- https://www.activebarcode.com/codes/checkdigit/modulo16.html
+
+CREATE OR REPLACE FUNCTION migration_tools.add_mod16_checkdigit (TEXT) RETURNS TEXT AS $$
+    my $barcode = shift;
+
+    my @digits = split //, $barcode;
+    my $total = 0;
+    foreach $digit (@digits) {
+        if ($digit =~ /[0-9]/) { $total += $digit;
+        } elsif ($digit eq '-') { $total += 10;
+        } elsif ($digit eq '$') { $total += 11;
+        } elsif ($digit eq ':') { $total += 12;
+        } elsif ($digit eq '/') { $total += 13;
+        } elsif ($digit eq '.') { $total += 14;
+        } elsif ($digit eq '+') { $total += 15;
+        } elsif ($digit eq 'A') { $total += 16;
+        } elsif ($digit eq 'B') { $total += 17;
+        } elsif ($digit eq 'C') { $total += 18;
+        } elsif ($digit eq 'D') { $total += 19;
+        } else { die "invalid digit <$digit>";
+        }
+    }
+    my $remainder = $total % 16;
+    my $difference = 16 - $remainder;
+    my $checkdigit;
+    if ($difference < 10) { $checkdigit = $difference;
+    } elsif ($difference == 10) { $checkdigit = '-';
+    } elsif ($difference == 11) { $checkdigit = '$';
+    } elsif ($difference == 12) { $checkdigit = ':';
+    } elsif ($difference == 13) { $checkdigit = '/';
+    } elsif ($difference == 14) { $checkdigit = '.';
+    } elsif ($difference == 15) { $checkdigit = '+';
+    } elsif ($difference == 16) { $checkdigit = 'A';
+    } elsif ($difference == 17) { $checkdigit = 'B';
+    } elsif ($difference == 18) { $checkdigit = 'C';
+    } elsif ($difference == 19) { $checkdigit = 'D';
+    } else { die "error calculating checkdigit";
+    }
+
+    return $barcode . $checkdigit;
+$$ LANGUAGE PLPERLU STRICT STABLE;
+
 CREATE OR REPLACE FUNCTION migration_tools.attempt_phone (TEXT,TEXT) RETURNS TEXT AS $$
   DECLARE
     phone TEXT := $1;
@@ -2821,6 +2882,84 @@ CREATE OR REPLACE FUNCTION migration_tools.handle_circmod (TEXT,TEXT) RETURNS VO
     END;
 $$ LANGUAGE PLPGSQL STRICT VOLATILE;
 
+-- convenience functions for handling item status maps
+
+CREATE OR REPLACE FUNCTION migration_tools.handle_status (TEXT,TEXT) RETURNS VOID AS $$
+    DECLARE
+        table_schema ALIAS FOR $1;
+        table_name ALIAS FOR $2;
+        proceed BOOLEAN;
+    BEGIN
+        EXECUTE 'SELECT EXISTS (
+            SELECT 1
+            FROM information_schema.columns
+            WHERE table_schema = $1
+            AND table_name = $2
+            and column_name = ''desired_status''
+        )' INTO proceed USING table_schema, table_name;
+        IF NOT proceed THEN
+            RAISE EXCEPTION 'Missing column desired_status'; 
+        END IF;
+
+        EXECUTE 'ALTER TABLE '
+            || quote_ident(table_name)
+            || ' DROP COLUMN IF EXISTS x_status';
+        EXECUTE 'ALTER TABLE '
+            || quote_ident(table_name)
+            || ' ADD COLUMN x_status INTEGER';
+
+        EXECUTE 'UPDATE ' || quote_ident(table_name) || ' a'
+            || ' SET x_status = id FROM config.copy_status b'
+            || ' WHERE BTRIM(UPPER(a.desired_status)) = BTRIM(UPPER(b.name))';
+
+        EXECUTE 'SELECT migration_tools.assert(
+            NOT EXISTS (SELECT 1 FROM ' || quote_ident(table_name) || ' WHERE desired_status <> '''' AND x_status IS NULL),
+            ''Cannot find a desired copy status'',
+            ''Found all desired copy statuses''
+        );';
+
+    END;
+$$ LANGUAGE PLPGSQL STRICT VOLATILE;
+
+-- convenience functions for handling org maps
+
+CREATE OR REPLACE FUNCTION migration_tools.handle_org (TEXT,TEXT) RETURNS VOID AS $$
+    DECLARE
+        table_schema ALIAS FOR $1;
+        table_name ALIAS FOR $2;
+        proceed BOOLEAN;
+    BEGIN
+        EXECUTE 'SELECT EXISTS (
+            SELECT 1
+            FROM information_schema.columns
+            WHERE table_schema = $1
+            AND table_name = $2
+            and column_name = ''desired_org''
+        )' INTO proceed USING table_schema, table_name;
+        IF NOT proceed THEN
+            RAISE EXCEPTION 'Missing column desired_org'; 
+        END IF;
+
+        EXECUTE 'ALTER TABLE '
+            || quote_ident(table_name)
+            || ' DROP COLUMN IF EXISTS x_org';
+        EXECUTE 'ALTER TABLE '
+            || quote_ident(table_name)
+            || ' ADD COLUMN x_org INTEGER';
+
+        EXECUTE 'UPDATE ' || quote_ident(table_name) || ' a'
+            || ' SET x_org = id FROM actor.org_unit b'
+            || ' WHERE BTRIM(a.desired_org) = BTRIM(b.shortname)';
+
+        EXECUTE 'SELECT migration_tools.assert(
+            NOT EXISTS (SELECT 1 FROM ' || quote_ident(table_name) || ' WHERE desired_org <> '''' AND x_org IS NULL),
+            ''Cannot find a desired org unit'',
+            ''Found all desired org units''
+        );';
+
+    END;
+$$ LANGUAGE PLPGSQL STRICT VOLATILE;
+
 -- convenience function for handling desired_not_migrate
 
 CREATE OR REPLACE FUNCTION migration_tools.handle_not_migrate (TEXT,TEXT) RETURNS VOID AS $$
@@ -2850,7 +2989,9 @@ CREATE OR REPLACE FUNCTION migration_tools.handle_not_migrate (TEXT,TEXT) RETURN
         EXECUTE 'UPDATE ' || quote_ident(table_name) || ' a'
             || ' SET x_migrate = CASE'
             || ' WHEN BTRIM(desired_not_migrate) = ''TRUE'' THEN FALSE'
+            || ' WHEN BTRIM(desired_not_migrate) = ''DNM'' THEN FALSE'
             || ' WHEN BTRIM(desired_not_migrate) = ''FALSE'' THEN TRUE'
+            || ' WHEN BTRIM(desired_not_migrate) = ''Migrate'' THEN TRUE'
             || ' WHEN BTRIM(desired_not_migrate) = '''' THEN TRUE'
             || ' END';
 
@@ -3161,8 +3302,8 @@ END
 $$ LANGUAGE plpgsql;
 
 -- alternate adding subfield 9 function in that it adds them to existing tags where the 856$u matches a correct value only
-DROP FUNCTION IF EXISTS add_sf9(TEXT,TEXT,TEXT);
-CREATE OR REPLACE FUNCTION add_sf9(marc TEXT, partial_u TEXT, new_9 TEXT)
+DROP FUNCTION IF EXISTS migration_tools.add_sf9(TEXT,TEXT,TEXT);
+CREATE OR REPLACE FUNCTION migration_tools.add_sf9(marc TEXT, partial_u TEXT, new_9 TEXT)
  RETURNS TEXT
  LANGUAGE plperlu
 AS $function$
@@ -3210,8 +3351,8 @@ return $marc_xml->as_xml_record();
 
 $function$;
 
-DROP FUNCTION IF EXISTS add_sf9(INTEGER, TEXT, TEXT);
-CREATE OR REPLACE FUNCTION add_sf9(bib_id INTEGER, target_u_text TEXT, sf9_text TEXT)
+DROP FUNCTION IF EXISTS migration_tools.add_sf9(BIGINT, TEXT, TEXT, REGCLASS);
+CREATE OR REPLACE FUNCTION migration_tools.add_sf9(bib_id BIGINT, target_u_text TEXT, sf9_text TEXT, bib_table REGCLASS)
     RETURNS BOOLEAN AS
 $BODY$
 DECLARE
@@ -3220,14 +3361,15 @@ DECLARE
     r             BOOLEAN;
 BEGIN
 
-    SELECT marc FROM biblio.record_entry WHERE id = bib_id INTO source_xml;
+    EXECUTE 'SELECT marc FROM ' || bib_table || ' WHERE id = ' || bib_id INTO source_xml;
 
     SELECT add_sf9(source_xml, target_u_text, sf9_text) INTO new_xml;
 
     r = FALSE;
+	new_xml = '$_$' || new_xml || '$_$';
 
     IF new_xml != source_xml THEN
-        UPDATE biblio.record_entry SET marc = new_xml WHERE id = bib_id;
+        EXECUTE 'UPDATE ' || bib_table || ' SET marc = ' || new_xml || ' WHERE id = ' || bib_id;
         r = TRUE;
     END IF;
 
@@ -3278,12 +3420,12 @@ CREATE OR REPLACE FUNCTION migration_tools.handle_item_barcode (TEXT,TEXT,TEXT,T
 
         IF btrim_desired THEN
             EXECUTE 'UPDATE ' || quote_ident(table_name) || ' a'
-                || ' SET x_item = id FROM asset_copy_legacy b'
+                || ' SET x_item = b.id FROM asset_copy_legacy b'
                 || ' WHERE BTRIM(a.' || quote_ident(foreign_column_name)
                 || ') = BTRIM(b.' || quote_ident(main_column_name) || ')';
         ELSE
             EXECUTE 'UPDATE ' || quote_ident(table_name) || ' a'
-                || ' SET x_item = id FROM asset_copy_legacy b'
+                || ' SET x_item = b.id FROM asset_copy_legacy b'
                 || ' WHERE a.' || quote_ident(foreign_column_name)
                 || ' = b.' || quote_ident(main_column_name);
         END IF;
@@ -3339,12 +3481,12 @@ CREATE OR REPLACE FUNCTION migration_tools.handle_user_barcode (TEXT,TEXT,TEXT,T
 
         IF btrim_desired THEN
             EXECUTE 'UPDATE ' || quote_ident(table_name) || ' a'
-                || ' SET x_user = id FROM actor_usr_legacy b'
+                || ' SET x_user = b.id FROM actor_usr_legacy b'
                 || ' WHERE BTRIM(a.' || quote_ident(foreign_column_name)
                 || ') = BTRIM(b.' || quote_ident(main_column_name) || ')';
         ELSE
             EXECUTE 'UPDATE ' || quote_ident(table_name) || ' a'
-                || ' SET x_user = id FROM actor_usr_legacy b'
+                || ' SET x_user = b.id FROM actor_usr_legacy b'
                 || ' WHERE a.' || quote_ident(foreign_column_name)
                 || ' = b.' || quote_ident(main_column_name);
         END IF;
@@ -3472,3 +3614,254 @@ CREATE OR REPLACE FUNCTION migration_tools.handle_link2 (TEXT,TEXT,TEXT,TEXT,TEX
 
     END;
 $$ LANGUAGE PLPGSQL STRICT VOLATILE;
+
+-- convenience function for handling desired asset stat cats
+
+CREATE OR REPLACE FUNCTION migration_tools.vivicate_asset_sc_and_sce (TEXT,TEXT,TEXT,TEXT) RETURNS VOID AS $$
+    DECLARE
+        table_schema ALIAS FOR $1;
+        table_name ALIAS FOR $2;
+        field_suffix ALIAS FOR $3; -- for distinguishing between desired_sce1, desired_sce2, etc.
+        org_shortname ALIAS FOR $4;
+        proceed BOOLEAN;
+        org INTEGER;
+        org_list INTEGER[];
+        sc TEXT;
+        sce TEXT;
+    BEGIN
+
+        SELECT 'desired_sc' || field_suffix INTO sc;
+        SELECT 'desired_sce' || field_suffix INTO sce;
+
+        EXECUTE 'SELECT EXISTS (
+            SELECT 1
+            FROM information_schema.columns
+            WHERE table_schema = $1
+            AND table_name = $2
+            and column_name = $3
+        )' INTO proceed USING table_schema, table_name, sc;
+        IF NOT proceed THEN
+            RAISE EXCEPTION 'Missing column %', sc; 
+        END IF;
+        EXECUTE 'SELECT EXISTS (
+            SELECT 1
+            FROM information_schema.columns
+            WHERE table_schema = $1
+            AND table_name = $2
+            and column_name = $3
+        )' INTO proceed USING table_schema, table_name, sce;
+        IF NOT proceed THEN
+            RAISE EXCEPTION 'Missing column %', sce; 
+        END IF;
+
+        SELECT id INTO org FROM actor.org_unit WHERE shortname = org_shortname;
+        IF org IS NULL THEN
+            RAISE EXCEPTION 'Cannot find org by shortname';
+        END IF;
+        SELECT INTO org_list ARRAY_ACCUM(id) FROM actor.org_unit_full_path( org );
+
+        -- caller responsible for their own truncates though we try to prevent duplicates
+        EXECUTE 'INSERT INTO asset_stat_cat (owner, name)
+            SELECT DISTINCT
+                 $1
+                ,BTRIM('||sc||')
+            FROM 
+                ' || quote_ident(table_name) || '
+            WHERE
+                NULLIF(BTRIM('||sc||'),'''') IS NOT NULL
+                AND NOT EXISTS (
+                    SELECT id
+                    FROM asset.stat_cat
+                    WHERE owner = ANY ($2)
+                    AND name = BTRIM('||sc||')
+                )
+                AND NOT EXISTS (
+                    SELECT id
+                    FROM asset_stat_cat
+                    WHERE owner = ANY ($2)
+                    AND name = BTRIM('||sc||')
+                )
+            ORDER BY 2;'
+        USING org, org_list;
+
+        EXECUTE 'INSERT INTO asset_stat_cat_entry (stat_cat, owner, value)
+            SELECT DISTINCT
+                COALESCE(
+                    (SELECT id
+                        FROM asset.stat_cat
+                        WHERE owner = ANY ($2)
+                        AND BTRIM('||sc||') = BTRIM(name))
+                   ,(SELECT id
+                        FROM asset_stat_cat
+                        WHERE owner = ANY ($2)
+                        AND BTRIM('||sc||') = BTRIM(name))
+                )
+                ,$1
+                ,BTRIM('||sce||')
+            FROM 
+                ' || quote_ident(table_name) || '
+            WHERE
+                    NULLIF(BTRIM('||sc||'),'''') IS NOT NULL
+                AND NULLIF(BTRIM('||sce||'),'''') IS NOT NULL
+                AND NOT EXISTS (
+                    SELECT id
+                    FROM asset.stat_cat_entry
+                    WHERE stat_cat = (
+                        SELECT id
+                        FROM asset.stat_cat
+                        WHERE owner = ANY ($2)
+                        AND BTRIM('||sc||') = BTRIM(name)
+                    ) AND value = BTRIM('||sce||')
+                )
+                AND NOT EXISTS (
+                    SELECT id
+                    FROM asset_stat_cat_entry
+                    WHERE stat_cat = (
+                        SELECT id
+                        FROM asset_stat_cat
+                        WHERE owner = ANY ($2)
+                        AND BTRIM('||sc||') = BTRIM(name)
+                    ) AND value = BTRIM('||sce||')
+                )
+            ORDER BY 1,3;'
+        USING org, org_list;
+    END;
+$$ LANGUAGE PLPGSQL STRICT VOLATILE;
+
+CREATE OR REPLACE FUNCTION migration_tools.handle_asset_sc_and_sce (TEXT,TEXT,TEXT,TEXT) RETURNS VOID AS $$
+    DECLARE
+        table_schema ALIAS FOR $1;
+        table_name ALIAS FOR $2;
+        field_suffix ALIAS FOR $3; -- for distinguishing between desired_sce1, desired_sce2, etc.
+        org_shortname ALIAS FOR $4;
+        proceed BOOLEAN;
+        org INTEGER;
+        org_list INTEGER[];
+        o INTEGER;
+        sc TEXT;
+        sce TEXT;
+    BEGIN
+        SELECT 'desired_sc' || field_suffix INTO sc;
+        SELECT 'desired_sce' || field_suffix INTO sce;
+        EXECUTE 'SELECT EXISTS (
+            SELECT 1
+            FROM information_schema.columns
+            WHERE table_schema = $1
+            AND table_name = $2
+            and column_name = $3
+        )' INTO proceed USING table_schema, table_name, sc;
+        IF NOT proceed THEN
+            RAISE EXCEPTION 'Missing column %', sc; 
+        END IF;
+        EXECUTE 'SELECT EXISTS (
+            SELECT 1
+            FROM information_schema.columns
+            WHERE table_schema = $1
+            AND table_name = $2
+            and column_name = $3
+        )' INTO proceed USING table_schema, table_name, sce;
+        IF NOT proceed THEN
+            RAISE EXCEPTION 'Missing column %', sce; 
+        END IF;
+
+        SELECT id INTO org FROM actor.org_unit WHERE shortname = org_shortname;
+        IF org IS NULL THEN
+            RAISE EXCEPTION 'Cannot find org by shortname';
+        END IF;
+
+        SELECT INTO org_list ARRAY_ACCUM(id) FROM actor.org_unit_full_path( org );
+
+        EXECUTE 'ALTER TABLE '
+            || quote_ident(table_name)
+            || ' DROP COLUMN IF EXISTS x_sc' || field_suffix;
+        EXECUTE 'ALTER TABLE '
+            || quote_ident(table_name)
+            || ' ADD COLUMN x_sc' || field_suffix || ' INTEGER';
+        EXECUTE 'ALTER TABLE '
+            || quote_ident(table_name)
+            || ' DROP COLUMN IF EXISTS x_sce' || field_suffix;
+        EXECUTE 'ALTER TABLE '
+            || quote_ident(table_name)
+            || ' ADD COLUMN x_sce' || field_suffix || ' INTEGER';
+
+
+        EXECUTE 'UPDATE ' || quote_ident(table_name) || '
+            SET
+                x_sc' || field_suffix || ' = id
+            FROM
+                (SELECT id, name, owner FROM asset_stat_cat
+                    UNION SELECT id, name, owner FROM asset.stat_cat) u
+            WHERE
+                    BTRIM(UPPER(u.name)) = BTRIM(UPPER(' || sc || '))
+                AND u.owner = ANY ($1);'
+        USING org_list;
+
+        EXECUTE 'UPDATE ' || quote_ident(table_name) || '
+            SET
+                x_sce' || field_suffix || ' = id
+            FROM
+                (SELECT id, stat_cat, owner, value FROM asset_stat_cat_entry
+                    UNION SELECT id, stat_cat, owner, value FROM asset.stat_cat_entry) u
+            WHERE
+                    u.stat_cat = x_sc' || field_suffix || '
+                AND BTRIM(UPPER(u.value)) = BTRIM(UPPER(' || sce || '))
+                AND u.owner = ANY ($1);'
+        USING org_list;
+
+        EXECUTE 'SELECT migration_tools.assert(
+            NOT EXISTS (SELECT 1 FROM ' || quote_ident(table_name) || ' WHERE desired_sc' || field_suffix || ' <> '''' AND x_sc' || field_suffix || ' IS NULL),
+            ''Cannot find a desired stat cat'',
+            ''Found all desired stat cats''
+        );';
+
+        EXECUTE 'SELECT migration_tools.assert(
+            NOT EXISTS (SELECT 1 FROM ' || quote_ident(table_name) || ' WHERE desired_sce' || field_suffix || ' <> '''' AND x_sce' || field_suffix || ' IS NULL),
+            ''Cannot find a desired stat cat entry'',
+            ''Found all desired stat cat entries''
+        );';
+
+    END;
+$$ LANGUAGE PLPGSQL STRICT VOLATILE;
+
+DROP FUNCTION IF EXISTS migration_tools.btrim_lcolumns(TEXT,TEXT);
+CREATE OR REPLACE FUNCTION migration_tools.btrim_lcolumns(s_name TEXT, t_name TEXT) RETURNS BOOLEAN
+ LANGUAGE plpgsql
+AS $function$
+DECLARE
+    c_name     TEXT;
+BEGIN
+
+    FOR c_name IN SELECT column_name FROM information_schema.columns WHERE 
+            table_name = t_name
+            AND table_schema = s_name
+            AND (data_type='text' OR data_type='character varying')
+            AND column_name like 'l_%'
+    LOOP
+       EXECUTE FORMAT('UPDATE ' || s_name || '.' || t_name || ' SET ' || c_name || ' = BTRIM(' || c_name || ')'); 
+    END LOOP;  
+
+    RETURN TRUE;
+END
+$function$;
+
+DROP FUNCTION IF EXISTS migration_tools.null_empty_lcolumns(TEXT,TEXT);
+CREATE OR REPLACE FUNCTION migration_tools.null_empty_lcolumns(s_name TEXT, t_name TEXT) RETURNS BOOLEAN
+ LANGUAGE plpgsql
+AS $function$
+DECLARE
+    c_name     TEXT;
+BEGIN
+
+    FOR c_name IN SELECT column_name FROM information_schema.columns WHERE 
+            table_name = t_name
+            AND table_schema = s_name
+            AND (data_type='text' OR data_type='character varying')
+            AND column_name like 'l_%'
+    LOOP
+       EXECUTE FORMAT('UPDATE ' || s_name || '.' || t_name || ' SET ' || c_name || ' = NULL WHERE ' || c_name || ' = '''' '); 
+    END LOOP;  
+
+    RETURN TRUE;
+END
+$function$;
+