X-Git-Url: http://git.equinoxoli.org/?p=migration-tools.git;a=blobdiff_plain;f=sql%2Fbase%2Fbase.sql;h=de80f0f8bb849241f80a39e704d406ca45e37f37;hp=8ea837ad4bce19becb9470e2fbb0efdd36da4fc5;hb=e9506234be916a9a719b844c7005c9cfa8251edf;hpb=5357ddfd207cc281548fec79809104267dc5d7b0 diff --git a/sql/base/base.sql b/sql/base/base.sql index 8ea837a..de80f0f 100644 --- a/sql/base/base.sql +++ b/sql/base/base.sql @@ -23,7 +23,7 @@ -- SELECT migration_tools.build('foo'); -- SELECT * FROM foo.fields_requiring_mapping; -- \d foo.actor_usr --- create some incoming ILS specific staging tables, like CREATE foo.legacy_items ( l_barcode TEXT, .. ) INHERITS foo.asset_copy; +-- create some incoming ILS specific staging tables, like CREATE foo.legacy_items ( l_barcode TEXT, .. ) INHERITS (foo.asset_copy); -- Do some mapping, like UPDATE foo.legacy_items SET barcode = TRIM(BOTH ' ' FROM l_barcode); -- Then, to move into production, do: select migration_tools.insert_base_into_production('foo') @@ -117,7 +117,7 @@ CREATE OR REPLACE FUNCTION migration_tools.init (TEXT) RETURNS VOID AS $$ END; PERFORM migration_tools.exec( $1, 'DROP TABLE IF EXISTS ' || migration_schema || '.config;' ); PERFORM migration_tools.exec( $1, 'CREATE TABLE ' || migration_schema || '.config ( key TEXT UNIQUE, value TEXT);' ); - PERFORM migration_tools.exec( $1, 'INSERT INTO ' || migration_schema || '.config (key,value) VALUES ( ''production_tables'', ''asset.call_number,asset.call_number_prefix,asset.call_number_suffix,asset.copy_location,asset.copy,asset.stat_cat,asset.stat_cat_entry,asset.stat_cat_entry_copy_map,asset.copy_note,actor.usr,actor.card,actor.usr_address,actor.stat_cat,actor.stat_cat_entry,actor.stat_cat_entry_usr_map,actor.usr_note,actor.usr_standing_penalty,action.circulation,action.hold_request,action.hold_notification,money.grocery,money.billing,money.cash_payment,money.forgive_payment,acq.provider,acq.provider_address,acq.provider_note,acq.fund,acq.fund_allocation,acq.fund_tag,acq.funding_source,acq.funding_source_credit,acq.lineitem,acq.purchase_order,acq.po_item,acq.invoice,acq.invoice_item,acq.invoice_entry,acq.lineitem_detail,acq.fund_debit,acq.fund_transfer,acq.po_note,config.circ_matrix_matchpoint,config.circ_matrix_limit_set_map,config.hold_matrix_matchpoint'' );' ); + PERFORM migration_tools.exec( $1, 'INSERT INTO ' || migration_schema || '.config (key,value) VALUES ( ''production_tables'', ''asset.call_number,asset.call_number_prefix,asset.call_number_suffix,asset.copy_location,asset.copy,asset.stat_cat,asset.stat_cat_entry,asset.stat_cat_entry_copy_map,asset.copy_note,actor.usr,actor.card,actor.usr_address,actor.stat_cat,actor.stat_cat_entry,actor.stat_cat_entry_usr_map,actor.usr_note,actor.usr_standing_penalty,action.circulation,action.hold_request,action.hold_notification,action.hold_request_note,action.hold_transit_copy,action.transit_copy,money.grocery,money.billing,money.cash_payment,money.forgive_payment,acq.provider,acq.provider_address,acq.provider_note,acq.provider_contact,acq.provider_contact_address,acq.fund,acq.fund_allocation,acq.fund_tag,acq.fund_tag_map,acq.funding_source,acq.funding_source_credit,acq.lineitem,acq.purchase_order,acq.po_item,acq.invoice,acq.invoice_item,acq.invoice_entry,acq.lineitem_detail,acq.fund_debit,acq.fund_transfer,acq.po_note,config.circ_matrix_matchpoint,config.circ_matrix_limit_set_map,config.hold_matrix_matchpoint'' );' ); PERFORM migration_tools.exec( $1, 'INSERT INTO ' || migration_schema || '.config (key,value) VALUES ( ''country_code'', ''USA'' );' ); PERFORM migration_tools.exec( $1, 'DROP TABLE IF EXISTS ' || migration_schema || '.fields_requiring_mapping;' ); PERFORM migration_tools.exec( $1, 'CREATE TABLE ' || migration_schema || '.fields_requiring_mapping( table_schema TEXT, table_name TEXT, column_name TEXT, data_type TEXT);' ); @@ -244,7 +244,7 @@ CREATE OR REPLACE FUNCTION migration_tools.build_specific_base_staging_table (TE BEGIN base_staging_table = REPLACE( production_table, '.', '_' ); --RAISE INFO 'In migration_tools.build_specific_base_staging_table(%,%) -> %', migration_schema, production_table, base_staging_table; - PERFORM migration_tools.exec( $1, 'CREATE TABLE ' || migration_schema || '.' || base_staging_table || ' ( LIKE ' || production_table || ' INCLUDING DEFAULTS EXCLUDING CONSTRAINTS );' ); + PERFORM migration_tools.exec( $1, 'CREATE UNLOGGED TABLE ' || migration_schema || '.' || base_staging_table || ' ( LIKE ' || production_table || ' INCLUDING DEFAULTS EXCLUDING CONSTRAINTS );' ); PERFORM migration_tools.exec( $1, ' INSERT INTO ' || migration_schema || '.fields_requiring_mapping SELECT table_schema, table_name, column_name, data_type @@ -380,38 +380,42 @@ CREATE OR REPLACE FUNCTION migration_tools.name_parse_out_last_first_middle_and_ prefix := 'Dr.'; temp := REGEXP_REPLACE( temp, E'DR\.\\s*', '', 'i' ); END IF; - IF temp ilike '%JR%' THEN + IF temp ilike '%JR.%' THEN suffix := 'Jr.'; - temp := REGEXP_REPLACE( temp, E'JR\.?\\s*', '', 'i' ); + temp := REGEXP_REPLACE( temp, E'JR\.\\s*', '', 'i' ); END IF; IF temp ilike '%JR,%' THEN suffix := 'Jr.'; temp := REGEXP_REPLACE( temp, E'JR,\\s*', ',', 'i' ); END IF; - IF temp ilike '%SR%' THEN + IF temp ilike '%SR.%' THEN suffix := 'Sr.'; - temp := REGEXP_REPLACE( temp, E'SR\.?\\s*', '', 'i' ); + temp := REGEXP_REPLACE( temp, E'SR\.\\s*', '', 'i' ); END IF; IF temp ilike '%SR,%' THEN suffix := 'Sr.'; temp := REGEXP_REPLACE( temp, E'SR,\\s*', ',', 'i' ); END IF; - IF temp ilike '%III%' THEN + IF temp like '%III%' THEN suffix := 'III'; - temp := REGEXP_REPLACE( temp, E'III', '', 'i' ); + temp := REGEXP_REPLACE( temp, E'III', '' ); END IF; - IF temp ilike '%II%' THEN + IF temp like '%II%' THEN suffix := 'II'; - temp := REGEXP_REPLACE( temp, E'II', '', 'i' ); + temp := REGEXP_REPLACE( temp, E'II', '' ); END IF; - IF temp ilike '%IV%' THEN + IF temp like '%IV%' THEN suffix := 'IV'; - temp := REGEXP_REPLACE( temp, E'IV', '', 'i' ); + temp := REGEXP_REPLACE( temp, E'IV', '' ); END IF; + temp := REGEXP_REPLACE( temp, '\(\)', ''); family_name := BTRIM( REGEXP_REPLACE(temp,E'^(\\S+).*$',E'\\1') ); - first_given_name := BTRIM( REGEXP_REPLACE(temp,E'^\\S+\\s+(\\S+).*$',E'\\1') ); - second_given_name := BTRIM( REGEXP_REPLACE(temp,E'^\\S+\\s+\\S+\\s*(.*)$',E'\\1') ); + family_name := REGEXP_REPLACE( family_name, ',', '' ); + first_given_name := CASE WHEN temp ~ E'^\\S+$' THEN 'N/A' ELSE BTRIM( REGEXP_REPLACE(temp,E'^\\S+\\s+(\\S+).*$',E'\\1') ) END; + first_given_name := REGEXP_REPLACE( first_given_name, ',', '' ); + second_given_name := CASE WHEN temp ~ E'^\\S+$' THEN '' ELSE BTRIM( REGEXP_REPLACE(temp,E'^\\S+\\s+\\S+\\s*(.*)$',E'\\1') ) END; + second_given_name := REGEXP_REPLACE( second_given_name, ',', '' ); RETURN ARRAY[ family_name, prefix, first_given_name, second_given_name, suffix ]; END; @@ -448,6 +452,74 @@ CREATE OR REPLACE FUNCTION migration_tools.address_parse_out_citystatezip (TEXT) END; $$ LANGUAGE PLPGSQL STRICT VOLATILE; +-- try to parse data like this: 100 MAIN ST$COVINGTON, GA 30016 +CREATE OR REPLACE FUNCTION migration_tools.parse_out_address (TEXT) RETURNS TEXT[] AS $$ + DECLARE + fullstring TEXT := $1; + address1 TEXT := ''; + address2 TEXT := ''; + scratch1 TEXT := ''; + scratch2 TEXT := ''; + city TEXT := ''; + state TEXT := ''; + zip TEXT := ''; + BEGIN + zip := CASE + WHEN fullstring ~ E'\\d\\d\\d\\d\\d' + THEN REGEXP_REPLACE( fullstring, E'^.*(\\d\\d\\d\\d\\d-?\\d*).*$', E'\\1' ) + ELSE '' + END; + fullstring := REGEXP_REPLACE( fullstring, E'^(.*)\\d\\d\\d\\d\\d-?\\d*(.*)$', E'\\1\\2'); + + IF fullstring ~ ',' THEN + state := REGEXP_REPLACE( fullstring, E'^(.*),(.*)$', E'\\2'); + scratch1 := REGEXP_REPLACE( fullstring, E'^(.*),(.*)$', E'\\1'); + ELSE + IF fullstring ~ E'\\s+[A-Z][A-Z]\\s*$' THEN + state := REGEXP_REPLACE( fullstring, E'^.*,?\\s+([A-Z][A-Z])\\s*$', E'\\1' ); + scratch1 := REGEXP_REPLACE( fullstring, E'^(.*?),?\\s+[A-Z][A-Z](\\s*)$', E'\\1\\2' ); + ELSE + IF fullstring ~ E'^\\S+$' THEN + scratch1 := fullstring; + state := 'N/A'; + ELSE + state := REGEXP_REPLACE( fullstring, E'^(.*?),?\\s*(\\S+)\\s*$', E'\\2'); + scratch1 := REGEXP_REPLACE( fullstring, E'^(.*?),?\\s*(\\S+)\\s*$', E'\\1'); + END IF; + END IF; + END IF; + + IF scratch1 ~ '[\$]' THEN + scratch2 := REGEXP_REPLACE( scratch1, E'^(.+)[\$](.+?)$', E'\\1'); + city := REGEXP_REPLACE( scratch1, E'^(.+)[\$](.+?)$', E'\\2'); + ELSE + IF scratch1 ~ '\s' THEN + scratch2 := REGEXP_REPLACE( scratch1, E'^(.+)\\s+(.+?)$', E'\\1'); + city := REGEXP_REPLACE( scratch1, E'^(.+)\\s+(.+?)$', E'\\2'); + ELSE + scratch2 := 'N/A'; + city := scratch1; + END IF; + END IF; + + IF scratch2 ~ '^\d' THEN + address1 := scratch2; + address2 := ''; + ELSE + address1 := REGEXP_REPLACE( scratch2, E'^(.+?)(\\d.+)$', E'\\1'); + address2 := REGEXP_REPLACE( scratch2, E'^(.+?)(\\d.+)$', E'\\2'); + END IF; + + RETURN ARRAY[ + TRIM(BOTH ' ' FROM address1) + ,TRIM(BOTH ' ' FROM address2) + ,TRIM(BOTH ' ' FROM city) + ,TRIM(BOTH ' ' FROM state) + ,TRIM(BOTH ' ' FROM zip) + ]; + END; +$$ LANGUAGE PLPGSQL STRICT VOLATILE; + CREATE OR REPLACE FUNCTION migration_tools.rebarcode (o TEXT, t BIGINT) RETURNS TEXT AS $$ DECLARE n TEXT := o; @@ -747,6 +819,45 @@ CREATE OR REPLACE FUNCTION migration_tools.attempt_timestamptz (TEXT,TEXT) RETUR END; $$ LANGUAGE PLPGSQL STRICT STABLE; +CREATE OR REPLACE FUNCTION migration_tools.attempt_hz_date (TEXT,TEXT) RETURNS DATE AS $$ + DECLARE + attempt_value ALIAS FOR $1; + fail_value ALIAS FOR $2; + output DATE; + BEGIN + FOR output IN + EXECUTE E'SELECT (\'1970-01-01\'::date + \'' || attempt_value || E' days\'::interval)::date AS a;' + LOOP + RETURN output; + END LOOP; + EXCEPTION + WHEN OTHERS THEN + FOR output IN + EXECUTE 'SELECT ' || quote_literal(fail_value) || '::date AS a;' + LOOP + RETURN output; + END LOOP; + END; +$$ LANGUAGE PLPGSQL STRICT STABLE; + +CREATE OR REPLACE FUNCTION migration_tools.attempt_sierra_timestamp (TEXT,TEXT) RETURNS TIMESTAMP AS $$ + DECLARE + attempt_value ALIAS FOR $1; + fail_value ALIAS FOR $2; + output TIMESTAMP; + BEGIN + output := REGEXP_REPLACE(attempt_value,E'^(..)(..)(..)(..)(..)$',E'20\\1-\\2-\\3 \\4:\\5')::TIMESTAMP; + RETURN output; + EXCEPTION + WHEN OTHERS THEN + FOR output IN + EXECUTE 'SELECT ' || quote_literal(fail_value) || '::TIMESTAMP AS a;' + LOOP + RETURN output; + END LOOP; + END; +$$ LANGUAGE PLPGSQL STRICT STABLE; + CREATE OR REPLACE FUNCTION migration_tools.attempt_money (TEXT,TEXT) RETURNS NUMERIC(8,2) AS $$ DECLARE attempt_value ALIAS FOR $1; @@ -2424,3 +2535,38 @@ CREATE OR REPLACE FUNCTION migration_tools.find_circ_matrix_matchpoint (INTEGER) ) ); $$ LANGUAGE SQL; + +CREATE OR REPLACE FUNCTION migration_tools.assert (BOOLEAN) RETURNS VOID AS $$ + DECLARE + test ALIAS FOR $1; + BEGIN + IF NOT test THEN + RAISE EXCEPTION 'assertion'; + END IF; + END; +$$ LANGUAGE PLPGSQL STRICT VOLATILE; + +CREATE OR REPLACE FUNCTION migration_tools.assert (BOOLEAN,TEXT) RETURNS VOID AS $$ + DECLARE + test ALIAS FOR $1; + msg ALIAS FOR $2; + BEGIN + IF NOT test THEN + RAISE EXCEPTION '%', msg; + END IF; + END; +$$ LANGUAGE PLPGSQL STRICT VOLATILE; + +CREATE OR REPLACE FUNCTION migration_tools.assert (BOOLEAN,TEXT,TEXT) RETURNS TEXT AS $$ + DECLARE + test ALIAS FOR $1; + fail_msg ALIAS FOR $2; + success_msg ALIAS FOR $3; + BEGIN + IF NOT test THEN + RAISE EXCEPTION '%', fail_msg; + END IF; + RETURN success_msg; + END; +$$ LANGUAGE PLPGSQL STRICT VOLATILE; +