-- SELECT migration_tools.build('foo');
-- SELECT * FROM foo.fields_requiring_mapping;
-- \d foo.actor_usr
--- create some incoming ILS specific staging tables, like CREATE foo.legacy_items ( l_barcode TEXT, .. ) INHERITS foo.asset_copy;
+-- create some incoming ILS specific staging tables, like CREATE foo.legacy_items ( l_barcode TEXT, .. ) INHERITS (foo.asset_copy);
-- Do some mapping, like UPDATE foo.legacy_items SET barcode = TRIM(BOTH ' ' FROM l_barcode);
-- Then, to move into production, do: select migration_tools.insert_base_into_production('foo')
END;
PERFORM migration_tools.exec( $1, 'DROP TABLE IF EXISTS ' || migration_schema || '.config;' );
PERFORM migration_tools.exec( $1, 'CREATE TABLE ' || migration_schema || '.config ( key TEXT UNIQUE, value TEXT);' );
- PERFORM migration_tools.exec( $1, 'INSERT INTO ' || migration_schema || '.config (key,value) VALUES ( ''production_tables'', ''asset.call_number,asset.call_number_prefix,asset.call_number_suffix,asset.copy_location,asset.copy,asset.stat_cat,asset.stat_cat_entry,asset.stat_cat_entry_copy_map,asset.copy_note,actor.usr,actor.card,actor.usr_address,actor.stat_cat,actor.stat_cat_entry,actor.stat_cat_entry_usr_map,actor.usr_note,actor.usr_standing_penalty,action.circulation,action.hold_request,action.hold_notification,money.grocery,money.billing,money.cash_payment,money.forgive_payment,acq.provider,acq.provider_address,acq.provider_note,acq.fund,acq.fund_allocation,acq.fund_tag,acq.funding_source,acq.funding_source_credit,acq.lineitem,acq.purchase_order,acq.po_item,acq.invoice,acq.invoice_item,acq.invoice_entry,acq.lineitem_detail,acq.fund_debit,acq.fund_transfer,acq.po_note,config.circ_matrix_matchpoint,config.circ_matrix_limit_set_map,config.hold_matrix_matchpoint'' );' );
+ PERFORM migration_tools.exec( $1, 'INSERT INTO ' || migration_schema || '.config (key,value) VALUES ( ''production_tables'', ''asset.call_number,asset.call_number_prefix,asset.call_number_suffix,asset.copy_location,asset.copy,asset.stat_cat,asset.stat_cat_entry,asset.stat_cat_entry_copy_map,asset.copy_note,actor.usr,actor.card,actor.usr_address,actor.stat_cat,actor.stat_cat_entry,actor.stat_cat_entry_usr_map,actor.usr_note,actor.usr_standing_penalty,actor.usr_setting,action.circulation,action.hold_request,action.hold_notification,action.hold_request_note,action.hold_transit_copy,action.transit_copy,money.grocery,money.billing,money.cash_payment,money.forgive_payment,acq.provider,acq.provider_address,acq.provider_note,acq.provider_contact,acq.provider_contact_address,acq.fund,acq.fund_allocation,acq.fund_tag,acq.fund_tag_map,acq.funding_source,acq.funding_source_credit,acq.lineitem,acq.purchase_order,acq.po_item,acq.invoice,acq.invoice_item,acq.invoice_entry,acq.lineitem_detail,acq.fund_debit,acq.fund_transfer,acq.po_note,config.circ_matrix_matchpoint,config.circ_matrix_limit_set_map,config.hold_matrix_matchpoint,asset.copy_tag,asset.copy_tag_copy_map,config.copy_tag_type,serial.item,serial.item_note,serial.record_entry'' );' );
PERFORM migration_tools.exec( $1, 'INSERT INTO ' || migration_schema || '.config (key,value) VALUES ( ''country_code'', ''USA'' );' );
PERFORM migration_tools.exec( $1, 'DROP TABLE IF EXISTS ' || migration_schema || '.fields_requiring_mapping;' );
PERFORM migration_tools.exec( $1, 'CREATE TABLE ' || migration_schema || '.fields_requiring_mapping( table_schema TEXT, table_name TEXT, column_name TEXT, data_type TEXT);' );
BEGIN
base_staging_table = REPLACE( production_table, '.', '_' );
--RAISE INFO 'In migration_tools.build_specific_base_staging_table(%,%) -> %', migration_schema, production_table, base_staging_table;
- PERFORM migration_tools.exec( $1, 'CREATE TABLE ' || migration_schema || '.' || base_staging_table || ' ( LIKE ' || production_table || ' INCLUDING DEFAULTS EXCLUDING CONSTRAINTS );' );
+ PERFORM migration_tools.exec( $1, 'CREATE UNLOGGED TABLE ' || migration_schema || '.' || base_staging_table || ' ( LIKE ' || production_table || ' INCLUDING DEFAULTS EXCLUDING CONSTRAINTS );' );
PERFORM migration_tools.exec( $1, '
INSERT INTO ' || migration_schema || '.fields_requiring_mapping
SELECT table_schema, table_name, column_name, data_type
prefix := 'Dr.';
temp := REGEXP_REPLACE( temp, E'DR\.\\s*', '', 'i' );
END IF;
- IF temp ilike '%JR%' THEN
+ IF temp ilike '%JR.%' THEN
suffix := 'Jr.';
- temp := REGEXP_REPLACE( temp, E'JR\.?\\s*', '', 'i' );
+ temp := REGEXP_REPLACE( temp, E'JR\.\\s*', '', 'i' );
END IF;
IF temp ilike '%JR,%' THEN
suffix := 'Jr.';
temp := REGEXP_REPLACE( temp, E'JR,\\s*', ',', 'i' );
END IF;
- IF temp ilike '%SR%' THEN
+ IF temp ilike '%SR.%' THEN
suffix := 'Sr.';
- temp := REGEXP_REPLACE( temp, E'SR\.?\\s*', '', 'i' );
+ temp := REGEXP_REPLACE( temp, E'SR\.\\s*', '', 'i' );
END IF;
IF temp ilike '%SR,%' THEN
suffix := 'Sr.';
temp := REGEXP_REPLACE( temp, E'SR,\\s*', ',', 'i' );
END IF;
- IF temp ilike '%III%' THEN
+ IF temp like '%III%' THEN
suffix := 'III';
- temp := REGEXP_REPLACE( temp, E'III', '', 'i' );
+ temp := REGEXP_REPLACE( temp, E'III', '' );
END IF;
- IF temp ilike '%II%' THEN
+ IF temp like '%II%' THEN
suffix := 'II';
- temp := REGEXP_REPLACE( temp, E'II', '', 'i' );
+ temp := REGEXP_REPLACE( temp, E'II', '' );
END IF;
- IF temp ilike '%IV%' THEN
+ IF temp like '%IV%' THEN
suffix := 'IV';
- temp := REGEXP_REPLACE( temp, E'IV', '', 'i' );
+ temp := REGEXP_REPLACE( temp, E'IV', '' );
END IF;
+ temp := REGEXP_REPLACE( temp, '\(\)', '');
family_name := BTRIM( REGEXP_REPLACE(temp,E'^(\\S+).*$',E'\\1') );
+ family_name := REGEXP_REPLACE( family_name, ',', '' );
first_given_name := CASE WHEN temp ~ E'^\\S+$' THEN 'N/A' ELSE BTRIM( REGEXP_REPLACE(temp,E'^\\S+\\s+(\\S+).*$',E'\\1') ) END;
+ first_given_name := REGEXP_REPLACE( first_given_name, ',', '' );
second_given_name := CASE WHEN temp ~ E'^\\S+$' THEN '' ELSE BTRIM( REGEXP_REPLACE(temp,E'^\\S+\\s+\\S+\\s*(.*)$',E'\\1') ) END;
+ second_given_name := REGEXP_REPLACE( second_given_name, ',', '' );
RETURN ARRAY[ family_name, prefix, first_given_name, second_given_name, suffix ];
END;
$$ LANGUAGE PLPGSQL STRICT IMMUTABLE;
+CREATE OR REPLACE FUNCTION migration_tools.name_parse_out_fuller_last_first_middle_and_random_affix (TEXT) RETURNS TEXT[] AS $$
+ DECLARE
+ full_name TEXT := $1;
+ temp TEXT;
+ family_name TEXT := '';
+ first_given_name TEXT := '';
+ second_given_name TEXT := '';
+ suffix TEXT := '';
+ prefix TEXT := '';
+ BEGIN
+ temp := BTRIM(full_name);
+ -- Use values, not structure, for prefix/suffix, unless we come up with a better idea
+ --IF temp ~ '^\S{2,}\.' THEN
+ -- prefix := REGEXP_REPLACE(temp, '^(\S{2,}\.).*$','\1');
+ -- temp := BTRIM(REGEXP_REPLACE(temp, '^\S{2,}\.(.*)$','\1'));
+ --END IF;
+ --IF temp ~ '\S{2,}\.$' THEN
+ -- suffix := REGEXP_REPLACE(temp, '^.*(\S{2,}\.)$','\1');
+ -- temp := REGEXP_REPLACE(temp, '^(.*)\S{2,}\.$','\1');
+ --END IF;
+ IF temp ilike '%MR.%' THEN
+ prefix := 'Mr.';
+ temp := BTRIM(REGEXP_REPLACE( temp, E'MR\.\\s*', '', 'i' ));
+ END IF;
+ IF temp ilike '%MRS.%' THEN
+ prefix := 'Mrs.';
+ temp := BTRIM(REGEXP_REPLACE( temp, E'MRS\.\\s*', '', 'i' ));
+ END IF;
+ IF temp ilike '%MS.%' THEN
+ prefix := 'Ms.';
+ temp := BTRIM(REGEXP_REPLACE( temp, E'MS\.\\s*', '', 'i' ));
+ END IF;
+ IF temp ilike '%DR.%' THEN
+ prefix := 'Dr.';
+ temp := BTRIM(REGEXP_REPLACE( temp, E'DR\.\\s*', '', 'i' ));
+ END IF;
+ IF temp ilike '%JR.%' THEN
+ suffix := 'Jr.';
+ temp := BTRIM(REGEXP_REPLACE( temp, E'JR\.\\s*', '', 'i' ));
+ END IF;
+ IF temp ilike '%JR,%' THEN
+ suffix := 'Jr.';
+ temp := BTRIM(REGEXP_REPLACE( temp, E'JR,\\s*', ',', 'i' ));
+ END IF;
+ IF temp ilike '%SR.%' THEN
+ suffix := 'Sr.';
+ temp := BTRIM(REGEXP_REPLACE( temp, E'SR\.\\s*', '', 'i' ));
+ END IF;
+ IF temp ilike '%SR,%' THEN
+ suffix := 'Sr.';
+ temp := BTRIM(REGEXP_REPLACE( temp, E'SR,\\s*', ',', 'i' ));
+ END IF;
+ IF temp like '%III%' THEN
+ suffix := 'III';
+ temp := BTRIM(REGEXP_REPLACE( temp, E'III', '' ));
+ END IF;
+ IF temp like '%II%' THEN
+ suffix := 'II';
+ temp := BTRIM(REGEXP_REPLACE( temp, E'II', '' ));
+ END IF;
+
+ IF temp ~ ',' THEN
+ family_name = BTRIM(REGEXP_REPLACE(temp,'^(.*?,).*$','\1'));
+ temp := BTRIM(REPLACE( temp, family_name, '' ));
+ family_name := REPLACE( family_name, ',', '' );
+ IF temp ~ ' ' THEN
+ first_given_name := BTRIM( REGEXP_REPLACE(temp,'^(.+)\s(.+)$','\1') );
+ second_given_name := BTRIM( REGEXP_REPLACE(temp,'^(.+)\s(.+)$','\2') );
+ ELSE
+ first_given_name := temp;
+ second_given_name := '';
+ END IF;
+ ELSE
+ IF temp ~ '^\S+\s+\S+\s+\S+$' THEN
+ first_given_name := BTRIM( REGEXP_REPLACE(temp,'^(\S+)\s*(\S+)\s*(\S+)$','\1') );
+ second_given_name := BTRIM( REGEXP_REPLACE(temp,'^(\S+)\s*(\S+)\s*(\S+)$','\2') );
+ family_name := BTRIM( REGEXP_REPLACE(temp,'^(\S+)\s*(\S+)\s*(\S+)$','\3') );
+ ELSE
+ first_given_name := BTRIM( REGEXP_REPLACE(temp,'^(\S+)\s*(\S+)$','\1') );
+ second_given_name := temp;
+ family_name := BTRIM( REGEXP_REPLACE(temp,'^(\S+)\s*(\S+)$','\2') );
+ END IF;
+ END IF;
+
+ RETURN ARRAY[ family_name, prefix, first_given_name, second_given_name, suffix ];
+ END;
+$$ LANGUAGE PLPGSQL STRICT IMMUTABLE;
CREATE OR REPLACE FUNCTION migration_tools.address_parse_out_citystatezip (TEXT) RETURNS TEXT[] AS $$
DECLARE
END;
$$ LANGUAGE PLPGSQL STRICT VOLATILE;
+-- try to parse data like this: 100 MAIN ST$COVINGTON, GA 30016
+CREATE OR REPLACE FUNCTION migration_tools.parse_out_address (TEXT) RETURNS TEXT[] AS $$
+ DECLARE
+ fullstring TEXT := $1;
+ address1 TEXT := '';
+ address2 TEXT := '';
+ scratch1 TEXT := '';
+ scratch2 TEXT := '';
+ city TEXT := '';
+ state TEXT := '';
+ zip TEXT := '';
+ BEGIN
+ zip := CASE
+ WHEN fullstring ~ E'\\d\\d\\d\\d\\d'
+ THEN REGEXP_REPLACE( fullstring, E'^.*(\\d\\d\\d\\d\\d-?\\d*).*$', E'\\1' )
+ ELSE ''
+ END;
+ fullstring := REGEXP_REPLACE( fullstring, E'^(.*)\\d\\d\\d\\d\\d-?\\d*(.*)$', E'\\1\\2');
+
+ IF fullstring ~ ',' THEN
+ state := REGEXP_REPLACE( fullstring, E'^(.*),(.*)$', E'\\2');
+ scratch1 := REGEXP_REPLACE( fullstring, E'^(.*),(.*)$', E'\\1');
+ ELSE
+ IF fullstring ~ E'\\s+[A-Z][A-Z]\\s*$' THEN
+ state := REGEXP_REPLACE( fullstring, E'^.*,?\\s+([A-Z][A-Z])\\s*$', E'\\1' );
+ scratch1 := REGEXP_REPLACE( fullstring, E'^(.*?),?\\s+[A-Z][A-Z](\\s*)$', E'\\1\\2' );
+ ELSE
+ IF fullstring ~ E'^\\S+$' THEN
+ scratch1 := fullstring;
+ state := 'N/A';
+ ELSE
+ state := REGEXP_REPLACE( fullstring, E'^(.*?),?\\s*(\\S+)\\s*$', E'\\2');
+ scratch1 := REGEXP_REPLACE( fullstring, E'^(.*?),?\\s*(\\S+)\\s*$', E'\\1');
+ END IF;
+ END IF;
+ END IF;
+
+ IF scratch1 ~ '[\$]' THEN
+ scratch2 := REGEXP_REPLACE( scratch1, E'^(.+)[\$](.+?)$', E'\\1');
+ city := REGEXP_REPLACE( scratch1, E'^(.+)[\$](.+?)$', E'\\2');
+ ELSE
+ IF scratch1 ~ '\s' THEN
+ scratch2 := REGEXP_REPLACE( scratch1, E'^(.+)\\s+(.+?)$', E'\\1');
+ city := REGEXP_REPLACE( scratch1, E'^(.+)\\s+(.+?)$', E'\\2');
+ ELSE
+ scratch2 := 'N/A';
+ city := scratch1;
+ END IF;
+ END IF;
+
+ IF scratch2 ~ '^\d' THEN
+ address1 := scratch2;
+ address2 := '';
+ ELSE
+ address1 := REGEXP_REPLACE( scratch2, E'^(.+?)(\\d.+)$', E'\\1');
+ address2 := REGEXP_REPLACE( scratch2, E'^(.+?)(\\d.+)$', E'\\2');
+ END IF;
+
+ RETURN ARRAY[
+ TRIM(BOTH ' ' FROM address1)
+ ,TRIM(BOTH ' ' FROM address2)
+ ,TRIM(BOTH ' ' FROM city)
+ ,TRIM(BOTH ' ' FROM state)
+ ,TRIM(BOTH ' ' FROM zip)
+ ];
+ END;
+$$ LANGUAGE PLPGSQL STRICT VOLATILE;
+
CREATE OR REPLACE FUNCTION migration_tools.rebarcode (o TEXT, t BIGINT) RETURNS TEXT AS $$
DECLARE
n TEXT := o;
END;
$$ LANGUAGE PLPGSQL STRICT STABLE;
+CREATE OR REPLACE FUNCTION migration_tools.attempt_sierra_timestamp (TEXT,TEXT) RETURNS TIMESTAMP AS $$
+ DECLARE
+ attempt_value ALIAS FOR $1;
+ fail_value ALIAS FOR $2;
+ output TIMESTAMP;
+ BEGIN
+ output := REGEXP_REPLACE(attempt_value,E'^(..)(..)(..)(..)(..)$',E'20\\1-\\2-\\3 \\4:\\5')::TIMESTAMP;
+ RETURN output;
+ EXCEPTION
+ WHEN OTHERS THEN
+ FOR output IN
+ EXECUTE 'SELECT ' || quote_literal(fail_value) || '::TIMESTAMP AS a;'
+ LOOP
+ RETURN output;
+ END LOOP;
+ END;
+$$ LANGUAGE PLPGSQL STRICT STABLE;
+
CREATE OR REPLACE FUNCTION migration_tools.attempt_money (TEXT,TEXT) RETURNS NUMERIC(8,2) AS $$
DECLARE
attempt_value ALIAS FOR $1;
)
);
$$ LANGUAGE SQL;
+
+CREATE OR REPLACE FUNCTION migration_tools.assert (BOOLEAN) RETURNS VOID AS $$
+ DECLARE
+ test ALIAS FOR $1;
+ BEGIN
+ IF NOT test THEN
+ RAISE EXCEPTION 'assertion';
+ END IF;
+ END;
+$$ LANGUAGE PLPGSQL STRICT VOLATILE;
+
+CREATE OR REPLACE FUNCTION migration_tools.assert (BOOLEAN,TEXT) RETURNS VOID AS $$
+ DECLARE
+ test ALIAS FOR $1;
+ msg ALIAS FOR $2;
+ BEGIN
+ IF NOT test THEN
+ RAISE EXCEPTION '%', msg;
+ END IF;
+ END;
+$$ LANGUAGE PLPGSQL STRICT VOLATILE;
+
+CREATE OR REPLACE FUNCTION migration_tools.assert (BOOLEAN,TEXT,TEXT) RETURNS TEXT AS $$
+ DECLARE
+ test ALIAS FOR $1;
+ fail_msg ALIAS FOR $2;
+ success_msg ALIAS FOR $3;
+ BEGIN
+ IF NOT test THEN
+ RAISE EXCEPTION '%', fail_msg;
+ END IF;
+ RETURN success_msg;
+ END;
+$$ LANGUAGE PLPGSQL STRICT VOLATILE;
+
+-- push bib sequence and return starting value for reserved range
+CREATE OR REPLACE FUNCTION migration_tools.push_bib_sequence(INTEGER) RETURNS BIGINT AS $$
+ DECLARE
+ bib_count ALIAS FOR $1;
+ output BIGINT;
+ BEGIN
+ PERFORM setval('biblio.record_entry_id_seq',(SELECT MAX(id) FROM biblio.record_entry) + bib_count + 2000);
+ FOR output IN
+ SELECT CEIL(MAX(id)/1000)*1000+1000 FROM biblio.record_entry WHERE id < (SELECT last_value FROM biblio.record_entry_id_seq)
+ LOOP
+ RETURN output;
+ END LOOP;
+ END;
+$$ LANGUAGE PLPGSQL STRICT VOLATILE;
+
+-- set a new salted password
+
+CREATE OR REPLACE FUNCTION migration_tools.set_salted_passwd(INTEGER,TEXT) RETURNS BOOLEAN AS $$
+ DECLARE
+ usr_id ALIAS FOR $1;
+ plain_passwd ALIAS FOR $2;
+ plain_salt TEXT;
+ md5_passwd TEXT;
+ BEGIN
+
+ SELECT actor.create_salt('main') INTO plain_salt;
+
+ SELECT MD5(plain_passwd) INTO md5_passwd;
+
+ PERFORM actor.set_passwd(usr_id, 'main', MD5(plain_salt || md5_passwd), plain_salt);
+
+ RETURN TRUE;
+
+ END;
+$$ LANGUAGE PLPGSQL STRICT VOLATILE;
+
+
+
+
projects / migration-tools.git / blobdiff