yet another bib ingest knob

When enabled, the ingest.assume_inserts_only flag disables
checking several tables for old index data to delete; skipping this
makes a small but noticeable difference when doing an initial bib
ingest with more than one process in parallel.

Note that this flag should not be used when *re*ingesting records
unless you clear the old rows beforehand.

Signed-off-by: Galen Charlton <gmc@esilibrary.com>

git-svn-id: svn://svn.open-ils.org/ILS/trunk@17863 dcc99617-32d9-48b4-a31d-7c20da2025e4

diff --git a/Open-ILS/src/sql/Pg/002.schema.config.sql b/Open-ILS/src/sql/Pg/002.schema.config.sql
index 22c6362..bfb4fb7 100644 (file)
--- a/Open-ILS/src/sql/Pg/002.schema.config.sql
+++ b/Open-ILS/src/sql/Pg/002.schema.config.sql
@@ -58,6 +58,7 @@ INSERT INTO config.internal_flag (name) VALUES ('ingest.disable_located_uri');
 INSERT INTO config.internal_flag (name) VALUES ('ingest.disable_metabib_full_rec');
 INSERT INTO config.internal_flag (name) VALUES ('ingest.disable_metabib_rec_descriptor');
 INSERT INTO config.internal_flag (name) VALUES ('ingest.disable_metabib_field_entry');
+INSERT INTO config.internal_flag (name) VALUES ('ingest.assume_inserts_only');
 
 CREATE TABLE config.global_flag (
     label   TEXT    NOT NULL
@@ -69,7 +70,7 @@ CREATE TABLE config.upgrade_log (
     install_date    TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW()
 );
 
-INSERT INTO config.upgrade_log (version) VALUES ('0410'); -- gmc
+INSERT INTO config.upgrade_log (version) VALUES ('0411'); -- gmc
 
 CREATE TABLE config.bib_source (
        id              SERIAL  PRIMARY KEY,

diff --git a/Open-ILS/src/sql/Pg/030.schema.metabib.sql b/Open-ILS/src/sql/Pg/030.schema.metabib.sql
index 7be8ff3..8efc3da 100644 (file)
--- a/Open-ILS/src/sql/Pg/030.schema.metabib.sql
+++ b/Open-ILS/src/sql/Pg/030.schema.metabib.sql
@@ -707,7 +707,10 @@ $func$ LANGUAGE PLPGSQL;
 
 CREATE OR REPLACE FUNCTION metabib.reingest_metabib_rec_descriptor( bib_id BIGINT ) RETURNS VOID AS $func$
 BEGIN
-    DELETE FROM metabib.rec_descriptor WHERE record = bib_id;
+    PERFORM * FROM config.internal_flag WHERE name = 'ingest.assume_inserts_only' AND enabled;
+    IF NOT FOUND THEN
+        DELETE FROM metabib.rec_descriptor WHERE record = bib_id;
+    END IF;
     INSERT INTO metabib.rec_descriptor (record, item_type, item_form, bib_level, control_type, enc_level, audience, lit_form, type_mat, cat_form, pub_status, item_lang, vr_format, date1, date2)
         SELECT  bib_id,
                 biblio.marc21_extract_fixed_field( bib_id, 'Type' ),
@@ -735,7 +738,10 @@ $func$ LANGUAGE PLPGSQL;
 
 CREATE OR REPLACE FUNCTION metabib.reingest_metabib_full_rec( bib_id BIGINT ) RETURNS VOID AS $func$
 BEGIN
-    DELETE FROM metabib.real_full_rec WHERE record = bib_id;
+    PERFORM * FROM config.internal_flag WHERE name = 'ingest.assume_inserts_only' AND enabled;
+    IF NOT FOUND THEN
+        DELETE FROM metabib.real_full_rec WHERE record = bib_id;
+    END IF;
     INSERT INTO metabib.real_full_rec (record, tag, ind1, ind2, subfield, value)
         SELECT record, tag, ind1, ind2, subfield, value FROM biblio.flatten_marc( bib_id );
 
@@ -748,12 +754,14 @@ DECLARE
     fclass          RECORD;
     ind_data        metabib.field_entry_template%ROWTYPE;
 BEGIN
-    FOR fclass IN SELECT * FROM config.metabib_class LOOP
-        -- RAISE NOTICE 'Emptying out %', fclass.name;
-        EXECUTE $$DELETE FROM metabib.$$ || fclass.name || $$_field_entry WHERE source = $$ || bib_id;
-    END LOOP;
-
-    DELETE FROM metabib.facet_entry WHERE source = bib_id;
+    PERFORM * FROM config.internal_flag WHERE name = 'ingest.assume_inserts_only' AND enabled;
+    IF NOT FOUND THEN
+        FOR fclass IN SELECT * FROM config.metabib_class LOOP
+            -- RAISE NOTICE 'Emptying out %', fclass.name;
+            EXECUTE $$DELETE FROM metabib.$$ || fclass.name || $$_field_entry WHERE source = $$ || bib_id;
+        END LOOP;
+        DELETE FROM metabib.facet_entry WHERE source = bib_id;
+    END IF;
 
     FOR ind_data IN SELECT * FROM biblio.extract_metabib_field_entry( bib_id ) LOOP
         IF ind_data.field < 0 THEN

diff --git a/Open-ILS/src/sql/Pg/upgrade/0411.schema.assume_inserts_only_flag.sql b/Open-ILS/src/sql/Pg/upgrade/0411.schema.assume_inserts_only_flag.sql
new file mode 100644 (file)
index 0000000..03194fd
--- /dev/null
+++ b/Open-ILS/src/sql/Pg/upgrade/0411.schema.assume_inserts_only_flag.sql
@@ -0,0 +1,86 @@
+BEGIN;
+
+INSERT INTO config.upgrade_log (version) VALUES ('0411'); -- gmc
+
+INSERT INTO config.internal_flag (name) VALUES ('ingest.assume_inserts_only');
+
+CREATE OR REPLACE FUNCTION metabib.reingest_metabib_rec_descriptor( bib_id BIGINT ) RETURNS VOID AS $func$
+BEGIN
+    PERFORM * FROM config.internal_flag WHERE name = 'ingest.assume_inserts_only' AND enabled;
+    IF NOT FOUND THEN
+        DELETE FROM metabib.rec_descriptor WHERE record = bib_id;
+    END IF;
+    INSERT INTO metabib.rec_descriptor (record, item_type, item_form, bib_level, control_type, enc_level, audience, lit_form, type_mat, cat_form, pub_status, item_lang, vr_format, date1, date2)
+        SELECT  bib_id,
+                biblio.marc21_extract_fixed_field( bib_id, 'Type' ),
+                biblio.marc21_extract_fixed_field( bib_id, 'Form' ),
+                biblio.marc21_extract_fixed_field( bib_id, 'BLvl' ),
+                biblio.marc21_extract_fixed_field( bib_id, 'Ctrl' ),
+                biblio.marc21_extract_fixed_field( bib_id, 'ELvl' ),
+                biblio.marc21_extract_fixed_field( bib_id, 'Audn' ),
+                biblio.marc21_extract_fixed_field( bib_id, 'LitF' ),
+                biblio.marc21_extract_fixed_field( bib_id, 'TMat' ),
+                biblio.marc21_extract_fixed_field( bib_id, 'Desc' ),
+                biblio.marc21_extract_fixed_field( bib_id, 'DtSt' ),
+                biblio.marc21_extract_fixed_field( bib_id, 'Lang' ),
+                (   SELECT  v.value
+                      FROM  biblio.marc21_physical_characteristics( bib_id) p
+                            JOIN config.marc21_physical_characteristic_subfield_map s ON (s.id = p.subfield)
+                            JOIN config.marc21_physical_characteristic_value_map v ON (v.id = p.value)
+                      WHERE p.ptype = 'v' AND s.subfield = 'e'    ),
+                biblio.marc21_extract_fixed_field( bib_id, 'Date1'),
+                biblio.marc21_extract_fixed_field( bib_id, 'Date2');
+
+    RETURN;
+END;
+$func$ LANGUAGE PLPGSQL;
+
+CREATE OR REPLACE FUNCTION metabib.reingest_metabib_full_rec( bib_id BIGINT ) RETURNS VOID AS $func$
+BEGIN
+    PERFORM * FROM config.internal_flag WHERE name = 'ingest.assume_inserts_only' AND enabled;
+    IF NOT FOUND THEN
+        DELETE FROM metabib.real_full_rec WHERE record = bib_id;
+    END IF;
+    INSERT INTO metabib.real_full_rec (record, tag, ind1, ind2, subfield, value)
+        SELECT record, tag, ind1, ind2, subfield, value FROM biblio.flatten_marc( bib_id );
+
+    RETURN;
+END;
+$func$ LANGUAGE PLPGSQL;
+
+CREATE OR REPLACE FUNCTION metabib.reingest_metabib_field_entries( bib_id BIGINT ) RETURNS VOID AS $func$
+DECLARE
+    fclass          RECORD;
+    ind_data        metabib.field_entry_template%ROWTYPE;
+BEGIN
+    PERFORM * FROM config.internal_flag WHERE name = 'ingest.assume_inserts_only' AND enabled;
+    IF NOT FOUND THEN
+        FOR fclass IN SELECT * FROM config.metabib_class LOOP
+            -- RAISE NOTICE 'Emptying out %', fclass.name;
+            EXECUTE $$DELETE FROM metabib.$$ || fclass.name || $$_field_entry WHERE source = $$ || bib_id;
+        END LOOP;
+        DELETE FROM metabib.facet_entry WHERE source = bib_id;
+    END IF;
+
+    FOR ind_data IN SELECT * FROM biblio.extract_metabib_field_entry( bib_id ) LOOP
+        IF ind_data.field < 0 THEN
+            ind_data.field = -1 * ind_data.field;
+            INSERT INTO metabib.facet_entry (field, source, value)
+                VALUES (ind_data.field, ind_data.source, ind_data.value);
+        ELSE
+            EXECUTE $$
+                INSERT INTO metabib.$$ || ind_data.field_class || $$_field_entry (field, source, value)
+                    VALUES ($$ ||
+                        quote_literal(ind_data.field) || $$, $$ ||
+                        quote_literal(ind_data.source) || $$, $$ ||
+                        quote_literal(ind_data.value) ||
+                    $$);$$;
+        END IF;
+
+    END LOOP;
+
+    RETURN;
+END;
+$func$ LANGUAGE PLPGSQL;
+
+COMMIT;