From 5bd5daad5a16cefe973f6fae88e59b3319794588 Mon Sep 17 00:00:00 2001 From: Ben Ostrowsky Date: Tue, 27 Oct 2009 17:39:05 +0000 Subject: [PATCH] Reforking unicorn importer; this is an unfinished version of a patron importer git-svn-id: svn://nox.esilibrary.com/migration-tools@624 eee7cc8d-164e-4af6-8e1b-092a69004917 --- unicorn/unicorn_patrons_to_tsv.pl | 154 +++++++++++++++++++++++++++++++++++++ 1 files changed, 154 insertions(+), 0 deletions(-) create mode 100755 unicorn/unicorn_patrons_to_tsv.pl diff --git a/unicorn/unicorn_patrons_to_tsv.pl b/unicorn/unicorn_patrons_to_tsv.pl new file mode 100755 index 0000000..fb36114 --- /dev/null +++ b/unicorn/unicorn_patrons_to_tsv.pl @@ -0,0 +1,154 @@ +#!/usr/bin/perl -w + +# Converts a Unicorn users.data, bill.data, or charge.data file to a tab-separated file. +# 2009-08-10 Ben Ostrowsky + +my @records; +my $serial = -1; +my $line = 0; +my $section = ''; +my $field = ''; +my %unique_fields; + + +# Load each record +while (<>) { + s/\r\n/\n/g; +# print STDERR "Loaded this line: " . $_; + + # Is this the start of a new record? + if ( /^... DOCUMENT BOUNDARY ...$/ ) { + $line = 0; + $serial++; + $section = ''; # just in case this didn't get reset in the previous record + # print STDERR "Processing record $serial.\n"; + next; + } + + # Is this a FORM= line (which can be ignored)? + if ( /^FORM=/ ) { + next; + } + + # If this isn't the start of the new record, it's a new line in the present record. + $line++; + + # Is this line the beginning of a block of data (typically an address or a note)? + if ( /^\.(.*?)_BEGIN.$/ ) { + # print STDERR "I think this might be the beginning of a beautiful " . $1 . ".\n"; + $section = "$1."; + next; + } + + # Is this line the beginning of a block of data (typically an address or a note)? + if ( /^\.(.*?)_END.$/ ) { + if ("$1." ne $section) { + print STDERR "Error in record $serial, line $line (input line $.): got an end-of-$1 but I thought I was in a $section block!\n"; + } + # print STDERR "It's been fun, guys, but... this is the end of the " . $1 . ".\n"; + $section = ''; + next; + } + + # Looks like we've got some actual data! Let's store it. + # FIXME: For large batches of data, we may run out of memory and should store this on disk. + if ( /^\.(.*?).\s+(\|a)?(.*)$/ ) { + + # Build the name of this field (taking note of whether we're in a named section of data) + $field = ''; + if ($section ne '') { + $field .= $section; + } + $field .= $1; + + # Store the field as a key of an array. If it already exists, oh well, now it still exists. + $unique_fields{$field} = 1; + + # Now we can actually store this line of data! + $records[$serial]{$field} = $3; + + # print STDERR "Data extracted: \$records[$serial]{'$field'} = '$3'\n"; + + next; + } + + # This is the continuation of the previous line. + else { + chomp($_); + $records[$serial]{$field} .= ' ' . $_; + # print STDERR "Appended data to previous field. \$records[$serial]{'$field'} is now '" . $records[$serial]{$field} . "'.\n"; + } + +} + +print STDERR "Loaded " . scalar(@records) . " records.\n"; + +# We're aiming to produce output that can be slurped in by this SQL code: \COPY m_anderson.actor_usr_unicorn ( l_user_id, l_user_altid, l_user_pin, l_user_profile, l_user_status, l_user_library, l_user_priv_granted, l_user_priv_expires, l_user_mailingaddr, +# l_birthdate, l_last_name, l_first_name, l_middle_name, l_suffix_name, l_note, l_note1, l_patron, l_comment, l_staff, l_webcatpref, l_user_category1, l_user_category2, l_user_category3, l_user_category4, l_dept, l_guardian, l_license, l_ssn, l_misc, l_aup, l_photo, +# l_notify_via, l_user_claims_ret, l_user_environment, l_user_department, l_ums_id, l_user_last_activity, l_placcard, l_user_email, l_addr1_std_line1, l_addr1_std_line2, l_addr1_std_city, l_addr1_std_state, l_addr1_std_zip, l_addr1_country, l_addr1_township, +# l_addr1_room, l_addr1_company, l_addr1_office, l_addr1_phone, l_addr1_dayphone, l_addr1_homephone, l_addr1_workphone, l_addr1_cellphone, l_addr1_fax, l_addr1_email, l_addr1_location, l_addr1_usefor, l_addr1_care_of, l_addr1_known_bad, l_addr1_ums_addrid, +# l_addr2_std_line1, l_addr2_std_line2, l_addr2_std_city, l_addr2_std_state, l_addr2_std_zip, l_addr2_country, l_addr2_township, l_addr2_room, l_addr2_company, l_addr2_office, l_addr2_phone, l_addr2_dayphone, l_addr2_homephone, l_addr2_workphone, l_addr2_cellphone, +# l_addr2_fax, l_addr2_email, l_addr2_location,l_addr2_usefor, l_addr2_care_of, l_addr2_known_bad, l_addr2_ums_addrid, l_addr3_std_line1, l_addr3_std_line2, l_addr3_std_city, l_addr3_std_state, l_addr3_std_zip, l_addr3_country, l_addr3_township, l_addr3_room, +# l_addr3_company, l_addr3_office, l_addr3_phone, l_addr3_dayphone, l_addr3_homephone, l_addr3_workphone, l_addr3_cellphone, l_addr3_fax, l_addr3_email, l_addr3_location, l_addr3_usefor, l_addr3_care_of, l_addr3_known_bad, l_addr3_ums_addrid, l_identific, l_noempl, +# l_profession, l_program, l_represent, l_userid_active, l_inactive_barcode1, l_inactive_barcode2 ) FROM './users.data.tsv' + +# Process the records: +for (my $u = 0; $u < @records; $u++) { + + # Some fields can be mapped straightforwardly: + foreach $f (qw( user_id user_alt_id user_pin user_profile user_status user_library user_priv_granted user_priv_expires user_mailingaddr user_claims_ret user_environment user_department user_last_activity )) { + $records[$u]{uc($f)} '' unless defined $records[$u]{uc($f)}; + $records[$u]{'l_' + $f} = $records[$u]{uc($f)}; + } + + # Addresses are a bit different: + foreach $a (qw( addr1 addr2 addr3 )) { + foreach $f (qw( std_line1 std_line2 std_city std_state std_zip country township room company office phone dayphone homephone workphone cellphone fax email location usefor care_of known_bad ums_addrid )) { + $records[$u]{uc('USER_' + $a + '.' + $f)} = '' unless defined $records[$u]{uc('USER_' + $a + '.' + $f)}; + $records[$u]{'l_' + $a + '_' + '$f'} = $records[$u]{uc('USER_' + $a + '.' + $f)}; + } + } + + # FIXME: handle fields that don't exactly match (e.g. parse USER_NAME into l_last_name etc.) + # l_birthdate, l_last_name, l_first_name, l_middle_name, l_suffix_name, l_note, l_note1, l_patron, l_comment, l_staff, l_webcatpref, l_user_category1, l_user_category2, l_user_category3, l_user_category4, l_dept, l_guardian, l_license, l_ssn, l_misc, l_aup, + # l_photo, l_notify_via, l_ums_id, l_placcard, l_user_email, l_identific, l_noempl, l_profession, l_program, l_represent, l_userid_active, l_inactive_barcode1, l_inactive_barcode2 + + # We can parse the name like so: + # Copy the name to a temp value + # Strip off a prefix, if there is one + # Strip off a suffix, if there is one + # Strip off the family name (before the comma) + # Of what remains, whatever is before the first space is the first name and the rest is the middle name + + +} + + + + +# Print the records +# ----------------- +# Print a header line +# FIXME: don't print a serial, and do print the fields in the order above +print "SERIAL\t"; +@sorted_fields = sort keys %unique_fields; +foreach $i (@sorted_fields) { + print "$i\t"; +} +print "\n"; + + +# Print the results +for (my $u = 0; $u < @records; $u++) { + print "$u\t"; + foreach $f (@sorted_fields) { + if (defined $records[$u]{$f}) { + print $records[$u]{$f}; + } + print "\t"; + } + print "\n"; +} + +print STDERR "Wrote " . scalar(@records) . " records.\n"; +# uh-bdee-uh-bdee-uh-bdee-uh- THAT'S ALL, FOLKS -- 1.7.2.5