From 9333c7283a057acdfc93ec27b835ace56fdac6b0 Mon Sep 17 00:00:00 2001
From: Ben Ostrowsky <ben@esilibrary.com>
Date: Mon, 26 Oct 2009 18:10:32 +0000
Subject: [PATCH] unicorn_to_tsv.pl has been generalized to use non-user-specific wording

git-svn-id: svn://nox.esilibrary.com/migration-tools@622 eee7cc8d-164e-4af6-8e1b-092a69004917
Signed-off-by: Galen Charlton <gmc@esilibrary.com>
---
 unicorn/unicorn_to_tsv.pl |  109 +++++++++++++++++++++++++++++++++++++++++++++
 1 files changed, 109 insertions(+), 0 deletions(-)
 create mode 100755 unicorn/unicorn_to_tsv.pl

diff --git a/unicorn/unicorn_to_tsv.pl b/unicorn/unicorn_to_tsv.pl
new file mode 100755
index 0000000..e864a31
--- /dev/null
+++ b/unicorn/unicorn_to_tsv.pl
@@ -0,0 +1,109 @@
+#!/usr/bin/perl -w
+
+# Converts a Unicorn users.data, bill.data, or charge.data file to a tab-separated file.
+# 2009-08-10 Ben Ostrowsky <ben@esilibrary.com>
+
+my @records;
+my $serial = -1;
+my $line = 0;
+my $section = '';
+my $field = '';
+my %unique_fields;
+
+
+# Load each record
+while (<>) {
+    s/\r\n/\n/g;
+# print STDERR "Loaded this line: " . $_;
+
+	# Is this the start of a new record?
+	if ( /^... DOCUMENT BOUNDARY ...$/ ) {
+		$line = 0;
+		$serial++;
+		$section = ''; # just in case this didn't get reset in the previous record
+		# print STDERR "Processing record $serial.\n";
+		next;
+	}
+
+	# Is this a FORM= line (which can be ignored)?
+	if ( /^FORM=/ ) {
+		next;
+	}
+
+	# If this isn't the start of the new record, it's a new line in the present record.
+	$line++;
+
+	# Is this line the beginning of a block of data (typically an address or a note)?
+	if ( /^\.(.*?)_BEGIN.$/ ) {
+		# print STDERR "I think this might be the beginning of a beautiful " . $1 . ".\n";
+		$section = "$1.";
+		next;
+	}
+
+	# Is this line the beginning of a block of data (typically an address or a note)?
+	if ( /^\.(.*?)_END.$/ ) {
+		if ("$1." ne $section) {
+			print STDERR "Error in record $serial, line $line (input line $.): got an end-of-$1 but I thought I was in a $section block!\n";
+		}
+		# print STDERR "It's been fun, guys, but... this is the end of the " . $1 . ".\n";
+		$section = '';
+		next;
+	}
+
+	# Looks like we've got some actual data!  Let's store it.
+	# FIXME: For large batches of data, we may run out of memory and should store this on disk.
+	if ( /^\.(.*?).\s+(\|a)?(.*)$/ ) {
+
+		# Build the name of this field (taking note of whether we're in a named section of data)
+		$field = '';
+		if ($section ne '') { 
+			$field .= $section;
+		}
+		$field .= $1;
+
+		# Store the field as a key of an array.  If it already exists, oh well, now it still exists.
+		$unique_fields{$field} = 1;
+
+		# Now we can actually store this line of data!
+		$records[$serial]{$field} = $3;		
+
+		# print STDERR "Data extracted: \$records[$serial]{'$field'} = '$3'\n";
+
+		next;
+	}	
+
+	# This is the continuation of the previous line.
+	else {
+		chomp($_);
+		$records[$serial]{$field} .= ' ' . $_;
+		# print STDERR "Appended data to previous field. \$records[$serial]{'$field'} is now '" . $records[$serial]{$field} . "'.\n";
+	}
+
+}
+
+print STDERR "Loaded " . scalar(@records) . " records.\n";
+
+
+# Print a header line
+print "SERIAL\t";
+@sorted_fields = sort keys %unique_fields;
+foreach $i (@sorted_fields) {
+	print "$i\t";
+}
+print "\n";
+
+
+# Print the results
+for (my $u = 0; $u < @records; $u++) {
+	print "$u\t";	
+	foreach $f (@sorted_fields) {
+		if (defined $records[$u]{$f}) {
+			print $records[$u]{$f};
+		}
+	print "\t";
+	}
+	print "\n";
+}
+
+print STDERR "Wrote " . scalar(@records) . " records.\n";
+# uh-bdee-uh-bdee-uh-bdee-uh- THAT'S ALL, FOLKS
-- 
1.7.2.5