);
use Pod::Usage;
use FindBin;
-my $mig_bin = "$FindBin::Bin/mig-bin/";
-use lib "$FindBin::Bin/mig-bin";
-use Mig;
+my $mig_bin = "$FindBin::Bin/emig.d/bin/";
+use lib "$FindBin::Bin/emig.d/bin";
+use EMig;
pod2usage(-verbose => 2) if ! $ARGV[0];
switch($ARGV[0]) {
sub standard_invocation {
my $cmd = shift;
- if ($cmd ne 'env') { Mig::die_if_no_env_migschema(); }
+ if ($cmd ne 'env') { EMig::die_if_no_env_migschema(); }
if (-e $mig_bin . "mig-$cmd") {
system( $mig_bin . "mig-$cmd", @_ );
} else {
-package Mig;
+package EMig;
use strict;
use Exporter;
use FindBin;
my $mig_bin = "$FindBin::Bin/";
use lib "$FindBin::Bin/";
-use Mig;
+use EMig;
pod2usage(-verbose => 2) if ! $ARGV[0] || $ARGV[0] eq '--help';
-Mig::die_if_no_env_migschema();
-Mig::die_if_mig_tracking_table_does_not_exist();
+EMig::die_if_no_env_migschema();
+EMig::die_if_mig_tracking_table_does_not_exist();
my $has_headers = 1;
my $headers_file;
return;
}
}
- if (Mig::check_for_tracked_file($file)) {
+ if (EMig::check_for_tracked_file($file)) {
print "File already tracked: $file\n";
} else {
print 'Adding (';
print ($headers ? ' with headers' : 'without headers');
}
print '): ' . "$file\n";
- my $dbh = Mig::db_connect();
+ my $dbh = EMig::db_connect();
my $rv = $dbh->do("
INSERT INTO $MIGSCHEMA.tracked_file (
base_filename
," . $dbh->quote($headers_file) . "
);
") || die "Error inserting into table $MIGSCHEMA.tracked_file: $!\n";
- Mig::db_disconnect($dbh);
+ EMig::db_disconnect($dbh);
}
}
use FindBin;
my $mig_bin = "$FindBin::Bin/";
use lib "$FindBin::Bin/";
-use Mig;
+use EMig;
use open ':encoding(utf8)';
pod2usage(-verbose => 2) if defined $ARGV[0] && $ARGV[0] eq '--help';
use FindBin;
my $mig_bin = "$FindBin::Bin/";
use lib "$FindBin::Bin/";
-use Mig;
+use EMig;
pod2usage(-verbose => 2) if ! $ARGV[0] || $ARGV[0] eq '--help';
-Mig::die_if_no_env_migschema();
-Mig::die_if_mig_tracking_table_does_not_exist();
+EMig::die_if_no_env_migschema();
+EMig::die_if_mig_tracking_table_does_not_exist();
my $file = abs_path($ARGV[0]);
if ($file =~ /^$MIGBASEWORKDIR/) {
my $file = abs_path(shift);
my @args = @_;
- my $tracked_file_id = Mig::check_for_tracked_file($file);
+ my $tracked_file_id = EMig::check_for_tracked_file($file);
if ($tracked_file_id) {
- my $data = Mig::status_this_file($file);
+ my $data = EMig::status_this_file($file);
if (! $data->{'utf8_filename'}) {
die "mig-iconv or mig-skip-iconv needed for UTF8 version of file: $file\n";
print join(' ',@args) . "\n";
system('clean_csv', @args, $utf8_file);
- my $dbh = Mig::db_connect();
+ my $dbh = EMig::db_connect();
my $clean_file = $dbh->quote($utf8_file . '.clean');
if (! -e $utf8_file . '.clean') {
print "clean file does not exist: $clean_file\n";
WHERE base_filename = " . $dbh->quote($file) . "
;
") || die "Error inserting into table $MIGSCHEMA.tracked_file: $!\n";
- Mig::db_disconnect($dbh);
+ EMig::db_disconnect($dbh);
} else {
print "File not currently tracked: $file\n";
}
use FindBin;
my $mig_bin = "$FindBin::Bin/";
use lib "$FindBin::Bin/";
-use Mig;
+use EMig;
pod2usage(-verbose => 2) if ! $ARGV[0] || $ARGV[0] eq '--help';
-Mig::die_if_no_env_migschema();
-Mig::die_if_mig_tracking_table_does_not_exist();
+EMig::die_if_no_env_migschema();
+EMig::die_if_mig_tracking_table_does_not_exist();
my $file = abs_path($ARGV[0]);
if ($file =~ /^$MIGBASEWORKDIR/) {
my @args = @_;
my $stage_sql_filename;
- my $tracked_file_id = Mig::check_for_tracked_file($file);
+ my $tracked_file_id = EMig::check_for_tracked_file($file);
if ($tracked_file_id) {
- my $data = Mig::status_this_file($file);
+ my $data = EMig::status_this_file($file);
if (! $data->{'utf8_filename'}) {
die "mig-iconv or mig-skip-iconv needed for UTF8 version of file: $file\n";
print "args: " . join(',',@args) . "\n";
system('csv2sql', @args, $clean_file);
- my $dbh = Mig::db_connect();
+ my $dbh = EMig::db_connect();
if (! -e $stage_sql_filename) {
print "SQL converted file does not exist: $stage_sql_filename\n";
$stage_sql_filename = '';
WHERE base_filename = " . $dbh->quote($file) . "
;
") || die "Error updating table $MIGSCHEMA.tracked_file: $!\n";
- Mig::db_disconnect($dbh);
+ EMig::db_disconnect($dbh);
} else {
print "File not currently tracked: $file\n";
}
use FindBin;
my $mig_bin = "$FindBin::Bin/";
use lib "$FindBin::Bin/";
-use Mig;
+use EMig;
use open ':encoding(utf8)';
pod2usage(-verbose => 2) if defined $ARGV[0] && $ARGV[0] eq '--help';
=head1 DESCRIPTION
For most invocations, B<mig-env> will either create or use a migration-specific
-file (~/.mig/<migration_schema>.env) for setting the following environment
+file (~/.emig/<migration_schema>.env) for setting the following environment
variables:
=over 15
=item B<use> <schema>
This command will spawn a bash shell that executes the corresponding
-~/.mig/<schema>.env script for setting up environment variables encoded during
+~/.emig/<schema>.env script for setting up environment variables encoded during
B<create>.
=item B<show> [schema]
-This command will show the contents of the corresponding ~/.mig/<schema>.env
+This command will show the contents of the corresponding ~/.emig/<schema>.env
script, or, if no schema is specified, then it will list pertinent variables in
the current environment if they exist.
=item B<list>
-This command will list migration schemas found in ~/.mig
+This command will list migration schemas found in ~/.emig
=item B<help>
pod2usage(-verbose => 2) if ! $ARGV[0];
my $migration_schema = $ARGV[1] || '';
-my $filename = "$HOME/.mig/$migration_schema.env";
+my $filename = "$HOME/.emig/$migration_schema.env";
switch($ARGV[0]) {
case "--help" {
pod2usage(-verbose => 2);
case "clone" {
pod2usage(-verbose => 1) if ! $ARGV[2];
$migration_schema = $ARGV[2] || '';
- $filename = "$HOME/.mig/$migration_schema.env";
+ $filename = "$HOME/.emig/$migration_schema.env";
mig_env_clone();
}
case "use" {
}
}
case "list" {
- opendir(my $dh, "$HOME/.mig") || die "can't open $HOME/.mig: $!";
+ opendir(my $dh, "$HOME/.emig") || die "can't open $HOME/.emig: $!";
while (readdir $dh) {
if (/^(.*)\.env$/) {
print "$1\n";
# create files and directories if needed
- mkdir "$HOME/.mig";
+ mkdir "$HOME/.emig";
make_path($MIGGITDIR, { verbose => 1 });
`touch $MIGGITDIR/README`;
make_path($MIGWORKDIR, { verbose => 1 });
sub mig_env_clone {
my $orig_migration_schema = $ARGV[1] || '';
- my $orig_filename = "$HOME/.mig/$orig_migration_schema.env";
+ my $orig_filename = "$HOME/.emig/$orig_migration_schema.env";
`cp $orig_filename $filename`;
`sed -i 's/export PGOPTIONS=.*/export PGOPTIONS='"'"'-c search_path=$migration_schema,public,evergreen'"'"'/' $filename`;
`sed -i 's/export MIGENVPROMPT=.*/export MIGENVPROMPT=$migration_schema/' $filename`;
use FindBin;
use lib "$FindBin::Bin/";
my $mig_bin = "$FindBin::Bin/";
-use Mig;
+use EMig;
use strict;
use Switch;
use Cwd 'abs_path';
pod2usage(-verbose => 2) if defined $ARGV[0] && $ARGV[0] eq '--help';
-Mig::die_if_no_env_migschema();
+EMig::die_if_no_env_migschema();
die_if_gsheet_tracked_table_does_not_exist();
die_if_gsheet_tracked_column_does_not_exist();
my $sth;
my @ws;
my @tracked_ws_names;
-my $authfile = $ENV{HOME} . '/.mig/oauth.env';
+my $authfile = $ENV{HOME} . '/.emig/oauth.env';
my $next_arg_is_authfile;
foreach my $arg (@ARGV) {
if ($var1 eq 'CLIENTSECRET') { $clientsecret = $var2; }
if ($var1 eq 'SESSIONFILE') { $sessionfile = $var2; }
}
-my $dbh = Mig::db_connect();
+my $dbh = EMig::db_connect();
my $spreadsheet = connect_gsheet($clientid,$clientsecret,$sessionfile);
abort('could not connect to google sheet') unless (defined $spreadsheet);
my $table_name = shift;
my $schema_name = shift;
my @headers;
- my $dbh = Mig::db_connect();
+ my $dbh = EMig::db_connect();
$sql = 'SELECT column_name FROM information_schema.columns WHERE table_schema = ' . $dbh->quote( $schema_name ) . ' AND table_name = ' . $dbh->quote( $table_name ) . ';';
$sth = $dbh->prepare($sql);
$ra = $sth->execute();
}
sub check_for_gsheet_tracked_table {
- my $dbh = Mig::db_connect();
+ my $dbh = EMig::db_connect();
my $sth = $dbh->prepare("
SELECT EXISTS(
SELECT 1
|| die "Error checking for table (tracked_gsheet_table): $!";
my @cols = $sth->fetchrow_array;
$sth->finish;
- Mig::db_disconnect($dbh);
+ EMig::db_disconnect($dbh);
return $cols[0];
}
sub check_for_gsheet_tracked_column {
- my $dbh = Mig::db_connect();
+ my $dbh = EMig::db_connect();
my $sth = $dbh->prepare("
SELECT EXISTS(
SELECT 1
|| die "Error checking for table (gsheet_tracked_column): $!";
my @cols = $sth->fetchrow_array;
$sth->finish;
- Mig::db_disconnect($dbh);
+ EMig::db_disconnect($dbh);
return $cols[0];
}
use FindBin;
my $mig_bin = "$FindBin::Bin/";
use lib "$FindBin::Bin/";
-use Mig;
+use EMig;
pod2usage(-verbose => 2) if ! $ARGV[0] || $ARGV[0] eq '--help';
-Mig::die_if_no_env_migschema();
-Mig::die_if_mig_tracking_table_does_not_exist();
+EMig::die_if_no_env_migschema();
+EMig::die_if_mig_tracking_table_does_not_exist();
my $file = abs_path($ARGV[0]);
if ($file =~ /^$MIGBASEWORKDIR/) {
my $file = abs_path(shift);
my @args = @_;
- my $tracked_file_id = Mig::check_for_tracked_file($file);
+ my $tracked_file_id = EMig::check_for_tracked_file($file);
if ($tracked_file_id) {
- my $data = Mig::status_this_file($file);
+ my $data = EMig::status_this_file($file);
print "iconv'ing tracked file: $file\n";
if (scalar(@args) == 0) {
system('iconv', @args, '-o', $file . '.utf8', $file);
system('touch', $file . '.utf8'); # handle 0-byte files
- my $dbh = Mig::db_connect();
+ my $dbh = EMig::db_connect();
my $utf8_file = $dbh->quote($file . '.utf8');
if (! -e $file . '.utf8') {
print "utf8 file does not exist: $utf8_file\n";
WHERE base_filename = " . $dbh->quote($file) . "
;
") || die "Error inserting into table $MIGSCHEMA.tracked_file: $!\n";
- Mig::db_disconnect($dbh);
+ EMig::db_disconnect($dbh);
} else {
print "File not currently tracked: $file\n";
}
use DBI;
use FindBin;
my $mig_bin = "$FindBin::Bin/";
-my $mig_sql = $mig_bin . "../mig-sql/init/";
+my $mig_sql = $mig_bin . "../sql/init/";
use lib "$FindBin::Bin/";
-use Mig;
+use EMig;
pod2usage(-verbose => 2) if $ARGV[0];
-Mig::die_if_no_env_migschema();
+EMig::die_if_no_env_migschema();
-if (! Mig::check_for_db_migschema()) {
+if (! EMig::check_for_db_migschema()) {
try_to_create_schema();
}
-if (! Mig::check_db_migschema_for_migration_tables()) {
+if (! EMig::check_db_migschema_for_migration_tables()) {
try_to_init_schema_with_migration_tools();
}
-Mig::die_if_mig_tracking_table_exists();
-Mig::die_if_mig_column_tracking_table_exists();
+EMig::die_if_mig_tracking_table_exists();
+EMig::die_if_mig_column_tracking_table_exists();
loop_through_mig_sql_templates();
exit 0;
if ($MIGSCHEMA =~ /[^\w_]/) {
die "$MIGSCHEMA is not suitable for a schema name in PostgreSQL\n";
}
- my $dbh = Mig::db_connect();
+ my $dbh = EMig::db_connect();
my $rv = $dbh->do("CREATE SCHEMA $MIGSCHEMA;")
|| die "Error creating migration schema ($MIGSCHEMA): $!\n";
print "Created schema $MIGSCHEMA\n";
- Mig::db_disconnect($dbh);
+ EMig::db_disconnect($dbh);
}
sub try_to_init_schema_with_migration_tools {
- Mig::die_if_no_migration_tools();
+ EMig::die_if_no_migration_tools();
print "Calling migration_tools.init() and .build()\n";
- my $dbh = Mig::db_connect();
+ my $dbh = EMig::db_connect();
my $rv = $dbh->do("SELECT migration_tools.init(" . $dbh->quote($MIGSCHEMA) . ");")
|| die "Error running migration_tools.init($MIGSCHEMA): $!\n";
print "migration_tools.init() finished\n";
my $rv2 = $dbh->do("SELECT migration_tools.build(" . $dbh->quote($MIGSCHEMA) . ");")
|| die "Error running migration_tools.build($MIGSCHEMA): $!\n";
print "migration_tools.build() finished\n";
- Mig::db_disconnect($dbh);
+ EMig::db_disconnect($dbh);
}
sub loop_through_mig_sql_templates {
use FindBin;
my $mig_bin = "$FindBin::Bin/";
use lib "$FindBin::Bin/";
-use Mig;
+use EMig;
pod2usage(-verbose => 2) if ! $ARGV[0] || $ARGV[0] eq '--help';
-Mig::die_if_no_env_migschema();
-Mig::die_if_mig_tracking_table_does_not_exist();
+EMig::die_if_no_env_migschema();
+EMig::die_if_mig_tracking_table_does_not_exist();
my $file = abs_path($ARGV[0]);
if ($file =~ /^$MIGBASEWORKDIR/) {
my $file = abs_path(shift);
my $table = shift;
- if (! Mig::check_db_migschema_for_specific_table($table)) {
+ if (! EMig::check_db_migschema_for_specific_table($table)) {
die "table not found in MIGSCHEMA ($MIGSCHEMA): $table\n";
}
- my $tracked_file_id = Mig::check_for_tracked_file($file);
+ my $tracked_file_id = EMig::check_for_tracked_file($file);
if ($tracked_file_id) {
- my $data = Mig::status_this_file($file);
+ my $data = EMig::status_this_file($file);
print "linking file to parent table: $file -> $table\n";
- my $dbh = Mig::db_connect();
+ my $dbh = EMig::db_connect();
my $sth = $dbh->prepare("
SELECT base_filename
FROM $MIGSCHEMA.tracked_file
WHERE base_filename = " . $dbh->quote($file) . "
;
") || die "Error updating table $MIGSCHEMA.tracked_file: $!\n";
- Mig::db_disconnect($dbh);
+ EMig::db_disconnect($dbh);
} else {
print "File not currently tracked: $file\n";
}
use FindBin;
my $mig_bin = "$FindBin::Bin/";
use lib "$FindBin::Bin/";
-use Mig;
+use EMig;
pod2usage(-verbose => 2) if ! $ARGV[0] || $ARGV[0] eq '--help';
-Mig::die_if_no_env_migschema();
-Mig::die_if_mig_tracking_table_does_not_exist();
+EMig::die_if_no_env_migschema();
+EMig::die_if_mig_tracking_table_does_not_exist();
my $column_filter = 1; # show all fields
my $file = abs_path($ARGV[0]);
my $fdata;
-my $tracked_file_id = Mig::check_for_tracked_file($file);
+my $tracked_file_id = EMig::check_for_tracked_file($file);
if ($tracked_file_id) {
- $fdata = Mig::status_this_file($file);
+ $fdata = EMig::status_this_file($file);
} else {
die "File not currently tracked: $file\n";
}
printf "%-30s", "---------";
printf "%-30s", "--------------";
print "\n";
- my $dbh = Mig::db_connect();
+ my $dbh = EMig::db_connect();
my $sth = $dbh->prepare("
SELECT *
FROM information_schema.columns
print "\n";
$sth->finish;
$sth2->finish;
- Mig::db_disconnect($dbh);
+ EMig::db_disconnect($dbh);
}
sub column_menu {
my $dtd_identifier = shift;
- my $dbh = Mig::db_connect();
+ my $dbh = EMig::db_connect();
my $sth = $dbh->prepare("
SELECT *
FROM information_schema.columns
|| die "Error retrieving data from information_schema: $!";
my $data = $sth->fetchrow_hashref;
$sth->finish;
- Mig::db_disconnect($dbh);
+ EMig::db_disconnect($dbh);
my $column = $data->{column_name};
sub list_ten {
my $column = shift;
- my $dbh = Mig::db_connect();
+ my $dbh = EMig::db_connect();
my $sth;
my $rv;
my @cols;
- $sth = $dbh->prepare(Mig::sql("
+ $sth = $dbh->prepare(EMig::sql("
SELECT " . (defined $column ? $column : '*') . "
FROM $MIGSCHEMA.$table
LIMIT 10;
my $min_value_as_timestamptz;
my $max_value_as_timestamptz;
- my $dbh = Mig::db_connect();
+ my $dbh = EMig::db_connect();
my $sth;
my $rv;
my @cols;
$sth->finish;
$max_value_as_timestamptz = $cols[0];
- Mig::db_disconnect($dbh);
+ EMig::db_disconnect($dbh);
print "\n";
print "# of rows = $count\n";
sub group_browse {
my ($column,$option) = (shift,shift||"GROUP BY 1 ORDER BY 1");
- my $dbh = Mig::db_connect();
+ my $dbh = EMig::db_connect();
my $sth;
my $rv;
- $sth = $dbh->prepare(Mig::sql("
+ $sth = $dbh->prepare(EMig::sql("
SELECT $column, COUNT(*)
FROM $MIGSCHEMA.$table
$option;
sub add_this_column {
my $column = shift;
- if (!Mig::check_for_tracked_column($table,$column)) {
- my $dbh = Mig::db_connect();
+ if (!EMig::check_for_tracked_column($table,$column)) {
+ my $dbh = EMig::db_connect();
my $rv = $dbh->do("
INSERT INTO $MIGSCHEMA.tracked_column (
base_filename
," . $dbh->quote($column) . "
);
") || die "Error inserting into table $MIGSCHEMA.tracked_column: $!\n";
- Mig::db_disconnect($dbh);
+ EMig::db_disconnect($dbh);
}
}
sub status_this_column {
my $column = shift;
- my $data = Mig::status_this_column($table,$column);
+ my $data = EMig::status_this_column($table,$column);
if (!$data) {
add_this_column($column);
- $data = Mig::status_this_column($table,$column);
+ $data = EMig::status_this_column($table,$column);
}
if ($$data{parent_table}) {
- my $dbh = Mig::db_connect();
+ my $dbh = EMig::db_connect();
my $sth = $dbh->prepare("
SELECT *
FROM $MIGSCHEMA.fields_requiring_mapping
$$data{required} = 0;
}
$sth->finish;
- Mig::db_disconnect($dbh);
+ EMig::db_disconnect($dbh);
}
return $data;
}
my ($column,$comment) = (shift,shift);
if ($comment) {
my $data = status_this_column($column);
- my $dbh = Mig::db_connect();
+ my $dbh = EMig::db_connect();
my $rv = $dbh->do("
UPDATE $MIGSCHEMA.tracked_column
SET comment = " . $dbh->quote($comment) . "
WHERE id = " . $dbh->quote($data->{id}) . ";
") || die "Error updating table $MIGSCHEMA.tracked_column: $!\n";
- Mig::db_disconnect($dbh);
+ EMig::db_disconnect($dbh);
}
}
my ($column,$transform) = (shift,shift);
if ($transform) {
my $data = status_this_column($column);
- my $dbh = Mig::db_connect();
+ my $dbh = EMig::db_connect();
my $rv = $dbh->do("
UPDATE $MIGSCHEMA.tracked_column
SET transform = " . $dbh->quote($transform) . "
WHERE id = " . $dbh->quote($data->{id}) . ";
") || die "Error updating table $MIGSCHEMA.tracked_column: $!\n";
- Mig::db_disconnect($dbh);
+ EMig::db_disconnect($dbh);
}
}
$target_column = $2;
}
my $data = status_this_column($column);
- my $dbh = Mig::db_connect();
+ my $dbh = EMig::db_connect();
my $rv = $dbh->do("
UPDATE $MIGSCHEMA.tracked_column
SET target_table = " . $dbh->quote($target_table) . "
,target_column = " . $dbh->quote($target_column) . "
WHERE id = " . $dbh->quote($data->{id}) . ";
") || die "Error updating table $MIGSCHEMA.tracked_column: $!\n";
- Mig::db_disconnect($dbh);
+ EMig::db_disconnect($dbh);
}
}
use FindBin;
my $mig_bin = "$FindBin::Bin/";
use lib "$FindBin::Bin/";
-use Mig;
+use EMig;
my @files = grep {!/^--/} @ARGV;
my %pass_thru = ('add'=>[],'skip-iconv'=>[],'clean'=>[],'convert'=>[],'stage'=>[]);
use FindBin;
my $mig_bin = "$FindBin::Bin/";
use lib "$FindBin::Bin/";
-use Mig;
+use EMig;
my $outtable = '';
my $outfile = '';
}
}
-Mig::die_if_no_env_migschema();
-Mig::die_if_mig_tracking_table_does_not_exist();
+EMig::die_if_no_env_migschema();
+EMig::die_if_mig_tracking_table_does_not_exist();
my $workbook;
my @worksheets = ();
}
close_workbook();
} else {
- if (Mig::check_db_migschema_for_specific_table($outtable)) {
+ if (EMig::check_db_migschema_for_specific_table($outtable)) {
drop_existing_outtable();
}
}
sub handle_file {
- my $tracked_file_id = Mig::check_for_tracked_file($file);
+ my $tracked_file_id = EMig::check_for_tracked_file($file);
if ($tracked_file_id) {
- $fdata = Mig::status_this_file($file);
+ $fdata = EMig::status_this_file($file);
} else {
die "File not currently tracked: $file\n";
}
}
}
if ($outtable) {
- if (Mig::check_db_migschema_for_specific_table($outtable)) {
+ if (EMig::check_db_migschema_for_specific_table($outtable)) {
if ($force) {
drop_existing_outtable();
} else {
sub drop_existing_outtable {
# we want a transaction for this one
- my $dbh = Mig::db_connect();
+ my $dbh = EMig::db_connect();
$dbh->{AutoCommit} = 0;
$dbh->{RaiseError} = 1;
die "Aborting mig-quicksheet\n";
};
- Mig::db_disconnect($dbh);
+ EMig::db_disconnect($dbh);
}
sub create_new_outtable {
- my $dbh = Mig::db_connect();
+ my $dbh = EMig::db_connect();
print "Creating table $MIGSCHEMA.$outtable\n";
my $rv = $dbh->do("
CREATE UNLOGGED TABLE $MIGSCHEMA.$outtable (
summary_table TEXT UNIQUE
);
") || die "Error creating outtable ($MIGSCHEMA.$outtable): $!\n";
- Mig::db_disconnect($dbh);
+ EMig::db_disconnect($dbh);
}
sub create_new_subtable {
my $subtable = shift;
- my $dbh = Mig::db_connect();
+ my $dbh = EMig::db_connect();
$dbh->{AutoCommit} = 0;
$dbh->{RaiseError} = 1;
die "Aborting mig-quicksheet\n";
};
- Mig::db_disconnect($dbh);
+ EMig::db_disconnect($dbh);
}
sub write_worksheets {
create_new_subtable( $first_table );
}
- my $dbh = Mig::db_connect();
+ my $dbh = EMig::db_connect();
my $sth = $dbh->prepare("
SELECT COUNT(*)
FROM $MIGSCHEMA.$table
};
}
- Mig::db_disconnect($dbh);
+ EMig::db_disconnect($dbh);
}
sub close_workbook {
###############################################################################
sub handle_list {
- my $dbh = Mig::db_connect();
+ my $dbh = EMig::db_connect();
my $sth = $dbh->prepare("
SELECT *
FROM " . $MIGSCHEMA. "." . $table . "
}
sub handle_columns {
- my $dbh = Mig::db_connect();
+ my $dbh = EMig::db_connect();
my $sth = $dbh->prepare("
SELECT *
FROM information_schema.columns
}
}
$sth->finish;
- Mig::db_disconnect($dbh);
+ EMig::db_disconnect($dbh);
}
sub column_summary {
my $column = shift;
- my $dbh = Mig::db_connect();
+ my $dbh = EMig::db_connect();
### non_empty_count
my $sth = $dbh->prepare("
sub group_by {
my ($column,$option) = (shift,"GROUP BY 2 ORDER BY 2");
- my $dbh = Mig::db_connect();
+ my $dbh = EMig::db_connect();
my $sth;
my $rv;
use FindBin;
my $mig_bin = "$FindBin::Bin/";
use lib "$FindBin::Bin/";
-use Mig;
+use EMig;
pod2usage(-verbose => 2) if ! $ARGV[0] || $ARGV[0] eq '--help';
-Mig::die_if_no_env_migschema();
-Mig::die_if_mig_tracking_table_does_not_exist();
+EMig::die_if_no_env_migschema();
+EMig::die_if_mig_tracking_table_does_not_exist();
foreach my $arg (@ARGV) {
my $file = abs_path($arg);
sub remove_this_file {
my $file = shift;
- my $tracked_file_id = Mig::check_for_tracked_file($file,{'allow_missing'=>1});
+ my $tracked_file_id = EMig::check_for_tracked_file($file,{'allow_missing'=>1});
if ($tracked_file_id) {
print "removing tracked file: $file\n";
- my $dbh = Mig::db_connect();
+ my $dbh = EMig::db_connect();
my $rv = $dbh->do("
DELETE FROM $MIGSCHEMA.tracked_file WHERE id = $tracked_file_id;
") || die "Error deleting from table $MIGSCHEMA.tracked_file (id = $tracked_file_id): $!\n";
- Mig::db_disconnect($dbh);
+ EMig::db_disconnect($dbh);
} else {
print "File not currently tracked: $file\n";
}
use FindBin;
my $mig_bin = "$FindBin::Bin/";
use lib "$FindBin::Bin/";
-use Mig;
+use EMig;
use open ':encoding(utf8)';
pod2usage(-verbose => 2) if defined $ARGV[0] && $ARGV[0] eq '--help';
}
if (defined $added_page_file) { $added_page_file = $MIGGITDIR . $added_page_file; }
-my $dbh = Mig::db_connect();
+my $dbh = EMig::db_connect();
my $report_file = create_report_name($report_title);
$report_file = $MIGGITDIR . $report_file;
use FindBin;
my $mig_bin = "$FindBin::Bin/";
use lib "$FindBin::Bin/";
-use Mig;
+use EMig;
pod2usage(-verbose => 2) if ! ($ARGV[0]||$ARGV[1]) || $ARGV[0] eq '--help';
-Mig::die_if_no_env_migschema();
-Mig::die_if_mig_tracking_table_does_not_exist();
+EMig::die_if_no_env_migschema();
+EMig::die_if_mig_tracking_table_does_not_exist();
my $file = abs_path($ARGV[0]);
my $clean_file;
my $file = shift;
my $clean_file = shift;
- my $tracked_file_id = Mig::check_for_tracked_file($file);
+ my $tracked_file_id = EMig::check_for_tracked_file($file);
if ($tracked_file_id) {
- my $data = Mig::status_this_file($file);
+ my $data = EMig::status_this_file($file);
if (! $data->{'utf8_filename'}) {
die "mig-iconv or mig-skip-iconv needed for UTF8 version of file: $file\n";
print "skipping cleaning of tracked file: $file\n";
- my $dbh = Mig::db_connect();
+ my $dbh = EMig::db_connect();
if (! $clean_file) {
$clean_file = $utf8_file . '.clean';
}
WHERE base_filename = " . $dbh->quote($file) . "
;
") || die "Error inserting into table $MIGSCHEMA.tracked_file: $!\n";
- Mig::db_disconnect($dbh);
+ EMig::db_disconnect($dbh);
} else {
die "File not currently tracked: $file\n";
}
use FindBin;
my $mig_bin = "$FindBin::Bin/";
use lib "$FindBin::Bin/";
-use Mig;
+use EMig;
pod2usage(-verbose => 2) if ! ($ARGV[0]||$ARGV[1]) || $ARGV[0] eq '--help';
-Mig::die_if_no_env_migschema();
-Mig::die_if_mig_tracking_table_does_not_exist();
+EMig::die_if_no_env_migschema();
+EMig::die_if_mig_tracking_table_does_not_exist();
my $file = abs_path($ARGV[0]);
my $utf8_file;
my $file = shift;
my $utf8_file = shift;
- my $tracked_file_id = Mig::check_for_tracked_file($file);
+ my $tracked_file_id = EMig::check_for_tracked_file($file);
if ($tracked_file_id) {
- my $data = Mig::status_this_file($file);
+ my $data = EMig::status_this_file($file);
print "skipping the iconv'ing of tracked file: $file\n";
- my $dbh = Mig::db_connect();
+ my $dbh = EMig::db_connect();
if (! $utf8_file) {
$utf8_file = $file . '.utf8';
}
WHERE base_filename = " . $dbh->quote($file) . "
;
") || die "Error inserting into table $MIGSCHEMA.tracked_file: $!\n";
- Mig::db_disconnect($dbh);
+ EMig::db_disconnect($dbh);
} else {
print "File not currently tracked: $file\n";
}
use FindBin;
my $mig_bin = "$FindBin::Bin/";
use lib "$FindBin::Bin/";
-use Mig;
+use EMig;
my @MYARGV = (
'psql'
use FindBin;
my $mig_bin = "$FindBin::Bin/";
use lib "$FindBin::Bin/";
-use Mig;
+use EMig;
pod2usage(-verbose => 2) if ! $ARGV[0] || $ARGV[0] eq '--help';
-Mig::die_if_no_env_migschema();
-Mig::die_if_mig_tracking_table_does_not_exist();
+EMig::die_if_no_env_migschema();
+EMig::die_if_mig_tracking_table_does_not_exist();
my $file = abs_path($ARGV[0]);
if ($file =~ /^$MIGBASEWORKDIR/) {
my $file = abs_path(shift);
my @args = @_;
- my $tracked_file_id = Mig::check_for_tracked_file($file);
+ my $tracked_file_id = EMig::check_for_tracked_file($file);
if ($tracked_file_id) {
- my $data = Mig::status_this_file($file);
+ my $data = EMig::status_this_file($file);
if (! $data->{'utf8_filename'}) {
die "mig-iconv or mig-skip-iconv needed for UTF8 version of file: $file\n";
my ($schema,$table) = split /\./, $schema_table;
if (defined $data->{'staged_table'} && $data->{'staged_table'} ne $table) {
- my $dbh2 = Mig::db_connect();
+ my $dbh2 = EMig::db_connect();
print "dropping previously staged table: $MIGSCHEMA.$data->{staged_table}\n";
my $rv2 = $dbh2->do("
DROP TABLE $MIGSCHEMA.$data->{staged_table};
SET target_table = " . $dbh2->quote($table) . "
WHERE target_table = " . $dbh2->quote($data->{staged_table}) . "
") || die "Error changing references to $data->{staged_table}: $!\n";
- Mig::db_disconnect($dbh2);
+ EMig::db_disconnect($dbh2);
}
print "running staging SQL: $stage_sql_filename\n";
if ($schema ne $MIGSCHEMA) {
die "Schema mismatch: env => $MIGSCHEMA sql => $schema\n";
}
- if (! Mig::check_db_migschema_for_specific_table($table)) {
+ if (! EMig::check_db_migschema_for_specific_table($table)) {
die "Missing staged table: $schema_table\n";
} else {
print "table staged: $schema_table\n";
}
- my $dbh = Mig::db_connect();
+ my $dbh = EMig::db_connect();
my $rv = $dbh->do("
UPDATE $MIGSCHEMA.tracked_file
SET staged_table = " . $dbh->quote($table) . "
WHERE base_filename = " . $dbh->quote($file) . "
;
") || die "Error updating table $MIGSCHEMA.tracked_file: $!\n";
- Mig::db_disconnect($dbh);
+ EMig::db_disconnect($dbh);
} else {
print "File not currently tracked: $file\n";
}
use UNIVERSAL;
my $mig_bin = "$FindBin::Bin/";
use lib "$FindBin::Bin/";
-use Mig;
+use EMig;
use Getopt::Long;
pod2usage(-verbose => 2) if defined $ARGV[0] && $ARGV[0] eq '--help';
my $source = 2;
my $x_source = 'default';
my $no_source_or_last_xact_id;
-my $dbh = Mig::db_connect();
+my $dbh = EMig::db_connect();
my $infile;
my $i = 0;
my $batch;
use FindBin;
my $mig_bin = "$FindBin::Bin/";
use lib "$FindBin::Bin/";
-use Mig;
+use EMig;
pod2usage(-verbose => 2) if scalar(@ARGV) > 0 && $ARGV[0] eq '--help';
-Mig::die_if_no_env_migschema();
-Mig::die_if_mig_tracking_table_does_not_exist();
+EMig::die_if_no_env_migschema();
+EMig::die_if_mig_tracking_table_does_not_exist();
my @files = @ARGV;
if (scalar(@files) == 0) {
@files = ();
- my $dbh = Mig::db_connect();
+ my $dbh = EMig::db_connect();
my $sth = $dbh->prepare("
SELECT base_filename
FROM $MIGSCHEMA.tracked_file
push @files, $row->[0]
}
$sth->finish;
- Mig::db_disconnect($dbh);
+ EMig::db_disconnect($dbh);
}
foreach my $arg (sort @files) {
my $file = abs_path($arg);
- my $data = Mig::status_this_file($file);
+ my $data = EMig::status_this_file($file);
print "=-=-=\n";
foreach my $key (
'base_filename'
use FindBin;
my $mig_bin = "$FindBin::Bin/";
use lib "$FindBin::Bin/";
-use Mig;
+use EMig;
pod2usage(-verbose => 2) if ! $ARGV[0] || $ARGV[0] eq '--help';
-Mig::die_if_no_env_migschema();
-Mig::die_if_mig_tracking_table_does_not_exist();
+EMig::die_if_no_env_migschema();
+EMig::die_if_mig_tracking_table_does_not_exist();
my $file = abs_path($ARGV[0]);
if ($file =~ /^$MIGBASEWORKDIR/) {
sub unlink_table {
my $file = abs_path(shift);
- my $tracked_file_id = Mig::check_for_tracked_file($file);
+ my $tracked_file_id = EMig::check_for_tracked_file($file);
if ($tracked_file_id) {
- my $data = Mig::status_this_file($file);
+ my $data = EMig::status_this_file($file);
my $table = $data->{'parent_table'} || '';
print "unlinking table ($table) from file: $file\n";
- my $dbh = Mig::db_connect();
+ my $dbh = EMig::db_connect();
my $rv = $dbh->do("
UPDATE $MIGSCHEMA.tracked_file
SET parent_table = ''
WHERE base_filename = " . $dbh->quote($file) . "
;
") || die "Error updating table $MIGSCHEMA.tracked_file: $!\n";
- Mig::db_disconnect($dbh);
+ EMig::db_disconnect($dbh);
} else {
print "File not currently tracked: $file\n";
}
=head1 NAME
-kmig - git-like program for tracking and manipulating legacy data files for
+mig - git-like program for tracking and manipulating legacy data files for
migrations. This variant of <mig> and is geared toward the Koha ILS and
MySql/MariaDB.
=head1 SYNOPSIS
-B<kmig> <command> [argument] [...]
+B<mig> <command> [argument] [...]
=head1 DESCRIPTION
-B<kmig> is used to track and manipulate CSV or CSV-like text files exported from
+B<mig> is used to track and manipulate CSV or CSV-like text files exported from
legacy systems for migration into Evergreen. It can be a wrapper for some
other migration tools and tracks state using a MySQL table for a given
database.
For most commands, if the current working directory falls outside of the
-directory specified by MIGWORKDIR, then kmig will assume that environment is
+directory specified by MIGWORKDIR, then mig will assume that environment is
also incorrect and bail before doing any actual work.
Only the B<env> and B<help> commands work without the MIGDATABASE environment
=head1 OVERVIEW
-Using B<kmig> should go something like this:
+Using B<mig> should go something like this:
=over 15
-=item kmig env create m_foo # Sets up the environment
+=item mig env create m_foo # Sets up the environment
-=item kmig env use m_foo # Spawns a shell using the configured environment
+=item mig env use m_foo # Spawns a shell using the configured environment
-=item kmig init # creates any needed auxilary tables
+=item mig init # creates any needed auxilary tables
-=item kmig add patrons.tsv # tracks an incoming data file; repeat for additional files
+=item mig add patrons.tsv # tracks an incoming data file; repeat for additional files
-=item kmig iconv patrons.tsv # convert it to UTF8, creating patrons.tsv.utf8
+=item mig iconv patrons.tsv # convert it to UTF8, creating patrons.tsv.utf8
-=item kmig clean patrons.tsv # cleans the file, creating patrons.tsv.utf8.clean
+=item mig clean patrons.tsv # cleans the file, creating patrons.tsv.utf8.clean
-=item kmig link patrons.tsv borrowers # models the soon-to-be staging table after table 'borrowers'
+=item mig link patrons.tsv borrowers # models the soon-to-be staging table after table 'borrowers'
-=item kmig convert patrons.tsv # creates a .sql file for staging the data
+=item mig convert patrons.tsv # creates a .sql file for staging the data
-=item kmig stage patrons.tsv # load said .sql file
+=item mig stage patrons.tsv # load said .sql file
-=item kmig mapper patrons.tsv # interactive tool for analyzing/mapping the staging table
+=item mig mapper patrons.tsv # interactive tool for analyzing/mapping the staging table
-=item kmig analysis patrons.tsv # writes a summary .tsv file of mapped/flagged fields from the staging table
+=item mig analysis patrons.tsv # writes a summary .tsv file of mapped/flagged fields from the staging table
-=item kmig map patrons.tsv # apply configured mappings
+=item mig map patrons.tsv # apply configured mappings
-=item kmig write_prod patrons.tsv # creates a .sql file for pushing the staging data into production
+=item mig write_prod patrons.tsv # creates a .sql file for pushing the staging data into production
-=item kmig reporter --analyst "Foo Fooer" --report_title "Foo Load Analysis" #creates an asciidoc report
+=item mig reporter --analyst "Foo Fooer" --report_title "Foo Load Analysis" #creates an asciidoc report
-=item kmig gsheet --pull foo_tab_name OR --push foo_pg_table_name
+=item mig gsheet --pull foo_tab_name OR --push foo_pg_table_name
-=item kmig stagebibs --file foo.xml
+=item mig stagebibs --file foo.xml
=back
specified, the tracker will assume the files have headers.
You can do crazy stuff like
-B<kmig add file1 --no-headers file2 file3 --headers file4>
+B<mig add file1 --no-headers file2 file3 --headers file4>
=item B<remove> <file> [file] [...]
=item B<skip-iconv> <file>
-If this is used instead of B<iconv>, then B<kmig> will look for an existing
+If this is used instead of B<iconv>, then B<mig> will look for an existing
<file>.utf8 and use it instead of attempting to create one.
=item B<clean> <file> [other arguments...]
=item B<skip-clean> <file>
-If this is used instead of B<clean>, then B<kmig> will look for an existing
+If this is used instead of B<clean>, then B<mig> will look for an existing
<file>.utf8.clean and use it instead of attempting to create one.
=item B<link> <file> <parent table>
--added_page_title and --added_page_file
If one is used both must be. The added page file can be plain text or asciidoc. This
-adds an extra arbitrary page of notes to the report. KMig assumes the page file is in the kmig git directory.
+adds an extra arbitrary page of notes to the report. KMig assumes the page file is in the mig git directory.
--tags
=item B<gsheet> --pull or --push spreadsheet_tab
This uses the gsheet_tracked_table and gsheet_tracked column tables to map a Google Docs Spreadsheet tabs
-with Postgres tables in the kmig schema. The spreadsheet is assumed to share the name as the kmig schema.
+with Postgres tables in the mig schema. The spreadsheet is assumed to share the name as the mig schema.
Tab names must be unique. Each spreadsheet column needs a header that matches the column name in the matching
-table. An oauth session key is also needed for your Google account and kmig gsheet will look for it in the
-.kmig directory.
+table. An oauth session key is also needed for your Google account and mig gsheet will look for it in the
+.mig directory.
=back
my $cmd = shift;
if ($cmd ne 'env') { KMig::die_if_no_env_migschema(); }
- if (-e $mig_bin . "kmig-$cmd") {
- system( $mig_bin . "kmig-$cmd", @_ );
+ if (-e $mig_bin . "mig-$cmd") {
+ system( $mig_bin . "mig-$cmd", @_ );
} else {
- system( "kmig-$cmd", @_ ) == 0 or die pod2usage(1);
+ system( "mig-$cmd", @_ ) == 0 or die pod2usage(1);
}
}
=head1 DESCRIPTION
For most invocations, B<mig-env> will either create or use a migration-specific
-file (~/.mig/<migration_schema>.env) for setting the following environment
+file (~/.kmig/<migration_schema>.env) for setting the following environment
variables:
=over 15
=item B<use> <schema>
This command will spawn a bash shell that executes the corresponding
-~/.mig/<schema>.env script for setting up environment variables encoded during
+~/.kmig/<schema>.env script for setting up environment variables encoded during
B<create>.
=item B<show> [schema]
-This command will show the contents of the corresponding ~/.mig/<schema>.env
+This command will show the contents of the corresponding ~/.kmig/<schema>.env
script, or, if no schema is specified, then it will list pertinent variables in
the current environment if they exist.
=item B<list>
-This command will list migration schemas found in ~/.mig
+This command will list migration schemas found in ~/.kmig
=item B<help>
pod2usage(-verbose => 2) if ! $ARGV[0];
my $migration_schema = $ARGV[1] || '';
-my $filename = "$HOME/.mig/$migration_schema.env";
+my $filename = "$HOME/.kmig/$migration_schema.env";
switch($ARGV[0]) {
case "--help" {
pod2usage(-verbose => 2);
case "clone" {
pod2usage(-verbose => 1) if ! $ARGV[2];
$migration_schema = $ARGV[2] || '';
- $filename = "$HOME/.mig/$migration_schema.env";
+ $filename = "$HOME/.kmig/$migration_schema.env";
mig_env_clone();
}
case "use" {
}
}
case "list" {
- opendir(my $dh, "$HOME/.mig") || die "can't open $HOME/.mig: $!";
+ opendir(my $dh, "$HOME/.kmig") || die "can't open $HOME/.kmig: $!";
while (readdir $dh) {
if (/^(.*)\.env$/) {
print "$1\n";
# create files and directories if needed
- mkdir "$HOME/.mig";
+ mkdir "$HOME/.kmig";
make_path($MIGGITDIR, { verbose => 1 });
`touch $MIGGITDIR/README`;
make_path($MIGWORKDIR, { verbose => 1 });
projects / migration-tools.git / commitdiff