From 483f7fb67b65840e55be6a04d64dc34c0e0ef87e Mon Sep 17 00:00:00 2001 From: Jason Etheridge Date: Fri, 10 Apr 2020 14:13:08 -0400 Subject: [PATCH] let's not name these differently than the emig.d/ counterparts --- kmig.d/bin/kmig-add | 125 ------------------ kmig.d/bin/kmig-clean | 127 ------------------- kmig.d/bin/kmig-env | 298 -------------------------------------------- kmig.d/bin/kmig-iconv | 107 ---------------- kmig.d/bin/kmig-init | 65 ---------- kmig.d/bin/kmig-remove | 64 ---------- kmig.d/bin/kmig-skip-iconv | 87 ------------- kmig.d/bin/kmig-sql | 46 ------- kmig.d/bin/kmig-status | 87 ------------- kmig.d/bin/mig-add | 125 ++++++++++++++++++ kmig.d/bin/mig-clean | 127 +++++++++++++++++++ kmig.d/bin/mig-env | 298 ++++++++++++++++++++++++++++++++++++++++++++ kmig.d/bin/mig-iconv | 107 ++++++++++++++++ kmig.d/bin/mig-init | 65 ++++++++++ kmig.d/bin/mig-remove | 64 ++++++++++ kmig.d/bin/mig-skip-iconv | 87 +++++++++++++ kmig.d/bin/mig-sql | 46 +++++++ kmig.d/bin/mig-status | 87 +++++++++++++ 18 files changed, 1006 insertions(+), 1006 deletions(-) delete mode 100755 kmig.d/bin/kmig-add delete mode 100755 kmig.d/bin/kmig-clean delete mode 100755 kmig.d/bin/kmig-env delete mode 100755 kmig.d/bin/kmig-iconv delete mode 100755 kmig.d/bin/kmig-init delete mode 100755 kmig.d/bin/kmig-remove delete mode 100755 kmig.d/bin/kmig-skip-iconv delete mode 100755 kmig.d/bin/kmig-sql delete mode 100755 kmig.d/bin/kmig-status create mode 100755 kmig.d/bin/mig-add create mode 100755 kmig.d/bin/mig-clean create mode 100755 kmig.d/bin/mig-env create mode 100755 kmig.d/bin/mig-iconv create mode 100755 kmig.d/bin/mig-init create mode 100755 kmig.d/bin/mig-remove create mode 100755 kmig.d/bin/mig-skip-iconv create mode 100755 kmig.d/bin/mig-sql create mode 100755 kmig.d/bin/mig-status diff --git a/kmig.d/bin/kmig-add b/kmig.d/bin/kmig-add deleted file mode 100755 index 3538e9f..0000000 --- a/kmig.d/bin/kmig-add +++ /dev/null @@ -1,125 +0,0 @@ -#!/usr/bin/perl -w -############################################################################### -=pod - -=head1 NAME - -kmig-add - This will add the specified files to the mig tracking table - ---headers (the default) and --no-headers are repeatable, and indicate whether -subsequent files have headers or not - ---headers-file specifies a text file defining the column headers for -the next added , which should contain one line per header - ---headers-file will automatically invoke --no-headers - -You'll need to invoke B prior to using commands like B - -=head1 SYNOPSIS - -B [--no-headers|--headers|--headers-file ] [file|--no-headers|--headers|--headers-file ] [...] - -=cut - -############################################################################### - -use strict; -use Switch; -use Env qw( - HOME MYSQL_HOST MYSQL_TCP_PORT MYSQL_USER MYSQL_DATABASE MYSQL_PW - MIGSCHEMA MIGBASEWORKDIR MIGBASEGITDIR MIGGITDIR MIGWORKDIR -); -use Pod::Usage; -use DBI; -use Cwd 'abs_path'; -use FindBin; -my $mig_bin = "$FindBin::Bin/"; -use lib "$FindBin::Bin/"; -use KMig; - -pod2usage(-verbose => 2) if ! $ARGV[0] || $ARGV[0] eq '--help'; - -KMig::die_if_no_env_migschema(); -KMig::die_if_mig_tracking_table_does_not_exist(); - -my $has_headers = 1; -my $headers_file; -my $next_arg_is_headers_file = 0; - -foreach my $arg (@ARGV) { - if ($next_arg_is_headers_file) { - $next_arg_is_headers_file = 0; - $headers_file = abs_path($arg); - next; - } - if ($arg eq '--headers') { - $has_headers = 1; - next; - } - if ($arg eq '--no-headers') { - $has_headers = 0; - next; - } - if ($arg eq '--headers-file') { - $next_arg_is_headers_file = 1; - $has_headers = 0; - next; - } - my $file = abs_path($arg); - if ($file =~ /^$MIGBASEWORKDIR/) { - if (-e $file) { - if (-f $file) { - add_this_file($file,$has_headers,$headers_file); - $headers_file = ''; # clear after applying to just one file - } else { - print "Not a real file: $file\n"; - } - } else { - print "Could not find file: $file\n"; - } - } else { - print "File falls outside of MIGWORKDIR ($MIGWORKDIR): $file\n"; - } -} - -exit 0; - -############################################################################### - -sub add_this_file { - my $file = shift; - my $headers = shift; - my $headers_file = shift; - if ($headers_file) { - if (! (-e $headers_file && -f $headers_file)) { - print "Could not find headers file $headers_file, skipping $file\n"; - return; - } - } - if (KMig::check_for_tracked_file($file)) { - print "File already tracked: $file\n"; - } else { - print 'Adding ('; - if ($headers_file) { - print "with headers file = $headers_file"; - } else { - print ($headers ? ' with headers' : 'without headers'); - } - print '): ' . "$file\n"; - my $dbh = KMig::db_connect(); - my $rv = $dbh->do(" - INSERT INTO m_tracked_file ( - base_filename - ,has_headers - ,headers_file - ) VALUES ( - " . $dbh->quote($file) . " - ," . $dbh->quote($headers) . " - ," . $dbh->quote($headers_file) . " - ); - ") || die "Error inserting into table m_tracked_file: $!\n"; - KMig::db_disconnect($dbh); - } -} - diff --git a/kmig.d/bin/kmig-clean b/kmig.d/bin/kmig-clean deleted file mode 100755 index 39522d6..0000000 --- a/kmig.d/bin/kmig-clean +++ /dev/null @@ -1,127 +0,0 @@ -#!/usr/bin/perl -w -############################################################################### -=pod - -=head1 NAME - -kmig-clean - -Attempts to invoke B on the specified tracked file, placing the -output in [file].clean - -If given no other arguments, the invocation will lool like - -=over 5 - -clean_csv --config scripts/clean.conf --fix --apply [--create-headers|--use-headers ] - -=back - -otherwise, the arguments will be passed through like so - -=over 5 - -clean_csv [other arguments...] - -=back - -You'll need to invoke B or B prior to using commands -like B - -=head1 SYNOPSIS - -B [other arguments...] - -=cut - -############################################################################### - -use strict; -use Switch; -use Env qw( - HOME MYSQL_HOST MYSQL_TCP_PORT MYSQL_USER MYSQL_DATABASE MYSQL_PW - MIGSCHEMA MIGBASEWORKDIR MIGBASEGITDIR MIGGITDIR MIGWORKDIR -); -use Pod::Usage; -use DBI; -use Cwd 'abs_path'; -use FindBin; -my $mig_bin = "$FindBin::Bin/"; -use lib "$FindBin::Bin/"; -use KMig; - -pod2usage(-verbose => 2) if ! $ARGV[0] || $ARGV[0] eq '--help'; - -KMig::die_if_no_env_migschema(); -KMig::die_if_mig_tracking_table_does_not_exist(); - -my $file = abs_path($ARGV[0]); -if ($file =~ /^$MIGBASEWORKDIR/) { - call_clean_csv(@ARGV); -} else { - print "File falls outside of MIGWORKDIR ($MIGWORKDIR): $file\n"; -} - -exit 0; - -############################################################################### - -sub call_clean_csv { - my $file = abs_path(shift); - my @args = @_; - - my $tracked_file_id = KMig::check_for_tracked_file($file); - if ($tracked_file_id) { - my $data = KMig::status_this_file($file); - - if (! $data->{'utf8_filename'}) { - die "kmig-iconv or kmig-skip-iconv needed for UTF8 version of file: $file\n"; - } - - my $utf8_file = $data->{'utf8_filename'}; - if (! -e $utf8_file) { - die "missing file: $utf8_file\n"; - } - - print "cleaning tracked file: $file\n"; - - if (scalar(@args) == 0) { - @args = ( - '--config' - ,'scripts/clean.conf' - ,'--fix' - ,'--apply' - ,'--backslash' - ,'--pad' - ); - if (! $data->{'has_headers'}) { - if ($data->{'headers_file'}) { - push @args, '--use-headers'; - push @args, $data->{'headers_file'}; - } else { - push @args, '--create-headers'; - } - } - } - - print join(' ',@args) . "\n"; - system('clean_csv', @args, $utf8_file); - - my $dbh = KMig::db_connect(); - my $clean_file = $dbh->quote($utf8_file . '.clean'); - if (! -e $utf8_file . '.clean') { - print "clean file does not exist: $clean_file\n"; - $clean_file = $dbh->quote(''); - } - - my $rv = $dbh->do(" - UPDATE m_tracked_file - SET clean_filename = $clean_file - WHERE base_filename = " . $dbh->quote($file) . " - ; - ") || die "Error inserting into table m_tracked_file: $!\n"; - KMig::db_disconnect($dbh); - } else { - print "File not currently tracked: $file\n"; - } -} diff --git a/kmig.d/bin/kmig-env b/kmig.d/bin/kmig-env deleted file mode 100755 index fce49bc..0000000 --- a/kmig.d/bin/kmig-env +++ /dev/null @@ -1,298 +0,0 @@ -#!/usr/bin/perl -w -############################################################################### -=pod - -=head1 NAME - -kmig-env - This tool is for tracking and setting environment variables used by -B and its sub-tools. - -=head1 SYNOPSIS - -B - -B [migration_schema] - -B [orig_migration_schema] [new_migration_schema] - -B - -B - -=head1 DESCRIPTION - -For most invocations, B will either create or use a migration-specific -file (~/.kmig/.env) for setting the following environment -variables: - -=over 15 - -=item MIGSCHEMA - -The name of the migration schema. In practice, this will match the name of the -Koha instance. - -=item MIGWORKDIR - -The base working directory for containing migration data, scripts, and other -files. - -=item MYSQL_HOST - -The IP address or hostname for the MySQL/MariaDB database used for a migration. - -=item MYSQL_TCP_PORT - -The TCP port for the database. - -=item MYSQL_USER - -The user to use for the database. - -=item MYSQL_PW - -The password to use for the database. - -=item MYSQL_DATABASE - -The name of the actual database/schema. In practice, this will match the -migration schema or Koha instance name, prefixed with 'koha_'. - -=back - -This script may also setup a symlink from a specified Git repository to a -scripts/ directory within the migration work directory. The default for this is -~/git/migration-work/MIGSCHEMA --> MIGWORKDIR/scripts - -It may also create the migration work directory if necessary. - -=head1 COMMANDS - -=over 15 - -=item B - -This invocation will prompt for various values and create a .env file for the -specified migration schema, and a symlink between the specified Git repository -and migration work directory (which will also be created if needed). - -=item B - -This command will spawn a bash shell that executes the corresponding -~/.kmig/.env script for setting up environment variables encoded during -B. - -=item B [schema] - -This command will show the contents of the corresponding ~/.kmig/.env -script, or, if no schema is specified, then it will list pertinent variables in -the current environment if they exist. - -=item B [orig schema] [new schema] - -FIXME: need to re-think this in a MySQL/MariaDB/Koha context - -This command will create a "shallow" clone of the orig schema, in that it will -share database credentials as well as git and data directories, but will have a -separate schema name. - -=item B - -This command will list migration schemas found in ~/.kmig - -=item B - -Display the documentation you're reading now. - -=back - -=cut - -############################################################################### - -use strict; -use 5.012; -use Switch; -use Env qw( - HOME MYSQL_HOST MYSQL_TCP_PORT MYSQL_USER MYSQL_DATABASE MYSQL_PW - MIGSCHEMA MIGBASEWORKDIR MIGBASEGITDIR MIGGITDIR MIGWORKDIR -); -use Pod::Usage; -use File::Path qw(make_path); -use FindBin; -my $mig_bin = "$FindBin::Bin/"; -use lib "$FindBin::Bin/"; - -pod2usage(-verbose => 2) if ! $ARGV[0]; - -my $migration_schema = $ARGV[1] || ''; -my $filename = "$HOME/.kmig/$migration_schema.env"; -switch($ARGV[0]) { - case "--help" { - pod2usage(-verbose => 2); - } - case "help" { - pod2usage(-verbose => 2); - } - case "create" { - pod2usage(-verbose => 1) if ! $ARGV[1]; - mig_env_create(); - } - case "clone" { - pod2usage(-verbose => 1) if ! $ARGV[2]; - $migration_schema = $ARGV[2] || ''; - $filename = "$HOME/.kmig/$migration_schema.env"; - mig_env_clone(); - } - case "use" { - pod2usage(-verbose => 1) if ! $ARGV[1]; - if (-e $filename) { - exec '/bin/bash', '--init-file', $filename; - } else { - die "\n$filename does not exist\n"; - } - } - case "show" { - if (-e $filename) { - exec '/bin/cat', $filename; - } else { - print `env | sort | egrep 'MIG|PG'`; - } - } - case "list" { - opendir(my $dh, "$HOME/.kmig") || die "can't open $HOME/.kmig: $!"; - while (readdir $dh) { - if (/^(.*)\.env$/) { - print "$1\n"; - } - } - closedir $dh; - } - else { - pod2usage(1); - } -} - -sub mig_env_create { - if (-e $filename) { - print "Re-Creating $filename\n"; - print `cat $filename`; - } else { - print "Creating $filename\n"; - } - print "\n"; - - # directories - - $MIGBASEWORKDIR = "$HOME/data/" unless $MIGBASEWORKDIR; - my $migworkdir_default = "$MIGBASEWORKDIR$migration_schema/"; - print "Main work directory (default $migworkdir_default): "; - my $MIGWORKDIR = ; - chomp $MIGWORKDIR; - if (! $MIGWORKDIR) { - $MIGWORKDIR = $migworkdir_default; - } - $MIGBASEGITDIR = "$HOME/git/migration-work/" unless $MIGBASEGITDIR; - my $miggitdir_default = "${MIGBASEGITDIR}/$migration_schema/"; - print "git repo for migration-specific scripts (default $miggitdir_default): "; - my $MIGGITDIR = ; - chomp $MIGGITDIR; - if (! $MIGGITDIR) { - $MIGGITDIR = $miggitdir_default; - } - - # MySQL/MariaDB - - my $mysqlhost; my $mysqldb; my $mysqlport; - my $mysqluser; my $mysqlpass; - if (-e '/usr/sbin/koha-list' && `/usr/sbin/koha-list` =~ $migration_schema - && `sudo -nl xmlstarlet` =~ 'xmlstarlet') { - my $kohaconfig="/etc/koha/sites/$migration_schema/koha-conf.xml"; - $mysqlhost=`sudo -n xmlstarlet sel -t -v 'yazgfs/config/hostname' $kohaconfig`; - $mysqldb=`sudo -n xmlstarlet sel -t -v 'yazgfs/config/database' $kohaconfig`; - $mysqlport=`sudo -n xmlstarlet sel -t -v 'yazgfs/config/port' $kohaconfig`; - $mysqluser=`sudo -n xmlstarlet sel -t -v 'yazgfs/config/user' $kohaconfig`; - $mysqlpass=`sudo -n xmlstarlet sel -t -v 'yazgfs/config/pass' $kohaconfig`; - chomp $mysqlhost; chomp $mysqldb; chomp $mysqlport; - chomp $mysqluser; chomp $mysqlpass; - } - - $MYSQL_HOST = $mysqlhost || 'localhost' unless $MYSQL_HOST; - my $mysql_host_default = $MYSQL_HOST; - print "MYSQL_HOST (default $mysql_host_default): "; - $MYSQL_HOST = ; - chomp $MYSQL_HOST; - if (! $MYSQL_HOST) { - $MYSQL_HOST = $mysql_host_default; - } - $MYSQL_TCP_PORT = $mysqlport || 3306 unless $MYSQL_TCP_PORT; - my $mysql_port_default = $MYSQL_TCP_PORT; - print "MYSQL_TCP_PORT (default $mysql_port_default): "; - $MYSQL_TCP_PORT = ; - chomp $MYSQL_TCP_PORT; - if (! $MYSQL_TCP_PORT) { - $MYSQL_TCP_PORT = $mysql_port_default; - } - $MYSQL_DATABASE = $mysqldb || 'koha_demo' unless $MYSQL_DATABASE; - my $mysql_database_default = $MYSQL_DATABASE; - print "MYSQL_DATABASE (default $mysql_database_default): "; - $MYSQL_DATABASE = ; - chomp $MYSQL_DATABASE; - if (! $MYSQL_DATABASE) { - $MYSQL_DATABASE = $mysql_database_default; - } - $MYSQL_USER = $mysqluser || $MYSQL_DATABASE unless $MYSQL_USER; - my $mysql_user_default = $MYSQL_USER; - print "MYSQL_USER (default $mysql_user_default): "; - my $MYSQL_USER = ; - chomp $MYSQL_USER; - if (! $MYSQL_USER) { - $MYSQL_USER = $mysql_user_default; - } - $MYSQL_PW = $mysqlpass || $MYSQL_USER unless $MYSQL_PW; - my $mysql_pw_default = $MYSQL_PW; - print "MYSQL_PW (default $mysql_pw_default): "; - my $MYSQL_PW = ; - chomp $MYSQL_PW; - if (! $MYSQL_PW) { - $MYSQL_PW = $mysql_pw_default; - } - - # create files and directories if needed - - mkdir "$HOME/.kmig"; - make_path($MIGGITDIR, { verbose => 1 }); - `touch $MIGGITDIR/README`; - make_path($MIGWORKDIR, { verbose => 1 }); - symlink $MIGGITDIR, "$MIGWORKDIR/scripts"; - open FILE, ">$filename"; - print FILE "export MYSQL_HOST=$MYSQL_HOST\n"; - print FILE "export MYSQL_TCP_PORT=$MYSQL_TCP_PORT\n"; - print FILE "export MYSQL_DATABASE=$MYSQL_DATABASE\n"; - print FILE "export MYSQL_USER=$MYSQL_USER\n"; - #TODO - brittle; need to escape the password string - print FILE "export MYSQL_PW=$MYSQL_PW\n"; - print FILE "export MIGENVPROMPT=$migration_schema\n"; - print FILE "export MIGSCHEMA=$migration_schema\n"; - print FILE "export MIGBASEWORKDIR=$MIGBASEWORKDIR\n"; - print FILE "export MIGWORKDIR=$MIGWORKDIR\n"; - print FILE "export MIGBASEGITDIR=$MIGBASEGITDIR\n"; - print FILE "export MIGGITDIR=$MIGGITDIR\n"; - print FILE "alias wcd='cd `mig wdir`'\n"; - print FILE "alias gcd='cd `mig gdir`'\n"; - print FILE "alias scd='cd `mig sdir`'\n"; - print FILE "source ~/.profile\n"; - print FILE "env | sort | egrep 'MYSQL|MIG'\n"; - print FILE 'echo shell PID = $$' . "\n"; - close FILE; - chmod 0600, $filename; # TODO: race condition worth worrying about? couldn't get sysopen to work -} - -sub mig_env_clone { - my $orig_migration_schema = $ARGV[1] || ''; - my $orig_filename = "$HOME/.kmig/$orig_migration_schema.env"; - `cp $orig_filename $filename`; - `sed -i 's/export MIGENVPROMPT=.*/export MIGENVPROMPT=$migration_schema/' $filename`; - `sed -i 's/export MIGSCHEMA=.*/export MIGSCHEMA=$migration_schema/' $filename`; -} - diff --git a/kmig.d/bin/kmig-iconv b/kmig.d/bin/kmig-iconv deleted file mode 100755 index 455e91a..0000000 --- a/kmig.d/bin/kmig-iconv +++ /dev/null @@ -1,107 +0,0 @@ -#!/usr/bin/perl -w -############################################################################### -=pod - -=head1 NAME - -kmig-iconv - -Attempts to invoke B on the specified tracked file, placing the -output in [file].iconv - -If given no other arguments, the invocation will lool like - -=over 5 - -iconv -f ISO-8859-1 -t UTF-8 -o .utf8 - -=back - -otherwise, the arguments will be passed through like so - -=over 5 - -iconv [other arguments...] -o .utf8 - -=back - -You'll need to invoke B prior to using commands like B - -=head1 SYNOPSIS - -B [other arguments...] - -=cut - -############################################################################### - -use strict; -use Switch; -use Env qw( - HOME MYSQL_HOST MYSQL_TCP_PORT MYSQL_USER MYSQL_DATABASE MYSQL_PW - MIGSCHEMA MIGBASEWORKDIR MIGBASEGITDIR MIGGITDIR MIGWORKDIR -); -use Pod::Usage; -use DBI; -use Cwd 'abs_path'; -use FindBin; -my $mig_bin = "$FindBin::Bin/"; -use lib "$FindBin::Bin/"; -use KMig; - -pod2usage(-verbose => 2) if ! $ARGV[0] || $ARGV[0] eq '--help'; - -KMig::die_if_no_env_migschema(); -KMig::die_if_mig_tracking_table_does_not_exist(); - -my $file = abs_path($ARGV[0]); -if ($file =~ /^$MIGBASEWORKDIR/) { - call_iconv(@ARGV); -} else { - print "File falls outside of MIGWORKDIR ($MIGWORKDIR): $file\n"; -} - -exit 0; - -############################################################################### - -sub call_iconv { - my $file = abs_path(shift); - my @args = @_; - - my $tracked_file_id = KMig::check_for_tracked_file($file); - if ($tracked_file_id) { - my $data = KMig::status_this_file($file); - print "iconv'ing tracked file: $file\n"; - - if (scalar(@args) == 0) { - @args = ( - '-f' - ,'ISO-8859-1' - ,'-t' - ,'UTF-8' - ,'--verbose' - ); - } - - system('iconv', @args, '-o', $file . '.utf8', $file); - system('touch', $file . '.utf8'); # handle 0-byte files - - my $dbh = KMig::db_connect(); - my $utf8_file = $dbh->quote($file . '.utf8'); - if (! -e $file . '.utf8') { - print "utf8 file does not exist: $utf8_file\n"; - $utf8_file = $dbh->quote(''); - } - - my $rv = $dbh->do(" - UPDATE m_tracked_file - SET utf8_filename = $utf8_file - WHERE base_filename = " . $dbh->quote($file) . " - ; - ") || die "Error inserting into table m_tracked_file: $!\n"; - KMig::db_disconnect($dbh); - } else { - print "File not currently tracked: $file\n"; - } -} diff --git a/kmig.d/bin/kmig-init b/kmig.d/bin/kmig-init deleted file mode 100755 index 7557168..0000000 --- a/kmig.d/bin/kmig-init +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/perl -w -############################################################################### -=pod - -=head1 NAME - -kmig-init - This will add or recreate tracking tables for the B toolset -to the database specified by the current kmig environment. - -In practice, you should invoke 'kmig env use schema_name' prior to calling -B - -=head1 SYNOPSIS - -B - -B - -=cut - -############################################################################### - -use strict; -use Switch; -use Env qw( - HOME MYSQL_HOST MYSQL_TCP_PORT MYSQL_USER MYSQL_DATABASE MYSQL_PW - MIGSCHEMA MIGBASEWORKDIR MIGBASEGITDIR MIGGITDIR MIGWORKDIR -); -use Pod::Usage; -use DBI; -use FindBin; -my $mig_bin = "$FindBin::Bin/"; -my $mig_sql = $mig_bin . "../sql/init/"; -use lib "$FindBin::Bin/"; -use KMig; - -pod2usage(-verbose => 2) if $ARGV[0]; - -KMig::die_if_no_env_migschema(); - -if (! KMig::check_for_db_migschema()) { - die "could not find the schema"; -} - -KMig::die_if_mig_tracking_table_exists(); -KMig::die_if_mig_column_tracking_table_exists(); -loop_through_mig_sql_templates(); - -exit 0; - -############################################################################### - -sub loop_through_mig_sql_templates { - print "Looping through kmig.d/sql/init/ templates\n"; - opendir my $dir, $mig_sql or die "Cannot open directory: $!"; - my @files = sort readdir $dir; - closedir $dir; - foreach my $file (@files) { - if ($file =~ /.sql$/) { - print "executing $file:\n"; - system( $mig_bin . "kmig-sql", ('-e',"source $mig_sql$file") ) - } - } -} - diff --git a/kmig.d/bin/kmig-remove b/kmig.d/bin/kmig-remove deleted file mode 100755 index 0ea81d8..0000000 --- a/kmig.d/bin/kmig-remove +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/perl -w -############################################################################### -=pod - -=head1 NAME - -kmig-remove - This will remove the specified files from the mig tracking table - -You'll need to invoke B prior to using commands like B - -=head1 SYNOPSIS - -B [file] [...] - -=cut - -############################################################################### - -use strict; -use Switch; -use Env qw( - HOME MYSQL_HOST MYSQL_TCP_PORT MYSQL_USER MYSQL_DATABASE MYSQL_PW - MIGSCHEMA MIGBASEWORKDIR MIGBASEGITDIR MIGGITDIR MIGWORKDIR -); -use Pod::Usage; -use DBI; -use Cwd 'abs_path'; -use FindBin; -my $mig_bin = "$FindBin::Bin/"; -use lib "$FindBin::Bin/"; -use KMig; - -pod2usage(-verbose => 2) if ! $ARGV[0] || $ARGV[0] eq '--help'; - -KMig::die_if_no_env_migschema(); -KMig::die_if_mig_tracking_table_does_not_exist(); - -foreach my $arg (@ARGV) { - my $file = abs_path($arg); - if ($file =~ /^$MIGBASEWORKDIR/) { - remove_this_file($file); - } else { - print "File falls outside of MIGWORKDIR ($MIGWORKDIR): $file\n"; - } -} - -exit 0; - -############################################################################### - -sub remove_this_file { - my $file = shift; - my $tracked_file_id = KMig::check_for_tracked_file($file,{'allow_missing'=>1}); - if ($tracked_file_id) { - print "removing tracked file: $file\n"; - my $dbh = KMig::db_connect(); - my $rv = $dbh->do(" - DELETE FROM m_tracked_file WHERE id = $tracked_file_id; - ") || die "Error deleting from table m_tracked_file (id = $tracked_file_id): $!\n"; - KMig::db_disconnect($dbh); - } else { - print "File not currently tracked: $file\n"; - } -} diff --git a/kmig.d/bin/kmig-skip-iconv b/kmig.d/bin/kmig-skip-iconv deleted file mode 100755 index c720256..0000000 --- a/kmig.d/bin/kmig-skip-iconv +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/perl -w -############################################################################### -=pod - -=head1 NAME - -kmig-skip-iconv - -Allows you to either use an existing file named .utf8 or a named -[utf8 file] as if it were the one created by mig-iconv - -=head1 SYNOPSIS - -B [utf8 file] - -=cut - -############################################################################### - -use strict; -use Switch; -use Env qw( - HOME MYSQL_HOST MYSQL_TCP_PORT MYSQL_USER MYSQL_DATABASE MYSQL_PW - MIGSCHEMA MIGBASEWORKDIR MIGBASEGITDIR MIGGITDIR MIGWORKDIR -); -use Pod::Usage; -use DBI; -use Cwd 'abs_path'; -use FindBin; -my $mig_bin = "$FindBin::Bin/"; -use lib "$FindBin::Bin/"; -use KMig; - -pod2usage(-verbose => 2) if ! ($ARGV[0]||$ARGV[1]) || $ARGV[0] eq '--help'; - -KMig::die_if_no_env_migschema(); -KMig::die_if_mig_tracking_table_does_not_exist(); - -my $file = abs_path($ARGV[0]); -my $utf8_file; -if ($ARGV[1]) { - $utf8_file = abs_path($ARGV[1]); -} else { - $utf8_file = $file; -} -if ($utf8_file && ! $utf8_file =~ /^$MIGBASEWORKDIR/) { - die "File falls outside of MIGWORKDIR ($MIGWORKDIR): $utf8_file\n"; -} - -if ($file =~ /^$MIGBASEWORKDIR/) { - skip_iconv($file,$utf8_file); -} else { - print "File falls outside of MIGWORKDIR ($MIGWORKDIR): $file\n"; -} - -exit 0; - -############################################################################### - -sub skip_iconv { - my $file = shift; - my $utf8_file = shift; - - my $tracked_file_id = KMig::check_for_tracked_file($file); - if ($tracked_file_id) { - my $data = KMig::status_this_file($file); - print "skipping the iconv'ing of tracked file: $file\n"; - - my $dbh = KMig::db_connect(); - if (! $utf8_file) { - $utf8_file = $file . '.utf8'; - } - if (! -e $utf8_file) { - die "utf8 file does not exist: $utf8_file\n"; - } - - my $rv = $dbh->do(" - UPDATE m_tracked_file - SET utf8_filename = " . $dbh->quote($utf8_file) . " - WHERE base_filename = " . $dbh->quote($file) . " - ; - ") || die "Error inserting into table m_tracked_file: $!\n"; - KMig::db_disconnect($dbh); - } else { - print "File not currently tracked: $file\n"; - } -} diff --git a/kmig.d/bin/kmig-sql b/kmig.d/bin/kmig-sql deleted file mode 100755 index 161dacd..0000000 --- a/kmig.d/bin/kmig-sql +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/perl -w -############################################################################### -=pod - -=head1 NAME - -kmig-sql - -A wrapper around the mysql client. - -=head1 SYNOPSIS - -B [arguments...] - -=cut - -############################################################################### - -use strict; -use Switch; -use Env qw( - HOME MYSQL_HOST MYSQL_TCP_PORT MYSQL_USER MYSQL_DATABASE MYSQL_PW - MIGSCHEMA MIGBASEWORKDIR MIGBASEGITDIR MIGGITDIR MIGWORKDIR -); -use Pod::Usage; -use DBI; -use Cwd 'abs_path'; -use FindBin; -my $mig_bin = "$FindBin::Bin/"; -use lib "$FindBin::Bin/"; -use KMig; - -my @MYARGV = ( - 'mysql' - ,'--host=' . $MYSQL_HOST - ,'--port=' . $MYSQL_TCP_PORT - ,'--user=' . $MYSQL_USER - ,'--password=' . $MYSQL_PW - ,"--init-command=set \@migschema = \"$MIGSCHEMA\";" - ,$MYSQL_DATABASE -); - -system(@MYARGV, @ARGV); - -exit 0; - diff --git a/kmig.d/bin/kmig-status b/kmig.d/bin/kmig-status deleted file mode 100755 index b5ca53b..0000000 --- a/kmig.d/bin/kmig-status +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/perl -w -############################################################################### -=pod - -=head1 NAME - -kmig-status - This will show tracking information for either the specified files -or all tracked files if no argument is given. - -You'll need to invoke B prior to using commands like B - -=head1 SYNOPSIS - -B [file] [...] - -=cut - -############################################################################### - -use strict; -use Switch; -use Env qw( - HOME MYSQL_HOST MYSQL_TCP_PORT MYSQL_USER MYSQL_DATABASE MYSQL_PW - MIGSCHEMA MIGBASEWORKDIR MIGBASEGITDIR MIGGITDIR MIGWORKDIR -); -use Pod::Usage; -use DBI; -use Cwd 'abs_path'; -use FindBin; -my $mig_bin = "$FindBin::Bin/"; -use lib "$FindBin::Bin/"; -use KMig; - -pod2usage(-verbose => 2) if scalar(@ARGV) > 0 && $ARGV[0] eq '--help'; - -KMig::die_if_no_env_migschema(); -KMig::die_if_mig_tracking_table_does_not_exist(); - -my @files = @ARGV; -if (scalar(@files) == 0) { - @files = (); - my $dbh = KMig::db_connect(); - my $sth = $dbh->prepare(" - SELECT base_filename - FROM m_tracked_file - ORDER BY 1;" - ); - my $rv = $sth->execute() - || die "Error retrieving data from table (tracked_file): $!"; - my $rows = $sth->fetchall_arrayref; - for my $row ( @$rows ) { - push @files, $row->[0] - } - $sth->finish; - KMig::db_disconnect($dbh); -} - -foreach my $arg (sort @files) { - my $file = abs_path($arg); - my $data = KMig::status_this_file($file); - print "=-=-=\n"; - foreach my $key ( - 'base_filename' - ,'has_headers' - ,'headers_file' - ,'utf8_filename' - ,'clean_filename' - ,'parent_table' - ,'stage_sql_filename' - ,'staged_table' - ,'map_sql_filename' - ,'prod_sql_filename' - ) { - printf "%-20s:\t", $key; - print $data->{$key} ? $data->{$key} : ""; - if ($key =~ /filename$/ && $data->{$key} && ! -e $data->{$key}) { - print " (FILE MISSING)"; - } - print "\n"; - } -} - -exit 0; - -############################################################################### - - diff --git a/kmig.d/bin/mig-add b/kmig.d/bin/mig-add new file mode 100755 index 0000000..ff4422b --- /dev/null +++ b/kmig.d/bin/mig-add @@ -0,0 +1,125 @@ +#!/usr/bin/perl -w +############################################################################### +=pod + +=head1 NAME + +mig-add - This will add the specified files to the mig tracking table + +--headers (the default) and --no-headers are repeatable, and indicate whether +subsequent files have headers or not + +--headers-file specifies a text file defining the column headers for +the next added , which should contain one line per header + +--headers-file will automatically invoke --no-headers + +You'll need to invoke B prior to using commands like B + +=head1 SYNOPSIS + +B [--no-headers|--headers|--headers-file ] [file|--no-headers|--headers|--headers-file ] [...] + +=cut + +############################################################################### + +use strict; +use Switch; +use Env qw( + HOME MYSQL_HOST MYSQL_TCP_PORT MYSQL_USER MYSQL_DATABASE MYSQL_PW + MIGSCHEMA MIGBASEWORKDIR MIGBASEGITDIR MIGGITDIR MIGWORKDIR +); +use Pod::Usage; +use DBI; +use Cwd 'abs_path'; +use FindBin; +my $mig_bin = "$FindBin::Bin/"; +use lib "$FindBin::Bin/"; +use KMig; + +pod2usage(-verbose => 2) if ! $ARGV[0] || $ARGV[0] eq '--help'; + +KMig::die_if_no_env_migschema(); +KMig::die_if_mig_tracking_table_does_not_exist(); + +my $has_headers = 1; +my $headers_file; +my $next_arg_is_headers_file = 0; + +foreach my $arg (@ARGV) { + if ($next_arg_is_headers_file) { + $next_arg_is_headers_file = 0; + $headers_file = abs_path($arg); + next; + } + if ($arg eq '--headers') { + $has_headers = 1; + next; + } + if ($arg eq '--no-headers') { + $has_headers = 0; + next; + } + if ($arg eq '--headers-file') { + $next_arg_is_headers_file = 1; + $has_headers = 0; + next; + } + my $file = abs_path($arg); + if ($file =~ /^$MIGBASEWORKDIR/) { + if (-e $file) { + if (-f $file) { + add_this_file($file,$has_headers,$headers_file); + $headers_file = ''; # clear after applying to just one file + } else { + print "Not a real file: $file\n"; + } + } else { + print "Could not find file: $file\n"; + } + } else { + print "File falls outside of MIGWORKDIR ($MIGWORKDIR): $file\n"; + } +} + +exit 0; + +############################################################################### + +sub add_this_file { + my $file = shift; + my $headers = shift; + my $headers_file = shift; + if ($headers_file) { + if (! (-e $headers_file && -f $headers_file)) { + print "Could not find headers file $headers_file, skipping $file\n"; + return; + } + } + if (KMig::check_for_tracked_file($file)) { + print "File already tracked: $file\n"; + } else { + print 'Adding ('; + if ($headers_file) { + print "with headers file = $headers_file"; + } else { + print ($headers ? ' with headers' : 'without headers'); + } + print '): ' . "$file\n"; + my $dbh = KMig::db_connect(); + my $rv = $dbh->do(" + INSERT INTO m_tracked_file ( + base_filename + ,has_headers + ,headers_file + ) VALUES ( + " . $dbh->quote($file) . " + ," . $dbh->quote($headers) . " + ," . $dbh->quote($headers_file) . " + ); + ") || die "Error inserting into table m_tracked_file: $!\n"; + KMig::db_disconnect($dbh); + } +} + diff --git a/kmig.d/bin/mig-clean b/kmig.d/bin/mig-clean new file mode 100755 index 0000000..705fd96 --- /dev/null +++ b/kmig.d/bin/mig-clean @@ -0,0 +1,127 @@ +#!/usr/bin/perl -w +############################################################################### +=pod + +=head1 NAME + +mig-clean + +Attempts to invoke B on the specified tracked file, placing the +output in [file].clean + +If given no other arguments, the invocation will lool like + +=over 5 + +clean_csv --config scripts/clean.conf --fix --apply [--create-headers|--use-headers ] + +=back + +otherwise, the arguments will be passed through like so + +=over 5 + +clean_csv [other arguments...] + +=back + +You'll need to invoke B or B prior to using commands +like B + +=head1 SYNOPSIS + +B [other arguments...] + +=cut + +############################################################################### + +use strict; +use Switch; +use Env qw( + HOME MYSQL_HOST MYSQL_TCP_PORT MYSQL_USER MYSQL_DATABASE MYSQL_PW + MIGSCHEMA MIGBASEWORKDIR MIGBASEGITDIR MIGGITDIR MIGWORKDIR +); +use Pod::Usage; +use DBI; +use Cwd 'abs_path'; +use FindBin; +my $mig_bin = "$FindBin::Bin/"; +use lib "$FindBin::Bin/"; +use KMig; + +pod2usage(-verbose => 2) if ! $ARGV[0] || $ARGV[0] eq '--help'; + +KMig::die_if_no_env_migschema(); +KMig::die_if_mig_tracking_table_does_not_exist(); + +my $file = abs_path($ARGV[0]); +if ($file =~ /^$MIGBASEWORKDIR/) { + call_clean_csv(@ARGV); +} else { + print "File falls outside of MIGWORKDIR ($MIGWORKDIR): $file\n"; +} + +exit 0; + +############################################################################### + +sub call_clean_csv { + my $file = abs_path(shift); + my @args = @_; + + my $tracked_file_id = KMig::check_for_tracked_file($file); + if ($tracked_file_id) { + my $data = KMig::status_this_file($file); + + if (! $data->{'utf8_filename'}) { + die "mig-iconv or mig-skip-iconv needed for UTF8 version of file: $file\n"; + } + + my $utf8_file = $data->{'utf8_filename'}; + if (! -e $utf8_file) { + die "missing file: $utf8_file\n"; + } + + print "cleaning tracked file: $file\n"; + + if (scalar(@args) == 0) { + @args = ( + '--config' + ,'scripts/clean.conf' + ,'--fix' + ,'--apply' + ,'--backslash' + ,'--pad' + ); + if (! $data->{'has_headers'}) { + if ($data->{'headers_file'}) { + push @args, '--use-headers'; + push @args, $data->{'headers_file'}; + } else { + push @args, '--create-headers'; + } + } + } + + print join(' ',@args) . "\n"; + system('clean_csv', @args, $utf8_file); + + my $dbh = KMig::db_connect(); + my $clean_file = $dbh->quote($utf8_file . '.clean'); + if (! -e $utf8_file . '.clean') { + print "clean file does not exist: $clean_file\n"; + $clean_file = $dbh->quote(''); + } + + my $rv = $dbh->do(" + UPDATE m_tracked_file + SET clean_filename = $clean_file + WHERE base_filename = " . $dbh->quote($file) . " + ; + ") || die "Error inserting into table m_tracked_file: $!\n"; + KMig::db_disconnect($dbh); + } else { + print "File not currently tracked: $file\n"; + } +} diff --git a/kmig.d/bin/mig-env b/kmig.d/bin/mig-env new file mode 100755 index 0000000..a8b5329 --- /dev/null +++ b/kmig.d/bin/mig-env @@ -0,0 +1,298 @@ +#!/usr/bin/perl -w +############################################################################### +=pod + +=head1 NAME + +mig-env - This tool is for tracking and setting environment variables used by +B and its sub-tools. + +=head1 SYNOPSIS + +B + +B [migration_schema] + +B [orig_migration_schema] [new_migration_schema] + +B + +B + +=head1 DESCRIPTION + +For most invocations, B will either create or use a migration-specific +file (~/.mig/.env) for setting the following environment +variables: + +=over 15 + +=item MIGSCHEMA + +The name of the migration schema. In practice, this will match the name of the +Koha instance. + +=item MIGWORKDIR + +The base working directory for containing migration data, scripts, and other +files. + +=item MYSQL_HOST + +The IP address or hostname for the MySQL/MariaDB database used for a migration. + +=item MYSQL_TCP_PORT + +The TCP port for the database. + +=item MYSQL_USER + +The user to use for the database. + +=item MYSQL_PW + +The password to use for the database. + +=item MYSQL_DATABASE + +The name of the actual database/schema. In practice, this will match the +migration schema or Koha instance name, prefixed with 'koha_'. + +=back + +This script may also setup a symlink from a specified Git repository to a +scripts/ directory within the migration work directory. The default for this is +~/git/migration-work/MIGSCHEMA --> MIGWORKDIR/scripts + +It may also create the migration work directory if necessary. + +=head1 COMMANDS + +=over 15 + +=item B + +This invocation will prompt for various values and create a .env file for the +specified migration schema, and a symlink between the specified Git repository +and migration work directory (which will also be created if needed). + +=item B + +This command will spawn a bash shell that executes the corresponding +~/.mig/.env script for setting up environment variables encoded during +B. + +=item B [schema] + +This command will show the contents of the corresponding ~/.mig/.env +script, or, if no schema is specified, then it will list pertinent variables in +the current environment if they exist. + +=item B [orig schema] [new schema] + +FIXME: need to re-think this in a MySQL/MariaDB/Koha context + +This command will create a "shallow" clone of the orig schema, in that it will +share database credentials as well as git and data directories, but will have a +separate schema name. + +=item B + +This command will list migration schemas found in ~/.mig + +=item B + +Display the documentation you're reading now. + +=back + +=cut + +############################################################################### + +use strict; +use 5.012; +use Switch; +use Env qw( + HOME MYSQL_HOST MYSQL_TCP_PORT MYSQL_USER MYSQL_DATABASE MYSQL_PW + MIGSCHEMA MIGBASEWORKDIR MIGBASEGITDIR MIGGITDIR MIGWORKDIR +); +use Pod::Usage; +use File::Path qw(make_path); +use FindBin; +my $mig_bin = "$FindBin::Bin/"; +use lib "$FindBin::Bin/"; + +pod2usage(-verbose => 2) if ! $ARGV[0]; + +my $migration_schema = $ARGV[1] || ''; +my $filename = "$HOME/.mig/$migration_schema.env"; +switch($ARGV[0]) { + case "--help" { + pod2usage(-verbose => 2); + } + case "help" { + pod2usage(-verbose => 2); + } + case "create" { + pod2usage(-verbose => 1) if ! $ARGV[1]; + mig_env_create(); + } + case "clone" { + pod2usage(-verbose => 1) if ! $ARGV[2]; + $migration_schema = $ARGV[2] || ''; + $filename = "$HOME/.mig/$migration_schema.env"; + mig_env_clone(); + } + case "use" { + pod2usage(-verbose => 1) if ! $ARGV[1]; + if (-e $filename) { + exec '/bin/bash', '--init-file', $filename; + } else { + die "\n$filename does not exist\n"; + } + } + case "show" { + if (-e $filename) { + exec '/bin/cat', $filename; + } else { + print `env | sort | egrep 'MIG|PG'`; + } + } + case "list" { + opendir(my $dh, "$HOME/.mig") || die "can't open $HOME/.mig: $!"; + while (readdir $dh) { + if (/^(.*)\.env$/) { + print "$1\n"; + } + } + closedir $dh; + } + else { + pod2usage(1); + } +} + +sub mig_env_create { + if (-e $filename) { + print "Re-Creating $filename\n"; + print `cat $filename`; + } else { + print "Creating $filename\n"; + } + print "\n"; + + # directories + + $MIGBASEWORKDIR = "$HOME/data/" unless $MIGBASEWORKDIR; + my $migworkdir_default = "$MIGBASEWORKDIR$migration_schema/"; + print "Main work directory (default $migworkdir_default): "; + my $MIGWORKDIR = ; + chomp $MIGWORKDIR; + if (! $MIGWORKDIR) { + $MIGWORKDIR = $migworkdir_default; + } + $MIGBASEGITDIR = "$HOME/git/migration-work/" unless $MIGBASEGITDIR; + my $miggitdir_default = "${MIGBASEGITDIR}/$migration_schema/"; + print "git repo for migration-specific scripts (default $miggitdir_default): "; + my $MIGGITDIR = ; + chomp $MIGGITDIR; + if (! $MIGGITDIR) { + $MIGGITDIR = $miggitdir_default; + } + + # MySQL/MariaDB + + my $mysqlhost; my $mysqldb; my $mysqlport; + my $mysqluser; my $mysqlpass; + if (-e '/usr/sbin/koha-list' && `/usr/sbin/koha-list` =~ $migration_schema + && `sudo -nl xmlstarlet` =~ 'xmlstarlet') { + my $kohaconfig="/etc/koha/sites/$migration_schema/koha-conf.xml"; + $mysqlhost=`sudo -n xmlstarlet sel -t -v 'yazgfs/config/hostname' $kohaconfig`; + $mysqldb=`sudo -n xmlstarlet sel -t -v 'yazgfs/config/database' $kohaconfig`; + $mysqlport=`sudo -n xmlstarlet sel -t -v 'yazgfs/config/port' $kohaconfig`; + $mysqluser=`sudo -n xmlstarlet sel -t -v 'yazgfs/config/user' $kohaconfig`; + $mysqlpass=`sudo -n xmlstarlet sel -t -v 'yazgfs/config/pass' $kohaconfig`; + chomp $mysqlhost; chomp $mysqldb; chomp $mysqlport; + chomp $mysqluser; chomp $mysqlpass; + } + + $MYSQL_HOST = $mysqlhost || 'localhost' unless $MYSQL_HOST; + my $mysql_host_default = $MYSQL_HOST; + print "MYSQL_HOST (default $mysql_host_default): "; + $MYSQL_HOST = ; + chomp $MYSQL_HOST; + if (! $MYSQL_HOST) { + $MYSQL_HOST = $mysql_host_default; + } + $MYSQL_TCP_PORT = $mysqlport || 3306 unless $MYSQL_TCP_PORT; + my $mysql_port_default = $MYSQL_TCP_PORT; + print "MYSQL_TCP_PORT (default $mysql_port_default): "; + $MYSQL_TCP_PORT = ; + chomp $MYSQL_TCP_PORT; + if (! $MYSQL_TCP_PORT) { + $MYSQL_TCP_PORT = $mysql_port_default; + } + $MYSQL_DATABASE = $mysqldb || 'koha_demo' unless $MYSQL_DATABASE; + my $mysql_database_default = $MYSQL_DATABASE; + print "MYSQL_DATABASE (default $mysql_database_default): "; + $MYSQL_DATABASE = ; + chomp $MYSQL_DATABASE; + if (! $MYSQL_DATABASE) { + $MYSQL_DATABASE = $mysql_database_default; + } + $MYSQL_USER = $mysqluser || $MYSQL_DATABASE unless $MYSQL_USER; + my $mysql_user_default = $MYSQL_USER; + print "MYSQL_USER (default $mysql_user_default): "; + my $MYSQL_USER = ; + chomp $MYSQL_USER; + if (! $MYSQL_USER) { + $MYSQL_USER = $mysql_user_default; + } + $MYSQL_PW = $mysqlpass || $MYSQL_USER unless $MYSQL_PW; + my $mysql_pw_default = $MYSQL_PW; + print "MYSQL_PW (default $mysql_pw_default): "; + my $MYSQL_PW = ; + chomp $MYSQL_PW; + if (! $MYSQL_PW) { + $MYSQL_PW = $mysql_pw_default; + } + + # create files and directories if needed + + mkdir "$HOME/.mig"; + make_path($MIGGITDIR, { verbose => 1 }); + `touch $MIGGITDIR/README`; + make_path($MIGWORKDIR, { verbose => 1 }); + symlink $MIGGITDIR, "$MIGWORKDIR/scripts"; + open FILE, ">$filename"; + print FILE "export MYSQL_HOST=$MYSQL_HOST\n"; + print FILE "export MYSQL_TCP_PORT=$MYSQL_TCP_PORT\n"; + print FILE "export MYSQL_DATABASE=$MYSQL_DATABASE\n"; + print FILE "export MYSQL_USER=$MYSQL_USER\n"; + #TODO - brittle; need to escape the password string + print FILE "export MYSQL_PW=$MYSQL_PW\n"; + print FILE "export MIGENVPROMPT=$migration_schema\n"; + print FILE "export MIGSCHEMA=$migration_schema\n"; + print FILE "export MIGBASEWORKDIR=$MIGBASEWORKDIR\n"; + print FILE "export MIGWORKDIR=$MIGWORKDIR\n"; + print FILE "export MIGBASEGITDIR=$MIGBASEGITDIR\n"; + print FILE "export MIGGITDIR=$MIGGITDIR\n"; + print FILE "alias wcd='cd `mig wdir`'\n"; + print FILE "alias gcd='cd `mig gdir`'\n"; + print FILE "alias scd='cd `mig sdir`'\n"; + print FILE "source ~/.profile\n"; + print FILE "env | sort | egrep 'MYSQL|MIG'\n"; + print FILE 'echo shell PID = $$' . "\n"; + close FILE; + chmod 0600, $filename; # TODO: race condition worth worrying about? couldn't get sysopen to work +} + +sub mig_env_clone { + my $orig_migration_schema = $ARGV[1] || ''; + my $orig_filename = "$HOME/.kmig/$orig_migration_schema.env"; + `cp $orig_filename $filename`; + `sed -i 's/export MIGENVPROMPT=.*/export MIGENVPROMPT=$migration_schema/' $filename`; + `sed -i 's/export MIGSCHEMA=.*/export MIGSCHEMA=$migration_schema/' $filename`; +} + diff --git a/kmig.d/bin/mig-iconv b/kmig.d/bin/mig-iconv new file mode 100755 index 0000000..c2552a5 --- /dev/null +++ b/kmig.d/bin/mig-iconv @@ -0,0 +1,107 @@ +#!/usr/bin/perl -w +############################################################################### +=pod + +=head1 NAME + +mig-iconv + +Attempts to invoke B on the specified tracked file, placing the +output in [file].iconv + +If given no other arguments, the invocation will lool like + +=over 5 + +iconv -f ISO-8859-1 -t UTF-8 -o .utf8 + +=back + +otherwise, the arguments will be passed through like so + +=over 5 + +iconv [other arguments...] -o .utf8 + +=back + +You'll need to invoke B prior to using commands like B + +=head1 SYNOPSIS + +B [other arguments...] + +=cut + +############################################################################### + +use strict; +use Switch; +use Env qw( + HOME MYSQL_HOST MYSQL_TCP_PORT MYSQL_USER MYSQL_DATABASE MYSQL_PW + MIGSCHEMA MIGBASEWORKDIR MIGBASEGITDIR MIGGITDIR MIGWORKDIR +); +use Pod::Usage; +use DBI; +use Cwd 'abs_path'; +use FindBin; +my $mig_bin = "$FindBin::Bin/"; +use lib "$FindBin::Bin/"; +use KMig; + +pod2usage(-verbose => 2) if ! $ARGV[0] || $ARGV[0] eq '--help'; + +KMig::die_if_no_env_migschema(); +KMig::die_if_mig_tracking_table_does_not_exist(); + +my $file = abs_path($ARGV[0]); +if ($file =~ /^$MIGBASEWORKDIR/) { + call_iconv(@ARGV); +} else { + print "File falls outside of MIGWORKDIR ($MIGWORKDIR): $file\n"; +} + +exit 0; + +############################################################################### + +sub call_iconv { + my $file = abs_path(shift); + my @args = @_; + + my $tracked_file_id = KMig::check_for_tracked_file($file); + if ($tracked_file_id) { + my $data = KMig::status_this_file($file); + print "iconv'ing tracked file: $file\n"; + + if (scalar(@args) == 0) { + @args = ( + '-f' + ,'ISO-8859-1' + ,'-t' + ,'UTF-8' + ,'--verbose' + ); + } + + system('iconv', @args, '-o', $file . '.utf8', $file); + system('touch', $file . '.utf8'); # handle 0-byte files + + my $dbh = KMig::db_connect(); + my $utf8_file = $dbh->quote($file . '.utf8'); + if (! -e $file . '.utf8') { + print "utf8 file does not exist: $utf8_file\n"; + $utf8_file = $dbh->quote(''); + } + + my $rv = $dbh->do(" + UPDATE m_tracked_file + SET utf8_filename = $utf8_file + WHERE base_filename = " . $dbh->quote($file) . " + ; + ") || die "Error inserting into table m_tracked_file: $!\n"; + KMig::db_disconnect($dbh); + } else { + print "File not currently tracked: $file\n"; + } +} diff --git a/kmig.d/bin/mig-init b/kmig.d/bin/mig-init new file mode 100755 index 0000000..8b207f8 --- /dev/null +++ b/kmig.d/bin/mig-init @@ -0,0 +1,65 @@ +#!/usr/bin/perl -w +############################################################################### +=pod + +=head1 NAME + +mig-init - This will add or recreate tracking tables for the B toolset +to the database specified by the current mig environment. + +In practice, you should invoke 'mig env use schema_name' prior to calling +B + +=head1 SYNOPSIS + +B + +B + +=cut + +############################################################################### + +use strict; +use Switch; +use Env qw( + HOME MYSQL_HOST MYSQL_TCP_PORT MYSQL_USER MYSQL_DATABASE MYSQL_PW + MIGSCHEMA MIGBASEWORKDIR MIGBASEGITDIR MIGGITDIR MIGWORKDIR +); +use Pod::Usage; +use DBI; +use FindBin; +my $mig_bin = "$FindBin::Bin/"; +my $mig_sql = $mig_bin . "../sql/init/"; +use lib "$FindBin::Bin/"; +use KMig; + +pod2usage(-verbose => 2) if $ARGV[0]; + +KMig::die_if_no_env_migschema(); + +if (! KMig::check_for_db_migschema()) { + die "could not find the schema"; +} + +KMig::die_if_mig_tracking_table_exists(); +KMig::die_if_mig_column_tracking_table_exists(); +loop_through_mig_sql_templates(); + +exit 0; + +############################################################################### + +sub loop_through_mig_sql_templates { + print "Looping through mig.d/sql/init/ templates\n"; + opendir my $dir, $mig_sql or die "Cannot open directory: $!"; + my @files = sort readdir $dir; + closedir $dir; + foreach my $file (@files) { + if ($file =~ /.sql$/) { + print "executing $file:\n"; + system( $mig_bin . "mig-sql", ('-e',"source $mig_sql$file") ) + } + } +} + diff --git a/kmig.d/bin/mig-remove b/kmig.d/bin/mig-remove new file mode 100755 index 0000000..2b625a2 --- /dev/null +++ b/kmig.d/bin/mig-remove @@ -0,0 +1,64 @@ +#!/usr/bin/perl -w +############################################################################### +=pod + +=head1 NAME + +mig-remove - This will remove the specified files from the mig tracking table + +You'll need to invoke B prior to using commands like B + +=head1 SYNOPSIS + +B [file] [...] + +=cut + +############################################################################### + +use strict; +use Switch; +use Env qw( + HOME MYSQL_HOST MYSQL_TCP_PORT MYSQL_USER MYSQL_DATABASE MYSQL_PW + MIGSCHEMA MIGBASEWORKDIR MIGBASEGITDIR MIGGITDIR MIGWORKDIR +); +use Pod::Usage; +use DBI; +use Cwd 'abs_path'; +use FindBin; +my $mig_bin = "$FindBin::Bin/"; +use lib "$FindBin::Bin/"; +use KMig; + +pod2usage(-verbose => 2) if ! $ARGV[0] || $ARGV[0] eq '--help'; + +KMig::die_if_no_env_migschema(); +KMig::die_if_mig_tracking_table_does_not_exist(); + +foreach my $arg (@ARGV) { + my $file = abs_path($arg); + if ($file =~ /^$MIGBASEWORKDIR/) { + remove_this_file($file); + } else { + print "File falls outside of MIGWORKDIR ($MIGWORKDIR): $file\n"; + } +} + +exit 0; + +############################################################################### + +sub remove_this_file { + my $file = shift; + my $tracked_file_id = KMig::check_for_tracked_file($file,{'allow_missing'=>1}); + if ($tracked_file_id) { + print "removing tracked file: $file\n"; + my $dbh = KMig::db_connect(); + my $rv = $dbh->do(" + DELETE FROM m_tracked_file WHERE id = $tracked_file_id; + ") || die "Error deleting from table m_tracked_file (id = $tracked_file_id): $!\n"; + KMig::db_disconnect($dbh); + } else { + print "File not currently tracked: $file\n"; + } +} diff --git a/kmig.d/bin/mig-skip-iconv b/kmig.d/bin/mig-skip-iconv new file mode 100755 index 0000000..db9c378 --- /dev/null +++ b/kmig.d/bin/mig-skip-iconv @@ -0,0 +1,87 @@ +#!/usr/bin/perl -w +############################################################################### +=pod + +=head1 NAME + +mig-skip-iconv + +Allows you to either use an existing file named .utf8 or a named +[utf8 file] as if it were the one created by mig-iconv + +=head1 SYNOPSIS + +B [utf8 file] + +=cut + +############################################################################### + +use strict; +use Switch; +use Env qw( + HOME MYSQL_HOST MYSQL_TCP_PORT MYSQL_USER MYSQL_DATABASE MYSQL_PW + MIGSCHEMA MIGBASEWORKDIR MIGBASEGITDIR MIGGITDIR MIGWORKDIR +); +use Pod::Usage; +use DBI; +use Cwd 'abs_path'; +use FindBin; +my $mig_bin = "$FindBin::Bin/"; +use lib "$FindBin::Bin/"; +use KMig; + +pod2usage(-verbose => 2) if ! ($ARGV[0]||$ARGV[1]) || $ARGV[0] eq '--help'; + +KMig::die_if_no_env_migschema(); +KMig::die_if_mig_tracking_table_does_not_exist(); + +my $file = abs_path($ARGV[0]); +my $utf8_file; +if ($ARGV[1]) { + $utf8_file = abs_path($ARGV[1]); +} else { + $utf8_file = $file; +} +if ($utf8_file && ! $utf8_file =~ /^$MIGBASEWORKDIR/) { + die "File falls outside of MIGWORKDIR ($MIGWORKDIR): $utf8_file\n"; +} + +if ($file =~ /^$MIGBASEWORKDIR/) { + skip_iconv($file,$utf8_file); +} else { + print "File falls outside of MIGWORKDIR ($MIGWORKDIR): $file\n"; +} + +exit 0; + +############################################################################### + +sub skip_iconv { + my $file = shift; + my $utf8_file = shift; + + my $tracked_file_id = KMig::check_for_tracked_file($file); + if ($tracked_file_id) { + my $data = KMig::status_this_file($file); + print "skipping the iconv'ing of tracked file: $file\n"; + + my $dbh = KMig::db_connect(); + if (! $utf8_file) { + $utf8_file = $file . '.utf8'; + } + if (! -e $utf8_file) { + die "utf8 file does not exist: $utf8_file\n"; + } + + my $rv = $dbh->do(" + UPDATE m_tracked_file + SET utf8_filename = " . $dbh->quote($utf8_file) . " + WHERE base_filename = " . $dbh->quote($file) . " + ; + ") || die "Error inserting into table m_tracked_file: $!\n"; + KMig::db_disconnect($dbh); + } else { + print "File not currently tracked: $file\n"; + } +} diff --git a/kmig.d/bin/mig-sql b/kmig.d/bin/mig-sql new file mode 100755 index 0000000..993826b --- /dev/null +++ b/kmig.d/bin/mig-sql @@ -0,0 +1,46 @@ +#!/usr/bin/perl -w +############################################################################### +=pod + +=head1 NAME + +mig-sql + +A wrapper around the mysql client. + +=head1 SYNOPSIS + +B [arguments...] + +=cut + +############################################################################### + +use strict; +use Switch; +use Env qw( + HOME MYSQL_HOST MYSQL_TCP_PORT MYSQL_USER MYSQL_DATABASE MYSQL_PW + MIGSCHEMA MIGBASEWORKDIR MIGBASEGITDIR MIGGITDIR MIGWORKDIR +); +use Pod::Usage; +use DBI; +use Cwd 'abs_path'; +use FindBin; +my $mig_bin = "$FindBin::Bin/"; +use lib "$FindBin::Bin/"; +use KMig; + +my @MYARGV = ( + 'mysql' + ,'--host=' . $MYSQL_HOST + ,'--port=' . $MYSQL_TCP_PORT + ,'--user=' . $MYSQL_USER + ,'--password=' . $MYSQL_PW + ,"--init-command=set \@migschema = \"$MIGSCHEMA\";" + ,$MYSQL_DATABASE +); + +system(@MYARGV, @ARGV); + +exit 0; + diff --git a/kmig.d/bin/mig-status b/kmig.d/bin/mig-status new file mode 100755 index 0000000..fe5a029 --- /dev/null +++ b/kmig.d/bin/mig-status @@ -0,0 +1,87 @@ +#!/usr/bin/perl -w +############################################################################### +=pod + +=head1 NAME + +mig-status - This will show tracking information for either the specified files +or all tracked files if no argument is given. + +You'll need to invoke B prior to using commands like B + +=head1 SYNOPSIS + +B [file] [...] + +=cut + +############################################################################### + +use strict; +use Switch; +use Env qw( + HOME MYSQL_HOST MYSQL_TCP_PORT MYSQL_USER MYSQL_DATABASE MYSQL_PW + MIGSCHEMA MIGBASEWORKDIR MIGBASEGITDIR MIGGITDIR MIGWORKDIR +); +use Pod::Usage; +use DBI; +use Cwd 'abs_path'; +use FindBin; +my $mig_bin = "$FindBin::Bin/"; +use lib "$FindBin::Bin/"; +use KMig; + +pod2usage(-verbose => 2) if scalar(@ARGV) > 0 && $ARGV[0] eq '--help'; + +KMig::die_if_no_env_migschema(); +KMig::die_if_mig_tracking_table_does_not_exist(); + +my @files = @ARGV; +if (scalar(@files) == 0) { + @files = (); + my $dbh = KMig::db_connect(); + my $sth = $dbh->prepare(" + SELECT base_filename + FROM m_tracked_file + ORDER BY 1;" + ); + my $rv = $sth->execute() + || die "Error retrieving data from table (tracked_file): $!"; + my $rows = $sth->fetchall_arrayref; + for my $row ( @$rows ) { + push @files, $row->[0] + } + $sth->finish; + KMig::db_disconnect($dbh); +} + +foreach my $arg (sort @files) { + my $file = abs_path($arg); + my $data = KMig::status_this_file($file); + print "=-=-=\n"; + foreach my $key ( + 'base_filename' + ,'has_headers' + ,'headers_file' + ,'utf8_filename' + ,'clean_filename' + ,'parent_table' + ,'stage_sql_filename' + ,'staged_table' + ,'map_sql_filename' + ,'prod_sql_filename' + ) { + printf "%-20s:\t", $key; + print $data->{$key} ? $data->{$key} : ""; + if ($key =~ /filename$/ && $data->{$key} && ! -e $data->{$key}) { + print " (FILE MISSING)"; + } + print "\n"; + } +} + +exit 0; + +############################################################################### + + -- 1.7.2.5